Update app.py

app.py

@@ -1,94 +1,150 @@
+
 import os
 import csv
+import json
 import logging
 import gradio as gr
-import nltk
-from datasets import Dataset, DatasetDict, DatasetInfo, Features, Value, ClassLabel
-from huggingface_hub import HfApi, Repository, create_repo
 from tqdm import tqdm
+import nltk
 from nltk.tokenize import word_tokenize
-from nltk.corpus import wordnet as wn
-import random
-import string
+from nltk.corpus import wordnet
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+from huggingface_hub import HfApi, Repository, login
+from datasets import Dataset
+import pandas as pd
+from datetime import datetime
+import secrets
 
-# Ensure necessary NLTK resources are downloaded
+# Download all NLTK data
 nltk.download('all')
-#nltk.download('wordnet')
 
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
+# Setup logging
+log_dir = "logs"
+os.makedirs(log_dir, exist_ok=True)
+logging.basicConfig(
+    filename=os.path.join(log_dir, f"app_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"),
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+
+# Error logging to Hugging Face
+error_dir = "errors"
+os.makedirs(error_dir, exist_ok=True)
+error_log_file = os.path.join(error_dir, f"errors_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log")
+
+def log_error(error_msg):
+    with open(error_log_file, 'a') as f:
+        f.write(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} - ERROR - {error_msg}\n")
+    try:
+        api = HfApi()
+        api.upload_file(
+            path_or_fileobj=error_log_file,
+            path_in_repo=f"errors_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log",
+            repo_id="katsukiai/errors",
+            repo_type="dataset"
+        )
+    except Exception as e:
+        logging.error(f"Failed to upload error log: {str(e)}")
 
-# Function to generate random words
-def generate_random_words(num_words=100):
-    words = []
-    for _ in range(num_words):
-        word_length = random.randint(3, 10)
-        word = ''.join(random.choices(string.ascii_lowercase, k=word_length))
-        words.append(word)
-    return words
+# Load Hugging Face models (300+ models available, using DeepSeek for long text)
+tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-6.7b-instruct")
+model = AutoModelForSeq2SeqLM.from_pretrained("deepseek-ai/deepseek-coder-6.7b-instruct")
+meaning_generator = pipeline("text2text-generation", model="google/flan-t5-large")
 
-# Function to get meanings of words using NLTK WordNet
-def get_word_meanings(words):
-    meanings = {}
-    for word in words:
-        synsets = wn.synsets(word)
-        if synsets:
-            meanings[word] = synsets[0].definition()
-        else:
-            meanings[word] = "No definition found."
-    return meanings
+# Hugging Face login
+HF_TOKEN = os.getenv("HF_TOKEN", secrets.token_hex(16))
+login(token=HF_TOKEN)
 
-# Function to convert data to CSV format
-def convert_to_csv(data, filename='dataset.csv'):
-    fieldnames = ['word', 'meaning']
-    with open(filename, mode='w', newline='', encoding='utf-8') as file:
-        writer = csv.DictWriter(file, fieldnames=fieldnames)
-        writer.writeheader()
-        for word, meaning in data.items():
-            writer.writerow({'word': word, 'meaning': meaning})
+# Dataset preparation
+dataset_dir = "dataset"
+os.makedirs(dataset_dir, exist_ok=True)
+csv_file = os.path.join(dataset_dir, "deepfocus_data.csv")
 
-# Function to create and push dataset to Hugging Face
-def create_and_push_dataset(csv_file='dataset.csv', repo_name='DeepFocus-X3'):
-    # Create a new dataset repository on Hugging Face
-    create_repo(repo_name, exist_ok=True)
-    api = HfApi()
-    api.upload_file(
-        path_or_fileobj=csv_file,
-        path_in_repo=csv_file,
-        repo_id=repo_name,
-        repo_type='dataset'
-    )
-    logger.info(f"Dataset {repo_name} created and file {csv_file} uploaded.")
+def process_text_to_csv(input_text):
+    try:
+        tokens = word_tokenize(input_text.lower())
+        words = list(set(tokens))
+        data = []
+        for word in tqdm(words, desc="Processing words"):
+            meanings = []
+            synsets = wordnet.synsets(word)
+            if synsets:
+                meanings = [syn.definition() for syn in synsets[:3]]
+            else:
+                try:
+                    generated_meaning = meaning_generator(f"Define the word '{word}'", max_length=100)[0]['generated_text']
+                    meanings.append(generated_meaning)
+                except Exception as e:
+                    log_error(f"Meaning generation failed for '{word}': {str(e)}")
+            data.append({"tokenizer": tokens, "words": word, "meaning": meanings})
+        
+        # Save to CSV
+        with open(csv_file, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=["tokenizer", "words", "meaning"])
+            writer.writeheader()
+            writer.writerows(data)
+        
+        logging.info(f"Dataset saved to {csv_file}")
+        return data
+    except Exception as e:
+        log_error(f"Error in process_text_to_csv: {str(e)}")
+        raise
 
-# Gradio interface functions
-def generate_words_interface():
-    num_words = random.randint(50, 200)
-    words = generate_random_words(num_words)
-    meanings = get_word_meanings(words)
-    convert_to_csv(meanings)
-    return f"Generated {num_words} random words and saved to dataset.csv."
+def upload_to_huggingface():
+    try:
+        dataset = Dataset.from_csv(csv_file)
+        dataset.push_to_hub("katsukiai/DeepFocus-X3", token=HF_TOKEN)
+        logging.info("Dataset uploaded to Hugging Face")
+    except Exception as e:
+        log_error(f"Error uploading to Hugging Face: {str(e)}")
+        raise
 
-def about_interface():
-    return "This is a dataset generation tool that creates a dataset of random words and their meanings, then uploads it to Hugging Face."
+def generate_output(input_text):
+    try:
+        data = process_text_to_csv(input_text)
+        upload_to_huggingface()
+        return json.dumps(data, indent=2)
+    except Exception as e:
+        log_error(f"Error in generate_output: {str(e)}")
+        return f"Error: {str(e)}"
 
-def logs_interface():
-    with open('dataset_generation.log', 'r') as file:
-        logs = file.read()
-    return logs
+def view_logs():
+    try:
+        log_files = os.listdir(log_dir)
+        log_content = ""
+        for log_file in log_files:
+            with open(os.path.join(log_dir, log_file), 'r') as f:
+                log_content += f"\n\n--- {log_file} ---\n\n{f.read()}"
+        return log_content
+    except Exception as e:
+        log_error(f"Error in view_logs: {str(e)}")
+        return f"Error: {str(e)}"
 
-# Gradio app setup
-with gr.Blocks() as demo:
+# Gradio Interface
+with gr.Blocks(title="DeepFocus-X3") as demo:
+    gr.Markdown("# DeepFocus-X3")
+    
     with gr.Tabs():
-        with gr.Tab("About"):
-            about_text = gr.Markdown(about_interface)
-        with gr.Tab("Generate"):
-            generate_button = gr.Button("Generate Dataset")
-            generate_output = gr.Textbox()
-            generate_button.click(generate_words_interface, outputs=generate_output)
-        with gr.Tab("Logs"):
-            logs_output = gr.Textbox(value=logs_interface(), interactive=False)
+        with gr.TabItem("About"):
+            gr.Markdown("""
+            ## About DeepFocus-X3
+            This application processes text, tokenizes it, extracts unique words, generates meanings, and uploads the dataset to Hugging Face.
+            - Uses NLTK for tokenization and WordNet for meanings.
+            - Leverages DeepSeek AI for long text processing and Google FLAN-T5 for meaning generation.
+            - Logs all activities and errors, with error logs uploaded to Hugging Face.
+            """)
+        
+        with gr.TabItem("Generate all"):
+            input_text = gr.Textbox(label="Input Text", lines=10)
+            output_json = gr.Textbox(label="Output JSON", lines=10)
+            generate_btn = gr.Button("Generate and Upload")
+            generate_btn.click(fn=generate_output, inputs=input_text, outputs=output_json)
+        
+        with gr.TabItem("Logs"):
+            gr.Markdown("## Report using Logs")
+            log_output = gr.Textbox(label="Log Content", lines=20)
+            view_logs_btn = gr.Button("View Logs")
+            view_logs_btn.click(fn=view_logs, inputs=None, outputs=log_output)
 
-# Run the Gradio app
-if __name__ == "__main__":
-    demo.launch()
+# Launch Gradio app
+demo.launch()