Update app.py

app.py

@@ -1,173 +1,94 @@
 import os
-import nltk
 import csv
 import logging
-from tqdm import tqdm
 import gradio as gr
-from transformers import pipeline
-from huggingface_hub import HfApi, upload_file, HfFolder
-import time
-
-# Setup Logging
-logging.basicConfig(filename='app.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+import nltk
+from datasets import Dataset, DatasetDict, DatasetInfo, Features, Value, ClassLabel
+from huggingface_hub import HfApi, Repository, create_repo
+from tqdm import tqdm
+from nltk.tokenize import word_tokenize
+from nltk.corpus import wordnet as wn
+import random
+import string
 
-# Download All NLTK Data
-nltk.download('all')
+# Ensure necessary NLTK resources are downloaded
+nltk.download('punkt')
+nltk.download('wordnet')
 
-# Constants
-HF_REPO = "katsukiai/DeepFocus-X3"
-TOKENIZER = 'bert-base-uncased'
-MODELS = ["bert-base-uncased", "gpt2", "roberta-base", "distilbert-base-uncased", "albert-base-v2"]  # Corrected model identifier
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-# Initialize Models
-try:
-    models = {model: pipeline('feature-extraction', model=model) for model in MODELS}
-except Exception as e:
-    logging.error(f"Error initializing models: {e}")
-    models = {}
+# Function to generate random words
+def generate_random_words(num_words=100):
+    words = []
+    for _ in range(num_words):
+        word_length = random.randint(3, 10)
+        word = ''.join(random.choices(string.ascii_lowercase, k=word_length))
+        words.append(word)
+    return words
 
-# Functions
-def process_text(text):
-    tokens = nltk.word_tokenize(text)
-    words = list(set(tokens))
-    means = {}
-    for word in tqdm(words, desc="Processing Words"):
-        word_means = {}
-        for model_name, model in models.items():
-            try:
-                output = model(word)
-                word_means[model_name] = output[0].mean().item()
-            except Exception as e:
-                logging.error(f"Error processing word {word} with model {model_name}: {e}")
-                word_means[model_name] = None
-        means[word] = word_means
-    return {"tokenizer": tokens, "words": words, "meaning": means}
+# Function to get meanings of words using NLTK WordNet
+def get_word_meanings(words):
+    meanings = {}
+    for word in words:
+        synsets = wn.synsets(word)
+        if synsets:
+            meanings[word] = synsets[0].definition()
+        else:
+            meanings[word] = "No definition found."
+    return meanings
 
-def save_to_csv(data, filename="output.csv"):
-    with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
-        writer = csv.DictWriter(csvfile, fieldnames=["word", "meanings"])
+# Function to convert data to CSV format
+def convert_to_csv(data, filename='dataset.csv'):
+    fieldnames = ['word', 'meaning']
+    with open(filename, mode='w', newline='', encoding='utf-8') as file:
+        writer = csv.DictWriter(file, fieldnames=fieldnames)
         writer.writeheader()
-        for word in data['words']:
-            writer.writerow({
-                "word": word,
-                "meanings": str(data['meaning'][word])
-            })
-
-def train_dataset():
-    text = "Your long text goes here..."
-    data = process_text(text)
-    save_to_csv(data)
-    logging.info("Dataset processed and saved to CSV.")
-
-def generate_report():
-    with open('app.log', 'r') as log_file:
-        log_content = log_file.read()
-    return log_content
-
-def get_uptime():
-    uptime = time.strftime('%H:%M:%S', time.gmtime(time.time() - start_time))
-    return f"Uptime: {uptime}"
+        for word, meaning in data.items():
+            writer.writerow({'word': word, 'meaning': meaning})
 
-# Gradio Interface
-def generate_all(text):
-    data = process_text(text)
-    save_to_csv(data)
-    return "Processed data saved to output.csv"
+# Function to create and push dataset to Hugging Face
+def create_and_push_dataset(csv_file='dataset.csv', repo_name='DeepFocus-X3'):
+    # Create a new dataset repository on Hugging Face
+    create_repo(repo_name, exist_ok=True)
+    api = HfApi()
+    api.upload_file(
+        path_or_fileobj=csv_file,
+        path_in_repo=csv_file,
+        repo_id=repo_name,
+        repo_type='dataset'
+    )
+    logger.info(f"Dataset {repo_name} created and file {csv_file} uploaded.")
 
-# Custom CSS for Tailwind CSS
-custom_css = """
-<style>
-@import url('https://cdn.jsdelivr.net/npm/[email protected]/dist/tailwind.min.css');
-body {
-    font-family: 'Arial', sans-serif;
-}
-#title {
-    text-align: center;
-    margin-bottom: 20px;
-}
-#input_text, #output_text, #log_output, #commit_input, #username_input, #metadata_input, #uptime_text {
-    width: 100%;
-    max-width: 600px;
-    margin: 10px 0;
-}
-#generate_button, #report_button, #save_settings_button {
-    width: 100%;
-    max-width: 200px;
-    margin: 10px 0;
-}
-#settings_container {
-    margin-top: 20px;
-}
-</style>
-"""
+# Gradio interface functions
+def generate_words_interface():
+    num_words = random.randint(50, 200)
+    words = generate_random_words(num_words)
+    meanings = get_word_meanings(words)
+    convert_to_csv(meanings)
+    return f"Generated {num_words} random words and saved to dataset.csv."
 
-start_time = time.time()
+def about_interface():
+    return "This is a dataset generation tool that creates a dataset of random words and their meanings, then uploads it to Hugging Face."
 
-with gr.Blocks(css=custom_css) as iface:
-    gr.Markdown("# DeepFocus-X3", elem_id="title")
-    
-    with gr.Tab("Generate All"):
-        with gr.Row():
-            input_text = gr.Textbox(label="Input Text", placeholder="Enter your text here...", elem_id="input_text")
-            output_text = gr.Textbox(label="Output", placeholder="Output will appear here...", elem_id="output_text")
-            generate_button = gr.Button("Generate", elem_id="generate_button")
-        generate_button.click(fn=generate_all, inputs=input_text, outputs=output_text)
-        
-    with gr.Tab("Logs"):
-        with gr.Row():
-            log_output = gr.Textbox(label="Logs", placeholder="Logs will appear here...", elem_id="log_output")
-            report_button = gr.Button("Report using Logs", elem_id="report_button")
-        report_button.click(fn=generate_report, outputs=log_output)
-        
-    with gr.Tab("Settings"):
-        with gr.Row():
-            commit_input = gr.Textbox(label="Commit", placeholder="Enter commit message", elem_id="commit_input")
-            username_input = gr.Textbox(label="Username", placeholder="Enter your username", elem_id="username_input")
-            metadata_input = gr.Textbox(label="Metadata", placeholder="Enter metadata", elem_id="metadata_input")
-            uptime_text = gr.Textbox(label="Uptime", placeholder="Uptime will appear here...", elem_id="uptime_text", interactive=False)
-            save_settings_button = gr.Button("Save Settings", elem_id="save_settings_button")
-        
-        save_settings_button.click(
-            fn=lambda commit, username, metadata: f"Settings saved: {commit}, {username}, {metadata}",
-            inputs=[commit_input, username_input, metadata_input],
-            outputs=[uptime_text]  # Reusing uptime_text for output to show settings saved message
-        )
-        
-        # Update uptime every 10 seconds
-        def update_uptime():
-            return get_uptime()
-        
-        gr.Every(10, fn=update_uptime, outputs=uptime_text)
+def logs_interface():
+    with open('dataset_generation.log', 'r') as file:
+        logs = file.read()
+    return logs
 
-# Run and Push to HuggingFace
-def run_and_push():
-    train_dataset()
-    try:
-        api = HfApi()
-        api.create_repo(repo_id=HF_REPO, private=False, exist_ok=True)
-        upload_file(
-            path_or_fileobj="output.csv",
-            path_in_repo="output.csv",
-            repo_id=HF_REPO
-        )
-        logging.info("Dataset pushed to HuggingFace.")
-    except Exception as e:
-        logging.error(f"Error uploading to HuggingFace: {e}")
-        try:
-            # Log the error to a separate errors repo
-            errors_repo = "katsukiai/errors"
-            api.create_repo(repo_id=errors_repo, private=False, exist_ok=True)
-            with open('upload_error.log', 'w') as error_file:
-                error_file.write(f"Error uploading to HuggingFace: {e}\n")
-            upload_file(
-                path_or_fileobj="upload_error.log",
-                path_in_repo="upload_error.log",
-                repo_id=errors_repo
-            )
-            logging.info("Error log pushed to HuggingFace errors repo.")
-        except Exception as e2:
-            logging.error(f"Failed to log error to HuggingFace errors repo: {e2}")
+# Gradio app setup
+with gr.Blocks() as demo:
+    with gr.Tabs():
+        with gr.Tab("About"):
+            about_text = gr.Markdown(about_interface)
+        with gr.Tab("Generate"):
+            generate_button = gr.Button("Generate Dataset")
+            generate_output = gr.Textbox()
+            generate_button.click(generate_words_interface, outputs=generate_output)
+        with gr.Tab("Logs"):
+            logs_output = gr.Textbox(value=logs_interface(), interactive=False)
 
+# Run the Gradio app
 if __name__ == "__main__":
-    iface.launch()
-    run_and_push()
+    demo.launch()