Update app.py

app.py

@@ -4,14 +4,15 @@ import logging
 import nltk
 from nltk import word_tokenize, pos_tag
 from tqdm import tqdm
-import gradio as gr
+import streamlit as st
 from transformers import AutoTokenizer, AutoModelForCausalLM
 from datasets import Dataset
 from huggingface_hub import HfApi
 import shutil
+import torch
 
 # Setup environment and logging
-os.environ["HF_TOKEN"] = os.getenv("HUGGINGFACE_API_TOKEN")
+os.environ["HF_TOKEN"] = os.getenv("HF_TOKEN", "your_hf_token_here")
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 logger = logging.getLogger(__name__)
 
@@ -19,10 +20,15 @@ logger = logging.getLogger(__name__)
 nltk.download('punkt')
 nltk.download('averaged_perceptron_tagger')
 
-# Load DeepSeek-R1 model and tokenizer
-model_name = "deepseek-ai/DeepSeek-R1"
+# Load DeepSeek-V3 model and tokenizer
+model_name = "deepseek-ai/DeepSeek-V3"  # Updated to V3
 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(model_name,trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    model_name, 
+    trust_remote_code=True, 
+    torch_dtype=torch.float32,  # CPU compatibility
+    device_map="cpu"
+)
 
 # Paths
 converted_dir = "converted/"
@@ -56,15 +62,15 @@ def push_to_hf(dataset_path):
     dataset.push_to_hub("katsukiai/DeepFocus-X3", token=os.environ["HF_TOKEN"])
     logger.info("Dataset pushed successfully")
 
-# Generate text using DeepSeek-R1
+# Generate text using DeepSeek-V3
 def generate_text(input_text):
-    inputs = tokenizer(input_text, return_tensors="pt", truncation=True, max_length=1024)
+    inputs = tokenizer(input_text, return_tensors="pt", truncation=True, max_length=1024).to("cpu")
     outputs = model.generate(**inputs, max_length=2048, num_return_sequences=1)
     return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
-# Gradio conversion function
-def gradio_convert(text):
-    logger.info("Processing text with Gradio...")
+# Streamlit conversion function
+def convert_text(text):
+    logger.info("Processing text with Streamlit...")
     long_text = generate_text(text) if len(text) > 100 else text
     dataset = prepare_dataset(long_text)
     output_file = os.path.join(converted_dir, "output.jsonl")
@@ -72,29 +78,29 @@ def gradio_convert(text):
     push_to_hf(output_file)
     return json.dumps(dataset, indent=2)
 
-# Gradio Interface
-with gr.Blocks(title="Text to JSON Converter") as demo:
-    gr.Markdown("# Text to JSON Converter")
-    
-    with gr.Tab("About"):
-        gr.Markdown("""
-        This tool converts text to JSONL format using NLTK for tokenization and DeepSeek-R1 for long text generation.
+# Streamlit Interface
+def main():
+    st.title("Text to JSON Converter")
+
+    # Tabs using Streamlit expander
+    tab = st.sidebar.selectbox("Select Tab", ["About", "Generate all"])
+
+    if tab == "About":
+        st.markdown("""
+        This tool converts text to JSONL format using NLTK for tokenization and DeepSeek-V3 for long text generation.
         The output is saved in 'converted/' folder and pushed to HuggingFace dataset 'katsukiai/DeepFocus-X3'.
         Format: {"tokenizer": tokens, "words": words, "meaning": means}
         """)
-    
-    with gr.Tab("Generate all"):
-        text_input = gr.Textbox(label="Input Text", lines=10)
-        output_json = gr.Textbox(label="JSON Output", lines=10)
-        convert_btn = gr.Button("Convert & Push")
-        convert_btn.click(
-            fn=gradio_convert,
-            inputs=text_input,
-            outputs=output_json
-        )
 
-# Launch Gradio app
-demo.launch()
+    elif tab == "Generate all":
+        text_input = st.text_area("Input Text", height=200)
+        if st.button("Convert & Push"):
+            with st.spinner("Processing..."):
+                result = convert_text(text_input)
+                st.text_area("JSON Output", value=result, height=200)
+
+if __name__ == "__main__":
+    main()
 
 # Cleanup (optional)
 shutil.rmtree(converted_dir, ignore_errors=True)