Update app.py

app.py

@@ -303,7 +303,7 @@ agents =[
     "PYTHON_CODE_DEV"
 ]
 def generate(
-        prompt, history, agent_name=agents[0], sys_prompt="", temperature=0.9, max_new_tokens=2048, top_p=0.95, repetition_penalty=1.0,
+    prompt, history, agent_name=agents[0], sys_prompt="", temperature=0.9, max_new_tokens=2048, top_p=0.95, repetition_penalty=1.0,
 ):
     seed = random.randint(1,1111111111111111)
 
@@ -320,14 +320,27 @@ def generate(
         temperature = 1e-2
     top_p = float(top_p)
 
-    generate_kwargs = dict(
-        temperature=temperature,
-        max_new_tokens=max_new_tokens,  # Reduced max_new_tokens
-        top_p=top_p,
-        repetition_penalty=repetition_penalty,
-        do_sample=True,
-        seed=seed,
-    )
+
+
+def generate_text_chunked(input_text, model, generation_parameters, max_tokens_to_generate):
+    """Generates text in chunks to avoid token limit errors."""
+    sentences = nltk.sent_tokenize(input_text)
+    generated_text = []
+    generator = pipeline('text-generation', model=model)
+
+    for sentence in sentences:
+        # Tokenize the sentence and check if it's within the limit
+        tokens = generator.tokenizer(sentence).input_ids
+        if len(tokens) + max_tokens_to_generate <= 32768:
+            # Generate text for this chunk
+            response = generator(sentence, max_length=max_tokens_to_generate, **generation_parameters)
+            generated_text.append(response[0]['generated_text'])
+        else:
+            # Handle cases where the sentence is too long
+            # You could split the sentence further or skip it
+            print(f"Sentence too long: {sentence}") 
+
+    return ''.join(generated_text)
 
     formatted_prompt = format_prompt(prompt, history, max_history_turns=5)  # Truncated history
     logging.info(f"Formatted Prompt: {formatted_prompt}")