Update document_scrapped.py

document_scrapped.py

@@ -27,7 +27,15 @@ from langchain_core.output_parsers import StrOutputParser
 from unidecode import unidecode
 from langchain_huggingface import HuggingFaceEndpoint
 import os
-
+def trim_input_words(input_str, max_new_tokens = 512, max_total_tokens=32768):
+    words = input_str.split()
+    max_input_tokens = max_total_tokens - max_new_tokens
+    
+    if len(words) > max_input_tokens - 100:
+        words = words[:max_input_tokens]
+    trimmed_input_str = ' '.join(words)
+    
+    return trimmed_input_str
 
 def select_words_until_char_limit(s, char_limit):
     s_no_punct = re.sub(r'[^\w\s]', '', s)  # remove punctuation, but leave spaces
@@ -40,7 +48,8 @@ def select_words_until_char_limit(s, char_limit):
             total_chars += len(word) + 1  # add 1 for the space
         else:
             break
-    return ' '.join(selected_words)
+    f = trim_input_words(' '.join(selected_words))
+    return f
 
 
 
@@ -95,7 +104,8 @@ def excel(link : str) -> str:
                       sample_df = df
             json_data = sample_df.to_json(orient='records')
             js = json.loads(json_data)
-            return f"{js}"
+            rs = trim_input_words(js)
+            return rs
         else:
             print("Failed to download file")
             return "No dat avaible error"
@@ -124,7 +134,7 @@ def csv(link : str) -> str:
 
       json_data = sample_df.to_json(orient='records')
       js = json.loads(json_data)
-      return f"{js}"
+      rs = trim_input_words(js)
 
   except Exception as e:
     return 'No data avaible'