Refactor translation error handling and remove debug print statements in vectorization

controllers/utils.py

@@ -164,8 +164,7 @@ def translate(text):
     for i in range(3):
         try:
             return GoogleTranslator(source='auto', target='en').translate(text)
-        except exceptions.RequestError as e:
-            print(f"Request failed: {e}. Retrying {i + 1}/{3}...")
+        except exceptions.RequestError:
             time.sleep(2)
             return GoogleTranslator(source='auto', target='en').translate(text)
     return ""
@@ -288,7 +287,6 @@ def update_content(report):
             }
         })
     vectorize(report)
-    print(response)
 
 
 def update_reference(report):
@@ -450,7 +448,6 @@ def extract_reference(row):
         reference_dates = re.findall(pattern['date_regex'], extracted_text)
         reference_titles = [s.replace(' ', '') for s in reference_titles]
         reference_dates = [s.replace(' ', '') for s in reference_dates]
-        print("%s - %s", reference_dates, reference_titles)
         if 'remove' in pattern:
             for remove_string in pattern['remove']:
                 reference_titles = [

controllers/vectorizer.py

@@ -37,20 +37,13 @@ def vectorize(article):
     Returns:
     None
     """
-    print("&"*50)
     article['id'] = str(article['id'])
     if isinstance(article, dict):
         article = [article]  # Convert single dictionary to list of dictionaries
     df = pd.DataFrame(article)
     df = df[['id','site','title','titleCN','category','author','content',
              'publishDate','link']]
-    # df = df[['id', 'publishdate', 'author', 'category',
-    #                      'content', 'referenceid', 'site', 'title', 'link']]
-    # df['sentimentScore'] = df['sentimentScore'].round(2)
-    # df['sentimentScore'] = df['sentimentScore'].astype(float)
     df['publishDate'] = pd.to_datetime(df['publishDate'])
-    print(df.columns)
-    print(df['content'].values[0])
     loader = DataFrameLoader(df, page_content_column="content")
     documents = loader.load()
     text_splitter = RecursiveCharacterTextSplitter(
@@ -61,14 +54,9 @@ def vectorize(article):
     )
 
     chunks = text_splitter.split_documents(documents)
-    for chunk in chunks:
-        print(chunk)
-        print("*"*50)
     ids = []
     for chunk in chunks:
         _id = f"{chunk.metadata['id']}-{str(uuid.uuid5(uuid.NAMESPACE_OID,chunk.page_content))}"
-        print(_id)
-        print("-"*50)
         ids.append(_id)
     inserted_ids = vstore.add_documents(chunks, ids=ids)
     print(inserted_ids)