refactor logging and streamline content update process

controllers/utils.py

@@ -231,7 +231,7 @@ def update_content(report):
     Returns:
         None
     """
-    print("Updating content for %s", report['id'])
+    logging.info("Updating content for %s", report['id'])
     dynamodb = get_client_connection()
     dynamodb.update_item(
         TableName="Article_China",
@@ -299,9 +299,6 @@ def update_content(report):
                 'L': []
             }
         })
-    # vectorize(report)
-    # openai_vectorize(report)
-
 
 def update_reference(report):
     """
@@ -458,24 +455,24 @@ def extract_reference(row):
         # Get the pattern for the given site. If not found, skip extraction.
         pattern = next((elem for elem in patterns if elem['site'] == row['site']), None)
         if pattern is None:
-            logging.warning(f"No reference pattern found for site {row['site']}. Skipping reference extraction.")
+            logging.warning("No reference pattern found for site %s. Skipping reference extraction.", row['site'])
             return []
-        
+
         # Extract text from PDF. If extraction fails, return an empty list.
         extracted_text = extract_from_pdf_by_pattern(row.get('attachment', ''), pattern)
         if not extracted_text:
-            logging.warning(f"PDF extraction returned empty text for record {row['id']}.")
+            logging.warning("PDF extraction returned empty text for record %s.", row['id'])
             return []
-        
+
         # Now safely attempt to extract reference titles and dates.
         reference_titles = re.findall(pattern.get('article_regex', ''), extracted_text) or []
         reference_dates = re.findall(pattern.get('date_regex', ''), extracted_text) or []
-        
+
         # Proceed only if reference_titles and reference_dates are non-empty.
         if not reference_titles or not reference_dates:
-            logging.info(f"No reference titles or dates found for record {row['id']}.")
+            logging.info("No reference titles or dates found for record %s.", row['id'])
             return []
-        
+
         reference_titles = [s.replace(' ', '') for s in reference_titles]
         reference_dates = [s.replace(' ', '') for s in reference_dates]
         if 'remove' in pattern:
@@ -719,8 +716,7 @@ def crawl_by_url(url, article):
                                article['titleCN'] + article['publishDate'])
     logging.info("%s - %s", article['id'], article['site'])
     article['referenceid'] = None
-    # update_content(article)
-    # vectorize(article)
-    # openai_vectorize(article)
+    update_content(article)
+    vectorize(article)
 
 data = download_files_from_s3('data')

source/eastmoney.py

@@ -96,8 +96,8 @@ def _crawl(url, article, retries=3):
     reference_id = extract_reference(article)
     if reference_id:
         article['referenceid'] = reference_id
-    # update_content(article)
-    # vectorize(article)
+    update_content(article)
+    vectorize(article)
     # openai_vectorize(article)
 
 @task(name = "Data Collection - eastmoney", log_prints = True)