Merge branch 'main' of https://github.com/oxbridge-econ/data-collection-china

controllers/utils.py

@@ -455,12 +455,27 @@ def extract_reference(row):
     """
     try:
         print("Extracting reference for %s", row['id'])
-        pattern = next(
-            (elem for elem in patterns if elem['site'] == row['site']), None)
-        extracted_text = extract_from_pdf_by_pattern(row['attachment'],
-                                                     pattern)
-        reference_titles = re.findall(pattern['article_regex'], extracted_text)
-        reference_dates = re.findall(pattern['date_regex'], extracted_text)
+        # Get the pattern for the given site. If not found, skip extraction.
+        pattern = next((elem for elem in patterns if elem['site'] == row['site']), None)
+        if pattern is None:
+            logging.warning(f"No reference pattern found for site {row['site']}. Skipping reference extraction.")
+            return []
+        
+        # Extract text from PDF. If extraction fails, return an empty list.
+        extracted_text = extract_from_pdf_by_pattern(row.get('attachment', ''), pattern)
+        if not extracted_text:
+            logging.warning(f"PDF extraction returned empty text for record {row['id']}.")
+            return []
+        
+        # Now safely attempt to extract reference titles and dates.
+        reference_titles = re.findall(pattern.get('article_regex', ''), extracted_text) or []
+        reference_dates = re.findall(pattern.get('date_regex', ''), extracted_text) or []
+        
+        # Proceed only if reference_titles and reference_dates are non-empty.
+        if not reference_titles or not reference_dates:
+            logging.info(f"No reference titles or dates found for record {row['id']}.")
+            return []
+        
         reference_titles = [s.replace(' ', '') for s in reference_titles]
         reference_dates = [s.replace(' ', '') for s in reference_dates]
         if 'remove' in pattern: