commit

utils.py

@@ -114,51 +114,54 @@ def isnot_substring(list_a, string_to_check):
     return True
 
 def extract_reference(row):
-    pattern = next((elem for elem in patterns if elem['site'] == row['site']), None)
-    extracted_text = extract_from_pdf_by_pattern(row['attachment'],pattern)
-    reference_titles = re.findall(pattern['article_regex'], extracted_text)
-    reference_dates = re.findall(pattern['date_regex'], extracted_text)
-    reference_titles = [s.replace(' ', '') for s in reference_titles]
-    reference_dates = [s.replace(' ', '') for s in reference_dates]
-    print(reference_dates, reference_titles)
-    if 'remove' in pattern:
-        for remove_string in pattern['remove']:
-            reference_titles = [s.replace(remove_string, '') for s in reference_titles]
-    for title, date in zip(reference_titles, reference_dates):
-        print(title, date)
-        try:
-            date = datetime.strptime(date, pattern['date_format'])
-        except:
-            date = datetime(2006, 1, 1)
-        dates = []
-        if 'date_range' in pattern:
-            for i in range(pattern['date_range'] + 1):
-                dates.append((date + timedelta(days=i)).strftime('%Y-%m-%d'))
-                dates.append((date - timedelta(days=i)).strftime('%Y-%m-%d'))
-        dates.append(date.strftime('%Y-%m-%d'))
-        date = date.strftime('%Y-%m-%d')
-        if 'split' in pattern:
-            for split_item in pattern['split']:
-                if 'exceptional_string' in split_item:
-                    if split_item['string'] in title and isnot_substring(split_item['exceptional_string'], title):
-                        title = re.split(split_item['string'], title)[split_item['index']]
-                else:
-                    if split_item['string'] in title:
-                        title = title.split(split_item['string'])[split_item['index']]
-        if len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]) == 0:
-            print("------------ = 0 ------------")
-            print(date, repr(title))
-        elif len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]) > 1:
-            print("------------ > 1 ------------")
-            print(date, repr(title))
-        else:
-            print("------------ = 1 ------------")
-            reference_df = data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]
-            row['referenceID'] = reference_df.iloc[0]['id']
-            row['link'] = reference_df.iloc[0]['link']
-            row['sourceID'] = row['id_x']
-            row['refID'] = uuid.uuid5(uuid.NAMESPACE_OID, str(row['sourceID'])+str(row['referenceID']))
-            print(date, repr(title), row['sourceID'],row['referenceID'])
+    try:
+        pattern = next((elem for elem in patterns if elem['site'] == row['site']), None)
+        extracted_text = extract_from_pdf_by_pattern(row['attachment'],pattern)
+        reference_titles = re.findall(pattern['article_regex'], extracted_text)
+        reference_dates = re.findall(pattern['date_regex'], extracted_text)
+        reference_titles = [s.replace(' ', '') for s in reference_titles]
+        reference_dates = [s.replace(' ', '') for s in reference_dates]
+        print(reference_dates, reference_titles)
+        if 'remove' in pattern:
+            for remove_string in pattern['remove']:
+                reference_titles = [s.replace(remove_string, '') for s in reference_titles]
+        for title, date in zip(reference_titles, reference_dates):
+            print(title, date)
+            try:
+                date = datetime.strptime(date, pattern['date_format'])
+            except:
+                date = datetime(2006, 1, 1)
+            dates = []
+            if 'date_range' in pattern:
+                for i in range(pattern['date_range'] + 1):
+                    dates.append((date + timedelta(days=i)).strftime('%Y-%m-%d'))
+                    dates.append((date - timedelta(days=i)).strftime('%Y-%m-%d'))
+            dates.append(date.strftime('%Y-%m-%d'))
+            date = date.strftime('%Y-%m-%d')
+            if 'split' in pattern:
+                for split_item in pattern['split']:
+                    if 'exceptional_string' in split_item:
+                        if split_item['string'] in title and isnot_substring(split_item['exceptional_string'], title):
+                            title = re.split(split_item['string'], title)[split_item['index']]
+                    else:
+                        if split_item['string'] in title:
+                            title = title.split(split_item['string'])[split_item['index']]
+            if len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]) == 0:
+                print("------------ = 0 ------------")
+                print(date, repr(title))
+            elif len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]) > 1:
+                print("------------ > 1 ------------")
+                print(date, repr(title))
+            else:
+                print("------------ = 1 ------------")
+                reference_df = data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]
+                row['referenceID'] = reference_df.iloc[0]['id']
+                row['link'] = reference_df.iloc[0]['link']
+                row['sourceID'] = row['id_x']
+                row['refID'] = uuid.uuid5(uuid.NAMESPACE_OID, str(row['sourceID'])+str(row['referenceID']))
+                print(date, repr(title), row['sourceID'],row['referenceID'])
+    except Exception as error:
+        print(error)
             # update_reference(row)
 
 def translate(text):