commit

glue.py

@@ -18,4 +18,8 @@ glue = get_client_connection()
 response = glue.start_job_run(
     JobName='Article Snapshot China'
 )
+print(response)
+response = glue.start_job_run(
+    JobName='Reference China'
+)
 print(response)

utils.py

@@ -19,8 +19,6 @@ from PyPDF2 import PdfReader
 
 AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
 AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
-# AWS_ACCESS_KEY_ID="AKIAQFXZMGHQYXKWUDWR"
-# AWS_SECRET_ACCESS_KEY="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"
 
 analyzer = pipeline("sentiment-analysis", model="ProsusAI/finbert")
 
@@ -29,7 +27,7 @@ translator = Translator()
 with open('xpath.json', 'r', encoding='UTF-8') as f:
     xpath_dict = json.load(f)
 
-with open('xpath.json', 'r', encoding='UTF-8') as f:
+with open('patterns.json', 'r', encoding='UTF-8') as f:
     patterns = json.load(f)
 
 def get_client_connection():
@@ -48,7 +46,7 @@ def update_reference(report):
                 TableName="reference_china",
                 Key={
                     'id': {'S': str(report['refID'])},
-                    'sourceID': {'S': report['sourceID']}
+                    'sourceID': {'S': str(report['sourceID'])}
                 },
                 UpdateExpression='SET link = :link, referenceID = :referenceID, LastModifiedDate = :LastModifiedDate',
                 ExpressionAttributeValues={
@@ -114,11 +112,8 @@ def isnot_substring(list_a, string_to_check):
     return True
 
 def extract_reference(row):
-    print(row['site'])
-    print(patterns)
     try:
         pattern = next((elem for elem in patterns if elem['site'] == row['site']), None)
-        print(pattern)
         extracted_text = extract_from_pdf_by_pattern(row['attachment'],pattern)
         reference_titles = re.findall(pattern['article_regex'], extracted_text)
         reference_dates = re.findall(pattern['date_regex'], extracted_text)
@@ -129,7 +124,6 @@ def extract_reference(row):
             for remove_string in pattern['remove']:
                 reference_titles = [s.replace(remove_string, '') for s in reference_titles]
         for title, date in zip(reference_titles, reference_dates):
-            print(title, date)
             try:
                 date = datetime.strptime(date, pattern['date_format'])
             except:
@@ -160,12 +154,13 @@ def extract_reference(row):
                 reference_df = data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]
                 row['referenceID'] = reference_df.iloc[0]['id']
                 row['link'] = reference_df.iloc[0]['link']
-                row['sourceID'] = row['id_x']
+                row['sourceID'] = row['id']
                 row['refID'] = uuid.uuid5(uuid.NAMESPACE_OID, str(row['sourceID'])+str(row['referenceID']))
                 print(date, repr(title), row['sourceID'],row['referenceID'])
+                update_reference(row)
     except Exception as error:
         print(error)
-            # update_reference(row)
+            
 
 def translate(text):
     return translator.translate(text, dest='en').text
@@ -251,7 +246,6 @@ def extract_from_pdf(url):
             # first_newline_index = text.find('。\n')
             # text = text[:first_newline_index+1].replace('\n', '') + text[first_newline_index+1:]
             text = text.replace('?\n', '?-\n').replace('！\n', '！-\n').replace('。\n', '。-\n').replace('\n','').replace('?-','?\n').replace('！-','！\n').replace('。-','。\n')
-            print(text)
             if text != '':
                 extracted_text += text
         try: