commit

mof.py

@@ -76,11 +76,10 @@ def extract_from_pdf(url):
 from datetime import datetime, timedelta
 from decimal import Decimal
 import boto3
+import os
 
-AWS_ACCESS_KEY_ID = "AKIAQFXZMGHQYXKWUDWR"
-AWS_SECRET_ACCESS_KEY = "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"
-
-print(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
+AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
 
 def get_db_connection():
     """Get dynamoDB connection"""

ndrc.py

@@ -2,12 +2,19 @@ import requests
 import uuid
 import time
 import urllib.request
+from datetime import datetime, timedelta
+from decimal import Decimal
+import boto3
+import os
 from lxml import etree
 from googletrans import Translator
 from transformers import pipeline
 from PyPDF2 import PdfReader
 analyzer = pipeline("sentiment-analysis", model="ProsusAI/finbert")
 
+AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
+AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
+
 translator = Translator()
 
 def datemodifier(date_string):
@@ -79,17 +86,6 @@ def extract_from_pdf(url):
             extracted_text += text
     return extracted_text, extracted_text_eng
 
-"""Upload file to dynamoDB"""
-# import datetime
-from datetime import datetime, timedelta
-from decimal import Decimal
-import boto3
-
-AWS_ACCESS_KEY_ID = "AKIAQFXZMGHQYXKWUDWR"
-AWS_SECRET_ACCESS_KEY = "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"
-
-print(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
-
 def get_db_connection():
     """Get dynamoDB connection"""
     dynamodb = boto3.resource(
@@ -109,9 +105,9 @@ def upsert_content(report):
         'id': str(report['id']),
         'site': report['site'],
         'title': report['title'],
-        'originalSite': report['originalSite'],
-        'originalTitle': report['originalTitle'],
-        'originalContent': report['originalContent'],
+        # 'originalSite': report['originalSite'],
+        # 'originalTitle': report['originalTitle'],
+        # 'originalContent': report['originalContent'],
         'category': report['category'],
         # 'author': report['author'],
         'content': report['content'],
@@ -126,7 +122,6 @@ def upsert_content(report):
     response = table.put_item(Item=item)
     print(response)
 
-reportList = []
 categoryu_urls = ["https://www.ndrc.gov.cn/xxgk/zcfb/fzggwl/", "https://www.ndrc.gov.cn/xxgk/zcfb/ghxwj/","https://www.ndrc.gov.cn/xxgk/zcfb/ghwb/","https://www.ndrc.gov.cn/xxgk/zcfb/gg/","https://www.ndrc.gov.cn/xxgk/zcfb/tz/","https://www.ndrc.gov.cn/xxgk/zcfb/pifu/","https://www.ndrc.gov.cn/xxgk/zcfb/qt/"]
 for categoryu_url in categoryu_urls:
     req = urllib.request.urlopen(categoryu_url)
@@ -196,7 +191,6 @@ for categoryu_url in categoryu_urls:
                     except Exception as error:
                         print(error)
 
-reportList = []
 categoryu_urls = ["https://www.ndrc.gov.cn/xxgk/jd/jd/index.html"]
 for categoryu_url in categoryu_urls:
     req = urllib.request.urlopen(categoryu_url)

requirements.txt

@@ -21,4 +21,4 @@ s3transfer==0.10.0
 six==1.16.0
 sniffio==1.3.1
 urllib3==2.0.7
-PyPDF2
+PyPDF2==3.0.1