Update pbc.py

pbc.py

@@ -84,8 +84,6 @@ import boto3
 AWS_ACCESS_KEY_ID = "AKIAQFXZMGHQYXKWUDWR"
 AWS_SECRET_ACCESS_KEY = "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"
 
-print(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
-
 def get_db_connection():
     """Get dynamoDB connection"""
     dynamodb = boto3.resource(
@@ -105,9 +103,9 @@ def upsert_content(report):
         'id': str(report['id']),
         'site': report['site'],
         'title': report['title'],
-        'originalSite': report['originalSite'],
-        'originalTitle': report['originalTitle'],
-        'originalContent': report['originalContent'],
+        # 'originalSite': report['originalSite'],
+        # 'originalTitle': report['originalTitle'],
+        # 'originalContent': report['originalContent'],
         'category': report['category'],
         # 'author': report['author'],
         'content': report['content'],
@@ -132,7 +130,6 @@ for categoryu_url in categoryu_urls:
     for url in urls:
         try:
             url = "http://www.pbc.gov.cn" + url
-            print(url)
             article = {}
             response = requests.get(url)
             response.encoding = 'utf-8'
@@ -142,18 +139,15 @@ for categoryu_url in categoryu_urls:
             for element in article['originalContent'].split("。"):
               content_eng += translator.translate(element, dest='en').text + ' '
             article['content'] = content_eng
-            print(article['content'])
             article['site'] = "The People's Bank of China"
             article['originalSite'] = "中国人民银行"
             article['originalTitle'] = page.xpath("//title/text()")[0]
-            print(article['originalTitle'])
             article['title'] = translator.translate(article['originalTitle'], dest='en').text
             article['url'] = url
             article['category']= "Policy Interpretation"
             article['publishDate'] = datemodifier(page.xpath("//meta[@name = '页面生成时间']/@content")[0])
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(article['publishDate'],"%Y-%m-%d")), "%Y-%m-%d")
-            if parsed_datetime < (datetime.today() - timedelta(days=180)):
-              print(article['publishDate'])
+            if parsed_datetime < (datetime.today() - timedelta(days=183)):
               continue
             article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
             label_dict = {
@@ -178,7 +172,6 @@ for categoryu_url in categoryu_urls:
                     sentiment_score = sentiment_score + 0
             article['sentimentScore'] = sentiment_score
             article['sentimentLabel'] = label_dict[sentiment_label]
-            print(article)
             upsert_content(article)
         except Exception as error:
             print(error)