commit

eastmoney.py

@@ -10,6 +10,10 @@ import os
 from datetime import datetime, timedelta
 from decimal import Decimal
 from transformers import pipeline
+
+AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
+AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
+
 analyzer = pipeline("sentiment-analysis", model="ProsusAI/finbert")
 
 translator = Translator()
@@ -50,9 +54,6 @@ def encode(content):
         text += line
     return text
 
-AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
-AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
-
 def get_db_connection():
     """Get dynamoDB connection"""
     dynamodb = boto3.resource(
@@ -90,10 +91,6 @@ def upsert_content(report):
     response = table.put_item(Item=item)
     print(response)
 
-
-reportList = []
-
-
 today = datetime.today().strftime('%Y-%m-%d')
 beginDate = (datetime.today() - timedelta(days=183)).strftime('%Y-%m-%d')
 i = 0
@@ -141,7 +138,10 @@ while i > -1:
                     report['author'] = translator.translate(report['researcher'], dest='en').text
                     report['originalAuthor'] = report['researcher']
                     report['originalContent'] = content
-                    report['content'] = translator.translate(content, dest='en').text
+                    content_eng = ''
+                        for element in article['originalContent'].split("。"):
+                            content_eng += translator.translate(element, dest='en').text + ' '
+                        article['content'] = content_eng
                     report['authorid'] = uuid.uuid5(uuid.NAMESPACE_OID, report['author'])
                     report['publishDate'] = datemodifier(report['publishDate'])
                     report['id'] = uuid.uuid5(uuid.NAMESPACE_OID, report['title']+report['publishDate'])
@@ -167,9 +167,7 @@ while i > -1:
                             sentiment_score = sentiment_score + 0
                     report['sentimentScore'] = sentiment_score
                     report['sentimentLabel'] = label_dict[sentiment_label]
-                    print(report)
                     upsert_content(report)
-                    reportList.append(report)
                 except Exception as error:
                     print(error)
         else:

mof.py

@@ -6,6 +6,14 @@ from lxml import etree
 from googletrans import Translator
 from transformers import pipeline
 from PyPDF2 import PdfReader
+from datetime import datetime, timedelta
+from decimal import Decimal
+import boto3
+import os
+
+AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
+AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
+
 analyzer = pipeline("sentiment-analysis", model="ProsusAI/finbert")
 
 translator = Translator()
@@ -71,16 +79,6 @@ def extract_from_pdf(url):
             extracted_text += text
     return extracted_text, extracted_text_eng
 
-"""Upload file to dynamoDB"""
-# import datetime
-from datetime import datetime, timedelta
-from decimal import Decimal
-import boto3
-import os
-
-AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
-AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
-
 def get_db_connection():
     """Get dynamoDB connection"""
     dynamodb = boto3.resource(
@@ -100,9 +98,9 @@ def upsert_content(report):
         'id': str(report['id']),
         'site': report['site'],
         'title': report['title'],
-        'originalSite': report['originalSite'],
-        'originalTitle': report['originalTitle'],
-        'originalContent': report['originalContent'],
+        # 'originalSite': report['originalSite'],
+        # 'originalTitle': report['originalTitle'],
+        # 'originalContent': report['originalContent'],
         'category': report['category'],
         # 'author': report['author'],
         'content': report['content'],
@@ -117,7 +115,6 @@ def upsert_content(report):
     response = table.put_item(Item=item)
     print(response)
 
-reportList = []
 categoryu_urls = ["https://www.mof.gov.cn/zhengwuxinxi/caizhengxinwen/"]
 for categoryu_url in categoryu_urls:
     req = urllib.request.urlopen(categoryu_url)
@@ -131,11 +128,10 @@ for categoryu_url in categoryu_urls:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=180)):
+            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:
-                        print(url)
                         article = {}
                         url = url.replace("./", "https://www.mof.gov.cn/zhengwuxinxi/caizhengxinwen/")
                         req = urllib.request.urlopen(url)
@@ -143,7 +139,10 @@ for categoryu_url in categoryu_urls:
                         html_text = text.decode("utf-8")
                         page = etree.HTML(html_text)
                         article['originalContent'] = encode(page.xpath("//div[contains(@class, 'TRS_Editor')]//p"))
-                        article['content'] = translator.translate(article['originalContent'], dest='en').text
+                        content_eng = ''
+                        for element in article['originalContent'].split("。"):
+                            content_eng += translator.translate(element, dest='en').text + ' '
+                        article['content'] = content_eng
                         article['site'] = "Ministry of Finance"
                         article['originalSite'] = "财政部"
                         article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
@@ -174,12 +173,10 @@ for categoryu_url in categoryu_urls:
                                 sentiment_score = sentiment_score + 0
                         article['sentimentScore'] = sentiment_score
                         article['sentimentLabel'] = label_dict[sentiment_label]
-                        print(article)
-                        # upsert_content(article)
+                        upsert_content(article)
                     except Exception as error:
                         print(error)
 
-reportList = []
 categoryu_urls = ["https://www.mof.gov.cn/zhengwuxinxi/zhengcefabu/"]
 for categoryu_url in categoryu_urls:
     req = urllib.request.urlopen(categoryu_url)
@@ -193,11 +190,10 @@ for categoryu_url in categoryu_urls:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=180)):
+            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:
-                        print(url)
                         article = {}
                         url = url.replace("./", categoryu_url)
                         req = urllib.request.urlopen(url)
@@ -205,7 +201,10 @@ for categoryu_url in categoryu_urls:
                         html_text = text.decode("utf-8")
                         page = etree.HTML(html_text)
                         article['originalContent'] = encode(page.xpath("//div[contains(@class, 'TRS_Editor')]//p"))
-                        article['content'] = translator.translate(article['originalContent'], dest='en').text
+                        content_eng = ''
+                        for element in article['originalContent'].split("。"):
+                            content_eng += translator.translate(element, dest='en').text + ' '
+                        article['content'] = content_eng
                         article['site'] = "Ministry of Finance"
                         article['originalSite'] = "财政部"
                         article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
@@ -236,12 +235,10 @@ for categoryu_url in categoryu_urls:
                                 sentiment_score = sentiment_score + 0
                         article['sentimentScore'] = sentiment_score
                         article['sentimentLabel'] = label_dict[sentiment_label]
-                        print(article)
-                        # upsert_content(article)
+                        upsert_content(article)
                     except Exception as error:
                         print(error)
 
-reportList = []
 categoryu_urls = ["https://www.mof.gov.cn/zhengwuxinxi/zhengcejiedu/"]
 for categoryu_url in categoryu_urls:
     req = urllib.request.urlopen(categoryu_url)
@@ -255,11 +252,10 @@ for categoryu_url in categoryu_urls:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=180)):
+            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:
-                        print(url)
                         article = {}
                         url = url.replace("./", categoryu_url)
                         req = urllib.request.urlopen(url)
@@ -267,7 +263,10 @@ for categoryu_url in categoryu_urls:
                         html_text = text.decode("utf-8")
                         page = etree.HTML(html_text)
                         article['originalContent'] = encode(page.xpath("//div[contains(@class, 'TRS_Editor')]//p"))
-                        article['content'] = translator.translate(article['originalContent'], dest='en').text
+                        content_eng = ''
+                        for element in article['originalContent'].split("。"):
+                            content_eng += translator.translate(element, dest='en').text + ' '
+                        article['content'] = content_eng
                         article['site'] = "Ministry of Finance"
                         article['originalSite'] = "财政部"
                         article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
@@ -298,7 +297,6 @@ for categoryu_url in categoryu_urls:
                                 sentiment_score = sentiment_score + 0
                         article['sentimentScore'] = sentiment_score
                         article['sentimentLabel'] = label_dict[sentiment_label]
-                        print(article)
-                        # upsert_content(article)
+                        upsert_content(article)
                     except Exception as error:
                         print(error)

ndrc.py

@@ -75,16 +75,14 @@ def extract_from_pdf(url):
         pdf_reader = PdfReader(f)
         num_pages = len(pdf_reader.pages)
         extracted_text = ""
-        extracted_text_eng = ""
         for page in range(num_pages):
             text = pdf_reader.pages[page].extract_text()
             if text and text[0].isdigit():
                 text = text[1:]
             first_newline_index = text.find('\n')
             text = text[:first_newline_index+1].replace('\n', ' ') + text[first_newline_index+1:].replace('\n', '')
-            extracted_text_eng += translator.translate(text, dest='en').text
             extracted_text += text
-    return extracted_text, extracted_text_eng
+    return extracted_text
 
 def get_db_connection():
     """Get dynamoDB connection"""
@@ -135,7 +133,7 @@ for categoryu_url in categoryu_urls:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y/%m/%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=180)):
+            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:
@@ -144,7 +142,6 @@ for categoryu_url in categoryu_urls:
                             url = url.replace("../../", "https://www.ndrc.gov.cn/xxgk/")
                         else:
                             url = url.replace("./", categoryu_url)
-                            print(url)
                             req = urllib.request.urlopen(url)
                             text = req.read()
                             html_text = text.decode("utf-8")
@@ -153,9 +150,12 @@ for categoryu_url in categoryu_urls:
                             for attachment_url in attachment_urls:
                                 if ".pdf" in attachment_url:
                                     pdf_url = url.rsplit('/', 1)[0] + attachment_url.replace('./','/')
-                                    pdf_content, extracted_text_eng = extract_from_pdf(pdf_url)
-                                    article['content'] = extracted_text_eng
+                                    pdf_content = extract_from_pdf(pdf_url)
                                     article['originalContent'] = pdf_content
+                                    content_eng = ''
+                                    for element in article['originalContent'].split("。"):
+                                        content_eng += translator.translate(element, dest='en').text + ' '
+                                    article['content'] = content_eng
                                     article['site'] = "National Development and Reform Commission"
                                     article['originalSite'] = "国家发展和改革委员会"
                                     article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
@@ -186,7 +186,6 @@ for categoryu_url in categoryu_urls:
                                             sentiment_score = sentiment_score + 0
                                     article['sentimentScore'] = sentiment_score
                                     article['sentimentLabel'] = label_dict[sentiment_label]
-                                    print(article)
                                     upsert_content(article)
                     except Exception as error:
                         print(error)
@@ -204,13 +203,12 @@ for categoryu_url in categoryu_urls:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y/%m/%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=180)):
+            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:
                         article = {}
                         if "https://www.gov.cn" in url:
-                          print(url)
                           req = urllib.request.urlopen(url)
                           text = req.read()
                           html_text = text.decode("utf-8")
@@ -220,7 +218,6 @@ for categoryu_url in categoryu_urls:
                           for element in article['originalContent'].split("。"):
                             content_eng += translator.translate(element, dest='en').text + ' '
                           article['content'] = content_eng
-                          print(article['content'])
                           article['site'] = "State Council"
                           article['originalSite'] = "国务院"
                           article['originalTitle'] = page.xpath("//title/text()")[0]
@@ -240,7 +237,6 @@ for categoryu_url in categoryu_urls:
                           for element in article['originalContent'].split("。"):
                             content_eng += translator.translate(element, dest='en').text + ' '
                           article['content'] = content_eng
-                          print(article['content'])
                           article['site'] = "National Development and Reform Commission"
                           article['originalSite'] = "国家发展和改革委员会"
                           article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
@@ -271,7 +267,6 @@ for categoryu_url in categoryu_urls:
                                 sentiment_score = sentiment_score + 0
                         article['sentimentScore'] = sentiment_score
                         article['sentimentLabel'] = label_dict[sentiment_label]
-                        print(article)
                         upsert_content(article)
                     except Exception as error:
                         print(error)