commit

cbirc.py

@@ -19,17 +19,17 @@ while i > -1:
             if  parsed_datetime < (datetime.today() - timedelta(days=183)):
                 i = -1
             else:
-                article['originalContent'] = extract_from_pdf("https://www.cbirc.gov.cn" + article['pdfFileUrl'])
-                if len(article['originalContent']) < 10:
+                article['originContent'] = extract_from_pdf("https://www.cbirc.gov.cn" + article['pdfFileUrl'])
+                if len(article['originContent']) < 10:
                     continue
                 CONTENT_ENG = ''
-                for element in article['originalContent'].split("。"):
-                    CONTENT_ENG += translate(element) + ' '
+                for element in article['originContent'].split("\n"):
+                    CONTENT_ENG += translate(element) + '\n'
                 article['content'] = CONTENT_ENG
                 article['site'] = "National Financial Regulatory Administration of China"
-                article['originalSite'] = "国家金融监督管理总局"
-                article['originalTitle'] = article['docSubtitle']
-                article['title'] = translate(article['originalTitle'])
+                article['originSite'] = "国家金融监督管理总局"
+                article['originTitle'] = article['docSubtitle']
+                article['title'] = translate(article['originTitle'])
                 article['url'] = "https://www.cbirc.gov.cn" + article['pdfFileUrl']
                 article['category']= "Policy Interpretation"
                 article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['publishDate'],"%Y-%m-%d %H:%M:%S"))

eastmoney.py

@@ -1,9 +1,45 @@
 import uuid
 import json
 import urllib.request
+from urllib.parse import urlparse
 from datetime import datetime, timedelta
 from lxml import etree
-from utils import encode, translate, datemodifier, sentiment_computation, upsert_content, fetch_url
+from utils import encode, translate, datemodifier, sentiment_computation, upsert_content, fetch_url, encode_content
+
+with open('xpath.json', 'r', encoding='UTF-8') as f:
+    xpath_dict = json.load(f)
+
+def crawl(url, article):
+    domain = urlparse(url).netloc
+    req = urllib.request.urlopen(url)
+    text = req.read()
+    html_text = text.decode("utf-8")
+    page = etree.HTML(html_text)
+    originContent, summary = encode_content(page.xpath(xpath_dict[domain]['content']))
+    article['attachment'] = encode(page.xpath(xpath_dict[domain]['attachment']))
+    article['link'] = url
+    if article['orgSName'] == "''":
+        article['site'] = translate(article['orgSName'])
+    else:
+        article['site'] = translate(article['orgName'])
+    article['originTitle'] = article['title']
+    article['title'] = translate(article['title'])
+    article['author'] = translate(article['researcher'])
+    article['originAuthor'] = article['researcher']
+    article['originContent'] = repr(originContent)
+    article['subtitle'] = translate(summary)
+    article['category'] = "Macroeconomic Research"
+    if len(article['originContent']) < 10:
+        return None
+    CONTENT_ENG = ''
+    for element in originContent.split("\n"):
+        CONTENT_ENG += translate(element) + '\n'
+    article['content'] = repr(CONTENT_ENG)
+    article['authorid'] = uuid.uuid5(uuid.NAMESPACE_OID, article['author'])
+    article['publishDate'] = datemodifier(article['publishDate'], xpath_dict[domain]['datetime']['format_string'])
+    article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
+    article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(originContent.replace("\n",""))
+    upsert_content(article)
 
 today = datetime.today().strftime('%Y-%m-%d')
 beginDate = (datetime.today() - timedelta(days=183)).strftime('%Y-%m-%d')
@@ -33,37 +69,7 @@ while i > -1:
             for article in reportinfo['data']:
                 try:
                     url = f"https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl={article['encodeUrl']}"
-                    req = urllib.request.urlopen(url)
-                    text = req.read()
-                    html_text = text.decode("utf-8")
-                    page = etree.HTML(html_text)
-                    content = encode(page.xpath("//div[contains(@class, 'ctx-content')]//p"))
-                    reporturl = encode(page.xpath("//a[contains(@class, 'pdf-link')]/@href"))
-                    article['url'] = url
-                    if article['orgSName'] == "''":
-                        article['site'] = translate(article['orgSName'])
-                        article['originalSite'] = article['orgSName']
-                    else:
-                        article['site'] = translate(article['orgName'])
-                        article['originalSite'] = article['orgSName']
-                    article['reporturl'] = reporturl
-                    article['originalTitle'] = article['title']
-                    article['title'] = translate(article['title'])
-                    article['author'] = translate(article['researcher'])
-                    article['originalAuthor'] = article['researcher']
-                    article['originalContent'] = content
-                    article['category'] = "Macroeconomic Research"
-                    if len(article['originalContent']) < 10:
-                        continue
-                    CONTENT_ENG = ''
-                    for element in article['originalContent'].split("。"):
-                        CONTENT_ENG += translate(element) + ' '
-                    article['content'] = CONTENT_ENG
-                    article['authorid'] = uuid.uuid5(uuid.NAMESPACE_OID, article['author'])
-                    article['publishDate'] = datemodifier(article['publishDate'], "%Y-%m-%d %H:%M:%S.%f")
-                    article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
-                    article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
-                    upsert_content(article)
+                    crawl(url,article)
                 except Exception as error:
                     print(error)
         else:

gov.py

@@ -1,9 +1,42 @@
 from datetime import datetime, timedelta
 import uuid
 import time
+from urllib.parse import urlparse
 import urllib.request
 from lxml import etree
-from utils import encode, translate, datemodifier, sentiment_computation, upsert_content
+from utils import encode, translate, datemodifier, sentiment_computation, upsert_content, encode_content
+import json
+
+with open('xpath.json', 'r', encoding='UTF-8') as f:
+    xpath_dict = json.load(f)
+
+def crawl(url, article):
+    domain = urlparse(url).netloc
+    req = urllib.request.urlopen(url)
+    text = req.read()
+    html_text = text.decode("utf-8")
+    page = etree.HTML(html_text)
+    originContent, summary  = encode_content(page.xpath(xpath_dict[domain]['content']))
+    article['originContent'] = repr(originContent)
+    article['subtitle'] = translate(summary)
+    if len(article['originContent']) < 10:
+        return None
+    CONTENT_ENG = ''
+    for element in originContent.split("\n"):
+        print(element)
+        CONTENT_ENG += translate(element) + '\n'
+    article['content'] = repr(CONTENT_ENG)
+    article['site'] = "State Council of China"
+    article['originSite'] = "国务院"
+    article['originTitle'] = encode(page.xpath(xpath_dict[domain]['title']))
+    article['title'] = translate(article['originTitle'])
+    article['author'] = translate(encode(page.xpath(xpath_dict[domain]['author'])))
+    article['link'] = url
+    article['publishDate'] = datemodifier(encode(page.xpath(xpath_dict[domain]['publishdate'])), xpath_dict[domain]['datetime']['format_string'])
+    article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
+    article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(originContent.replace("\n",""))
+    article['attachment'] = ""
+    upsert_content(article)
 
 i = 0
 while i > -1:
@@ -32,27 +65,8 @@ while i > -1:
                         article = {}
                         url = url.replace('../', 'https://www.gov.cn/zhengce/')
                         if "https://www.gov.cn" in url:
-                            req = urllib.request.urlopen(url)
-                            text = req.read()
-                            html_text = text.decode("utf-8")
-                            page = etree.HTML(html_text)
-                            article['originalContent'] = encode(page.xpath("//div[contains(@id, 'UCAP-CONTENT')]//p"))
-                            if len(article['originalContent']) < 10:
-                                continue
-                            CONTENT_ENG = ''
-                            for element in article['originalContent'].split("。"):
-                                CONTENT_ENG += translate(element) + ' '
-                            article['content'] = CONTENT_ENG
-                            article['site'] = "State Council of China"
-                            article['originalSite'] = "国务院"
-                            article['originalTitle'] = page.xpath("//title/text()")[0]
-                            article['title'] = translate(article['originalTitle'])
-                            article['url'] = url
                             article['category']= "Policy Interpretation"
-                            article['publishDate'] = datemodifier(page.xpath("//meta[@name = 'firstpublishedtime']/@content")[0], "%Y-%m-%d-%H:%M:%S")
-                            article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
-                            article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
-                            upsert_content(article)
+                            crawl(url, article)
                     except Exception as error:
                         print(error)
 
@@ -83,26 +97,7 @@ while i > -1:
                         article = {}
                         url = url.replace('../', 'https://www.gov.cn/zhengce/')
                         if "https://www.gov.cn" in url:
-                            req = urllib.request.urlopen(url)
-                            text = req.read()
-                            html_text = text.decode("utf-8")
-                            page = etree.HTML(html_text)
-                            article['originalContent'] = encode(page.xpath("//div[contains(@id, 'UCAP-CONTENT')]//p"))
-                            if len(article['originalContent']) < 10:
-                                continue
-                            CONTENT_ENG = ''
-                            for element in article['originalContent'].split("。"):
-                                CONTENT_ENG += translate(article['originalContent']) + ' '
-                            article['content'] = CONTENT_ENG
-                            article['site'] = "State Council of China"
-                            article['originalSite'] = "国务院"
-                            article['originalTitle'] = page.xpath("//title/text()")[0]
-                            article['title'] = translate(article['originalTitle'])
-                            article['url'] = url
                             article['category']= "Policy Release"
-                            article['publishDate'] = datemodifier(page.xpath("//meta[@name = 'firstpublishedtime']/@content")[0], "%Y-%m-%d-%H:%M:%S")
-                            article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
-                            article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
-                            upsert_content(article)
+                            crawl(url, article)
                     except Exception as error:
                         print(error)

utils.py

@@ -10,10 +10,10 @@ from googletrans import Translator
 from transformers import pipeline
 from PyPDF2 import PdfReader
 
-AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
-AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
-# AWS_ACCESS_KEY_ID="AKIAQFXZMGHQYXKWUDWR"
-# AWS_SECRET_ACCESS_KEY="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"
+# AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
+# AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
+AWS_ACCESS_KEY_ID="AKIAQFXZMGHQYXKWUDWR"
+AWS_SECRET_ACCESS_KEY="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"
 
 analyzer = pipeline("sentiment-analysis", model="ProsusAI/finbert")
 
@@ -44,6 +44,38 @@ def translist(infolist):
     return out
 
 def encode(content):
+    """Encode Function"""
+    text = ''
+    for element in content:
+        if isinstance(element, etree._Element):
+            subelement = etree.tostring(element).decode()
+            subpage = etree.HTML(subelement)
+            tree = subpage.xpath('//text()')
+            line = ''.join(translist(tree)).\
+                replace('\n','').replace('\t','').replace('\r','').replace('  ','').strip()
+        else:
+            line = element
+        text += line
+    return text
+
+# def encode(content):
+#     """Encode Function"""
+#     text = ''
+#     for element in content:
+#         if isinstance(element, etree._Element):
+#             subelement = etree.tostring(element).decode()
+#             subpage = etree.HTML(subelement)
+#             tree = subpage.xpath('//text()')
+#             line = ''.join(translist(tree)).\
+#                 replace('\n','').replace('\t','').replace('\r','').replace('  ','').strip()
+#         else:
+#             line = element
+#         text += line
+#         index = text.find('打印本页')
+#         if index != -1:
+#           text = text[:index]
+
+def encode_content(content):
     """Encode Function"""
     text = ''
     for element in content:
@@ -109,9 +141,9 @@ def sentiment_computation(content):
     }
     sentiment_score = 0
     maximum_value = 0
-    raw_sentiment = analyzer(content[:512], return_all_scores=True)
+    raw_sentiment = analyzer(content[:512], top_k=None)
     sentiment_label = None
-    for sentiment_dict in raw_sentiment[0]:
+    for sentiment_dict in raw_sentiment:
         value = sentiment_dict["score"]
         if value > maximum_value:
             sentiment_label = sentiment_dict["label"]
@@ -127,25 +159,25 @@ def sentiment_computation(content):
 def upsert_content(report):
     """Upsert the content records"""
     dynamodb = get_db_connection()
-    table = dynamodb.Table('article_china')
+    table = dynamodb.Table('article_test')
         # Define the item data
     item = {
         'id': str(report['id']),
         'site': report['site'],
         'title': report['title'],
-        # 'originalSite': report['originalSite'],
-        # 'originalTitle': report['originalTitle'],
+        # 'originSite': report['originSite'],
+        'originTitle': report['originTitle'],
         'originContent': report['originContent'],
         'category': report['category'],
-        # 'author': report['author'],
+        'author': report['author'],
         'content': report['content'],
-        'publishDate': report['publishdate'],
+        'publishDate': report['publishDate'],
         'link': report['link'],
-        # 'attachment': report['reporturl'],
+        'attachment': report['attachment'],
         # 'authorID': str(report['authorid']),
-        'entityList': report['entitylist'],
-        'sentimentScore': Decimal(str(report['sentimentscore'])).quantize(Decimal('0.01')),
-        'sentimentLabel': report['sentimentlabel'],
+        # 'entityList': report['entitylist'],
+        'sentimentScore': Decimal(str(report['sentimentScore'])).quantize(Decimal('0.01')),
+        'sentimentLabel': report['sentimentLabel'],
         'LastModifiedDate': datetime.now().strftime("%Y-%m-%dT%H:%M:%S"),
         'subtitle': report['subtitle']
     }
@@ -165,7 +197,7 @@ def get_client_connection():
 def delete_records(item):
   dynamodb_client = get_client_connection()
   dynamodb_client.delete_item(
-            TableName="article_china",
+            TableName="article_test",
             Key={
                 'id': {'S': item['id']},
                 'site': {'S': item['site']}
@@ -175,7 +207,7 @@ def delete_records(item):
 def update_content(report):
   dynamodb = get_client_connection()
   response = dynamodb.update_item(
-              TableName="article_china",
+              TableName="article_test",
               Key={
                   'id': {'S': report['id']},
                   'site': {'S': report['site']}

xpath.json

@@ -0,0 +1,19 @@
+{
+    "data.eastmoney.com": {
+        "attachment": "//a[contains(@class, 'pdf-link')]/@href",
+        "content": "//div[contains(@class, 'ctx-content')]//p",
+        "datetime": {
+            "format_string": "%Y-%m-%d %H:%M:%S.%f"
+        }
+    },
+    "www.gov.cn": {
+        "title": "//title/text()",
+        "subtitle": "//meta[@name = 'description']/@content",
+        "author": "//meta[@name = 'author']/@content",
+        "publishdate": "//meta[@name = 'firstpublishedtime']/@content",
+        "content": "//div[contains(@id, 'UCAP-CONTENT')]//p",
+        "datetime": {
+            "format_string": "%Y-%m-%d-%H:%M:%S"
+        }
+    }
+}