commit

cbirc.py

@@ -22,20 +22,20 @@ while i > -1:
                 i = -1
             else:
                 contentCN, summary = extract_from_pdf("https://www.cbirc.gov.cn" + article['pdfFileUrl'])
-                article['contentCN'] = repr(contentCN)
+                article['contentCN'] = repr(contentCN)[1:-1].strip()
                 if len(contentCN) < 10:
                     continue
                 CONTENT_ENG = ''
                 for element in article['contentCN'].split("\n"):
                     CONTENT_ENG += translate(element) + '\n'
-                article['content'] = repr(CONTENT_ENG)
+                article['content'] = repr(CONTENT_ENG)[1:-1].strip()
                 article['site'] = "National Financial Regulatory Administration of China"
                 article['originSite'] = "国家金融监督管理总局"
                 article['titleCN'] = article['docSubtitle']
                 article['title'] = translate(article['docSubtitle'])
                 article['link'] = "https://www.cbirc.gov.cn" + str(article['pdfFileUrl'])
                 article['category']= "Policy Interpretation"                
-                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
+                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['titleCN']+article['publishDate'])
                 article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
                 article['attachment'] = ''
                 article['author'] = ''
@@ -45,40 +45,40 @@ while i > -1:
             print(error)
 
 
-ssl._create_default_https_context = ssl._create_stdlib_context
-i = 0
-while i > -1:
-    CATEGORY_URL = f"https://www.chinatax.gov.cn/search5/search/s?siteCode=bm29000002&searchWord=&type=1&xxgkResolveType=%E6%96%87%E5%AD%97&pageNum={i}&pageSize=10&cwrqStart=&cwrqEnd=&column=%E6%94%BF%E7%AD%96%E8%A7%A3%E8%AF%BB&likeDoc=0&wordPlace=0&videoreSolveType="
-    i = i + 1
-    urllib3.disable_warnings()
-    try:
-        req = urllib.request.urlopen(CATEGORY_URL, timeout=30)
-    except:
-        break
-    content = req.read().decode("utf-8")
-    reportinfo = json.loads(content)
-    for article in reportinfo['searchResultAll']['searchTotal']:
-        try:
-            parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(article['cwrq'],"%Y-%m-%d %H:%M:%S")), "%Y-%m-%d")
-            if  parsed_datetime < (datetime.today() - timedelta(days=183)):
-                i = -1
-            else:
-                article['originalContent'] = article['content'].replace('\\u','')
-                if len(article['originalContent']) < 10:
-                    continue
-                CONTENT_ENG = ''
-                for element in article['originalContent'].split("。"):
-                    CONTENT_ENG += translate(element) + ' '
-                article['content'] = CONTENT_ENG
-                article['site'] = "State Taxation Administration of China"
-                article['originalSite'] = "国家税务总局"
-                article['originalTitle'] = article['title']
-                article['title'] = translate(article['originalTitle'])
-                article['url'] = article['snapshotUrl']
-                article['category']= "Policy Interpretation"
-                article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['cwrq'],"%Y-%m-%d %H:%M:%S"))
-                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
-                article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
-                upsert_content(article)
-        except Exception as error:
-            print(error)
+# ssl._create_default_https_context = ssl._create_stdlib_context
+# i = 0
+# while i > -1:
+#     CATEGORY_URL = f"https://www.chinatax.gov.cn/search5/search/s?siteCode=bm29000002&searchWord=&type=1&xxgkResolveType=%E6%96%87%E5%AD%97&pageNum={i}&pageSize=10&cwrqStart=&cwrqEnd=&column=%E6%94%BF%E7%AD%96%E8%A7%A3%E8%AF%BB&likeDoc=0&wordPlace=0&videoreSolveType="
+#     i = i + 1
+#     urllib3.disable_warnings()
+#     try:
+#         req = urllib.request.urlopen(CATEGORY_URL, timeout=30)
+#     except:
+#         break
+#     content = req.read().decode("utf-8")
+#     reportinfo = json.loads(content)
+#     for article in reportinfo['searchResultAll']['searchTotal']:
+#         try:
+#             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(article['cwrq'],"%Y-%m-%d %H:%M:%S")), "%Y-%m-%d")
+#             if  parsed_datetime < (datetime.today() - timedelta(days=183)):
+#                 i = -1
+#             else:
+#                 article['originalContent'] = article['content'].replace('\\u','')
+#                 if len(article['originalContent']) < 10:
+#                     continue
+#                 CONTENT_ENG = ''
+#                 for element in article['originalContent'].split("。"):
+#                     CONTENT_ENG += translate(element) + ' '
+#                 article['content'] = CONTENT_ENG
+#                 article['site'] = "State Taxation Administration of China"
+#                 article['originalSite'] = "国家税务总局"
+#                 article['originalTitle'] = article['title']
+#                 article['title'] = translate(article['originalTitle'])
+#                 article['url'] = article['snapshotUrl']
+#                 article['category']= "Policy Interpretation"
+#                 article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['cwrq'],"%Y-%m-%d %H:%M:%S"))
+#                 article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
+#                 article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
+#                 upsert_content(article)
+#         except Exception as error:
+#             print(error)

chinatax.py

@@ -6,7 +6,7 @@ import time
 import urllib.request
 import urllib3
 from lxml import etree
-from utils import encode, translate, sentiment_computation, upsert_content
+from utils import encode, translate, sentiment_computation, upsert_content, encode_content
 
 ssl._create_default_https_context = ssl._create_stdlib_context
 
@@ -25,22 +25,25 @@ while i > -1:
                 print(parsed_datetime)
                 i = -1
             else:
-                article['originalContent'] = article['content'].replace('\\u','')
-                if len(article['originalContent']) < 10:
+                article['category']= "Policy Interpretation"
+                contentCN = article['content'].replace('\\u','')
+                article['contentCN'] = repr(contentCN)[1:-1].strip()
+                if len(contentCN) < 10:
                     continue
                 CONTENT_ENG = ''
-                for element in article['originalContent'].split("。"):
+                for element in contentCN.split("。"):
                     CONTENT_ENG += translate(element) + ' '
                 article['content'] = CONTENT_ENG
                 article['site'] = "State Taxation Administration of China"
                 article['originalSite'] = "国家税务总局"
-                article['originalTitle'] = article['title']
+                article['titleCN'] = article['title']
                 article['title'] = translate(article['originalTitle'])
                 article['url'] = article['snapshotUrl']
-                article['category']= "Policy Interpretation"
+                article['author'] = ""
+                article['attachment'] = ""
                 article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['cwrq'],"%Y-%m-%d %H:%M:%S"))
-                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
-                article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
+                article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(CONTENT_ENG.replace("\n",""))
+                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['titleCN']+article['publishDate'])
                 upsert_content(article)
         except Exception as error:
             print(error)
@@ -73,21 +76,24 @@ while i > -1:
                 text = req.read()
                 html_text = text.decode("utf-8")
                 page = etree.HTML(html_text)
-                article['originalContent'] = encode(page.xpath("//div[contains(@class, 'article')]//p"))
-                if len(article['originalContent']) < 10:
+                contentCN= encode_content(page.xpath("//div[contains(@class, 'article')]//p"))
+                if len(contentCN) < 10:
                     continue
                 CONTENT_ENG = ''
-                for element in article['originalContent'].split("。"):
+                for element in contentCN.split("。"):
                     CONTENT_ENG += translate(element) + ' '
-                article['content'] = CONTENT_ENG
+                article['contentCN'] = repr(contentCN)[1:-1].strip()
+                article['content'] = repr(CONTENT_ENG)[1:-1].strip()
                 article['site'] = "State Taxation Administration of China"
                 article['originalSite'] = "国家税务总局"
-                article['originalTitle'] = article['title']
+                article['titleCN'] = article['title']
                 article['title'] = translate(article['originalTitle'])
                 article['url'] = article['url']
+                article['attachment'] = ""
+                article['author'] = ""
                 article['category']= "Policy Interpretation"
                 article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['publishedTimeStr'],"%Y-%m-%d %H:%M:%S"))
-                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
+                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['titleCN']+article['publishDate'])
                 article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
                 upsert_content(article)
             except Exception as error:

csrc.py

@@ -52,22 +52,22 @@ while i > -1:
                 article['category']= "Financial News"
                 article['site'] = "Securities Regulatory Commission of China"
                 article['originSite'] = "证监会"
-                article['originTitle'] = article['title']
-                article['title'] = translate(article['originTitle'])
+                article['titleCN'] = article['title']
+                article['title'] = translate(article['titleCN'])
                 article['author'] = ''
-                article['originContent'] = repr(article['content'])
-                if len(article['originContent']) < 10:
+                article['contentCN'] = repr(article['content'])[1:-1].strip()
+                if len(article['contentCN']) < 10:
                     continue
                 CONTENT_ENG = ''
-                for element in article['originContent'].split("。"):
+                for element in article['contentCN'].split("。"):
                     CONTENT_ENG += translate(element) + ' '
-                article['content'] = repr(CONTENT_ENG)
+                article['content'] = repr(CONTENT_ENG)[1:-1].strip()
                 article['subtitle'] = article['memo']
                 article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['publishedTimeStr'],"%Y-%m-%d %H:%M:%S"))
                 article['link'] = article['url']
                 article['attachment'] = ""
                 article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
-                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
+                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['titleCN']+article['publishDate'])
                 upsert_content(article)
         except Exception as error:
             print(error)

utils.py

@@ -124,9 +124,12 @@ def extract_from_pdf(url):
             text = pdf_reader.pages[page].extract_text()
             if text and text[0].isdigit():
                 text = text[1:]
-            first_newline_index = text.find('\n')
-            text = text[:first_newline_index+1].replace('\n', ' ') + text[first_newline_index+1:]
-            extracted_text += text
+            # first_newline_index = text.find('。\n')
+            # text = text[:first_newline_index+1].replace('\n', '') + text[first_newline_index+1:]
+            text = text.replace('?\n', '?-\n').replace('！\n', '！-\n').replace('。\n', '。-\n').replace('\n','').replace('?-','?\n').replace('！-','！\n').replace('。-','。\n')
+            print(text)
+            if text != '':
+                extracted_text += text
         try:
             summary = '\n'.join(extracted_text.split('\n')[:2])
         except:
@@ -202,7 +205,7 @@ def crawl(url, article):
 def upsert_content(report):
     """Upsert the content records"""
     dynamodb = get_db_connection()
-    table = dynamodb.Table('article_china')
+    table = dynamodb.Table('article_test')
         # Define the item data
     item = {
         'id': str(report['id']),