commit

.gitignore

@@ -1,3 +1,5 @@
 env
 __pycache__
-downloaded_file.pdf
+downloaded_file.pdf
+downloaded_file.docx
+downloaded_file.doc

cbirc.py

@@ -26,7 +26,7 @@ while i > -1:
                 for element in article['originalContent'].split("。"):
                     CONTENT_ENG += translate(element) + ' '
                 article['content'] = CONTENT_ENG
-                article['site'] = "National Financial Regulatory Administration"
+                article['site'] = "National Financial Regulatory Administration of China"
                 article['originalSite'] = "国家金融监督管理总局"
                 article['originalTitle'] = article['docSubtitle']
                 article['title'] = translate(article['originalTitle'])
@@ -62,7 +62,7 @@ while i > -1:
                 for element in article['originalContent'].split("。"):
                     CONTENT_ENG += translate(element) + ' '
                 article['content'] = CONTENT_ENG
-                article['site'] = "State Taxation Administration"
+                article['site'] = "State Taxation Administration of China"
                 article['originalSite'] = "国家税务总局"
                 article['originalTitle'] = article['title']
                 article['title'] = translate(article['originalTitle'])

chinatax.py

@@ -32,7 +32,7 @@ while i > -1:
                 for element in article['originalContent'].split("。"):
                     CONTENT_ENG += translate(element) + ' '
                 article['content'] = CONTENT_ENG
-                article['site'] = "State Taxation Administration"
+                article['site'] = "State Taxation Administration of China"
                 article['originalSite'] = "国家税务总局"
                 article['originalTitle'] = article['title']
                 article['title'] = translate(article['originalTitle'])
@@ -80,7 +80,7 @@ while i > -1:
                 for element in article['originalContent'].split("。"):
                     CONTENT_ENG += translate(element) + ' '
                 article['content'] = CONTENT_ENG
-                article['site'] = "State Taxation Administration"
+                article['site'] = "State Taxation Administration of China"
                 article['originalSite'] = "国家税务总局"
                 article['originalTitle'] = article['title']
                 article['title'] = translate(article['originalTitle'])

csrc.py

@@ -43,7 +43,7 @@ while i > -1:
                         for element in article['originalContent'].split("。"):
                             CONTENT_ENG += translate(element) + ' '
                         article['content'] = CONTENT_ENG
-                        article['site'] = "Securities Regulatory Commission"
+                        article['site'] = "Securities Regulatory Commission of China"
                         article['originalSite'] = "证监会"
                         article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
                         article['title'] = translate(article['originalTitle'])
@@ -75,7 +75,7 @@ while i > -1:
                 for element in article['originalContent'].split("。"):
                     CONTENT_ENG += translate(element) + ' '
                 article['content'] = CONTENT_ENG
-                article['site'] = "Securities Regulatory Commission"
+                article['site'] = "Securities Regulatory Commission of China"
                 article['originalSite'] = "证监会"
                 article['originalTitle'] = article['title']
                 article['title'] = translate(article['originalTitle'])

gov.py

@@ -43,7 +43,7 @@ while i > -1:
                             for element in article['originalContent'].split("。"):
                                 CONTENT_ENG += translate(element) + ' '
                             article['content'] = CONTENT_ENG
-                            article['site'] = "State Council"
+                            article['site'] = "State Council of China"
                             article['originalSite'] = "国务院"
                             article['originalTitle'] = page.xpath("//title/text()")[0]
                             article['title'] = translate(article['originalTitle'])
@@ -94,7 +94,7 @@ while i > -1:
                             for element in article['originalContent'].split("。"):
                                 CONTENT_ENG += translate(article['originalContent']) + ' '
                             article['content'] = CONTENT_ENG
-                            article['site'] = "State Council"
+                            article['site'] = "State Council of China"
                             article['originalSite'] = "国务院"
                             article['originalTitle'] = page.xpath("//title/text()")[0]
                             article['title'] = translate(article['originalTitle'])

mof.py

@@ -43,7 +43,7 @@ while i > -1:
                         for element in article['originalContent'].split("。"):
                             CONTENT_ENG += translate(element) + ' '
                         article['content'] = CONTENT_ENG
-                        article['site'] = "Ministry of Finance"
+                        article['site'] = "Ministry of Finance of China"
                         article['originalSite'] = "财政部"
                         article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
                         article['title'] = translate(article['originalTitle'])
@@ -93,7 +93,7 @@ while i > -1:
                         for element in article['originalContent'].split("。"):
                             CONTENT_ENG += translate(element) + ' '
                         article['content'] = CONTENT_ENG
-                        article['site'] = "Ministry of Finance"
+                        article['site'] = "Ministry of Finance of China"
                         article['originalSite'] = "财政部"
                         article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
                         article['title'] = translate(article['originalTitle'])

mofcom.py

@@ -48,7 +48,7 @@ for category in categories:
                             for element in article['originalContent'].split("。"):
                                 CONTENT_ENG += translate(element) + ' '
                             article['content'] = CONTENT_ENG
-                            article['site'] = "Ministry of Commerce"
+                            article['site'] = "Ministry of Commerce of China"
                             article['originalSite'] = "商务部"
                             article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
                             article['title'] = translate(article['originalTitle'])

ndrc.py

@@ -43,7 +43,7 @@ while i > -1:
                             for element in article['originalContent'].split("。"):
                                 CONTENT_ENG += translate(element) + ' '
                             article['content'] = CONTENT_ENG
-                            article['site'] = "State Council"
+                            article['site'] = "State Council of China"
                             article['originalSite'] = "国务院"
                             article['originalTitle'] = page.xpath("//title/text()")[0]
                             article['title'] = translate(article['originalTitle'])
@@ -64,7 +64,7 @@ while i > -1:
                             for element in article['originalContent'].split("。"):
                                 CONTENT_ENG += translate(element) + ' '
                             article['content'] = CONTENT_ENG
-                            article['site'] = "National Development and Reform Commission"
+                            article['site'] = "National Development and Reform Commission of China"
                             article['originalSite'] = "国家发展和改革委员会"
                             article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
                             article['title'] = translate(article['originalTitle'])
@@ -86,7 +86,7 @@ while i > -1:
                             for element in article['originalContent'].split("。"):
                                 CONTENT_ENG += translate(element) + ' '
                             article['content'] = CONTENT_ENG
-                            article['site'] = "National Development and Reform Commission"
+                            article['site'] = "National Development and Reform Commission of China"
                             article['originalSite'] = "国家发展和改革委员会"
                             article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
                             article['title'] = translate(article['originalTitle'])

safe.py

@@ -42,7 +42,7 @@ while i > -1:
                         for element in article['originalContent'].split("。"):
                             CONTENT_ENG += translate(element) + ' '
                         article['content'] = CONTENT_ENG
-                        article['site'] = "State Administration of Foregin Exchange"
+                        article['site'] = "State Administration of Foregin Exchange of China"
                         article['originalSite'] = "外汇管理局"
                         article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
                         article['title'] = translate(article['originalTitle'])
@@ -92,7 +92,7 @@ while i > -1:
                         for element in article['originalContent'].split("。"):
                             CONTENT_ENG += translate(element) + ' '
                         article['content'] = CONTENT_ENG
-                        article['site'] = "State Administration of Foregin Exchange"
+                        article['site'] = "State Administration of Foregin Exchange of China"
                         article['originalSite'] = "外汇管理局"
                         article['originalTitle'] = page.xpath("//meta[@name = 'ArticleTitle']/@content")[0]
                         article['title'] = translate(article['originalTitle'])

stats.py

@@ -30,7 +30,7 @@ while i > -1:
                 for url in urls:
                     try:
                         article = {}
-                        url = url.replace('./', "https://www.stats.gov.cn/sj/sjjd/", )
+                        url = url.replace('./', "https://www.stats.gov.cn/sj/sjjd/")
                         req = urllib.request.urlopen(url)
                         text = req.read()
                         html_text = text.decode("utf-8")
@@ -42,7 +42,7 @@ while i > -1:
                         for element in article['originalContent'].split("。"):
                             CONTENT_ENG += translate(element) + ' '
                         article['content'] = CONTENT_ENG
-                        article['site'] = "National Bureau of Statistics"
+                        article['site'] = "National Bureau of Statistics of China"
                         article['originalSite'] = "国家统计局"
                         article['originalTitle'] = page.xpath("//title/text()")[0]
                         article['title'] = translate(article['originalTitle'])

utils.py

@@ -55,12 +55,16 @@ def encode(content):
                 replace('\n','').replace('\t','').replace('\r','').replace('  ','').strip()
         else:
             line = element
+        line = line + '\n'
         text += line
         index = text.find('打印本页')
         if index != -1:
           text = text[:index]
-        
-    return text
+        try:
+          summary = '\n'.join(text.split('\n')[:2])
+        except:
+          summary = text
+    return text, summary
 
 def extract_from_pdf(url):
     # Send a GET request to the URL and retrieve the PDF content
@@ -131,17 +135,54 @@ def upsert_content(report):
         'title': report['title'],
         # 'originalSite': report['originalSite'],
         # 'originalTitle': report['originalTitle'],
-        # 'originalContent': report['originalContent'],
+        'originContent': report['originContent'],
         'category': report['category'],
         # 'author': report['author'],
         'content': report['content'],
-        'publishDate': report['publishDate'],
-        'link': report['url'],
+        'publishDate': report['publishdate'],
+        'link': report['link'],
         # 'attachment': report['reporturl'],
         # 'authorID': str(report['authorid']),
-        'sentimentScore': str(Decimal(report['sentimentScore']).quantize(Decimal('0.01'))),
-        'sentimentLabel': report['sentimentLabel'],
-        'LastModifiedDate': datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+        'entityList': report['entitylist'],
+        'sentimentScore': Decimal(str(report['sentimentscore'])).quantize(Decimal('0.01')),
+        'sentimentLabel': report['sentimentlabel'],
+        'LastModifiedDate': datetime.now().strftime("%Y-%m-%dT%H:%M:%S"),
+        'subtitle': report['subtitle']
     }
     response = table.put_item(Item=item)
     print(response)
+
+def get_client_connection():
+    """Get dynamoDB connection"""
+    dynamodb = boto3.client(
+        service_name='dynamodb',
+        region_name='us-east-1',
+        aws_access_key_id=AWS_ACCESS_KEY_ID,
+        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
+    )
+    return dynamodb
+
+def delete_records(item):
+  dynamodb_client = get_client_connection()
+  dynamodb_client.delete_item(
+            TableName="article_china",
+            Key={
+                'id': {'S': item['id']},
+                'site': {'S': item['site']}
+            }
+        )
+
+def update_content(report):
+  dynamodb = get_client_connection()
+  response = dynamodb.update_item(
+              TableName="article_china",
+              Key={
+                  'id': {'S': report['id']},
+                  'site': {'S': report['site']}
+              },
+              UpdateExpression='SET sentimentScore = :sentimentScore, sentimentLabel = :sentimentLabel',
+              ExpressionAttributeValues={
+                  ':sentimentScore': {'N': str(Decimal(str(report['sentimentscore'])).quantize(Decimal('0.01')))},
+                  ':sentimentLabel': {'S': report['sentimentlabel']}
+              }
+          )