commit

eastmoney.py

@@ -15,30 +15,30 @@ def crawl(url, article):
     text = req.read()
     html_text = text.decode("utf-8")
     page = etree.HTML(html_text)
-    originContent, summary = encode_content(page.xpath(xpath_dict[domain]['content']))
+    contentCN, summary = encode_content(page.xpath(xpath_dict[domain]['content']))
     article['attachment'] = encode(page.xpath(xpath_dict[domain]['attachment']))
     article['link'] = url
     if article['orgSName'] == "''":
         article['site'] = translate(article['orgSName'])
     else:
         article['site'] = translate(article['orgName'])
-    article['originTitle'] = article['title']
+    article['titleCN'] = article['title']
     article['title'] = translate(article['title'])
     article['author'] = translate(article['researcher'])
     article['originAuthor'] = article['researcher']
-    article['originContent'] = repr(originContent)
+    article['contentCN'] = repr(contentCN)[1:-1].strip()
     article['subtitle'] = translate(summary)
     article['category'] = "Macroeconomic Research"
-    if len(article['originContent']) < 10:
+    if len(article['contentCN']) < 10:
         return None
     CONTENT_ENG = ''
-    for element in originContent.split("\n"):
+    for element in contentCN.split("\n"):
         CONTENT_ENG += translate(element) + '\n'
-    article['content'] = repr(CONTENT_ENG)
+    article['content'] = repr(CONTENT_ENG)[1:-1].strip()
     article['authorid'] = uuid.uuid5(uuid.NAMESPACE_OID, article['author'])
-    article['publishDate'] = datemodifier(article['publishDate'], xpath_dict[domain]['datetime']['format_string'])
-    article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
-    article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(originContent.replace("\n",""))
+    article['publishDate'] = datemodifier(article['publishDate'], xpath_dict[domain]['datetime_format'])
+    article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['titleCN']+article['publishDate'])
+    article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(contentCN.replace("\n",""))
     upsert_content(article)
 
 today = datetime.today().strftime('%Y-%m-%d')

utils.py

@@ -94,15 +94,16 @@ def encode_content(content):
                 replace('\n','').replace('\t','').replace('\r','').replace('  ','').strip()
         else:
             line = element
-        line = line + '\n'
-        text += line
+        if line != '':
+          line = line + '\n'
+          text += line
         index = text.find('打印本页')
-        if index != -1:
-          text = text[:index]
-        try:
-          summary = '\n'.join(text.split('\n')[:2])
-        except:
-          summary = text
+    if index != -1:
+      text = text[:index]
+    try:
+      summary = '\n'.join(text.split('\n')[:2])
+    except:
+      summary = text
     return text, summary
 
 def extract_from_pdf(url):
@@ -180,13 +181,13 @@ def crawl(url, article):
         article['author'] = translate(encode(page.xpath(xpath_dict[domain]['author'])))
     else:
         article['author'] = ""
-    article['contentCN'] = repr(contentCN)
+    article['contentCN'] = repr(contentCN)[1:-1].strip()
     if len(article['contentCN']) < 10:
         return None
     CONTENT_ENG = ''
     for element in contentCN.split("\n"):
         CONTENT_ENG += translate(element) + '\n'
-    article['content'] = repr(CONTENT_ENG)
+    article['content'] = repr(CONTENT_ENG)[1:-1].strip()
     if 'subtitle' in xpath_dict[domain]:
         article['subtitle'] = translate(encode(page.xpath(xpath_dict[domain]['subtitle'])))
     else:
@@ -201,7 +202,7 @@ def crawl(url, article):
 def upsert_content(report):
     """Upsert the content records"""
     dynamodb = get_db_connection()
-    table = dynamodb.Table('article_test')
+    table = dynamodb.Table('article_china')
         # Define the item data
     item = {
         'id': str(report['id']),
@@ -261,4 +262,4 @@ def update_content(report):
                 }
             )
     print(response)
-  
+  

xpath.json

@@ -2,7 +2,7 @@
     "data.eastmoney.com": {
         "attachment": "//a[contains(@class, 'pdf-link')]/@href",
         "content": "//div[contains(@class, 'ctx-content')]//p",
-        "datetime": "%Y-%m-%d %H:%M:%S.%f"
+        "datetime_format": "%Y-%m-%d %H:%M:%S.%f"
     },
     "gov.cn": {
         "title": "//title/text()",