commit

chinatax.py

@@ -28,12 +28,12 @@ while i > -1:
                 article['category']= "Policy Interpretation"
                 contentCN = article['content'].replace('\\u','')
                 article['contentCN'] = repr(contentCN)[1:-1].strip()
-                if len(contentCN) < 10:
+                if len(article['contentCN']) < 10:
                     continue
                 CONTENT_ENG = ''
-                for element in contentCN.split("。"):
-                    CONTENT_ENG += translate(element) + ' '
-                article['content'] = CONTENT_ENG
+                for element in contentCN.split("\n"):
+                    CONTENT_ENG += translate(element) + '\n'
+                article['content'] = repr(CONTENT_ENG)[1:-1].strip()
                 article['site'] = "State Taxation Administration of China"
                 article['originalSite'] = "国家税务总局"
                 article['titleCN'] = article['title']
@@ -76,19 +76,20 @@ while i > -1:
                 text = req.read()
                 html_text = text.decode("utf-8")
                 page = etree.HTML(html_text)
-                contentCN= encode_content(page.xpath("//div[contains(@class, 'article')]//p"))
-                if len(contentCN) < 10:
+                contentCN, summary = encode_content(page.xpath("//div[contains(@class, 'article')]//p"))
+                article['contentCN'] = repr(contentCN)[1:-1].strip()
+                if len(article['contentCN']) < 10:
                     continue
                 CONTENT_ENG = ''
-                for element in contentCN.split("。"):
-                    CONTENT_ENG += translate(element) + ' '
-                article['contentCN'] = repr(contentCN)[1:-1].strip()
+                for element in contentCN.split("\n"):
+                    CONTENT_ENG += translate(element) + '\n'
                 article['content'] = repr(CONTENT_ENG)[1:-1].strip()
                 article['site'] = "State Taxation Administration of China"
                 article['originalSite'] = "国家税务总局"
                 article['titleCN'] = article['title']
                 article['title'] = translate(article['originalTitle'])
                 article['url'] = article['url']
+                article['subtitle'] = translate(summary)
                 article['attachment'] = ""
                 article['author'] = ""
                 article['category']= "Policy Interpretation"

mof.py

@@ -11,6 +11,7 @@ while i > -1:
     else:
         CATEGORY_URL = f"https://www.mof.gov.cn/zhengwuxinxi/caizhengxinwen/index_{i}.htm"
     i = i + 1
+    print(CATEGORY_URL)
     req = urllib.request.urlopen(CATEGORY_URL)
     text = req.read()
     html_text = text.decode("utf-8")
@@ -31,6 +32,7 @@ while i > -1:
                         article = {}
                         url = url.replace("../", "https://www.mof.gov.cn/zhengwuxinxi/")
                         url = url.replace("./", "https://www.mof.gov.cn/zhengwuxinxi/caizhengxinwen/")
+                        print(url)
                         article['category']= "Financial News"
                         crawl(url, article)
                     except Exception as error:
@@ -43,6 +45,7 @@ while i > -1:
     else:
         CATEGORY_URL = f"https://www.mof.gov.cn/zhengwuxinxi/zhengcejiedu/index_{i}.htm"
     i = i + 1
+    print(CATEGORY_URL)
     req = urllib.request.urlopen(CATEGORY_URL)
     text = req.read()
     html_text = text.decode("utf-8")
@@ -63,6 +66,7 @@ while i > -1:
                         article = {}
                         url = url.replace("./", CATEGORY_URL)
                         article['category']= "Policy Interpretation"
+                        print(url)
                         crawl(url, article)
                     except Exception as error:
                         print(error)