commit

.github/workflows/eastmoney.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   build:
 
-    runs-on: data-collection
+    runs-on: self-hosted
     timeout-minutes: 7200
     steps:
     - uses: actions/checkout@v3

.github/workflows/gov.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   build:
 
-    runs-on: data-collection
+    runs-on: self-hosted
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v3

.github/workflows/mof.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   build:
 
-    runs-on: data-collection
+    runs-on: self-hosted
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v3

.github/workflows/ndrc.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   build:
 
-    runs-on: data-collection
+    runs-on: self-hosted
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v3

.github/workflows/pbc.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   build:
 
-    runs-on: data-collection
+    runs-on: self-hosted
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v3

gov.py

@@ -125,6 +125,7 @@ while i > -1:
       categoryu_url = "https://www.gov.cn/zhengce/jiedu/home.htm"
     else:
       categoryu_url = f"https://www.gov.cn/zhengce/jiedu/home_{i}.htm"
+    i = i + 1
     req = urllib.request.urlopen(categoryu_url)
     text = req.read()
     html_text = text.decode("utf-8")
@@ -137,7 +138,6 @@ while i > -1:
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d")), "%Y-%m-%d")
             if  parsed_datetime < (datetime.today() - timedelta(days=183)):
-                print(categoryu_url)
                 i = -1
             else:
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
@@ -195,6 +195,7 @@ while i > -1:
       categoryu_url = "https://www.gov.cn/zhengce/zuixin/home.htm"
     else:
       categoryu_url = f"https://www.gov.cn/zhengce/zuixin/home_{i}.htm"
+    i = i + 1
     req = urllib.request.urlopen(categoryu_url)
     text = req.read()
     html_text = text.decode("utf-8")
@@ -206,7 +207,9 @@ while i > -1:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
+            if  parsed_datetime < (datetime.today() - timedelta(days=183)):
+                i = -1
+            else:
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:

mof.py

@@ -121,6 +121,7 @@ while i > -1:
       categoryu_url = "https://www.mof.gov.cn/zhengwuxinxi/caizhengxinwen/"
     else:
       categoryu_url = f"https://www.mof.gov.cn/zhengwuxinxi/caizhengxinwen/index_{i}.htm"
+    i = i + 1
     req = urllib.request.urlopen(categoryu_url)
     text = req.read()
     html_text = text.decode("utf-8")
@@ -132,7 +133,9 @@ while i > -1:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
+            if  parsed_datetime < (datetime.today() - timedelta(days=183)):
+                i = -1
+            else:
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:
@@ -250,6 +253,7 @@ while i > -1:
       categoryu_url = "https://www.mof.gov.cn/zhengwuxinxi/zhengcejiedu/"
     else:
       categoryu_url = f"https://www.mof.gov.cn/zhengwuxinxi/zhengcejiedu/index_{i}.htm"
+    i = i + 1
     req = urllib.request.urlopen(categoryu_url)
     text = req.read()
     html_text = text.decode("utf-8")
@@ -261,7 +265,9 @@ while i > -1:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
+            if  parsed_datetime < (datetime.today() - timedelta(days=183)):
+                i = -1
+            else:
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:

ndrc.py

@@ -196,6 +196,7 @@ while i > -1:
       categoryu_url = "https://www.ndrc.gov.cn/xxgk/jd/jd/index.html"
     else:
       categoryu_url = f"https://www.ndrc.gov.cn/xxgk/jd/jd/index_{i}.html"
+    i = i + 1
     req = urllib.request.urlopen(categoryu_url)
     text = req.read()
     html_text = text.decode("utf-8")
@@ -207,7 +208,9 @@ while i > -1:
             subpage = etree.HTML(subelement)
             date = subpage.xpath("//span/text()")[0]
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y/%m/%d")), "%Y-%m-%d")
-            if  parsed_datetime > (datetime.today() - timedelta(days=183)):
+            if  parsed_datetime < (datetime.today() - timedelta(days=183)):
+                i = -1
+            else:
                 urls = subpage.xpath("//a[contains(@target, '_blank')]/@href")
                 for url in urls:
                     try:

pbc.py

@@ -127,7 +127,7 @@ while i > -1:
     else:
         j = i + 1
         categoryu_url = f"http://www.pbc.gov.cn/rmyh/3963412/3963426/index_{j}.html"
-
+    i = i + 1
     response = requests.get(categoryu_url)
     page = etree.HTML(response.text)
     urls = page.xpath("//td[contains(@height,'22')]//a[contains(@target, '_blank')]/@href")
@@ -153,30 +153,31 @@ while i > -1:
             article['publishDate'] = datemodifier(page.xpath("//meta[@name = '页面生成时间']/@content")[0])
             parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(article['publishDate'],"%Y-%m-%d")), "%Y-%m-%d")
             if parsed_datetime < (datetime.today() - timedelta(days=183)):
-              continue
-            article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
-            label_dict = {
-                "positive": "+",
-                "negative": "-",
-                "neutral": "0",
-            }
-            sentiment_score = 0
-            maximum_value = 0
-            raw_sentiment = analyzer(article['content'][:512], return_all_scores=True)
-            sentiment_label = None
-            for sentiment_dict in raw_sentiment[0]:
-                value = sentiment_dict["score"]
-                if value > maximum_value:
-                    sentiment_label = sentiment_dict["label"]
-                    maximum_value = value
-                if sentiment_dict["label"] == "positive":
-                    sentiment_score = sentiment_score + value
-                if sentiment_dict["label"] == "negative":
-                    sentiment_score = sentiment_score - value
-                else:
-                    sentiment_score = sentiment_score + 0
-            article['sentimentScore'] = sentiment_score
-            article['sentimentLabel'] = label_dict[sentiment_label]
-            upsert_content(article)
+                i = -1
+            else:
+                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
+                label_dict = {
+                    "positive": "+",
+                    "negative": "-",
+                    "neutral": "0",
+                }
+                sentiment_score = 0
+                maximum_value = 0
+                raw_sentiment = analyzer(article['content'][:512], return_all_scores=True)
+                sentiment_label = None
+                for sentiment_dict in raw_sentiment[0]:
+                    value = sentiment_dict["score"]
+                    if value > maximum_value:
+                        sentiment_label = sentiment_dict["label"]
+                        maximum_value = value
+                    if sentiment_dict["label"] == "positive":
+                        sentiment_score = sentiment_score + value
+                    if sentiment_dict["label"] == "negative":
+                        sentiment_score = sentiment_score - value
+                    else:
+                        sentiment_score = sentiment_score + 0
+                article['sentimentScore'] = sentiment_score
+                article['sentimentLabel'] = label_dict[sentiment_label]
+                upsert_content(article)
         except Exception as error:
             print(error)