commit

controllers/utils.py

@@ -534,6 +534,7 @@ def extract_reference(row):
                     update_reference(row)
     except Exception as error:
         logging.error(error)
+        return None
 
 
 def translist(infolist):
@@ -651,11 +652,6 @@ def crawl_by_url(url, article):
             contenteng += translate(element) + '. '
     article['content'] = repr(contenteng)[1:-1].strip()
     article['subtitle'] = summarize(article['content'])
-    # if 'subtitle' in xpath_dict[domain]:
-    #     article['subtitle'] = translate(
-    #         encode(page.xpath(xpath_dict[domain]['subtitle'])))
-    # else:
-    #     article['subtitle'] = translate(summary)
     article['publishDate'] = datemodifier(
         encode(page.xpath(xpath_dict[domain]['publishdate'])),
         xpath_dict[domain]['datetime_format'])

main.py

@@ -9,6 +9,7 @@ import os
 from dotenv import load_dotenv
 
 from source import cbirc, csrc, eastmoney, gov, mofcom, ndrc, safe, stats, mof
+from glue import glue_job_run
 
 load_dotenv()
 
@@ -29,3 +30,4 @@ if __name__ == '__main__':
     mofcom.crawl(delta)
     ndrc.crawl(delta)
     mof.crawl(delta)
+    glue_job_run()

requirements.txt

@@ -192,3 +192,4 @@ websocket-client==1.8.0
 Werkzeug==3.0.3
 wrapt==1.16.0
 yarl==1.9.4
+prefect==2.20.2

source/cbirc.py

@@ -4,6 +4,7 @@ import logging
 import time
 import uuid
 from datetime import datetime, timedelta
+from prefect import flow
 
 from controllers.summarizer import summarize
 from controllers.utils import (
@@ -14,7 +15,7 @@ from controllers.utils import (
         update_content,
 )
 
-
+@flow(name = "Data Collection - cbirc")
 def crawl(delta):
     """
     Crawls the website 'https://www.cbirc.gov.cn' to fetch and process articles.

source/csrc.py

@@ -6,6 +6,7 @@ import urllib.request
 import uuid
 from datetime import datetime, timedelta
 
+from prefect import flow
 from lxml import etree
 
 from controllers.summarizer import summarize
@@ -18,7 +19,7 @@ from controllers.utils import (
     update_content,
 )
 
-
+@flow(name = "Data Collection - csrc")
 def crawl(delta):
     """
     Crawls the website http://www.csrc.gov.cn to retrieve articles based on the specified delta.

source/eastmoney.py

@@ -6,6 +6,7 @@ import uuid
 from datetime import datetime, timedelta
 from urllib.parse import urlparse
 
+from prefect import flow
 from lxml import etree
 
 from controllers.summarizer import summarize
@@ -45,7 +46,7 @@ def _crawl(url, article):
     text = req.read()
     html_text = text.decode("utf-8")
     page = etree.HTML(html_text)
-    contentcn, summary = encode_content(
+    contentcn, _ = encode_content(
         page.xpath(xpath_dict[domain]['content']))
     article['attachment'] = encode(page.xpath(
         xpath_dict[domain]['attachment']))
@@ -54,6 +55,7 @@ def _crawl(url, article):
         article['site'] = translate(article['orgSName'])
     else:
         article['site'] = translate(article['orgName'])
+    logging.info(article)
     article['titleCN'] = article['title']
     article['title'] = translate(article['title'])
     article['author'] = translate(article['researcher'])
@@ -66,6 +68,7 @@ def _crawl(url, article):
     for element in contentcn.split("\n"):
         contenteng += translate(element) + '\n'
     article['content'] = repr(contenteng)[1:-1].strip()
+    logging.info(article)
     article['subtitle'] = summarize(article['content'])
     article['authorid'] = uuid.uuid5(uuid.NAMESPACE_OID, article['author'])
     article['publishDate'] = datemodifier(
@@ -74,11 +77,11 @@ def _crawl(url, article):
                                article['titleCN'] + article['publishDate'])
     article['sentimentScore'], article[
         'sentimentLabel'] = sentiment_computation(contentcn.replace("\n", ""))
+    logging.info(article)
     extract_reference(article)
     update_content(article)
-    logging.info(article)
-
 
+@flow(name = "Data Collection - eastmoney")
 def crawl(delta):
     """
     Crawls the website data.eastmoney.com and retrieves reports within a specified time range.

source/gov.py

@@ -5,10 +5,11 @@ import urllib.request
 from datetime import datetime, timedelta
 
 from lxml import etree
+from prefect import flow
 
 from controllers.utils import crawl_by_url
 
-
+@flow(name = "Data Collection - gov")
 def crawl(delta):
     """
     Crawls the government website for policy interpretation and latest news articles.

source/mof.py

@@ -5,10 +5,11 @@ import urllib.request
 from datetime import datetime, timedelta
 
 from lxml import etree
+from prefect import flow
 
 from controllers.utils import crawl_by_url
 
-
+@flow(name = "Data Collection - mof")
 def crawl(delta):
     """
     Crawls the website to retrieve articles based on the specified delta.

source/mofcom.py

@@ -5,10 +5,11 @@ import urllib.request
 from datetime import datetime, timedelta
 
 from lxml import etree
+from prefect import flow
 
 from controllers.utils import crawl_by_url
 
-
+@flow(name = "Data Collection - mofcom")
 def crawl(delta):
     """
     Crawls the website http://www.mofcom.gov.cn to retrieve articles based on the specified delta.

source/ndrc.py

@@ -5,10 +5,11 @@ import urllib.request
 from datetime import datetime, timedelta
 
 from lxml import etree
+from prefect import flow
 
 from controllers.utils import crawl_by_url
 
-
+@flow(name = "Data Collection - ndrc")
 def crawl(delta):
     """
     Crawls the website "https://www.ndrc.gov.cn/xxgk/jd/jd/" and retrieves articles based on the specified time delta.

source/safe.py

@@ -5,10 +5,11 @@ import urllib.request
 from datetime import datetime, timedelta
 
 from lxml import etree
+from prefect import flow
 
 from controllers.utils import crawl_by_url
 
-
+@flow(name = "Data Collection - safe")
 def crawl(delta):
     """
     Crawls the website "https://www.safe.gov.cn" to retrieve policy interpretation and data interpretation articles.

source/stats.py

@@ -5,10 +5,11 @@ import urllib.request
 from datetime import datetime, timedelta
 
 from lxml import etree
+from prefect import flow
 
 from controllers.utils import crawl_by_url, encode
 
-
+@flow(name = "Data Collection - stats")
 def crawl(delta):
     """
     Crawls the website "https://www.stats.gov.cn/sj/sjjd/" and retrieves articles within a specified time range.

utils.py

@@ -14,7 +14,7 @@ import requests
 import boto3
 from dotenv import load_dotenv
 from lxml import etree
-from googletrans import Translator
+# from googletrans import Translator
 from transformers import pipeline
 from PyPDF2 import PdfReader
 from langdetect import detect