source/mofcom.py

"""Module to crawl the website 'https://www.mofcom.gov.cn' to fetch and process articles."""
import time
import urllib.request
from datetime import datetime, timedelta

from lxml import etree
from prefect import task, get_run_logger

from controllers.utils import crawl_by_url

@task(name = "Data Collection - mofcom", log_prints = True)
def crawl(delta):
    """
    Crawls the website http://www.mofcom.gov.cn to retrieve articles based on the specified delta.

    Parameters:
    - delta (int): The number of days in the past from today to retrieve articles.

    Returns:
    None
    """
    logger = get_run_logger()
    logger.info("mofcom.gov.cn")
    categories = ['jdzhsw', 'jdgnmy', 'jddwmy', 'jdtzhz']
    for category in categories:
        i = 1
        while i > -1:
            if i == 1:
                url = f"http://www.mofcom.gov.cn/article/zcjd/{category}/"
            else:
                url = f"http://www.mofcom.gov.cn/article/zcjd/{category}/?{i}"
            i = i + 1
            try:
                req = urllib.request.urlopen(url, timeout=60)
                text = req.read()
                html_text = text.decode("utf-8")
                page = etree.HTML(html_text)
                articlelist = page.xpath(
                        "//section[contains(@class, 'listCon iListCon f-mt30')]/ul/li")
                for article in articlelist:
                    if isinstance(article, etree._Element):
                        subelement = etree.tostring(article).decode()
                        subpage = etree.HTML(subelement)
                        date = subpage.xpath("//span/text()")[0]
                        parsed_datetime = datetime.strptime(
                                time.strftime("%Y-%m-%d",
                                              time.strptime(date, "%Y-%m-%d %H:%M:%S")),
                                              "%Y-%m-%d")
                        if parsed_datetime < (datetime.today() - timedelta(days=delta)):
                            i = -1
                        else:
                            urls = subpage.xpath("//a/@href")
                            for url in urls:
                                try:
                                    article = {}
                                    if '/article/zcjd' in url:
                                        url = "http://www.mofcom.gov.cn" + url
                                        article['category'] = "Policy Interpretation"
                                    else:
                                        article['category'] = "Policy Release"
                                    crawl_by_url(url, article)
                                except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                                    logger.error(error)
            except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                i = -1
                logger.error(error)