|
|
"""Module to crawl the website 'https://www.ndrc.gov.cn' to fetch and process articles.""" |
|
|
import time |
|
|
import urllib.request |
|
|
from datetime import datetime, timedelta |
|
|
|
|
|
from lxml import etree |
|
|
from prefect import task, get_run_logger |
|
|
|
|
|
from controllers.utils import crawl_by_url |
|
|
|
|
|
@task(name = "Data Collection - ndrc", log_prints = True) |
|
|
def crawl(delta): |
|
|
""" |
|
|
Crawls the website "https://www.ndrc.gov.cn/xxgk/jd/jd/" and retrieves articles based on the specified time delta. |
|
|
|
|
|
Args: |
|
|
delta (int): The number of days to look back for articles. |
|
|
|
|
|
Returns: |
|
|
None |
|
|
|
|
|
Raises: |
|
|
None |
|
|
""" |
|
|
logger = get_run_logger() |
|
|
logger.info("ndrc.gov.cn") |
|
|
i = 0 |
|
|
while i > -1: |
|
|
if i == 0: |
|
|
category_url = "https://www.ndrc.gov.cn/xxgk/jd/jd/index.html" |
|
|
else: |
|
|
category_url = f"https://www.ndrc.gov.cn/xxgk/jd/jd/index_{i}.html" |
|
|
i = i + 1 |
|
|
req = urllib.request.urlopen(category_url, timeout=60) |
|
|
text = req.read() |
|
|
html_text = text.decode("utf-8") |
|
|
page = etree.HTML(html_text) |
|
|
articlelist = page.xpath( |
|
|
"//div[contains(@class, 'list')]/ul/li[not(@class = 'empty')]") |
|
|
for article in articlelist: |
|
|
if isinstance(article, etree._Element): |
|
|
subelement = etree.tostring(article).decode() |
|
|
subpage = etree.HTML(subelement) |
|
|
date = subpage.xpath("//span/text()")[0] |
|
|
parsed_datetime = datetime.strptime( |
|
|
time.strftime("%Y-%m-%d", time.strptime(date, "%Y/%m/%d")), |
|
|
"%Y-%m-%d") |
|
|
if parsed_datetime < (datetime.today() - timedelta(days=delta)): |
|
|
i = -1 |
|
|
else: |
|
|
urls = subpage.xpath("//a[contains(@target, '_blank')]/@href") |
|
|
for url in urls: |
|
|
try: |
|
|
article = {} |
|
|
if "www.gov.cn" in url: |
|
|
article['category'] = "Policy Release" |
|
|
elif "../../zcfb/" in url: |
|
|
url = url.replace("../../zcfb/", |
|
|
"https://www.ndrc.gov.cn/xxgk/zcfb/") |
|
|
article['category'] = "Policy Release" |
|
|
else: |
|
|
url = url.replace("../../", |
|
|
"https://www.ndrc.gov.cn/xxgk/jd/jd/") |
|
|
url = url.replace("./", "https://www.ndrc.gov.cn/xxgk/jd/jd/") |
|
|
article['category'] = "Policy Interpretation" |
|
|
crawl_by_url(url, article) |
|
|
except (urllib.error.URLError, etree.XMLSyntaxError) as error: |
|
|
logger.error(error) |
|
|
|
