| """ | |
| This script is used to crawl and collect data from the website of the China Securities Regulatory Commission (CSRC). | |
| It retrieves policy interpretation articles and financial news articles from the CSRC website. | |
| The collected data is then processed and stored in a database. | |
| The script consists of two main parts: | |
| 1. Crawl and process policy interpretation articles from the CSRC website. | |
| 2. Crawl and process financial news articles from the CSRC website. | |
| The script uses various libraries and functions to handle web scraping, data processing, and database operations. | |
| Note: This script assumes the presence of the following dependencies: | |
| - urllib | |
| - lxml | |
| - json | |
| - datetime | |
| - time | |
| - utils (custom module) | |
| Please make sure to install these dependencies before running the script. | |
| """ | |
| import uuid | |
| import json | |
| import time | |
| import urllib.request | |
| from datetime import datetime, timedelta | |
| from lxml import etree | |
| from utils import encode, translate, sentiment_computation, upsert_content, fetch_url, crawl | |
| i = 1 | |
| while i > -1: | |
| if i == 1: | |
| CATEGORY_URL = "http://www.csrc.gov.cn/csrc/c100039/common_list.shtml" | |
| else: | |
| CATEGORY_URL = f"http://www.csrc.gov.cn/csrc/c100039/common_list_{i}.shtml" | |
| i = i + 1 | |
| req = urllib.request.urlopen(CATEGORY_URL) | |
| text = req.read() | |
| html_text = text.decode("utf-8") | |
| page = etree.HTML(html_text) | |
| articlelist = page.xpath("//div[contains(@class, 'main-right fr common-list')]/ul/li") | |
| for article in articlelist: | |
| if isinstance(article, etree._Element): | |
| subelement = etree.tostring(article).decode() | |
| subpage = etree.HTML(subelement) | |
| date = encode(subpage.xpath("//span[@class='date']")) | |
| parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d")), "%Y-%m-%d") | |
| if parsed_datetime < (datetime.today() - timedelta(days=183)): | |
| i = -1 | |
| else: | |
| urls = subpage.xpath("//a/@href") | |
| for url in urls: | |
| try: | |
| article = {} | |
| url = "http://www.csrc.gov.cn" + url | |
| article['category']= "Policy Interpretation" | |
| crawl(url, article) | |
| except Exception as error: | |
| print(error) | |
| i = 1 | |
| while i > -1: | |
| CATEGORY_URL = f"http://www.csrc.gov.cn/searchList/a1a078ee0bc54721ab6b148884c784a8?_isAgg=true&_isJson=true&_pageSize=18&_template=index&_rangeTimeGte=&_channelName=&page={i}" | |
| i = i + 1 | |
| content = fetch_url(CATEGORY_URL) | |
| reportinfo = json.loads(content) | |
| for article in reportinfo['data']['results']: | |
| try: | |
| parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(article['publishedTimeStr'],"%Y-%m-%d %H:%M:%S")), "%Y-%m-%d") | |
| if parsed_datetime < (datetime.today() - timedelta(days=183)): | |
| i = -1 | |
| else: | |
| article['category']= "Financial News" | |
| article['site'] = "Securities Regulatory Commission of China" | |
| article['originSite'] = "证监会" | |
| article['titleCN'] = article['title'] | |
| article['title'] = translate(article['titleCN']) | |
| article['author'] = '' | |
| article['contentCN'] = repr(article['content'])[1:-1].strip() | |
| if len(article['contentCN']) < 10: | |
| continue | |
| CONTENT_ENG = '' | |
| for element in article['contentCN'].split("。"): | |
| CONTENT_ENG += translate(element) + ' ' | |
| article['content'] = repr(CONTENT_ENG)[1:-1].strip() | |
| article['subtitle'] = article['memo'] | |
| article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['publishedTimeStr'],"%Y-%m-%d %H:%M:%S")) | |
| article['link'] = article['url'] | |
| article['attachment'] = "" | |
| article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content']) | |
| article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['titleCN']+article['publishDate']) | |
| upsert_content(article) | |
| except Exception as error: | |
| print(error) | |