import json
import ssl
import uuid
import time
import urllib.request
import urllib3
from datetime import datetime, timedelta
from utils import translate, sentiment_computation, upsert_content, fetch_url, extract_from_pdf

i = 1
while i > -1:
    CATEGORY_URL = f"https://www.cbirc.gov.cn/cn/static/data/DocInfo/SelectDocByItemIdAndChild/data_itemId=917,pageIndex={i},pageSize=18.json"
    i = i + 1
    content = fetch_url(CATEGORY_URL)
    reportinfo = json.loads(content)
    for article in reportinfo['data']['rows']:
        try:
            parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(article['publishDate'],"%Y-%m-%d %H:%M:%S")), "%Y-%m-%d")
            if  parsed_datetime < (datetime.today() - timedelta(days=183)):
                i = -1
            else:
                article['originalContent'] = extract_from_pdf("https://www.cbirc.gov.cn" + article['pdfFileUrl'])
                if len(article['originalContent']) < 10:
                    continue
                CONTENT_ENG = ''
                for element in article['originalContent'].split("。"):
                    CONTENT_ENG += translate(element) + ' '
                article['content'] = CONTENT_ENG
                article['site'] = "National Financial Regulatory Administration"
                article['originalSite'] = "国家金融监督管理总局"
                article['originalTitle'] = article['docSubtitle']
                article['title'] = translate(article['originalTitle'])
                article['url'] = "https://www.cbirc.gov.cn" + article['pdfFileUrl']
                article['category']= "Policy Interpretation"
                article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['publishDate'],"%Y-%m-%d %H:%M:%S"))
                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
                article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
                upsert_content(article)
        except Exception as error:
            print(error)


ssl._create_default_https_context = ssl._create_stdlib_context
i = 0
while i > -1:
    CATEGORY_URL = f"https://www.chinatax.gov.cn/search5/search/s?siteCode=bm29000002&searchWord=&type=1&xxgkResolveType=%E6%96%87%E5%AD%97&pageNum={i}&pageSize=10&cwrqStart=&cwrqEnd=&column=%E6%94%BF%E7%AD%96%E8%A7%A3%E8%AF%BB&likeDoc=0&wordPlace=0&videoreSolveType="
    i = i + 1
    urllib3.disable_warnings()
    req = urllib.request.urlopen(CATEGORY_URL)
    content = req.read().decode("utf-8")
    reportinfo = json.loads(content)
    for article in reportinfo['searchResultAll']['searchTotal']:
        try:
            parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(article['cwrq'],"%Y-%m-%d %H:%M:%S")), "%Y-%m-%d")
            if  parsed_datetime < (datetime.today() - timedelta(days=183)):
                i = -1
            else:
                article['originalContent'] = article['content'].replace('\\u','')
                if len(article['originalContent']) < 10:
                    continue
                CONTENT_ENG = ''
                for element in article['originalContent'].split("。"):
                    CONTENT_ENG += translate(element) + ' '
                article['content'] = CONTENT_ENG
                article['site'] = "State Taxation Administration"
                article['originalSite'] = "国家税务总局"
                article['originalTitle'] = article['title']
                article['title'] = translate(article['originalTitle'])
                article['url'] = article['snapshotUrl']
                article['category']= "Policy Interpretation"
                article['publishDate'] = time.strftime("%Y-%m-%d", time.strptime(article['cwrq'],"%Y-%m-%d %H:%M:%S"))
                article['id'] = uuid.uuid5(uuid.NAMESPACE_OID, article['title']+article['publishDate'])
                article['sentimentScore'], article['sentimentLabel'] = sentiment_computation(article['content'])
                upsert_content(article)
        except Exception as error:
            print(error)