commit

daily.py

@@ -354,32 +354,37 @@ for category in categories:
         else:
             URL = f"http://www.mofcom.gov.cn/article/zcjd/{category}/?{i}"
         i = i + 1
-        req = urllib.request.urlopen(URL)
-        text = req.read()
-        html_text = text.decode("utf-8")
-        page = etree.HTML(html_text)
-        articlelist = page.xpath("//section[contains(@class, 'listCon iListCon f-mt30')]/ul/li")
-        for article in articlelist:
-            if isinstance(article, etree._Element):
-                subelement = etree.tostring(article).decode()
-                subpage = etree.HTML(subelement)
-                date = subpage.xpath("//span/text()")[0]
-                parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d %H:%M:%S")), "%Y-%m-%d")
-                if  parsed_datetime < (datetime.today() - timedelta(days=DELTA)):
-                    i = -1
-                else:
-                    urls = subpage.xpath("//a/@href")
-                    for url in urls:
-                        try:
-                            article = {}
-                            if '/article/zcjd' in url:
-                                url = "http://www.mofcom.gov.cn" + url
-                                article['category']= "Policy Interpretation"
-                            else:
-                                article['category']= "Policy Release"
-                            crawl(url, article)
-                        except Exception as error:
-                            print(error)
+        try:
+            req = urllib.request.urlopen(URL)
+            text = req.read()
+            html_text = text.decode("utf-8")
+            page = etree.HTML(html_text)
+            articlelist = page.xpath("//section[contains(@class, 'listCon iListCon f-mt30')]/ul/li")
+            for article in articlelist:
+                if isinstance(article, etree._Element):
+                    subelement = etree.tostring(article).decode()
+                    subpage = etree.HTML(subelement)
+                    date = subpage.xpath("//span/text()")[0]
+                    parsed_datetime = datetime.strptime(time.strftime("%Y-%m-%d", time.strptime(date,"%Y-%m-%d %H:%M:%S")), "%Y-%m-%d")
+                    if  parsed_datetime < (datetime.today() - timedelta(days=DELTA)):
+                        i = -1
+                    else:
+                        urls = subpage.xpath("//a/@href")
+                        for url in urls:
+                            try:
+                                article = {}
+                                if '/article/zcjd' in url:
+                                    url = "http://www.mofcom.gov.cn" + url
+                                    article['category']= "Policy Interpretation"
+                                else:
+                                    article['category']= "Policy Release"
+                                crawl(url, article)
+                            except Exception as error:
+                                print(error)
+        except Exception as error:
+            i = -1
+            print(error)
+
 
 print("ndrc.gov.cn")
 i = 0