add exponential backoff logic for persistent network issues

source/safe.py

@@ -2,6 +2,7 @@
 import time
 import urllib.request
 from datetime import datetime, timedelta
+import random
 
 from lxml import etree
 from prefect import task, get_run_logger
@@ -28,15 +29,26 @@ def crawl(delta):
         else:
             category_url = f"https://www.safe.gov.cn/safe/zcfgjd/index_{i}.html"
         i = i + 1
-        try:
-            req = urllib.request.urlopen(category_url, timeout=60)
-            text = req.read()
-            html_text = text.decode("utf-8")
-            page = etree.HTML(html_text)
-            articlelist = page.xpath("//div[contains(@class, 'list_conr')]/ul/li")
-        except (urllib.error.URLError, TimeoutError) as error:
-            logger.error(error)
-            continue
+        max_retries = 5
+        backoff_factor = 2
+        retries = max_retries
+        while retries > 0:
+            try:
+                req = urllib.request.urlopen(category_url, timeout=120)  # Increased timeout
+                text = req.read()
+                html_text = text.decode("utf-8")
+                page = etree.HTML(html_text)
+                articlelist = page.xpath("//div[contains(@class, 'list_conr')]/ul/li")
+                break  # Success, exit retry loop
+            except (urllib.error.URLError, TimeoutError) as error:
+                logger.error(f"Network error: {error}. Retries left: {retries-1}")
+                retries -= 1
+                if retries > 0:
+                    sleep_time = backoff_factor ** (max_retries - retries) + random.uniform(0, 1)
+                    time.sleep(sleep_time)
+                else:
+                    logger.error(f"Failed to fetch {category_url} after {max_retries} attempts.")
+                    articlelist = []
         for article in articlelist:
             if isinstance(article, etree._Element):
                 subelement = etree.tostring(article).decode()

source/stats.py

@@ -3,6 +3,7 @@ import time
 import urllib.request
 import http.client
 from datetime import datetime, timedelta
+import random
 
 from lxml import etree
 from prefect import task, get_run_logger
@@ -33,21 +34,26 @@ def crawl(delta):
         else:
             category_url = f"https://www.stats.gov.cn/sj/sjjd/index_{i}.html"
         i = i + 1
-        retries = 3
+        max_retries = 5
+        backoff_factor = 2
+        retries = max_retries
         while retries > 0:
             try:
                 req = urllib.request.urlopen(category_url, timeout=60)
-                retries -= 1
                 text = req.read()
                 html_text = text.decode("utf-8")
                 page = etree.HTML(html_text)
                 articlelist = page.xpath("//div[contains(@class, 'list-content')]/ul/li")
+                break  # Success, exit retry loop
             except (urllib.error.URLError, http.client.IncompleteRead, TimeoutError) as error:
-                logger.info(error)
+                logger.info(f"Network error: {error}. Retries left: {retries-1}")
+                retries -= 1
                 if retries > 0:
-                    time.sleep(5)  # Wait for 5 seconds before retrying
+                    sleep_time = backoff_factor ** (max_retries - retries) + random.uniform(0, 1)
+                    time.sleep(sleep_time)
                 else:
-                    continue  # Skip to the next URL after retries are exhausted
+                    logger.error(f"Failed to fetch {category_url} after {max_retries} attempts.")
+                    articlelist = []  # Prevents UnboundLocalError
         for article in articlelist:
             if isinstance(article, etree._Element):
                 subelement = etree.tostring(article).decode()