Refactor exception handling in multiple files to specify exception types and improve logging

.gitignore

@@ -1,4 +1,4 @@
-.env
+.venv
 data
 venv
 __pycache__

controllers/utils.py

@@ -115,7 +115,7 @@ def encode_content(content):
         text = text[:index]
     try:
         summary = '\n'.join(text.split('\n')[:2])
-    except Exception as e:
+    except (IndexError, AttributeError) as e:
         logging.error(e)
         summary = text
     return text, summary
@@ -382,7 +382,8 @@ def extract_from_pdf_by_pattern(url, pattern):
                 else:
                     text = text.strip()
                 extracted_text += text
-    except:
+    except (requests.exceptions.RequestException, requests.exceptions.ReadTimeout) as e:
+        logging.error(e)
         extracted_text = ''
     return extracted_text.replace('?\n', '?-\n').replace(
         '！\n', '！-\n').replace('。\n', '。-\n').replace('\n', ' ').replace(
@@ -446,7 +447,7 @@ def extract_reference(row):
             for title, date in zip(reference_titles, reference_dates):
                 try:
                     date = datetime.strptime(date, pattern['date_format'])
-                except:
+                except ValueError:
                     date = datetime(2006, 1, 1)
                 dates = []
                 if 'date_range' in pattern:
@@ -590,8 +591,9 @@ def extract_from_pdf(url):
                 extracted_text += text
         try:
             summary = '\n'.join(extracted_text.split('\n')[:2])
-        except:
-            summary = text
+        except (ValueError, KeyError, TypeError) as e:
+            logging.error(e)
+            summary = extracted_text
     return extracted_text, summary
 
 
@@ -651,7 +653,7 @@ def crawl_by_url(url, article):
     article['content'] = repr(contenteng)[1:-1].strip()
     try:
         article['subtitle'] = summarize(article['content'])
-    except:
+    except (ValueError, KeyError, TypeError):
         article['subtitle'] = translate(summary)
     article['publishDate'] = datemodifier(
         encode(page.xpath(xpath_dict[domain]['publishdate'])),

source/cbirc.py

@@ -72,8 +72,8 @@ def crawl(delta):
                         article['author'] = ''
                         try:
                             article['subtitle'] = summarize(article['content'])
-                        except:
+                        except (RuntimeError, ValueError):
                             article['subtitle'] = translate(summary)
                         update_content(article)
-                except Exception as error:
+                except (ValueError, KeyError, TypeError) as error:
                     logger.error(error)

source/csrc.py

@@ -15,7 +15,6 @@ from controllers.utils import (
     fetch_url,
     sentiment_computation,
     translate,
-    update_content,
 )
 
 @task(name = "Data Collection - csrc", log_prints = True)
@@ -73,9 +72,9 @@ def crawl(delta):
                                 article['category'] = "Policy Interpretation"
                                 logger.info(f"Processing article URL: {url}")
                                 crawl_by_url(url, article)
-                            except Exception as error:
+                            except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                                 logger.error(error)
-        except Exception as error:
+        except (urllib.error.URLError, etree.XMLSyntaxError, ValueError) as error:
             i = -1
             logger.error(error)
 
@@ -117,8 +116,8 @@ def crawl(delta):
                         article['content'] = repr(contenteng)[1:-1].strip()
                         try:
                             article['subtitle'] = summarize(article['content'])
-                        except:
-                            article['subtitle'] = translate(summary)
+                        except (RuntimeError, ValueError):
+                            article['subtitle'] = ""
                         article['publishDate'] = time.strftime(
                                 "%Y-%m-%d",
                                 time.strptime(article['publishedTimeStr'],
@@ -132,7 +131,6 @@ def crawl(delta):
                                 article['titleCN'] + article['publishDate'])
                         logger.info(article)
                         # update_content(article)
-        except Exception as error:
+        except (ValueError, KeyError, TypeError) as error:
             i = -1
             logger.error(error)
-

source/eastmoney.py

@@ -70,7 +70,7 @@ def _crawl(url, article):
     print(f'INFO - {article}')
     try:
         article['subtitle'] = summarize(article['content'])
-    except:
+    except (RuntimeError, ValueError):
         article['subtitle'] = translate(summary)
     article['authorid'] = uuid.uuid5(uuid.NAMESPACE_OID, article['author'])
     article['publishDate'] = datemodifier(
@@ -127,7 +127,7 @@ def crawl(delta):
                     try:
                         url = f"https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl={article['encodeUrl']}"
                         _crawl(url, article)
-                    except Exception as error:
+                    except (urllib.error.URLError, json.JSONDecodeError, KeyError) as error:
                         logger.error(error)
             else:
                 i = -1

source/gov.py

@@ -52,7 +52,7 @@ def crawl(delta):
                             if "https://www.gov.cn" in url:
                                 article['category'] = "Policy Interpretation"
                                 crawl_by_url(url, article)
-                        except Exception as error:
+                        except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                             logger.error(error)
     i = 0
     while i > -1:
@@ -85,5 +85,5 @@ def crawl(delta):
                             if "https://www.gov.cn" in url:
                                 article['site'] = "State Council of China"
                                 crawl_by_url(url, article)
-                        except Exception as error:
+                        except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                             logger.error(error)

source/mof.py

@@ -55,7 +55,7 @@ def crawl(delta):
                                     "./", "https://www.mof.gov.cn/zhengwuxinxi/caizhengxinwen/")
                             article['category'] = "Financial News"
                             crawl_by_url(url, article)
-                        except Exception as error:
+                        except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                             logger.error(error)
 
     i = 0
@@ -90,5 +90,5 @@ def crawl(delta):
                             url = url.replace("./", category_url)
                             article['category'] = "Policy Interpretation"
                             crawl_by_url(url, article)
-                        except Exception as error:
+                        except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                             logger.error(error)

source/mofcom.py

@@ -59,8 +59,8 @@ def crawl(delta):
                                     else:
                                         article['category'] = "Policy Release"
                                     crawl_by_url(url, article)
-                                except Exception as error:
+                                except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                                     logger.error(error)
-            except Exception as error:
+            except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                 i = -1
                 logger.error(error)

source/ndrc.py

@@ -64,5 +64,5 @@ def crawl(delta):
                                 url = url.replace("./", "https://www.ndrc.gov.cn/xxgk/jd/jd/")
                                 article['category'] = "Policy Interpretation"
                             crawl_by_url(url, article)
-                        except Exception as error:
+                        except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                             logger.error(error)

source/safe.py

@@ -51,7 +51,7 @@ def crawl(delta):
                             url = "https://www.safe.gov.cn" + url
                             article['category'] = "Policy Interpretation"
                             crawl_by_url(url, article)
-                        except Exception as error:
+                        except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                             logger.error(error)
 
     i = 1
@@ -84,5 +84,5 @@ def crawl(delta):
                             url = "https://www.safe.gov.cn" + url
                             article['category'] = "Data Interpretation"
                             crawl_by_url(url, article)
-                        except Exception as error:
+                        except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                             logger.error(error)

source/stats.py

@@ -54,5 +54,5 @@ def crawl(delta):
                             url = url.replace('./', "https://www.stats.gov.cn/sj/sjjd/")
                             article['category'] = "Data Interpretation"
                             crawl_by_url(url, article)
-                        except Exception as error:
+                        except (urllib.error.URLError, etree.XMLSyntaxError) as error:
                             logger.info(error)