Add validation for content length and enhance error handling in crawl_by_url function

controllers/utils.py

@@ -660,6 +660,8 @@ def crawl_by_url(url, article):
     page = etree.HTML(html_text)
     contentcn, summary = encode_content(
         page.xpath(xpath_dict[domain]['content']))
+    if contentcn is None or len(contentcn) < 10:
+        return
     article['originSite'] = xpath_dict[domain]['siteCN']
     article['site'] = xpath_dict[domain]['site']
     article['titleCN'] = encode(page.xpath(xpath_dict[domain]['title']))
@@ -679,7 +681,8 @@ def crawl_by_url(url, article):
         if detect(contenteng) != 'en':
             for element in contentcn.split("。"):
                 contenteng += translate(element) + '. '
-    except Exception as e:
+    except (requests.exceptions.RequestException, requests.exceptions.ReadTimeout,
+            PyPDF2.errors.PdfReadError, PyPDF2.errors.DependencyError) as e:
         print(f"An unexpected error occurred: {e}")
     article['content'] = repr(contenteng)[1:-1].strip()
     try:
@@ -702,5 +705,4 @@ def crawl_by_url(url, article):
     vectorize(article)
     openai_vectorize(article)
 
-
 data = download_files_from_s3('data')