Update app.py

app.py

@@ -23,6 +23,8 @@ from datetime import datetime
 from dateutil import parser as date_parser
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
+from trafilatura import fetch_url, extract
+import json
 
 
 # Set up basic configuration for logging
@@ -276,15 +278,49 @@ def generate_chunked_response(prompt, model, max_tokens=10000, num_calls=3, temp
     print(f"Final clean response: {final_response[:100]}...")
     return final_response
 
-def duckduckgo_search(query):
-    with DDGS() as ddgs:
-        results = list(ddgs.text(query, max_results=10))
+class SimpleDDGSearch:
+    def search(self, query: str, num_results: int = 5):
+        results = DDGS().text(query, region='wt-wt', safesearch='off', max_results=num_results)
+        return [res["href"] for res in results]
+
+class TrafilaturaWebCrawler:
+    def get_website_content_from_url(self, url: str) -> str:
+        try:
+            downloaded = fetch_url(url)
+            if downloaded is None:
+                return f"Failed to fetch content from URL: {url}"
+            
+            result = extract(downloaded, output_format='json', include_comments=False, with_metadata=True, url=url)
+            if result:
+                result_dict = json.loads(result)
+                title = result_dict.get('title', 'No title found')
+                content = result_dict.get('text', 'No content extracted')
+                
+                if content == 'No content extracted':
+                    content = extract(downloaded, include_comments=False)
+                
+                return f'=========== Website Title: {title} ===========\n\n=========== Website URL: {url} ===========\n\n=========== Website Content ===========\n\n{content}\n\n=========== Website Content End ===========\n\n'
+            else:
+                return f"No content extracted from URL: {url}"
+        except Exception as e:
+            return f"An error occurred while processing {url}: {str(e)}"
+
+def search_and_crawl(query: str, num_results: int = 10):
+    searcher = SimpleDDGSearch()
+    search_results = searcher.search(query, num_results=num_results)
     
-    # Add date to results, defaulting to current date if not available
-    for result in results:
-        result['date'] = date_parser.parse(result.get('published', datetime.now().isoformat()))
+    crawler = TrafilaturaWebCrawler()
+    output = ""
     
-    return results
+    for i, url in enumerate(search_results):
+        output += f"Results for URL {i+1}: {url}\n\n"
+        output += crawler.get_website_content_from_url(url) + "\n"
+        output += "------------------------------------------------------------\n\n"
+    
+    return output
+
+def duckduckgo_search(query):
+    return search_and_crawl(query, num_results=10)
 
 class CitingSources(BaseModel):
     sources: List[str] = Field(