Add rate limiting and retry logic for DuckDuckGo search

search_engine.py

@@ -8,6 +8,10 @@ import time
 import json
 import os
 from urllib.parse import urlparse
+import logging
+import random
+
+logger = logging.getLogger(__name__)
 
 class ModelManager:
     """Manages different AI models for specific tasks"""
@@ -63,9 +67,22 @@ class WebSearchEngine:
     def __init__(self):
         self.processor = ContentProcessor()
         self.session = requests.Session()
-        self.request_delay = 1.0
+        self.request_delay = 2.0  # Increased delay between requests
         self.last_request_time = 0
-        self.ddgs = DDGS()
+        self.max_retries = 3
+        self.ddgs = None
+        self.initialize_search()
+    
+    def initialize_search(self):
+        """Initialize DuckDuckGo search with retries"""
+        for _ in range(self.max_retries):
+            try:
+                self.ddgs = DDGS()
+                return
+            except Exception as e:
+                logger.error(f"Error initializing DDGS: {str(e)}")
+                time.sleep(random.uniform(1, 3))
+        raise Exception("Failed to initialize DuckDuckGo search after multiple attempts")
         
     def is_valid_url(self, url: str) -> bool:
         """Check if URL is valid for crawling"""
@@ -92,11 +109,12 @@ class WebSearchEngine:
             return {'error': f"Invalid URL: {url}"}
             
         try:
-            # Rate limiting
+            # Rate limiting with random delay
             current_time = time.time()
             time_since_last = current_time - self.last_request_time
             if time_since_last < self.request_delay:
-                time.sleep(self.request_delay - time_since_last)
+                delay = self.request_delay - time_since_last + random.uniform(0.5, 1.5)
+                time.sleep(delay)
             
             response = self.session.get(url, timeout=10)
             self.last_request_time = time.time()
@@ -134,10 +152,34 @@ class WebSearchEngine:
     def search(self, query: str, max_results: int = 5) -> Dict:
         """Perform search and process results"""
         try:
-            # Search using DuckDuckGo
+            # Initialize search if needed
+            if self.ddgs is None:
+                self.initialize_search()
+            
+            # Add delay before search
+            time.sleep(random.uniform(1, 2))
+            
+            # Search using DuckDuckGo with retries
             search_results = []
-            for result in self.ddgs.text(query, max_results=max_results):
-                search_results.append(result)
+            retry_count = 0
+            
+            while retry_count < self.max_retries:
+                try:
+                    for result in self.ddgs.text(query, max_results=max_results):
+                        search_results.append(result)
+                        # Add small delay between results
+                        time.sleep(random.uniform(0.2, 0.5))
+                    break
+                except Exception as e:
+                    retry_count += 1
+                    if retry_count >= self.max_retries:
+                        return {'error': f"Search failed after {self.max_retries} attempts: {str(e)}"}
+                    logger.warning(f"Search attempt {retry_count} failed: {str(e)}")
+                    time.sleep(random.uniform(2, 5))
+                    self.initialize_search()
+            
+            if not search_results:
+                return {'error': 'No results found'}
             
             results = []
             for result in search_results:
@@ -145,6 +187,8 @@ class WebSearchEngine:
                     processed = self.process_url(result['link'])
                     if 'error' not in processed:
                         results.append(processed)
+                        # Add delay between processing URLs
+                        time.sleep(random.uniform(0.5, 1.0))
                         
             # Generate insights from results
             all_content = " ".join([r['summary'] for r in results if 'summary' in r])