Update utils.py

utils.py

@@ -496,11 +496,11 @@ def run_research_agent(
     """
     Low-Call approach:
       1) Tavily search (up to 20 URLs).
-      2) Firecrawl scrape => combined text
-      3) Truncate to 12k tokens total
-      4) Split into chunks (each 4500 tokens) => Summarize each chunk individually => summaries
-      5) Single final merge => final PDF
-      => 2 or more total LLM calls (but no more than 10) to reduce the chance of rate limit errors.
+      2) Firecrawl scrape => combined text from the URLs.
+      3) (No truncation) Use the full richness of the scraped materials.
+      4) Split the text into chunks (each 4500 tokens) and summarize each chunk individually.
+      5) Use a single final merge call to produce a comprehensive, detailed, and exhaustive research report.
+         The final report must adhere to world-class research report guidelines.
     """
     print(f"[LOG] Starting LOW-CALL research agent for topic: {topic}")
 
@@ -541,16 +541,17 @@ def run_research_agent(
             print("[LOG] Could not retrieve content from any search results. Exiting.")
             return "Could not retrieve content from any of the search results."
 
-        # Step 3: Truncate to 12k tokens total
-        print("[LOG] Step 3: Truncating combined text to 12,000 tokens if needed.")
-        combined_content = truncate_text_tokens(combined_content, max_tokens=12000)
+        # Input Sanitization: Remove any chain-of-thought markers from the scraped content.
+        combined_content = re.sub(r"<think>.*?</think>", "", combined_content, flags=re.DOTALL)
+
+        # Note: The previous truncation to 12,000 tokens is removed so the full content is used.
 
         # Step 4: Splitting text into chunks (4500 tokens each) and summarizing each chunk.
         print("[LOG] Step 4: Splitting text into chunks (4500 tokens each). Summarizing each chunk.")
         tokenizer = tiktoken.get_encoding("cl100k_base")
         tokens = tokenizer.encode(combined_content)
         chunk_size = 4500          # Reduced chunk size to avoid exceeding the LLM's TPM limit.
-        max_chunks = 10            # Allow up to 10 chunks (and thus 10 LLM calls).
+        max_chunks = 10            # Allow up to 10 chunks (and thus up to 10 LLM calls).
         summaries = []
         start = 0
         chunk_index = 1
@@ -564,8 +565,7 @@ def run_research_agent(
             prompt = f"""
 You are a specialized summarization engine. Summarize the following text 
 for a professional research report. Provide accurate details but do not 
-include chain-of-thought or internal reasoning. Keep it concise, but 
-include key data points and context:
+include any chain-of-thought or internal planning. Keep it concise, yet capture all key points:
 {chunk_text}
 """
             data = {
@@ -580,36 +580,56 @@ include key data points and context:
             start = end
             chunk_index += 1
 
-        # Step 5: Single final merge call
-        print("[LOG] Step 5: Doing one final merge of chunk summaries.")
+        # Step 5: Single final merge call with enhanced instructions.
+        print("[LOG] Step 5: Merging chunk summaries into the final research report.")
         references_text = "\n".join(f"- {url}" for url in references_list) if references_list else "None"
         truncated_summaries = [truncate_text_for_llm(s, max_tokens=1000) for s in summaries]
         merged_input = "\n\n".join(truncated_summaries)
 
+        # Enhanced final prompt including world-class report guidelines.
         final_prompt = f"""
-IMPORTANT: Do NOT include chain-of-thought or hidden planning. 
-Produce a long, academic-style research paper with the following structure:
-- Title Page (concise descriptive title)
-- Table of Contents
-- Executive Summary
-- Introduction
-- Historical or Contextual Background
-- Multiple Thematic Sections (with subheadings)
-- Detailed Analysis (multi-paragraph sections)
-- Footnotes or inline citations referencing the URLs
-- Conclusion
-- References / Bibliography (list these URLs at the end)
-Requirements:
-- Minimal bullet points, prefer multi-paragraph
-- Each section at least 2-3 paragraphs
-- Aim for 1500+ words if possible
-- Under 6000 tokens total
-- Professional, academic tone
+IMPORTANT: Do NOT include any chain-of-thought, internal planning, or hidden reasoning in the final output.
+Draft a professional, world-class research report that adheres to the following tenets:
+
+I. Essential Principles and Qualities:
+   - Accuracy: Present accurate facts with no spelling or grammatical errors.
+   - Clarity: Use clear, straightforward language.
+   - Brevity: Be concise yet complete.
+   - Objectivity: Avoid personal bias.
+   - Simplicity: Use simple language, and explain any necessary technical jargon briefly.
+   - Logical Sequence: Arrange points in a logical order with proper planning.
+   - Proper Form and Presentation: Follow required formats with an attractive presentation.
+   - Selectiveness: Include only necessary content.
+   - Comprehensiveness: Provide complete and detailed coverage.
+   - Reliability, Coherence, and Relevance: Ensure a logical flow and relevance to the research questions.
+
+II. Structure the Report as Follows:
+   - Title Page (with a concise descriptive title)
+   - Table of Contents
+   - Executive Summary
+   - Introduction (clearly outlining the research purpose and objectives)
+   - Historical or Contextual Background
+   - Detailed Findings organized into coherent thematic sections
+   - Conclusion (with recommendations and insights)
+   - References/Bibliography (listing the provided URLs)
+
+III. Content and Writing Style:
+   - Use consistent and clear language.
+   - Support arguments with reliable evidence.
+   - Write in active voice with clear headings and a logical flow.
+   - Develop each section in multiple detailed paragraphs.
+
+IV. Steps for Writing the Report:
+   - Write a clear thesis statement.
+   - Prepare an outline and develop content sequentially.
+
+Use the following partial summaries and references as source materials to produce a detailed and exhaustive research report.
 Partial Summaries:
 {merged_input}
 References (URLs):
 {references_text}
-Now, merge these partial summaries into one thoroughly expanded research paper:
+
+Now, merge these partial summaries into one thoroughly expanded, detailed, and exhaustive research report:
 """
         final_data = {
             "model": MODEL_COMBINATION,
@@ -620,6 +640,9 @@ Now, merge these partial summaries into one thoroughly expanded research paper:
         final_response = call_llm_with_retry(groq_client, **final_data)
         final_text = final_response.choices[0].message.content.strip()
 
+        # Post-process final_text to remove any lingering chain-of-thought markers.
+        final_text = re.sub(r"<think>.*?</think>", "", final_text, flags=re.DOTALL)
+
         # Step 6: PDF generation
         print("[LOG] Step 6: Generating final PDF from the merged text.")
         final_report = generate_report(final_text)