Update utils.py

utils.py

@@ -52,6 +52,7 @@ def call_llm_with_retry(groq_client, **payload):
         try:
             print(f"[DEBUG] call_llm_with_retry attempt {attempt+1}")
             response = groq_client.chat.completions.create(**payload)
+            # Short sleep to avoid bursting usage
             time.sleep(0.3)
             print("[DEBUG] LLM call succeeded, returning response.")
             return response
@@ -114,61 +115,178 @@ def pitch_shift(audio: AudioSegment, semitones: int) -> AudioSegment:
 
 
 ###############################################################################
-#      ITERATIVE MERGING HELPER FUNCTION (BATCH PROCESSING STRATEGY)
+#                      PODCAST SCRIPT GENERATION (Single Call)
 ###############################################################################
 
-def iterative_merge_summaries(summaries: List[str], groq_client, references_text: str) -> str:
+def generate_script(
+    system_prompt: str,
+    input_text: str,
+    tone: str,
+    target_length: str,
+    host_name: str = "Jane",
+    guest_name: str = "John",
+    sponsor_style: str = "Separate Break",
+    sponsor_provided=None
+):
     """
-    Iteratively merge a list of summaries into one final consolidated summary.
-    Summaries are grouped into batches (default batch size 3) whose combined token count is kept below a threshold.
+    If you do a single call to generate the entire script.
+    Uses DEEPSEEK_R1. Just ensure you parse the JSON.
     """
-    tokenizer = tiktoken.get_encoding("cl100k_base")
-    max_merge_input_tokens = 2000  # Safe threshold for each merge call
-    round_index = 1
-    current_summaries = summaries
-    while len(current_summaries) > 1:
-        print(f"[LOG] Iterative merging round {round_index}: {len(current_summaries)} summaries to merge.")
-        new_summaries = []
-        i = 0
-        while i < len(current_summaries):
-            batch = []
-            batch_tokens = 0
-            while i < len(current_summaries):
-                summary = current_summaries[i]
-                summary_tokens = len(tokenizer.encode(summary))
-                if batch_tokens + summary_tokens <= max_merge_input_tokens or not batch:
-                    batch.append(summary)
-                    batch_tokens += summary_tokens
-                    i += 1
-                else:
-                    break
-            batch_text = "\n\n".join(batch)
-            merge_prompt = f"""
-You are a specialized summarization engine. Merge the following summaries into one comprehensive summary.
-Summaries:
-{batch_text}
-References (if any):
-{references_text}
-Please output the merged summary.
-"""
-            data = {
-                "model": MODEL_COMBINATION,
-                "messages": [{"role": "user", "content": merge_prompt}],
-                "temperature": 0.3,
-                "max_tokens": 4096
-            }
-            merge_response = call_llm_with_retry(groq_client, **data)
-            merged_batch = merge_response.choices[0].message.content.strip()
-            merged_batch = re.sub(r"<think>.*?</think>", "", merged_batch, flags=re.DOTALL).strip()
-            new_summaries.append(merged_batch)
-        current_summaries = new_summaries
-        print(f"[LOG] Iterative merge round {round_index} produced {len(current_summaries)} consolidated summaries.")
-        round_index += 1
-    return current_summaries[0] if current_summaries else ""
+    print("[LOG] Generating script with tone:", tone, "and length:", target_length)
+
+    language_selection = st.session_state.get("language_selection", "English (American)")
+    if (host_name == "Jane" or not host_name) and language_selection in ["English (Indian)", "Hinglish", "Hindi"]:
+        host_name = "Isha"
+    if (guest_name == "John" or not guest_name) and language_selection in ["English (Indian)", "Hinglish", "Hindi"]:
+        guest_name = "Aarav"
+
+    words_per_minute = 150
+    numeric_minutes = 3
+    match = re.search(r"(\d+)", target_length)
+    if match:
+        numeric_minutes = int(match.group(1))
+
+    min_words = max(50, numeric_minutes * 100)
+    max_words = numeric_minutes * words_per_minute
+
+    tone_map = {
+        "Humorous": "funny and exciting, makes people chuckle",
+        "Formal": "business-like, well-structured, professional",
+        "Casual": "like a conversation between close friends, relaxed and informal",
+        "Youthful": "like how teenagers might chat, energetic and lively"
+    }
+    chosen_tone = tone_map.get(tone, "casual")
+
+    if sponsor_provided:
+        if sponsor_style == "Separate Break":
+            sponsor_instructions = (
+                "If sponsor content is provided, include it in a separate ad break (~30 seconds). "
+                "Use 'Now a word from our sponsor...' and end with 'Back to the show', etc."
+            )
+        else:
+            sponsor_instructions = (
+                "If sponsor content is provided, blend it naturally (~30 seconds) into conversation. "
+                "Avoid abrupt transitions."
+            )
+    else:
+        sponsor_instructions = ""
+
+    prompt = (
+        f"{system_prompt}\n"
+        f"TONE: {chosen_tone}\n"
+        f"TARGET LENGTH: {target_length} (~{min_words}-{max_words} words)\n"
+        f"INPUT TEXT: {input_text}\n\n"
+        f"# Sponsor Style Instruction:\n{sponsor_instructions}\n\n"
+        "Please provide the output in the following JSON format without any extra text:\n"
+        "{\n"
+        '   "dialogue": [\n'
+        '     { "speaker": "Jane", "text": "..." },\n'
+        '     { "speaker": "John", "text": "..." }\n'
+        "   ]\n"
+        "}"
+    )
+    if language_selection == "Hinglish":
+        prompt += "\n\nPlease generate the script in Romanized Hindi.\n"
+    elif language_selection == "Hindi":
+        prompt += "\n\nPlease generate the script exclusively in Hindi.\n"
+
+    print("[LOG] Sending script generation prompt to LLM.")
+    try:
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('DEEPSEEK_API_KEY')}",
+            "Content-Type": "application/json"
+        }
+        data = {
+            "model": "deepseek/deepseek-r1",
+            "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": 2048,
+            "temperature": 0.7
+        }
+        resp = requests.post("https://openrouter.ai/api/v1/chat/completions",
+                             headers=headers, data=json.dumps(data))
+        resp.raise_for_status()
+        raw_content = resp.json()["choices"][0]["message"]["content"].strip()
+    except Exception as e:
+        print("[ERROR] LLM error generating script:", e)
+        raise ValueError(f"Error generating script: {str(e)}")
+
+    start_idx = raw_content.find("{")
+    end_idx = raw_content.rfind("}")
+    if start_idx == -1 or end_idx == -1:
+        raise ValueError("No JSON found in LLM response for script generation.")
+
+    json_str = raw_content[start_idx:end_idx+1]
+    try:
+        data_js = json.loads(json_str)
+        dialogue_list = data_js.get("dialogue", [])
+
+        # Adjust speaker names if they match
+        for d in dialogue_list:
+            raw_speaker = d.get("speaker", "Jane")
+            if raw_speaker.lower() == host_name.lower():
+                d["speaker"] = "Jane"
+                d["display_speaker"] = host_name
+            elif raw_speaker.lower() == guest_name.lower():
+                d["speaker"] = "John"
+                d["display_speaker"] = guest_name
+            else:
+                d["speaker"] = "Jane"
+                d["display_speaker"] = raw_speaker
+
+        new_dialogue_items = []
+        for d in dialogue_list:
+            if "display_speaker" not in d:
+                d["display_speaker"] = d["speaker"]
+            new_dialogue_items.append(DialogueItem(**d))
+
+        return Dialogue(dialogue=new_dialogue_items)
+
+    except json.JSONDecodeError as e:
+        print("[ERROR] JSON decoding failed for script generation:", e)
+        raise ValueError(f"Script parse error: {str(e)}")
+    except Exception as e:
+        print("[ERROR] Unknown error parsing script JSON:", e)
+        raise ValueError(f"Script parse error: {str(e)}")
 
 
 ###############################################################################
-#                      AUDIO GENERATION (TTS) AND BG MUSIC MIX
+#                      YOUTUBE TRANSCRIPTION (RAPIDAPI)
+###############################################################################
+
+def transcribe_youtube_video(video_url: str) -> str:
+    print("[LOG] Transcribing YouTube video:", video_url)
+    match = re.search(r"(?:v=|/)([0-9A-Za-z_-]{11})", video_url)
+    if not match:
+        raise ValueError(f"Invalid YouTube URL: {video_url}, cannot extract video ID.")
+    video_id = match.group(1)
+    print("[LOG] Extracted video ID:", video_id)
+
+    base_url = "https://youtube-transcriptor.p.rapidapi.com/transcript"
+    params = {"video_id": video_id, "lang": "en"}
+    headers = {
+        "x-rapidapi-host": "youtube-transcriptor.p.rapidapi.com",
+        "x-rapidapi-key": os.environ.get("RAPIDAPI_KEY")
+    }
+    try:
+        resp = requests.get(base_url, headers=headers, params=params, timeout=30)
+        resp.raise_for_status()
+        data = resp.json()
+        if not isinstance(data, list) or not data:
+            raise ValueError(f"Unexpected transcript format or empty transcript: {data}")
+
+        transcript_as_text = data[0].get("transcriptionAsText", "").strip()
+        if not transcript_as_text:
+            raise ValueError("transcriptionAsText missing or empty in RapidAPI response.")
+
+        print("[LOG] Transcript retrieval successful. Sample:", transcript_as_text[:200], "...")
+        return transcript_as_text
+    except Exception as e:
+        print("[ERROR] YouTube transcription error:", e)
+        raise ValueError(f"Error transcribing YouTube video: {str(e)}")
+
+
+###############################################################################
+#                  AUDIO GENERATION (TTS) AND BG MUSIC MIX
 ###############################################################################
 
 def _preprocess_text_for_tts(text: str, speaker: str) -> str:
@@ -181,11 +299,13 @@ def _preprocess_text_for_tts(text: str, speaker: str) -> str:
         if abbr in abbreviations_as_words:
             return abbr
         return ".".join(list(abbr)) + "."
+
     text = re.sub(r"\b([A-Z]{2,})\b", insert_periods_for_abbrev, text)
     text = re.sub(r"\.\.", ".", text)
 
     def remove_periods_for_tts(m):
         return m.group().replace(".", " ").strip()
+
     text = re.sub(r"[A-Z]\.[A-Z](?:\.[A-Z])*\.", remove_periods_for_tts, text)
     text = re.sub(r"-", " ", text)
     text = re.sub(r"\b(ha(ha)?|heh|lol)\b", "(* laughs *)", text, flags=re.IGNORECASE)
@@ -204,9 +324,12 @@ def _preprocess_text_for_tts(text: str, speaker: str) -> str:
         text = re.sub(keywords_pattern, insert_thinking_pause, text, flags=re.IGNORECASE)
         conj_pattern = r"\b(and|but|so|because|however)\b"
         text = re.sub(conj_pattern, lambda m: f"{m.group()}...", text, flags=re.IGNORECASE)
+
     text = re.sub(r"\b(uh|um|ah)\b", "", text, flags=re.IGNORECASE)
+
     def capitalize_after_sentence(m):
         return m.group().upper()
+
     text = re.sub(r'(^\s*\w)|([.!?]\s*\w)', capitalize_after_sentence, text)
     return text.strip()
 
@@ -230,6 +353,7 @@ def generate_audio_mp3(text: str, speaker: str) -> str:
             body = {"text": processed_text}
             r = requests.post(deepgram_api_url, params=params, headers=headers, json=body, stream=True)
             r.raise_for_status()
+
             content_type = r.headers.get("Content-Type", "")
             if "audio/mpeg" not in content_type:
                 raise ValueError("Unexpected content-type from Deepgram TTS.")
@@ -238,6 +362,7 @@ def generate_audio_mp3(text: str, speaker: str) -> str:
                     if chunk:
                         mp3_file.write(chunk)
                 mp3_path = mp3_file.name
+
             audio_seg = AudioSegment.from_file(mp3_path, format="mp3")
             audio_seg = effects.normalize(audio_seg)
             final_mp3_path = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3").name
@@ -245,6 +370,7 @@ def generate_audio_mp3(text: str, speaker: str) -> str:
             if os.path.exists(mp3_path):
                 os.remove(mp3_path)
             return final_mp3_path
+
         else:
             print("[LOG] Using Murf API for TTS. Language=", language_selection)
             from indic_transliteration.sanscript import transliterate, DEVANAGARI, IAST
@@ -263,6 +389,7 @@ def generate_audio_mp3(text: str, speaker: str) -> str:
                 voice_id = "hi-IN-kabir" if speaker == "John" else "hi-IN-shweta"
             else:
                 voice_id = "en-IN-aarav" if speaker == "John" else "en-IN-isha"
+
             payload = {
                 "audioDuration": 0,
                 "channelType": "MONO",
@@ -287,9 +414,11 @@ def generate_audio_mp3(text: str, speaker: str) -> str:
                 raise ValueError("No audioFile URL from Murf API.")
             audio_resp = requests.get(audio_url)
             audio_resp.raise_for_status()
+
             with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as wav_file:
                 wav_file.write(audio_resp.content)
                 wav_path = wav_file.name
+
             audio_seg = AudioSegment.from_file(wav_path, format="wav")
             audio_seg = effects.normalize(audio_seg)
             final_mp3_path = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3").name
@@ -308,11 +437,13 @@ def mix_with_bg_music(spoken: AudioSegment, custom_music_path=None) -> AudioSegm
         music_path = custom_music_path
     else:
         music_path = "bg_music.mp3"
+
     try:
         bg_music = AudioSegment.from_file(music_path, format="mp3")
     except Exception as e:
         print("[ERROR] Failed to load background music:", e)
         return spoken
+
     bg_music = bg_music - 18.0
     total_length_ms = len(spoken) + 2000
     looped_music = AudioSegment.empty()
@@ -352,6 +483,61 @@ def call_groq_api_for_qa(system_prompt: str) -> str:
         return json.dumps(fallback)
 
 
+###############################################################################
+#      ITERATIVE MERGING HELPER FUNCTION (BATCH PROCESSING STRATEGY)
+###############################################################################
+
+def iterative_merge_summaries(summaries: List[str], groq_client, references_text: str) -> str:
+    """
+    Iteratively merge a list of summaries into one final report summary.
+    This function groups summaries into batches whose total token count is below a set threshold,
+    merges each batch, and then recursively merges the batch outputs until only one final summary remains.
+    """
+    tokenizer = tiktoken.get_encoding("cl100k_base")
+    max_merge_input_tokens = 2000  # Set a safe threshold for each merge call
+
+    round_index = 1
+    while len(summaries) > 1:
+        print(f"[LOG] Iterative merging round {round_index}: {len(summaries)} summaries to merge.")
+        new_summaries = []
+        i = 0
+        while i < len(summaries):
+            batch = []
+            batch_tokens = 0
+            # Group summaries until the token count exceeds threshold
+            while i < len(summaries):
+                summary = summaries[i]
+                summary_tokens = len(tokenizer.encode(summary))
+                if batch_tokens + summary_tokens <= max_merge_input_tokens or not batch:
+                    batch.append(summary)
+                    batch_tokens += summary_tokens
+                    i += 1
+                else:
+                    break
+            batch_text = "\n\n".join(batch)
+            merge_prompt = f"""
+You are a specialized summarization engine. Merge the following summaries into one comprehensive summary.
+Summaries:
+{batch_text}
+References (if any):
+{references_text}
+Please output the merged summary.
+"""
+            data = {
+                "model": MODEL_COMBINATION,
+                "messages": [{"role": "user", "content": merge_prompt}],
+                "temperature": 0.3,
+                "max_tokens": 4096
+            }
+            merge_response = call_llm_with_retry(groq_client, **data)
+            merged_batch = merge_response.choices[0].message.content.strip()
+            merged_batch = re.sub(r"<think>.*?</think>", "", merged_batch, flags=re.DOTALL).strip()
+            new_summaries.append(merged_batch)
+        summaries = new_summaries
+        round_index += 1
+    return summaries[0]
+
+
 ###############################################################################
 #      LOW-CALL RESEARCH AGENT (Minimizing LLM Calls)
 ###############################################################################
@@ -370,11 +556,10 @@ def run_research_agent(
       2) Firecrawl scrape => combined text.
       3) Use the full combined text without truncation.
       4) Split into chunks (each 4500 tokens) => Summarize each chunk individually => summaries.
-      5) Iteratively merge the summaries into a consolidated summary.
-      6) Final merge using an enhanced prompt with detailed guidelines to produce the full research report.
-         If the report output is incomplete, the model will output "CONTINUE" so that additional calls
-         can be made to retrieve the rest of the report.
-         (Total LLM calls: 2 or more, but no more than 10.)
+      5) Iteratively merge the summaries into a final report.
+      If the report output is incomplete, the model will output "CONTINUE" so that additional calls
+      can be made to retrieve the rest of the report.
+      => 2 or more total LLM calls (but no more than 10) to reduce the chance of rate limit errors.
     """
     print(f"[LOG] Starting LOW-CALL research agent for topic: {topic}")
 
@@ -388,6 +573,7 @@ def run_research_agent(
         if not search_results:
             print("[LOG] No relevant search results found by Tavily.")
             return "No relevant search results found."
+
         references_list = [r["url"] for r in search_results if "url" in r]
 
         # Step 2: Firecrawl scraping
@@ -460,87 +646,15 @@ include key data points and context:
         # Step 5: Iteratively merge the chunk summaries.
         print("[LOG] Step 5: Iteratively merging chunk summaries.")
         references_text = "\n".join(f"- {url}" for url in references_list) if references_list else "None"
-        consolidated_summary = iterative_merge_summaries(summaries, groq_client, references_text)
-        print("[LOG] Iterative merge produced a consolidated summary.")
-        consolidated_summary = re.sub(r"<think>.*?</think>", "", consolidated_summary, flags=re.DOTALL).strip()
-
-        # Step 6: Final merge to generate the full research report.
-        final_prompt = f"""
-IMPORTANT: Do NOT include any chain-of-thought, internal planning, or hidden reasoning in the final output.
-Draft a professional, world-class research report that adheres to the following tenets:
-
-I. Essential Principles and Qualities:
-   - Accuracy: Present accurate facts with no spelling or grammatical errors.
-   - Clarity: Use clear, straightforward language.
-   - Brevity: Be concise yet complete.
-   - Objectivity: Avoid personal bias.
-   - Simplicity: Use simple language, and explain any necessary technical jargon briefly.
-   - Logical Sequence: Arrange points in a logical order with proper planning.
-   - Proper Form and Presentation: Follow required formats with an attractive presentation.
-   - Selectiveness: Include only necessary content.
-   - Comprehensiveness: Provide complete and detailed coverage.
-   - Reliability, Coherence, and Relevance: Ensure a logical flow and relevance to the research questions.
-
-II. Structure the Report as Follows:
-   - Title Page (with a concise descriptive title)
-   - Table of Contents
-   - Executive Summary
-   - Introduction (clearly outlining the research purpose and objectives)
-   - Historical or Contextual Background
-   - Detailed Findings organized into coherent thematic sections
-   - Conclusion (with recommendations and insights)
-   - References/Bibliography (listing the provided URLs)
-
-III. Content and Writing Style:
-   - Use consistent and clear language.
-   - Support arguments with reliable evidence.
-   - Write in active voice with clear headings and a logical flow.
-   - Develop each section in multiple detailed paragraphs.
-
-IV. Steps for Writing the Report:
-   - Write a clear thesis statement.
-   - Prepare an outline and develop content sequentially.
-
-Consolidated Summary:
-{consolidated_summary}
-
-References (URLs):
-{references_text}
-
-Now, merge the above into one thoroughly expanded, detailed, and exhaustive research report.
-If the report is incomplete, please output "CONTINUE" at the end; otherwise, end with "END_OF_REPORT".
-"""
-        final_data = {
-            "model": MODEL_COMBINATION,
-            "messages": [{"role": "user", "content": final_prompt}],
-            "temperature": 0.3,
-            "max_tokens": 4096
-        }
-        final_response = call_llm_with_retry(groq_client, **final_data)
-        final_text = final_response.choices[0].message.content.strip()
-
-        # Continuation loop: if the report does not include END_OF_REPORT, ask for continuation.
-        while "END_OF_REPORT" not in final_text:
-            print("[LOG] Final output incomplete. Requesting continuation...")
-            continuation_prompt = "The previous report ended with 'CONTINUE'. Please continue the report from where it left off, and when finished, output 'END_OF_REPORT'."
-            cont_data = {
-                "model": MODEL_COMBINATION,
-                "messages": [{"role": "user", "content": continuation_prompt}],
-                "temperature": 0.3,
-                "max_tokens": 4096
-            }
-            cont_response = call_llm_with_retry(groq_client, **cont_data)
-            cont_text = cont_response.choices[0].message.content.strip()
-            final_text += "\n" + cont_text
+        final_text = iterative_merge_summaries(summaries, groq_client, references_text)
 
         # --- NEW POST-PROCESSING STEP ---
-        # Remove any lingering chain-of-thought markers and the END/CONTINUE tokens.
-        final_text = re.sub(r"<think>.*?</think>", "", final_text, flags=re.DOTALL)
-        final_text = final_text.replace("END_OF_REPORT", "").replace("CONTINUE", "").strip()
+        # Remove any lingering chain-of-thought markers.
+        final_text = re.sub(r"<think>.*?</think>", "", final_text, flags=re.DOTALL).strip()
         # ------------------------------
 
-        # Step 7: PDF generation
-        print("[LOG] Step 7: Generating final PDF from the merged text.")
+        # Step 6: PDF generation
+        print("[LOG] Step 6: Generating final PDF from the merged text.")
         final_report = generate_report(final_text)
 
         print("[LOG] Done! Returning PDF from run_research_agent (low-call).")
@@ -548,4 +662,4 @@ If the report is incomplete, please output "CONTINUE" at the end; otherwise, end
 
     except Exception as e:
         print(f"[ERROR] Error in run_research_agent: {e}")
-        return f"Sorry, encountered an error: {str(e)}"
+        return f"Sorry, encountered an error: {str(e)}"