import gradio as gr from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler from transformers import AutoProcessor, AutoModelForVision2Seq, AutoModelForCausalLM, AutoTokenizer import torch from PIL import Image, ImageDraw, ImageFont import numpy as np import textwrap import os import gc import re from datetime import datetime import spaces from kokoro import KPipeline import soundfile as sf # Initialize models at startup - outside of functions print("Loading models...") # Load SmolVLM for image analysis processor_vlm = AutoProcessor.from_pretrained("HuggingFaceTB/SmolVLM-500M-Instruct") model_vlm = AutoModelForVision2Seq.from_pretrained( "HuggingFaceTB/SmolVLM-500M-Instruct", torch_dtype=torch.bfloat16, use_safetensors=True ) # Load SmolLM2 for story and prompt generation checkpoint = "HuggingFaceTB/SmolLM2-1.7B-Instruct" tokenizer_lm = AutoTokenizer.from_pretrained(checkpoint) model_lm = AutoModelForCausalLM.from_pretrained( checkpoint, use_safetensors=True ) # Load Stable Diffusion pipeline pipe = StableDiffusionPipeline.from_pretrained( "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True ) pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) # Move models to GPU if available if torch.cuda.is_available(): model_vlm = model_vlm.to("cuda") model_lm = model_lm.to("cuda") pipe = pipe.to("cuda") @torch.inference_mode() @spaces.GPU(duration=30) def generate_image(): """Generate a random landscape image.""" torch.cuda.empty_cache() default_prompt = "a beautiful, professional landscape photograph" default_negative_prompt = "blurry, bad quality, distorted, deformed" default_steps = 30 default_guidance = 7.5 default_seed = torch.randint(0, 2**32 - 1, (1,)).item() generator = torch.Generator("cuda").manual_seed(default_seed) image = pipe( prompt=default_prompt, negative_prompt=default_negative_prompt, num_inference_steps=default_steps, guidance_scale=default_guidance, generator=generator, ).images[0] return image @torch.inference_mode() @spaces.GPU(duration=30) def analyze_image(image): if image is None: return "Please generate an image first." torch.cuda.empty_cache() if isinstance(image, np.ndarray): image = Image.fromarray(image) messages = [ { "role": "user", "content": [ {"type": "image"}, {"type": "text", "text": "Describe this image very briefly in five sentences or less."} ] } ] prompt = processor_vlm.apply_chat_template(messages, add_generation_prompt=True) inputs = processor_vlm( text=prompt, images=[image], return_tensors="pt" ).to('cuda') outputs = model_vlm.generate( input_ids=inputs.input_ids, pixel_values=inputs.pixel_values, attention_mask=inputs.attention_mask, num_return_sequences=1, no_repeat_ngram_size=2, max_new_tokens=500, min_new_tokens=10 ) description = processor_vlm.decode(outputs[0], skip_special_tokens=True) description = re.sub(r".*?Assistant:\s*", "", description, flags=re.DOTALL).strip() return description @torch.inference_mode() @spaces.GPU(duration=30) def generate_story(image_description): torch.cuda.empty_cache() story_prompt = f"""Write a short children's story (one chapter, about 500 words) based on this scene: {image_description} Requirements: 1. Main character: An English bulldog named Champ 2. Include these values: confidence, teamwork, caring, and hope 3. Theme: "We are stronger together than as individuals" 4. Keep it simple and engaging for young children 5. End with a simple moral lesson""" messages = [{"role": "user", "content": story_prompt}] input_text = tokenizer_lm.apply_chat_template(messages, tokenize=False) inputs = tokenizer_lm.encode(input_text, return_tensors="pt").to("cuda") outputs = model_lm.generate( inputs, max_new_tokens=750, temperature=0.7, top_p=0.9, do_sample=True, repetition_penalty=1.2 ) story = tokenizer_lm.decode(outputs[0]) story = clean_story_output(story) return story @torch.inference_mode() @spaces.GPU(duration=30) def generate_image_prompts(story_text): torch.cuda.empty_cache() paragraphs = split_into_paragraphs(story_text) all_prompts = [] prompt_instruction = '''Here is a story paragraph: {paragraph} Start your response with "Watercolor bulldog" and describe what Champ is doing in this scene. Add where it takes place and one mood detail. Keep it short.''' for i, paragraph in enumerate(paragraphs, 1): messages = [{"role": "user", "content": prompt_instruction.format(paragraph=paragraph)}] input_text = tokenizer_lm.apply_chat_template(messages, tokenize=False) inputs = tokenizer_lm.encode(input_text, return_tensors="pt").to("cuda") outputs = model_lm.generate( inputs, max_new_tokens=30, temperature=0.5, top_p=0.9, do_sample=True, repetition_penalty=1.2 ) prompt = process_generated_prompt(tokenizer_lm.decode(outputs[0]), paragraph) section = f"Paragraph {i}:\n{paragraph}\n\nScenery Prompt {i}:\n{prompt}\n\n{'='*50}" all_prompts.append(section) return '\n'.join(all_prompts) @torch.inference_mode() @spaces.GPU(duration=60) def generate_story_image(prompt): torch.cuda.empty_cache() pipe.load_lora_weights("Prof-Hunt/lora-bulldog") enhanced_prompt = f"{prompt}, watercolor style, children's book illustration, soft colors" image = pipe( prompt=enhanced_prompt, negative_prompt="deformed, ugly, blurry, bad art, poor quality, distorted", num_inference_steps=50, guidance_scale=15, ).images[0] return image @torch.inference_mode() @spaces.GPU(duration=180) # Longer duration for multiple image generation def generate_all_scenes(prompts_text): generated_images = [] formatted_prompts = [] sections = prompts_text.split('='*50) for section in sections: if not section.strip(): continue lines = [line.strip() for line in section.split('\n') if line.strip()] scene_prompt = None for i, line in enumerate(lines): if 'Scenery Prompt' in line: scene_num = line.split('Scenery Prompt')[1].split(':')[0].strip() if i + 1 < len(lines): scene_prompt = lines[i + 1] formatted_prompts.append(f"Scene {scene_num}: {scene_prompt}") break if scene_prompt: try: torch.cuda.empty_cache() image = generate_story_image(scene_prompt) if image is not None: generated_images.append(np.array(image)) except Exception as e: print(f"Error generating image: {str(e)}") continue return generated_images, "\n\n".join(formatted_prompts) # Helper functions without GPU usage def clean_story_output(story): story = story.replace("<|im_end|>", "") story_start = story.find("Once upon") if story_start == -1: possible_starts = ["One day", "In a", "There was", "Champ"] for marker in possible_starts: story_start = story.find(marker) if story_start != -1: break if story_start != -1: story = story[story_start:] lines = story.split('\n') cleaned_lines = [] for line in lines: line = line.strip() if line and not any(skip in line.lower() for skip in ['requirement', 'include these values', 'theme:', 'keep it simple', 'end with', 'write a']): if not line.startswith(('1.', '2.', '3.', '4.', '5.')): cleaned_lines.append(line) return '\n\n'.join(cleaned_lines).strip() def split_into_paragraphs(text): paragraphs = [] current_paragraph = [] for line in text.split('\n'): line = line.strip() if not line: if current_paragraph: paragraphs.append(' '.join(current_paragraph)) current_paragraph = [] else: current_paragraph.append(line) if current_paragraph: paragraphs.append(' '.join(current_paragraph)) return [p for p in paragraphs if not any(skip in p.lower() for skip in ['requirement', 'include these values', 'theme:', 'keep it simple', 'end with', 'write a'])] def process_generated_prompt(prompt, paragraph): prompt = prompt.replace("<|im_start|>", "").replace("<|im_end|>", "") prompt = prompt.replace("assistant", "").replace("system", "").replace("user", "") cleaned_lines = [line.strip() for line in prompt.split('\n') if line.strip().lower().startswith("watercolor bulldog")] if cleaned_lines: prompt = cleaned_lines[0] else: setting = "quiet town" if "quiet town" in paragraph.lower() else "park" mood = "hopeful" if "wished" in paragraph.lower() else "peaceful" prompt = f"Watercolor bulldog watching friends play in {setting}, {mood} atmosphere." if not prompt.endswith('.'): prompt = prompt + '.' return prompt def overlay_text_on_image(image, text): if isinstance(image, np.ndarray): image = Image.fromarray(image) img = image.convert('RGB') draw = ImageDraw.Draw(img) try: font_size = int(img.width * 0.025) font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", font_size) except: font = ImageFont.load_default() y_position = int(img.height * 0.005) x_margin = int(img.width * 0.005) available_width = img.width - (2 * x_margin) wrapped_text = textwrap.fill(text, width=int(available_width / (font_size * 0.6))) outline_color = (255, 255, 255) text_color = (0, 0, 0) offsets = [-2, -1, 1, 2] for dx in offsets: for dy in offsets: draw.multiline_text( (x_margin + dx, y_position + dy), wrapped_text, font=font, fill=outline_color ) draw.multiline_text( (x_margin, y_position), wrapped_text, font=font, fill=text_color ) return img # Initialize Kokoro TTS pipeline pipeline = KPipeline(lang_code='a') # 'a' for American English def generate_combined_audio_from_story(story_text, voice='af_heart', speed=1): """Generate a single audio file for all paragraphs in the story.""" if not story_text: return None # Split story into paragraphs paragraphs = [] current_paragraph = [] for line in story_text.split('\n'): line = line.strip() if not line: # Empty line indicates paragraph break if current_paragraph: paragraphs.append(' '.join(current_paragraph)) current_paragraph = [] else: current_paragraph.append(line) if current_paragraph: paragraphs.append(' '.join(current_paragraph)) # Combine audio for all paragraphs combined_audio = [] for paragraph in paragraphs: if not paragraph.strip(): continue # Skip empty paragraphs generator = pipeline( paragraph, voice=voice, speed=speed, split_pattern=r'\n+' # Split on newlines ) for _, _, audio in generator: combined_audio.extend(audio) # Append audio data # Convert combined audio to NumPy array and save combined_audio = np.array(combined_audio) filename = "combined_story.wav" sf.write(filename, combined_audio, 24000) # Save audio as .wav return filename def add_text_to_scenes(gallery_images, prompts_text): if not isinstance(gallery_images, list): return [], [] sections = prompts_text.split('='*50) overlaid_images = [] output_files = [] temp_dir = "temp_book_pages" os.makedirs(temp_dir, exist_ok=True) for i, (image_data, section) in enumerate(zip(gallery_images, sections)): if not section.strip(): continue lines = [line.strip() for line in section.split('\n') if line.strip()] paragraph = None for j, line in enumerate(lines): if line.startswith('Paragraph'): if j + 1 < len(lines): paragraph = lines[j + 1] break if paragraph and image_data is not None: try: overlaid_img = overlay_text_on_image(image_data, paragraph) if overlaid_img is not None: overlaid_array = np.array(overlaid_img) overlaid_images.append(overlaid_array) output_path = os.path.join(temp_dir, f"panel_{i+1}.png") overlaid_img.save(output_path) output_files.append(output_path) except Exception as e: print(f"Error processing image: {str(e)}") continue return overlaid_images, output_files def create_interface(): theme = gr.themes.Soft().set( body_background_fill="*primary_50", button_primary_background_fill="rgb(173, 216, 230)", # light blue button_secondary_background_fill="rgb(255, 182, 193)", # light red button_primary_background_fill_hover="rgb(135, 206, 235)", # slightly darker blue for hover button_secondary_background_fill_hover="rgb(255, 160, 180)", # slightly darker red for hover block_title_text_color="*primary_500", block_label_text_color="*secondary_500", ) with gr.Blocks(theme=theme) as demo: gr.Markdown("# Tech Tales: Story Creation") with gr.Row(): generate_btn = gr.Button("1. Generate Random Landscape") image_output = gr.Image(label="Generated Image", type="pil") with gr.Row(): analyze_btn = gr.Button("2. Get Brief Description") analysis_output = gr.Textbox(label="Image Description", lines=3) with gr.Row(): story_btn = gr.Button("3. Create Children's Story") story_output = gr.Textbox(label="Generated Story", lines=10) with gr.Row(): prompts_btn = gr.Button("4. Generate Scene Prompts") prompts_output = gr.Textbox(label="Generated Scene Prompts", lines=20) with gr.Row(): generate_scenes_btn = gr.Button("5. Generate Story Scenes", variant="primary") with gr.Row(): scene_prompts_display = gr.Textbox( label="Scenes Being Generated", lines=8, interactive=False ) with gr.Row(): gallery = gr.Gallery( label="Story Scenes", show_label=True, columns=2, height="auto" ) with gr.Row(): add_text_btn = gr.Button("6. Add Text to Scenes", variant="primary") with gr.Row(): final_gallery = gr.Gallery( label="Story Book Pages", show_label=True, columns=2, height="auto" ) with gr.Row(): download_btn = gr.File( label="Download Story Book", file_count="multiple", interactive=False ) with gr.Row(): tts_btn = gr.Button("7. Read Story Aloud") audio_output = gr.Audio(label="Story Audio") # Event handlers generate_btn.click( fn=generate_image, outputs=image_output ) analyze_btn.click( fn=analyze_image, inputs=[image_output], outputs=analysis_output ) story_btn.click( fn=generate_story, inputs=[analysis_output], outputs=story_output ) prompts_btn.click( fn=generate_image_prompts, inputs=[story_output], outputs=prompts_output ) generate_scenes_btn.click( fn=generate_all_scenes, inputs=[prompts_output], outputs=[gallery, scene_prompts_display] ) add_text_btn.click( fn=add_text_to_scenes, inputs=[gallery, prompts_output], outputs=[final_gallery, download_btn] ) tts_btn.click( fn=generate_combined_audio_from_story, inputs=[story_output], outputs=audio_output ) return demo if __name__ == "__main__": demo = create_interface() demo.launch()