########################################## # Step 0: Import required libraries ########################################## import streamlit as st # For building the web application from transformers import ( pipeline, SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan, AutoModelForCausalLM, AutoTokenizer ) # For emotion analysis, text-to-speech, and text generation from datasets import load_dataset # For loading datasets (e.g., speaker embeddings) import torch # For tensor operations import soundfile as sf # For saving audio as .wav files ########################################## # Streamlit application title and input ########################################## st.title("🚀 Just Comment") # Application title displayed to users st.write("I'm listening to you, my friend~") # Application description for users text = st.text_area("Enter your comment", "") # Text area for user input of comments ########################################## # Step 1: Sentiment Analysis Function ########################################## def analyze_dominant_emotion(user_review): """ Analyze the dominant emotion in the user's review using a text classification model. """ emotion_classifier = pipeline("text-classification", model="Thea231/jhartmann_emotion_finetuning", return_all_scores=True) # Load emotion classification model emotion_results = emotion_classifier(user_review)[0] # Get emotion scores for the review dominant_emotion = max(emotion_results, key=lambda x: x['score']) # Find the emotion with the highest confidence return dominant_emotion # Return the dominant emotion (as a dict with label and score) ########################################## # Step 2: Response Generation Function ########################################## def response_gen(user_review): """ Generate a response based on the sentiment of the user's review. """ dominant_emotion = analyze_dominant_emotion(user_review) # Get dominant emotion for the input emotion_label = dominant_emotion['label'].lower() # Extract emotion label # Define response templates for each emotion emotion_prompts = { "anger": "I appreciate your feedback and apologize for the inconvenience caused by '{review}'. We're committed to resolving this issue promptly and will ensure it doesn't happen again. Thank you for your patience.", "joy": "Thank you for your positive feedback on '{review}'! We're thrilled to hear you had a great experience and hope to serve you again soon.", "disgust": "We regret that your experience with '{review}' did not meet our standards. We will take immediate steps to address this issue and appreciate your understanding.", "fear": "Your safety is our priority. Regarding your concern about '{review}', we ensure that all our products meet strict safety standards. Please feel free to reach out for further assistance.", "neutral": "Thank you for your feedback on '{review}'. We value your input and would love to hear more about your experience to improve our services.", "sadness": "I'm sorry to hear that you were disappointed with '{review}'. We're here to help and would like to offer you a solution tailored to your needs.", "surprise": "We're glad to hear that '{review}' exceeded your expectations! Thank you for sharing your excitement with us." } # Format the prompt with the user's review prompt = emotion_prompts.get(emotion_label, "Neutral").format(review=user_review) # Load a pre-trained text generation model tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B") # Load tokenizer model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen1.5-0.5B") # Load model inputs = tokenizer(prompt, return_tensors="pt") # Tokenize the prompt outputs = model.generate(**inputs, max_new_tokens=100) # Generate a response response = tokenizer.decode(outputs[0], skip_special_tokens=True) # Decode the generated text return response.strip()[:200] # Return a response trimmed to 200 characters ########################################## # Step 3: Text-to-Speech Conversion Function ########################################## def sound_gen(response): """ Convert the generated response to speech and save as a .wav file. """ # Load the pre-trained TTS models processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts") # Load processor model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts") # Load TTS model vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan") # Load vocoder # Load speaker embeddings (e.g., neutral female voice) embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation") # Load dataset speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0) # Get speaker embeddings # Process the input text and generate a spectrogram inputs = processor(text=response, return_tensors="pt") # Process the text spectrogram = model.generate_speech(inputs["input_ids"], speaker_embeddings) # Generate spectrogram # Use the vocoder to generate a waveform with torch.no_grad(): speech = vocoder(spectrogram) # Generate speech waveform # Save the generated speech as a .wav file sf.write("customer_service_response.wav", speech.numpy(), samplerate=16000) # Save audio st.audio("customer_service_response.wav") # Play the audio in Streamlit ########################################## # Main Function ########################################## def main(): """ Main function to orchestrate the workflow of sentiment analysis, response generation, and text-to-speech. """ if text: # Check if the user entered a comment response = response_gen(text) # Generate a response st.write(f"Generated response: {response}") # Display the generated response sound_gen(response) # Convert the response to speech and play it # Run the main function if __name__ == "__main__": main() # Execute the main function