Update app.py

app.py

@@ -4,7 +4,7 @@ import pandas as pd
 import plotly.express as px
 
 # ------------------------------
-# Load pretrained text model
+# Load pretrained models
 # ------------------------------
 text_classifier = pipeline(
     "text-classification",
@@ -12,6 +12,11 @@ text_classifier = pipeline(
     top_k=None  # returns all scores
 )
 
+audio_classifier = pipeline(
+    "audio-classification",
+    model="Dpngtm/wav2vec2-emotion-recognition"
+)
+
 # ------------------------------
 # Map emotion to emoji
 # ------------------------------
@@ -22,9 +27,40 @@ EMOJI_MAP = {
     "joy": "😄",
     "neutral": "😐",
     "sadness": "😢",
-    "surprise": "😲"
+    "surprise": "😲",
+    "hap": "😄",  # for audio model
+    "neu": "😐",
+    "sad": "😢",
+    "ang": "😡"
 }
 
+# ------------------------------
+# Fusion function
+# ------------------------------
+def fuse_predictions(text_preds=None, audio_preds=None, w_text=0.5, w_audio=0.5):
+    labels = set()
+    if text_preds:
+        labels |= {p['label'] for p in text_preds}
+    if audio_preds:
+        labels |= {p['label'] for p in audio_preds}
+    scores = {l: 0.0 for l in labels}
+
+    def normalize(preds):
+        s = sum(p['score'] for p in preds)
+        return {p['label']: p['score']/s for p in preds}
+
+    if text_preds:
+        t_norm = normalize(text_preds)
+        for l in labels:
+            scores[l] += w_text * t_norm.get(l, 0)
+    if audio_preds:
+        a_norm = normalize(audio_preds)
+        for l in labels:
+            scores[l] += w_audio * a_norm.get(l, 0)
+
+    best = max(scores.items(), key=lambda x: x[1]) if scores else ("none", 0)
+    return {"fused_label": best[0], "fused_score": round(best[1], 3), "all_scores": scores}
+
 # ------------------------------
 # Create bar chart
 # ------------------------------
@@ -43,45 +79,46 @@ def make_bar_chart(scores_dict, title="Emotion Scores"):
 # ------------------------------
 # Prediction function
 # ------------------------------
-def predict(text, w_text=1.0):
-    if not text:
-        return "Please enter text.", None
-    preds = text_classifier(text)[0]  # get all scores
-    scores = {p['label']: p['score'] for p in preds}
-    best_label = max(scores, key=scores.get)
-    emoji = EMOJI_MAP.get(best_label, "")
-    
-    # Animate emoji with simple bouncing
-    final_emotion_html = f"""
-    <div style="font-size:80px; text-align:center; animation: bounce 1s infinite;">
-        {emoji}
-    </div>
-    <h3 style="text-align:center;">{best_label.upper()} (score: {scores[best_label]:.2f})</h3>
-    <style>
-    @keyframes bounce {{
-        0%, 20%, 50%, 80%, 100% {{transform: translateY(0);}}
-        40% {{transform: translateY(-20px);}}
-        60% {{transform: translateY(-10px);}}
-    }}
-    </style>
-    """
-    chart = make_bar_chart(scores, "Text Emotion Scores")
-    return final_emotion_html, chart
+def predict(text, audio, w_text, w_audio):
+    text_preds, audio_preds = None, None
+    if text:
+        text_preds = text_classifier(text)[0]
+    if audio:
+        audio_preds = audio_classifier(audio)
+    fused = fuse_predictions(text_preds, audio_preds, w_text, w_audio)
+
+    # Display final predicted emotion with emoji
+    label = fused['fused_label']
+    emoji = EMOJI_MAP.get(label, "")
+    final_emotion = f"### Final Predicted Emotion: {label.upper()} {emoji} (score: {fused['fused_score']})"
+
+    # Bar charts
+    charts = []
+    if text_preds:
+        charts.append(make_bar_chart({p['label']: p['score'] for p in text_preds}, "Text Emotion Scores"))
+    if audio_preds:
+        charts.append(make_bar_chart({p['label']: p['score'] for p in audio_preds}, "Audio Emotion Scores"))
+    charts.append(make_bar_chart(fused['all_scores'], "Fused Emotion Scores"))
+
+    return final_emotion, charts
 
 # ------------------------------
 # Build Gradio interface
 # ------------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("## 🎭 Text Emotion Classification with Emoji Animation")
+    gr.Markdown("## 🎭 Multimodal Emotion Classification (Text + Speech)")
 
     with gr.Row():
         with gr.Column():
             txt = gr.Textbox(label="Text input", placeholder="Type something emotional...")
+            aud = gr.Audio(type="filepath", label="Upload speech (wav/mp3)")
+            w1 = gr.Slider(minimum=0.0, maximum=1.0, value=0.5, label="Text weight (w_text)")
+            w2 = gr.Slider(minimum=0.0, maximum=1.0, value=0.5, label="Audio weight (w_audio)")
             btn = gr.Button("Predict")
         with gr.Column():
-            final_label = gr.HTML(label="Predicted Emotion")
+            final_label = gr.Markdown(label="Predicted Emotion")
             chart_output = gr.Plot(label="Emotion Scores")
 
-    btn.click(fn=predict, inputs=[txt], outputs=[final_label, chart_output])
+    btn.click(fn=predict, inputs=[txt, aud, w1, w2], outputs=[final_label, chart_output])
 
 demo.launch()