update

app.py

@@ -1,20 +1,32 @@
 import gradio as gr
 import torch
-from transformers import AutoModel, AutoTokenizer
+from transformers import AutoModel, AutoTokenizer, pipeline
 from PIL import Image
 from decord import VideoReader, cpu
 import base64
 import io
 import spaces
 import time
+import os
+from transformers.pipelines.audio_utils import ffmpeg_read
+import moviepy.editor as mp
 
-# Load model
+# Load models
 model_path = 'openbmb/MiniCPM-V-2_6'
 model = AutoModel.from_pretrained(model_path, trust_remote_code=True, torch_dtype=torch.bfloat16)
 model = model.to(device='cuda')
 tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
 model.eval()
 
+# Load Whisper model
+whisper_model = "openai/whisper-large-v3"
+asr_pipeline = pipeline(
+    task="automatic-speech-recognition",
+    model=whisper_model,
+    chunk_length_s=30,
+    device="cuda" if torch.cuda.is_available() else "cpu",
+)
+
 MAX_NUM_FRAMES = 64
 
 def encode_image(image):
@@ -43,6 +55,22 @@ def encode_video(video_path):
     video = [encode_image(v) for v in video]
     return video
 
+def extract_audio(video_path):
+    video = mp.VideoFileClip(video_path)
+    audio_path = "temp_audio.wav"
+    video.audio.write_audiofile(audio_path)
+    return audio_path
+
+def transcribe_audio(audio_file):
+    with open(audio_file, "rb") as f:
+        inputs = f.read()
+
+    inputs = ffmpeg_read(inputs, asr_pipeline.feature_extractor.sampling_rate)
+    inputs = {"array": inputs, "sampling_rate": asr_pipeline.feature_extractor.sampling_rate}
+
+    transcription = asr_pipeline(inputs, batch_size=8, generate_kwargs={"task": "transcribe"}, return_timestamps=True)["text"]
+    return transcription
+
 @spaces.GPU
 def analyze_video(prompt, video):
     start_time = time.time()
@@ -54,7 +82,15 @@ def analyze_video(prompt, video):
     
     encoded_video = encode_video(video_path)
     
+    # Extract audio and transcribe
+    audio_path = extract_audio(video_path)
+    transcription = transcribe_audio(audio_path)
+    
+    # Clean up temporary audio file
+    os.remove(audio_path)
+    
     context = [
+        {"role": "system", "content": f"Transcription of the video: {transcription}"},
         {"role": "user", "content": [prompt] + encoded_video}
     ]
     
@@ -74,19 +110,21 @@ def analyze_video(prompt, video):
     
     end_time = time.time()
     processing_time = end_time - start_time
-    
-    return f"Analysis Result:\n{response}\n\nProcessing Time: {processing_time:.2f} seconds"
+    analysis_result = f"Analysis Result:\n{response}\n\n"
+    processing_time = f"Processing Time: {processing_time:.2f} seconds"
+    return analysis_result, processing_time
 
 with gr.Blocks() as demo:
     gr.Markdown("# Video Analyzer")
     with gr.Row():
         with gr.Column():
-            prompt_input = gr.Textbox(label="Prompt")
+            prompt_input = gr.Textbox(label="Prompt", value="What is the video about?")
             video_input = gr.Video(label="Upload Video")
         with gr.Column():
-            output = gr.Textbox(label="Analysis Result and Processing Time")
+            analysis_result = gr.Textbox(label="Analysis Result")
+            processing_time = gr.Textbox(label="Processing Time")
     
     analyze_button = gr.Button("Analyze Video")
-    analyze_button.click(fn=analyze_video, inputs=[prompt_input, video_input], outputs=output)
+    analyze_button.click(fn=analyze_video, inputs=[prompt_input, video_input], outputs=[analysis_result, processing_time])
 
 demo.launch()

app2.py

@@ -1,571 +1,211 @@
-#!/usr/bin/env python
-# encoding: utf-8
-import spaces
-import torch
-import argparse
-from transformers import AutoModel, AutoTokenizer
 import gradio as gr
-from PIL import Image
-from decord import VideoReader, cpu
-import io
+import yt_dlp
+from dotenv import load_dotenv
 import os
-import copy
-import requests
-import base64
-import json
-import traceback
+import google.generativeai as genai
 import re
-import modelscope_studio as mgr
-
-
-# README, How to run demo on different devices
-
-# For Nvidia GPUs.
-# python web_demo_2.6.py --device cuda
-
-# For Mac with MPS (Apple silicon or AMD GPUs).
-# PYTORCH_ENABLE_MPS_FALLBACK=1 python web_demo_2.6.py --device mps
-
-# Argparser
-parser = argparse.ArgumentParser(description='demo')
-parser.add_argument('--device', type=str, default='cuda', help='cuda or mps')
-parser.add_argument('--multi-gpus', action='store_true', default=False, help='use multi-gpus')
-args = parser.parse_args()
-device = args.device
-assert device in ['cuda', 'mps']
-
-# Load model
-model_path = 'openbmb/MiniCPM-V-2_6'
-if 'int4' in model_path:
-    if device == 'mps':
-        print('Error: running int4 model with bitsandbytes on Mac is not supported right now.')
-        exit()
-    model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
-else:
-    if False: #args.multi_gpus:
-        from accelerate import load_checkpoint_and_dispatch, init_empty_weights, infer_auto_device_map
-        with init_empty_weights():
-            #model = AutoModel.from_pretrained(model_path, trust_remote_code=True, attn_implementation='sdpa', torch_dtype=torch.bfloat16)
-            model = AutoModel.from_pretrained(model_path, trust_remote_code=True, torch_dtype=torch.bfloat16)
-        device_map = infer_auto_device_map(model, max_memory={0: "10GB", 1: "10GB"},
-            no_split_module_classes=['SiglipVisionTransformer', 'Qwen2DecoderLayer'])
-        device_id = device_map["llm.model.embed_tokens"]
-        device_map["llm.lm_head"] = device_id # firtt and last layer should be in same device
-        device_map["vpm"] = device_id
-        device_map["resampler"] = device_id
-        device_id2 = device_map["llm.model.layers.26"]
-        device_map["llm.model.layers.8"] = device_id2
-        device_map["llm.model.layers.9"] = device_id2
-        device_map["llm.model.layers.10"] = device_id2
-        device_map["llm.model.layers.11"] = device_id2
-        device_map["llm.model.layers.12"] = device_id2
-        device_map["llm.model.layers.13"] = device_id2
-        device_map["llm.model.layers.14"] = device_id2
-        device_map["llm.model.layers.15"] = device_id2
-        device_map["llm.model.layers.16"] = device_id2
-        #print(device_map)
-
-        #model = load_checkpoint_and_dispatch(model, model_path, dtype=torch.bfloat16, device_map=device_map)
-        model = AutoModel.from_pretrained(model_path, trust_remote_code=True, torch_dtype=torch.bfloat16, device_map=device_map)
-    else:
-        #model = AutoModel.from_pretrained(model_path, trust_remote_code=True, attn_implementation='sdpa', torch_dtype=torch.bfloat16)
-        model = AutoModel.from_pretrained(model_path, trust_remote_code=True, torch_dtype=torch.bfloat16)
-        model = model.to(device=device)
-tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
-model.eval()
-
-
-
-
-ERROR_MSG = "Error, please retry"
-model_name = 'MiniCPM-V 2.6'
-MAX_NUM_FRAMES = 64
-IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.webp'}
-VIDEO_EXTENSIONS = {'.mp4', '.mkv', '.mov', '.avi', '.flv', '.wmv', '.webm', '.m4v'}
-
-def get_file_extension(filename):
-    return os.path.splitext(filename)[1].lower()
-
-def is_image(filename):
-    return get_file_extension(filename) in IMAGE_EXTENSIONS
-
-def is_video(filename):
-    return get_file_extension(filename) in VIDEO_EXTENSIONS
-
-
-form_radio = {
-    'choices': ['Beam Search', 'Sampling'],
-    #'value': 'Beam Search',
-    'value': 'Sampling',
-    'interactive': True,
-    'label': 'Decode Type'
-}
+import torch
+from transformers import pipeline
+from transformers.pipelines.audio_utils import ffmpeg_read
+import time
+import spaces
 
+load_dotenv()
+default_gemini_api_key = os.getenv('gemini_api_key')
 
-def create_component(params, comp='Slider'):
-    if comp == 'Slider':
-        return gr.Slider(
-            minimum=params['minimum'],
-            maximum=params['maximum'],
-            value=params['value'],
-            step=params['step'],
-            interactive=params['interactive'],
-            label=params['label']
-        )
-    elif comp == 'Radio':
-        return gr.Radio(
-            choices=params['choices'],
-            value=params['value'],
-            interactive=params['interactive'],
-            label=params['label']
-        )
-    elif comp == 'Button':
-        return gr.Button(
-            value=params['value'],
-            interactive=True
-        )
+device = 0 if torch.cuda.is_available() else "cpu"
 
+def load_pipeline(model_name):
+    return pipeline(
+        task="automatic-speech-recognition",
+        model=model_name,
+        chunk_length_s=30,
+        device=device,
+    )
 
-def create_multimodal_input(upload_image_disabled=False, upload_video_disabled=False):
-    return mgr.MultimodalInput(value=None, upload_image_button_props={'label': 'Upload Image', 'disabled': upload_image_disabled, 'file_count': 'multiple'}, 
-                                        upload_video_button_props={'label': 'Upload Video', 'disabled': upload_video_disabled, 'file_count': 'single'},
-                                        submit_button_props={'label': 'Submit'})
+def configure_genai(api_key, model_variant):
+    genai.configure(api_key=api_key)
+    return genai.GenerativeModel(model_variant)
 
+def extract_youtube_id(youtube_url):
+    # Extract the YouTube video ID from various URL formats
+    youtube_id_match = re.search(r'(?:v=|\/)([0-9A-Za-z_-]{11}).*', youtube_url)
+    if youtube_id_match:
+        return youtube_id_match.group(1)
+    return None
 
-@spaces.GPU(duration=120)
-def chat(img, msgs, ctx, params=None, vision_hidden_states=None):
+def download_youtube_audio(youtube_url, output_filename):
+    ydl_opts = {
+        'format': 'bestaudio/best',
+        'postprocessors': [{
+            'key': 'FFmpegExtractAudio',
+            'preferredcodec': 'mp3',
+            'preferredquality': '192',
+        }],
+        'outtmpl': output_filename,
+    }
+    
     try:
-        if msgs[-1]['role'] == 'assistant':
-            msgs = msgs[:-1] # remove last which is added for streaming
-        print('msgs:', msgs)
-        answer = model.chat(
-            image=None,
-            msgs=msgs,
-            tokenizer=tokenizer,
-            **params
-        )
-        if params['stream'] is False:
-            res = re.sub(r'(<box>.*</box>)', '', answer)
-            res = res.replace('<ref>', '')
-            res = res.replace('</ref>', '')
-            res = res.replace('<box>', '')
-            answer = res.replace('</box>', '')
-        print('answer:')
-        for char in answer:
-            print(char, flush=True, end='')
-            yield char
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            ydl.download([youtube_url])
+        
+        print(f"Downloaded audio from YouTube URL: {youtube_url}")
+        return output_filename
     except Exception as e:
-        print(e)
-        traceback.print_exc()
-        yield ERROR_MSG
-
-
-def encode_image(image):
-    if not isinstance(image, Image.Image):
-        if hasattr(image, 'path'):
-            image = Image.open(image.path).convert("RGB")
-        else:
-            image = Image.open(image.file.path).convert("RGB")
-    # resize to max_size
-    max_size = 448*16 
-    if max(image.size) > max_size:
-        w,h = image.size
-        if w > h:
-            new_w = max_size
-            new_h = int(h * max_size / w)
-        else:
-            new_h = max_size
-            new_w = int(w * max_size / h)
-        image = image.resize((new_w, new_h), resample=Image.BICUBIC)
-    return image
-    ## save by BytesIO and convert to base64
-    #buffered = io.BytesIO()
-    #image.save(buffered, format="png")
-    #im_b64 = base64.b64encode(buffered.getvalue()).decode()
-    #return {"type": "image", "pairs": im_b64}
-
-
-def encode_video(video):
-    def uniform_sample(l, n):
-        gap = len(l) / n
-        idxs = [int(i * gap + gap / 2) for i in range(n)]
-        return [l[i] for i in idxs]
-
-    if hasattr(video, 'path'):
-        vr = VideoReader(video.path, ctx=cpu(0))
-    else:
-        vr = VideoReader(video.file.path, ctx=cpu(0))
-    sample_fps = round(vr.get_avg_fps() / 1)  # FPS
-    frame_idx = [i for i in range(0, len(vr), sample_fps)]
-    if len(frame_idx)>MAX_NUM_FRAMES:
-        frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES)
-    video = vr.get_batch(frame_idx).asnumpy()
-    video = [Image.fromarray(v.astype('uint8')) for v in video]
-    video = [encode_image(v) for v in video]
-    print('video frames:', len(video))
-    return video
+        print(f"Error downloading YouTube audio: {str(e)}")
+        raise gr.Error(f"Failed to download YouTube audio: {str(e)}")
 
+def summarize_transcription(transcription, model, gemini_prompt):
+    try:
+        prompt = f"{gemini_prompt}:\n\n{transcription}"
+        response = model.generate_content(prompt)
+        return response.text
+    except Exception as e:
+        print(f"Error summarizing transcription: {str(e)}")
+        return f"Error summarizing transcription: {str(e)}"
+    
+@spaces.GPU(duration=180)
+def process_audio(audio_file, language, whisper_model):
+    print("Starting transcription...")
+    start_time = time.time()
 
-def check_mm_type(mm_file):
-    if hasattr(mm_file, 'path'):
-        path = mm_file.path
+    if device == 0:
+        pipe = load_pipeline(whisper_model)
     else:
-        path = mm_file.file.path
-    if is_image(path):
-        return "image"
-    if is_video(path):
-        return "video"
-    return None
-
-
-def encode_mm_file(mm_file):
-    if check_mm_type(mm_file) == 'image':
-        return [encode_image(mm_file)]
-    if check_mm_type(mm_file) == 'video':
-        return encode_video(mm_file)
-    return None
-
-def make_text(text):
-    #return {"type": "text", "pairs": text} # # For remote call
-    return text
-
-def encode_message(_question):
-    files = _question.files
-    question = _question.text
-    pattern = r"\[mm_media\]\d+\[/mm_media\]"
-    matches = re.split(pattern, question)
-    message = []
-    if len(matches) != len(files) + 1:
-        gr.Warning("Number of Images not match the placeholder in text, please refresh the page to restart!")
-    assert len(matches) == len(files) + 1
-
-    text = matches[0].strip()
-    if text:
-        message.append(make_text(text))
-    for i in range(len(files)):
-        message += encode_mm_file(files[i])
-        text = matches[i + 1].strip()
-        if text:
-            message.append(make_text(text))
-    return message
-
-
-def check_has_videos(_question):
-    images_cnt = 0
-    videos_cnt = 0
-    for file in _question.files:
-        if check_mm_type(file) == "image":
-            images_cnt += 1 
-        else:
-            videos_cnt += 1
-    return images_cnt, videos_cnt 
-
-
-def count_video_frames(_context):
-    num_frames = 0
-    for message in _context:
-        for item in message["content"]:
-            #if item["type"] == "image": # For remote call
-            if isinstance(item, Image.Image):
-                num_frames += 1
-    return num_frames
-
+        pipe = load_pipeline("openai/whisper-tiny")
 
-def request(_question, _chat_bot, _app_cfg):
-    images_cnt = _app_cfg['images_cnt']
-    videos_cnt = _app_cfg['videos_cnt']
-    files_cnts = check_has_videos(_question)
-    if files_cnts[1] + videos_cnt > 1 or (files_cnts[1] + videos_cnt == 1 and files_cnts[0] + images_cnt > 0):
-        gr.Warning("Only supports single video file input right now!")
-        return _question, _chat_bot, _app_cfg
-    if files_cnts[1] + videos_cnt + files_cnts[0] + images_cnt <= 0:
-        gr.Warning("Please chat with at least one image or video.")
-        return _question, _chat_bot, _app_cfg
-    _chat_bot.append((_question, None))
-    images_cnt += files_cnts[0]
-    videos_cnt += files_cnts[1]
-    _app_cfg['images_cnt'] = images_cnt
-    _app_cfg['videos_cnt'] = videos_cnt
-    upload_image_disabled = videos_cnt > 0
-    upload_video_disabled = videos_cnt > 0 or images_cnt > 0
-    return create_multimodal_input(upload_image_disabled, upload_video_disabled), _chat_bot, _app_cfg
+    with open(audio_file, "rb") as f:
+        inputs = f.read()
 
+    inputs = ffmpeg_read(inputs, pipe.feature_extractor.sampling_rate)
+    inputs = {"array": inputs, "sampling_rate": pipe.feature_extractor.sampling_rate}
 
-def respond(_chat_bot, _app_cfg, params_form):
-    if len(_app_cfg) == 0:
-        yield (_chat_bot, _app_cfg)
-    elif _app_cfg['images_cnt'] == 0 and _app_cfg['videos_cnt'] == 0:
-        yield(_chat_bot, _app_cfg)
+    if language:
+        print(f"Using language: {language}")
+        transcription = pipe(inputs, batch_size=8, generate_kwargs={"task": "transcribe", "language": language}, return_timestamps=True)["text"]
     else:
-        _question = _chat_bot[-1][0]
-        _context = _app_cfg['ctx'].copy()
-        _context.append({'role': 'user', 'content': encode_message(_question)})
-
-        videos_cnt = _app_cfg['videos_cnt']
+        print("No language defined, using default language")
+        transcription = pipe(inputs, batch_size=8, generate_kwargs={"task": "transcribe"}, return_timestamps=True)["text"]
+    
+    end_time = time.time()
+    processing_time = round(end_time - start_time, 2)
+    return transcription, processing_time
 
-        if params_form == 'Beam Search':
-            params = {
-                'sampling': False,
-                'stream': False,
-                'num_beams': 3,
-                'repetition_penalty': 1.2,
-                "max_new_tokens": 2048
-            }
+def transcribe(youtube_url, audio_file, whisper_model, gemini_api_key, gemini_prompt, gemini_model_variant, language, progress=gr.Progress()):
+    try:
+        progress(0, desc="Initializing")
+        if not gemini_api_key:
+            gemini_api_key = default_gemini_api_key
+        model = configure_genai(gemini_api_key, gemini_model_variant)
+
+        if youtube_url:
+            progress(0.1, desc="Extracting YouTube ID")
+            youtube_id = extract_youtube_id(youtube_url)
+            if youtube_id:
+                output_filename = f"{youtube_id}"
+            else:
+                output_filename = f"unknown"
+            progress(0.2, desc="Downloading YouTube audio")
+            audio_file = download_youtube_audio(youtube_url, output_filename)
+            audio_file = f"{audio_file}.mp3"
+            print(f"Audio file downloaded: {audio_file}")
         else:
-            params = {
-                'sampling': True,
-                'stream': True,
-                'top_p': 0.8,
-                'top_k': 100,
-                'temperature': 0.7,
-                'repetition_penalty': 1.05,
-                "max_new_tokens": 2048
-            }
-        params["max_inp_length"] = 4352 # 4096+256
-
-        if videos_cnt > 0:
-            #params["max_inp_length"] = 4352 # 4096+256
-            params["use_image_id"] = False
-            params["max_slice_nums"] = 1 if count_video_frames(_context) > 16 else 2
-
-        gen = chat("", _context, None, params)
-
-        _context.append({"role": "assistant", "content": [""]}) 
-        _chat_bot[-1][1] = ""
-
-        for _char in gen:
-            _chat_bot[-1][1] += _char
-            _context[-1]["content"][0] += _char
-            yield (_chat_bot, _app_cfg)
+            progress(0.2, desc="Reading audio file")
+            audio_file = f"{audio_file.name}"
+            print(f"Audio file read: {audio_file}")
         
-        _app_cfg['ctx']=_context
-        yield (_chat_bot, _app_cfg)
-
+        progress(0.4, desc="Starting transcription")
+        transcription, processing_time = process_audio(audio_file, language, whisper_model)
+        
+        progress(0.6, desc="Cleaning up")
+        # Delete the audio file after transcription
+        if os.path.exists(f"{audio_file}.mp3"):
+            os.remove(f"{audio_file}.mp3")
+            print(f"Deleted audio file: {audio_file}.mp3")
+        
+        progress(0.7, desc="Summarizing transcription")
+        # Summarize the transcription
+        summary = summarize_transcription(transcription, model, gemini_prompt)
+        
+        progress(0.8, desc="Preparing output")
+        # Prepare the transcription and summary message
+        transcription_message = f"{transcription}" if transcription else ""
+        
+        summary_message = f"{summary}" if summary else ""
+        
+        progress(0.9, desc="Saving output to file")
+        print("Saving transcription and summary to file...")
+        # Save transcription and summary to separate text files
+        transcription_file = "transcription_output.txt"
+        summary_file = "summary_output.txt"
+        with open(transcription_file, "w", encoding="utf-8") as f:
+            f.write(transcription_message)
+        with open(summary_file, "w", encoding="utf-8") as f:
+            f.write(summary_message)
+        
+        progress(1, desc="Complete")
+        print("Transcription and summarization complete.")
+        return transcription_message, summary_message, transcription_file, summary_file, processing_time
+    except gr.Error as e:
+        # Re-raise Gradio errors
+        raise e
+    except Exception as e:
+        print(f"Error during transcription or summarization: {str(e)}")
+        raise gr.Error(f"Transcription or summarization failed: {str(e)}")
 
-def fewshot_add_demonstration(_image, _user_message, _assistant_message, _chat_bot, _app_cfg):
-    ctx = _app_cfg["ctx"]
-    message_item = []
-    if _image is not None:
-        image = Image.open(_image).convert("RGB")
-        ctx.append({"role": "user", "content": [encode_image(image), make_text(_user_message)]})
-        message_item.append({"text": "[mm_media]1[/mm_media]" + _user_message, "files": [_image]})
-        _app_cfg["images_cnt"] += 1
-    else:
-        if _user_message:
-            ctx.append({"role": "user", "content": [make_text(_user_message)]})
-            message_item.append({"text": _user_message, "files": []})
+def toggle_input(choice):
+        if choice == "YouTube URL":
+            return gr.update(visible=True), gr.update(visible=False, value=None)
         else:
-            message_item.append(None)
-    if _assistant_message:
-        ctx.append({"role": "assistant", "content": [make_text(_assistant_message)]})
-        message_item.append({"text": _assistant_message, "files": []})
-    else:
-        message_item.append(None)
-
-    _chat_bot.append(message_item)
-    return None, "", "", _chat_bot, _app_cfg
-
-
-def fewshot_request(_image, _user_message, _chat_bot, _app_cfg):
-    if _app_cfg["images_cnt"] == 0 and not _image:
-        gr.Warning("Please chat with at least one image.")
-        return None, '', '', _chat_bot, _app_cfg
-    if _image:
-        _chat_bot.append([
-            {"text": "[mm_media]1[/mm_media]" + _user_message, "files": [_image]},
-            ""        
-        ])
-        _app_cfg["images_cnt"] += 1
+            return gr.update(visible=False, value=None), gr.update(visible=True)
+        
+def toggle_language(choice):
+    if choice == True:
+        return gr.update(visible=True, value="id")
     else:
-        _chat_bot.append([
-            {"text": _user_message, "files": [_image]},
-            ""
-        ])
-
-    return None, '', '', _chat_bot, _app_cfg
-
-
-def regenerate_button_clicked(_chat_bot, _app_cfg):
-    if len(_chat_bot) <= 1 or not _chat_bot[-1][1]:
-        gr.Warning('No question for regeneration.')
-        return None, None, '', '', _chat_bot, _app_cfg
-    if _app_cfg["chat_type"] == "Chat":
-        images_cnt = _app_cfg['images_cnt']
-        videos_cnt = _app_cfg['videos_cnt']
-        _question = _chat_bot[-1][0]
-        _chat_bot = _chat_bot[:-1]
-        _app_cfg['ctx'] = _app_cfg['ctx'][:-2]
-        files_cnts = check_has_videos(_question)
-        images_cnt -= files_cnts[0]
-        videos_cnt -= files_cnts[1]
-        _app_cfg['images_cnt'] = images_cnt
-        _app_cfg['videos_cnt'] = videos_cnt
+        return gr.update(visible=False, value="")
 
-        _question, _chat_bot, _app_cfg = request(_question, _chat_bot, _app_cfg)
-        return _question, None, '', '', _chat_bot, _app_cfg
-    else: 
-        last_message = _chat_bot[-1][0]
-        last_image = None
-        last_user_message = ''
-        if last_message.text:
-            last_user_message = last_message.text
-        if last_message.files:
-            last_image = last_message.files[0].file.path
-        _chat_bot[-1][1] = ""
-        _app_cfg['ctx'] = _app_cfg['ctx'][:-2]
-        return _question, None, '', '', _chat_bot, _app_cfg
-
-
-def flushed():
-    return gr.update(interactive=True)
-
-
-def clear(txt_message, chat_bot, app_session):
-    txt_message.files.clear()
-    txt_message.text = ''
-    chat_bot = copy.deepcopy(init_conversation)
-    app_session['sts'] = None
-    app_session['ctx'] = []
-    app_session['images_cnt'] = 0
-    app_session['videos_cnt'] = 0
-    return create_multimodal_input(), chat_bot, app_session, None, '', ''
+with gr.Blocks(theme='NoCrypt/miku') as demo:
+    gr.Label('Youtube Summarizer WebUI created with ❤️ by Ryusui', show_label=False)
     
-
-def select_chat_type(_tab, _app_cfg):
-    _app_cfg["chat_type"] = _tab
-    return _app_cfg
-
-
-init_conversation = [
-    [
-        None,
-        {
-            # The first message of bot closes the typewriter.
-            "text": "You can talk to me now",
-            "flushing": False
-        }
-    ],
-]
-
-
-css = """
-.example label { font-size: 16px;}
-"""
-
-introduction = """
-
-## Features:
-1. Chat with single image
-2. Chat with multiple images
-3. Chat with video
-4. In-context few-shot learning
-
-Click `How to use` tab to see examples.
-"""
-
-
-with gr.Blocks(css=css) as demo:
-    with gr.Tab(model_name):
-        with gr.Row():
-            with gr.Column(scale=1, min_width=300):
-                gr.Markdown(value=introduction)
-                params_form = create_component(form_radio, comp='Radio')
-                regenerate = create_component({'value': 'Regenerate'}, comp='Button')
-                clear_button = create_component({'value': 'Clear History'}, comp='Button')
-
-            with gr.Column(scale=3, min_width=500):
-                app_session = gr.State({'sts':None,'ctx':[], 'images_cnt': 0, 'videos_cnt': 0, 'chat_type': 'Chat'})
-                chat_bot = mgr.Chatbot(label=f"Chat with {model_name}", value=copy.deepcopy(init_conversation), height=600, flushing=False, bubble_full_width=False)
-                
-                with gr.Tab("Chat") as chat_tab:
-                    txt_message = create_multimodal_input()
-                    chat_tab_label = gr.Textbox(value="Chat", interactive=False, visible=False)
-
-                    txt_message.submit(
-                        request,
-                        [txt_message, chat_bot, app_session], 
-                        [txt_message, chat_bot, app_session]
-                    ).then(
-                        respond,
-                        [chat_bot, app_session, params_form],
-                        [chat_bot, app_session]
-                    )
-
-                with gr.Tab("Few Shot") as fewshot_tab:
-                    fewshot_tab_label = gr.Textbox(value="Few Shot", interactive=False, visible=False)
-                    with gr.Row():
-                        with gr.Column(scale=1):
-                            image_input = gr.Image(type="filepath", sources=["upload"])
-                        with gr.Column(scale=3):
-                            user_message = gr.Textbox(label="User")
-                            assistant_message = gr.Textbox(label="Assistant")
-                            with gr.Row():
-                                add_demonstration_button = gr.Button("Add Example")
-                                generate_button = gr.Button(value="Generate", variant="primary")
-                    add_demonstration_button.click(
-                        fewshot_add_demonstration,
-                        [image_input, user_message, assistant_message, chat_bot, app_session],
-                        [image_input, user_message, assistant_message, chat_bot, app_session]
-                    )
-                    generate_button.click(
-                        fewshot_request,
-                        [image_input, user_message, chat_bot, app_session],
-                        [image_input, user_message, assistant_message, chat_bot, app_session]
-                    ).then(
-                        respond,
-                        [chat_bot, app_session, params_form],
-                        [chat_bot, app_session]
-                    )
-
-                chat_tab.select(
-                    select_chat_type,
-                    [chat_tab_label, app_session],
-                    [app_session]
-                )
-                chat_tab.select( # do clear
-                    clear,
-                    [txt_message, chat_bot, app_session],
-                    [txt_message, chat_bot, app_session, image_input, user_message, assistant_message]
-                )
-                fewshot_tab.select(
-                    select_chat_type,
-                    [fewshot_tab_label, app_session],
-                    [app_session]
-                )
-                fewshot_tab.select( # do clear
-                    clear,
-                    [txt_message, chat_bot, app_session],
-                    [txt_message, chat_bot, app_session, image_input, user_message, assistant_message]
-                )
-                chat_bot.flushed(
-                    flushed,
-                    outputs=[txt_message]
-                )
-                regenerate.click(
-                    regenerate_button_clicked,
-                    [chat_bot, app_session],
-                    [txt_message, image_input, user_message, assistant_message, chat_bot, app_session]
-                ).then(
-                    respond,
-                    [chat_bot, app_session, params_form],
-                    [chat_bot, app_session]
-                )
-                clear_button.click(
-                    clear,
-                    [txt_message, chat_bot, app_session],
-                    [txt_message, chat_bot, app_session, image_input, user_message, assistant_message]
-                )
-
-    with gr.Tab("How to use"):
+    with gr.Accordion("Input"):
         with gr.Column():
+            input_type = gr.Radio(["YouTube URL", "Audio File"], label="Input Type", value="Audio File", info="Please consider using the audio file if you face any issues with the YouTube URL. Currently youtube is banning HuggingFace IP Addresses.")
             with gr.Row():
-                image_example = gr.Image(value="http://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/m_bear2.gif", label='1. Chat with single or multiple images', interactive=False, width=400, elem_classes="example")
-                example2 = gr.Image(value="http://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/video2.gif", label='2. Chat with video', interactive=False, width=400, elem_classes="example")
-                example3 = gr.Image(value="http://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/fshot.gif", label='3. Few shot', interactive=False, width=400, elem_classes="example")
-
-
-# launch
-#demo.launch(share=False, debug=True, show_api=False, server_port=8885, server_name="0.0.0.0")
-demo.queue()
-demo.launch(show_api=False)
-
+                youtube_url = gr.Textbox(label="YouTube URL", visible=False, info="Input the full URL of the YouTube video you want to transcribe and summarize. Example: https://www.youtube.com/watch?v=VIDEO_ID")
+                audio_file = gr.File(label="Upload Audio File", visible=True, file_types=['.wav', '.flac', '.mp3'])
+                whisper_model = gr.Dropdown(["openai/whisper-tiny", "openai/whisper-base", "openai/whisper-small", "openai/whisper-medium", "openai/whisper-large-v3"], label="Whisper Model", value="openai/whisper-large-v3", info="Tiny is the fastest model, but it's not the best quality. large-v3 is the best quality, but it's the slowest model.")
+                gemini_model_variant = gr.Dropdown(["gemini-1.5-flash", "gemini-1.5-pro"], label="Gemini Model Variant", value="gemini-1.5-pro", info="Gemini-1.5-flash is the fastest model, but it's not the best quality. Gemini-1.5-pro is the best quality, but it's slower")
+            define_language = gr.Checkbox(label="Define Language", value=False, info="If you want to define the language, check this box")
+            language = gr.Dropdown(["id","en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], label="Language", value=None, info="Select the language for transcription", visible=False)
+            gemini_api_key = gr.Textbox(label="Gemini API Key (Optional)", placeholder="Enter your Gemini API key or leave blank to use default", info="If you facing error on transcription, please try to use your own API key")
+            gemini_prompt = gr.Textbox(label="Gemini Prompt", value="Buatkan resume dari transkrip ini")
+            transcribe_button = gr.Button("Transcribe and Summarize")
+
+    with gr.Accordion("Output"):
+        with gr.Column():
+            transcription_output = gr.Textbox(label="Transcription Output")
+            summary_output = gr.Textbox(label="Summary Output")
+        transcription_file = gr.File(label="Download Transcription")
+        summary_file = gr.File(label="Download Summary")
+        processing_time = gr.Textbox(label="Transcription Processing Time (seconds)")
+    
+    input_type.change(fn=toggle_input, inputs=input_type, outputs=[youtube_url, audio_file])
+    define_language.change(fn=toggle_language, inputs=define_language, outputs=[language])
+
+    transcribe_button.click(
+        fn=transcribe,
+        inputs=[
+            youtube_url,
+            audio_file,
+            whisper_model,
+            gemini_api_key,
+            gemini_prompt,
+            gemini_model_variant,
+            language,
+        ],
+        outputs=[transcription_output, summary_output, transcription_file, summary_file, processing_time]
+    )
+
+print("Launching Gradio interface...")
+demo.launch()