Upload 3 files

README.md

@@ -1,13 +1,14 @@
 ---
-title: Anime Speech Japanese Refiner Demo
-emoji: 🐨
-colorFrom: yellow
-colorTo: purple
+title: Anime-Speech-Japanese-Refiner-Demo
+emoji: ⚡
+colorFrom: green
+colorTo: green
 sdk: gradio
 sdk_version: 5.49.1
 app_file: app.py
 pinned: false
 license: cc-by-nc-4.0
+startup_duration_timeout: 1h
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+import os
+import re
+from typing import Dict, List, Optional, Tuple
+
+import gradio as gr
+import spaces
+import torch
+from qwen_omni_utils import process_mm_info
+from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor
+
+# -------------------------
+# Config
+# -------------------------
+
+# Model ID to use for captioning
+MODEL_ID = "NandemoGHS/Anime-Speech-Japanese-Refiner"
+
+# Generation defaults
+DEFAULT_TEMPERATURE = 0.6
+DEFAULT_TOP_P = 0.95
+DEFAULT_MAX_NEW_TOKENS = 1024
+
+model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(
+    MODEL_ID,
+    trust_remote_code=True,
+    device_map="auto",
+    dtype="auto",
+)
+
+processor = Qwen3OmniMoeProcessor.from_pretrained(MODEL_ID)
+
+
+# -------------------------
+# Utilities
+# -------------------------
+def _build_prompt(original_transcription: str) -> str:
+    """Build the Japanese instruction prompt (kept from your notebook, minor formatting)."""
+    return f"""これから与えられる音声クリップとその文字起こしについて、声の特徴と読み上げスタイル、感情などをアノテーションしたうえで、日本語の短いキャプションで要約してください。
+出力には以下の項目を含めてください。
+
+profile: 話者プロファイル（例: お姉さん的な女性声／落ち着いた男性声／少女声 等）
+mood: 感情・ムード（例: 明るい／落ち着いた／緊張／怒り／恐怖／悲しみ／快楽 等）
+speed: 話速（例: とても遅い／やや速い／一定／(1.2×) 等）
+prosody: 抑揚・リズム（例: 平坦／メリハリ／語尾上げ下げ／ため息混じり 等）
+pitch_timbre: ピッチ/声質（例: 高め／低め／息多め／張りのある／囁き 等）
+style: 発話スタイル（例: ナレーション風／会話調／朗読調／プレゼン調／囁き／喘ぎ／嗚咽／叫び 等）
+emotion: 感情タグ（次のリストから1つ選択: ["angry", "sad", "disdainful", "excited", "surprised", "satisfied", "unhappy", "anxious", "hysterical", "delighted", "scared", "worried", "indifferent", "upset", "impatient", "nervous", "guilty", "scornful", "frustrated", "depressed", "panicked", "furious", "empathetic", "embarrassed", "reluctant", "disgusted", "keen", "moved", "proud", "relaxed", "grateful", "confident", "interested", "curious", "confused", "joyful", "disapproving", "negative", "denying", "astonished", "serious", "sarcastic", "conciliative", "comforting", "sincere", "sneering", "hesitating", "yielding", "painful", "awkward", "amused", "loving", "dating", "longing", "aroused", "seductive", "ecstatic", "shy"]）
+notes: 特記事項（間の取り方、笑い・ため・ブレス、ノイズ感、キス音、効果音、チュパ音 等）
+caption: 上記を1〜2文・全角30〜80文字で自然文に要約
+refined_text: 元の文字起こしテキストに、必要に応じて特殊タグを音声中のイベントの描写として文章のどこかに挿入したもの（必要なければ元テキストをそのまま出力）。
+
+元の文字起こしテキスト: {original_transcription}
+元の音声クリップ："""
+
+
+KEYS = [
+    "profile",
+    "mood",
+    "speed",
+    "prosody",
+    "pitch_timbre",
+    "style",
+    "emotion",
+    "notes",
+    "caption",
+    "refined_text",
+]
+
+
+def _decode_sequences(processor, sequences, input_ids_len: int) -> str:
+    """Decode generated tokens into text, skipping prompt tokens."""
+    out = processor.batch_decode(
+        sequences[:, input_ids_len:],
+        skip_special_tokens=True,
+        clean_up_tokenization_spaces=False,
+    )
+    return out[0] if out else ""
+
+
+def _run_transformers_inference(
+    audio_path: str,
+    original_transcription: str,
+    temperature: float,
+    top_p: float,
+    max_new_tokens: int,
+) -> str:
+    """
+    Prepare chat messages, encode with processor, and generate with the model.
+    Returns raw text output from the model.
+    """
+
+    # Compose the chat content (text + audio) following the Omni chat template.
+    prompt = _build_prompt(original_transcription)
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": prompt},
+                {"type": "audio", "audio": audio_path},
+            ],
+        }
+    ]
+
+    # Prepare inputs
+    text = processor.apply_chat_template(
+        messages, add_generation_prompt=True, tokenize=False
+    )
+    audios, images, videos = process_mm_info(messages, use_audio_in_video=True)
+
+    inputs = processor(
+        text=text,
+        audio=audios,
+        images=images,
+        videos=videos,
+        return_tensors="pt",
+        padding=True,
+        use_audio_in_video=True,
+    )
+
+    inputs = inputs.to(model.device).to(model.dtype)
+
+    gen_kwargs = dict(
+        thinker_max_new_tokens=max_new_tokens,
+        thinker_do_sample=True,
+        thinker_top_p=top_p,
+        thinker_temperature=temperature,
+    )
+
+    # Some Qwen Omni variants support extra kwargs; try them first, then fall back.
+    sequences = None
+    with torch.no_grad():
+        text_ids, _ = model.generate(
+            **inputs,
+            thinker_return_dict_in_generate=True,
+            output_scores=False,
+            use_audio_in_video=True,
+            return_audio=False,
+            **gen_kwargs,
+        )
+        sequences = text_ids.sequences
+
+    input_ids_len = inputs["input_ids"].shape[1]
+    response_text = _decode_sequences(processor, sequences, input_ids_len)
+    return response_text.strip()
+
+
+# -------------------------
+# Gradio Inference
+# -------------------------
+
+
+@spaces.GPU(duration=120)  # Request ephemeral GPU for up to 120s per call
+def infer(
+    audio_file: Optional[str],
+    original_transcription: str,
+    temperature: float,
+    top_p: float,
+    max_new_tokens: int,
+    progress=gr.Progress(track_tqdm=True),
+):
+    # Validate inputs
+    if not original_transcription or not original_transcription.strip():
+        gr.Warning(
+            "元の文字起こしテキストは必須です。/ The original transcription is required."
+        )
+        return None, None, None
+
+    if not audio_file:
+        gr.Warning(
+            "音声ファイルをアップロードしてください。/ Please upload an audio file."
+        )
+        return None, None, None
+
+    progress(0.45, desc="Encoding & generating...")
+    raw_text = _run_transformers_inference(
+        audio_path=audio_file,
+        original_transcription=original_transcription.strip(),
+        temperature=temperature,
+        top_p=top_p,
+        max_new_tokens=int(max_new_tokens),
+    )
+
+    progress(1.0, desc="Done.")
+    return raw_text
+
+
+# -------------------------
+# Gradio UI
+# -------------------------
+
+DESCRIPTION = """
+# Anime Speech Japanese Captioner (Transformers / ZeroGPU)
+
+Upload an audio clip and provide its **original transcription (required)**.
+The model analyzes voice profile, mood, prosody, style, and produces a concise Japanese caption along with a refined text.
+
+**Model:** `NandemoGHS/Anime-Speech-Japanese-Refiner`
+"""
+
+with gr.Blocks(theme="soft") as demo:
+    gr.Markdown(DESCRIPTION)
+
+    with gr.Row():
+        audio_in = gr.Audio(
+            label="Audio (required)",
+            type="filepath",  # we pass the local path to the processor
+            sources=["upload", "microphone"],
+            waveform_options={"show_controls": True},
+        )
+        text_in = gr.Textbox(
+            label="Original Transcription (required)",
+            placeholder="音声の元の文字起こしを入力してください。",
+            lines=6,
+        )
+
+    with gr.Row():
+        temperature = gr.Slider(
+            0.0, 1.0, step=0.05, value=DEFAULT_TEMPERATURE, label="Temperature"
+        )
+        top_p = gr.Slider(0.0, 1.0, step=0.05, value=DEFAULT_TOP_P, label="Top-p")
+        max_new_tokens = gr.Slider(
+            64, 2048, step=32, value=DEFAULT_MAX_NEW_TOKENS, label="Max new tokens"
+        )
+
+    run_btn = gr.Button("Caption", variant="primary")
+
+    out = gr.Textbox(label="Output", lines=16)
+
+    run_btn.click(
+        infer,
+        inputs=[audio_in, text_in, temperature, top_p, max_new_tokens],
+        outputs=[out],
+        api_name="infer",
+    )
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,5 @@
+transformers
+accelerate
+soundfile
+qwen_omni_utils
+torchvision