add audio

app.py

@@ -22,32 +22,92 @@ def image_text_zeroshot(image, text_list):
     with torch.no_grad():
         embeddings = model(inputs)
 
-    scores = torch.softmax(
-        embeddings[ModalityType.VISION] @ embeddings[ModalityType.TEXT].T,
-        dim=-1
-    ).squeeze(0).tolist()
+    scores = (
+        torch.softmax(
+            embeddings[ModalityType.VISION] @ embeddings[ModalityType.TEXT].T, dim=-1
+        )
+        .squeeze(0)
+        .tolist()
+    )
 
-    score_dict = {label:score for label, score in zip(labels, scores)}
+    score_dict = {label: score for label, score in zip(labels, scores)}
 
     return score_dict
 
 
-inputs = [
-    gr.inputs.Image(type='filepath',
-                    label="Input image"),
-    gr.inputs.Textbox(lines=1,
-                      label="Candidate texts"),
-]
-
-iface = gr.Interface(image_text_zeroshot,
-            inputs,
-            "label",
-            examples=[["assets/dog_image.jpg", "A dog|A car|A bird"],
-                      ["assets/car_image.jpg", "A dog|A car|A bird"],
-                      ["assets/bird_image.jpg", "A dog|A car|A bird"]],
-            description="""<p>This is a simple demo of ImageBind for zeroshot image classification. Please refer to the original <a href='https://arxiv.org/abs/2305.05665' target='_blank'>paper</a> and <a href='https://github.com/facebookresearch/ImageBind' target='_blank'>repo</a> for more details.<br>
-                To test your own cases, you can upload an image, and provide the candidate texts separated by "|".<br>
-                You can duplicate this space and run it privately: <a href='https://huggingface.co/spaces/OFA-Sys/chinese-clip-zero-shot-image-classification?duplicate=true'><img src='https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14' alt='Duplicate Space'></a></p>""",
-            title="ImageBind: Zero-shot Image Classification")
-
-iface.launch()
+def audio_text_zeroshot(audio, text_list):
+    audio_paths = [audio]
+    labels = [label.strip(" ") for label in text_list.strip(" ").split("|")]
+    inputs = {
+        ModalityType.TEXT: data.load_and_transform_text(labels, device),
+        ModalityType.AUDIO: data.load_and_transform_vision_data(audio_paths, device),
+    }
+
+    with torch.no_grad():
+        embeddings = model(inputs)
+
+    scores = (
+        torch.softmax(
+            embeddings[ModalityType.AUDIO] @ embeddings[ModalityType.TEXT].T, dim=-1
+        )
+        .squeeze(0)
+        .tolist()
+    )
+
+    score_dict = {label: score for label, score in zip(labels, scores)}
+
+    return score_dict
+
+
+def inference(
+    task,
+    image=None,
+    audio=None,
+    text_list=None,
+):
+    if task == "image-text":
+        result = image_text_zeroshot(image, text_list)
+    elif task == "audio-text":
+        result = audio_text_zeroshot(audio, text_list)
+    else:
+        raise NotImplementedError
+    return result
+
+
+def main():
+    inputs = [
+        gr.inputs.Radio(
+            choices=[
+                "image-text",
+                "audio-text",
+            ],
+            type="value",
+            default="image-text",
+            label="Task",
+        ),
+        gr.inputs.Image(type="filepath", label="Input image"),
+        gr.inputs.Audio(type="filepath", label="Input audio"),
+        gr.inputs.Textbox(lines=1, label="Candidate texts"),
+    ]
+
+    iface = gr.Interface(
+        inference,
+        inputs,
+        "label",
+        examples=[
+            ["image-text", "assets/dog_image.jpg", None, "A dog|A car|A bird"],
+            ["image-text", "assets/car_image.jpg", None, "A dog|A car|A bird"],
+            ["audio-text", None, "assets/bird_audio.wav", "A dog|A car|A bird"],
+            ["audio-text", None, "assets/dog_audio.wav", "A dog|A car|A bird"],
+        ],
+        description="""<p>This is a simple demo of ImageBind for zero-shot cross-modal understanding (now including image classification and audio classification). Please refer to the original <a href='https://arxiv.org/abs/2305.05665' target='_blank'>paper</a> and <a href='https://github.com/facebookresearch/ImageBind' target='_blank'>repo</a> for more details.<br>
+                    To test your own cases, you can upload an image or an audio, and provide the candidate texts separated by "|".<br>
+                    You can duplicate this space and run it privately: <a href='https://huggingface.co/spaces/OFA-Sys/chinese-clip-zero-shot-image-classification?duplicate=true'><img src='https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14' alt='Duplicate Space'></a></p>""",
+        title="ImageBind: Zero-shot Cross-modal Understanding",
+    )
+
+    iface.launch()
+
+
+if __name__ == "__main__":
+    main()