add app.py and readme

README.md

@@ -1,13 +1,13 @@
 ---
-title: ImageBind Zeroshot Demo
-emoji: 💻
-colorFrom: blue
-colorTo: indigo
+title: ImageBind
+emoji: 🔥
+colorFrom: yellow
+colorTo: pink
 sdk: gradio
-sdk_version: 3.29.0
+sdk_version: 3.12.0
 app_file: app.py
 pinned: false
 license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,51 @@
+import data
+import torch
+import gradio as gr
+from models import imagebind_model
+from models.imagebind_model import ModalityType
+
+
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+model = imagebind_model.imagebind_huge(pretrained=True)
+model.eval()
+model.to(device)
+
+
+def image_text_zeroshot(image, text_list):
+    image_paths = [image]
+    labels = [label.strip(" ") for label in text_list.strip(" ").split(",")]
+    inputs = {
+        ModalityType.TEXT: data.load_and_transform_text(text_list, device),
+        ModalityType.VISION: data.load_and_transform_vision_data(image_paths, device),
+    }
+
+    with torch.no_grad():
+        embeddings = model(inputs)
+
+    scores = torch.softmax(
+        embeddings[ModalityType.VISION] @ embeddings[ModalityType.AUDIO].T,
+        dim=-1
+    ).squeeze(0).tolist()
+
+    score_dict = {label:score for label, score in zip(labels, scores)}
+
+    return score_dict
+
+
+inputs = [
+    gr.inputs.Image(type='file',
+                    label="Input image"),
+    gr.inputs.Textbox(lines=1,
+                      label="Candidate texts"),
+]
+
+iface = gr.Interface(image_text_zeroshot,
+            inputs,
+            "label",
+            examples=[[".assets/dog_image.jpg", "A dog|A car|A bird"],
+                      [".assets/car_image.jpg", "A dog|A car|A bird"],
+                      [".assets/bird_image.jpg", "A dog|A car|A bird"]],
+            description="""Zeroshot test""",
+            title="Zero-shot Classification")
+
+iface.launch()