Duplicate from SDbiaseval/find-my-butterfly

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+gradio_cached_examples/

README.md

@@ -0,0 +1,14 @@
+---
+title: Find My Butterfly 🦋
+emoji: 🦋
+colorFrom: yellow
+colorTo: blue
+sdk: gradio
+sdk_version: 3.12.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+duplicated_from: SDbiaseval/find-my-butterfly
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,71 @@
+import pickle
+import gradio as gr
+from datasets import load_dataset
+from transformers import AutoModel, AutoFeatureExtractor
+import wikipedia
+
+
+# Only runs once when the script is first run.
+with open("index_768_cosine.pickle", "rb") as handle:
+    index = pickle.load(handle)
+
+# Load model for computing embeddings.
+feature_extractor = AutoFeatureExtractor.from_pretrained("sasha/autotrain-butterfly-similarity-2490576840")
+model = AutoModel.from_pretrained("sasha/autotrain-butterfly-similarity-2490576840")
+
+# Candidate images.
+dataset = load_dataset("sasha/butterflies_10k_names_multiple")
+ds = dataset["train"]
+
+
+def query(image, top_k=4):
+    inputs = feature_extractor(image, return_tensors="pt")
+    model_output = model(**inputs)
+    embedding = model_output.pooler_output.detach()
+    results = index.query(embedding, k=top_k)
+    inx = results[0][0].tolist()
+    logits = results[1][0].tolist()
+    images = ds.select(inx)["image"]
+    captions = ds.select(inx)["name"]
+    images_with_captions = [(i, c) for i, c in zip(images,captions)]
+    labels_with_probs = dict(zip(captions,logits))
+    labels_with_probs = {k: 1- v for k, v in labels_with_probs.items()}
+    try:
+    	description = wikipedia.summary(captions[0], sentences = 1)
+    	description = "### " + description
+    	url = wikipedia.page(captions[0]).url
+    	url = " You can learn more about your butterfly [here](" + str(url) + ")!"
+    	description = description + url
+    except:
+    	description = "### Butterflies are insects in the order Lepidoptera, which also includes moths. Adult butterflies have large, often brightly coloured wings."
+    	url = "https://en.wikipedia.org/wiki/Butterfly"
+    	url = " You can learn more about butterflies [here](" + str(url) + ")!"
+    	description = description + url
+    return images_with_captions, labels_with_probs, description
+
+
+with gr.Blocks() as demo:
+	gr.Markdown("# Find my Butterfly 🦋")
+	gr.Markdown("## Use this Space to find your butterfly, based on the [iNaturalist butterfly dataset](https://huggingface.co/datasets/huggan/inat_butterflies_top10k)!")
+	with gr.Row():
+		with gr.Column(min_width= 900):
+			inputs = gr.Image(shape=(800, 1600))
+			btn = gr.Button("Find my butterfly!")
+			description = gr.Markdown()
+			
+		with gr.Column():
+			outputs=gr.Gallery().style(grid=[2], height="auto")
+			labels = gr.Label()
+			
+	gr.Markdown("### Image Examples")
+	gr.Examples(
+	examples=["elton.jpg", "ken.jpg", "gaga.jpg", "taylor.jpg"],
+	inputs=inputs,
+	outputs=[outputs,labels],
+	fn=query,
+	cache_examples=True,
+	)
+	btn.click(query, inputs, [outputs, labels, description])
+	
+demo.launch()
+    

index_768_cosine.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:864fe29de71f0e5b56ca87b04d559ea707d4cd3798429f80f04b8a58a07e3721
+size 53168791

requirements.txt

@@ -0,0 +1,7 @@
+transformers==4.25.1
+datasets==2.7.1
+numpy==1.21.6
+torch==1.12.1
+torchvision
+pynndescent
+wikipedia

similarity_utils.py

@@ -0,0 +1,175 @@
+from typing import List, Union
+
+import datasets
+import numpy as np
+import torch
+import torchvision.transforms as T
+from PIL import Image
+from tqdm.auto import tqdm
+from transformers import AutoFeatureExtractor, AutoModel
+
+seed = 42
+hash_size = 8
+hidden_dim = 768  # ViT-base
+np.random.seed(seed)
+
+
+# Device.
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# Load model for computing embeddings..
+model_ckpt = "nateraw/vit-base-beans"
+extractor = AutoFeatureExtractor.from_pretrained(model_ckpt)
+
+# Data transformation chain.
+transformation_chain = T.Compose(
+    [
+        # We first resize the input image to 256x256 and then we take center crop.
+        T.Resize(int((256 / 224) * extractor.size["height"])),
+        T.CenterCrop(extractor.size["height"]),
+        T.ToTensor(),
+        T.Normalize(mean=extractor.image_mean, std=extractor.image_std),
+    ]
+)
+
+
+# Define random vectors to project with.
+random_vectors = np.random.randn(hash_size, hidden_dim).T
+
+
+def hash_func(embedding, random_vectors=random_vectors):
+    """Randomly projects the embeddings and then computes bit-wise hashes."""
+    if not isinstance(embedding, np.ndarray):
+        embedding = np.array(embedding)
+    if len(embedding.shape) < 2:
+        embedding = np.expand_dims(embedding, 0)
+
+    # Random projection.
+    bools = np.dot(embedding, random_vectors) > 0
+    return [bool2int(bool_vec) for bool_vec in bools]
+
+
+def bool2int(x):
+    y = 0
+    for i, j in enumerate(x):
+        if j:
+            y += 1 << i
+    return y
+
+
+def compute_hash(model: Union[torch.nn.Module, str]):
+    """Computes hash on a given dataset."""
+    device = model.device
+
+    def pp(example_batch):
+        # Prepare the input images for the model.
+        image_batch = example_batch["image"]
+        image_batch_transformed = torch.stack(
+            [transformation_chain(image) for image in image_batch]
+        )
+        new_batch = {"pixel_values": image_batch_transformed.to(device)}
+
+        # Compute embeddings and pool them i.e., take the representations from the [CLS]
+        # token.
+        with torch.no_grad():
+            embeddings = model(**new_batch).last_hidden_state[:, 0].cpu().numpy()
+
+        # Compute hashes for the batch of images.
+        hashes = [hash_func(embeddings[i]) for i in range(len(embeddings))]
+        example_batch["hashes"] = hashes
+        return example_batch
+
+    return pp
+
+
+class Table:
+    def __init__(self, hash_size: int):
+        self.table = {}
+        self.hash_size = hash_size
+
+    def add(self, id: int, hashes: List[int], label: int):
+        # Create a unique indentifier.
+        entry = {"id_label": str(id) + "_" + str(label)}
+
+        # Add the hash values to the current table.
+        for h in hashes:
+            if h in self.table:
+                self.table[h].append(entry)
+            else:
+                self.table[h] = [entry]
+
+    def query(self, hashes: List[int]):
+        results = []
+
+        # Loop over the query hashes and determine if they exist in
+        # the current table.
+        for h in hashes:
+            if h in self.table:
+                results.extend(self.table[h])
+        return results
+
+
+class LSH:
+    def __init__(self, hash_size, num_tables):
+        self.num_tables = num_tables
+        self.tables = []
+        for i in range(self.num_tables):
+            self.tables.append(Table(hash_size))
+
+    def add(self, id: int, hash: List[int], label: int):
+        for table in self.tables:
+            table.add(id, hash, label)
+
+    def query(self, hashes: List[int]):
+        results = []
+        for table in self.tables:
+            results.extend(table.query(hashes))
+        return results
+
+
+class BuildLSHTable:
+    def __init__(
+        self,
+        model: Union[torch.nn.Module, None],
+        batch_size: int = 48,
+        hash_size: int = hash_size,
+        dim: int = hidden_dim,
+        num_tables: int = 10,
+    ):
+        self.hash_size = hash_size
+        self.dim = dim
+        self.num_tables = num_tables
+        self.lsh = LSH(self.hash_size, self.num_tables)
+
+        self.batch_size = batch_size
+        self.hash_fn = compute_hash(model.to(device))
+
+    def build(self, ds: datasets.DatasetDict):
+        dataset_hashed = ds.map(self.hash_fn, batched=True, batch_size=self.batch_size)
+
+        for id in tqdm(range(len(dataset_hashed))):
+            hash, label = dataset_hashed[id]["hashes"], dataset_hashed[id]["labels"]
+            self.lsh.add(id, hash, label)
+
+    def query(self, image, verbose=True):
+        if isinstance(image, str):
+            image = Image.open(image).convert("RGB")
+
+        # Compute the hashes of the query image and fetch the results.
+        example_batch = dict(image=[image])
+        hashes = self.hash_fn(example_batch)["hashes"][0]
+
+        results = self.lsh.query(hashes)
+        if verbose:
+            print("Matches:", len(results))
+
+        # Calculate Jaccard index to quantify the similarity.
+        counts = {}
+        for r in results:
+            if r["id_label"] in counts:
+                counts[r["id_label"]] += 1
+            else:
+                counts[r["id_label"]] = 1
+        for k in counts:
+            counts[k] = float(counts[k]) / self.dim
+        return counts