Update: transformers 4.35, examples

.gitattributes

@@ -37,3 +37,6 @@ assets/captioning_example_2.png filter=lfs diff=lfs merge=lfs -text
 assets/vqa_example_1.png filter=lfs diff=lfs merge=lfs -text
 assets/vqa_example_2.png filter=lfs diff=lfs merge=lfs -text
 assets/docvqa_example.png filter=lfs diff=lfs merge=lfs -text
+assets/food.png filter=lfs diff=lfs merge=lfs -text
+assets/girl_hat.png filter=lfs diff=lfs merge=lfs -text
+assets/jobs.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -2,15 +2,13 @@ import gradio as gr
 import re
 import torch
 from PIL import Image
-from transformers import AutoTokenizer, FuyuForCausalLM, FuyuImageProcessor, FuyuProcessor
+from transformers import FuyuForCausalLM, FuyuProcessor
 
 model_id = "adept/fuyu-8b"
 dtype = torch.bfloat16
-device = "cuda"
 
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-model = FuyuForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=dtype)
-processor = FuyuProcessor(image_processor=FuyuImageProcessor(), tokenizer=tokenizer)
+model = FuyuForCausalLM.from_pretrained(model_id, device_map="cuda", torch_dtype=dtype)
+processor = FuyuProcessor.from_pretrained(model_id)
 
 CAPTION_PROMPT = "Generate a coco-style caption.\n"
 DETAILED_CAPTION_PROMPT = "What is happening in this image?"
@@ -38,12 +36,11 @@ def pad_to_size(image, canvas_width=1920, canvas_height=1080):
 
 def predict(image, prompt):
     # image = image.convert('RGB')
-    model_inputs = processor(text=prompt, images=[image])
-    model_inputs = {k: v.to(dtype=dtype if torch.is_floating_point(v) else v.dtype, device=device) for k,v in model_inputs.items()}
+    model_inputs = processor(text=prompt, images=[image]).to(device=model.device)
 
     generation_output = model.generate(**model_inputs, max_new_tokens=50)
     prompt_len = model_inputs["input_ids"].shape[-1]
-    return tokenizer.decode(generation_output[0][prompt_len:], skip_special_tokens=True)
+    return processor.decode(generation_output[0][prompt_len:], skip_special_tokens=True)
 
 def caption(image, detailed_captioning):
     if detailed_captioning:
@@ -55,43 +52,6 @@ def caption(image, detailed_captioning):
 def set_example_image(example: list) -> dict:
     return gr.Image.update(value=example[0])
 
-def scale_factor_to_fit(original_size, target_size=(1920, 1080)):
-    width, height = original_size
-    max_width, max_height = target_size
-    if width <= max_width and height <= max_height:
-        return 1.0
-    return min(max_width/width, max_height/height)
-    
-def tokens_to_box(tokens, original_size):
-    bbox_start = tokenizer.convert_tokens_to_ids("<0x00>")
-    bbox_end = tokenizer.convert_tokens_to_ids("<0x01>")
-    try:
-        # Assumes a single box
-        bbox_start_pos = (tokens == bbox_start).nonzero(as_tuple=True)[0].item()
-        bbox_end_pos = (tokens == bbox_end).nonzero(as_tuple=True)[0].item()
-        
-        if bbox_end_pos != bbox_start_pos + 5:
-            return tokens
-
-        # Retrieve transformed coordinates from tokens
-        coords = tokenizer.convert_ids_to_tokens(tokens[bbox_start_pos+1:bbox_end_pos])
-
-        # Scale back to original image size and multiply by 2
-        scale = scale_factor_to_fit(original_size)
-        top, left, bottom, right = [2 * int(float(c)/scale) for c in coords]
-        
-        # Replace the IDs so they get detokenized right
-        replacement = f" <box>{top}, {left}, {bottom}, {right}</box>"
-        replacement = tokenizer.tokenize(replacement)[1:]
-        replacement = tokenizer.convert_tokens_to_ids(replacement)
-        replacement = torch.tensor(replacement).to(tokens)
-
-        tokens = torch.cat([tokens[:bbox_start_pos], replacement, tokens[bbox_end_pos+1:]], 0)
-        return tokens
-    except:
-        gr.Error("Can't convert tokens.")
-        return tokens
-
 def coords_from_response(response):
     # y1, x1, y2, x2
     pattern = r"<box>(\d+),\s*(\d+),\s*(\d+),\s*(\d+)</box>"
@@ -111,14 +71,12 @@ def localize(image, query):
     padded = resize_to_max(image)
     padded = pad_to_size(padded)
 
-    model_inputs = processor(text=prompt, images=[padded])
-    model_inputs = {k: v.to(dtype=dtype if torch.is_floating_point(v) else v.dtype, device=device) for k,v in model_inputs.items()}
+    model_inputs = processor(text=prompt, images=[padded]).to(device=model.device)
     
-    generation_output = model.generate(**model_inputs, max_new_tokens=40)
-    prompt_len = model_inputs["input_ids"].shape[-1]
-    tokens = generation_output[0][prompt_len:]
-    tokens = tokens_to_box(tokens, image.size)
-    decoded = tokenizer.decode(tokens, skip_special_tokens=True)
+    outputs = model.generate(**model_inputs, max_new_tokens=40)
+    post_processed_bbox_tokens = processor.post_process_box_coordinates(outputs)[0]
+    decoded = processor.decode(post_processed_bbox_tokens, skip_special_tokens=True)
+    decoded = decoded.split('\x04', 1)[1] if '\x04' in decoded else ''
     coords = coords_from_response(decoded)
     return image, [(coords, f"Location of \"{query}\"")]
 
@@ -145,6 +103,13 @@ with gr.Blocks(css=css) as demo:
         """
     )
     with gr.Tab("Visual Question Answering"):
+        gr.Markdown(
+            """
+            You can use natural-language questions to ask about the image. However, since this is a base model not fine-tuned for \
+            chat instructions, you may get better results by following a prompt format similar to the one used during training. See the \
+            examples below for details!
+            """
+        )
         with gr.Row():
             with gr.Column():
                 image_input = gr.Image(label="Upload your Image", type="pil")
@@ -154,8 +119,14 @@ with gr.Blocks(css=css) as demo:
         vqa_btn = gr.Button("Answer Visual Question")
         
         gr.Examples(
-            [["assets/vqa_example_1.png", "How is this made?"], ["assets/vqa_example_2.png", "What is this flower and where is it's origin?"],
-            ["assets/docvqa_example.png", "How many items are sold?"], ["assets/screen2words_ui_example.png", "What is this app about?"]],
+            [
+                ["assets/vqa_example_1.png", "What's the name of this dessert, and how is it made?\n"],
+                ["assets/vqa_example_2.png", "What is this flower and where is it's origin?"],
+                ["assets/food.png", "Answer the following VQAv2 question based on the image.\nWhat type of foods are in the image?"],
+                ["assets/jobs.png", "Answer the following DocVQA question based on the image.\nWhich is the metro in California that has a good job Outlook?"],
+                ["assets/docvqa_example.png", "How many items are sold?"],
+                ["assets/screen2words_ui_example.png", "What is this app about?"],
+            ],
             inputs = [image_input, text_input],
             outputs = [vqa_output],
             fn=predict,
@@ -173,7 +144,7 @@ with gr.Blocks(css=css) as demo:
         captioning_btn = gr.Button("Generate Caption")
 
         gr.Examples(
-            [["assets/captioning_example_1.png", False], ["assets/captioning_example_2.png", True]],
+            [["assets/captioning_example_1.png", False], ["assets/girl_hat.png", True]],
             inputs = [captioning_input, detailed_captioning_checkbox],
             outputs = [captioning_output],
             fn=caption,

requirements.txt

@@ -1,3 +1,3 @@
-git+https://github.com/huggingface/transformers.git
+transformers>=4.35.0
 accelerate
 torch==2.0.1