Update app.py

app.py

@@ -1,19 +1,20 @@
 import spaces
-import argparse
 import torch
 import re
 import gradio as gr
 from threading import Thread
 from transformers import TextIteratorStreamer, AutoTokenizer, AutoModelForCausalLM
 
-parser = argparse.ArgumentParser()
+import subprocess
+subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
 
 model_id = "vikhyatk/moondream2"
 revision = "2024-04-02"
 tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision)
 moondream = AutoModelForCausalLM.from_pretrained(
     model_id, trust_remote_code=True, revision=revision,
-    torch_dtype=torch.float32
+    torch_dtype=torch.bfloat16, device_map={"": "cuda"},
+    attn_implementation="flash_attention_2"
 )
 moondream.eval()
 
@@ -48,7 +49,7 @@ with gr.Blocks() as demo:
         """
     )
     with gr.Row():
-        prompt = gr.Textbox(label="Input", placeholder="Type here...", scale=4)
+        prompt = gr.Textbox(label="Input", value="Describe this image.", scale=4)
         submit = gr.Button("Submit")
     with gr.Row():
         img = gr.Image(type="pil", label="Upload an Image")