Feature(MInference): add dtype

app.py

@@ -57,7 +57,7 @@ h1 {
 # Load the tokenizer and model
 model_name = "gradientai/Llama-3-8B-Instruct-262k"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")  # to("cuda:0") 
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", device_map="auto")  # to("cuda:0") 
 
 from minference import MInference
 minference_patch = MInference("minference", model_name)