Update main.py

main.py

@@ -4,12 +4,12 @@ from llama_cpp import Llama
 import uvicorn
 import prompt_style
 import os
+from huggingface_hub import hf_hub_download
 
-model_id = "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3-GGUF"
-model = Llama.from_pretrained(repo_id=model_id, filename="*-v3_q6.gguf", n_gpu_layers=-1, n_ctx=4096, verbose=False)
 
-# model_id = "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3"
-# client = InferenceClient(token=os.getenv('HF_TOKEN'), model=model_id)
+model_id = "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3-GGUF"
+model_path = hf_hub_download(repo_id=model_id, filename="*-v3_q6.gguf", token=os.environment["HF_TOKEN"])
+model = Llama(model_path=model_path, n_gpu_layers=-1, n_ctx=4096, verbose=False)
 
 class Item(BaseModel):
     prompt: str
@@ -51,4 +51,5 @@ async def generate_text(item: Item):
 
 @app.get("/")
 def read_root():
-    return {"Hello": "World!"}
+    
+    return {"Hello": os.environment["HF_TOKEN"]}