use correct llama

README.md

@@ -4,7 +4,7 @@ emoji: 📈
 colorFrom: indigo
 colorTo: green
 sdk: gradio
-sdk_version: 4.32.2
+sdk_version: 4.31.3
 app_file: app.py
 pinned: false
 ---

app.py

@@ -1,5 +1,6 @@
 from gradio_huggingfacehub_search import HuggingfaceHubSearch
 from llama_cpp.llama_speculative import LlamaPromptLookupDecoding
+from llama_cpp_cuda_tensorcores import Llama
 from huggingface_hub import hf_hub_download
 from huggingface_hub import HfApi
 import matplotlib.pyplot as plt
@@ -8,7 +9,6 @@ import pandas as pd
 import gradio as gr
 import duckdb
 import requests
-import llama_cpp
 import instructor
 import spaces
 import enum
@@ -22,11 +22,11 @@ view_name = "dataset_view"
 hf_api = HfApi()
 conn = duckdb.connect()
 
-gpu_layers = int(os.environ.get("GPU_LAYERS", 81))
+gpu_layers = int(os.environ.get("GPU_LAYERS", 0))
 draft_pred_tokens = int(os.environ.get("DRAFT_PRED_TOKENS", 2))
 
 repo_id = os.getenv("MODEL_REPO_ID", "NousResearch/Hermes-2-Pro-Llama-3-8B-GGUF")
-model_file_name = os.getenv("MODEL_FILE_NAME", "Hermes-2-Pro-Llama-3-8B-Q8_0.gguf")
+model_file_name = os.getenv("MODEL_FILE_NAME", "Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf")
 
 hf_hub_download(
     repo_id=repo_id,
@@ -88,7 +88,7 @@ CREATE TABLE {} (
 
 @spaces.GPU(duration=120)
 def generate_query(ddl: str, query: str) -> dict:
-    llama = llama_cpp.Llama(
+    llama = Llama(
         model_path=f"models/{model_file_name}",
         n_gpu_layers=gpu_layers,
         chat_format="chatml",
@@ -200,4 +200,8 @@ with gr.Blocks() as demo:
 
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch(
+        show_error=True,
+        quiet=False,
+        debug=True,
+    )

requirements.txt

@@ -40,7 +40,6 @@ Jinja2==3.1.4
 jsonschema==4.22.0
 jsonschema-specifications==2023.12.1
 kiwisolver==1.4.5
-llama_cpp_python==0.2.77
 markdown-it-py==3.0.0
 MarkupSafe==2.1.5
 matplotlib==3.9.0
@@ -92,3 +91,5 @@ uvloop==0.19.0
 watchfiles==0.22.0
 websockets==11.0.3
 yarl==1.9.4
+https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.69+cpuavx2-cp310-cp310-linux_x86_64.whl
+https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.69+cu121-cp310-cp310-linux_x86_64.whl