Duplicate from MrOvkill/gemma-2-inference-endpoint-GGUF

Co-authored-by: Samuel L Meyers <MrOvkill@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+gemma-2b.q8_0.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,31 @@
+---
+license: apache-2.0
+language:
+- en
+library_name: llama.cpp
+pipeline_tag: text-generation
+---
+
+# Gemma 2 - Inference Endpoint
+
+## <span style="color: red">NOTICE:</span> This model does, in fact run on inference endpoints. Just click deploy, unlike with regular GGUF models. The model is no longer stored, merely linked. Enjoy <span style="color: red">&lt;3</span>
+
+<label>Code Sample ( One-Shot )</label>
+```javascript
+
+{
+  "inputs": "A plain old prompt with nothing else"
+}
+```
+
+## Multi turn coming soon...
+
+Hello! I wrote a simple container that allows for easy running of llama-cpp-python with GGUF models. My goal here was a cheap way to play with Gemma, but then I thought maybe i'd share just in case it's helpful. I'll probably make a bunch of these, so if you have any requests for GGUF or otherwise quantized Llama.cpp models to become inference endpoints, please feel free to reach out!
+
+# Files
+
+I used the excellent quant by [lmstudio-ai/gemma-2b-it-GGUF](https://huggingface.co/lmstudio-ai/gemma-2b-it-GGUF), 
+
+My email is newp@justkidding.net
+
+Just kidding, it's sam att samuellmeyers DOT... com

gemma_tools.py

@@ -0,0 +1,38 @@
+
+def get_args_or_none(data):
+    system_prompt = data.pop("system-prompt", "You are Gemma. Assist user with whatever they require, in a safe and moral manner.")
+    inputs = data.pop("inputs", "")
+    temperature = data.pop("temperature", None)
+    if not temperature or temperature is None:
+        temperature = data.pop("temp", 0.33)
+    if temperature > 3 or temperature < 0:
+        return {
+            0: False,
+            "status": "error",
+            "reason": "temperature",
+            "reason": "invalid temperature ( 0.01 - 1.00 only allowed )"
+        }
+    top_p = data.pop("top-p", 0.85)
+    if top_p > 3 or top_p < 0:
+        return {
+            0: False,
+            "status": "error",
+            "reason": "top_p",
+            "description": "invalid top percentage ( 0.01 - 1.00 only allowed )"
+        }
+    top_k = data.pop("top-k", 42)
+    if top_k > 100 or top_k < 0:
+        return {
+            0: False,
+            "status": "error",
+            "reason": "top_k",
+            "description": "invalid top k ( 1 - 99 only allowed )"
+        }
+    return {
+        0: True,
+        "inputs": inputs,
+        "system_prompt": system_prompt,
+        "temperature": temperature,
+        "top_p": top_p,
+        "top_k": top_k
+    }

handler.py

@@ -0,0 +1,40 @@
+import json
+import os
+from typing import Dict, List, Any
+from llama_cpp import Llama
+import gemma_tools as gem
+
+MAX_TOKENS=8192
+
+class EndpointHandler():
+    def __init__(self, data):
+        self.model = Llama.from_pretrained("lmstudio-ai/gemma-2b-it-GGUF", filename="gemma-2b-it-q4_k_m.gguf", n_ctx=8192)
+
+    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
+        args = gem.get_args_or_none(data)
+        fmat = "<startofturn>system\n{system_prompt} <endofturn>\n<startofturn>user\n{prompt} <endofturn>\n<startofturn>model"
+        print(args, fmat)
+        if not args[0]:
+            return {
+                "status": args["status"],
+                "message": args["description"]
+            }
+        try:
+            fmat = fmat.format(system_prompt = args["system_prompt"], prompt = args["inputs"])
+        except Exception as e:
+            return json.dumps({
+                "status": "error",
+                "reason": "invalid format"
+            })
+        max_length = data.pop("max_length", 512)
+        try:
+            max_length = int(max_length)
+        except Exception as e:
+            return json.dumps({
+                "status": "error",
+                "reason": "max_length was passed as something that was absolutely not a plain old int"
+            })
+        
+        res = self.model(fmat, temperature=args["temperature"], top_p=args["top_p"], top_k=args["top_k"], max_tokens=max_length)
+
+        return res

requirements.txt

@@ -0,0 +1 @@
+llama-cpp-python