app file

app.py

@@ -0,0 +1,118 @@
+import os
+from datetime import date
+import gradio as gr
+import openai
+
+MODEL_CONFIGS = {
+    "Falcon-H1-34B-Instruct": {
+        "model_id":     "tiiuae/Falcon-H1-34B-Instruct",
+        "api_key_env":  "XXL_API_KEY",
+        "base_url_env": "XXL_URL",
+    },
+    "Falcon-H1-7B-Instruct": {
+        "model_id":     "tiiuae/Falcon-H1-7B-Instruct",
+        "api_key_env":  "L_API_KEY",
+        "base_url_env": "L_URL",
+    },
+    "Falcon-H1-3B-Instruct": {
+        "model_id":     "tiiuae/Falcon-H1-3B-Instruct",
+        "api_key_env":  "M_API_KEY",
+        "base_url_env": "M_URL",
+    },
+    "Falcon-H1-1.5B-Deep-Instruct": {
+        "model_id":     "tiiuae/Falcon-H1-1.5B-Deep-Instruct",
+        "api_key_env":  "S_API_KEY",
+        "base_url_env": "S_URL",
+    },
+    "Falcon-H1-1.5B-Instruct": {
+        "model_id":     "tiiuae/Falcon-H1-1.5B-Instruct",
+        "api_key_env":  "XS_API_KEY",
+        "base_url_env": "XS_URL",
+    },
+    "Falcon-H1-0.5B-Instruct": {
+        "model_id":     "tiiuae/Falcon-H1-0.5B-Instruct",
+        "api_key_env":  "XXS_API_KEY",
+        "base_url_env": "XXS_URL",
+    },
+}
+
+today = date.today()
+TITLE = "<h1><center>Private multi-backend playground</center></h1>"
+SUB_TITLE = "<center>Keys & endpoints stay server-side; the browser never sees them.</center>"
+CSS = """.duplicate-button{margin:auto!important;color:#fff!important;background:#000!important;border-radius:100vh!important}h3{text-align:center;}"""
+
+def stream_chat(
+    message: str,
+    history: list,
+    model_label: str,
+    temperature: float = 0.7,
+    max_new_tokens: int = 1024,
+    top_p: float = 1.0,
+    top_k: int = 20,          
+    penalty: float = 1.2,
+):
+    cfg = MODEL_CONFIGS[model_label]
+
+    api_key   = os.getenv(cfg["api_key_env"])
+    base_url  = os.getenv(cfg.get("base_url_env", ""), None)
+
+    if not api_key:
+        yield f"❌ Env-var `{cfg['api_key_env']}` not set."
+        return
+    if cfg.get("base_url_env") and not base_url:
+        yield f"❌ Env-var `{cfg['base_url_env']}` not set."
+        return
+
+    client = openai.OpenAI(api_key=api_key, base_url=base_url)
+
+    for u, a in history:
+        msgs += [{"role": "user", "content": u},
+                 {"role": "assistant", "content": a}]
+    msgs.append({"role": "user", "content": message})
+
+    stream = client.chat.completions.create(
+        model=cfg["model_id"],
+        messages=msgs,
+        temperature=temperature,
+        top_p=top_p,
+        max_tokens=max_new_tokens,
+        presence_penalty=penalty,
+        stream=True,
+    )
+
+    partial = ""
+    for chunk in stream:
+        if (delta := chunk.choices[0].delta).content:
+            partial += delta.content
+            yield partial
+
+
+chatbot = gr.Chatbot(height=600)
+
+with gr.Blocks(css=CSS, theme="soft") as demo:
+    gr.HTML(TITLE)
+    gr.HTML(SUB_TITLE)
+    gr.DuplicateButton(value="Duplicate Space", elem_classes="duplicate-button")
+
+    gr.ChatInterface(
+        fn=stream_chat,
+        chatbot=chatbot,
+        fill_height=True,
+        additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False),
+        additional_inputs=[
+            gr.Dropdown(
+                choices=list(MODEL_CONFIGS.keys()),
+                value=list(MODEL_CONFIGS.keys())[0],
+                label="Model",
+            ),
+            gr.Slider(0, 1, 0.1, 0.7,  label="Temperature"),
+            gr.Slider(64, 4096, 1, 1024, label="Max new tokens"),
+            gr.Slider(0, 1, 0.05, 1.0,  label="top_p"),
+            gr.Slider(1, 20, 1, 20,     label="top_k (ignored)"),
+            gr.Slider(0, 2, 0.1, 1.2,   label="Presence penalty"),
+        ],
+        cache_examples=False,
+    )
+
+if __name__ == "__main__":
+    demo.launch()