endpoint

api_endpoints.json

@@ -131,6 +131,18 @@
         "text-arena": true,
         "vision-arena": false
     },
+    "gpt-4-turbo-2024-04-09": {
+        "model_name": "gpt-4-turbo-2024-04-09",
+        "api_type": "openai",
+        "api_base": "https://api.openai.com/v1",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
     "gpt-4o-mini-2024-07-18": {
         "model_name": "gpt-4o-mini-2024-07-18",
         "api_type": "openai",

local/local_setup

@@ -35,6 +35,10 @@ python -m vllm.entrypoints.openai.api_server --model tokyotech-llm/Llama-3.1-Swa
 #export CUDA_VISIBLE_DEVICES=1
 #../llama-server -m gemma-2-2B-jpn-it-BF16.gguf --n_gpu_layers 100 --port 8020
 
+#tanuki 8x8b
+export CUDA_VISIBLE_DEVICES=1
+python -m vllm.entrypoints.openai.api_server --model team-hatakeyama-phase2/Tanuki-8x8B-dpo-v1.0-AWQ --max-model-len 4096 --port 8020 --gpu-memory-utilization 0.2 --trust-remote-code --quantization awq --api-key $VLLM_API_KEY
+
 #########################
 #launch ngrok
 ngrok http http://localhost:8765