update

README.md

@@ -21,4 +21,12 @@ An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https
 直接从源码安装，推理速度较慢，因此加入以下参数。
 ```sh
 pip install git+https://github.com/abetlen/llama-cpp-python.git -C cmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS"
-```
+```
+
+
+## Serverless Inference API
+
+
+client.py
+
+## Reference

app.py

@@ -123,47 +123,49 @@ with gr.Blocks(head=None) as demo:
                     label="Top-k",
                 )
 
+    # TODO: gr.State 不能通过API传参。
     history = gr.State([{"role": "system", "content": system_list[0]}])  # 有用信息只有个system，其他和chatbot内容重叠
     system.change(reset_state, inputs=[system], outputs=[chatbot, history])
 
-    ######## tab1
-    generate_btn.click(generate, [chatbot, history], outputs=[chatbot, history],
+    ######## tab1: self-chat
+    generate_btn.click(chat, [chatbot, history], outputs=[chatbot, history],
                        show_progress="full")
-    retry_btn.click(undo_generate, [chatbot, history], outputs=[chatbot, history]) \
-        .then(generate, [chatbot, history], outputs=[chatbot, history],
-              show_progress="full")
-    undo_btn.click(undo_generate, [chatbot, history], outputs=[chatbot, history])
-    clear_btn.click(reset_state, inputs=[system], outputs=[chatbot, history])
-
-    ######## tab2
-    generate_btn_2.click(append_user, [input_text_2, chatbot, history], outputs=[chatbot, history]) \
+    retry_btn.click(undo_generate, [chatbot, history], outputs=[chatbot, history], show_api=False) \
+        .then(chat, [chatbot, history], outputs=[chatbot, history],
+              show_progress="full", show_api=False)
+    undo_btn.click(undo_generate, [chatbot, history], outputs=[chatbot, history], show_api=False)
+    clear_btn.click(reset_state, inputs=[system], outputs=[chatbot, history], show_api=False)
+
+    ######## tab2:  response-generator
+    generate_btn_2.click(append_user_to_history, [input_text_2, chatbot, history], outputs=[chatbot, history],
+                         show_api=False) \
         .then(generate_assistant_message, [chatbot, history], outputs=[chatbot, history],
-              show_progress="full")
-    retry_btn_2.click(undo_generate, [chatbot, history], outputs=[chatbot, history]) \
-        .then(generate, [chatbot, history], outputs=[chatbot, history],
-              show_progress="full")
-    undo_btn_2.click(undo_generate, [chatbot, history], outputs=[chatbot, history])
-    clear_btn_2.click(reset_state, inputs=[system], outputs=[chatbot, history]) \
-        .then(reset_user_input, outputs=[input_text_2])
-
-    ######## tab3
-    generate_btn_3.click(append_assistant, [input_text_3, chatbot, history], outputs=[chatbot, history]) \
-        .then(generate_assistant_message, [chatbot, history], outputs=[chatbot, history],
-              show_progress="full")
-    retry_btn_3.click(undo_generate, [chatbot, history], outputs=[chatbot, history]) \
-        .then(generate, [chatbot, history], outputs=[chatbot, history],
-              show_progress="full")
-    undo_btn_3.click(undo_generate, [chatbot, history], outputs=[chatbot, history])
-    clear_btn_3.click(reset_state, inputs=[system], outputs=[chatbot, history]) \
-        .then(reset_user_input, outputs=[input_text_3])
+              show_progress="full", show_api=False)
+    retry_btn_2.click(undo_generate, [chatbot, history], outputs=[chatbot, history], show_api=False) \
+        .then(chat, [chatbot, history], outputs=[chatbot, history],
+              show_progress="full", show_api=False)
+    undo_btn_2.click(undo_generate, [chatbot, history], outputs=[chatbot, history], show_api=False)
+    clear_btn_2.click(reset_state, inputs=[system], outputs=[chatbot, history], show_api=False) \
+        .then(reset_user_input, outputs=[input_text_2], show_api=False)
+
+    ######## tab3: user-simulator
+    generate_btn_3.click(append_assistant_to_history, [input_text_3, chatbot, history], outputs=[chatbot, history],
+                         show_api=False) \
+        .then(generate_user_message, [chatbot, history], outputs=[chatbot, history],
+              show_progress="full", show_api=False)
+    retry_btn_3.click(undo_generate, [chatbot, history], outputs=[chatbot, history], show_api=False) \
+        .then(chat, [chatbot, history], outputs=[chatbot, history],
+              show_progress="full", show_api=False)
+    undo_btn_3.click(undo_generate, [chatbot, history], outputs=[chatbot, history], show_api=False)
+    clear_btn_3.click(reset_state, inputs=[system], outputs=[chatbot, history], show_api=False) \
+        .then(reset_user_input, outputs=[input_text_3], show_api=False)
 
     slider_max_new_tokens.change(set_max_new_tokens, inputs=[slider_max_new_tokens])
     slider_temperature.change(set_temperature, inputs=[slider_temperature])
     slider_top_p.change(set_top_p, inputs=[slider_top_p])
     slider_top_k.change(set_top_k, inputs=[slider_top_k])
 
-
-    demo.load(lambda: gr.update(value=random.choice(system_list)), None, system)
+    demo.load(lambda: gr.update(value=random.choice(system_list)), None, system, show_api=False)
 
 # demo.queue().launch(share=False, server_name="0.0.0.0", debug=True)
 # demo.queue().launch(concurrency_count=1, max_size=5)

app_util.py

@@ -1,5 +1,6 @@
 import json
 import gradio as gr
+from typing import List, Dict
 from utils.logging_util import logger
 from models.cpp_qwen2 import Qwen2Simulator as Bot
 # from models.hf_qwen2 import Qwen2Simulator as Bot
@@ -68,7 +69,13 @@ def generate_assistant_message(chatbot, history):
         yield chatbot, history
 
 
-def generate(chatbot, history):
+def chat(chatbot: List[str], history: List[Dict]):
+    """
+    self chat
+    :param chatbot:
+    :param history:
+    :return:
+    """
     request_param = json.dumps({'chatbot': chatbot, 'history': history}, ensure_ascii=False)
     logger.info(f"request_param: {request_param}")
     streamer = None
@@ -83,7 +90,14 @@ def generate(chatbot, history):
         yield out
 
 
-def append_user(input_content, chatbot, history):
+def append_user_to_history(input_content, chatbot, history):
+    """
+
+    :param input_content:
+    :param chatbot:
+    :param history:
+    :return:
+    """
     if history[-1]["role"] == "user":
         gr.Warning('You should generate assistant-response.')
         return chatbot, history
@@ -93,7 +107,7 @@ def append_user(input_content, chatbot, history):
     return chatbot, history
 
 
-def append_assistant(input_content, chatbot, history):
+def append_assistant_to_history(input_content, chatbot, history):
     if history[-1]["role"] != "user":
         gr.Warning('You should generate or type user-input first.')
         return chatbot, history

config.py

@@ -5,5 +5,5 @@ MAX_SEQUENCE_LENGTH = 8192  #
 
 DEFAULT_MAX_NEW_TOKENS = 128
 DEFAULT_TOP_K = 100
-DEFAULT_TOP_P = 0.95
-DEFAULT_TEMPERATURE = 5
+DEFAULT_TOP_P = 0.98
+DEFAULT_TEMPERATURE = 2