update

log_util.py

@@ -0,0 +1,12 @@
+
+import logging
+
+logging.basicConfig(
+    format='[%(asctime)s] [%(levelname)s] [%(process)d:%(thread)d] [%(filename)s:%(lineno)d:%(funcName)s] %(message)s',
+    level=logging.INFO,
+    datefmt="%Y-%m-%d %H:%M:%S",
+
+)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)

models/cpp_qwen2.py

@@ -15,34 +15,38 @@ python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen1.5-0.5B-C
 
 """
 
+import json
 from simulator import Simulator
 import llama_cpp
 # import llama_cpp.llama_tokenizer
 from transformers import AutoTokenizer
+from log_util import logger
 
 
 class Qwen2Simulator(Simulator):
 
-    def __init__(self, model_name_or_path=None):
-        self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Chat")
-        self.llm = llama_cpp.Llama.from_pretrained(
-            repo_id="Qwen/Qwen2-0.5B-Instruct-GGUF",
-            filename="*fp16.gguf",
-            tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
-            verbose=False,
-        )
+    def __init__(self, from_local=False):
+        if from_local:
+            self.hf_tokenizer = AutoTokenizer.from_pretrained("/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/")
+            self.llm = llama_cpp.Llama(
+                model_path="/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct-GGUF/qwen2-0_5b-instruct-fp16.gguf",
+                tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
+                verbose=False,
+            )
+        else:
+            self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
+            self.llm = llama_cpp.Llama.from_pretrained(
+                repo_id="Qwen/Qwen2-0.5B-Instruct-GGUF",
+                filename="*fp16.gguf",
+                tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
+                verbose=False,
+            )
+            logger.info(f"llm has been initialized: {self.llm}")
+        # warmup
+
 
         ### local
-        # self.hf_tokenizer = AutoTokenizer.from_pretrained("/workspace/xusong/huggingface/models/Qwen2-0.5B-Chat/")
-        # self.llm = Llama(
-        #     model_path="/workspace/xusong/huggingface/models/Qwen2-0.5B-Chat-GGUF/qwen2-0_5b-chat-q8_0.gguf",
-        #     # model_path="/workspace/xusong/huggingface/models/Qwen2-0.5B-Chat/Qwen2-0.5B-Chat-F16.gguf",
-        #     # n_gpu_layers=-1, # Uncomment to use GPU acceleration
-        #     # seed=1337, # Uncomment to set a specific seed
-        #     # n_ctx=2048, # Uncomment to increase the context window
-        #     tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
-        #     verbose=False,
-        # )
+
 
 
     def generate_query(self, messages):
@@ -51,6 +55,7 @@ class Qwen2Simulator(Simulator):
         :return:
         """
         assert messages[-1]["role"] != "user"
+        logger.info(f"generating {json.dumps(messages)}")
         inputs = self.hf_tokenizer.apply_chat_template(
             messages,
             tokenize=False,
@@ -63,6 +68,7 @@ class Qwen2Simulator(Simulator):
 
     def generate_response(self, messages):
         assert messages[-1]["role"] == "user"
+        logger.info(f"generating {json.dumps(messages)}")
         inputs = self.hf_tokenizer.apply_chat_template(
             messages,
             tokenize=False,