{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/jeff/miniconda3/envs/py10/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n", "Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n" ] } ], "source": [ "from transformers import AutoProcessor, AutoModel\n", "import torch\n", "import os\n", "\n", "os.environ['TORCH_USE_CUDA_DSA']=\"1\"\n", "os.environ['CUDA_LAUNCH_BLOCKING']=\"1\"\n", "os.environ['TORCH_DISABLE_SDPA'] = '1'\n", "\n", "model_id = \"/home/jeff/jeff/codes/llm/InCar/gemma-3-4b-it-omni\"\n", "revision = \"main\"\n", "\n", "# model = AutoModel.from_pretrained(\n", "# model_id, device_map=\"cuda\", \n", "# revision = revision, trust_remote_code=True,\n", "# ).eval()\n", "\n", "processor = AutoProcessor.from_pretrained(\n", " model_id, revision = revision, trust_remote_code=True\n", ")\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/jeff/miniconda3/envs/py10/lib/python3.10/site-packages/torchaudio/_backend/utils.py:213: UserWarning: In 2.9, this function's implementation will be changed to use torchaudio.load_with_torchcodec` under the hood. Some parameters like ``normalize``, ``format``, ``buffer_size``, and ``backend`` will be ignored. We recommend that you port your code to rely directly on TorchCodec's decoder instead: https://docs.pytorch.org/torchcodec/stable/generated/torchcodec.decoders.AudioDecoder.html#torchcodec.decoders.AudioDecoder.\n", " warnings.warn(\n", "/home/jeff/miniconda3/envs/py10/lib/python3.10/site-packages/torchaudio/_backend/ffmpeg.py:88: UserWarning: torio.io._streaming_media_decoder.StreamingMediaDecoder has been deprecated. This deprecation is part of a large refactoring effort to transition TorchAudio into a maintenance phase. The decoding and encoding capabilities of PyTorch for both audio and video are being consolidated into TorchCodec. Please see https://github.com/pytorch/audio/issues/3902 for more information. It will be removed from the 2.9 release. \n", " s = torchaudio.io.StreamReader(src, format, None, buffer_size)\n" ] } ], "source": [ "from ASRDataset import *\n", "datasets = MultiturnAudioDataset(processor=processor,json_path='/home/jeff/jeff/codes/llm/InCar/data/test_data/nav_0730_noisy.json')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'我在找台中太平逸境'" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from rapidfuzz import process, fuzz\n", "from pypinyin import pinyin, Style\n", "\n", "def correct_sentence_with_pinyin(user_input_sentence, location_dict, score_cutoff=50):\n", " pinyin_dict = {}\n", " for location in location_dict:\n", " pinyin_name = ''.join([item[0] for item in pinyin(location, style=Style.NORMAL)])\n", " pinyin_dict[pinyin_name] = location\n", "\n", " user_pinyin_sentence = ''.join([item[0] for item in pinyin(user_input_sentence, style=Style.NORMAL)])\n", "\n", " best_match_pinyin = process.extractOne(\n", " query=user_pinyin_sentence,\n", " choices=list(pinyin_dict.keys()), # 傳入拼音作為搜尋目標\n", " scorer=fuzz.token_set_ratio,\n", " score_cutoff=score_cutoff\n", " )\n", "\n", " if best_match_pinyin:\n", " best_pinyin_name = best_match_pinyin[0]\n", " corrected_location_name = pinyin_dict[best_pinyin_name]\n", "\n", " best_user_substring = None\n", " max_substring_score = 0\n", " \n", " for i in range(len(user_input_sentence)):\n", " for j in range(i + 2, min(i + 16, len(user_input_sentence) + 1)):\n", " substring = user_input_sentence[i:j]\n", " \n", " score = fuzz.ratio(substring, corrected_location_name)\n", " \n", " if score > max_substring_score:\n", " max_substring_score = score\n", " best_user_substring = substring\n", " \n", " if best_user_substring and max_substring_score > score_cutoff:\n", " return user_input_sentence.replace(best_user_substring, corrected_location_name, 1)\n", " else:\n", " return user_input_sentence\n", " return user_input_sentence\n", "large_location_dict = [\n", " \"台北太平逸\",\n", " \"台中太平逸境\",\n", " \"台北信義區\",\n", " \"高雄駁二藝術特區\",\n", " \"台南安平古堡\",\n", " \"台中逢甲夜市\",\n", " \"台北101\",\n", " \"淡水老街\"\n", " ]\n", "\n", "user_input_1 = \"我在找台東太平逸境\"\n", "correct_sentence_with_pinyin(user_input_1, large_location_dict, score_cutoff=50)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import json\n", "data = json.load(open('/home/jeff/jeff/codes/llm/InCar/data/test_data/nav_0730_noisy.json'))" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import copy\n", "class InferenceClass:\n", " def __init__(self,model_id):\n", " self.model = AutoModel.from_pretrained(\n", " model_id, device_map=\"cuda\", \n", " torch_dtype=torch.bfloat16,\n", " trust_remote_code=True,\n", " attn_implementation=\"eager\"\n", " ).eval()\n", "\n", " self.processor = AutoProcessor.from_pretrained(\n", " model_id, trust_remote_code=True\n", " )\n", " self.remove_words_signs = lambda x:x.replace('User transcribe is :','').replace('GPT output is :','').replace('\\n','').\\\n", " replace(' ','').replace('?','').replace('？','').replace('!','').replace('。','').\\\n", " replace('！','')\n", " def call_gpt(self,inputs_tensor):\n", " with torch.inference_mode():\n", " inputs = {k:inputs_tensor[k].to('cuda') for k in inputs_tensor}\n", " generate_ids = self.model.generate(**inputs, max_new_tokens=128, do_sample=False)\n", " generate_ids = generate_ids[:, inputs['input_ids'].shape[1] :]\n", " model_output = self.processor.batch_decode(\n", " generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False\n", " )[0]\n", " return model_output\n", " def call_function_fake(self,messages=[],obs=\"\"):\n", " messages.append({'from': 'observation', 'value': obs})\n", " return messages\n", " def generate(self,chat_history,tools=\"\",audio_path=None):\n", " '''\n", " input:\n", " audio_path : str\n", " chat_history : dict\n", " return:\n", " model_output : dict\n", " '''\n", " chat_history = copy.deepcopy(chat_history)\n", " if type(audio_path)!=type(None):\n", " chat_history.append({'from': 'human',\n", " 'value': [{'type': 'audio',\n", " 'audio': audio_path}]})\n", " words_from_poi = []\n", " for hist in chat_history:\n", " if hist['from']=='observation' and '地點查詢成功' in hist['value'] and 'poi' in hist['value']:\n", " tmp = json.loads(hist['value'])\n", " for i,poi in enumerate(tmp['poi']):\n", " words_from_poi.append(poi['name'])\n", " for hist in chat_history:\n", " if hist['from']=='human' and type(hist['value'])==str:\n", " hist['value'] = correct_sentence_with_pinyin(hist['value'],words_from_poi)\n", " elif hist['from']=='function_call' and \"arguments\" in hist['value'] and 'keyword' in hist['value'][\"arguments\"]:\n", " hist['value'][\"arguments\"] = eval(hist['value'][\"arguments\"])\n", " if 'keyword' in hist['value'][\"arguments\"]:\n", " hist['value'][\"arguments\"]['keyword'] = correct_sentence_with_pinyin(hist['value'][\"arguments\"]['keyword'],words_from_poi)\n", " hist['value'][\"arguments\"] = str(hist['value'][\"arguments\"])\n", " # model_input_history = copy.deepcopy(chat_history)\n", " # num2ch = {1:'一',2:'二',3:'三',4:'四',5:'五',6:'六'}\n", " # for hist in model_input_history:\n", " # if hist['from']=='observation' and '地點查詢成功' in hist['value'] and 'poi' in hist['value']:\n", " # tmp = json.loads(hist['value'])\n", " # new_poi = []\n", " # for i,poi in enumerate(tmp['poi']):\n", " # new_poi.append('第{}個 : '.format(num2ch[i+1])+str(poi))\n", " # tmp['poi'] = new_poi\n", " # hist['value'] = json.dumps(tmp, ensure_ascii=False)\n", "\n", " inputs_text = self.processor.apply_chat_template(\n", " chat_history, add_generation_prompt=True, tokenize=False,\n", " return_dict=True, return_tensors=\"pt\", tools=json.loads(tools)\n", " )\n", " inputs_tensor = self.processor(text=inputs_text, \n", " audio=[torchaudio.load(audio_path)[0]] if type(audio_path)!=type(None) else None, \n", " add_special_tokens=False, \n", " return_tensors='pt'\n", " )\n", " model_output = self.call_gpt(inputs_tensor)\n", " if chat_history[-1]['from']=='observation':\n", " chat_history.append({'from': 'gpt', 'value': correct_sentence_with_pinyin(model_output,words_from_poi)})\n", " return chat_history\n", " if ((not ';\\n' in model_output) or (not 'User transcribe is :' in model_output) or (not 'GPT output is :' in model_output)\\\n", " or len(model_output.split(';\\n'))<2 ):\n", " if chat_history[-1]['value']!=\"抱歉我聽不清楚 能麻煩您再說一次嗎\":\n", " chat_history.append({'from': 'human',\n", " 'value': 'HUMAN_VOICE_IS_NOT_RECOGNIZED'}),\n", " chat_history.append({'from': 'gpt', 'value': '抱歉我聽不清楚 能麻煩您再說一次嗎'})\n", " return chat_history\n", " output_t,output_o = model_output.split(';\\n')[:2]\n", " output_t,output_o = self.remove_words_signs(output_t),self.remove_words_signs(output_o)\n", " chat_history[-1]['value'] = correct_sentence_with_pinyin(output_t,words_from_poi)\n", " if 'Action:' in output_o and 'ActionInput:' in output_o: # function calling\n", " function_name,function_arg = output_o.split('ActionInput:')\n", " function_name = function_name.replace('Action:','')\n", " if \"keyword\" in function_arg:\n", " function_arg = json.loads(function_arg)\n", " if \"keyword\" in function_arg:\n", " function_arg[\"keyword\"] = correct_sentence_with_pinyin(function_arg[\"keyword\"],words_from_poi)\n", " chat_history.append({'from': 'function_call', 'value': {\"name\": function_name, \"arguments\": str(function_arg)}})\n", " else: # gpt response\n", " chat_history.append({'from': 'gpt', 'value': correct_sentence_with_pinyin(output_o,words_from_poi)})\n", " return chat_history\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/jeff/.cache/huggingface/modules/transformers_modules/gemma-3-4b-it-omni/speech_conformer_encoder.py:2775: FutureWarning: Please specify CheckpointImpl.NO_REENTRANT as CheckpointImpl.REENTRANT will soon be removed as the default and eventually deprecated.\n", " lambda i: encoder_checkpoint_wrapper(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "######################## speech lora #############\n", "######################## text lora #############\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Loading checkpoint shards: 100%|██████████| 3/3 [00:03<00:00, 1.22s/it]\n" ] } ], "source": [ "pipeline = InferenceClass(model_id)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/jeff/miniconda3/envs/py10/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:636: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n", " warnings.warn(\n", "/home/jeff/miniconda3/envs/py10/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:653: UserWarning: `do_sample` is set to `False`. However, `top_k` is set to `64` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_k`.\n", " warnings.warn(\n", "/home/jeff/miniconda3/envs/py10/lib/python3.10/site-packages/torch/utils/checkpoint.py:85: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '直接將最後排左邊溫度調弱到31度'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_TEMPERATURE_SET\",\"areaId\":\"SEAT_ROW_3_LEFT\",\"operation\":\"set\",\"value\":\"31\"}]}'}}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '直接將最後排左邊溫度調弱到31度'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_TEMPERATURE_SET\",\"areaId\":\"SEAT_ROW_3_LEFT\",\"operation\":\"set\",\"value\":\"31\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第三排左座空調已設定為31度'}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '直接將最後排左邊溫度調弱到31度'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_TEMPERATURE_SET\",\"areaId\":\"SEAT_ROW_3_LEFT\",\"operation\":\"set\",\"value\":\"31\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第三排左座空調已設定為31度'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '直接將最後排左邊溫度調弱到31度'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_TEMPERATURE_SET\",\"areaId\":\"SEAT_ROW_3_LEFT\",\"operation\":\"set\",\"value\":\"31\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第三排左座空調已設定為31度'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '請開啟電滑門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_SLIDING_DOOR_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"true\"}]}'}}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '直接將最後排左邊溫度調弱到31度'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_TEMPERATURE_SET\",\"areaId\":\"SEAT_ROW_3_LEFT\",\"operation\":\"set\",\"value\":\"31\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第三排左座空調已設定為31度'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '請開啟電滑門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_SLIDING_DOOR_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"true\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電滑門已開啟'}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '直接將最後排左邊溫度調弱到31度'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_TEMPERATURE_SET\",\"areaId\":\"SEAT_ROW_3_LEFT\",\"operation\":\"set\",\"value\":\"31\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第三排左座空調已設定為31度'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '請開啟電滑門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_SLIDING_DOOR_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"true\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電滑門已開啟'}, {'from': 'human', 'value': '中間這排左邊關空調'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_AC_ON\",\"areaId\":\"SEAT_ROW_2_LEFT\",\"operation\":\"set\",\"value\":\"false\"}]}'}}]\n", "[{'from': 'human', 'value': '馬上關充電口'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"EV_CHARGE_PORT_OPEN\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '充電口已關閉'}, {'from': 'human', 'value': '立刻關掉電尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': '請第2排中間靠背向後'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SEAT_BACKREST_ANGLE_1_POS\",\"areaId\":\"SEAT_ROW_2_CENTER\",\"operation\":\"decrease\",\"value\":\"5\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排中座椅背已向後調整'}, {'from': 'human', 'value': '幫我關閉電動尾門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_POWER_TAILGATE_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電動尾門已關閉'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '直接將最後排左邊溫度調弱到31度'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_TEMPERATURE_SET\",\"areaId\":\"SEAT_ROW_3_LEFT\",\"operation\":\"set\",\"value\":\"31\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第三排左座空調已設定為31度'}, {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'}, {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'}, {'from': 'human', 'value': '請開啟電滑門'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"SRDC_SLIDING_DOOR_ON\",\"areaId\":\"GLOBAL\",\"operation\":\"set\",\"value\":\"true\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '電滑門已開啟'}, {'from': 'human', 'value': '中間這排左邊關空調'}, {'from': 'function_call', 'value': {'name': 'control_car_properties', 'arguments': '{\"properties\":[{\"propertyId\":\"HVAC_AC_ON\",\"areaId\":\"SEAT_ROW_2_LEFT\",\"operation\":\"set\",\"value\":\"false\"}]}'}}, {'from': 'observation', 'value': '{\"name\": \"control_car_properties\", \"status\": \"success\", \"message\": \"控制指令執行完成\"}'}, {'from': 'gpt', 'value': '第二排左座已關閉空調'}]\n" ] } ], "source": [ "import json\n", "data = json.load(open('/home/jeff/jeff/codes/llm/InCar/data/test_data/ctrl_0730_noisy.json'))\n", "for all_conv in data:\n", " history = []\n", " tools = all_conv['tools']\n", " for idx,conv in enumerate(all_conv['conversations']):\n", " if conv['from']=='function_call' or conv['from']=='gpt':continue\n", " elif conv['from']=='human':\n", " history = pipeline.generate(history,tools=tools,audio_path=conv['audio_path'])\n", " elif conv['from']=='observation':\n", " history = pipeline.call_function_fake(history,conv['value'])\n", " history = pipeline.generate(history,tools=tools)\n", " print(history)\n", " break" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'我在找台中太平逸境'" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'keyword': '台東太平逸境'}" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "eval(history[1]['value'][\"arguments\"])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'{\"name\": \"search_and_show_place\", \"arguments\": {\"keyword\": \"台中太平逸境\"}}'" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_conv['conversations'][1]['value']" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'from': 'human', 'value': '我在找台東太平逸境'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"台東太平逸境\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"台中太平逸境\", \"address\": \"台中市太平區新仁路一段88號\", \"latitude\": 24.109912, \"longitude\": 120.737201, \"fromMemory\": false}]}'},\n", " {'from': 'gpt', 'value': '我找到「台中太平逸境」，請問是要到這裡嗎？'},\n", " {'from': 'human', 'value': '去第一個'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'nav_start',\n", " 'arguments': '{\"keyword\":\"台東太平逸境\",\"name\":\"台中太平逸境\",\"address\":\"台中市太平區新仁路一段88號\",\"latitude\":24.109912,\"longitude\":120.737201,\"fromMemory\":false}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"nav_start\", \"status\": \"success\", \"message\": \"導航已啟動\"}'},\n", " {'from': 'gpt', 'value': '導航已設定，往「台中太平逸境」出發。'},\n", " {'from': 'human', 'value': '請帶我去澳洲大堡道科韓的書店'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"澳洲大堡道科韓的書店\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"咔啦咔啦雞\", \"address\": \"南投市中山路120號\", \"latitude\": 23.9133, \"longitude\": 120.685, \"fromMemory\": true}, {\"name\": \"炸物車站\", \"address\": \"新竹市香山區中華路六段88號\", \"latitude\": 24.776, \"longitude\": 120.9331, \"fromMemory\": true}]}'},\n", " {'from': 'gpt', 'value': '上次您是去「咔啦咔啦雞」，這次還要去這裡嗎？'},\n", " {'from': 'human', 'value': '這好像不像吧'},\n", " {'from': 'gpt',\n", " 'value': '請告訴我目的地附近的標誌性景點或描述「澳洲大堡道科韓的書店」的具體特徵，這樣我可以更精準地幫你搜尋並導航到正確的地址'},\n", " {'from': 'human', 'value': '這裡好像不對'},\n", " {'from': 'gpt',\n", " 'value': '請告訴我目的地附近的標誌性景點或描述「澳洲大堡道科韓的書店」的具體特徵，這樣我可以更精準地幫你搜尋並導航到正確的地址'},\n", " {'from': 'human', 'value': '我有印象他們有大片玻璃門面'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"澳洲大堡道科韓的書店大片玻璃門面\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"雞丁炸舖\", \"address\": \"台東市中華路一段210號\", \"latitude\": 22.7575, \"longitude\": 121.1457, \"fromMemory\": false}, {\"name\": \"即刻炸\", \"address\": \"桃園市桃園區中正路400號\", \"latitude\": 24.9933, \"longitude\": 121.3121, \"fromMemory\": false}, {\"name\": \"BOOM炸雞\", \"address\": \"台南市永康區中華路88號\", \"latitude\": 23.0349, \"longitude\": 120.25, \"fromMemory\": false}, {\"name\": \"雞咔咔速食站\", \"address\": \"台中市南區建成路160號\", \"latitude\": 24.1217, \"longitude\": 120.6852, \"fromMemory\": false}, {\"name\": \"炸雞一下\", \"address\": \"彰化市民生路188號\", \"latitude\": 24.0801, \"longitude\": 120.5429, \"fromMemory\": false}]}'},\n", " {'from': 'gpt', 'value': '我找到以下與「澳洲大堡道科韓的書店」相關的地點，請問您要導航到哪一個地點？'},\n", " {'from': 'human', 'value': '還是算了，去比較好'},\n", " {'from': 'gpt', 'value': '已為您取消導航清單'},\n", " {'from': 'human', 'value': '還是算了，不去比較好'},\n", " {'from': 'gpt', 'value': '已為您取消導航清單'}]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "history" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a = []\n", "import json\n", "json.loads(str(a))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'},\n", " {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'},\n", " {'from': 'human', 'value': '請帶我去義式鄉村風的海灘式漢堡'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"義式鄉村風的海灘式漢堡\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"Benny’s漢堡坊\", \"address\": \"新北市三重區重新路四段201號\", \"latitude\": 25.0601, \"longitude\": 121.4876, \"fromMemory\": true}, {\"name\": \"Chubby’s Burger\", \"address\": \"台中市西屯區黎明路三段122號\", \"latitude\": 24.1812, \"longitude\": 120.6422, \"fromMemory\": true}]}'},\n", " {'from': 'gpt', 'value': '上次您是去「Benny’s漢堡坊」，這次還要去這裡嗎？'},\n", " {'from': 'human', 'value': '這個看起來不符合'},\n", " {'from': 'gpt',\n", " 'value': '請告訴我目的地附近的標誌性景點或描述「義式鄉村風的海灘式漢堡」的具體特徵，這樣我可以更精準地幫你搜尋並導航到正確的地址'},\n", " {'from': 'human', 'value': '店名叫大和水셰店'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"義式鄉村風的海灘式漢堡大和水셰店\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"Classic Burger屋\", \"address\": \"高雄市鼓山區裕誠路350號\", \"latitude\": 22.6503, \"longitude\": 120.2911, \"fromMemory\": false}, {\"name\": \"Burger Kingdom\", \"address\": \"花蓮市中正路310號\", \"latitude\": 23.9782, \"longitude\": 121.6023, \"fromMemory\": false}, {\"name\": \"大口咬咬漢堡店\", \"address\": \"台北市大安區安和路二段95號\", \"latitude\": 25.03, \"longitude\": 121.5509, \"fromMemory\": false}]}'},\n", " {'from': 'gpt', 'value': '我找到以下與「義式鄉村風的海灘式漢堡」相關的地點，請問您要導航到哪一個地點？'},\n", " {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'},\n", " {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'},\n", " {'from': 'human', 'value': '全錯'},\n", " {'from': 'gpt',\n", " 'value': '請告訴我目的地附近的標誌性景點或描述「義式鄉村風的海灘式漢堡」的具體特徵，這樣我可以更精準地幫你搜尋並導航到正確的地址'},\n", " {'from': 'human', 'value': '地點選在松山文創園區旁邊，特色為日式壽司'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"義式鄉村風的海灘式漢堡松山文創園區旁邊日式壽司\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"Station 13漢堡屋\", \"address\": \"新北市土城區中央路三段260號\", \"latitude\": 24.979, \"longitude\": 121.4456, \"fromMemory\": false}, {\"name\": \"牛仔漢堡屋\", \"address\": \"台南市永康區中正南路150號\", \"latitude\": 23.0271, \"longitude\": 120.2485, \"fromMemory\": false}, {\"name\": \"Happy Cow美式餐館\", \"address\": \"高雄市左營區自由三路88號\", \"latitude\": 22.6845, \"longitude\": 120.3077, \"fromMemory\": false}, {\"name\": \"Chef John’s Burger\", \"address\": \"台中市太平區太平路250號\", \"latitude\": 24.1275, \"longitude\": 120.728, \"fromMemory\": false}, {\"name\": \"Rolling Burger\", \"address\": \"嘉義市東區忠孝路88號\", \"latitude\": 23.4866, \"longitude\": 120.4511, \"fromMemory\": false}, {\"name\": \"Wagyu Burger\", \"address\": \"新竹市東區金山街101號\", \"latitude\": 24.8001, \"longitude\": 121.0102, \"fromMemory\": false}]}'},\n", " {'from': 'gpt', 'value': '我找到以下與「義式鄉村風的海灘式漢堡」相關的地點，請問您要導航到哪一個地點？'},\n", " {'from': 'human', 'value': '沒有選中'},\n", " {'from': 'gpt',\n", " 'value': '請告訴我目的地附近的標誌性景點或描述「義式鄉村風的海灘式漢堡」的具體特徵，這樣我可以更精準地幫你搜尋並導航到正確的地址'},\n", " {'from': 'human', 'value': '地點應該是板橋大遠百對面,那家好像叫什麼懸的'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"義式鄉村風的海灘式漢堡板橋大遠百對面懸\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"Classic Burger屋\", \"address\": \"高雄市鼓山區裕誠路350號\", \"latitude\": 22.6503, \"longitude\": 120.2911, \"fromMemory\": false}, {\"name\": \"大漢堡俱樂部\", \"address\": \"台中市北區雙十路二段170號\", \"latitude\": 24.1578, \"longitude\": 120.6842, \"fromMemory\": false}, {\"name\": \"Burger Bank\", \"address\": \"新竹市北區湳雅街55號\", \"latitude\": 24.8154, \"longitude\": 120.9766, \"fromMemory\": false}, {\"name\": \"熊堡堡\", \"address\": \"高雄市楠梓區德賢路89號\", \"latitude\": 22.7261, \"longitude\": 120.3319, \"fromMemory\": false}]}'},\n", " {'from': 'gpt', 'value': '我找到以下與「義式鄉村風的海灘式漢堡」相關的地點，請問您要導航到哪一個地點？'},\n", " {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'},\n", " {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'},\n", " {'from': 'human', 'value': '沒看到對的'},\n", " {'from': 'gpt',\n", " 'value': '請告訴我目的地附近的標誌性景點或描述「義式鄉村風的海灘式漢堡」的具體特徵，這樣我可以更精準地幫你搜尋並導航到正確的地址'},\n", " {'from': 'human', 'value': '位在公館商圈，有西班牙燉飯'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"義式鄉村風的海灘式漢堡公館商圈西班牙燉飯\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"Rolling Burger\", \"address\": \"嘉義市東區忠孝路88號\", \"latitude\": 23.4866, \"longitude\": 120.4511, \"fromMemory\": false}, {\"name\": \"Barrel & Beef\", \"address\": \"宜蘭市女中路三段122號\", \"latitude\": 24.7576, \"longitude\": 121.7523, \"fromMemory\": false}]}'},\n", " {'from': 'gpt', 'value': '我找到以下與「義式鄉村風的海灘式漢堡」相關的地點，請問您要導航到哪一個地點？'},\n", " {'from': 'human', 'value': '我找不到我要的選項'},\n", " {'from': 'gpt',\n", " 'value': '請告訴我目的地附近的標誌性景點或描述「義式鄉村風的海灘式漢堡」的具體特徵，這樣我可以更精準地幫你搜尋並導航到正確的地址'},\n", " {'from': 'human', 'value': '印象中是什麼B開頭的店名'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'search_and_show_place',\n", " 'arguments': '{\"keyword\":\"義式鄉村風的海灘式漢堡B\"}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"search_and_show_place\", \"status\": \"success\", \"message\": \"地點查詢成功\", \"poi\": [{\"name\": \"Station 13漢堡屋\", \"address\": \"新北市土城區中央路三段260號\", \"latitude\": 24.979, \"longitude\": 121.4456, \"fromMemory\": false}, {\"name\": \"Burger Bank\", \"address\": \"新竹市北區湳雅街55號\", \"latitude\": 24.8154, \"longitude\": 120.9766, \"fromMemory\": false}, {\"name\": \"Big Jack漢堡屋\", \"address\": \"台北市中山區民生東路三段56號\", \"latitude\": 25.0563, \"longitude\": 121.5387, \"fromMemory\": false}, {\"name\": \"Happy Cow美式餐館\", \"address\": \"高雄市左營區自由三路88號\", \"latitude\": 22.6845, \"longitude\": 120.3077, \"fromMemory\": false}, {\"name\": \"美式胖堡\", \"address\": \"台南市中西區西門路二段210號\", \"latitude\": 22.9987, \"longitude\": 120.1976, \"fromMemory\": false}, {\"name\": \"大口咬咬漢堡店\", \"address\": \"台北市大安區安和路二段95號\", \"latitude\": 25.03, \"longitude\": 121.5509, \"fromMemory\": false}]}'},\n", " {'from': 'gpt', 'value': '我找到以下與「義式鄉村風的海灘式漢堡」相關的地點，請問您要導航到哪一個地點？'},\n", " {'from': 'human', 'value': '我記得去了'},\n", " {'from': 'gpt', 'value': '已為您取消導航清單'},\n", " {'from': 'human', 'value': '直奔第五個'},\n", " {'from': 'function_call',\n", " 'value': {'name': 'nav_start',\n", " 'arguments': '{\"keyword\":\"義式鄉村風的海灘式漢堡\",\"name\":\"美式胖堡\",\"address\":\"台南市中西區西門路二段210號\",\"latitude\":22.9987,\"longitude\":120.1976,\"fromMemory\":false}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"nav_start\", \"status\": \"success\", \"message\": \"導航已啟動\"}'},\n", " {'from': 'gpt', 'value': '導航已設定，往「美式胖堡」出發，祝您用餐愉快！'},\n", " {'from': 'human', 'value': 'HUMAN_VOICE_CANNOT_RECOGNIZE'},\n", " {'from': 'gpt', 'value': '抱歉我聽不清楚能麻煩您再說一次嗎'},\n", " {'from': 'human', 'value': '請終止目前的路線安排'},\n", " {'from': 'function_call', 'value': {'name': 'nav_stop', 'arguments': '{}'}},\n", " {'from': 'observation',\n", " 'value': '{\"name\": \"nav_stop\", \"status\": \"success\", \"message\": \"導航已取消\"}'},\n", " {'from': 'gpt', 'value': '已為您取消導航，有需要再出發的話可以隨時告訴我喔！'}]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "history" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "data[1]['conversations']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(pipeline.processor.apply_chat_template(\n", " history, add_generation_prompt=True, tokenize=False,\n", " return_dict=True, return_tensors=\"pt\", tools=json.loads(tools)\n", " ))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "history = [{'from': 'function_call', 'value': {\"name\": json.loads(tmp['value'])[\"name\"], \"arguments\": str(json.loads(tmp['value'])[\"arguments\"])}} \n", " if tmp['from']=='function_call' else tmp for tmp in data[0]['conversations']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pipeline.generate(history[:4],tools=data[0]['tools'],audio_path='/home/jeff/jeff/codes/llm/InCar/data/test_data/audio_noisy/ctrl_toolcall_train 2-00015.wav')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import json\n", "data = json.load(open('/home/jeff/jeff/codes/llm/InCar/data/test_data/nav_0730_noisy.json'))\n", "data[0]['conversations'][-2] = {'from':'human',\n", " 'value':[{\"type\": \"audio\", \"audio\": '/home/jeff/jeff/codes/llm/InCar/data/test_data/audio_noisy/nav_toolcall_train_0730-00005.wav'}]\n", "}\n", "{\n", " \"role\": \"user\",\n", " \"content\": [\n", " # ans is what_is_shown_in_this_image\n", " {\"type\": \"audio\", \"audio\": '/home/jeff/jeff/codes/llm/InCar/data/test_data/audio_noisy/nav_toolcall_train_0730-00003_purenoisy.wav'},\n", " {\"type\": \"text\", \"text\": \"Transcribe this audio clip into text.\"}\n", " ]\n", " }\n", "for conv in data[0]['conversations']:\n", " if conv['from']=='function_call':\n", " conv['value']=json.loads(conv['value'])\n", " conv['value']['arguments'] = str(conv['value']['arguments'])\n", "print(processor.apply_chat_template(data[0]['conversations'], add_generation_prompt=True, tokenize=False,\n", " return_dict=True,tools=json.loads(data[0]['tools'])\n", " ))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "audio_path = '/home/jeff/jeff/codes/llm/InCar/data/test_data/audio_noisy/nav_toolcall_train_0730-00005.wav'\n", "messages = [\n", " {\n", " \"from\": \"human\",\n", " \"value\": [\n", " # ans is what_is_shown_in_this_image\n", " {\"type\": \"audio\", \"audio\": audio_path},\n", " {\"type\": \"text\", \"text\": \"Transcribe this audio clip into text.\"}\n", " ]\n", " }\n", "]\n", "\n", "inputs_text = processor.apply_chat_template(\n", " data[0]['conversations'][:-1], add_generation_prompt=True, tokenize=False,\n", " return_dict=True, return_tensors=\"pt\", tools={}\n", ")\n", "inputs = processor(text=inputs_text, \n", " audio=[torchaudio.load(audio_path)[0]], \n", " add_special_tokens=False, \n", " return_tensors='pt'\n", " )\n", "\n", "with torch.inference_mode():\n", " inputs = {k:inputs[k].to('cuda') for k in inputs}\n", " generate_ids = model.generate(**inputs, max_new_tokens=128, do_sample=False)\n", " generate_ids = generate_ids[:, inputs['input_ids'].shape[1] :]\n", " response = processor.batch_decode(\n", " generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False\n", " )[0]\n", "print(response)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "inputs_text" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "inputs = processor.apply_chat_template(\n", " messages, add_generation_prompt=True, tokenize=True,\n", " return_dict=True, return_tensors=\"pt\", tools=json.loads(data[0]['tools'])\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from transformers import AutoProcessor, AutoModel,AutoModelForCausalLM\n", "import torch\n", "model_id = \"/home/jeff/codes/llm/InCar/Phi-4-multimodal-instruct\"\n", "revision = \"main\"\n", "model2 = AutoModelForCausalLM.from_pretrained(#AutoModel.from_pretrained(\n", " model_id, device_map=\"cpu\", \n", " revision = revision, trust_remote_code=True,\n", " _attn_implementation='flash_attention_2'\n", " # torch_dtype=torch.float16\n", ").eval()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "messages = [\n", " {\n", " \"role\": \"user\",\n", " \"content\": [\n", " # ans is what_is_shown_in_this_image\n", " {\"type\": \"audio\", \"audio\": \"https://huggingface.co/microsoft/Phi-4-multimodal-instruct/resolve/main/examples/what_is_shown_in_this_image.wav\"},\n", " {\"type\": \"text\", \"text\": \"Transcribe this audio clip into text.\"}\n", " ]\n", " }\n", "]\n", "\n", "inputs = processor.apply_chat_template(\n", " messages, add_generation_prompt=True, tokenize=True,\n", " return_dict=True, return_tensors=\"pt\"\n", ")\n", "\n", "# with torch.inference_mode():\n", "# inputs = {k:inputs[k].to('cuda') for k in inputs}\n", "# generate_ids = model.generate(**inputs, max_new_tokens=128, do_sample=False)\n", "# generate_ids = generate_ids[:, inputs['input_ids'].shape[1] :]\n", "# response = processor.batch_decode(\n", "# generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False\n", "# )[0]\n", "# print(response)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "len(inputs['input_ids'][0]),len(inputs['input_audio_embeds'][0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model2.model.embed_tokens_extend.audio_embed.encoder(inputs['input_audio_embeds'],None)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import json\n", "with open('/mnt/data-2t/jeff/codes/llm/InCar/gemma-3-4b-it-omni/output_org_nav_2025-07-24 08:34.json') as f:\n", " nav_res = json.load(f)\n", "remove_sign = lambda x:x.replace('User transcribe is','').replace('GPT output is','').replace('\\n','').\\\n", " replace(' ','').replace('?','').replace('？','').replace('!','').replace('。','').\\\n", " replace('.','').replace('！','')\n", "total_func_call=0\n", "func_error=0\n", "errors=[]\n", "for res in nav_res:\n", " if res['cer']!=0:\n", " errors.append(res)\n", " if 'Action:' in res['label']:\n", " func_error+=remove_sign(res['label'])!=remove_sign(res['output'])\n", " total_func_call+=1\n", "avg_cer = sum(a['cer'] for a in nav_res)/len(nav_res)\n", "total_error = sum(a['cer']!=0 for a in nav_res)\n", "print('total',len(nav_res))\n", "print('total_error & rate',total_error,total_error/len(nav_res))\n", "print('avg_cer',avg_cer)\n", "print('total_func_call',total_func_call)\n", "print('func_error & rate',func_error,',',func_error/total_func_call)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "488/11046" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "aa = []\n", "for e in errors:\n", " if '}' in e['output'] and remove_sign(e['output'][:e['output'].index('}')+1])==remove_sign(e['label']):continue\n", " if remove_sign(e['label']) in remove_sign(e['output']):continue\n", " aa.append(e)\n", "len(aa)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "5/4830" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "len(aa)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "aa" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import json\n", "with open('/mnt/data-2t/jeff/codes/llm/InCar/gemma-3-4b-it-omni/output_org_ctrl_2025-07-24 05:43.json') as f:\n", " nav_res = json.load(f)\n", "remove_sign = lambda x:x.replace('User transcribe is','').replace('GPT output is','').replace('\\n','').\\\n", " replace(' ','').replace('?','').replace('？','').replace('!','').replace('。','').\\\n", " replace('.','').replace('！','')\n", "total_func_call=0\n", "func_error=0\n", "errors=[]\n", "for res in nav_res:\n", " if res['cer']!=0:\n", " errors.append(res)\n", " if 'Action:' in res['label']:\n", " func_error+=remove_sign(res['label'])!=remove_sign(res['output'])\n", " total_func_call+=1\n", "avg_cer = sum(a['cer'] for a in nav_res)/len(nav_res)\n", "total_error = sum(a['cer']!=0 for a in nav_res)\n", "print('total',len(nav_res))\n", "print('total_error & rate',total_error,total_error/len(nav_res))\n", "print('avg_cer',avg_cer)\n", "print('total_func_call',total_func_call)\n", "print('func_error & rate',func_error,',',func_error/total_func_call)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "errors" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(errors[0]['input'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "py10", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.18" } }, "nbformat": 4, "nbformat_minor": 2 }