| input_stream: "HTTP_REQUEST_PAYLOAD:input" | |
| output_stream: "HTTP_RESPONSE_PAYLOAD:output" | |
| node: { | |
| name: "LLMExecutor" | |
| calculator: "HttpLLMCalculator" | |
| input_stream: "LOOPBACK:loopback" | |
| input_stream: "HTTP_REQUEST_PAYLOAD:input" | |
| input_side_packet: "LLM_NODE_RESOURCES:llm" | |
| output_stream: "LOOPBACK:loopback" | |
| output_stream: "HTTP_RESPONSE_PAYLOAD:output" | |
| input_stream_info: { | |
| tag_index: 'LOOPBACK:0', | |
| back_edge: true | |
| } | |
| node_options: { | |
| [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { | |
| models_path: "./", | |
| plugin_config: '{}', | |
| enable_prefix_caching: false, | |
| cache_size: 2, | |
| max_num_seqs: 256, | |
| device: "GPU", | |
| } | |
| } | |
| input_stream_handler { | |
| input_stream_handler: "SyncSetInputStreamHandler", | |
| options { | |
| [mediapipe.SyncSetInputStreamHandlerOptions.ext] { | |
| sync_set { | |
| tag_index: "LOOPBACK:0" | |
| } | |
| } | |
| } | |
| } | |
| } |