src/components/p5/P5Sketch.tsx

/**
 * Copyright 2024 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
import { FunctionDeclaration, SchemaType } from "@google/generative-ai";
import { useEffect, useRef, useState, memo } from "react";
import vegaEmbed from "vega-embed";
import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
import { ToolCall } from "../../multimodal-live-types";

function P5SketchComponent() {
  const containerRef = useRef<HTMLDivElement>(null);
  const [jsonString, setJSONString] = useState<string>("");
  const { client, setConfig } = useLiveAPIContext();

  useEffect(() => {
    const w = window as any;
    if (containerRef.current && w.initSketch) {
      w.initSketch(containerRef.current);
    }
  }, [containerRef]);

  useEffect(() => {
    setConfig({
      model: "models/gemini-2.0-flash-exp",
      generationConfig: {
        responseModalities: "text",
        //responseModalities: "audio",
        /*
        speechConfig: {
          voiceConfig: { prebuiltVoiceConfig: { voiceName: "Kobe" } },
        },
        */
      },
      systemInstruction: {
        parts: [
          {
            text: `We have circles on a canvas that we can move and resize. Every time that I ask you something, I want you to use the "get_circles" function to understand where the circles are. Once you receive the response for "get_circles" I want you to use "change_circle" to modify one or multiple circle properties to accomplish our mutual goal. Every time I ask you ask you to do something you should call "get_circles" then "change_circle" after evaluating the response.
            
            The circles blend additively using screen mode, so if I ask you mix colors, you can make the color by moving circles to the same position to blend their colors.
            `
          },
        ],
      },
      tools: [
        // there is a free-tier quota for search
        { googleSearch: {} },
        {
          functionDeclarations: [
            {
              name: "get_circles",
              description: "Get a list of circles in my application",
            },
            {
              name: "change_circle",
              description: "Change one of the circles",
              parameters: {
                type: SchemaType.OBJECT,
                properties: {
                  color: {
                    type: SchemaType.STRING,
                  },
                  x: {
                    type: SchemaType.NUMBER,
                  },
                  y: {
                    type: SchemaType.NUMBER,
                  },
                  radius: {
                    type: SchemaType.NUMBER,
                  },
                },
                required: ["color", "x", "y", "radius"],
              },
            },
          ],
        },
      ],
    });
  }, [setConfig]);

  useEffect(() => {
    const onToolCall = (toolCall: ToolCall) => {
      console.log(`got toolcall`, toolCall);
      toolCall.functionCalls.forEach((fc) => {
        const w = window as any;
        const func: Function = w[fc.name];
        if (fc.name === "get_circles") {
          setTimeout(
            () =>
              client.sendToolResponse({
                functionResponses: [
                  {
                    response: w.get_circles(),
                    id: fc.id,
                  },
                ],
              }),
            500,
          );
        } else if (func && typeof func === "function") {
          func(fc.args);
          client.sendToolResponse({
            functionResponses: [
              {
                response: {
                  content: { ...w.get_circles() },
                },
                id: fc.id,
              },
            ],
          });
        } else {
          console.error(`Unhandled function call for ${fc.name}`);
        }
      });
    };
    client.on("toolcall", onToolCall);
    return () => {
      client.off("toolcall", onToolCall);
    };
  }, [client]);

  const embedRef = useRef<HTMLDivElement>(null);

  useEffect(() => {
    if (embedRef.current && jsonString) {
      vegaEmbed(embedRef.current, JSON.parse(jsonString));
    }
  }, [embedRef, jsonString]);
  return (
    <div
      ref={containerRef}
      style={{
        width: "100vw",
        height: "100vh",
        alignItems: "center",
        justifyContent: "center",
        display: "flex",
      }}
    />
  );
}

export const P5Sketch = memo(P5SketchComponent);