import { useState, useRef, useEffect, useCallback } from "react"; import WebcamCapture from "./WebcamCapture"; import PromptInput from "./PromptInput"; import LiveCaption, { type HistoryEntry } from "./LiveCaption"; import { useVLMContext } from "../context/useVLMContext"; import { PROMPTS, TIMING } from "../constants"; interface CaptioningViewProps { videoRef: React.RefObject; } function useCaptioningLoop( videoRef: React.RefObject, isRunning: boolean, promptRef: React.RefObject, onCaptionUpdate: (caption: string) => void, onError: (error: string) => void, onGenerationComplete: (caption: string) => void, onStatsUpdate: (stats: { tps?: number; ttft?: number }) => void, ) { const { isLoaded, runInference } = useVLMContext(); const abortControllerRef = useRef(null); const onCaptionUpdateRef = useRef(onCaptionUpdate); const onErrorRef = useRef(onError); const onGenerationCompleteRef = useRef(onGenerationComplete); const onStatsUpdateRef = useRef(onStatsUpdate); useEffect(() => { onCaptionUpdateRef.current = onCaptionUpdate; }, [onCaptionUpdate]); useEffect(() => { onErrorRef.current = onError; }, [onError]); useEffect(() => { onGenerationCompleteRef.current = onGenerationComplete; }, [onGenerationComplete]); useEffect(() => { onStatsUpdateRef.current = onStatsUpdate; }, [onStatsUpdate]); useEffect(() => { abortControllerRef.current?.abort(); if (!isRunning || !isLoaded) return; abortControllerRef.current = new AbortController(); const signal = abortControllerRef.current.signal; const video = videoRef.current; const captureLoop = async () => { while (!signal.aborted) { if ( video && video.readyState >= 2 && !video.paused && video.videoWidth > 0 ) { try { const currentPrompt = promptRef.current || ""; const result = await runInference( video, currentPrompt, onCaptionUpdateRef.current, (stats) => onStatsUpdateRef.current(stats), ); if (result && !signal.aborted) { onCaptionUpdateRef.current(result); onGenerationCompleteRef.current(result); } } catch (error) { if (!signal.aborted) { const message = error instanceof Error ? error.message : String(error); onErrorRef.current(message); console.error("Error processing frame:", error); } } } if (signal.aborted) break; await new Promise((resolve) => setTimeout(resolve, TIMING.FRAME_CAPTURE_DELAY), ); } }; // NB: Wrap with a setTimeout to ensure abort controller can run before starting the loop // This is necessary for React's strict mode which calls effects twice in development. setTimeout(captureLoop, 0); return () => { abortControllerRef.current?.abort(); }; }, [isRunning, isLoaded, runInference, promptRef, videoRef]); } export default function CaptioningView({ videoRef }: CaptioningViewProps) { const { imageSize, setImageSize } = useVLMContext(); const [caption, setCaption] = useState(""); const [isLoopRunning, setIsLoopRunning] = useState(true); const [currentPrompt, setCurrentPrompt] = useState(PROMPTS.default); const [error, setError] = useState(null); const [history, setHistory] = useState([]); const [stats, setStats] = useState<{ tps?: number; ttft?: number }>({}); // Use ref to store current prompt to avoid loop restarts const promptRef = useRef(currentPrompt); // Update prompt ref when state changes useEffect(() => { promptRef.current = currentPrompt; }, [currentPrompt]); const handleCaptionUpdate = useCallback((newCaption: string) => { setCaption(newCaption); setError(null); }, []); const handleError = useCallback((errorMessage: string) => { setError(errorMessage); setCaption(`Error: ${errorMessage}`); }, []); const handleGenerationComplete = useCallback((text: string) => { const now = new Date(); const timeString = now.toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit", }); setHistory((prev) => [ { timestamp: timeString, text: text, }, ...prev, ].slice(0, 50), ); }, []); const handleStatsUpdate = useCallback( (newStats: { tps?: number; ttft?: number }) => { setStats((prev) => ({ ...prev, ...newStats })); }, [], ); useCaptioningLoop( videoRef, isLoopRunning, promptRef, handleCaptionUpdate, handleError, handleGenerationComplete, handleStatsUpdate, ); const handlePromptChange = useCallback((prompt: string) => { setCurrentPrompt(prompt); setError(null); }, []); const handleToggleLoop = useCallback(() => { setIsLoopRunning((prev) => !prev); if (error) setError(null); }, [error]); return (
{/* Prompt Input - Bottom Left */}
{/* Live Caption - Bottom Right */}
); }