Video Examples

backend/examples.py

@@ -10,7 +10,7 @@ def group_files_by_index(file_paths, data_type="audio"):
     if data_type == "audio":
         pattern = r"audio_(\d+).(png|wav)"
     elif data_type == "video":
-        pattern = r"video_(\d+).(png|mkv)"
+        pattern = r"video_(\d+).(png|mp4)"
     else:
         pattern = r"img_(\d+).png"
     # Dictionary to store the grouped files
@@ -110,11 +110,12 @@ def build_infos(abs_path: Path, datatype: str, dataset_name: str, db_key: str):
         eval_results_path = abs_path + f"{dataset_name}_1k/examples_eval_results.json"
     elif datatype == "video":
         quality_metrics = ["psnr", "ssim", "lpips", "msssim", "vmaf"]
-        extensions = ["mkv"]
+        extensions = ["mp4"]
         datatype_abbr = "video"
         eval_results_path = abs_path + f"{dataset_name}/examples_eval_results.json"
 
     response = requests.get(eval_results_path)
+    print(eval_results_path)
     if response.status_code == 200:
         results_data = response.json()
     else:
@@ -130,7 +131,7 @@ def build_infos(abs_path: Path, datatype: str, dataset_name: str, db_key: str):
     ]
 
     file_patterns = generate_file_patterns(prefixes, extensions)
-
+    print(f"File patterns: {file_patterns}")
     infos = {}
     for model_name in dataset.keys():
         model_infos = {}
@@ -192,7 +193,7 @@ def build_infos(abs_path: Path, datatype: str, dataset_name: str, db_key: str):
                             file_info["image_url"] = file.replace(".wav", ".png")
                             file_info["audio_url"] = file
                         elif datatype == "video":
-                            file_info["image_url"] = file.replace(".mkv", ".png")
+                            # file_info["image_url"] = file.replace(".mp4", ".png")
                             file_info["video_url"] = file
                         else:
                             file_info["image_url"] = file
@@ -263,7 +264,8 @@ def image_examples_tab(abs_path: Path):
 
 
 def video_examples_tab(abs_path: Path):
-    dataset_name = "sav_val_full"
+    # dataset_name = "sav_val_full"
+    dataset_name = "sav_val_full_v2"
     datatype = "video"
     db_key = "sa-v_sav_val_videos"
 

frontend/src/components/VideoGallery.tsx

@@ -4,6 +4,7 @@ import { groupByNameAndVariant } from './galleryUtils'
 import ExampleVariantMetricsTable from './ExampleVariantMetricsTable'
 import ExampleDetailsSection from './ExampleDetailsSection'
 import ExampleVariantSelector from './ExampleVariantSelector'
+import ExampleVariantToggle from './ExampleVariantToggle'
 
 interface GalleryProps {
   selectedModel: string
@@ -23,6 +24,68 @@ const VideoGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, e
   const variants = grouped[selectedVideo] || {}
   const variantKeys = Object.keys(variants)
   const [selectedVariant, setSelectedVariant] = React.useState(variantKeys[0] || '')
+  const [toggleMode, setToggleMode] = React.useState<'wmd' | 'attacked'>('wmd')
+  // Add state for rewind seconds
+  const [rewindSeconds, setRewindSeconds] = React.useState(0.5)
+  // State for video scale
+  const [videoScale, setVideoScale] = React.useState(1)
+
+  // Playback time ref for syncing position
+  const playbackTimeRef = React.useRef(0)
+  // Refs for all video elements
+  const videoRefs = React.useMemo(() => {
+    const refs: Record<string, React.RefObject<HTMLVideoElement>> = {}
+    variantKeys.forEach((v) => {
+      refs[v] = React.createRef<HTMLVideoElement>()
+    })
+    return refs
+  }, [variantKeys.join(',')])
+
+  // Track which variant is currently playing
+  const [playingVariant, setPlayingVariant] = React.useState<string | null>(null)
+
+  // Play handler: pause all others, sync time
+  const handlePlay = (variant: string) => {
+    setPlayingVariant(variant)
+    variantKeys.forEach((v) => {
+      if (v !== variant && videoRefs[v]?.current) {
+        videoRefs[v]?.current?.pause()
+      }
+    })
+  }
+  // Pause handler
+  const handlePause = (variant: string) => {
+    if (playingVariant === variant) setPlayingVariant(null)
+  }
+
+  // When selectedVariant changes, sync playback position, rewind, and play state
+  React.useEffect(() => {
+    if (!selectedVariant) return
+    // Rewind playbackTimeRef by rewindSeconds, clamp to 0
+    playbackTimeRef.current = Math.max(0, playbackTimeRef.current - rewindSeconds)
+    // Set all videos to the new time, pause all except selected
+    variantKeys.forEach((v) => {
+      const ref = videoRefs[v]?.current
+      if (ref) {
+        ref.currentTime = playbackTimeRef.current
+        if (v !== selectedVariant) {
+          ref.pause()
+        }
+      }
+    })
+    // If a video was playing, continue playing the swapped variant
+    if (playingVariant && videoRefs[selectedVariant]?.current) {
+      videoRefs[selectedVariant].current.play()
+    }
+  }, [selectedVariant])
+
+  // When the selected video plays, update playbackTimeRef
+  const handleTimeUpdate = (variantKey: string) => {
+    const ref = videoRefs[variantKey]?.current
+    if (ref && variantKey === selectedVariant) {
+      playbackTimeRef.current = ref.currentTime
+    }
+  }
 
   React.useEffect(() => {
     setSelectedVariant(variantKeys[0] || '')
@@ -72,14 +135,60 @@ const VideoGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, e
                 selectedVariant={selectedVariant}
                 setSelectedVariant={setSelectedVariant}
               />
+              <ExampleVariantToggle
+                toggleMode={toggleMode}
+                setToggleMode={setToggleMode}
+                type="button"
+                selectedVariant={selectedVariant}
+                setSelectedVariant={setSelectedVariant}
+                variantKeys={variantKeys}
+              />
+              <div className="flex items-center gap-4 mt-2">
+                <label htmlFor="rewind-seconds" className="font-mono text-xs">Rewind Seconds:</label>
+                <input
+                  id="rewind-seconds"
+                  type="number"
+                  min={0}
+                  step={0.1}
+                  value={rewindSeconds}
+                  onChange={(e) => setRewindSeconds(Math.max(0, Number(e.target.value)))}
+                  className="input input-bordered input-xs w-20"
+                  placeholder="Seconds"
+                />
+                <label htmlFor="video-scale" className="font-mono text-xs ml-4">Scale:</label>
+                <input
+                  id="video-scale"
+                  type="range"
+                  min={0.3}
+                  max={1}
+                  step={0.01}
+                  value={videoScale}
+                  onChange={e => setVideoScale(Number(e.target.value))}
+                  className="range range-xs w-40"
+                  style={{ verticalAlign: 'middle' }}
+                />
+                <span className="ml-2 font-mono text-xs">{(videoScale * 100).toFixed(0)}%</span>
+              </div>
               <div className="flex flex-col items-center gap-4">
-                {variants[selectedVariant].video_url && (
-                  <video
-                    controls
-                    src={variants[selectedVariant].video_url}
-                    className="example-video"
-                    style={{ maxWidth: 400, maxHeight: 300 }}
-                  />
+                {variantKeys.map((variantKey) =>
+                  variants[variantKey].video_url ? (
+                    <video
+                      key={variantKey}
+                      ref={videoRefs[variantKey]}
+                      controls
+                      src={variants[variantKey].video_url}
+                      className="example-video"
+                      style={{
+                        width: `${videoScale * 100}%`,
+                        height: 'auto',
+                        display: selectedVariant === variantKey ? 'block' : 'none',
+                        maxWidth: '100%',
+                      }}
+                      onTimeUpdate={() => handleTimeUpdate(variantKey)}
+                      onPlay={() => handlePlay(variantKey)}
+                      onPause={() => handlePause(variantKey)}
+                    />
+                  ) : null
                 )}
               </div>
             </ExampleDetailsSection>