let's hear it

src/scheduler/processTask.mts

@@ -1,3 +1,5 @@
+import { v4 as uuidv4 } from "uuid"
+
 import { saveCompletedTask } from "./saveCompletedTask.mts"
 import { savePendingTask } from "./savePendingTask.mts"
 import { updatePendingTask } from "./updatePendingTask.mts"
@@ -12,6 +14,9 @@ import { postInterpolation } from "../production/postInterpolation.mts"
 import { moveVideoFromPendingToCompleted } from "../utils/moveVideoFromPendingToCompleted.mts"
 import { assembleShots } from "../production/assembleShots.mts"
 import { copyVideoFromPendingToCompleted } from "../utils/copyVideoFromPendingToCompleted.mts"
+import { generateAudio } from "../production/generateAudio.mts"
+import { mergeAudio } from "../production/mergeAudio.mts"
+import { addAudioToVideo } from "../production/addAudioToVideo.mts"
 
 export const processTask = async (task: VideoTask) => {
   console.log(`processing video task ${task.id}`)
@@ -192,6 +197,34 @@ export const processTask = async (task: VideoTask) => {
       }
     }
 
+
+    let foregroundAudioFileName = `${task.ownerId}_${task.id}_${shot.id}_${uuidv4()}.m4a`
+
+    if (!shot.hasGeneratedForegroundAudio && shot.foregroundAudioPrompt) {
+      console.log("generating foreground audio..")
+   
+      try {
+        await generateAudio(shot.foregroundAudioPrompt, foregroundAudioFileName)
+
+        shot.hasGeneratedForegroundAudio = true
+        shot.nbCompletedSteps++
+        nbCompletedSteps++
+        shot.progressPercent = Math.round((shot.nbCompletedSteps / shot.nbTotalSteps) * 100)
+        task.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
+
+        await addAudioToVideo(shot.fileName, foregroundAudioFileName)
+
+        await updatePendingTask(task)
+
+      } catch (err) {
+        console.error(`failed to generate foreground audio for ${shot.id} (${err})`)
+        // something is wrong, let's put the whole thing back into the queue
+        task.error = `failed to generate foreground audio ${shot.id} (will try again later)`
+        await updatePendingTask(task)
+        break
+      }
+    }
+
       
     if (!shot.hasPostProcessedVideo) {
       console.log("post-processing video..")
@@ -204,7 +237,7 @@ export const processTask = async (task: VideoTask) => {
     console.log('performing final scaling (1280x720 @ 24 FPS)')
 
       try {
-        await postInterpolation(shot.fileName, shot.durationMs, shot.fps)
+        await postInterpolation(shot.fileName, shot.durationMs, shot.fps, shot.noiseAmount)
     
         shot.hasPostProcessedVideo = true
         shot.nbCompletedSteps++

src/utils/parseShotRequest.mts

@@ -21,7 +21,7 @@ export const parseShotRequest = async (sequence: VideoSequence, maybeShotMeta: P
     backgroundAudioPrompt: `${maybeShotMeta.backgroundAudioPrompt || ""}`,
 
     // describe the foreground audio (cars revving, footsteps, objects breaking, explosion etc)
-    foregroundAudioPrompt: `${maybeShotMeta.foregroundAudioPrompt || ""}`,
+    foregroundAudioPrompt: `${maybeShotMeta.foregroundAudioPrompt || maybeShotMeta.shotPrompt || ""}`,
 
     // describe the main actor visible in the shot (optional)
     actorPrompt: `${maybeShotMeta.actorPrompt || ""}`,
@@ -70,15 +70,14 @@ export const parseShotRequest = async (sequence: VideoSequence, maybeShotMeta: P
     hasPostProcessedVideo: false,
     nbCompletedSteps: 0,
 
-    // 0. in queue
-    // 1. generate with Zeroscope
-    // 2. upscale with Zeroscope XL
-    // 3. interpolate with FILE
-    // 4. generate audio background
-    // 5. generate audio foreground
-    // 6. add audio to video
-    // 7. post-processing
-    nbTotalSteps: 7,
+    // - generate with Zeroscope
+    // - upscale with Zeroscope XL
+    // - interpolate with FILE
+    // - generate audio background
+    // - generate audio foreground
+    // - add audio to video
+    // - post-processing
+    nbTotalSteps: 5,
     progressPercent: 0,
     completedAt: '',
     completed: false,