import gradio as gr import spaces import torch from diffusers import DiffusionPipeline import numpy as np from PIL import Image import tempfile import os from moviepy.editor import ImageSequenceClip, AudioFileClip import soundfile as sf from transformers import pipeline import time from typing import List, Tuple, Optional import json from config import Config from utils import VideoGenerator, AudioGenerator, ImageGenerator # Initialize generators image_gen = ImageGenerator() audio_gen = AudioGenerator() video_gen = VideoGenerator() @spaces.GPU(duration=1500) def compile_transformer(): """Compile the Stable Diffusion transformer for faster inference""" with spaces.aoti_capture(image_gen.pipe.transformer) as call: image_gen.pipe("test compilation prompt") exported = torch.export.export( image_gen.pipe.transformer, args=call.args, kwargs=call.kwargs, ) return spaces.aoti_compile(exported) # Compile during startup print("Compiling AI models for optimal performance...") compiled_transformer = compile_transformer() spaces.aoti_apply(compiled_transformer, image_gen.pipe.transformer) print("✅ Models compiled successfully!") @spaces.GPU(duration=120) def generate_video( prompt: str, duration: int, fps: int, audio_type: str, voice_gender: str, music_style: str, num_images: int, image_size: int, motion_strength: float, progress=gr.Progress() ) -> str: """ Generate a video from text prompt with AI-generated images and audio Args: prompt: Text description for the video content duration: Duration of the video in seconds fps: Frames per second for the video audio_type: Type of audio to generate (narration/music/both) voice_gender: Gender for voice narration music_style: Style of background music num_images: Number of unique images to generate image_size: Size of generated images motion_strength: Strength of motion between frames Returns: Path to the generated video file """ try: progress(0.1, desc="Starting video generation...") # Calculate timing total_frames = duration * fps frames_per_image = total_frames // num_images progress(0.2, desc="Generating images...") # Generate images images = [] for i in range(num_images): # Slightly vary the prompt for each image varied_prompt = f"{prompt}, frame {i+1}, cinematic lighting" image = image_gen.generate_image( prompt=varied_prompt, size=(image_size, image_size) ) images.append(image) progress(0.2 + (i / num_images) * 0.3, desc=f"Generated image {i+1}/{num_images}") progress(0.5, desc="Generating audio...") # Generate audio audio_path = None if audio_type in ["narration", "both"]: narration_path = audio_gen.generate_narration( text=prompt, gender=voice_gender, duration=duration ) audio_path = narration_path if audio_type in ["music", "both"]: music_path = audio_gen.generate_music( style=music_style, duration=duration ) if audio_path and audio_type == "both": # Mix narration and music audio_path = audio_gen.mix_audio(audio_path, music_path) elif not audio_path: audio_path = music_path progress(0.7, desc="Creating video frames...") # Create video frames with motion video_frames = video_gen.create_motion_frames( images=images, frames_per_image=frames_per_image, motion_strength=motion_strength ) progress(0.9, desc="Composing final video...") # Create video video_path = video_gen.create_video( frames=video_frames, fps=fps, audio_path=audio_path, duration=duration ) progress(1.0, desc="Video generation complete!") return video_path except Exception as e: raise gr.Error(f"Error generating video: {str(e)}") @spaces.GPU(duration=60) def generate_sample_image(prompt: str, style: str) -> Image.Image: """Generate a sample image for preview""" styled_prompt = f"{prompt}, {style} style, high quality, detailed" return image_gen.generate_image( prompt=styled_prompt, size=(512, 512) ) def create_demo(): """Create the Gradio demo interface""" with gr.Blocks( title="AI Video Generator", theme=gr.themes.Soft(), css=""" .gradio-container { max-width: 1200px !important; } .header-text { text-align: center; margin-bottom: 2rem; } .preview-box { border: 2px dashed #ccc; border-radius: 10px; padding: 20px; text-align: center; } """ ) as demo: gr.HTML("""
Create stunning videos from text prompts using AI-powered image and audio generation