adding 3d

.env.example

@@ -18,4 +18,7 @@ MCP_SERVER_URL="your-mcp-server-url-here"
 #Used to test mcp function calling
 OPENAI_API_KEY="your-openai-api-key-here"
 #Used to Parse the PDFs for story enrichment
-LLAMA_CLOUD_API_KEY="your-llama-cloud-api-key-here"
+LLAMA_CLOUD_API_KEY="your-llama-cloud-api-key-here"
+
+# Used for 3D mesh generation
+THREE_D_API_URL="your-3d-api-url-here"

app.py

@@ -26,6 +26,7 @@ from ui.components import (
     create_chapter_navigation,
     create_chapter_accordion,
     create_story_melody_section,
+    create_3d_model_viewer,
 )
 from ui.mcp_components import letter_counter_component
 from controllers.app_controller import (
@@ -34,7 +35,6 @@ from controllers.app_controller import (
 )
 from ui.events import setup_event_handlers
 
-
 # Configure logging
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
@@ -152,6 +152,9 @@ def create_app():
                         outputs=[melody_output, melody_status],
                     )
 
+                    # 3D Model Viewer Tab (demo)
+                    create_3d_model_viewer(story_text)
+
             # TODO DELETE THIS AS IT IRRELEVENT TO STORY GENERATION
             """ with gr.Tab("Character Counter"):
                 letter_counter_component() """

controllers/app_controller.py

@@ -46,16 +46,19 @@ def process_story_generation(
 
         # Process PDF if provided
         pdf_content = ""
+        summarized_pdf = ""  # Initialize with empty string by default
+
         if pdf_file:
             logger.info("Extracting text from PDF")
             pdf_content = extract_text_from_pdf(pdf_file)
             # summarize the PDF content for better prompting using mistral
-            mistral_api = MistralAPI()
-            summarized_pdf = mistral_api.send_request(
-                f"Summarize the following Text content into a single-sentence children's story without any explanations, tags, or formatting—just plain text in one line.: {pdf_content}"
-            )["choices"][0]["message"]["content"]
-            logger.info(f"summarized_pdf: {summarized_pdf}")
-            if pdf_content.startswith("Error:"):
+            if pdf_content and not pdf_content.startswith("Error:"):
+                mistral_api = MistralAPI()
+                summarized_pdf = mistral_api.send_request(
+                    f"Summarize the following Text content into a single-sentence children's story without any explanations, tags, or formatting—just plain text in one line.: {pdf_content}"
+                )["choices"][0]["message"]["content"]
+                logger.info(f"summarized_pdf: {summarized_pdf}")
+            else:
                 logger.error(f"PDF extraction error: {pdf_content}")
 
         # Generate story
@@ -73,7 +76,7 @@ def process_story_generation(
 
         if story_response.startswith("Error:"):
             logger.error(f"Story generation error: {story_response}")
-            return "", story_response, ""
+            return "", story_response, gr.update(interactive=False)
 
         try:
             # Parse JSON response
@@ -84,12 +87,16 @@ def process_story_generation(
             return (title, story, gr.update(interactive=True, visible=True))
         except json.JSONDecodeError:
             logger.error("Failed to parse story JSON response")
-            return "", f"Error: Failed to parse story response: {story_response}"
+            return (
+                "",
+                f"Error: Failed to parse story response: {story_response}",
+                gr.update(interactive=False),
+            )
 
     except Exception as e:
         error_msg = f"Unexpected error during story generation: {str(e)}"
         logger.error(error_msg, exc_info=True)
-        return f"Error: {error_msg}"
+        return "", f"Error: {error_msg}", gr.update(interactive=False)
 
 
 def process_chapters(

services/mesh_service.py

@@ -0,0 +1,96 @@
+import requests
+import json
+import os
+from typing import Dict, Any, Optional, TypedDict, Union, Literal
+from pathlib import Path
+import base64
+
+
+class ModelData(TypedDict):
+    prompt: str
+    model: str
+    format: str
+    mesh_base64: str
+
+
+class MeshResponse(TypedDict):
+    text: str
+    model_data: ModelData
+    model: str
+    image_base64: str
+    used_image_input: bool
+
+
+class ErrorResponse(TypedDict):
+    error: str
+
+
+OutputFormatType = Literal["glb", "obj", "stl"]
+
+
+def get_mesh_base64(
+    text: str, apply_texture: bool = False, output_format: OutputFormatType = "glb"
+) -> Union[MeshResponse, ErrorResponse]:
+    """
+    Retrieve the base64 encoded mesh data for the 3D model viewer by making an API call.
+
+    Args:
+        text (str): The description of the 3D model.
+        apply_texture (bool): Whether to apply texture to the model.
+        output_format (OutputFormatType): The desired output format ('glb', 'obj', or 'stl').
+
+    Returns:
+        Union[MeshResponse, ErrorResponse]: The API response as JSON or an error message.
+    """
+    url = os.getenv("THREE_D_API_URL", "https://3d-model-inference-api.com")
+
+    headers = {"Content-Type": "application/json"}
+    payload = {
+        "text": text,
+        "apply_texture": apply_texture,
+        "output_format": output_format,
+    }
+
+    try:
+        response = requests.post(
+            url,
+            headers=headers,
+            data=json.dumps(payload),
+        )
+        response.raise_for_status()  # Raise an exception for HTTP errors
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        return {"error": f"Error retrieving mesh data: {str(e)}"}
+    except json.JSONDecodeError:
+        return {"error": "Failed to parse API response as JSON"}
+
+
+def transform_base64_to_glb_file(base64_data, output_path=None):
+    """
+    Transform a base64-encoded GLB model to a .glb file and return the path for gr.Model3D
+
+    Args:
+        base64_data (str): The base64-encoded GLB data
+        output_path (str, optional): Path where to save the .glb file.
+                                    If None, creates a temp file in assets/models/
+
+    Returns:
+        str: The path to the saved .glb file
+    """
+    if output_path is None:
+        # Create models directory if it doesn't exist
+        models_dir = Path("assets/models")
+        models_dir.mkdir(parents=True, exist_ok=True)
+        output_path = models_dir / "model.glb"
+
+    # Ensure parent directories exist
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+    # Decode the base64 data
+    binary_data = base64.b64decode(base64_data)
+
+    # Write the binary data to a .glb file
+    with open(output_path, "wb") as f:
+        f.write(binary_data)
+
+    return str(output_path)

ui/components.py

@@ -5,6 +5,9 @@ UI components for the Magic Story Creator application
 import gradio as gr
 from config import constants
 
+from services.mesh_service import get_mesh_base64, transform_base64_to_glb_file
+from util.mistral_api_client import MistralAPI
+
 
 def create_header():
     """Create the main header section of the application"""
@@ -415,3 +418,141 @@ def create_story_melody_section():
                 )
 
     return generate_melody_button, melody_status, melody_output
+
+
+# TODO split this further later
+def create_3d_model_viewer(story: str = "simple yellow duck"):
+    """
+    Create a 3D model viewer component that displays a mesh from a base64 string.
+    For now, this uses a placeholder HTML with a message and a download link for the mesh.
+    In a real app, you would use a JS 3D viewer (e.g., Three.js) embedded in the HTML.
+    """
+
+    def get_prompt_from_story(story_text):
+        """
+        Extract a prompt from the story text for generating a 3D model.
+
+        Uses Mistral API to analyze the story and generate an appropriate
+        3D model prompt based on characters, objects, or scenes in the story.
+
+        Args:
+            story_text (str): The story text to analyze
+
+        Returns:
+            str: A concise prompt suitable for 3D model generation
+        """
+        try:
+            import logging
+
+            logger = logging.getLogger(__name__)
+
+            # Initialize Mistral API
+            mistral_api = MistralAPI()
+
+            # Create a prompt for extracting a 3D model description
+            system_prompt = """
+            You are an assistant that analyzes children's stories and extracts 
+            concise descriptions of key objects, characters, or scenes that would 
+            make a good 3D model. Focus on something iconic from the story that 
+            would be visually interesting and meaningful to the child.
+            """
+
+            instruction_prompt = f"""
+            Read this children's story and suggest a simple, clear description for a 3D model 
+            that represents an important element from the story (character, object, or scene).
+            
+            Keep your response to a single phrase or short sentence (max 10 words) 
+            that clearly describes what to model in 3D. Don't use any formatting, 
+            explanations, or additional text - just the model description itself.
+            
+            Story: {story_text[:1500]}...
+            """
+
+            # Send request to Mistral API
+            logger.info("Generating 3D model prompt from story text")
+            response = mistral_api.send_request(instruction_prompt, system_prompt)
+
+            # Extract the model prompt from the response
+            if "choices" in response and len(response["choices"]) > 0:
+                model_prompt = response["choices"][0]["message"]["content"].strip()
+
+                # Clean up the prompt to ensure it's suitable for 3D generation
+                # Remove quotes if present and limit length
+                model_prompt = model_prompt.strip("\"'").split(".")[0]
+                if len(model_prompt) > 100:
+                    model_prompt = model_prompt[:100]
+
+                logger.info(f"Generated 3D model prompt: {model_prompt}")
+                return model_prompt
+            else:
+                logger.error("Failed to generate model prompt from Mistral API")
+                return "magical storybook character"
+
+        except Exception as e:
+            logger.error(f"Error generating 3D model prompt: {e}")
+            return "friendly cartoon character from a children's story"
+
+    generated_prompt_from_story = get_prompt_from_story(story)
+    prompt_state = gr.State(value=generated_prompt_from_story)
+
+    gr.HTML("""
+        <div class="image-header-wrapper"  style="margin: 30px 0 15px 0;">
+            <h1 class="sub-header" style="margin-left: 16px; padding-bottom: 36px;">
+                Create a Magical 3D Model from Your Story!
+            </h1>
+        </div>
+        <p class="text" style="font-size: 1.2em; text-align: center;">Click the button below to generate a 3D model from your story!</p>
+    """)
+
+    with gr.Blocks(elem_classes="melody-box"):
+        with gr.Row():
+            with gr.Column(scale=1, min_width=200):
+                generate_model_button = gr.Button(
+                    "Create 3D Model 🎨",
+                    variant="primary",
+                    elem_classes="melody-button",
+                )
+
+            with gr.Column(scale=2):
+                model_viewer = gr.Model3D(label="3D Model Viewer", clear_color=[0.0, 0.0, 0.0, 0.0])
+            
+            model_status = gr.Markdown("", visible=False)
+
+    # Add status message and model display
+
+    def generate_3d_model(prompt):
+        model_response = get_mesh_base64(
+            text=prompt, apply_texture=False, output_format="glb"
+        )
+
+        # Check if response contains an error
+        if "error" in model_response:
+            return None, f"Error: {model_response['error']}"
+
+        # Check if the expected data structure exists
+        if (
+            "model_data" not in model_response
+            or "mesh_base64" not in model_response["model_data"]
+        ):
+            return (
+                None,
+                "Error: Received unexpected response format from 3D model API",
+            )
+
+        try:
+            glb_file_path = transform_base64_to_glb_file(
+                model_response["model_data"]["mesh_base64"]
+            )
+            return (
+                glb_file_path,
+                f"Successfully generated 3D model: {prompt}",
+            )
+        except Exception as e:
+            return None, f"Error processing model data: {str(e)}"
+
+    # Connect the button to the generation function
+    generate_model_button.click(
+        generate_3d_model,
+        inputs=[prompt_state],
+        outputs=[model_viewer, model_status],
+    )

util/mistral_api_client.py

@@ -8,8 +8,9 @@ logger = logging.getLogger(__name__)
 
 # Constants
 MISTRAL_API_ENDPOINT = "https://api.mistral.ai/v1/chat/completions"
-MISTRAL_MODEL = "mistral-medium-latest"
+MISTRAL_MODEL = "ministral-8b-latest"
 API_KEY_ENV_VAR = "MISTRAL_API_KEY"
+DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant. Please respond to the user's queries with just text and no additional formatting or explanations."
 
 
 class MistralAPI:
@@ -26,7 +27,9 @@ class MistralAPI:
             )
         self.mistral_model = mistral_model
 
-    def send_request(self, prompt: str) -> Dict:
+    def send_request(
+        self, prompt: str, system_prompt: str = DEFAULT_SYSTEM_PROMPT
+    ) -> Dict:
         """
         Send a request to the Mistral API with the given prompt.
 
@@ -44,7 +47,10 @@ class MistralAPI:
 
         data = {
             "model": self.mistral_model,
-            "messages": [{"role": "user", "content": prompt}],
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": prompt},
+            ],
         }
 
         try: