ADD generate-instruction (#3)

* ADD generate_instruction

* CHG update_trace_step exception handling

* ADD frontend objets

cua2-core/src/cua2_core/models/models.py

@@ -315,3 +315,17 @@ class AvailableModelsResponse(BaseModel):
     """Response for available models"""
 
     models: list[str]
+
+
+class GenerateInstructionRequest(BaseModel):
+    """Request model for generating task instruction"""
+
+    model_id: str
+    prompt: Optional[str] = None
+
+
+class GenerateInstructionResponse(BaseModel):
+    """Response model for generated task instruction"""
+
+    instruction: str
+    model_id: str

cua2-core/src/cua2_core/routes/routes.py

@@ -3,12 +3,15 @@ from datetime import datetime
 # Get services from app state
 from cua2_core.models.models import (
     AvailableModelsResponse,
+    GenerateInstructionRequest,
+    GenerateInstructionResponse,
     HealthResponse,
     UpdateStepRequest,
     UpdateStepResponse,
 )
 from cua2_core.services.agent_service import AgentService
 from cua2_core.services.agent_utils.get_model import AVAILABLE_MODELS
+from cua2_core.services.instruction_service import InstructionService
 from cua2_core.websocket.websocket_manager import WebSocketManager
 from fastapi import APIRouter, Depends, HTTPException, Request
 
@@ -44,6 +47,30 @@ async def get_available_models():
     return AvailableModelsResponse(models=AVAILABLE_MODELS)
 
 
+@router.post("/generate-instruction", response_model=GenerateInstructionResponse)
+async def generate_task_instruction(
+    request: GenerateInstructionRequest,
+):
+    """Generate a task instruction using a specified model"""
+    try:
+        instruction = InstructionService.generate_instruction(
+            model_id=request.model_id, prompt=request.prompt
+        )
+
+        return GenerateInstructionResponse(
+            instruction=instruction, model_id=request.model_id
+        )
+
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Error generating instruction: {str(e)}",
+        )
+
+
 @router.patch("/traces/{trace_id}/steps/{step_id}", response_model=UpdateStepResponse)
 async def update_trace_step(
     trace_id: str,
@@ -62,7 +89,7 @@ async def update_trace_step(
             success=True,
             message="Step updated successfully",
         )
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
     except FileNotFoundError as e:
         raise HTTPException(status_code=404, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))

cua2-core/src/cua2_core/services/agent_utils/desktop_agent.py

@@ -206,12 +206,8 @@ class E2BVisionAgent(CodeAgent):
             Args:
                 url: The URL to open
             """
-            # Make sure URL has http/https prefix
-            if not url.startswith(("http://", "https://")):
-                url = "https://" + url
-
             self.desktop.open(url)
-            # Give it time to load
+
             time.sleep(2)
             self.logger.log(f"Opening URL: {url}")
             return f"Opened URL: {url}"

cua2-core/src/cua2_core/services/agent_utils/get_model.py

@@ -2,10 +2,6 @@ from smolagents import InferenceClientModel, Model
 
 # Available model IDs
 AVAILABLE_MODELS = [
-    "Qwen/Qwen3-VL-2B-Instruct",
-    "Qwen/Qwen3-VL-2B-Thinking",
-    "Qwen/Qwen3-VL-4B-Instruct",
-    "Qwen/Qwen3-VL-4B-Thinking",
     "Qwen/Qwen3-VL-8B-Instruct",
     "Qwen/Qwen3-VL-8B-Thinking",
     "Qwen/Qwen3-VL-30B-A3B-Instruct",

cua2-core/src/cua2_core/services/instruction_service.py

@@ -0,0 +1,173 @@
+import logging
+import random
+import time
+
+from cua2_core.services.agent_utils.get_model import AVAILABLE_MODELS, get_model
+from smolagents import ChatMessage, Model
+
+logger = logging.getLogger(__name__)
+
+
+class InstructionService:
+    """Service for generating task instructions using LLM models"""
+
+    available_models = AVAILABLE_MODELS
+    seed_topics = [
+        "web browsing",
+        "email management",
+        "calendar scheduling",
+        "file management",
+        "note-taking",
+        "system settings",
+        "text editing",
+        "terminal commands",
+    ]
+
+    prompt_templates = [
+        (
+            "Generate a clear and specific task instruction for a desktop automation agent. "
+            "The task should involve {topic} and be completable using a desktop computer. "
+            "Do not assume any pre-existing files, emails, or resources exist on the system. "
+            "Return only the task instruction, nothing else. Keep it simple and focused on a single action."
+        ),
+        (
+            "Create a practical task instruction for desktop automation related to {topic}. "
+            "The task should be straightforward and achievable in one application. "
+            "Do not reference specific files or resources that may not exist locally. "
+            "Provide only the task description without any additional explanation."
+        ),
+        (
+            "Generate a specific {topic} task that a desktop automation agent can perform. "
+            "The task should be concrete and not require multiple applications. "
+            "Avoid assuming pre-existing documents, files, or local resources. "
+            "Return just the task instruction."
+        ),
+        (
+            "Provide a single, clear task instruction involving {topic} for a desktop agent. "
+            "The task should be simple and focused. "
+            "Do not assume any specific files or resources already exist on the computer. "
+            "Output only the instruction."
+        ),
+        (
+            "Think of a realistic {topic} task suitable for desktop automation. "
+            "Keep it simple and achievable in one application. "
+            "The task should not depend on pre-existing local files or resources. "
+            "Return only the task."
+        ),
+    ]
+
+    web_browsing_templates = [
+        (
+            "Generate a clear and specific web browsing task instruction for a desktop automation agent. "
+            "The task should be goal-centric, focused on retrieving information or performing an action online. "
+            "You can specify a URL or website to visit. "
+            "Return only the task instruction, nothing else. Keep it simple and focused on a single goal."
+        ),
+        (
+            "Create a practical web browsing task for desktop automation. "
+            "The task should focus on finding specific information or completing an online action. "
+            "Include a specific URL or website name if relevant to the goal. "
+            "Provide only the task description without any additional explanation."
+        ),
+        (
+            "Generate a specific web browsing task that a desktop automation agent can perform. "
+            "The task should be about retrieving information or performing an action on a website. "
+            "You may specify URLs or web addresses. Keep it concrete and single-purpose. "
+            "Return just the task instruction."
+        ),
+        (
+            "Provide a goal-oriented web browsing task instruction for a desktop agent. "
+            "Focus on what information to find or what action to perform online. "
+            "Specify a URL or website if it helps achieve the goal. "
+            "Output only the instruction."
+        ),
+        (
+            "Think of a realistic web browsing task suitable for desktop automation. "
+            "The task should be about accessing online information or performing a web-based action. "
+            "Include specific URLs or websites as needed. Keep it simple and goal-focused. "
+            "Return only the task."
+        ),
+    ]
+
+    default_prompt = (
+        "Generate a clear and specific task instruction for a desktop automation agent. "
+        "The task should be something that can be completed using a desktop computer, "
+        "such as opening applications, browsing websites, or manipulating files. "
+        "Do not assume any pre-existing files, emails, or resources exist on the system. "
+        "Return only the task instruction, nothing else. the instruction must be not to complexe and not multi-app task. "
+    )
+
+    @staticmethod
+    def get_random_prompt() -> str:
+        """
+        Generate a random prompt by selecting a random topic and template.
+        Uses special templates for web browsing that allow URL specification.
+        """
+        random.seed(time.time_ns())
+
+        topic = random.choice(InstructionService.seed_topics)
+
+        if topic == "web browsing":
+            template = random.choice(InstructionService.web_browsing_templates)
+            return template
+
+        template = random.choice(InstructionService.prompt_templates)
+        return template.format(topic=topic)
+
+    @staticmethod
+    def generate_instruction(
+        model_id: str, prompt: str | None = None, use_random: bool = True
+    ) -> str:
+        """
+        Generate a task instruction using the specified model
+
+        Args:
+            model_id: The ID of the model to use
+            prompt: Optional custom prompt. If None, uses default or random prompt
+            use_random: If True, uses random prompts for variety. If False, uses default prompt
+        """
+
+        if model_id not in InstructionService.available_models:
+            available_models_str = ", ".join(InstructionService.available_models)
+            raise ValueError(
+                f"Invalid model_id '{model_id}'. Must be one of: {available_models_str}"
+            )
+
+        try:
+            logger.info(f"Generating instruction with model: {model_id}")
+
+            model: Model = get_model(model_id)
+
+            if prompt:
+                generation_prompt = prompt
+            elif use_random:
+                generation_prompt = InstructionService.get_random_prompt()
+            else:
+                generation_prompt = InstructionService.default_prompt
+
+            instruction = model([ChatMessage(role="user", content=generation_prompt)])
+            logger.info(
+                f"Successfully generated instruction with {model_id}: {instruction.content[:100]}..."
+            )
+            return instruction.content
+
+        except Exception as e:
+            logger.error(f"Error generating instruction with {model_id}: {str(e)}")
+            raise Exception(f"Failed to generate instruction: {str(e)}")
+
+    @staticmethod
+    def get_available_models() -> list[str]:
+        """Get the list of available model IDs"""
+        return InstructionService.available_models
+
+    @staticmethod
+    def get_random_topic() -> str:
+        """Get a random topic from the seed topics"""
+        return random.choice(InstructionService.seed_topics)
+
+
+if __name__ == "__main__":
+    instruction = InstructionService.generate_instruction(
+        model_id="Qwen/Qwen3-VL-8B-Instruct"
+    )
+    print(instruction)

cua2-core/tests/test_routes.py

@@ -1,4 +1,4 @@
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
 
 import pytest
 from cua2_core.models.models import AvailableModelsResponse, UpdateStepResponse
@@ -95,7 +95,7 @@ class TestGetAvailableModels:
 
         # Check for some specific models
         expected_models = [
-            "Qwen/Qwen3-VL-2B-Instruct",
+            "Qwen/Qwen3-VL-8B-Instruct",
             "Qwen/Qwen3-VL-30B-A3B-Instruct",
         ]
 
@@ -284,6 +284,53 @@ class TestUpdateTraceStep:
         assert update_response.message == "Step updated successfully"
 
 
+class TestGenerateInstruction:
+    """Test suite for POST /generate-instruction endpoint"""
+
+    @patch("cua2_core.routes.routes.InstructionService.generate_instruction")
+    def test_generate_instruction_success(self, mock_generate, client):
+        """Test successful instruction generation with mocked model"""
+        # Mock the instruction generation
+        mock_instruction = "Open Google Chrome and navigate to example.com"
+        mock_generate.return_value = mock_instruction
+
+        request_data = {
+            "model_id": "Qwen/Qwen3-VL-8B-Instruct",
+            "prompt": "Generate a web browsing task",
+        }
+
+        response = client.post("/generate-instruction", json=request_data)
+
+        assert response.status_code == 200
+        data = response.json()
+
+        assert data["instruction"] == mock_instruction
+        assert data["model_id"] == request_data["model_id"]
+
+        # Verify the service was called correctly
+        mock_generate.assert_called_once_with(
+            model_id=request_data["model_id"], prompt=request_data["prompt"]
+        )
+
+    @patch("cua2_core.routes.routes.InstructionService.generate_instruction")
+    def test_generate_instruction_invalid_model(self, mock_generate, client):
+        """Test instruction generation with invalid model_id"""
+        # Mock the service to raise ValueError for invalid model
+        mock_generate.side_effect = ValueError(
+            "Invalid model_id 'invalid-model'. Must be one of: Qwen/Qwen3-VL-2B-Instruct, ..."
+        )
+
+        request_data = {
+            "model_id": "invalid-model",
+            "prompt": "Generate a task",
+        }
+
+        response = client.post("/generate-instruction", json=request_data)
+
+        assert response.status_code == 400
+        assert "Invalid model_id" in response.json()["detail"]
+
+
 class TestRoutesIntegration:
     """Integration tests for multiple routes"""
 

cua2-front/src/types/agent.ts

@@ -97,3 +97,13 @@ export interface UpdateStepResponse {
   success: boolean;
   message: string;
 }
+
+export interface GenerateInstructionRequest {
+  model_id: string;
+  prompt?: string;
+}
+
+export interface GenerateInstructionResponse {
+  instruction: string;
+  model_id: string;
+}