generated, filled in and verfied 250 eval questions

eval/amp_batch_solve.py

@@ -1,106 +0,0 @@
-import asyncio
-import json
-import os
-from pathlib import Path
-import threading
-
-from amp_sdk import AmpOptions, execute
-
-# Thread-safe file writing
-file_lock = threading.Lock()
-
-
-async def solve_task(
-    question: str, difficulty: str, task_idx: int, total: int, semaphore: asyncio.Semaphore
-) -> dict:
-    """Solve a single task using Amp SDK."""
-    async with semaphore:
-        print(f"[{task_idx}/{total}] Starting: {question[:60]}...")
-        
-        messages = []
-        solution = None
-        
-        try:
-            async for message in execute(
-                question,
-                AmpOptions(
-                    cwd=os.getcwd(),
-                    visibility="workspace",
-                    dangerously_allow_all=True,
-                ),
-            ):
-                messages.append(message.model_dump())
-                
-                # Extract the final text response as solution
-                if message.type == "assistant":
-                    content = message.message.get("content", [])
-                    for item in content:
-                        if isinstance(item, dict) and item.get("type") == "text":
-                            solution = item.get("text")
-                elif message.type == "result":
-                    if message.result:
-                        solution = message.result
-            
-            print(f"[{task_idx}/{total}] ✓ Done: {question[:60]}...")
-            return {
-                "question": question,
-                "difficulty": difficulty,
-                "solution": solution,
-                "messages": messages,
-            }
-        except Exception as e:
-            print(f"[{task_idx}/{total}] ✗ Error: {e}")
-            return {
-                "question": question,
-                "difficulty": difficulty,
-                "solution": None,
-                "messages": messages,
-                "error": str(e),
-            }
-
-
-def write_result(output_path: Path, result: dict):
-    """Thread-safe write to output file."""
-    with file_lock:
-        with open(output_path, "a") as f:
-            f.write(json.dumps(result) + "\n")
-
-
-async def main():
-    # Load tasks
-    tasks_path = Path(__file__).parent / "generated_tasks_with_difficulty.json"
-    with open(tasks_path) as f:
-        tasks = json.load(f)
-    
-    # Output file - clear it first
-    output_path = Path(__file__).parent / "solved_tasks.jsonl"
-    output_path.write_text("")
-    
-    # Semaphore to limit concurrency
-    max_concurrent = 20
-    semaphore = asyncio.Semaphore(max_concurrent)
-    
-    total = len(tasks)
-    print(f"Processing {total} tasks with {max_concurrent} concurrent agents...")
-    
-    async def process_and_save(question: str, difficulty: str, idx: int):
-        result = await solve_task(question, difficulty, idx, total, semaphore)
-        write_result(output_path, result)
-        return result
-    
-    # Create all tasks
-    coroutines = [
-        process_and_save(question, difficulty, i + 1)
-        for i, (question, difficulty) in enumerate(tasks.items())
-    ]
-    
-    # Run all concurrently (semaphore limits actual parallelism)
-    results = await asyncio.gather(*coroutines, return_exceptions=True)
-    
-    successful = sum(1 for r in results if isinstance(r, dict) and "error" not in r)
-    print(f"\nCompleted: {successful}/{total} successful")
-    print(f"Results saved to {output_path}")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

eval/amp_solve.py

@@ -1,31 +0,0 @@
-import asyncio
-import os
-
-from amp_sdk import AmpOptions, execute
-
-prompt = """
-what account am I logged in as?
-"""
-
-
-async def main():
-    # Use the toolbox directory to share tools with Amp
-    toolbox_dir = os.path.join(os.getcwd(), "toolbox")
-    messages = []
-    async for message in execute(
-        prompt,
-        AmpOptions(
-            cwd=os.getcwd(),
-            toolbox=toolbox_dir,
-            visibility="workspace",
-            dangerously_allow_all=True,
-        ),
-    ):
-        messages.append(message)
-
-    for msg in messages:
-        print(msg.model_dump_json(indent=2))
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

eval/check_completeness.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""
+Minimal script to check if tasks in solved_tasks.jsonl were fully completed and verified.
+Uses an LLM to assess completion status and adds the result to each row.
+"""
+
+import argparse
+import json
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import litellm
+from dotenv import load_dotenv
+from pydantic import BaseModel
+
+load_dotenv()
+
+
+class CompletionCheck(BaseModel):
+    reasoning: str
+    completed: bool
+    verified: bool
+
+
+PROMPT = """You are evaluating whether an AI agent fully completed a task AND verified its completion.
+
+Task: {question}
+
+Agent's final answer: {solution}
+
+Agent's trace (tool calls and responses):
+{trace}
+
+Evaluate:
+1. **completed**: Did the agent actually complete the task? (not just explain what could be done, but actually do it)
+2. **verified**: Did the agent verify/confirm that the task was completed correctly? (e.g., checked output, validated results, confirmed success)
+
+Be strict:
+- If the agent asked for more information or said "please provide...", it's NOT completed.
+- If the agent only explained how to do something but didn't do it, it's NOT completed.
+- If the agent just made a plan of how to complete it but didn't do it, it's NOT completed.
+- If there's an error in the trace and no recovery, it's NOT completed.
+- If the agent didn't check/confirm the code/command completed succesfully or the result is correct somehow, it's NOT verified.
+
+Return JSON with: completed (bool), verified (bool), reasoning (brief explanation)."""
+
+
+def format_trace(messages: list) -> str:
+    """Format messages trace for the prompt."""
+    if not messages:
+        return "(No trace)"
+
+    parts = []
+    for msg in messages:
+        role = msg.get("role", "unknown")
+        if role == "system":
+            continue
+
+        content = msg.get("content", "")
+        tool_calls = msg.get("tool_calls", [])
+
+        if tool_calls:
+            for tc in tool_calls:
+                if isinstance(tc, dict) and "function" in tc:
+                    name = tc["function"].get("name", "?")
+                    parts.append(f"[TOOL CALL] {name}")
+
+        if content:
+            # Truncate long content
+            if len(content) > 5000:
+                content = content[:4000] + "..." + content[-1000:]
+            parts.append(f"[{role.upper()}] {content}")
+
+    return "\n".join(parts) if parts else "(Empty trace)"
+
+
+def check_row(row: dict, model: str) -> CompletionCheck | None:
+    """Check if a single task was completed and verified."""
+    prompt = PROMPT.format(
+        question=row["question"],
+        solution=row.get("solution", "(No solution)"),
+        trace=format_trace(row.get("messages", [])),
+    )
+
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            response_format=CompletionCheck,
+            timeout=60,
+        )
+        return CompletionCheck.model_validate_json(response.choices[0].message.content)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return None
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Check task completion status")
+    parser.add_argument("--infile", type=str, default="eval/solved_tasks.jsonl")
+    parser.add_argument(
+        "--outfile", type=str, default="eval/solved_tasks_checked.jsonl"
+    )
+    parser.add_argument(
+        "--model", type=str, default="anthropic/claude-sonnet-4-5-20250929"
+    )
+    parser.add_argument("--max-concurrent", type=int, default=30)
+    args = parser.parse_args()
+
+    # Load data
+    print(f"Loading {args.infile}...")
+    rows = []
+    with open(args.infile) as f:
+        for line in f:
+            rows.append(json.loads(line))
+    print(f"Loaded {len(rows)} rows")
+
+    # Process in parallel
+    print(f"Checking completion with {args.model}...")
+    with ThreadPoolExecutor(max_workers=args.max_concurrent) as executor:
+        futures = {
+            executor.submit(check_row, row, args.model): i for i, row in enumerate(rows)
+        }
+        results = [None] * len(rows)
+
+        for future in as_completed(futures):
+            idx = futures[future]
+            results[idx] = future.result()
+            print(
+                f"Done: {sum(1 for r in results if r is not None)}/{len(rows)}",
+                end="\r",
+            )
+
+    print()
+
+    # Merge results
+    output_rows = []
+    for row, result in zip(rows, results):
+        if result:
+            row["task_completed"] = result.completed
+            row["task_verified"] = result.verified
+            row["completion_reasoning"] = result.reasoning
+        else:
+            row["task_completed"] = None
+            row["task_verified"] = None
+            row["completion_reasoning"] = "Error during check"
+        output_rows.append(row)
+
+    # Write output
+    print(f"Writing to {args.outfile}...")
+    with open(args.outfile, "w") as f:
+        for row in output_rows:
+            f.write(json.dumps(row, default=str) + "\n")
+
+    # Summary
+    completed = sum(1 for r in results if r and r.completed)
+    verified = sum(1 for r in results if r and r.verified)
+    print("\nSummary:")
+    print(f"  Completed: {completed}/{len(rows)}")
+    print(f"  Verified: {verified}/{len(rows)}")
+
+
+if __name__ == "__main__":
+    main()

eval/claude_batch_solve.py

@@ -0,0 +1,230 @@
+import asyncio
+import json
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+from claude_agent_sdk import (
+    AssistantMessage,
+    ClaudeAgentOptions,
+    ResultMessage,
+    SystemMessage,
+    TextBlock,
+    ToolResultBlock,
+    ToolUseBlock,
+    UserMessage,
+    query,
+)
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Thread-safe file writing
+file_lock = threading.Lock()
+
+
+def convert_message_to_chat_format(message: Any) -> dict | None:
+    """Convert SDK message to standard chat format with role/content/tool_calls."""
+
+    if isinstance(message, SystemMessage):
+        # Extract tools list from init data for system message
+        if message.subtype == "init":
+            tools = message.data.get("tools", [])
+            tools_desc = "\n".join(f"- {tool}" for tool in tools)
+            return {
+                "role": "system",
+                "content": f"You are a helpful assistant with access to the following tools:\n{tools_desc}",
+            }
+        return None
+
+    elif isinstance(message, AssistantMessage):
+        text_content = ""
+        tool_calls = []
+
+        for block in message.content:
+            if isinstance(block, TextBlock):
+                text_content += block.text
+            elif isinstance(block, ToolUseBlock):
+                tool_calls.append(
+                    {
+                        "id": block.id,
+                        "function": {
+                            "name": block.name,
+                            "arguments": block.input,
+                        },
+                    }
+                )
+
+        result = {"role": "assistant", "content": text_content}
+        if tool_calls:
+            result["tool_calls"] = tool_calls
+        return result
+
+    elif isinstance(message, UserMessage):
+        # UserMessage can contain tool results or text
+        if isinstance(message.content, str):
+            return {"role": "user", "content": message.content}
+        elif isinstance(message.content, list):
+            # Check for tool results
+            tool_results = []
+            text_content = ""
+            for block in message.content:
+                if isinstance(block, ToolResultBlock):
+                    # Format tool result content
+                    if isinstance(block.content, str):
+                        content = block.content
+                    elif isinstance(block.content, list):
+                        content = json.dumps(block.content)
+                    else:
+                        content = str(block.content) if block.content else ""
+
+                    tool_results.append(
+                        {
+                            "tool_use_id": block.tool_use_id,
+                            "content": content,
+                            "is_error": block.is_error,
+                        }
+                    )
+                elif isinstance(block, TextBlock):
+                    text_content += block.text
+
+            if tool_results:
+                return {
+                    "role": "user",
+                    "content": f"<tool_response>\n{json.dumps(tool_results, indent=2)}\n</tool_response>",
+                }
+            else:
+                return {"role": "user", "content": text_content}
+        return None
+
+    elif isinstance(message, ResultMessage):
+        # ResultMessage is metadata, not a conversation message
+        return None
+
+    return None
+
+
+async def solve_task(
+    question: str,
+    difficulty: str,
+    task_idx: int,
+    total: int,
+    semaphore: asyncio.Semaphore,
+) -> dict:
+    """Solve a single task using Claude Agent SDK."""
+    async with semaphore:
+        print(f"[{task_idx}/{total}] Starting: {question[:60]}...")
+
+        messages = []
+        solution = None
+
+        try:
+            async for message in query(
+                prompt=question,
+                options=ClaudeAgentOptions(
+                    cwd=os.getcwd(),
+                    permission_mode="bypassPermissions",
+                    disallowed_tools=["Write", "Edit", "Bash", "Glob", "Grep"],
+                    mcp_servers={
+                        "huggingface": {
+                            "type": "http",
+                            "url": "https://huggingface.co/mcp",
+                            "headers": {
+                                "Authorization": f"Bearer {os.environ['HF_TOKEN']}"
+                            },
+                        }
+                    },
+                ),
+            ):
+                # Convert to chat format and append if valid
+                chat_msg = convert_message_to_chat_format(message)
+                if chat_msg:
+                    messages.append(chat_msg)
+
+                # Extract text from assistant messages
+                if isinstance(message, AssistantMessage):
+                    for block in message.content:
+                        if isinstance(block, TextBlock):
+                            solution = block.text
+                # Check for result messages
+                elif isinstance(message, ResultMessage):
+                    if message.is_error:
+                        print(f"[{task_idx}/{total}] ✗ Agent error: {message.subtype}")
+                        return {
+                            "question": question,
+                            "difficulty": difficulty,
+                            "solution": None,
+                            "messages": messages,
+                            "error": f"Agent error: {message.subtype}",
+                        }
+                    elif message.result:
+                        solution = message.result
+
+            print(f"[{task_idx}/{total}] ✓ Done: {question[:60]}...")
+            return {
+                "question": question,
+                "difficulty": difficulty,
+                "solution": solution,
+                "messages": messages,
+                "error": None,
+            }
+        except Exception as e:
+            print(f"[{task_idx}/{total}] ✗ Error: {e}")
+            return {
+                "question": question,
+                "difficulty": difficulty,
+                "solution": None,
+                "messages": messages,
+                "error": str(e),
+            }
+
+
+def write_result(output_path: Path, result: dict):
+    """Thread-safe write to output file."""
+    with file_lock:
+        with open(output_path, "a") as f:
+            f.write(json.dumps(result) + "\n")
+
+
+async def main():
+    # Load tasks from filled_tasks.jsonl
+    tasks_path = Path(__file__).parent / "filled_tasks.jsonl"
+    tasks = []
+    with open(tasks_path) as f:
+        for line in f:
+            tasks.append(json.loads(line))
+
+    # Output file - clear it first
+    output_path = Path(__file__).parent / "solved_tasks.jsonl"
+    output_path.write_text("")
+
+    # Semaphore to limit concurrency
+    max_concurrent = 5
+    semaphore = asyncio.Semaphore(max_concurrent)
+
+    total = len(tasks)
+    print(f"Processing {total} tasks with {max_concurrent} concurrent agents...")
+
+    async def process_and_save(task: dict, idx: int):
+        result = await solve_task(
+            task["question"], task["difficulty"], idx, total, semaphore
+        )
+        write_result(output_path, result)
+        return result
+
+    # Create all tasks
+    coroutines = [process_and_save(task, i + 1) for i, task in enumerate(tasks)]
+
+    # Run all concurrently (semaphore limits actual parallelism)
+    results = await asyncio.gather(*coroutines, return_exceptions=True)
+
+    successful = sum(
+        1 for r in results if isinstance(r, dict) and r.get("error") is None
+    )
+    print(f"\nCompleted: {successful}/{total} successful")
+    print(f"Results saved to {output_path}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

eval/generate_rubrics.py

@@ -38,19 +38,22 @@ load_dotenv()
 # Rubric generation prompt template based on RaR paper
 
 
-PROMPT_TEMPLATE = """You are an expert rubric writer. Your job is to generate a self-contained set of evaluation criteria (“rubrics”) for judging
-how good, helpful and complete an agent's trajectory is to a given user question/request. 
+PROMPT_TEMPLATE = """You are an expert rubric writer. Your job is to generate a self-contained set of evaluation criteria ("rubrics") for judging how good, helpful and complete an agent's trajectory is to a given user question/request. 
 
 Rubrics can cover aspects of a response such as, but not limited to, factual correctness, helpfulness, completeness, harmlessness, correctness of using Hugging Face best practices (based on HF documentation), depth of
 reasoning, contextual relevance and usefulness. Each item must be self-contained – non expert readers should not need to
-infer anything or consult external information. Begin each description with its category: “Essential Criteria: . . . ”, “Important
-Criteria: . . . ”, “Optional Criteria: . . . ”, or “Pitfall Criteria: Does not mention . . . ”.
+infer anything or consult external information. Begin each description with its category: "Essential Criteria: . . . ", "Important
+Criteria: . . . ", "Optional Criteria: . . . ", or "Pitfall Criteria: Does not mention . . . ".
 
 
-Inputs: !!!
+Inputs:
 - question: <<<{question}>>>
-- reference_answer (ideal solution): <<<{reference_answer}>>>
-- thread: <<<{thread}>>>
+- example_solution (NOT ground truth - just an okay attempt): <<<{example_solution}>>>
+- example_trace (NOT ground truth - just an okay attempt showing what tool usage might look like): <<<{example_trace}>>>
+
+IMPORTANT: The example_solution and example_trace provided are NOT ground truth or ideal solutions. They represent 
+an attempt at solving the task - they give you a general idea of the shape of the problem and what tool usage 
+might look like, but they contain mistakes and incomplete solutions, suboptimal approaches, or incomplete answers. Your rubrics MUST be designed to fairly grade a PERFECT solution. The perfect solution is complete in all aspects of solving the task and verifing it's correctness before giving the final answer. It tells the user what was done and why, and provides the final answer clearly answering the user's question.
 
 Total items:
 • Choose 7–20 rubric items based on the complexity of the question.
@@ -69,47 +72,97 @@ Category guidance:
 • Important: Key reasoning, completeness, or clarity; strongly affects quality and usefulness (weight 3–4).
 • Optional: Helpfulness in educating the user or providing extra depth; nice to have but not deal-breaking (weight 1–2).
 • Pitfall: Common mistakes or omissions specific to this prompt—identify things a respondent often forgets or misstates.
-Each Pitfall description must begin with “Pitfall Criteria: Does not mention . . . ” or “Pitfall Criteria: Recommends . . . ”
+Each Pitfall description must begin with "Pitfall Criteria: Does not mention . . . " or "Pitfall Criteria: Recommends . . . "
 and use weight –1 or –2.
 
 To ensure self-contained guidance:
-• When referring to answer choices, explicitly say “Identifies (A)”, “Identifies (B)”, etc., rather than vague phrasing.
+• When referring to answer choices, explicitly say "Identifies (A)", "Identifies (B)", etc., rather than vague phrasing.
 • If the format requires an action like calling a tool or launching a training run, include a rubric item such as:
 – Essential Criteria: Includes a clear statement "Launches the training with hf-jobs.".
 • If reasoning should precede the answer, include a rubric like:
 – Important Criteria: Presents the explanation and reasoning before stating the final answer.
 • If brevity is valued, include a rubric like:
 – Optional Criteria: Remains concise and avoids unnecessary detail.
-• If the question context demands mention of specific findings/best practices, include that explicitly (e.g., “Essential Criteria: Mentions
-that training data must be in "messages" column for LLM training”).
+• If the question context demands mention of specific findings/best practices, include that explicitly (e.g., "Essential Criteria: Mentions
+that training data must be in "messages" column for LLM training").
 
 Output: Provide a JSON array of rubric objects. Each object must contain exactly three keys—title, description, and weight.
-Do not copy large blocks of the question or reference_answer into the text. Each description must begin with its category
+Do not copy large blocks of the question or example_solution into the text. Each description must begin with its category
 prefix, and no extra keys are allowed.
-Now, given the question, thread and reference_answer, generate the rubric as described. The reference answer is an good and helpful response
-but not necessarily exhaustive; use it only as guidance."""
+
+Remember: The example_solution and example_trace are NOT ideal answers - they are just rough attempts to show the 
+general approach. Design rubrics that can fairly evaluate any solution, including ones that are better than the example."""
 
 
 def build_prompt(
-    question: str, reference_answer: str, thread: List[Dict[str, str]]
+    question: str,
+    example_solution: str,
+    example_trace: List[Dict[str, Any]],
 ) -> List[Dict[str, str]]:
     """
     Build the messages list for LiteLLM completion.
 
     Args:
         question: The question/task to evaluate
-        reference_answer: The reference/accepted solution
+        difficulty: The difficulty level of the task
+        example_solution: An example solution attempt (not ground truth)
+        example_trace: The agent's message trace showing tool usage
 
     Returns:
         List of message dicts for LiteLLM
     """
+    # Format the trace for readability - only include key parts
+    formatted_trace = format_trace_for_prompt(example_trace)
+
     prompt = PROMPT_TEMPLATE.format(
-        question=question, reference_answer=reference_answer, thread=thread
+        question=question,
+        example_solution=example_solution,
+        example_trace=formatted_trace,
     )
 
     return [{"role": "user", "content": prompt}]
 
 
+def format_trace_for_prompt(messages: List[Dict[str, Any]]) -> str:
+    """
+    Format the agent message trace for inclusion in the prompt.
+    Extracts key information while keeping it readable.
+    """
+    if not messages:
+        return "(No trace available)"
+
+    formatted_parts = []
+    for msg in messages:
+        role = msg.get("role", "unknown")
+        content = msg.get("content", "")
+
+        # Skip system messages
+        if role == "system":
+            continue
+
+        # Handle tool calls
+        if "tool_calls" in msg and msg["tool_calls"]:
+            tool_info = []
+            for tc in msg["tool_calls"]:
+                if isinstance(tc, dict) and "function" in tc:
+                    func = tc["function"]
+                    tool_name = func.get("name", "unknown_tool")
+                    tool_info.append(f"  - Called: {tool_name}")
+            if tool_info:
+                formatted_parts.append(
+                    "[Assistant Tool Calls]\n" + "\n".join(tool_info)
+                )
+
+        # Handle regular content
+        if content:
+            # Truncate very long content
+            if len(content) > 500:
+                content = content[:500] + "... (truncated)"
+            formatted_parts.append(f"[{role.title()}]\n{content}")
+
+    return "\n\n".join(formatted_parts) if formatted_parts else "(Empty trace)"
+
+
 def validate_rubric(rubric_list: List[Dict[str, Any]]) -> bool:
     """
     Validate that rubric meets basic requirements.
@@ -151,8 +204,7 @@ def generate_rubric(row: pd.Series, model: str, timeout: int = 120) -> Dict[str,
     Generate rubric for a single question using LiteLLM.
 
     Args:
-        question: The question text
-        reference_answer: The reference solution
+        row: DataFrame row containing question, difficulty, solution, and messages
         model: Model name for LiteLLM
         timeout: Request timeout in seconds
 
@@ -160,7 +212,11 @@ def generate_rubric(row: pd.Series, model: str, timeout: int = 120) -> Dict[str,
         Dict with rubric_list and rubric_count, or None on failure
     """
 
-    messages = build_prompt(row["question"], row["solution"], row["thread"])
+    messages = build_prompt(
+        question=row["question"],
+        example_solution=row["solution"],
+        example_trace=row.get("messages", []),
+    )
 
     try:
         response = litellm.completion(
@@ -206,17 +262,19 @@ def load_input_data(infile: str) -> pd.DataFrame:
 
     # Validate required columns
     required_cols = [
-        "discussion_title",
-        "discussion_url",
         "question",
-        "thread",
         "solution",
     ]
+    optional_cols = ["difficulty", "messages", "error"]
     missing_cols = [col for col in required_cols if col not in df.columns]
 
     if missing_cols:
         raise ValueError(f"Missing required columns: {missing_cols}")
 
+    # Log available optional columns
+    available_optional = [col for col in optional_cols if col in df.columns]
+    print(f"Found optional columns: {available_optional}")
+
     return df
 
 
@@ -307,6 +365,7 @@ def main():
 
         # Merge with original data
         output_row = row.to_dict()
+        output_row["messages"] = json.dumps(output_row["messages"])
         output_row["rubric"] = rubric_result
         output_rows.append(output_row)
         success_count += 1