api samples input just List[dict]

api/app.py

@@ -7,8 +7,6 @@ from concurrent.futures import ProcessPoolExecutor, as_completed
 from typing import Dict, List, Tuple
 import gc
 
-from pydantic import BaseModel
-
 from fastapi import FastAPI
 from fastapi.responses import RedirectResponse
 
@@ -17,14 +15,6 @@ from api.code_execution import untrusted_check
 
 Result = Tuple[str, List[bool]]
 
-class SampleDate(BaseModel):
-    task_id: str
-    solution: str
-    code_prompt: str
-    test: str
-    entry_point: str
-    res_id: int
-
 def create_app() -> FastAPI:
 
     level = os.environ.get("LOG_LEVEL", default=logging.INFO)
@@ -43,7 +33,7 @@ def create_app() -> FastAPI:
 
     @app.post("/evaluate/")
     async def evaluate(
-        samples: List[SampleDate],
+        samples: List[dict],
         calibrate: bool = True,
         parallel: int = -1,
         min_time_limit: float = 1,

api/bigcodebench_data.py

@@ -24,6 +24,15 @@ def load_solutions(samples) -> Iterable[Dict]:
     """
     """
     for i, sample in enumerate(samples):
+        assert "task_id" in sample, "No task_id found in sample!"
+        assert "res_id" in sample, "No res_id found in sample!"
+        assert "test" in sample, "No test found in sample!"
+        assert "solution" in sample, "No solution found in sample!"
+        assert "entry_point" in sample, "No entry_point found in sample!"
+        assert isinstance(
+            sample["solution"], str
+        ), "Solution must be a string! If you have multiple solutions, please repeat the task_id."
+
         sample["_identifier"] = (
             sample["task_id"] + f" (line {i+1} )"
         )