Demo-2025 / mission_planner.py
zye0616's picture
update: added yolo v8 defense
4cb2d06
from __future__ import annotations
import json
import logging
from dataclasses import asdict, dataclass
from typing import Dict, List, Tuple
from prompt import mission_planner_system_prompt, mission_planner_user_prompt
from utils.openai_client import get_openai_client
YOLO_CLASSES: Tuple[str, ...] = (
"person",
"bicycle",
"car",
"motorcycle",
"airplane",
"bus",
"train",
"truck",
"boat",
"traffic light",
"fire hydrant",
"stop sign",
"parking meter",
"bench",
"bird",
"cat",
"dog",
"horse",
"sheep",
"cow",
"elephant",
"bear",
"zebra",
"giraffe",
"backpack",
"umbrella",
"handbag",
"tie",
"suitcase",
"frisbee",
"skis",
"snowboard",
"sports ball",
"kite",
"baseball bat",
"baseball glove",
"skateboard",
"surfboard",
"tennis racket",
"bottle",
"wine glass",
"cup",
"fork",
"knife",
"spoon",
"bowl",
"banana",
"apple",
"sandwich",
"orange",
"broccoli",
"carrot",
"hot dog",
"pizza",
"donut",
"cake",
"chair",
"couch",
"potted plant",
"bed",
"dining table",
"toilet",
"tv",
"laptop",
"mouse",
"remote",
"keyboard",
"cell phone",
"microwave",
"oven",
"toaster",
"sink",
"refrigerator",
"book",
"clock",
"vase",
"scissors",
"teddy bear",
"hair drier",
"toothbrush",
)
DEFAULT_OPENAI_MODEL = "gpt-4o-mini"
@dataclass
class MissionClass:
name: str
score: float
rationale: str
@dataclass
class MissionPlan:
mission: str
relevant_classes: List[MissionClass]
def queries(self) -> List[str]:
return [entry.name for entry in self.relevant_classes]
def to_dict(self) -> dict:
return {
"mission": self.mission,
"classes": [asdict(entry) for entry in self.relevant_classes],
}
def to_json(self) -> str:
return json.dumps(self.to_dict())
class MissionReasoner:
def __init__(
self,
*,
model_name: str = DEFAULT_OPENAI_MODEL,
top_k: int = 10,
) -> None:
self._model_name = model_name
self._top_k = top_k
def plan(self, mission: str) -> MissionPlan:
mission = (mission or "").strip()
if not mission:
raise ValueError("Mission prompt cannot be empty.")
response_payload = self._query_llm(mission)
relevant = self._parse_plan(response_payload, fallback_mission=mission)
return MissionPlan(mission=response_payload.get("mission", mission), relevant_classes=relevant[: self._top_k])
def _query_llm(self, mission: str) -> Dict[str, object]:
client = get_openai_client()
system_prompt = mission_planner_system_prompt()
user_prompt = mission_planner_user_prompt(mission, YOLO_CLASSES, self._top_k)
completion = client.chat.completions.create(
model=self._model_name,
temperature=0.2,
response_format={"type": "json_object"},
messages=[
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
],
)
content = completion.choices[0].message.content or "{}"
try:
return json.loads(content)
except json.JSONDecodeError:
logging.exception("LLM returned non-JSON content: %s", content)
return {"mission": mission, "classes": []}
def _parse_plan(self, payload: Dict[str, object], fallback_mission: str) -> List[MissionClass]:
entries = payload.get("classes") or payload.get("relevant_classes") or []
mission = payload.get("mission") or fallback_mission
parsed: List[MissionClass] = []
seen = set()
for entry in entries:
if not isinstance(entry, dict):
continue
name = str(entry.get("name") or "").strip()
if not name or name not in YOLO_CLASSES or name in seen:
continue
seen.add(name)
score_raw = entry.get("score")
try:
score = float(score_raw)
except (TypeError, ValueError):
score = 0.5
rationale = str(entry.get("rationale") or f"Track '{name}' for mission '{mission}'.")
parsed.append(MissionClass(name=name, score=max(0.0, min(1.0, score)), rationale=rationale))
if not parsed:
logging.warning("LLM returned no usable classes. Falling back to default YOLO list.")
parsed = [
MissionClass(
name=label,
score=1.0 - (idx * 0.05),
rationale=f"Fallback selection for mission '{mission}'.",
)
for idx, label in enumerate(YOLO_CLASSES[: self._top_k])
]
return parsed
_REASONER: MissionReasoner | None = None
def get_mission_plan(mission: str) -> MissionPlan:
global _REASONER
if _REASONER is None:
_REASONER = MissionReasoner()
return _REASONER.plan(mission)