update: added gps and location intelligence

app.py

@@ -58,6 +58,8 @@ def _schedule_cleanup(background_tasks: BackgroundTasks, path: str) -> None:
 class CachedMission:
     prompt: str
     detector: Optional[str]
+    latitude: float
+    longitude: float
     plan: MissionPlan
     created_at: float
 
@@ -77,12 +79,20 @@ def _prune_mission_cache() -> None:
         MISSION_CACHE.pop(key, None)
 
 
-def _store_mission_plan(prompt: str, detector: Optional[str], plan: MissionPlan) -> str:
+def _store_mission_plan(
+    prompt: str,
+    detector: Optional[str],
+    plan: MissionPlan,
+    latitude: float,
+    longitude: float,
+) -> str:
     _prune_mission_cache()
     mission_id = uuid4().hex
     MISSION_CACHE[mission_id] = CachedMission(
         prompt=prompt,
         detector=detector,
+        latitude=latitude,
+        longitude=longitude,
         plan=plan,
         created_at=time.time(),
     )
@@ -97,28 +107,80 @@ def _get_cached_mission(mission_id: str) -> CachedMission:
     return entry
 
 
-def _resolve_mission_plan(prompt: Optional[str], mission_id: Optional[str]) -> tuple[MissionPlan, str]:
+def _require_coordinates(
+    latitude: Optional[float],
+    longitude: Optional[float],
+) -> tuple[float, float]:
+    if latitude is None or longitude is None:
+        raise HTTPException(status_code=400, detail="Mission location coordinates are required.")
+    return float(latitude), float(longitude)
+
+
+def _resolve_mission_plan(
+    prompt: Optional[str],
+    mission_id: Optional[str],
+    latitude: Optional[float],
+    longitude: Optional[float],
+) -> tuple[MissionPlan, str]:
     if mission_id:
         cached = _get_cached_mission(mission_id)
         return cached.plan, cached.prompt
     normalized_prompt = (prompt or "").strip()
     if not normalized_prompt:
         raise HTTPException(status_code=400, detail="Mission prompt is required.")
+    _require_coordinates(latitude, longitude)
     plan = get_mission_plan(normalized_prompt)
     return plan, normalized_prompt
 
 
-def _validate_inputs(video: UploadFile | None, prompt: str | None, mission_id: str | None) -> None:
+def _validate_inputs(
+    video: UploadFile | None,
+    prompt: str | None,
+    mission_id: Optional[str],
+    latitude: Optional[float],
+    longitude: Optional[float],
+) -> None:
     if video is None:
         raise HTTPException(status_code=400, detail="Video file is required.")
-    if not prompt and not mission_id:
+    if mission_id:
+        return
+    if not prompt:
         raise HTTPException(status_code=400, detail="Mission prompt is required.")
+    _require_coordinates(latitude, longitude)
+
+
+def _location_only_prompt(latitude: float, longitude: float) -> str:
+    return (
+        "Threat reconnaissance mission. "
+        f"Identify and prioritize potential hostile or suspicious object classes around latitude {latitude:.4f}, "
+        f"longitude {longitude:.4f}. Consider common threats for this environment when selecting classes."
+    )
+
+
+@app.post("/location_context")
+async def location_context(
+    latitude: float = Form(...),
+    longitude: float = Form(...),
+):
+    prompt = _location_only_prompt(latitude, longitude)
+    try:
+        plan = get_mission_plan(prompt)
+    except Exception as exc:
+        logging.exception("Location-only planning failed.")
+        raise HTTPException(status_code=500, detail=str(exc))
+    return {
+        "mission_plan": plan.to_dict(),
+        "location": {"latitude": latitude, "longitude": longitude},
+        "prompt_used": prompt,
+    }
 
 
 @app.post("/mission_plan")
 async def mission_plan_endpoint(
     prompt: str = Form(...),
     detector: Optional[str] = Form(None),
+    latitude: float = Form(...),
+    longitude: float = Form(...),
 ):
     normalized_prompt = (prompt or "").strip()
     if not normalized_prompt:
@@ -129,8 +191,12 @@ async def mission_plan_endpoint(
         logging.exception("Mission planning failed.")
         raise HTTPException(status_code=500, detail=str(exc))
 
-    mission_id = _store_mission_plan(normalized_prompt, detector, plan)
-    return {"mission_id": mission_id, "mission_plan": plan.to_dict()}
+    mission_id = _store_mission_plan(normalized_prompt, detector, plan, latitude, longitude)
+    return {
+        "mission_id": mission_id,
+        "mission_plan": plan.to_dict(),
+        "location": {"latitude": latitude, "longitude": longitude},
+    }
 
 
 @app.post("/process_video")
@@ -140,8 +206,10 @@ async def process_video(
     prompt: Optional[str] = Form(None),
     mission_id: Optional[str] = Form(None),
     detector: Optional[str] = Form(None),
+    latitude: Optional[float] = Form(None),
+    longitude: Optional[float] = Form(None),
 ):
-    _validate_inputs(video, prompt, mission_id)
+    _validate_inputs(video, prompt, mission_id, latitude, longitude)
 
     try:
         input_path = _save_upload_to_tmp(video)
@@ -154,7 +222,7 @@ async def process_video(
     fd, output_path = tempfile.mkstemp(prefix="output_", suffix=".mp4", dir="/tmp")
     os.close(fd)
 
-    mission_plan, mission_prompt = _resolve_mission_plan(prompt, mission_id)
+    mission_plan, mission_prompt = _resolve_mission_plan(prompt, mission_id, latitude, longitude)
     try:
         output_path, _, _ = run_inference(
             input_path,
@@ -192,9 +260,11 @@ async def mission_summary(
     prompt: Optional[str] = Form(None),
     mission_id: Optional[str] = Form(None),
     detector: Optional[str] = Form(None),
+    latitude: Optional[float] = Form(None),
+    longitude: Optional[float] = Form(None),
 ):
-    _validate_inputs(video, prompt, mission_id)
-    mission_plan, mission_prompt = _resolve_mission_plan(prompt, mission_id)
+    _validate_inputs(video, prompt, mission_id, latitude, longitude)
+    mission_plan, mission_prompt = _resolve_mission_plan(prompt, mission_id, latitude, longitude)
     try:
         input_path = _save_upload_to_tmp(video)
     except Exception:

demo.html

@@ -4,6 +4,7 @@
 <meta charset="UTF-8">
 <title>Mission Console</title>
 
+<link rel="stylesheet" href="https://unpkg.com/leaflet/dist/leaflet.css">
 <style>
 body {
     margin: 0;
@@ -107,6 +108,11 @@ button:disabled {
     background-color: #3a3a3a;
 }
 
+.secondary-button.small {
+    padding: 8px 14px;
+    font-size: 14px;
+}
+
 .prompt-actions {
     display: flex;
     flex-wrap: wrap;
@@ -114,6 +120,40 @@ button:disabled {
     margin-top: 8px;
 }
 
+.location-section {
+    margin-top: 20px;
+    padding: 16px;
+    border: 1px solid #2c2c2c;
+    border-radius: 10px;
+    background: #141414;
+}
+
+.location-hint {
+    font-size: 13px;
+    color: #bdbdbd;
+    margin-top: 6px;
+}
+
+.location-map-container {
+    margin-top: 12px;
+    height: 260px;
+    border: 1px solid #2c2c2c;
+    border-radius: 8px;
+    overflow: hidden;
+}
+
+#missionMap {
+    height: 100%;
+    width: 100%;
+}
+
+.location-readout {
+    display: flex;
+    gap: 20px;
+    margin-top: 12px;
+    font-size: 14px;
+}
+
 .output-section {
     margin-top: 35px;
     padding: 20px;
@@ -167,6 +207,11 @@ button:disabled {
     font-size: 13px;
     min-height: 18px;
 }
+
+.status-message.small {
+    font-size: 12px;
+    margin-top: 10px;
+}
 </style>
 
 </head>
@@ -190,6 +235,20 @@ button:disabled {
         </div>
     </div>
 
+    <div class="location-section">
+        <label class="label">MISSION LOCATION</label>
+        <div class="location-hint">We will auto-detect your GPS (if allowed). Otherwise, click anywhere on the map to select the mission location.</div>
+        <div id="missionMapContainer" class="location-map-container">
+            <div id="missionMap"></div>
+        </div>
+        <div class="location-readout">
+            <div>Latitude: <span id="latitudeDisplay">--</span></div>
+            <div>Longitude: <span id="longitudeDisplay">--</span></div>
+        </div>
+        <div id="locationStatus" class="status-message small"></div>
+        <div id="locationIntel" class="status-message small" style="margin-top:6px;"></div>
+    </div>
+
     <label class="label">OBJECT DETECTOR</label>
     <select id="detectorSelect">
         <option value="owlv2" selected>OWL-V2</option>
@@ -217,11 +276,13 @@ button:disabled {
 
 </div>
 
+<script src="https://unpkg.com/leaflet/dist/leaflet.js"></script>
 <script>
 const API_BASE_URL = "https://biaslab2025-demo-2025.hf.space";
 const PROCESS_VIDEO_URL = `${API_BASE_URL}/process_video`;
 const SUMMARY_URL = `${API_BASE_URL}/mission_summary`;
 const MISSION_PLAN_URL = `${API_BASE_URL}/mission_plan`;
+const LOCATION_INTEL_URL = `${API_BASE_URL}/location_context`;
 
 const missionInputEl = document.getElementById("missionPrompt");
 const detectorSelectEl = document.getElementById("detectorSelect");
@@ -231,12 +292,21 @@ const summaryEl = document.getElementById("summary");
 const statusEl = document.getElementById("status");
 const setMissionButton = document.getElementById("setMissionButton");
 const executeButton = document.getElementById("executeButton");
+const latitudeDisplayEl = document.getElementById("latitudeDisplay");
+const longitudeDisplayEl = document.getElementById("longitudeDisplay");
+const locationStatusEl = document.getElementById("locationStatus");
+const locationIntelEl = document.getElementById("locationIntel");
 
 let originalVideoUrl = null;
 let currentMissionId = null;
 let missionReady = false;
 let missionRequestPending = false;
 let videoProcessing = false;
+let locationReady = false;
+let selectedLat = null;
+let selectedLon = null;
+let missionMap = null;
+let missionMapMarker = null;
 
 missionInputEl.addEventListener("input", handleMissionPromptEdit);
 
@@ -252,6 +322,84 @@ videoInputEl.addEventListener("change", () => {
     }
 });
 
+initializeMissionMap();
+attemptAutoGps();
+
+function initializeMissionMap() {
+    if (missionMap) {
+        return;
+    }
+    missionMap = L.map("missionMap").setView([20, 0], 2);
+    L.tileLayer("https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", {
+        maxZoom: 19,
+        attribution: "&copy; OpenStreetMap contributors",
+    }).addTo(missionMap);
+    missionMap.on("click", (event) => {
+        setSelectedLocation(event.latlng.lat, event.latlng.lng, "map");
+    });
+}
+
+function attemptAutoGps() {
+    if (!navigator.geolocation) {
+        setLocationStatus("GPS unavailable. Click on the map to choose a location.", "error");
+        return;
+    }
+    setLocationStatus("Detecting GPS location...");
+    navigator.geolocation.getCurrentPosition(
+        (position) => {
+            setSelectedLocation(position.coords.latitude, position.coords.longitude, "gps");
+        },
+        (error) => {
+            setLocationStatus(`GPS failed (${error.message}). Click on the map to select a location.`, "error");
+        },
+        { enableHighAccuracy: true, timeout: 12000, maximumAge: 0 }
+    );
+}
+
+function setSelectedLocation(lat, lon, source) {
+    selectedLat = lat;
+    selectedLon = lon;
+    locationReady = true;
+    placeMapMarker(lat, lon);
+    updateLocationDisplay();
+    setLocationStatus(
+        `Location set via ${source.toUpperCase()}: ${lat.toFixed(4)}, ${lon.toFixed(4)}`,
+        "success"
+    );
+    if (currentMissionId) {
+        missionReady = false;
+        currentMissionId = null;
+        setStatus("Mission location changed. Set mission prompt again.");
+    }
+    updateControlState();
+    sendLocationIntel(lat, lon);
+}
+
+function updateLocationDisplay() {
+    latitudeDisplayEl.textContent = selectedLat !== null ? selectedLat.toFixed(4) : "--";
+    longitudeDisplayEl.textContent = selectedLon !== null ? selectedLon.toFixed(4) : "--";
+}
+
+function placeMapMarker(lat, lon) {
+    if (!missionMap) {
+        return;
+    }
+    const point = [lat, lon];
+    if (!missionMapMarker) {
+        missionMapMarker = L.circleMarker(point, {
+            radius: 8,
+            color: "#ff4d4d",
+            fillColor: "#ff4d4d",
+            fillOpacity: 0.95,
+            weight: 2,
+        }).addTo(missionMap);
+    } else {
+        missionMapMarker.setLatLng(point);
+    }
+    const targetZoom = missionMap.getZoom() < 8 ? 8 : missionMap.getZoom();
+    missionMap.setView(point, targetZoom);
+}
+
 function handleMissionPromptEdit() {
     if (!missionReady && !currentMissionId) {
         return;
@@ -266,7 +414,7 @@ function updateControlState() {
     const disableVideoActions = !missionReady || missionRequestPending || videoProcessing;
     videoInputEl.disabled = disableVideoActions;
     executeButton.disabled = disableVideoActions;
-    setMissionButton.disabled = missionRequestPending;
+    setMissionButton.disabled = missionRequestPending || !locationReady;
 }
 
 function setStatus(message, tone = "info") {
@@ -280,12 +428,70 @@ function setStatus(message, tone = "info") {
     }
 }
 
+function setLocationStatus(message, tone = "info") {
+    locationStatusEl.textContent = message || "";
+    if (tone === "error") {
+        locationStatusEl.style.color = "#ff7b7b";
+    } else if (tone === "success") {
+        locationStatusEl.style.color = "#7bffb3";
+    } else {
+        locationStatusEl.style.color = "#bdbdbd";
+    }
+}
+
+let locationIntelAbortController = null;
+
+async function sendLocationIntel(lat, lon) {
+    if (locationIntelAbortController) {
+        locationIntelAbortController.abort();
+    }
+    locationIntelAbortController = new AbortController();
+    locationIntelEl.textContent = "Analyzing location-based threats...";
+    locationIntelEl.style.color = "#ffa95c";
+    try {
+        const form = new FormData();
+        form.append("latitude", lat.toString());
+        form.append("longitude", lon.toString());
+        const response = await fetch(LOCATION_INTEL_URL, {
+            method: "POST",
+            body: form,
+            signal: locationIntelAbortController.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Location intelligence failed (${response.status})`);
+        }
+        const payload = await response.json();
+        const classes = (payload.mission_plan?.classes || []).slice(0, 4);
+        if (!classes.length) {
+            locationIntelEl.textContent = "Location intel: no threat classes suggested.";
+            locationIntelEl.style.color = "#bdbdbd";
+            return;
+        }
+        const classSummary = classes
+            .map((entry) => entry.name)
+            .join(", ");
+        locationIntelEl.textContent = `Likely threats near this location: ${classSummary}`;
+        locationIntelEl.style.color = "#7bffb3";
+    } catch (error) {
+        if (error.name === "AbortError") {
+            return;
+        }
+        console.error("[location_intel] error", error);
+        locationIntelEl.textContent = error.message || "Failed to analyze location.";
+        locationIntelEl.style.color = "#ff7b7b";
+    }
+}
+
 async function stageMissionPrompt() {
     const mission = missionInputEl.value.trim();
     if (!mission) {
         alert("Enter a mission prompt before setting it.");
         return;
     }
+    if (!locationReady || selectedLat === null || selectedLon === null) {
+        alert("Set mission location via GPS or the map before proceeding.");
+        return;
+    }
     missionRequestPending = true;
     missionReady = false;
     currentMissionId = null;
@@ -295,6 +501,8 @@ async function stageMissionPrompt() {
         const form = new FormData();
         form.append("prompt", mission);
         form.append("detector", detectorSelectEl.value);
+        form.append("latitude", selectedLat.toString());
+        form.append("longitude", selectedLon.toString());
         const response = await fetch(MISSION_PLAN_URL, {
             method: "POST",
             body: form,
@@ -347,6 +555,8 @@ async function executeMission() {
         videoForm.append("video", videoFile);
         videoForm.append("prompt", mission);
         videoForm.append("mission_id", currentMissionId);
+        videoForm.append("latitude", selectedLat?.toString() || "");
+        videoForm.append("longitude", selectedLon?.toString() || "");
         videoForm.append("detector", detector);
 
         console.log("[process_video] submitting request", { detector, missionLength: mission.length, fileSize: videoFile.size });
@@ -395,6 +605,8 @@ async function executeMission() {
         summaryForm.append("video", videoFile);
         summaryForm.append("prompt", mission);
         summaryForm.append("mission_id", currentMissionId);
+        summaryForm.append("latitude", selectedLat?.toString() || "");
+        summaryForm.append("longitude", selectedLon?.toString() || "");
         summaryForm.append("detector", detector);
 
         console.log("[mission_summary] submitting request", { detector, missionLength: mission.length, fileSize: videoFile.size });
@@ -439,6 +651,9 @@ function setOriginalPreview(file) {
     originalVideoEl.src = originalVideoUrl;
     originalVideoEl.load();
 }
+
+setLocationStatus("Awaiting GPS permission or map selection...");
+updateControlState();
 </script>
 
 </body>