post_merge fix with reset option

post_merge.py

@@ -128,6 +128,79 @@ def update_dataset_card(api: HfApi) -> None:
     logger.info("Updated dataset card with %d configs: %s", len(configs), configs)
 
 
+def full_rebuild() -> dict:
+    """Rebuild manifest.json from scratch and regenerate the dataset card.
+
+    Scans ALL data files on main, recomputes every hash/fingerprint,
+    and writes a fresh manifest (replacing the old one entirely).
+    """
+    logger.info("Starting full rebuild of manifest + dataset card")
+
+    all_data_files: list[str] = []
+    for entry in api.list_repo_tree(
+        repo_id=DATASET_REPO_ID,
+        repo_type="dataset",
+        revision="main",
+        recursive=True,
+    ):
+        if not hasattr(entry, "rfilename"):
+            continue
+        path = entry.rfilename
+        if path.startswith("data/") and (path.endswith(".json") or path.endswith(".jsonl")):
+            all_data_files.append(path)
+
+    logger.info("Found %d data files to index", len(all_data_files))
+
+    now = datetime.now(timezone.utc).isoformat()
+    manifest = {"files": {}}
+
+    for file_path in all_data_files:
+        try:
+            local_path = hf_hub_download(
+                repo_id=DATASET_REPO_ID,
+                filename=file_path,
+                repo_type="dataset",
+                revision="main",
+            )
+            with open(local_path, "rb") as f:
+                content = f.read()
+
+            sha256 = compute_sha256(content)
+            if file_path.endswith(".json"):
+                fingerprint = compute_fingerprint(content)
+            else:
+                fingerprint = sha256
+
+            manifest["files"][file_path] = {
+                "sha256": sha256,
+                "fingerprint": fingerprint,
+                "added_at": now,
+            }
+            logger.info("Indexed %s", file_path)
+        except Exception:
+            logger.exception("Failed to index %s", file_path)
+
+    # Upload fresh manifest
+    manifest_bytes = json.dumps(manifest, indent=2, sort_keys=True).encode()
+    api.upload_file(
+        path_or_fileobj=io.BytesIO(manifest_bytes),
+        path_in_repo="manifest.json",
+        repo_id=DATASET_REPO_ID,
+        repo_type="dataset",
+        commit_message="Full rebuild of manifest.json",
+    )
+    logger.info("Uploaded rebuilt manifest.json (%d files)", len(manifest["files"]))
+
+    # Regenerate dataset card
+    update_dataset_card(api)
+
+    return {
+        "status": "ok",
+        "action": "full_rebuild",
+        "files_indexed": len(manifest["files"]),
+    }
+
+
 def handle_merge(pr_num: int) -> dict:
     """Run all post-merge actions for a PR."""
     logger.info("Handling merge for PR #%d", pr_num)
@@ -175,3 +248,12 @@ def handle_merge(pr_num: int) -> dict:
         "pr": pr_num,
         "files_added_to_manifest": len(untracked),
     }
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+    result = full_rebuild()
+    print(json.dumps(result, indent=2))