Update app.py

app.py

@@ -1,4 +1,5 @@
 import os
+import io
 import json
 import hashlib
 import gradio as gr
@@ -11,31 +12,26 @@ from pipelines.openai_ingest import (
 from pipelines.parsing import normalize_resume
 from pipelines.merge import merge_normalized_records
 from pipelines.skills import extract_skills
+from pipelines.anonymize import anonymize_text, render_anonymized_pdf
 from pipelines.scoring import compute_quality_score
+from pipelines.storage import persist_to_hf
 from pipelines.utils import detect_filetype, load_doc_text
 
-APP_TITLE = "候補者インテーク & レジュメ標準化（安定・軽量版）"
+APP_TITLE = "候補者インテーク & レジュメ標準化（OpenAI版）"
 
 
 def process_resumes(filepaths, candidate_id: str, additional_notes: str = ""):
-    """
-    軽量安定版：
-      - OCR/抽出 → 構造化 → 正規化 → 統合 → スキル抽出 → 要約 → スコア
-      - 匿名化PDF・HF保存など重い処理は全てOFF
-    入力: gr.Files(type="filepath") のパス配列
-    出力: JSON文字列 / スキルJSON文字列 / スコアJSON文字列 / 要約(3種)
-    """
     if not filepaths:
         raise gr.Error("少なくとも1ファイルをアップロードしてください。")
 
     partial_records = []
-    merged_plain_texts = []
+    raw_texts = []
 
+    # gr.Files(type="filepath") から渡されるのは文字列パス
     for path in filepaths:
+        with open(path, "rb") as f:
+            raw_bytes = f.read()
         fname = os.path.basename(path)
-        with open(path, "rb") as rf:
-            raw_bytes = rf.read()
-
         filetype = detect_filetype(fname, raw_bytes)
 
         # 1) テキスト抽出
@@ -45,7 +41,7 @@ def process_resumes(filepaths, candidate_id: str, additional_notes: str = ""):
             base_text = load_doc_text(filetype, raw_bytes)
             text = extract_text_with_openai(base_text.encode("utf-8"), filename=fname, filetype="txt")
 
-        merged_plain_texts.append(text)
+        raw_texts.append({"filename": fname, "text": text})
 
         # 2) 構造化 → 3) 正規化
         structured = structure_with_openai(text)
@@ -55,7 +51,6 @@ def process_resumes(filepaths, candidate_id: str, additional_notes: str = ""):
             "certifications": structured.get("certifications_raw", ""),
             "skills": ", ".join(structured.get("skills_list", [])),
         })
-
         partial_records.append({
             "source": fname,
             "text": text,
@@ -63,11 +58,11 @@ def process_resumes(filepaths, candidate_id: str, additional_notes: str = ""):
             "normalized": normalized,
         })
 
-    # 4) 統合（複数ファイル→1候補者）
+    # 4) 統合
     merged = merge_normalized_records([r["normalized"] for r in partial_records])
 
-    # 5) スキル抽出（軽量辞書/正規表現）
-    merged_text = "\n\n".join(merged_plain_texts)
+    # 5) スキル抽出
+    merged_text = "\n\n".join([r["text"] for r in partial_records])
     skills = extract_skills(merged_text, {
         "work_experience": merged.get("raw_sections", {}).get("work_experience", ""),
         "education": merged.get("raw_sections", {}).get("education", ""),
@@ -75,43 +70,66 @@ def process_resumes(filepaths, candidate_id: str, additional_notes: str = ""):
         "skills": ", ".join(merged.get("skills", [])),
     })
 
-    # 6) 品質スコア
+    # 6) 匿名化（軽量正規表現ベース）
+    anonymized_text, anon_map = anonymize_text(merged_text)
+    anon_pdf_bytes = render_anonymized_pdf(anonymized_text)
+
+    # 7) 品質スコア
     score = compute_quality_score(merged_text, merged)
 
-    # 7) 要約（300/100/1文）
+    # 8) 要約
     summaries = summarize_with_openai(merged_text)
 
-    # 8) 構造化出力
+    # 9) 出力組み立て
+    cid = candidate_id or hashlib.sha256(merged_text.encode("utf-8")).hexdigest()[:16]
     result_json = {
-        "candidate_id": candidate_id or hashlib.sha256(merged_text.encode("utf-8")).hexdigest()[:16],
+        "candidate_id": cid,
         "files": [os.path.basename(p) for p in filepaths],
         "merged": merged,
         "skills": skills,
         "quality_score": score,
         "summaries": summaries,
+        "anonymization_map": anon_map,
         "notes": additional_notes,
     }
 
+    # 10) HF Datasets 保存（任意）
+    dataset_repo = os.environ.get("DATASET_REPO")
+    commit_info = None
+    if dataset_repo:
+        commit_info = persist_to_hf(
+            dataset_repo=dataset_repo,
+            record=result_json,
+            anon_pdf_bytes=anon_pdf_bytes,
+            parquet_path=f"candidates/{cid}.parquet",
+            json_path=f"candidates/{cid}.json",
+            pdf_path=f"candidates/{cid}.anon.pdf",
+        )
+
+    anon_pdf = (f"{cid}.anon.pdf", anon_pdf_bytes)
+
     return (
-        json.dumps(result_json, ensure_ascii=False, indent=2),  # out_json -> Code
-        json.dumps(skills, ensure_ascii=False, indent=2),       # out_skills -> Code
-        json.dumps(score, ensure_ascii=False, indent=2),        # out_score -> Code
+        json.dumps(result_json, ensure_ascii=False, indent=2),
+        json.dumps(skills, ensure_ascii=False, indent=2),     # ← JSONは Code で返す
+        json.dumps(score, ensure_ascii=False, indent=2),
         summaries.get("300chars", ""),
         summaries.get("100chars", ""),
         summaries.get("onesent", ""),
+        anon_pdf,
+        json.dumps(commit_info or {"status": "skipped (DATASET_REPO not set)"}, ensure_ascii=False, indent=2),
     )
 
 
 with gr.Blocks(title=APP_TITLE) as demo:
-    gr.Markdown(f"# {APP_TITLE}\nOpenAIでOCR/構造化/要約→統合→スコア（匿名化・保存なし）")
+    gr.Markdown(f"# {APP_TITLE}\n複数ファイルを統合→OpenAIで読み込み/構造化/要約→匿名化→Datasets保存")
 
     with gr.Row():
-        # ★ Gradio v4 の仕様に合わせて 'filepath' を使用（'file' は不可）
+        # ★ Gradio v4 は type="file" 非対応。'filepath' を使用
         in_files = gr.Files(
             label="レジュメ類 (PDF/画像/Word/テキスト) 複数可",
             file_count="multiple",
             file_types=[".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".bmp", ".docx", ".txt"],
-            type="filepath"
+            type="filepath",
         )
         candidate_id = gr.Textbox(label="候補者ID（任意。未入力なら自動生成）")
     notes = gr.Textbox(label="補足メモ（任意）", lines=3)
@@ -122,7 +140,7 @@ with gr.Blocks(title=APP_TITLE) as demo:
         out_json = gr.Code(label="統合出力 (JSON)")
 
     with gr.Tab("抽出スキル"):
-        # ★ GradioのJSONスキーマ推論バグを避けるため Code に統一
+        # ★ JSONウィジェットの型推論バグを回避。Codeで安全に表示
         out_skills = gr.Code(label="スキル一覧 (JSON)")
 
     with gr.Tab("品質スコア"):
@@ -133,13 +151,19 @@ with gr.Blocks(title=APP_TITLE) as demo:
         out_sum_100 = gr.Textbox(label="100字要約")
         out_sum_1 = gr.Textbox(label="1文要約")
 
+    with gr.Tab("匿名PDF"):
+        out_pdf = gr.File(label="匿名PDFダウンロード")
+
+    with gr.Tab("Datasets 保存ログ"):
+        out_commit = gr.Code(label="コミット情報")
+
     run_btn.click(
         process_resumes,
         inputs=[in_files, candidate_id, notes],
-        outputs=[out_json, out_skills, out_score, out_sum_300, out_sum_100, out_sum_1],
+        outputs=[out_json, out_skills, out_score, out_sum_300, out_sum_100, out_sum_1, out_pdf, out_commit],
     )
 
 
 if __name__ == "__main__":
-    # ローカル到達不可環境でも動くように share=True を明示
+    # ★ ローカル到達不可環境対策
     demo.launch(share=True, server_name="0.0.0.0", server_port=7860)