Update app.py

app.py

@@ -616,22 +616,61 @@ def _extract_json(raw: str) -> str:
     json_text = json_text.strip()
     return json_text
 
+from concurrent.futures import ThreadPoolExecutor
+
 def process_pdf(file):
     if file is None:
-        return "Please upload a PDF file."
+        return (
+            "Please upload a PDF file.",
+            "",
+            "",
+            "",
+            {},
+            "",
+            [],
+            {},
+            {},
+            "No file uploaded."
+        )
+
     try:
         extracted_text = extract_text_from_pdf(file.name)
         responsibilities = extract_section_from_pdf(extracted_text, section_title="Responsibilities and Accountabilities")
         if not responsibilities:
             log_debug(f"Skipping {os.path.basename(file.name)} - no responsibilities section found")
-            return None
-        job_family = classify_job_family(responsibilities)
-        log_debug(f"Identified {job_family} ")
-        occ_group = classify_occupational_group_by_level(responsibilities)
-        esco_occ = classify_esco_by_hierarchical_level(responsibilities)
-        qualification = extract_qualification(responsibilities)
-        skills = extract_skills(responsibilities)
+            return (
+                os.path.basename(file.name),
+                "",
+                "",
+                "",
+                {},
+                "",
+                [],
+                {},
+                {},
+                "No responsibilities section found."
+            )
+
+        # Use ThreadPoolExecutor to parallelize independent tasks
+        with ThreadPoolExecutor() as executor:
+            # Submit tasks to the executor
+            job_family_future = executor.submit(classify_job_family, responsibilities)
+            occ_group_future = executor.submit(classify_occupational_group_by_level, responsibilities)
+            esco_occ_future = executor.submit(classify_esco_by_hierarchical_level, responsibilities)
+            qualification_future = executor.submit(extract_qualification, responsibilities)
+            skills_future = executor.submit(extract_skills, responsibilities)
+
+            # Retrieve results from futures
+            job_family = job_family_future.result()
+            occ_group = occ_group_future.result()
+            esco_occ = esco_occ_future.result()
+            qualification = qualification_future.result()
+            skills = skills_future.result()
+
+        log_debug(f"Identified {job_family}")
+
         skill_map = map_proficiency_and_assessment(skills, responsibilities)
+
         has_esco = esco_occ.get("Level_5_ESCO_code") is not None
         skill_esco_extract = []
         skill_esco_map = []
@@ -641,6 +680,7 @@ def process_pdf(file):
             skill_esco_map = map_proficiency_and_assessment(skill_esco_extract, responsibilities)
         else:
             log_debug(f"No Level 5 ESCO code found for {os.path.basename(file.name)}, skipping ESCO skills mapping")
+
         time.sleep(6)
         assessment_lookup = {item['skill_name']: item for item in skill_map}
         joined_skills = [
@@ -657,6 +697,7 @@ def process_pdf(file):
             }
             for skill in skills
         ]
+
         joined_skills_esco = []
         if has_esco and skill_esco_extract:
             assessment_esco_lookup = {item['skill_name']: item for item in skill_esco_map}
@@ -669,44 +710,54 @@ def process_pdf(file):
                 }
                 for skill in skill_esco_extract
             ]
+
         interview = build_interview(responsibilities, skills)
-        result = {
-            "file": os.path.basename(file.name),
-            "responsibilities": responsibilities, 
-            "classified_job_family": job_family,
-            **{f"Level_{i}_CCOG_{field}": occ_group.get(f"Level_{i}_CCOG_{field}")
-               for i in range(1, 5) for field in ["code", "name", "desc"]},
-            "qualification": qualification,
-            "interview": interview,
-            "skills": {
-                "file": os.path.basename(file.name),                
+
+        # Prepare the results for each output component
+        ccoq_levels = {f"Level_{i}_CCOG_{field}": occ_group.get(f"Level_{i}_CCOG_{field}")
+                       for i in range(1, 5) for field in ["code", "name", "desc"]}
+
+        if has_esco:
+            esco_levels = {f"Level_{i}_ESCO_{field}": esco_occ.get(f"Level_{i}_ESCO_{field}")
+                           for i in range(1, 6) for field in ["code", "name", "desc"]}
+            esco_skills = {
+                "file": os.path.basename(file.name),
                 "classified_job_family": job_family,
-                "skills": joined_skills
+                "skills": joined_skills_esco
             }
-        }
-        if has_esco:
-            result.update({
-                **{f"Level_{i}_ESCO_{field}": esco_occ.get(f"Level_{i}_ESCO_{field}")
-                   for i in range(1, 6) for field in ["code", "name", "desc"]},
-                "skills_esco": {
-                    "file": os.path.basename(file.name),
-                    "classified_job_family": job_family,
-                    "skills": joined_skills_esco
-                }
-            })
         else:
-            result.update({
-                **{f"Level_{i}_ESCO_{field}": None
-                   for i in range(1, 6) for field in ["code", "name", "desc"]},
-                "skills_esco": None
-            })
-            
+            esco_levels = {f"Level_{i}_ESCO_{field}": None
+                           for i in range(1, 6) for field in ["code", "name", "desc"]}
+            esco_skills = None
+
         debug_message = "Processing completed successfully."
-        return result, debug_message
+        return (
+            os.path.basename(file.name),
+            responsibilities,
+            job_family,
+            "\n".join(qualification),
+            ccoq_levels,
+            "\n".join(interview),
+            joined_skills,
+            esco_levels,
+            esco_skills,
+            debug_message if DEBUG else None
+        )
 
     except Exception as e:
         error_message = f"Error processing PDF: {str(e)}"
-        return error_message, error_message
+        return (
+            error_message,
+            "",
+            "",
+            "",
+            {},
+            "",
+            [],
+            {},
+            {},
+            error_message
+        )
 
 from docx import Document