baseline08_beta01_27Sept25: zipped, yield output, log in

file_handler/file_utils.py

@@ -2,6 +2,9 @@
 #import os
 from pathlib import Path
 import sys
+import shutil
+import tempfile
+
 from itertools import chain
 from typing import List, Union, Any, Mapping
 from PIL import Image
@@ -95,8 +98,8 @@ def resolve_grandparent_object(gp_object:str):
         current_path = Path(sys.argv[0]).resolve()        
     except IndexError:        
         # Handle cases where sys.argv[0] might not exist (e.g., in some IDEs)
-        #current_path = Path(__file__).resolve()
-        current_path = Path('.').resolve()
+        current_path = Path(__file__).resolve()
+        #current_path = Path('.').resolve()    ##unreliable
 
     parent_dir = current_path.parent
     grandparent_dir = current_path.parent.parent
@@ -109,7 +112,7 @@ def resolve_grandparent_object(gp_object:str):
     #print(f"resolve: sys.path[1]:  {sys.path[1]}")  ##debug
 
 
-def check_create_logfile(filename: str, dir_path: Union[str, Path]="logs") -> Path:
+def check_create_logfile(filename: str, dir_path: Union[str, Path]="logs", tz_hours=None, date_format="%Y-%m-%d") -> Path:
     """
     check if log file exists, else create one and return the file path.
 
@@ -123,6 +126,7 @@ def check_create_logfile(filename: str, dir_path: Union[str, Path]="logs") -> Pa
     import datetime
     import warnings
     import tempfile
+    from utils.utils import get_time_now_str
 
     # 1. Get the path of the current script's parent directory (the project folder).
     # `__file__` is a special variable that holds the path to the current script.
@@ -151,7 +155,8 @@ def check_create_logfile(filename: str, dir_path: Union[str, Path]="logs") -> Pa
     
     # 4. Create log file with a timestamp inside the new logs directory.
     # This ensures a unique log file is created for the day the script runs.
-    timestamp = datetime.datetime.now().strftime("%Y-%m-%d")  #.strftime("%Y-%m-%d_%H-%M-%S")
+    #timestamp = datetime.datetime.now().strftime("%Y-%m-%d")  #.strftime("%Y-%m-%d_%H-%M-%S")
+    timestamp = get_time_now_str(tz_hours=tz_hours, date_format="%Y-%m-%d")
     log_file = logs_dir / f"{Path(filename).stem}_{timestamp}.log"
     
     # 5. Check if the file exists (it won't, if it's not the same day).
@@ -171,8 +176,33 @@ resolve_grandparent_object("file_handler")
 print(f'file: {check_create_logfile("app_logging.log")}')
 '''
 
-##SMY: to revisit. Make generic for any file apart from log files
-def check_create_file(filename: str, dir_path: Union[str, Path]="logs") -> Path:
+##SMY:DONE -  to revisit. Make generic for any file apart from log files
+def check_create_dir(dir_name: Union[str, Path]) -> Path:
+    """
+    check if directory exists, else create one and return the directory path.
+
+    Args:
+        directory_path (str): The path to the directory.
+        filename (str): The name of the directory to check/create.
+    Returns:
+        The pathlib.Path object for the directory
+    """
+
+    import warnings
+    
+    try:
+        dir_path = Path(dir_name)
+        #if dir_path.is_dir():
+        #    dir_path.mkdir(parents=True, exist_ok=True)  #, mode=0o2755)
+        dir_path.mkdir(parents=True, exist_ok=True)  #, mode=0o2755)
+    except PermissionError: ##[Errno 13] Permission denied: '/home/user/app/logs/app_logging_2025-09-18.log'
+        warnings.warn("[Errno 13] Permission denied, possibly insufficient permission or Persistent Storage not enable: attempting chmod 0o2644")
+        dir_path.mkdir(mode=0o2755, parents=True, exist_ok=True)
+        dir_path.chmod(0o2755)
+    
+    return dir_path
+
+def check_create_file(filename: Union[str, Path]) -> Path:
     """
     check if File exists, else create one and return the file path.
 
@@ -182,35 +212,124 @@ def check_create_file(filename: str, dir_path: Union[str, Path]="logs") -> Path:
     Returns:
         The pathlib.Path object for the file
     """
-    # Get project root
-    #project_root = Path(__file__).resolve().parent.parent  ##SMY:  `__file__` is a special variable pointing to current file`
-    project_root = Path(__file__).resolve().parents[1]      ##SMY: leverages parents. Get 2nd level
-
-    #file_dir = Path("logs") / file_dir if not isinstance(file_dir, Path) else Path(file_dir)
-    dir_path = dir_path if isinstance(dir_path, Path) else Path(dir_path)
-
-    # Ensure the directory exists
-    # Create the file parent directory, relative to the project root, if it doesn't exist.
-    # `parents=True` creates any missing parent directories.
-    # `exist_ok=True` prevents an error if the directory already exists.
-    dir_path = project_root / dir_path
-    if not dir_path.is_dir():
-        dir_path.mkdir(parents=True, exist_ok=True, mode=0o2755)  #, mode=0o2644)
-    #dir_path.chmod(0) 
+
+    import warnings
+    
+    try:
+        filename_path = Path(filename)
+        filename_path.touch(exist_ok=True)  #, mode=0o2755)
+    except PermissionError: ##[Errno 13] Permission denied: '/home/user/app/logs/app_logging_2025-09-18.log'
+        warnings.warn("[Errno 13] Permission denied, possibly insufficient permission or Persistent Storage not enable: attempting chmod 0o2644")
+        filename_path.touch(exist_ok=True, mode=0o2755)  # Creates an empty file if it doesn't exists
+        filename_path.chmod(0)
     
-    file_path = dir_path / filename  # Concatenate directory and filename to get full path
-    #print(f"file_path:  {file_path}")  ##debug
+    return filename_path
+
+def zip_processed_files(root_dir: str, file_paths: list[str], tz_hours=None, date_format='%d%b%Y_%H-%M-%S') -> Path:
+    """
+    Creates a zip file from a list of file paths (strings) and returns the Path object.
+    It preserves the directory structure relative to the specified root directory.
+
+    Args:
+        root_dir (str): The root directory against which relative paths are calculated.
+        file_paths (list[str]): A list of string paths to the files to be zipped.
+
+    Returns:
+        str(Path): The string of the Path object of the newly created zip file.
+    """
+
+    import zipfile
+    from file_handler import file_utils
+    from utils import utils
+
+    root_path = Path(root_dir)
+    if not root_path.is_dir():
+        raise ValueError(f"Root directory does not exist: {root_path}")
+
+    # Create a temporary directory in a location where Gradio can access it.
+    gradio_output_dir = Path(tempfile.gettempdir()) / "gradio_temp_output"
+    #gradio_output_dir.mkdir(exist_ok=True)
+    file_utils.check_create_dir(gradio_output_dir)
+    final_zip_path = gradio_output_dir / f"outputs_processed_{utils.get_time_now_str(tz_hours=tz_hours, date_format=date_format)}.zip"
+
+    # Use a context manager to create the zip file: use zipfile() opposed to shutil.make_archive
+    # 'w' mode creates a new file, overwriting if it already exists. 
+    zip_unprocessed = 0
+    with zipfile.ZipFile(final_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        for file_path_str in file_paths:
+            file_path = Path(file_path_str)
+            if file_path.exists() and file_path.is_file():
+                # Calculate the relative path from the root_dir.
+                # The `arcname` parameter tells `zipfile` what the path inside the zip file should be.
+                arcname = file_path.relative_to(root_path)
+                zipf.write(file_path, arcname=arcname)
+            else:
+                #print(f"Warning: Skipping {file_path_str}, as it is not a valid file.")
+                zip_processed_files += 1  ##SMY:future - to be implemented
+
+    #return final_zip_path
+    return str(final_zip_path)
+
+
+def process_and_zip(input_dir_path):
+    """
+    Finds dynamic directories, copies files from a source directory to a temporary directory, zips it,
+    and returns the path to the zip file.
     
-    #file_path.touch(exist_ok=True, mode=0o2664)  # Creates an empty file if it doesn't exists
-
-    #'''
-    if not file_path.exists():       # check if file doesn't exist
-        file_path.touch(exist_ok=True)  #, mode=0o2664)  # Creates an empty file if it doesn't exists
-        #file_dir.touch(mode=0o2644, exist_ok=True)  #, parents=True)  ##SMY: Note Permission Errno13 - https://stackoverflow.com/a/57454275
-        #file_dir.chmod(0)
-    #''' 
+    Args:
+        input_dir_path (str): The path to the directory containing files to be processed.
     
-    return file_path
+    Returns:
+        pathlib.Path: The path to the generated zip file.
+    """
+    # Convert the input path to a Path object
+    #input_path = Path(input_dir_path)
+    parent_input_path = Path(input_dir_path)  #.parent
+
+    # Check if the input directory exists
+    if not parent_input_path.is_dir():
+        raise ValueError(f"Input directory does not exist: {parent_input_path}")
+        
+    # Create a temporary directory using a context manager
+    with tempfile.TemporaryDirectory() as temp_dir_str:
+        temp_dir_path = Path(temp_dir_str)
+        
+        # Define the path for the output structure inside the temporary directory
+        temp_output_path = temp_dir_path / "output_dir"
+        
+        # Copy all extracted files to the temporary directory
+        # We use semantic accurate and performant .iterdir than more robust glob to get all files and folders
+        
+        for input_subdir in parent_input_path.iterdir():
+            if input_subdir.is_dir():
+            # Create the corresponding subdirectory in the temp directory
+                temp_output_subdir = temp_output_path / input_subdir.name
+                #temp_output_subdir.mkdir(parents=True, exist_ok=True)   #, mode=0o2755)
+                #file_handler.file_utils.check_create_dir(temp_output_subdir)
+                check_create_dir(temp_output_subdir)
+
+                # Copy the files from the source subdirectory to the temp subdirectory
+                #for item_path in input_path.glob('*'):
+                for item_path in input_subdir.iterdir():    
+                    if item_path.is_dir():
+                        shutil.copytree(src=item_path, dst=temp_output_subdir / item_path.name)
+                    else:
+                        shutil.copy2(item_path, temp_output_subdir)
+        
+        # Create the zip file from the temporary directory
+        zip_base_name = temp_dir_path / "outputs_processed_files"
+        zip_file_path = shutil.make_archive(
+            base_name=str(zip_base_name),   ##zip file's name
+            format='zip',
+            root_dir=str( temp_output_path)  #(temp_dir_path)     ##exclude from the archive
+        )
+        # Manually move the completed zip file to the Gradio-managed temporary directory
+        final_zip_file_path = parent_input_path / Path(zip_file_path).name
+        shutil.move(src=zip_file_path, dst=final_zip_file_path)
+        
+    # The shutil function returns a string, so we convert it back to a Path object in gr.File
+    return str(final_zip_file_path)
+
 
 def is_file_with_extension(path_obj: Path) -> bool:
     """

llm/hf_client.py

@@ -6,7 +6,7 @@ import time
 import traceback
 from huggingface_hub import InferenceClient, login, logout as hf_logout
 
-from llm.llm_login import login_huggingface, is_login_huggingface
+from llm.llm_login import login_huggingface, is_loggedin_huggingface   #,is_login_huggingface
 
 from utils.logger import get_logger
 
@@ -101,9 +101,9 @@ class HFChatClient:
                 #pass
         '''
         
-        login_huggingface(self.token) if not is_login_huggingface() else logger.log(level=20, msg=f"You are logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
+        login_huggingface(self.token) if not is_loggedin_huggingface() else logger.log(level=20, msg=f"You are logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
         ##SMY: TODO: Mapped with openai_client.py
-        #self.islogged_in = is_login_huggingface()
+        #self.islogged_in = is_loggedin_huggingface()
 
     @staticmethod
     def _normalise_history(history: list, system_message: str, latest_user_message: str) -> list[dict]:

llm/llm_login.py

@@ -1,4 +1,4 @@
-from huggingface_hub import HfApi, login, logout, get_token
+from huggingface_hub import HfApi, login, logout, get_token, whoami
 import os
 import traceback
 from time import sleep
@@ -9,6 +9,11 @@ from utils.logger import get_logger
 ## Get logger instance
 logger = get_logger(__name__)
 
+def disable_immplicit_token():
+    # Disable implicit token propagation for determinism
+    # Explicitly disable implicit token propagation; we rely on explicit auth or env var
+    os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
+
 def login_huggingface(token: Optional[str] = None):
     """
     Login to Hugging Face account. Prioritize CLI login for privacy and determinism.
@@ -25,13 +30,15 @@ def login_huggingface(token: Optional[str] = None):
         
     # Disable implicit token propagation for determinism
     # Explicitly disable implicit token propagation; we rely on explicit auth or env var
-    os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
+    #os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
+    disable_immplicit_token()
 
     token = token
     # Privacy-first login: try interactive CLI first; fallback to provided/env token only if needed
     try:
-        if HfApi.whoami():
-            logger.info("✔️ hf_login already", extra={"mode": "cli"})
+        #if HfApi.whoami():   ##SMY requires 'self' = HfApi. Alternatively HfApi().whoami()
+        if whoami():  ##SMY: Call HF API to know "whoami".
+            logger.info("✔️ hf_login already", extra={"mode": "HF Oauth"})
             #return True
         else:
             login()
@@ -53,13 +60,15 @@ def login_huggingface(token: Optional[str] = None):
             # Silent fallback; client will still work if token is passed directly
             #pass
 
-def is_login_huggingface():
-    from huggingface_hub import HfApi
+#def is_login_huggingface():
+def is_loggedin_huggingface():
+    #from huggingface_hub import HfApi
     from huggingface_hub.utils import HfHubHTTPError
 
     try:
         HfApi().whoami()
         logger.log(level=20, msg=("✔️ You are logged in."), extra={"is_logged_in": True})
+        disable_immplicit_token()
         return True
     except HfHubHTTPError as exc:
         # A 401 status code indicates an authentication error.

llm/openai_client.py

@@ -7,7 +7,7 @@ from typing import Optional   #Iterable, Literal
 import traceback
 #from huggingface_hub import InferenceClient, login, logout as hf_logout
 
-from llm.llm_login import login_huggingface, is_login_huggingface
+from llm.llm_login import login_huggingface, is_loggedin_huggingface  #, is_login_huggingface
 
 import dotenv
 #dotenv.load_dotenv(".env")
@@ -42,15 +42,17 @@ class OpenAIChatClient:
             self.model_id = f"{model_id}:{hf_provider}" if hf_provider is not None else model_id  ##concatenate so HF can pipe to Hf provider
             self.hf_provider = hf_provider
             self.base_url = base_url  #"https://router.huggingface.co/v1"  #%22"  #HF API proxy
-            #self.token = api_token if api_token else None   ##debug
-            self.token = openai_api_key_env if openai_api_key_env else api_token  #dotenv.get_key(".env", "OPENAI_API_KEY")
+            self.token = api_token if api_token else openai_api_key_env   ##None   ##debug
+            #self.token = openai_api_key_env if openai_api_key_env else api_token  #dotenv.get_key(".env", "OPENAI_API_KEY")
             #self.token = token or os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")  ## not preferred
-            login_huggingface(self.token) if not is_login_huggingface() else logger.log(level=20, msg=f"You are logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
             #self.fake_token = api_token or "a1b2c3" #or os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")
+            
             self.openai_api_key = self.token  #self.fake_token
             self.temperature = temperature
             self.top_p = top_p
-            self.islogged_in = is_login_huggingface()
+            self.islogged_in = is_loggedin_huggingface()
+            ##SMY: log in now handled at higher entry level.
+            #login_huggingface(self.token) if not is_loggedin_huggingface() else logger.log(level=20, msg=f"You are logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.            
 
             logger.log(level=2, msg="initialised OpenAIChatClient:", extra={"base_url": self.base_url, "openai_api_key": self.openai_api_key})
             
@@ -60,7 +62,7 @@ class OpenAIChatClient:
             logger.exception(f'✗ OpenAI client_init_failed", extra={"error": str(exc)}\n{tb}', exc_info=True)
             raise RuntimeError(f"✗ Failed to initialise OpenAI client: {exc}\n{tb}")
       
-        #login_huggingface(self.token) if not is_login_huggingface() else logger.log(level=20, msg=f"logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
+        #login_huggingface(self.token) if not is_loggedin_huggingface() else logger.log(level=20, msg=f"logged in to HF Hub already") ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
 
 ####IN PROGRESS
 # 

ui/gradio_ui.py

@@ -11,10 +11,10 @@ import file_handler
 import file_handler.file_utils
 from utils.config import TITLE, DESCRIPTION, DESCRIPTION_PDF_HTML, DESCRIPTION_PDF, DESCRIPTION_HTML, DESCRIPTION_MD
 from utils.utils import is_dict, is_list_of_dicts
-from file_handler.file_utils import process_dicts_data, collect_pdf_paths, collect_html_paths, collect_markdown_paths, create_outputdir  ## should move to handling file
+from file_handler.file_utils import zip_processed_files, process_dicts_data, collect_pdf_paths, collect_html_paths, collect_markdown_paths, create_outputdir  ## should move to handling file
 #from llm.hf_client import HFChatClient  ## SMY: unused. See converters.extraction_converter
 from llm.provider_validator import is_valid_provider, suggest_providers
-from llm.llm_login import login_huggingface
+from llm.llm_login import is_loggedin_huggingface, login_huggingface
 
 from converters.extraction_converter import DocumentConverter as docconverter  #DocumentExtractor #as docextractor
 from converters.pdf_to_md import PdfToMarkdownConverter, init_worker
@@ -34,6 +34,7 @@ pdf2md_converter = PdfToMarkdownConverter()
 #md2pdf_converter = MarkdownToPdfConverter()
 
 # pool executor to convert files called by Gradio
+##SMY: TODO: future: refactor to gradio_process.py
 def convert_batch(
     pdf_files, #: list[str],
     pdf_files_count: int,
@@ -59,8 +60,9 @@ def convert_batch(
     #output_dir: Optional[Union[str, Path]] = "output_dir",
     output_dir_string: str = "output_dir_default",
     use_llm: bool = False,   #Optional[bool] = False,  #True,
-    page_range: str = None,  #Optional[str] = None,  
-) -> str:
+    page_range: str = None,  #Optional[str] = None,
+    tz_hours: str = None,
+    ): #-> str:
     """
     Handles the conversion process using multiprocessing.
     Spins up a pool and converts all uploaded files in parallel.
@@ -71,13 +73,18 @@ def convert_batch(
     # explicitly wrap file object in a list
     #pdf_files = pdf_files_wrap(pdf_files)   ##Flag:  deprecation
 
+    # Update the Gradio UI to improve user-friendly eXperience
+    #outputs=[process_button, log_output, files_individual_JSON, files_individual_downloads],
+    yield gr.update(interactive=False), f"Processing files...", {"process": "Processing files"}, f"__init__.py"
+    
     ## debug
     #logger.log(level=30, msg="pdf_files_inputs", extra={"input_arg[0]:": pdf_files[0]})
 
     #if not files:
     if not pdf_files or pdf_files is None:  ## Check if files is None. This handles the case where no files are uploaded.
         logger.log(level=30, msg="Initialising ProcessPool: No files uploaded.", extra={"pdf_files": pdf_files, "files_len": pdf_files_count})
-        return "Initialising ProcessPool: No files uploaded."
+        #outputs=[log_output, files_individual_JSON, files_individual_downloads],
+        return gr.update(interactive=True), "Initialising ProcessPool: No files uploaded.", {"Upload":"No files uploaded"}, f"__init__.py"
     
     # Get config values if not provided
     config_file = find_file("config.ini")  ##from file_handler.file_utils
@@ -117,11 +124,20 @@ def convert_batch(
     
     #global docextractor   ##SMY: deprecated.
     try:
+        ##SMY: might deprecate. To replace with oauth login from Gradio ui or integrate cleanly.
         login_huggingface(api_token)  ## attempt login if not already logged in. NB: HF CLI login prompt would not display in Process Worker.
+        
+        if is_loggedin_huggingface() and (api_token is None or api_token == ""):
+            api_token = get_token()
+        else:
+            login_huggingface()
+        # login: Update the Gradio UI to improve user-friendly eXperience
+        yield gr.update(interactive=False), f"login to HF: Processing files...", {"process": "Processing files"}, f"__init__.py"
+        
     except Exception as exc:  # Catch all exceptions
         tb = traceback.format_exc()
         logger.exception(f"✗ Error during login_huggingface → {exc}\n{tb}", exc_info=True) # Log the full traceback
-        return f"✗ An error occurred during login_huggingface → {exc}\n{tb}", f"Error: {exc}", f"Error: {exc}"  # return the exception message
+        return gr.update(interactive=True), f"✗ An error occurred during login_huggingface → {exc}\n{tb}", {"Error":f"Error: {exc}"}, f"__init__.py"  # return the exception message
 
     try:
         # Create a pool with init_worker initialiser
@@ -146,38 +162,48 @@ def convert_batch(
                 #result_convert = pool.map(pdf2md_converter.convert_files, pdf_files, max_retries)
                 results = pool.map(pdf2md_converter.convert_files, pdf_files)  ##SMY iterables  #output_dir_string)
             except Exception as exc:
-                # Raise the exception to stop the Gradio app
-                #raise  # Re-raise the exception to halt execution
+                # Raise the exception to stop the Gradio app: exception to halt execution
                 logger.exception("Error during pooling file conversion", exc_info=True)  # Log the full traceback
                 traceback.print_exc()  # Print the exception traceback
-                return f"An error occurred during pool.map: {str(exc)}", f"Error: {exc}", f"Error: {exc}"  ## return the exception message
+                #return f"An error occurred during pool.map: {str(exc)}", f"Error: {exc}", f"Error: {exc}"  ## return the exception message
+                yield gr.update(interactive=True), f"An error occurred during pool.map: {str(exc)}", {"Error":f"Error: {exc}"}, f"__init__.py"  ## return the exception message
     
             #'''
-            logger.log(level=20, msg="ProcessPoolExecutor pool result:", extra={"results": str(results)})
-            logs = []
-            logs_files_images = []
-            #logs.extend(results)   ## performant pythonic
-            #logs = list[results]  ## 
-            logs = [result for result in results]  ## pythonic list comprehension
-            ## logs : [file , images , filepath, image_path]
-            
-            #logs_files_images = logs_files.extend(logs_images)  #zip(logs_files, logs_images)   ##SMY: in progress
-            for log in logs:
-                #logs_files_images.append(log.get("filepath", "Error or No filepath"))  # if all(isinstance(log, dict) for item in logs))
-                #logs_files_images.extend(list(image for image in log.get("image_path", "Error or no image_path")))
-
-                logs_files_images.append(log.get("filepath") if is_dict(logs) or isinstance(log, Path) else "Error or no image_path")  # isinstance(log, (dict, str))
-                logs_files_images.extend(list(image for image in log.get("image_path", "Error or no image_path")))
+            try:
+                logger.log(level=20, msg="ProcessPoolExecutor pool result:", extra={"results": str(results)})
+                logs = []
+                logs_files_images = []
+                #logs.extend(results)   ## performant pythonic
+                #logs = list[results]  ## 
+                logs = [result for result in results]  ## pythonic list comprehension
+                ## logs : [file , images , filepath, image_path]
                 
+                #logs_files_images = logs_files.extend(logs_images)  #zip(logs_files, logs_images)   ##SMY: in progress
+                logs_count =  0
+                #for log in logs:
+                for i, log in enumerate(logs):
+                    logs_files_images.append(log.get("filepath") if is_dict(log) or is_list_of_dicts(logs) else "Error or no file_path")  # isinstance(log, (dict, str))
+                    logs_files_images.extend(list(image for image in log.get("image_path", "Error or no image_path")))
+                    i_image = log.get("images", 0)
+                    # Update the Gradio UI to improve user-friendly eXperience
+                    yield gr.update(interactive=False), f"Processing files: {logs_files_images[logs_count]}", {"process": "Processing files"}, f"__init__.py"
+                    logs_count = i+i_image
+                
+                #logs_files_images.append(logs_filepath) ## to del
+                #logs_files_images.extend(logs_images)   ## to del
+            except Exception as exc:
+                logger.exception("Error during processing results logs → {exc}\n{tb}", exc_info=True)  # Log the full traceback
+                traceback.print_exc()  # Print the exception traceback
+                #return f"An error occurred during processing results logs: {str(exc)}\n{tb}", f"Error: {exc}", f"Error: {exc}"  ## return the exception message
+                yield gr.update(interactive=True), f"An error occurred during processing results logs: {str(exc)}\n{tb}", {"Error":f"Error: {exc}"}, f"__init__.py"  ## return the exception message
             
-            #logs_files_images.append(logs_filepath) ## to del
-            #logs_files_images.extend(logs_images)   ## to del
             #'''
     except Exception as exc:
         tb = traceback.format_exc()
         logger.exception(f"✗ Error during ProcessPoolExecutor → {exc}\n{tb}" , exc_info=True)  # Log the full traceback
         #traceback.print_exc()  # Print the exception traceback
-        return f"✗ An error occurred during ProcessPoolExecutor→ {exc}\n{tb}", f"Error: {exc}", f"Error: {exc}"  # return the exception message
+        #return gr.update(interactive=True), f"✗ An error occurred during ProcessPoolExecutor→ {exc}\n{tb}", f"Error: {exc}", f"Error: {exc}"  # return the exception message
+        yield gr.update(interactive=True), f"✗ An error occurred during ProcessPoolExecutor→ {exc}\n{tb}", {"Error":f"Error: {exc}"}, f"__init__.py"  # return the exception message
 
     '''
     logger.log(level=20, msg="ProcessPoolExecutor pool result:", extra={"results": str(results)})
@@ -187,8 +213,25 @@ def convert_batch(
     logs = [result for result in results]  ## pythonic list comprehension
     '''
 
+    # Zip Processed md Files and images. Insert to first index
+    try:  ##from file_handler.file_utils
+        zipped_processed_files = zip_processed_files(root_dir=f"data/{output_dir_string}", file_paths=logs_files_images, tz_hours=tz_hours, date_format='%d%b%Y')
+        logs_files_images.insert(0, zipped_processed_files)
+        #logs_files_images.insert(1, "====================")
+        yield gr.update(interactive=False), f"Processing zip and files: {logs_files_images}", {"process": "Processing files"}, f"__init__.py"
+    
+    except Exception as exc:
+        tb = traceback.format_exc()
+        logger.exception(f"✗ Error during zipping processed files → {exc}\n{tb}" , exc_info=True)  # Log the full traceback
+        #traceback.print_exc()  # Print the exception traceback
+        #return gr.update(interactive=True), f"✗ An error occurred during zipping files → {exc}\n{tb}", f"Error: {exc}", f"Error: {exc}"  # return the exception message
+        yield gr.update(interactive=True), f"✗ An error occurred during zipping files → {exc}\n{tb}", {"Error":f"Error: {exc}"}, f"__init__.py"  # return the exception message
+
+    
+    # Return processed files log
     try:
-        logs_return = file_handler.file_utils.process_dicts_data(logs)   #"\n".join(log for log in logs)  ##SMY outputs to gr.JSON component with no need for json.dumps(data, indent=)
+        ## # Convert logs list of dicts to formatted json string
+        logs_return_formatted_json_string = file_handler.file_utils.process_dicts_data(logs)   #"\n".join(log for log in logs)  ##SMY outputs to gr.JSON component with no need for json.dumps(data, indent=)
         #logs_files_images_return = "\n".join(path for path in logs_files_images)  ##TypeError: sequence item 0: expected str instance, WindowsPath found
         
         ##convert the List of Path objects to List of string for gr.Files output
@@ -196,13 +239,20 @@ def convert_batch(
         
         ## # Convert any Path objects to strings, but leave strings as-is
         logs_files_images_return = list(str(path) if isinstance(path, Path) else path for path in logs_files_images)
-        return logs_return, logs_return, logs_files_images_return
+        logger.log(level=20, msg="File conversion complete. Sending outcome to Gradio:", extra={"logs_files_image_return": str(logs_files_images_return)})  ## debug: FileNotFoundError: [WinError 2] The system cannot find the file specified: 'Error or no image_path'
+        
+        #outputs=[process_button, log_output, files_individual_JSON, files_individual_downloads],
         #return "\n".join(logs), "\n".join(logs_files_images)    #"\n".join(logs_files)
+        #return logs_return_formatted_json_string, logs_return_formatted_json_string, logs_files_images_return
+        #return gr.update(interactive=True), gr.update(value=logs_return_formatted_json_string), gr.update(value=logs_return_formatted_json_string, visible=True), gr.update(value=logs_files_images_return, visible=True)
+        yield  gr.update(interactive=True), gr.update(), gr.update(visible=True), gr.update(visible=True)
+        yield gr.update(interactive=True), logs_return_formatted_json_string, logs_return_formatted_json_string, logs_files_images_return
+        
     except Exception as exc:
         tb = traceback.format_exc()
         logger.exception(f"✗ Error during returning result logs → {exc}\n{tb}" , exc_info=True)  # Log the full traceback
         #traceback.print_exc()  # Print the exception traceback
-        return f"✗ An error occurred during returning result logs→ {exc}\n{tb}", f"Error: {exc}", f"Error: {exc}"  # return the exception message
+        return gr.update(interactive=True), f"✗ An error occurred during returning result logs→ {exc}\n{tb}", {"Error":f"Error: {exc}"}, f"__init__.py"  # return the exception message
 
 
     #return "\n".join(log for log in logs), "\n".join(str(path) for path in logs_files_images)
@@ -346,6 +396,7 @@ def build_interface() -> gr.Blocks:
     }
     """
 
+    ##SMY: flagged; to move to file_handler.file_utils
     def is_file_with_extension(path_obj: Path) -> bool:
         """
         Checks if a pathlib.Path object is a file and has a non-empty extension.
@@ -353,6 +404,7 @@ def build_interface() -> gr.Blocks:
         path_obj = path_obj if isinstance(path_obj, Path) else Path(path_obj) if isinstance(path_obj, str) else None
         return path_obj.is_file() and bool(path_obj.suffix)
 
+    ##SMY: flagged; to move to file_handler.file_utils
     def accumulate_files(uploaded_files, current_state):
         """
         Accumulates newly uploaded files with the existing state.
@@ -396,18 +448,18 @@ def build_interface() -> gr.Blocks:
             gr.Markdown(f"#### **Backend Configuration**")
             system_message = gr.Textbox(
                 label="System Message",
-                lines=2
+                lines=2,
             )
             with gr.Row():
                 provider_dd = gr.Dropdown(
                     choices=["huggingface", "openai"],
                     label="Provider",
                     value="huggingface",
-                    #allow_custom_value=True
+                    #allow_custom_value=True,
                 )
                 backend_choice = gr.Dropdown(
                     choices=["model-id", "provider", "endpoint"],
-                    label="HF Backend Choice"
+                    label="HF Backend Choice",
                 )  ## SMY: ensure HFClient maps correctly 
                 model_tb = gr.Textbox(
                     label="Model ID",
@@ -415,7 +467,7 @@ def build_interface() -> gr.Blocks:
                 )
                 endpoint_tb = gr.Textbox(
                     label="Endpoint",
-                    placeholder="Optional custom endpoint"
+                    placeholder="Optional custom endpoint",
                 )
             with gr.Row():
                 max_token_sl = gr.Slider(
@@ -423,26 +475,29 @@ def build_interface() -> gr.Blocks:
                     minimum=1,
                     maximum=131172,  #65536,  #32768,  #16384,  #8192,
                     value=1024,  #512,
-                    step=1
+                    step=1,
                 )
                 temperature_sl = gr.Slider(
                     label="Temperature",
                     minimum=0.0,
                     maximum=1.0,
                     value=0.0,
-                    step=0.1  #0.01
+                    step=0.1,  #0.01
                 )
                 top_p_sl = gr.Slider(
                     label="Top-p",
                     minimum=0.0,
                     maximum=1.0,
                     value=0.1,
-                    step=0.1  #0.01
-                )
-                stream_cb = gr.Checkbox(
-                    label="LLM Streaming",
-                    value=False
+                    step=0.1,  #0.01
                 )
+                with gr.Column():
+                    stream_cb = gr.Checkbox(
+                        label="LLM Streaming",
+                        value=False,
+                    )
+                    #tz_hours_tb = gr.Textbox(value=None, label="TZ Hours", placeholder="Timezone in numbers", max_lines=1,)
+                    tz_hours_num = gr.Number(label="TZ Hours", placeholder="Timezone in numbers", min_width=5,)
             with gr.Row():
                 api_token_tb = gr.Textbox(
                     label="API Token [OPTIONAL]",
@@ -524,6 +579,7 @@ def build_interface() -> gr.Blocks:
         # Initialise gr.State
         state_max_workers = gr.State(4)  #max_workers_sl,
         state_max_retries = gr.State(2) #max_retries_sl,
+        state_tz_hours    = gr.State(value=None)
 
         def update_state_stored_value(new_component_input):
             """ Updates stored state: use for max_workers and max_retries """
@@ -532,30 +588,51 @@ def build_interface() -> gr.Blocks:
         # Update gr.State values on slider components change. NB: initial value of `gr.State` must be able to be deepcopied
         max_workers_sl.change(update_state_stored_value, inputs=max_workers_sl, outputs=state_max_workers)
         max_retries_sl.change(update_state_stored_value, inputs=max_retries_sl, outputs=state_max_retries)
+        tz_hours_num.change(update_state_stored_value, inputs=tz_hours_num, outputs=state_tz_hours)
 
 
         with gr.Accordion("🤗 HuggingFace Client Logout", open=True):  #, open=False):
             # Logout controls
-            def do_logout():
+            '''def do_logout():
                 try:
                     #ok = docextractor.client.logout()
                     ok = docconverter.client.logout()
                     # Reset token textbox on successful logout
-                    msg = "✅ Logged out of HuggingFace and cleared tokens. Remember to log out of HuggingFace completely." if ok else "⚠️ Logout failed."
-                    return gr.update(value=""), gr.update(visible=True, value=msg), gr.update(value="Sign in to HuggingFace 🤗")
+                    #msg = "✅ Logged out of HuggingFace and cleared tokens. Remember to log out of HuggingFace completely." if ok else "⚠️ Logout failed."
+                    msg = "✅ Session Cleared. Remember to browser." if ok else "⚠️ Logout failed."
+                    return gr.update(value=""), gr.update(visible=True, value=msg), gr.update(value="Sign in to HuggingFace 🤗"), gr.update(value="Clear session")
                 except AttributeError:
-                    return gr.update(value=""), gr.update(visible=True, value=msg), gr.update(value="Sign in to HuggingFace 🤗")
-            
+                    msg = "⚠️ Logout."
+                    return gr.update(value=""), gr.update(visible=True, value=msg), gr.update(value="Sign in to HuggingFace 🤗"), gr.update(value="Clear session", interactive=False)
+            '''
+            def do_logout_hf():
+                try:
+                    ok = docconverter.client.logout()
+                    # Reset token textbox on successful logout
+                    msg = "✅ Session Cleared. Remember to close browser." if ok else "⚠️ Logout & Session Cleared"
+                    #return gr.update(value=""), gr.update(visible=True, value=msg), gr.update(value="Sign in to HuggingFace 🤗"), gr.update(value="Clear session", interactive=False)
+                    #return msg
+                    yield msg
+                except AttributeError:
+                    msg = "⚠️ Logout. No HF session"
+                    #return msg
+                    yield msg
+                
             def custom_do_logout():
-                return gr.update(value="Sign in to HuggingFace 🤗")
+                #do_logout()
+                #return gr.update(value="Sign in to HuggingFace 🤗")
+                msg = do_logout_hf()
+                #return gr.update(value="Sign in to HuggingFace 🤗"), gr.update(value=""), gr.update(visible=True, value=msg)
+                yield gr.update(value="Sign in to HuggingFace 🤗"), gr.update(value=""), gr.update(visible=True, value=msg)
             
             logout_status = gr.Markdown(visible=False)
             with gr.Row():
-                hf_login_logout_btn = gr.LoginButton(value="Sign in to HuggingFace 🤗", logout_value="Logout of HF: ({})", variant="huggingface")
-                logout_btn = gr.Button("Logout from session and Hugging Face (inference) Client", variant="stop", )
+                hf_login_logout_btn = gr.LoginButton(value="Sign in to HuggingFace 🤗", logout_value="Clear Session & Logout of HF: ({})", variant="huggingface")
+                #logout_btn = gr.Button("Logout from session and Hugging Face (inference) Client", variant="stop", )
 
-            hf_login_logout_btn.click(fn=custom_do_logout, inputs=None, outputs=hf_login_logout_btn)
-            logout_btn.click(fn=do_logout, inputs=None, outputs=[api_token_tb, logout_status, hf_login_logout_btn]) 
+            #hf_login_logout_btn.click(fn=custom_do_logout, inputs=None, outputs=hf_login_logout_btn)
+            hf_login_logout_btn.click(fn=custom_do_logout, inputs=None, outputs=[hf_login_logout_btn, api_token_tb, logout_status])
+            #logout_btn.click(fn=do_logout, inputs=None, outputs=[api_token_tb, logout_status, hf_login_logout_btn, logout_btn])
 
         
         # The gr.State component to hold the accumulated list of files
@@ -695,18 +772,26 @@ def build_interface() -> gr.Blocks:
             '''
 
         # A Files component to display individual processed files as download links
-        with gr.Accordion("⏬ View and Download processed files", open=False):
+        with gr.Accordion("⏬ View and Download processed files", open=True):  #, open=False
+            processed_file_state = gr.State([])
+
+            ##SMY: future
+            zip_btn = gr.DownloadButton("Download Zip file of all processed files", visible=False)   #.Button()
+            
+            # Placeholder to download zip file of processed files
+            download_zip_file = gr.File(label="Download processed Files (ZIP)", interactive=False, visible=False)  #, height="1"
+
             with gr.Row():
-                files_individual_JSON = gr.JSON(label="Serialised JSON list", max_height=250)
-                files_individual_downloads = gr.Files(label="Individual Processed Files")
+                files_individual_JSON = gr.JSON(label="Serialised JSON list", max_height=250, visible=False)
+                files_individual_downloads = gr.Files(label="Individual Processed Files", visible=False)
 
         ## Displays processed file paths
-        with gr.Accordion("View processing log", open=False):
+        with gr.Accordion("View processing log", open=True): #open=False):
             log_output = gr.Textbox(
                 label="Conversion Logs",
                 lines=5,
                 #max_lines=25,
-                interactive=False
+                #interactive=False
             )
         
         # file inputs
@@ -745,6 +830,7 @@ def build_interface() -> gr.Blocks:
             output_dir_tb,
             use_llm_cb,
             page_range_tb,
+            tz_hours_num,
         ]
 
         ## debug
@@ -756,12 +842,14 @@ def build_interface() -> gr.Blocks:
             #pdf_files.upload( 
                 fn=convert_batch,
                 inputs=inputs_arg,
-                outputs=[log_output, files_individual_JSON, files_individual_downloads],
+                outputs=[process_button, log_output, files_individual_JSON, files_individual_downloads],
             )
         except Exception as exc:
             tb = traceback.format_exc()
             logger.exception(f"✗ Error during process_button.click → {exc}\n{tb}", exc_info=True)
-            return f"✗ An error occurred during process_button.click → {exc}\n{tb}"            
+            msg = "✗ An error occurred during process_button.click"  # →
+            #return f"✗ An error occurred during process_button.click → {exc}\n{tb}"
+            return gr.update(interactive=True), f"{msg} → {exc}\n{tb}", f"{msg} → {exc}", f"{msg} → {exc}"
 
         ##gr.File .upload() event, fire only after a file has been uploaded
         # Event handler for the pdf file upload button
@@ -774,37 +862,9 @@ def build_interface() -> gr.Blocks:
         btn_pdf_convert.click(
         #pdf_files.upload( 
             fn=convert_batch,
-            outputs=[log_output, files_individual_downloads],
+            outputs=[btn_pdf_convert, log_output, files_individual_JSON, files_individual_downloads],
             inputs=inputs_arg, 
-        )
-        '''
-            inputs = [
-                pdf_files,
-                #pdf_files_wrap(pdf_files),  # wrap pdf_files in a list (if not already)            
-                pdf_files_count,
-                provider_dd,
-                model_tb,
-                hf_provider_dd,
-                endpoint_tb,
-                backend_choice,
-                system_message,
-                max_token_sl,
-                temperature_sl,
-                top_p_sl,
-                stream_cb,
-                api_token_tb,
-                #gr.State(4),   # max_workers
-                #gr.State(3),    # max_retries
-                openai_base_url_tb,
-                openai_image_format_dd,
-                state_max_workers, #gr.State(max_workers_sl),  #max_workers_sl,
-                state_max_retries, #gr.State(max_retries_sl), #max_retries_sl,
-                output_format_dd,
-                output_dir_tb,
-                use_llm_cb,
-                page_range_tb,
-            ],
-        '''   
+        ) 
         #    )
 
         # reuse the same business logic for HTML tab
@@ -818,7 +878,7 @@ def build_interface() -> gr.Blocks:
         btn_html_convert.click(
             fn=convert_batch,
             inputs=inputs_arg,
-            outputs=[log_output, files_individual_downloads]
+            outputs=[btn_html_convert,log_output, files_individual_JSON, files_individual_downloads]
         )
 
         def get_file_count(file_list):

utils/get_config.py

@@ -9,7 +9,7 @@ import traceback
 import sys
 from pathlib import Path
 #base_grandparent = Path(__file__).resolve().parent.parent
-grandparent_dir = Path('.').resolve() #.parent.parent
+grandparent_dir = Path('.').resolve() #.parent.parent      ##unreliable
 sys.path.insert(0, f"{grandparent_dir}")  #\\file_handler")
 ##end debug
 #'''

utils/logger.py

@@ -45,7 +45,7 @@ class JsonFormatter(logging.Formatter):
         return json.dumps(payload, ensure_ascii=False)
 
 #def setup_logging(level: int = logging.INFO) -> None:  ## Causing non-stop logging on HF spaces
-def setup_logging(level: int = None) -> None:
+def setup_logging(level: int = None, tz_hours=None, date_format:str="%d%b%Y") -> None:  #'%Y-%m-%d
     """Configure root logger with JSON output to both stdout and file.
     
     Args:
@@ -66,7 +66,7 @@ def setup_logging(level: int = None) -> None:
     #file_handler = logging.FileHandler("logs/app_logging_scrap.log", mode="a", encoding="utf-8")
     #file_handler = logging.FileHandler("logs/app_logging.log", mode="a", encoding="utf-8")
     from file_handler.file_utils import check_create_logfile
-    file_handler = logging.FileHandler(check_create_logfile("app_logging.log"), mode="a", encoding="utf-8")
+    file_handler = logging.FileHandler(check_create_logfile(filename="app_logging.log", tz_hours=tz_hours, date_format=date_format), mode="a", encoding="utf-8")
     ## Getting filepermission error
     
     file_handler.setFormatter(JsonFormatter())

utils/utils.py

@@ -1,3 +1,4 @@
+
 def is_dict(variable):
     """Checks if a variable is a dict."""
     if isinstance(variable, dict):
@@ -12,4 +13,26 @@ def is_list_of_dicts(variable):
         # Return True only if the list is empty or all elements are dicts.
         return all(isinstance(item, dict) for item in variable)
         
-    return False
+    return False
+
+
+def get_time_now_str(tz_hours=None, date_format:str='%Y-%m-%d'):  #date_format:str='%d%b%Y'):
+    """Returns the current time in a specific format + local time: ("%Y-%m-%d %H:%M:%S.%f %Z")."""
+    from datetime import datetime, timezone, timedelta
+
+    # Get the current time or UTC time
+    if tz_hours is not None:
+        current_utc_time = datetime.now(tz=timezone.utc) + timedelta(hours=tz_hours)
+        current_time = current_utc_time
+    else:
+        current_time = datetime.now()
+
+    # Format the time as a string
+    #formatted_time = current_utc_time.strftime(date_format)  #("%Y-%m-%d %H:%M:%S.%f %Z")
+    formatted_time = current_time.strftime(date_format)  #("%Y-%m-%d %H:%M:%S.%f %Z")
+
+    #print(f"Current time: {formatted_time}")   ##debug
+    return formatted_time
+
+#get_time_now_str() ##debug
+