Upload 2 files
Browse files
HFSmartWorkflow_Jan2026.ipynb
ADDED
|
@@ -0,0 +1,1020 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"metadata": {},
|
| 6 |
+
"source": [
|
| 7 |
+
"# <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Welcome to the Hugging Face File Uploader! \n",
|
| 8 |
+
"\n",
|
| 9 |
+
"# Welcome to the Hugging Face Backup & Image Zipper\n",
|
| 10 |
+
"\n",
|
| 11 |
+
"This notebook provides a suite of interactive widgets designed to streamline the entire process of preparing and uploading files to your Hugging Face repositories.\n",
|
| 12 |
+
"\n",
|
| 13 |
+
"Each step has been enhanced with \"smart\" features to provide clear feedback, prevent common errors, and accelerate your workflow.\n",
|
| 14 |
+
"\n",
|
| 15 |
+
"## Workflow at a Glance\n",
|
| 16 |
+
"\n",
|
| 17 |
+
"This notebook is organized into a simple, step-by-step process. Just run the cells in order.\n",
|
| 18 |
+
"\n",
|
| 19 |
+
"1. **βοΈ Setup & Validate Environment:** The first cell installs all necessary packages and then **validates** the environment, confirming that all tools and the `hf` CLI are ready to use.\n",
|
| 20 |
+
"2. **π Secure Authentication:** The login cell **checks your current login status** first. If you need to log in, it will then **validate your token** to ensure it has the correct `write` permissions required for uploading.\n",
|
| 21 |
+
"3. **ποΈ (Optional) Zip Your Images:** Use the **Smart Image Zipper** to prepare your image datasets. It allows you to filter by file type and **analyze a folder** to see a preview of the archive size *before* zipping.\n",
|
| 22 |
+
"4. **π Upload to the Hub:** The **Smart Uploader** widget provides a powerful interface for uploading your files, supporting concurrent uploads, single-commit mode, and automatic repository creation.\n",
|
| 23 |
+
"\n",
|
| 24 |
+
"## Key Features Across the Toolkit\n",
|
| 25 |
+
"- **Interactive Widgets:** Manage your entire workflow without writing complex scripts.\n",
|
| 26 |
+
"- **Environment Validation:** Confidence that your setup is correct from the very beginning.\n",
|
| 27 |
+
"- **Secure Login with Permission Checks:** Prevents upload failures due to incorrect token permissions.\n",
|
| 28 |
+
"- **Pre-Zip Analysis:** Analyze image folders to know the size and file count before you zip.\n",
|
| 29 |
+
"- **Advanced Upload Options:** Choose between single-commit mode for clean history or concurrent uploads for speed.\n",
|
| 30 |
+
"- **Fast Uploads:** Automatically uses `hf_transfer` to speed up large file transfers.\n",
|
| 31 |
+
"- **Live Progress Bars:** Monitor progress during both zipping and uploading.\n",
|
| 32 |
+
"\n",
|
| 33 |
+
"---\n",
|
| 34 |
+
"\n",
|
| 35 |
+
"**Community & Support:**\n",
|
| 36 |
+
"\n",
|
| 37 |
+
"* **GitHub:** [HuggingFace\\_Backup Repository on GitHub](https://github.com/Ktiseos-Nyx/HuggingFace_Backup) (for the latest version, updates, bug reports, and contributions)\n",
|
| 38 |
+
"* **Discord:**\n",
|
| 39 |
+
" * [Ktiseos Nyx AI/ML Discord](https://discord.gg/HhBSvM9gBY)\n",
|
| 40 |
+
" * [Earth & Dusk Media](https://discord.gg/5t2kYxt7An)\n",
|
| 41 |
+
"\n",
|
| 42 |
+
"This toolkit is designed to simplify every step of getting your files onto the Hugging Face Hub. We hope you find it useful!"
|
| 43 |
+
]
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"cell_type": "markdown",
|
| 47 |
+
"metadata": {},
|
| 48 |
+
"source": [
|
| 49 |
+
" # <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Install Dependencies\n",
|
| 50 |
+
"\n",
|
| 51 |
+
"\n",
|
| 52 |
+
"### βοΈ Environment Setup & Validation\n",
|
| 53 |
+
"\n",
|
| 54 |
+
"This cell installs and verifies the required packages to ensure your environment is correctly configured.\n",
|
| 55 |
+
"\n",
|
| 56 |
+
"### Key Packages & Versions:\n",
|
| 57 |
+
"* `huggingface_hub==1.3.0`: The latest library for interacting with the Hub, including the powerful `hf` command-line interface (CLI).\n",
|
| 58 |
+
"* `hf_transfer==0.1.9`: The current version of the library that dramatically **accelerates uploads**.\n",
|
| 59 |
+
"* `ipywidgets`: Powers the interactive uploader widget.\n",
|
| 60 |
+
"\n",
|
| 61 |
+
"After installation, the cell will **validate** the entire setupβchecking package versions and confirming that the `hf` CLI is available and ready to use.\n",
|
| 62 |
+
"\n",
|
| 63 |
+
"---"
|
| 64 |
+
]
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"cell_type": "code",
|
| 68 |
+
"execution_count": 1,
|
| 69 |
+
"metadata": {},
|
| 70 |
+
"outputs": [
|
| 71 |
+
{
|
| 72 |
+
"name": "stdout",
|
| 73 |
+
"output_type": "stream",
|
| 74 |
+
"text": [
|
| 75 |
+
"Installing/updating required packages...\n",
|
| 76 |
+
"Package installation/update process complete.\n"
|
| 77 |
+
]
|
| 78 |
+
}
|
| 79 |
+
],
|
| 80 |
+
"source": [
|
| 81 |
+
"# Cell 1: Environment Setup and Validation\n",
|
| 82 |
+
"# -----------------------------------------------------------------------------\n",
|
| 83 |
+
"import sys\n",
|
| 84 |
+
"import os\n",
|
| 85 |
+
"import subprocess\n",
|
| 86 |
+
"import pkg_resources\n",
|
| 87 |
+
"\n",
|
| 88 |
+
"print(\"βοΈ Setting up the environment with specified package versions...\")\n",
|
| 89 |
+
"\n",
|
| 90 |
+
"# --- 1. Install exact versions for a reproducible environment ---\n",
|
| 91 |
+
"# Using '==' ensures that this notebook will work consistently.\n",
|
| 92 |
+
"# The '-U' is still useful to ensure that if an older version is present, it's replaced.\n",
|
| 93 |
+
"!{sys.executable} -m pip install -U \"huggingface_hub==1.3.0\" \"ipywidgets>=8.0.0\" \"hf_transfer==0.1.9\" --no-color --disable-pip-version-check\n",
|
| 94 |
+
"\n",
|
| 95 |
+
"print(\"\\nβ
Installation complete.\")\n",
|
| 96 |
+
"print(\"π Validating environment and tools...\")\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"# --- 2. Validate Python package imports and confirm versions ---\n",
|
| 99 |
+
"try:\n",
|
| 100 |
+
" # Use pkg_resources to be absolutely sure of the installed version\n",
|
| 101 |
+
" hf_hub_version = pkg_resources.get_distribution(\"huggingface_hub\").version\n",
|
| 102 |
+
" ipywidgets_version = pkg_resources.get_distribution(\"ipywidgets\").version\n",
|
| 103 |
+
" print(f\" - βοΈ huggingface_hub version: {hf_hub_version}\")\n",
|
| 104 |
+
" print(f\" - βοΈ ipywidgets version: {ipywidgets_version}\")\n",
|
| 105 |
+
" \n",
|
| 106 |
+
" if hf_hub_version != \"1.3.0\":\n",
|
| 107 |
+
" print(f\" β οΈ Warning: Expected v1.3.0, but found v{hf_hub_version}. This might cause issues.\")\n",
|
| 108 |
+
"\n",
|
| 109 |
+
"except (pkg_resources.DistributionNotFound, ImportError) as e:\n",
|
| 110 |
+
" print(f\"β Critical package failed to be validated: {e}. Please check the installation log above.\")\n",
|
| 111 |
+
"\n",
|
| 112 |
+
"# --- 3. Validate hf_transfer for accelerated uploads ---\n",
|
| 113 |
+
"try:\n",
|
| 114 |
+
" hf_transfer_version = pkg_resources.get_distribution(\"hf-transfer\").version\n",
|
| 115 |
+
" print(f\" - βοΈ hf-transfer version: {hf_transfer_version}. Uploads will be accelerated.\")\n",
|
| 116 |
+
" os.environ['HF_HUB_ENABLE_HF_TRANSFER'] = '1'\n",
|
| 117 |
+
"except (pkg_resources.DistributionNotFound, ImportError):\n",
|
| 118 |
+
" print(f\" - β οΈ hf-transfer is not installed. Uploads may be slow.\")\n",
|
| 119 |
+
"\n",
|
| 120 |
+
"# --- 4. Validate that the 'hf' Command-Line Interface (CLI) is working ---\n",
|
| 121 |
+
"# This confirms the core tool you're interested in is available from the shell.\n",
|
| 122 |
+
"try:\n",
|
| 123 |
+
" result = subprocess.run(['hf', 'version'], capture_output=True, text=True, check=True)\n",
|
| 124 |
+
" print(f\" - βοΈ Hugging Face CLI is ready: ({result.stdout.strip()})\")\n",
|
| 125 |
+
"except (subprocess.CalledProcessError, FileNotFoundError):\n",
|
| 126 |
+
" print(\" - β The 'hf' CLI command could not be found or failed to run.\")\n",
|
| 127 |
+
" print(\" This might indicate an issue with your system's PATH or a broken installation.\")\n",
|
| 128 |
+
"\n",
|
| 129 |
+
"print(\"\\nπ‘ Tip: If the widgets in the uploader do not appear later, try restarting the Jupyter kernel (Kernel -> Restart).\")\n",
|
| 130 |
+
" "
|
| 131 |
+
]
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"cell_type": "markdown",
|
| 135 |
+
"metadata": {
|
| 136 |
+
"id": "Xs1mb1VKLuUW"
|
| 137 |
+
},
|
| 138 |
+
"source": [
|
| 139 |
+
"# β¨ <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Connecting to Hugging Face: Authentication\n",
|
| 140 |
+
"## π How To Use\n",
|
| 141 |
+
"\n",
|
| 142 |
+
"This smart cell securely handles your login and validates your credentials. You will need a token with **write** permissions to upload files.\n",
|
| 143 |
+
"\n",
|
| 144 |
+
"### What this cell does:\n",
|
| 145 |
+
"1. **Checks Your Status:** It first checks if you are already logged in.\n",
|
| 146 |
+
"2. **Prompts if Needed:** If you aren't logged in, it will display a login box.\n",
|
| 147 |
+
"3. **Validates Your Token:** After you enter a token, it immediately confirms that the token is valid and checks its permissions.\n",
|
| 148 |
+
"\n",
|
| 149 |
+
"### Instructions:\n",
|
| 150 |
+
"1. **Create a Token:** Go to your [Hugging Face Tokens page](https://huggingface.co/settings/tokens), click \"New token\", and give it the **`write`** role.\n",
|
| 151 |
+
"2. **Copy the Token:** Copy the newly generated token to your clipboard.\n",
|
| 152 |
+
"3. **Run this Cell:** Execute the code cell below. If prompted, paste your token into the box and press `Login`.\n",
|
| 153 |
+
"\n",
|
| 154 |
+
"### After Running:\n",
|
| 155 |
+
"* β
If successful, a confirmation message will appear with your username and token permissions.\n",
|
| 156 |
+
"* β οΈ If your token only has `read` access, a warning will be displayed.\n",
|
| 157 |
+
"* β If the login fails, an error message will help you diagnose the issue."
|
| 158 |
+
]
|
| 159 |
+
},
|
| 160 |
+
{
|
| 161 |
+
"cell_type": "code",
|
| 162 |
+
"execution_count": 11,
|
| 163 |
+
"metadata": {},
|
| 164 |
+
"outputs": [
|
| 165 |
+
{
|
| 166 |
+
"data": {
|
| 167 |
+
"text/html": [
|
| 168 |
+
"\n",
|
| 169 |
+
" <div style=\"padding: 10px; border: 1px solid #2E8B57; border-radius: 5px;\">\n",
|
| 170 |
+
" β
<b>Already logged in!</b><br>\n",
|
| 171 |
+
" Welcome back, <b>Duskfallcrew</b>. Your token has <b>WRITE</b> permissions.\n",
|
| 172 |
+
" </div>\n",
|
| 173 |
+
" <div style=\"margin-top: 10px;\"><i>If you need to switch accounts, please restart the kernel and run this cell again.</i></div>\n",
|
| 174 |
+
" "
|
| 175 |
+
],
|
| 176 |
+
"text/plain": [
|
| 177 |
+
"<IPython.core.display.HTML object>"
|
| 178 |
+
]
|
| 179 |
+
},
|
| 180 |
+
"metadata": {},
|
| 181 |
+
"output_type": "display_data"
|
| 182 |
+
}
|
| 183 |
+
],
|
| 184 |
+
"source": [
|
| 185 |
+
"# Cell 2: Hugging Face Authentication Setup (Smart Version)\n",
|
| 186 |
+
"# -----------------------------------------------------------------------------\n",
|
| 187 |
+
"# This cell securely logs you into Hugging Face.\n",
|
| 188 |
+
"# It checks if you're already logged in, validates your token after entry,\n",
|
| 189 |
+
"# and confirms the permissions (read/write) of your token.\n",
|
| 190 |
+
"# -----------------------------------------------------------------------------\n",
|
| 191 |
+
"from huggingface_hub import notebook_login, whoami\n",
|
| 192 |
+
"from IPython.display import display, HTML, clear_output\n",
|
| 193 |
+
"\n",
|
| 194 |
+
"print(\"Checking Hugging Face authentication status...\")\n",
|
| 195 |
+
"\n",
|
| 196 |
+
"try:\n",
|
| 197 |
+
" # 1. Check if we're already logged in by making a simple API call.\n",
|
| 198 |
+
" user_info = whoami()\n",
|
| 199 |
+
" username = user_info.get(\"name\")\n",
|
| 200 |
+
" auth_scope = \"write\" if user_info.get(\"auth\", {}).get(\"accessToken\", {}).get(\"role\") == \"write\" else \"read\"\n",
|
| 201 |
+
" \n",
|
| 202 |
+
" # If the call succeeds, we are already authenticated.\n",
|
| 203 |
+
" clear_output(wait=True)\n",
|
| 204 |
+
" display(HTML(f\"\"\"\n",
|
| 205 |
+
" <div style=\"padding: 10px; border: 1px solid #2E8B57; border-radius: 5px;\">\n",
|
| 206 |
+
" β
<b>Already logged in!</b><br>\n",
|
| 207 |
+
" Welcome back, <b>{username}</b>. Your token has <b>{auth_scope.upper()}</b> permissions.\n",
|
| 208 |
+
" </div>\n",
|
| 209 |
+
" <div style=\"margin-top: 10px;\"><i>If you need to switch accounts, please restart the kernel and run this cell again.</i></div>\n",
|
| 210 |
+
" \"\"\"))\n",
|
| 211 |
+
"\n",
|
| 212 |
+
"except Exception:\n",
|
| 213 |
+
" # 2. If whoami() fails, it means we're not logged in.\n",
|
| 214 |
+
" clear_output(wait=True)\n",
|
| 215 |
+
" print(\"You are not logged in. Please proceed with authentication.\")\n",
|
| 216 |
+
" print(\"A login widget will appear below. Paste your Hugging Face token with 'write' permissions.\")\n",
|
| 217 |
+
" \n",
|
| 218 |
+
" # Display the standard login widget.\n",
|
| 219 |
+
" notebook_login()\n",
|
| 220 |
+
"\n",
|
| 221 |
+
" # 3. After the user submits, validate the new token immediately.\n",
|
| 222 |
+
" try:\n",
|
| 223 |
+
" clear_output(wait=True) # Remove the login widget for a clean output\n",
|
| 224 |
+
" print(\"Validating token...\")\n",
|
| 225 |
+
" user_info = whoami()\n",
|
| 226 |
+
" username = user_info.get(\"name\")\n",
|
| 227 |
+
" auth_scope = \"write\" if user_info.get(\"auth\", {}).get(\"accessToken\", {}).get(\"role\") == \"write\" else \"read\"\n",
|
| 228 |
+
" \n",
|
| 229 |
+
" display(HTML(f\"\"\"\n",
|
| 230 |
+
" <div style=\"padding: 10px; border: 1px solid #2E8B57; border-radius: 5px;\">\n",
|
| 231 |
+
" β
<b>Login Successful!</b><br>\n",
|
| 232 |
+
" Welcome, <b>{username}</b>. Your token has been saved with <b>{auth_scope.upper()}</b> permissions.\n",
|
| 233 |
+
" </div>\n",
|
| 234 |
+
" \"\"\"))\n",
|
| 235 |
+
" \n",
|
| 236 |
+
" if auth_scope != 'write':\n",
|
| 237 |
+
" display(HTML(f\"\"\"\n",
|
| 238 |
+
" <div style=\"margin-top:10px; padding: 10px; border: 1px solid #FFD700; border-radius: 5px;\">\n",
|
| 239 |
+
" β οΈ <b>Warning:</b> Your token only has 'read' permissions. You will not be able to upload files. \n",
|
| 240 |
+
" Please generate a new token with 'write' permissions on the Hugging Face website.\n",
|
| 241 |
+
" </div>\n",
|
| 242 |
+
" \"\"\"))\n",
|
| 243 |
+
"\n",
|
| 244 |
+
" except Exception as e:\n",
|
| 245 |
+
" clear_output(wait=True)\n",
|
| 246 |
+
" display(HTML(f\"\"\"\n",
|
| 247 |
+
" <div style=\"padding: 10px; border: 1px solid #DC143C; border-radius: 5px;\">\n",
|
| 248 |
+
" β <b>Login Failed.</b><br>\n",
|
| 249 |
+
" The token you provided could not be validated. Please check your token and try again.\n",
|
| 250 |
+
" <pre style=\"white-space: pre-wrap; margin-top: 5px;\">Error: {e}</pre>\n",
|
| 251 |
+
" </div>\n",
|
| 252 |
+
" \"\"\"))\n",
|
| 253 |
+
"\n"
|
| 254 |
+
]
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"cell_type": "markdown",
|
| 258 |
+
"metadata": {},
|
| 259 |
+
"source": [
|
| 260 |
+
"# π <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Using the Hugging Face File Uploader\n",
|
| 261 |
+
"\n",
|
| 262 |
+
"## Uploader Checklist\n",
|
| 263 |
+
"Follow these steps to upload your files.\n",
|
| 264 |
+
"\n",
|
| 265 |
+
"1. Fill in Repository Details\n",
|
| 266 |
+
"- Owner (your username/org)\n",
|
| 267 |
+
"- Repo Name\n",
|
| 268 |
+
"- Repo Type (model, dataset, etc.)\n",
|
| 269 |
+
"\n",
|
| 270 |
+
"2. Select Local Files\n",
|
| 271 |
+
"- Set the Source Directory to your local folder path.\n",
|
| 272 |
+
"- Click the π List Files button.\n",
|
| 273 |
+
"- Select your desired files from the list that appears.\n",
|
| 274 |
+
"\n",
|
| 275 |
+
"3. Review Upload Settings (Optional)\n",
|
| 276 |
+
"- Add a Commit Message to describe your changes.\n",
|
| 277 |
+
"- Choose whether to Create a Pull Request for this upload.\n",
|
| 278 |
+
"\n",
|
| 279 |
+
"4. Start the Upload\n",
|
| 280 |
+
"- Click the β¬οΈ Upload Selected Files button and monitor the output."
|
| 281 |
+
]
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"cell_type": "code",
|
| 285 |
+
"execution_count": 5,
|
| 286 |
+
"metadata": {
|
| 287 |
+
"cellView": "form",
|
| 288 |
+
"id": "J851eLx6Ii3h"
|
| 289 |
+
},
|
| 290 |
+
"outputs": [],
|
| 291 |
+
"source": [
|
| 292 |
+
"# --- Essential Imports for the Uploader ---\n",
|
| 293 |
+
"import glob\n",
|
| 294 |
+
"import os\n",
|
| 295 |
+
"import time\n",
|
| 296 |
+
"import traceback # --- NEW: Explicitly import traceback ---\n",
|
| 297 |
+
"from pathlib import Path\n",
|
| 298 |
+
"import math\n",
|
| 299 |
+
"from concurrent.futures import ThreadPoolExecutor, as_completed # --- NEW: For concurrent uploads ---\n",
|
| 300 |
+
"from typing import List, Tuple, Optional # --- NEW: For type hinting ---\n",
|
| 301 |
+
"\n",
|
| 302 |
+
"from huggingface_hub import HfApi, CommitOperationAdd\n",
|
| 303 |
+
"from ipywidgets import (Text, Dropdown, Button, SelectMultiple, VBox, HBox,\n",
|
| 304 |
+
" Output, Layout, Checkbox, HTML, Textarea, Label,\n",
|
| 305 |
+
"\n",
|
| 306 |
+
" FloatProgress)\n",
|
| 307 |
+
"from IPython.display import display, clear_output\n",
|
| 308 |
+
"\n",
|
| 309 |
+
"# Attempt to enable hf_transfer.\n",
|
| 310 |
+
"os.environ['HF_HUB_ENABLE_HF_TRANSFER'] = '1'\n",
|
| 311 |
+
"\n",
|
| 312 |
+
"class SmartHuggingFaceUploader:\n",
|
| 313 |
+
" \"\"\"\n",
|
| 314 |
+
" A \"smarter\" Jupyter widget-based tool to upload files to the Hugging Face Hub.\n",
|
| 315 |
+
" Enhancements:\n",
|
| 316 |
+
" - Fetches user/org repositories.\n",
|
| 317 |
+
" - Option for single commit for all files.\n",
|
| 318 |
+
" - Option to create the repo if it doesn't exist.\n",
|
| 319 |
+
" - Displays file sizes in the picker.\n",
|
| 320 |
+
" - Disables UI elements during long operations.\n",
|
| 321 |
+
" - Supports concurrent uploads for better performance.\n",
|
| 322 |
+
" \"\"\"\n",
|
| 323 |
+
"\n",
|
| 324 |
+
" def __init__(self) -> None:\n",
|
| 325 |
+
" self.api = HfApi()\n",
|
| 326 |
+
" self.user_info = self.api.whoami()\n",
|
| 327 |
+
" self.file_types = [\n",
|
| 328 |
+
" # ... (your file types are great, no change needed)\n",
|
| 329 |
+
" ('SafeTensors', 'safetensors'), ('PyTorch Models', 'pt'), ('PyTorch Legacy', 'pth'),\n",
|
| 330 |
+
" ('ONNX Models', 'onnx'), ('TensorFlow Models', 'pb'), ('Keras Models', 'h5'),\n",
|
| 331 |
+
" ('Checkpoints', 'ckpt'), ('Binary Files', 'bin'),\n",
|
| 332 |
+
" ('JSON Files', 'json'), ('YAML Files', 'yaml'), ('YAML Alt', 'yml'),\n",
|
| 333 |
+
" ('Text Files', 'txt'), ('CSV Files', 'csv'), ('Pickle Files', 'pkl'),\n",
|
| 334 |
+
" ('PNG Images', 'png'), ('JPEG Images', 'jpg'), ('JPEG Alt', 'jpeg'),\n",
|
| 335 |
+
" ('WebP Images', 'webp'), ('GIF Images', 'gif'),\n",
|
| 336 |
+
" ('ZIP Archives', 'zip'), ('TAR Files', 'tar'), ('GZ Archives', 'gz')\n",
|
| 337 |
+
" ]\n",
|
| 338 |
+
" self.current_directory = os.getcwd()\n",
|
| 339 |
+
" self.hf_transfer_active = self._check_hf_transfer_availability()\n",
|
| 340 |
+
" self._create_widgets()\n",
|
| 341 |
+
" self._bind_events()\n",
|
| 342 |
+
" self._update_files(None) # Initial file list update\n",
|
| 343 |
+
"\n",
|
| 344 |
+
" def _check_hf_transfer_availability(self) -> bool:\n",
|
| 345 |
+
" if os.environ.get(\"HF_HUB_ENABLE_HF_TRANSFER\") == \"1\":\n",
|
| 346 |
+
" try:\n",
|
| 347 |
+
" import hf_transfer\n",
|
| 348 |
+
" return True\n",
|
| 349 |
+
" except ImportError:\n",
|
| 350 |
+
" return False\n",
|
| 351 |
+
" return False\n",
|
| 352 |
+
"\n",
|
| 353 |
+
" def _create_widgets(self) -> None:\n",
|
| 354 |
+
" # --- Repository Info ---\n",
|
| 355 |
+
" self.repo_info_html = HTML(value=\"<b>π Repository Details</b>\")\n",
|
| 356 |
+
" \n",
|
| 357 |
+
" # --- NEW: Dynamic Repo Fetching ---\n",
|
| 358 |
+
" self.org_name_text = Text(value=self.user_info['name'], placeholder='Organization or Username', description='Owner:', style={'description_width': 'initial'})\n",
|
| 359 |
+
" self.repo_name_dropdown = Dropdown(options=[], description='Repo:', style={'description_width': 'initial'}, layout=Layout(flex='1'))\n",
|
| 360 |
+
" self.fetch_repos_btn = Button(description=\"Fetch Repos\", button_style='info', tooltip=\"Fetch this owner's repositories\", layout=Layout(width='auto'))\n",
|
| 361 |
+
" \n",
|
| 362 |
+
" self.repo_type_dropdown = Dropdown(options=['model', 'dataset', 'space'], value='model', description='Repo Type:', style={'description_width': 'initial'})\n",
|
| 363 |
+
" self.repo_folder_text = Text(placeholder='Optional: e.g., models/v1', description='Remote Folder:', style={'description_width': 'initial', \"flex\": \"1 1 auto\"}, layout=Layout(width='auto'))\n",
|
| 364 |
+
"\n",
|
| 365 |
+
" # --- File Selection ---\n",
|
| 366 |
+
" self.file_section_html = HTML(value=\"<b>ποΈ File Selection & Source</b>\")\n",
|
| 367 |
+
" self.file_type_dropdown = Dropdown(options=self.file_types, value='safetensors', description='File Type:', style={'description_width': 'initial'})\n",
|
| 368 |
+
" self.sort_by_dropdown = Dropdown(options=['name', 'date', 'size'], value='name', description='Sort By:', style={'description_width': 'initial'}) # --- NEW: Sort by size\n",
|
| 369 |
+
" self.recursive_search_checkbox = Checkbox(value=False, description='Search Subdirectories', indent=False)\n",
|
| 370 |
+
"\n",
|
| 371 |
+
" self.directory_label = Label(value=\"Source Directory:\", layout=Layout(width='auto'))\n",
|
| 372 |
+
" self.directory_text = Text(value=self.current_directory, description=\"\", style={'description_width': '0px'}, layout=Layout(width=\"auto\", flex='1 1 auto'))\n",
|
| 373 |
+
" self.directory_update_btn = Button(description='π List Files', button_style='info', tooltip='Change source directory and refresh file list', layout=Layout(width='auto'))\n",
|
| 374 |
+
"\n",
|
| 375 |
+
" # --- Commit Details ---\n",
|
| 376 |
+
" self.commit_section_html = HTML(value=\"<b>π Commit Details</b>\")\n",
|
| 377 |
+
" self.commit_msg_textarea = Textarea(value=\"Upload files via SmartUploader\", placeholder='Enter your commit message', description='Message:', style={'description_width': 'initial'}, layout=Layout(width='98%', height='60px'))\n",
|
| 378 |
+
" \n",
|
| 379 |
+
" # --- NEW: Single Commit & Repo Creation Options ---\n",
|
| 380 |
+
" self.single_commit_checkbox = Checkbox(value=True, description='Single Commit', indent=False, tooltip=\"Upload all files in one commit.\")\n",
|
| 381 |
+
" self.create_repo_checkbox = Checkbox(value=True, description='Create repo if not exists', indent=False)\n",
|
| 382 |
+
" self.private_repo_checkbox = Checkbox(value=False, description='Make repo private', indent=False)\n",
|
| 383 |
+
"\n",
|
| 384 |
+
" # --- Upload Settings ---\n",
|
| 385 |
+
" self.upload_section_html = HTML(value=\"<b>π Upload Settings</b>\")\n",
|
| 386 |
+
" self.create_pr_checkbox = Checkbox(value=False, description='Create Pull Request', indent=False)\n",
|
| 387 |
+
" self.clear_after_checkbox = Checkbox(value=True, description='Clear output after upload', indent=False)\n",
|
| 388 |
+
" # --- NEW: Concurrent Uploads ---\n",
|
| 389 |
+
" self.concurrent_uploads_checkbox = Checkbox(value=True, description='Enable Concurrent Uploads (faster)', indent=False)\n",
|
| 390 |
+
"\n",
|
| 391 |
+
" # --- Action Buttons ---\n",
|
| 392 |
+
" self.upload_button = Button(description='β¬οΈ Upload Selected Files', button_style='success', tooltip='Start upload process', layout=Layout(width='auto', height='auto'))\n",
|
| 393 |
+
" self.clear_output_button = Button(description='π§Ή Clear Output Log', button_style='warning', tooltip='Clear the output log area', layout=Layout(width='auto'))\n",
|
| 394 |
+
"\n",
|
| 395 |
+
" # --- File Picker & Output ---\n",
|
| 396 |
+
" self.file_picker_selectmultiple = SelectMultiple(options=[], description='Files:', layout=Layout(width=\"98%\", height=\"200px\"), style={'description_width': 'initial'})\n",
|
| 397 |
+
" self.output_area = Output(layout=Layout(padding='10px', border='1px solid #ccc', margin_top='10px', width='98%', max_height='400px', overflow_y='auto'))\n",
|
| 398 |
+
"\n",
|
| 399 |
+
" # --- Progress Display Area ---\n",
|
| 400 |
+
" self.current_file_label = Label(value=\"N/A\")\n",
|
| 401 |
+
" self.file_count_label = Label(value=\"File 0/0\")\n",
|
| 402 |
+
" self.progress_bar = FloatProgress(value=0, min=0, max=100, description='Overall:', bar_style='info', layout=Layout(width='85%'))\n",
|
| 403 |
+
" self.progress_percent_label = Label(value=\"0%\")\n",
|
| 404 |
+
"\n",
|
| 405 |
+
" self.progress_display_box = VBox([\n",
|
| 406 |
+
" HBox([Label(\"Current File:\", layout=Layout(width='100px')), self.current_file_label]),\n",
|
| 407 |
+
" HBox([Label(\"File Count:\", layout=Layout(width='100px')), self.file_count_label]),\n",
|
| 408 |
+
" HBox([self.progress_bar, self.progress_percent_label], layout=Layout(align_items='center'))\n",
|
| 409 |
+
" ], layout=Layout(visibility='hidden', margin='10px 0', padding='10px', border='1px solid #ddd', width='98%'))\n",
|
| 410 |
+
"\n",
|
| 411 |
+
" def _set_ui_busy_state(self, busy: bool) -> None:\n",
|
| 412 |
+
" \"\"\" --- NEW: Disables key UI elements during operations. --- \"\"\"\n",
|
| 413 |
+
" self.upload_button.disabled = busy\n",
|
| 414 |
+
" self.directory_update_btn.disabled = busy\n",
|
| 415 |
+
" self.fetch_repos_btn.disabled = busy\n",
|
| 416 |
+
" self.upload_button.icon = 'spinner' if busy else ''\n",
|
| 417 |
+
"\n",
|
| 418 |
+
" def _bind_events(self) -> None:\n",
|
| 419 |
+
" self.directory_update_btn.on_click(self._update_directory_and_files)\n",
|
| 420 |
+
" self.fetch_repos_btn.on_click(self._fetch_user_repos) # --- NEW ---\n",
|
| 421 |
+
" self.upload_button.on_click(self._upload_files_handler)\n",
|
| 422 |
+
" self.clear_output_button.on_click(lambda _: self.output_area.clear_output(wait=True))\n",
|
| 423 |
+
" # Update file list when any relevant option changes\n",
|
| 424 |
+
" for widget in [self.file_type_dropdown, self.sort_by_dropdown, self.recursive_search_checkbox]:\n",
|
| 425 |
+
" widget.observe(self._update_files, names='value')\n",
|
| 426 |
+
"\n",
|
| 427 |
+
" def _fetch_user_repos(self, _) -> None:\n",
|
| 428 |
+
" \"\"\" --- NEW: Fetches repositories for the specified owner. --- \"\"\"\n",
|
| 429 |
+
" owner = self.org_name_text.value.strip()\n",
|
| 430 |
+
" if not owner:\n",
|
| 431 |
+
" with self.output_area: print(\"β Please enter an owner (user/org) name.\")\n",
|
| 432 |
+
" return\n",
|
| 433 |
+
" \n",
|
| 434 |
+
" with self.output_area:\n",
|
| 435 |
+
" clear_output(wait=True)\n",
|
| 436 |
+
" print(f\"Fetching repos for '{owner}'...\")\n",
|
| 437 |
+
" try:\n",
|
| 438 |
+
" self._set_ui_busy_state(True)\n",
|
| 439 |
+
" repos = list(self.api.list_repos(author=owner, repo_type=self.repo_type_dropdown.value))\n",
|
| 440 |
+
" repo_names = sorted([repo.repo_id.split('/')[1] for repo in repos])\n",
|
| 441 |
+
" self.repo_name_dropdown.options = repo_names\n",
|
| 442 |
+
" if repo_names:\n",
|
| 443 |
+
" self.repo_name_dropdown.value = repo_names[0]\n",
|
| 444 |
+
" print(f\"β
Found {len(repo_names)} repositories.\")\n",
|
| 445 |
+
" except Exception as e:\n",
|
| 446 |
+
" print(f\"β Could not fetch repositories: {e}\")\n",
|
| 447 |
+
" finally:\n",
|
| 448 |
+
" self._set_ui_busy_state(False)\n",
|
| 449 |
+
"\n",
|
| 450 |
+
" def _update_directory_and_files(self, _) -> None:\n",
|
| 451 |
+
" new_dir = self.directory_text.value.strip()\n",
|
| 452 |
+
" if not new_dir or not os.path.isdir(new_dir):\n",
|
| 453 |
+
" with self.output_area:\n",
|
| 454 |
+
" clear_output(wait=True)\n",
|
| 455 |
+
" print(f\"β Invalid or empty directory path: {new_dir}\")\n",
|
| 456 |
+
" return\n",
|
| 457 |
+
"\n",
|
| 458 |
+
" self.current_directory = os.path.abspath(new_dir)\n",
|
| 459 |
+
" self.directory_text.value = self.current_directory\n",
|
| 460 |
+
" self._update_files(None)\n",
|
| 461 |
+
"\n",
|
| 462 |
+
" def _update_files(self, _) -> None:\n",
|
| 463 |
+
" self._set_ui_busy_state(True)\n",
|
| 464 |
+
" file_extension = self.file_type_dropdown.value\n",
|
| 465 |
+
" self.output_area.clear_output(wait=True)\n",
|
| 466 |
+
" try:\n",
|
| 467 |
+
" source_path = Path(self.current_directory)\n",
|
| 468 |
+
" if not source_path.is_dir():\n",
|
| 469 |
+
" with self.output_area: print(f\"β οΈ Source directory '{self.current_directory}' is not valid.\")\n",
|
| 470 |
+
" self.file_picker_selectmultiple.options = []\n",
|
| 471 |
+
" return\n",
|
| 472 |
+
"\n",
|
| 473 |
+
" # --- MODIFIED: Cleaner file search and sorting ---\n",
|
| 474 |
+
" pattern = f'**/*.{file_extension}' if self.recursive_search_checkbox.value else f'*.{file_extension}'\n",
|
| 475 |
+
" found_paths = list(source_path.glob(pattern))\n",
|
| 476 |
+
" \n",
|
| 477 |
+
" # Use a dictionary to hold file info for easier sorting\n",
|
| 478 |
+
" files_info = {}\n",
|
| 479 |
+
" for p in found_paths:\n",
|
| 480 |
+
" if p.is_file(): # More robust check\n",
|
| 481 |
+
" stat = p.stat()\n",
|
| 482 |
+
" files_info[str(p)] = {'mtime': stat.st_mtime, 'size': stat.st_size, 'name': p.name.lower()}\n",
|
| 483 |
+
"\n",
|
| 484 |
+
" sort_key = self.sort_by_dropdown.value\n",
|
| 485 |
+
" reverse_sort = sort_key in ['date', 'size']\n",
|
| 486 |
+
" \n",
|
| 487 |
+
" sorted_paths = sorted(files_info.keys(), key=lambda p: files_info[p][sort_key], reverse=reverse_sort)\n",
|
| 488 |
+
" \n",
|
| 489 |
+
" # --- MODIFIED: Display relative paths with file size ---\n",
|
| 490 |
+
" display_options = []\n",
|
| 491 |
+
" for abs_path_str in sorted_paths:\n",
|
| 492 |
+
" file_size = files_info[abs_path_str]['size']\n",
|
| 493 |
+
" display_name = f\"{os.path.relpath(abs_path_str, self.current_directory)} ({self._format_size(file_size)})\"\n",
|
| 494 |
+
" display_options.append((display_name, abs_path_str))\n",
|
| 495 |
+
" \n",
|
| 496 |
+
" self.file_picker_selectmultiple.options = display_options\n",
|
| 497 |
+
" \n",
|
| 498 |
+
" with self.output_area:\n",
|
| 499 |
+
" if not display_options:\n",
|
| 500 |
+
" print(f\"π€· No '.{file_extension}' files found in '{self.current_directory}'.\")\n",
|
| 501 |
+
" else:\n",
|
| 502 |
+
" print(f\"β¨ Found {len(display_options)} '.{file_extension}' files. Select files to upload.\")\n",
|
| 503 |
+
"\n",
|
| 504 |
+
" except Exception as e:\n",
|
| 505 |
+
" with self.output_area:\n",
|
| 506 |
+
" clear_output(wait=True); print(f\"β Error listing files: {e}\"); traceback.print_exc()\n",
|
| 507 |
+
" finally:\n",
|
| 508 |
+
" self._set_ui_busy_state(False)\n",
|
| 509 |
+
"\n",
|
| 510 |
+
" def _format_size(self, size_bytes: int) -> str:\n",
|
| 511 |
+
" if size_bytes < 0: return \"Invalid size\"\n",
|
| 512 |
+
" if size_bytes == 0: return \"0 B\"\n",
|
| 513 |
+
" units = (\"B\", \"KB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\")\n",
|
| 514 |
+
" i = math.floor(math.log(size_bytes, 1024)) if size_bytes > 0 else 0\n",
|
| 515 |
+
" if i >= len(units): i = len(units) - 1\n",
|
| 516 |
+
" s = round(size_bytes / (1024 ** i), 2)\n",
|
| 517 |
+
" return f\"{s} {units[i]}\"\n",
|
| 518 |
+
"\n",
|
| 519 |
+
" def _upload_files_handler(self, _) -> None:\n",
|
| 520 |
+
" org_or_user = self.org_name_text.value.strip()\n",
|
| 521 |
+
" repo_name = self.repo_name_dropdown.value.strip() # --- MODIFIED: Use dropdown value\n",
|
| 522 |
+
"\n",
|
| 523 |
+
" if not org_or_user or not repo_name:\n",
|
| 524 |
+
" with self.output_area: clear_output(wait=True); print(\"β Please fill in 'Owner' and select a 'Repo'.\")\n",
|
| 525 |
+
" return\n",
|
| 526 |
+
"\n",
|
| 527 |
+
" repo_id = f\"{org_or_user}/{repo_name}\"\n",
|
| 528 |
+
" selected_file_paths = list(self.file_picker_selectmultiple.value)\n",
|
| 529 |
+
"\n",
|
| 530 |
+
" if not selected_file_paths:\n",
|
| 531 |
+
" with self.output_area: clear_output(wait=True); print(\"π Nothing selected for upload.\")\n",
|
| 532 |
+
" return\n",
|
| 533 |
+
"\n",
|
| 534 |
+
" self._set_ui_busy_state(True)\n",
|
| 535 |
+
" self.output_area.clear_output(wait=True)\n",
|
| 536 |
+
" \n",
|
| 537 |
+
" try:\n",
|
| 538 |
+
" # --- NEW: Automatic Repo Creation ---\n",
|
| 539 |
+
" if self.create_repo_checkbox.value:\n",
|
| 540 |
+
" with self.output_area: print(f\"Ensuring repo '{repo_id}' exists...\")\n",
|
| 541 |
+
" self.api.create_repo(\n",
|
| 542 |
+
" repo_id=repo_id,\n",
|
| 543 |
+
" repo_type=self.repo_type_dropdown.value,\n",
|
| 544 |
+
" private=self.private_repo_checkbox.value,\n",
|
| 545 |
+
" exist_ok=True\n",
|
| 546 |
+
" )\n",
|
| 547 |
+
"\n",
|
| 548 |
+
" with self.output_area:\n",
|
| 549 |
+
" print(f\"π― Preparing to upload to: https://huggingface.co/{repo_id}\")\n",
|
| 550 |
+
" if self.hf_transfer_active: print(\"π HF_TRANSFER is enabled.\")\n",
|
| 551 |
+
" else: print(\"βΉοΈ For faster uploads, run `%pip install -q hf_transfer` and restart kernel.\")\n",
|
| 552 |
+
"\n",
|
| 553 |
+
" # --- MODIFIED: Handle single vs. multi-commit ---\n",
|
| 554 |
+
" if self.single_commit_checkbox.value:\n",
|
| 555 |
+
" self._upload_as_single_commit(repo_id, selected_file_paths)\n",
|
| 556 |
+
" else:\n",
|
| 557 |
+
" self._upload_as_multiple_commits(repo_id, selected_file_paths)\n",
|
| 558 |
+
"\n",
|
| 559 |
+
" except Exception as e:\n",
|
| 560 |
+
" with self.output_area:\n",
|
| 561 |
+
" print(f\"β An unexpected error occurred: {e}\")\n",
|
| 562 |
+
" traceback.print_exc()\n",
|
| 563 |
+
" finally:\n",
|
| 564 |
+
" self._set_ui_busy_state(False)\n",
|
| 565 |
+
" if self.clear_after_checkbox.value:\n",
|
| 566 |
+
" time.sleep(5)\n",
|
| 567 |
+
" self.output_area.clear_output(wait=True)\n",
|
| 568 |
+
" self.progress_display_box.layout.visibility = 'hidden'\n",
|
| 569 |
+
"\n",
|
| 570 |
+
" def _upload_as_single_commit(self, repo_id: str, file_paths: List[str]) -> None:\n",
|
| 571 |
+
" \"\"\" --- NEW: Logic for uploading all files in a single commit. --- \"\"\"\n",
|
| 572 |
+
" self.progress_display_box.layout.visibility = 'visible'\n",
|
| 573 |
+
" self.progress_bar.value = 0\n",
|
| 574 |
+
" self.progress_percent_label.value = \"0%\"\n",
|
| 575 |
+
" self.current_file_label.value = \"Preparing operations...\"\n",
|
| 576 |
+
" \n",
|
| 577 |
+
" repo_folder_prefix = self.repo_folder_text.value.strip().replace('\\\\', '/')\n",
|
| 578 |
+
" \n",
|
| 579 |
+
" operations = []\n",
|
| 580 |
+
" for path_str in file_paths:\n",
|
| 581 |
+
" path_in_repo_base = os.path.relpath(path_str, self.current_directory).replace('\\\\', '/')\n",
|
| 582 |
+
" path_in_repo = f\"{repo_folder_prefix}/{path_in_repo_base}\" if repo_folder_prefix.strip('/') else path_in_repo_base\n",
|
| 583 |
+
" operations.append(CommitOperationAdd(path_in_repo=path_in_repo, path_or_fileobj=path_str))\n",
|
| 584 |
+
" \n",
|
| 585 |
+
" commit_message = self.commit_msg_textarea.value or f\"Upload {len(operations)} files\"\n",
|
| 586 |
+
" \n",
|
| 587 |
+
" with self.output_area:\n",
|
| 588 |
+
" print(f\"π Starting upload of {len(operations)} files in a single commit...\")\n",
|
| 589 |
+
" \n",
|
| 590 |
+
" start_time = time.time()\n",
|
| 591 |
+
" \n",
|
| 592 |
+
" # Note: Progress bar for single commit is harder. Here we just show completion.\n",
|
| 593 |
+
" try:\n",
|
| 594 |
+
" commit_info = self.api.create_commit(\n",
|
| 595 |
+
" repo_id=repo_id,\n",
|
| 596 |
+
" operations=operations,\n",
|
| 597 |
+
" commit_message=commit_message,\n",
|
| 598 |
+
" repo_type=self.repo_type_dropdown.value,\n",
|
| 599 |
+
" create_pr=self.create_pr_checkbox.value\n",
|
| 600 |
+
" )\n",
|
| 601 |
+
" duration = time.time() - start_time\n",
|
| 602 |
+
" self.progress_bar.value = 100\n",
|
| 603 |
+
" self.progress_percent_label.value = \"100%\"\n",
|
| 604 |
+
" self.current_file_label.value = \"Completed.\"\n",
|
| 605 |
+
" with self.output_area:\n",
|
| 606 |
+
" print(f\"β
Successfully committed {len(operations)} files in {duration:.1f}s.\")\n",
|
| 607 |
+
" print(f\" View commit: {commit_info.commit_url}\")\n",
|
| 608 |
+
" except Exception as e:\n",
|
| 609 |
+
" with self.output_area:\n",
|
| 610 |
+
" print(f\"β Commit failed: {e}\")\n",
|
| 611 |
+
" traceback.print_exc()\n",
|
| 612 |
+
"\n",
|
| 613 |
+
" def _upload_as_multiple_commits(self, repo_id: str, file_paths: List[str]) -> None:\n",
|
| 614 |
+
" \"\"\" --- MODIFIED: Original logic now in its own function, with concurrency. --- \"\"\"\n",
|
| 615 |
+
" self.progress_display_box.layout.visibility = 'visible'\n",
|
| 616 |
+
" self.progress_bar.value = 0\n",
|
| 617 |
+
" total_files = len(file_paths)\n",
|
| 618 |
+
" \n",
|
| 619 |
+
" repo_type = self.repo_type_dropdown.value\n",
|
| 620 |
+
" repo_folder_prefix = self.repo_folder_text.value.strip().replace('\\\\', '/')\n",
|
| 621 |
+
" base_commit_msg = self.commit_msg_textarea.value or \"Upload file\"\n",
|
| 622 |
+
" \n",
|
| 623 |
+
" success_count = 0\n",
|
| 624 |
+
" files_processed = 0\n",
|
| 625 |
+
"\n",
|
| 626 |
+
" # --- NEW: Concurrent Upload Logic ---\n",
|
| 627 |
+
" use_concurrency = self.concurrent_uploads_checkbox.value\n",
|
| 628 |
+
" max_workers = 4 if use_concurrency else 1\n",
|
| 629 |
+
"\n",
|
| 630 |
+
" with ThreadPoolExecutor(max_workers=max_workers) as executor:\n",
|
| 631 |
+
" future_to_path = {}\n",
|
| 632 |
+
" for path_str in file_paths:\n",
|
| 633 |
+
" path_in_repo_base = os.path.relpath(path_str, self.current_directory).replace('\\\\', '/')\n",
|
| 634 |
+
" path_in_repo = f\"{repo_folder_prefix}/{path_in_repo_base}\" if repo_folder_prefix.strip('/') else path_in_repo_base\n",
|
| 635 |
+
" commit_message_for_file = f\"{base_commit_msg} ({Path(path_str).name})\"\n",
|
| 636 |
+
"\n",
|
| 637 |
+
" future = executor.submit(\n",
|
| 638 |
+
" self.api.upload_file,\n",
|
| 639 |
+
" path_or_fileobj=path_str,\n",
|
| 640 |
+
" path_in_repo=path_in_repo,\n",
|
| 641 |
+
" repo_id=repo_id,\n",
|
| 642 |
+
" repo_type=repo_type,\n",
|
| 643 |
+
" create_pr=self.create_pr_checkbox.value,\n",
|
| 644 |
+
" commit_message=commit_message_for_file,\n",
|
| 645 |
+
" )\n",
|
| 646 |
+
" future_to_path[future] = path_str\n",
|
| 647 |
+
"\n",
|
| 648 |
+
" for future in as_completed(future_to_path):\n",
|
| 649 |
+
" local_path_str = future_to_path[future]\n",
|
| 650 |
+
" file_name = Path(local_path_str).name\n",
|
| 651 |
+
" self.current_file_label.value = file_name\n",
|
| 652 |
+
" \n",
|
| 653 |
+
" try:\n",
|
| 654 |
+
" response_url = future.result()\n",
|
| 655 |
+
" with self.output_area: print(f\"β
Uploaded '{file_name}'\\n View at: {response_url}\")\n",
|
| 656 |
+
" success_count += 1\n",
|
| 657 |
+
" except Exception as e:\n",
|
| 658 |
+
" with self.output_area: \n",
|
| 659 |
+
" print(f\"β Error uploading {file_name}: {e}\")\n",
|
| 660 |
+
" traceback.print_exc()\n",
|
| 661 |
+
" finally:\n",
|
| 662 |
+
" files_processed += 1\n",
|
| 663 |
+
" percentage = int((files_processed / total_files) * 100)\n",
|
| 664 |
+
" self.progress_bar.value = percentage\n",
|
| 665 |
+
" self.progress_percent_label.value = f\"{percentage}%\"\n",
|
| 666 |
+
" self.file_count_label.value = f\"File {files_processed}/{total_files}\"\n",
|
| 667 |
+
"\n",
|
| 668 |
+
" with self.output_area:\n",
|
| 669 |
+
" print(f\"\\nβ¨ Upload complete. {success_count}/{total_files} files processed. β¨\")\n",
|
| 670 |
+
" # Final links logic (unchanged)\n",
|
| 671 |
+
"\n",
|
| 672 |
+
" def display(self) -> None:\n",
|
| 673 |
+
" # --- MODIFIED: Layout updated for new widgets ---\n",
|
| 674 |
+
" repo_select_box = HBox([self.org_name_text, self.repo_name_dropdown, self.fetch_repos_btn], layout=Layout(flex_flow='wrap', justify_content='space-between', align_items='center'))\n",
|
| 675 |
+
" repo_opts_box = HBox([self.repo_type_dropdown, self.create_repo_checkbox, self.private_repo_checkbox], layout=Layout(flex_flow='wrap', justify_content='space-between', align_items='center', margin='5px 0'))\n",
|
| 676 |
+
" \n",
|
| 677 |
+
" dir_select_box = HBox([self.directory_label, self.directory_text, self.directory_update_btn], layout=Layout(width='100%', align_items='center'))\n",
|
| 678 |
+
" file_opts_box = HBox([self.file_type_dropdown, self.sort_by_dropdown, self.recursive_search_checkbox], layout=Layout(flex_flow='wrap', justify_content='space-between', align_items='center'))\n",
|
| 679 |
+
" \n",
|
| 680 |
+
" commit_opts_box = HBox([self.single_commit_checkbox], layout=Layout(margin='5px 0'))\n",
|
| 681 |
+
" \n",
|
| 682 |
+
" upload_opts_box = HBox([self.create_pr_checkbox, self.clear_after_checkbox, self.concurrent_uploads_checkbox], layout=Layout(margin='5px 0', flex_flow='wrap'))\n",
|
| 683 |
+
" action_buttons_box = HBox([self.upload_button, self.clear_output_button], layout=Layout(margin='10px 0 0 0', spacing='10px'))\n",
|
| 684 |
+
"\n",
|
| 685 |
+
" main_layout = VBox([\n",
|
| 686 |
+
" self.repo_info_html, repo_select_box, repo_opts_box, self.repo_folder_text,\n",
|
| 687 |
+
" HTML(\"<hr>\"),\n",
|
| 688 |
+
" self.file_section_html, file_opts_box, dir_select_box,\n",
|
| 689 |
+
" self.file_picker_selectmultiple,\n",
|
| 690 |
+
" HTML(\"<hr>\"),\n",
|
| 691 |
+
" self.commit_section_html, self.commit_msg_textarea, commit_opts_box,\n",
|
| 692 |
+
" HTML(\"<hr>\"),\n",
|
| 693 |
+
" self.upload_section_html, upload_opts_box,\n",
|
| 694 |
+
" action_buttons_box,\n",
|
| 695 |
+
" self.progress_display_box,\n",
|
| 696 |
+
" self.output_area\n",
|
| 697 |
+
" ], layout=Layout(width='800px', padding='10px', border='1px solid lightgray'))\n",
|
| 698 |
+
" \n",
|
| 699 |
+
" display(main_layout)\n",
|
| 700 |
+
"\n",
|
| 701 |
+
"# How to use it:\n",
|
| 702 |
+
"# uploader = SmartHuggingFaceUploader()\n",
|
| 703 |
+
"# uploader.display()"
|
| 704 |
+
]
|
| 705 |
+
},
|
| 706 |
+
{
|
| 707 |
+
"cell_type": "markdown",
|
| 708 |
+
"metadata": {},
|
| 709 |
+
"source": [
|
| 710 |
+
"# π <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Uploader Widget! \n",
|
| 711 |
+
"\n",
|
| 712 |
+
"**Run the next cell to initiate the uploader widget!**\n"
|
| 713 |
+
]
|
| 714 |
+
},
|
| 715 |
+
{
|
| 716 |
+
"cell_type": "code",
|
| 717 |
+
"execution_count": 6,
|
| 718 |
+
"metadata": {},
|
| 719 |
+
"outputs": [
|
| 720 |
+
{
|
| 721 |
+
"name": "stdout",
|
| 722 |
+
"output_type": "stream",
|
| 723 |
+
"text": [
|
| 724 |
+
"π Initializing the Smart Hugging Face Uploader...\n"
|
| 725 |
+
]
|
| 726 |
+
},
|
| 727 |
+
{
|
| 728 |
+
"data": {
|
| 729 |
+
"application/vnd.jupyter.widget-view+json": {
|
| 730 |
+
"model_id": "8d07f13a250d4d42a157b89c15148ae5",
|
| 731 |
+
"version_major": 2,
|
| 732 |
+
"version_minor": 0
|
| 733 |
+
},
|
| 734 |
+
"text/plain": [
|
| 735 |
+
"VBox(children=(HTML(value='<b>π Repository Details</b>'), HBox(children=(Text(value='Duskfallcrew', descriptioβ¦"
|
| 736 |
+
]
|
| 737 |
+
},
|
| 738 |
+
"metadata": {},
|
| 739 |
+
"output_type": "display_data"
|
| 740 |
+
},
|
| 741 |
+
{
|
| 742 |
+
"name": "stdout",
|
| 743 |
+
"output_type": "stream",
|
| 744 |
+
"text": [
|
| 745 |
+
"β
Uploader interface is ready. You can now select files and upload.\n"
|
| 746 |
+
]
|
| 747 |
+
}
|
| 748 |
+
],
|
| 749 |
+
"source": [
|
| 750 |
+
"# --- Uploader Widget ---\n",
|
| 751 |
+
"# This cell creates and displays the uploader interface.\n",
|
| 752 |
+
"# Make sure you have run the cell containing the SmartHuggingFaceUploader class definition first!\n",
|
| 753 |
+
"\n",
|
| 754 |
+
"print(\"π Initializing the Smart Hugging Face Uploader...\")\n",
|
| 755 |
+
"\n",
|
| 756 |
+
"# Use the new class name here\n",
|
| 757 |
+
"uploader = SmartHuggingFaceUploader() \n",
|
| 758 |
+
"uploader.display()\n",
|
| 759 |
+
"\n",
|
| 760 |
+
"print(\"β
Uploader interface is ready. You can now select files and upload.\")"
|
| 761 |
+
]
|
| 762 |
+
},
|
| 763 |
+
{
|
| 764 |
+
"cell_type": "markdown",
|
| 765 |
+
"metadata": {},
|
| 766 |
+
"source": [
|
| 767 |
+
"## ποΈ Smart Image Zipper\n",
|
| 768 |
+
"\n",
|
| 769 |
+
"This widget helps you create a zip archive of images from a specified folder. It's designed to give you more control and feedback than a simple zipping script.\n",
|
| 770 |
+
"\n",
|
| 771 |
+
"### Smart Features:\n",
|
| 772 |
+
"* **Selective Zipping:** Choose which image file types (`.png`, `.jpg`, etc.) you want to include.\n",
|
| 773 |
+
"* **Analyze Before Zipping:** Click \"Analyze Folder\" to get a preview of how many files will be included and their total size.\n",
|
| 774 |
+
"* **Live Progress:** A progress bar shows the zipping process in real-time, which is essential for large datasets.\n",
|
| 775 |
+
"* **Download Link:** Once complete, it provides a direct download link for your new archive."
|
| 776 |
+
]
|
| 777 |
+
},
|
| 778 |
+
{
|
| 779 |
+
"cell_type": "code",
|
| 780 |
+
"execution_count": 13,
|
| 781 |
+
"metadata": {},
|
| 782 |
+
"outputs": [],
|
| 783 |
+
"source": [
|
| 784 |
+
"# Cell: Smart Image Zipper Widget\n",
|
| 785 |
+
"# -----------------------------------------------------------------------------\n",
|
| 786 |
+
"import ipywidgets as widgets\n",
|
| 787 |
+
"from IPython.display import display, FileLink, HTML, clear_output\n",
|
| 788 |
+
"import zipfile\n",
|
| 789 |
+
"import os\n",
|
| 790 |
+
"from pathlib import Path\n",
|
| 791 |
+
"\n",
|
| 792 |
+
"class SmartZipper:\n",
|
| 793 |
+
" \"\"\"A widget to selectively zip image files with analysis and progress feedback.\"\"\"\n",
|
| 794 |
+
" \n",
|
| 795 |
+
" def __init__(self):\n",
|
| 796 |
+
" self.files_to_zip = []\n",
|
| 797 |
+
" self._create_widgets()\n",
|
| 798 |
+
" self._bind_events()\n",
|
| 799 |
+
"\n",
|
| 800 |
+
" def _create_widgets(self):\n",
|
| 801 |
+
" # --- 1. Folder & File Naming ---\n",
|
| 802 |
+
" self.folder_path_text = widgets.Text(\n",
|
| 803 |
+
" value=os.getcwd(),\n",
|
| 804 |
+
" placeholder='Enter the path to the folder containing images',\n",
|
| 805 |
+
" description='Source Folder:',\n",
|
| 806 |
+
" style={'description_width': 'initial'},\n",
|
| 807 |
+
" layout=widgets.Layout(width='98%')\n",
|
| 808 |
+
" )\n",
|
| 809 |
+
" self.zip_name_text = widgets.Text(\n",
|
| 810 |
+
" value='image_archive',\n",
|
| 811 |
+
" placeholder='Name for the final .zip file',\n",
|
| 812 |
+
" description='Zip Name:',\n",
|
| 813 |
+
" style={'description_width': 'initial'},\n",
|
| 814 |
+
" layout=widgets.Layout(width='50%')\n",
|
| 815 |
+
" )\n",
|
| 816 |
+
"\n",
|
| 817 |
+
" # --- 2. File Type Selection ---\n",
|
| 818 |
+
" self.file_types_label = widgets.Label(value=\"Select image types to include:\")\n",
|
| 819 |
+
" self.image_types_checkboxes = [\n",
|
| 820 |
+
" widgets.Checkbox(description=ext, value=True, indent=False) \n",
|
| 821 |
+
" for ext in ['.jpg', '.jpeg', '.png', '.webp', '.gif', '.bmp', '.tiff']\n",
|
| 822 |
+
" ]\n",
|
| 823 |
+
" self.image_types_box = widgets.HBox(self.image_types_checkboxes, layout=widgets.Layout(flex_flow='wrap'))\n",
|
| 824 |
+
"\n",
|
| 825 |
+
" # --- 3. Action Buttons & Progress ---\n",
|
| 826 |
+
" self.analyze_button = widgets.Button(\n",
|
| 827 |
+
" description=\"1. Analyze Folder\", \n",
|
| 828 |
+
" button_style='info', \n",
|
| 829 |
+
" icon='search',\n",
|
| 830 |
+
" tooltip=\"Scan the folder and see what will be zipped.\"\n",
|
| 831 |
+
" )\n",
|
| 832 |
+
" self.zip_button = widgets.Button(\n",
|
| 833 |
+
" description=\"2. Create Zip Archive\", \n",
|
| 834 |
+
" button_style='success', \n",
|
| 835 |
+
" icon='archive',\n",
|
| 836 |
+
" tooltip=\"Start the zipping process.\",\n",
|
| 837 |
+
" disabled=True # Disabled until analysis is complete\n",
|
| 838 |
+
" )\n",
|
| 839 |
+
" self.progress_bar = widgets.FloatProgress(\n",
|
| 840 |
+
" value=0, min=0, max=1.0, description='Zipping:', \n",
|
| 841 |
+
" bar_style='info', orientation='horizontal',\n",
|
| 842 |
+
" layout=widgets.Layout(visibility='hidden', width='98%')\n",
|
| 843 |
+
" )\n",
|
| 844 |
+
" \n",
|
| 845 |
+
" # --- 4. Output Area ---\n",
|
| 846 |
+
" self.output_area = widgets.Output(layout=widgets.Layout(padding='10px', border='1px solid #ccc', margin_top='10px', width='98%'))\n",
|
| 847 |
+
"\n",
|
| 848 |
+
" # --- 5. Assemble the Layout ---\n",
|
| 849 |
+
" self.layout = widgets.VBox([\n",
|
| 850 |
+
" widgets.HTML(\"<h3>1. Select Source and Name</h3>\"),\n",
|
| 851 |
+
" self.folder_path_text,\n",
|
| 852 |
+
" self.zip_name_text,\n",
|
| 853 |
+
" widgets.HTML(\"<hr><h3>2. Choose File Types</h3>\"),\n",
|
| 854 |
+
" self.image_types_box,\n",
|
| 855 |
+
" widgets.HTML(\"<hr><h3>3. Execute</h3>\"),\n",
|
| 856 |
+
" widgets.HBox([self.analyze_button, self.zip_button]),\n",
|
| 857 |
+
" self.progress_bar,\n",
|
| 858 |
+
" self.output_area\n",
|
| 859 |
+
" ], layout=widgets.Layout(width='700px', padding='10px', border='1px solid lightgray'))\n",
|
| 860 |
+
"\n",
|
| 861 |
+
" def _bind_events(self):\n",
|
| 862 |
+
" self.analyze_button.on_click(self._analyze_folder)\n",
|
| 863 |
+
" self.zip_button.on_click(self._create_zip_archive)\n",
|
| 864 |
+
"\n",
|
| 865 |
+
" def _set_busy_state(self, busy):\n",
|
| 866 |
+
" \"\"\"Disable buttons during long operations.\"\"\"\n",
|
| 867 |
+
" self.analyze_button.disabled = busy\n",
|
| 868 |
+
" self.zip_button.disabled = busy\n",
|
| 869 |
+
" self.analyze_button.icon = 'spinner' if busy else 'search'\n",
|
| 870 |
+
"\n",
|
| 871 |
+
" def _analyze_folder(self, b):\n",
|
| 872 |
+
" self.zip_button.disabled = True\n",
|
| 873 |
+
" self.files_to_zip.clear()\n",
|
| 874 |
+
" self.output_area.clear_output()\n",
|
| 875 |
+
" self._set_busy_state(True)\n",
|
| 876 |
+
" \n",
|
| 877 |
+
" source_folder = self.folder_path_text.value.strip()\n",
|
| 878 |
+
" selected_extensions = [cb.description for cb in self.image_types_checkboxes if cb.value]\n",
|
| 879 |
+
" \n",
|
| 880 |
+
" with self.output_area:\n",
|
| 881 |
+
" if not Path(source_folder).is_dir():\n",
|
| 882 |
+
" display(HTML(\"<b style='color:red;'>Error: The specified source folder is not a valid directory.</b>\"))\n",
|
| 883 |
+
" self._set_busy_state(False)\n",
|
| 884 |
+
" return\n",
|
| 885 |
+
" \n",
|
| 886 |
+
" print(f\"π Scanning '{source_folder}' for {', '.join(selected_extensions)} files...\")\n",
|
| 887 |
+
" \n",
|
| 888 |
+
" total_size = 0\n",
|
| 889 |
+
" for file_path in Path(source_folder).rglob('*'):\n",
|
| 890 |
+
" if file_path.is_file() and file_path.suffix.lower() in selected_extensions:\n",
|
| 891 |
+
" self.files_to_zip.append(file_path)\n",
|
| 892 |
+
" total_size += file_path.stat().st_size\n",
|
| 893 |
+
" \n",
|
| 894 |
+
" # Convert size to human-readable format\n",
|
| 895 |
+
" size_mb = total_size / (1024 * 1024)\n",
|
| 896 |
+
" \n",
|
| 897 |
+
" if not self.files_to_zip:\n",
|
| 898 |
+
" display(HTML(\"<b style='color:orange;'>Warning: No matching image files were found.</b>\"))\n",
|
| 899 |
+
" self._set_busy_state(False)\n",
|
| 900 |
+
" return\n",
|
| 901 |
+
" \n",
|
| 902 |
+
" display(HTML(f\"β
<b>Analysis Complete:</b> Found <b>{len(self.files_to_zip)}</b> matching image files, with a total size of <b>{size_mb:.2f} MB</b>.\"))\n",
|
| 903 |
+
" display(HTML(\"You can now proceed by clicking 'Create Zip Archive'.\"))\n",
|
| 904 |
+
" self.zip_button.disabled = False\n",
|
| 905 |
+
" \n",
|
| 906 |
+
" self._set_busy_state(False)\n",
|
| 907 |
+
"\n",
|
| 908 |
+
" def _create_zip_archive(self, b):\n",
|
| 909 |
+
" self._set_busy_state(True)\n",
|
| 910 |
+
" self.progress_bar.value = 0\n",
|
| 911 |
+
" self.progress_bar.layout.visibility = 'visible'\n",
|
| 912 |
+
" self.output_area.clear_output()\n",
|
| 913 |
+
" \n",
|
| 914 |
+
" zip_name = self.zip_name_text.value.strip()\n",
|
| 915 |
+
" final_zip_path = Path.cwd() / f\"{zip_name}.zip\"\n",
|
| 916 |
+
" \n",
|
| 917 |
+
" with self.output_area:\n",
|
| 918 |
+
" if not zip_name:\n",
|
| 919 |
+
" display(HTML(\"<b style='color:red;'>Error: Please provide a name for the zip file.</b>\"))\n",
|
| 920 |
+
" self._set_busy_state(False)\n",
|
| 921 |
+
" return\n",
|
| 922 |
+
" \n",
|
| 923 |
+
" print(f\"π¦ Creating archive at: {final_zip_path}...\")\n",
|
| 924 |
+
" \n",
|
| 925 |
+
" try:\n",
|
| 926 |
+
" with zipfile.ZipFile(final_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:\n",
|
| 927 |
+
" total_files = len(self.files_to_zip)\n",
|
| 928 |
+
" for i, file_path in enumerate(self.files_to_zip):\n",
|
| 929 |
+
" relative_path = file_path.relative_to(self.folder_path_text.value.strip())\n",
|
| 930 |
+
" zipf.write(file_path, relative_path)\n",
|
| 931 |
+
" self.progress_bar.value = (i + 1) / total_files\n",
|
| 932 |
+
"\n",
|
| 933 |
+
" display(HTML(f\"π <b>Successfully created '{final_zip_path.name}'!</b>\"))\n",
|
| 934 |
+
" display(HTML(\"You can now use this file in the uploader widget above, or download it using the link below.\"))\n",
|
| 935 |
+
" display(FileLink(str(final_zip_path)))\n",
|
| 936 |
+
"\n",
|
| 937 |
+
" except Exception as e:\n",
|
| 938 |
+
" display(HTML(f\"<b style='color:red;'>Error creating zip file: {e}</b>\"))\n",
|
| 939 |
+
" finally:\n",
|
| 940 |
+
" self.progress_bar.layout.visibility = 'hidden'\n",
|
| 941 |
+
" self._set_busy_state(False)\n",
|
| 942 |
+
" self.zip_button.disabled = True # Force re-analysis for next run\n",
|
| 943 |
+
"\n",
|
| 944 |
+
" def display(self):\n",
|
| 945 |
+
" \"\"\"Renders the widget in the notebook.\"\"\"\n",
|
| 946 |
+
" display(self.layout)\n"
|
| 947 |
+
]
|
| 948 |
+
},
|
| 949 |
+
{
|
| 950 |
+
"cell_type": "markdown",
|
| 951 |
+
"metadata": {},
|
| 952 |
+
"source": [
|
| 953 |
+
"## ποΈ Smart Image Zipper\n",
|
| 954 |
+
"\n",
|
| 955 |
+
"This one helps display the widget! "
|
| 956 |
+
]
|
| 957 |
+
},
|
| 958 |
+
{
|
| 959 |
+
"cell_type": "code",
|
| 960 |
+
"execution_count": 14,
|
| 961 |
+
"metadata": {},
|
| 962 |
+
"outputs": [
|
| 963 |
+
{
|
| 964 |
+
"data": {
|
| 965 |
+
"application/vnd.jupyter.widget-view+json": {
|
| 966 |
+
"model_id": "e289eb19638e46c6a0448a807f0a8ac7",
|
| 967 |
+
"version_major": 2,
|
| 968 |
+
"version_minor": 0
|
| 969 |
+
},
|
| 970 |
+
"text/plain": [
|
| 971 |
+
"VBox(children=(HTML(value='<h3>1. Select Source and Name</h3>'), Text(value='/workspace/stable-diffusion-webuiβ¦"
|
| 972 |
+
]
|
| 973 |
+
},
|
| 974 |
+
"metadata": {},
|
| 975 |
+
"output_type": "display_data"
|
| 976 |
+
}
|
| 977 |
+
],
|
| 978 |
+
"source": [
|
| 979 |
+
"zipper = SmartZipper()\n",
|
| 980 |
+
"zipper.display()"
|
| 981 |
+
]
|
| 982 |
+
},
|
| 983 |
+
{
|
| 984 |
+
"cell_type": "code",
|
| 985 |
+
"execution_count": null,
|
| 986 |
+
"metadata": {},
|
| 987 |
+
"outputs": [],
|
| 988 |
+
"source": []
|
| 989 |
+
}
|
| 990 |
+
],
|
| 991 |
+
"metadata": {
|
| 992 |
+
"colab": {
|
| 993 |
+
"collapsed_sections": [
|
| 994 |
+
"IZ_JYwvBLrg-",
|
| 995 |
+
"PNF2kdyeO3Dn"
|
| 996 |
+
],
|
| 997 |
+
"private_outputs": true,
|
| 998 |
+
"provenance": []
|
| 999 |
+
},
|
| 1000 |
+
"kernelspec": {
|
| 1001 |
+
"display_name": "Python3 (ipykernel)",
|
| 1002 |
+
"language": "python",
|
| 1003 |
+
"name": "python3"
|
| 1004 |
+
},
|
| 1005 |
+
"language_info": {
|
| 1006 |
+
"codemirror_mode": {
|
| 1007 |
+
"name": "ipython",
|
| 1008 |
+
"version": 3
|
| 1009 |
+
},
|
| 1010 |
+
"file_extension": ".py",
|
| 1011 |
+
"mimetype": "text/x-python",
|
| 1012 |
+
"name": "python",
|
| 1013 |
+
"nbconvert_exporter": "python",
|
| 1014 |
+
"pygments_lexer": "ipython3",
|
| 1015 |
+
"version": "3.10.12"
|
| 1016 |
+
}
|
| 1017 |
+
},
|
| 1018 |
+
"nbformat": 4,
|
| 1019 |
+
"nbformat_minor": 4
|
| 1020 |
+
}
|
HuggingFace_Backup_Jupyter_2025_Sept_Update.ipynb
ADDED
|
@@ -0,0 +1,550 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"metadata": {
|
| 6 |
+
"jp-MarkdownHeadingCollapsed": true
|
| 7 |
+
},
|
| 8 |
+
"source": [
|
| 9 |
+
"# <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Welcome to the Hugging Face File Uploader! \n",
|
| 10 |
+
"\n",
|
| 11 |
+
"# Welcome to the Hugging Face File Uploader\n",
|
| 12 |
+
"\n",
|
| 13 |
+
"Use this notebook's widget to easily upload files to your Hugging Face repositories.\n",
|
| 14 |
+
"\n",
|
| 15 |
+
"## Getting Started\n",
|
| 16 |
+
"\n",
|
| 17 |
+
"- Install Libraries: Run the first code cell.\n",
|
| 18 |
+
"- Log In: Run the notebook_login() cell with a Hugging Face write token.\n",
|
| 19 |
+
"- Launch Uploader: Run the final cell to display the uploader widget.\n",
|
| 20 |
+
"\n",
|
| 21 |
+
"## Uploading Files\n",
|
| 22 |
+
"\n",
|
| 23 |
+
"- Repo Info: Fill in the destination repository details.\n",
|
| 24 |
+
"- File Selection: Point to a local directory and select your files.\n",
|
| 25 |
+
"- Upload: Click the upload button to start the process.\n",
|
| 26 |
+
"\n",
|
| 27 |
+
"## Key Features\n",
|
| 28 |
+
"- Fast Uploads: Automatically uses hf_transfer to speed up large file transfers.\n",
|
| 29 |
+
"- Live Progress: Monitor upload status and speed in the output log.\n",
|
| 30 |
+
"- Simple Interface: All controls are contained within a single, easy-to-use widget.\n",
|
| 31 |
+
"\n",
|
| 32 |
+
"\n",
|
| 33 |
+
"**Community & Support:**\n",
|
| 34 |
+
"\n",
|
| 35 |
+
"* **GitHub:** [HuggingFace\\_Backup Repository on GitHub](https://github.com/Ktiseos-Nyx/HuggingFace_Backup) (for the latest version, updates, bug reports, and contributions)\n",
|
| 36 |
+
"* **Discord:**\n",
|
| 37 |
+
" * [Ktiseos Nyx AI/ML Discord](https://discord.gg/HhBSvM9gBY)\n",
|
| 38 |
+
" * [Earth & Dusk Media](https://discord.gg/5t2kYxt7An)\n",
|
| 39 |
+
"\n",
|
| 40 |
+
"This uploader is designed to simplify the process of getting your files onto the Hugging Face Hub. We hope you find it useful!"
|
| 41 |
+
]
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"cell_type": "markdown",
|
| 45 |
+
"metadata": {},
|
| 46 |
+
"source": [
|
| 47 |
+
" # <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Install Dependencies"
|
| 48 |
+
]
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"cell_type": "code",
|
| 52 |
+
"execution_count": 1,
|
| 53 |
+
"metadata": {},
|
| 54 |
+
"outputs": [
|
| 55 |
+
{
|
| 56 |
+
"name": "stdout",
|
| 57 |
+
"output_type": "stream",
|
| 58 |
+
"text": [
|
| 59 |
+
"Installing/updating required packages...\n",
|
| 60 |
+
"Package installation/update process complete.\n"
|
| 61 |
+
]
|
| 62 |
+
}
|
| 63 |
+
],
|
| 64 |
+
"source": [
|
| 65 |
+
"# Cell 1: Install Required Python Packages\n",
|
| 66 |
+
"# -----------------------------------------------------------------------------\n",
|
| 67 |
+
"print(\"Installing/updating required packages...\")\n",
|
| 68 |
+
"!pip install -q huggingface_hub\n",
|
| 69 |
+
"!pip install -q ipywidgets \n",
|
| 70 |
+
"!pip install -q hf_transfer\n",
|
| 71 |
+
"print(\"Package installation/update process complete.\")\n"
|
| 72 |
+
]
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"cell_type": "markdown",
|
| 76 |
+
"metadata": {
|
| 77 |
+
"id": "Xs1mb1VKLuUW"
|
| 78 |
+
},
|
| 79 |
+
"source": [
|
| 80 |
+
"# β¨ <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Connecting to Hugging Face: Authentication\n",
|
| 81 |
+
"\n",
|
| 82 |
+
"## π Hugging Face Login\n",
|
| 83 |
+
"You need a write-enabled token to upload files.\n",
|
| 84 |
+
"\n",
|
| 85 |
+
"### Instructions:\n",
|
| 86 |
+
"\n",
|
| 87 |
+
"- Create a Token: Go [here](https://huggingface.co/settings/tokens), click \"New token\", and give it the write role.\n",
|
| 88 |
+
"- Copy the Token: Copy the new token to your clipboard.\n",
|
| 89 |
+
"- Run Login Cell: Execute the code cell below, paste your token when prompted, and press Enter.\n",
|
| 90 |
+
"\n",
|
| 91 |
+
"\n",
|
| 92 |
+
"Troubleshooting: If you get an error, the most common problem is that your token is missing write permissions. Double-check the token's role on the Hugging Face website."
|
| 93 |
+
]
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"cell_type": "code",
|
| 97 |
+
"execution_count": 1,
|
| 98 |
+
"metadata": {},
|
| 99 |
+
"outputs": [
|
| 100 |
+
{
|
| 101 |
+
"data": {
|
| 102 |
+
"application/vnd.jupyter.widget-view+json": {
|
| 103 |
+
"model_id": "e4bf0d699a1c403e978ece176575b5cd",
|
| 104 |
+
"version_major": 2,
|
| 105 |
+
"version_minor": 0
|
| 106 |
+
},
|
| 107 |
+
"text/plain": [
|
| 108 |
+
"VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svβ¦"
|
| 109 |
+
]
|
| 110 |
+
},
|
| 111 |
+
"metadata": {},
|
| 112 |
+
"output_type": "display_data"
|
| 113 |
+
}
|
| 114 |
+
],
|
| 115 |
+
"source": [
|
| 116 |
+
"# Cell 2: Hugging Face Authentication Setup\n",
|
| 117 |
+
"# -----------------------------------------------------------------------------\n",
|
| 118 |
+
"from huggingface_hub import notebook_login\n",
|
| 119 |
+
"import os\n",
|
| 120 |
+
"notebook_login();\n",
|
| 121 |
+
"\n"
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"cell_type": "markdown",
|
| 126 |
+
"metadata": {},
|
| 127 |
+
"source": [
|
| 128 |
+
"# π <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Using the Hugging Face File Uploader\n",
|
| 129 |
+
"\n",
|
| 130 |
+
"## Uploader Checklist\n",
|
| 131 |
+
"Follow these steps to upload your files.\n",
|
| 132 |
+
"\n",
|
| 133 |
+
"1. Fill in Repository Details\n",
|
| 134 |
+
"- Owner (your username/org)\n",
|
| 135 |
+
"- Repo Name\n",
|
| 136 |
+
"- Repo Type (model, dataset, etc.)\n",
|
| 137 |
+
"\n",
|
| 138 |
+
"2. Select Local Files\n",
|
| 139 |
+
"- Set the Source Directory to your local folder path.\n",
|
| 140 |
+
"- Click the π List Files button.\n",
|
| 141 |
+
"- Select your desired files from the list that appears.\n",
|
| 142 |
+
"\n",
|
| 143 |
+
"3. Review Upload Settings (Optional)\n",
|
| 144 |
+
"- Add a Commit Message to describe your changes.\n",
|
| 145 |
+
"- Choose whether to Create a Pull Request for this upload.\n",
|
| 146 |
+
"\n",
|
| 147 |
+
"4. Start the Upload\n",
|
| 148 |
+
"- Click the β¬οΈ Upload Selected Files button and monitor the output."
|
| 149 |
+
]
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"cell_type": "code",
|
| 153 |
+
"execution_count": 4,
|
| 154 |
+
"metadata": {
|
| 155 |
+
"cellView": "form",
|
| 156 |
+
"id": "J851eLx6Ii3h"
|
| 157 |
+
},
|
| 158 |
+
"outputs": [],
|
| 159 |
+
"source": [
|
| 160 |
+
"# --- Essential Imports for the Uploader ---\n",
|
| 161 |
+
"import glob\n",
|
| 162 |
+
"import os\n",
|
| 163 |
+
"import time\n",
|
| 164 |
+
"from pathlib import Path\n",
|
| 165 |
+
"import math # For math.log and math.floor in _format_size\n",
|
| 166 |
+
"\n",
|
| 167 |
+
"from huggingface_hub import HfApi\n",
|
| 168 |
+
"from ipywidgets import (Text, Dropdown, Button, SelectMultiple, VBox, HBox,\n",
|
| 169 |
+
" Output, Layout, Checkbox, HTML, Textarea, Label,\n",
|
| 170 |
+
" FloatProgress)\n",
|
| 171 |
+
"from IPython.display import display, clear_output\n",
|
| 172 |
+
"\n",
|
| 173 |
+
"# Attempt to enable hf_transfer.\n",
|
| 174 |
+
"os.environ['HF_HUB_ENABLE_HF_TRANSFER'] = '1'\n",
|
| 175 |
+
"\n",
|
| 176 |
+
"class HuggingFaceUploader:\n",
|
| 177 |
+
" \"\"\"\n",
|
| 178 |
+
" A Jupyter widget-based tool to upload files to the Hugging Face Hub.\n",
|
| 179 |
+
" \"\"\"\n",
|
| 180 |
+
"\n",
|
| 181 |
+
" def __init__(self):\n",
|
| 182 |
+
" self.api = HfApi()\n",
|
| 183 |
+
" self.file_types = [\n",
|
| 184 |
+
" # AI Model Files π€\n",
|
| 185 |
+
" ('SafeTensors', 'safetensors'), ('PyTorch Models', 'pt'), ('PyTorch Legacy', 'pth'),\n",
|
| 186 |
+
" ('ONNX Models', 'onnx'), ('TensorFlow Models', 'pb'), ('Keras Models', 'h5'),\n",
|
| 187 |
+
" # Checkpoint Files π―\n",
|
| 188 |
+
" ('Checkpoints', 'ckpt'), ('Binary Files', 'bin'),\n",
|
| 189 |
+
" # Config & Data Files π\n",
|
| 190 |
+
" ('JSON Files', 'json'), ('YAML Files', 'yaml'), ('YAML Alt', 'yml'),\n",
|
| 191 |
+
" ('Text Files', 'txt'), ('CSV Files', 'csv'), ('Pickle Files', 'pkl'),\n",
|
| 192 |
+
" # Image Files π¨\n",
|
| 193 |
+
" ('PNG Images', 'png'), ('JPEG Images', 'jpg'), ('JPEG Alt', 'jpeg'),\n",
|
| 194 |
+
" ('WebP Images', 'webp'), ('GIF Images', 'gif'),\n",
|
| 195 |
+
" # Archive Files π¦\n",
|
| 196 |
+
" ('ZIP Archives', 'zip'), ('TAR Files', 'tar'), ('GZ Archives', 'gz')\n",
|
| 197 |
+
" ]\n",
|
| 198 |
+
" self.current_directory = os.getcwd()\n",
|
| 199 |
+
" self.hf_transfer_active = self._check_hf_transfer_availability()\n",
|
| 200 |
+
" self._create_widgets()\n",
|
| 201 |
+
" self._bind_events()\n",
|
| 202 |
+
" self._update_files(None) # Initial file list update\n",
|
| 203 |
+
"\n",
|
| 204 |
+
" def _check_hf_transfer_availability(self):\n",
|
| 205 |
+
" if os.environ.get(\"HF_HUB_ENABLE_HF_TRANSFER\") == \"1\":\n",
|
| 206 |
+
" try:\n",
|
| 207 |
+
" import hf_transfer\n",
|
| 208 |
+
" return True\n",
|
| 209 |
+
" except ImportError:\n",
|
| 210 |
+
" return False\n",
|
| 211 |
+
" return False\n",
|
| 212 |
+
"\n",
|
| 213 |
+
" def _create_widgets(self):\n",
|
| 214 |
+
" # --- Repository Info ---\n",
|
| 215 |
+
" self.repo_info_html = HTML(value=\"<b>π Repository Details</b>\")\n",
|
| 216 |
+
" self.org_name_text = Text(placeholder='Organization or Username', description='Owner:', style={'description_width': 'initial'})\n",
|
| 217 |
+
" self.repo_name_text = Text(placeholder='Repository Name', description='Repo:', style={'description_width': 'initial'})\n",
|
| 218 |
+
" self.repo_type_dropdown = Dropdown(options=['model', 'dataset', 'space'], value='model', description='Repo Type:', style={'description_width': 'initial'})\n",
|
| 219 |
+
" self.repo_folder_text = Text(placeholder='Optional: e.g., models/v1', description='Remote Folder:', style={'description_width': 'initial', \"flex\": \"1 1 auto\"}, layout=Layout(width='auto'))\n",
|
| 220 |
+
"\n",
|
| 221 |
+
" # --- File Selection ---\n",
|
| 222 |
+
" self.file_section_html = HTML(value=\"<b>ποΈ File Selection & Source</b>\")\n",
|
| 223 |
+
" self.file_type_dropdown = Dropdown(options=self.file_types, value='safetensors', description='File Type:', style={'description_width': 'initial'})\n",
|
| 224 |
+
" self.sort_by_dropdown = Dropdown(options=['name', 'date'], value='name', description='Sort By:', style={'description_width': 'initial'})\n",
|
| 225 |
+
" self.recursive_search_checkbox = Checkbox(value=False, description='Search Subdirectories', indent=False)\n",
|
| 226 |
+
"\n",
|
| 227 |
+
" self.directory_label = Label(value=\"Source Directory:\", layout=Layout(width='auto'))\n",
|
| 228 |
+
" self.directory_text = Text(value=self.current_directory, description=\"\", style={'description_width': '0px'}, layout=Layout(width=\"auto\", flex='1 1 auto'))\n",
|
| 229 |
+
" self.directory_update_btn = Button(description='π List Files', button_style='info', tooltip='Change source directory and refresh file list', layout=Layout(width='auto'))\n",
|
| 230 |
+
"\n",
|
| 231 |
+
" # --- Commit Details ---\n",
|
| 232 |
+
" self.commit_section_html = HTML(value=\"<b>π Commit Details</b>\")\n",
|
| 233 |
+
" self.commit_msg_textarea = Textarea(value=\"Upload via HuggingFaceUploader Widget π€\", placeholder='Enter your commit message (optional)', description='Message:', style={'description_width': 'initial'}, layout=Layout(width='98%', height='60px'))\n",
|
| 234 |
+
"\n",
|
| 235 |
+
" # --- Upload Settings ---\n",
|
| 236 |
+
" self.upload_section_html = HTML(value=\"<b>π Upload Settings</b>\")\n",
|
| 237 |
+
" self.create_pr_checkbox = Checkbox(value=False, description='Create Pull Request', indent=False)\n",
|
| 238 |
+
" self.clear_after_checkbox = Checkbox(value=True, description='Clear output after upload', indent=False)\n",
|
| 239 |
+
"\n",
|
| 240 |
+
" # --- Action Buttons ---\n",
|
| 241 |
+
" self.upload_button = Button(description='β¬οΈ Upload Selected Files', button_style='success', tooltip='Start upload process', layout=Layout(width='auto', height='auto'))\n",
|
| 242 |
+
" self.clear_output_button = Button(description='π§Ή Clear Output Log', button_style='warning', tooltip='Clear the output log area', layout=Layout(width='auto'))\n",
|
| 243 |
+
"\n",
|
| 244 |
+
" # --- File Picker & Output ---\n",
|
| 245 |
+
" self.file_picker_selectmultiple = SelectMultiple(options=[], description='Files:', layout=Layout(width=\"98%\", height=\"200px\"), style={'description_width': 'initial'})\n",
|
| 246 |
+
" self.output_area = Output(layout=Layout(padding='10px', border='1px solid #ccc', margin_top='10px', width='98%', max_height='400px', overflow_y='auto'))\n",
|
| 247 |
+
"\n",
|
| 248 |
+
" # --- Progress Display Area ---\n",
|
| 249 |
+
" self.current_file_label = Label(value=\"N/A\")\n",
|
| 250 |
+
" self.file_count_label = Label(value=\"File 0/0\")\n",
|
| 251 |
+
" self.progress_bar = FloatProgress(value=0, min=0, max=100, description='Overall:', bar_style='info', layout=Layout(width='85%'))\n",
|
| 252 |
+
" self.progress_percent_label = Label(value=\"0%\")\n",
|
| 253 |
+
"\n",
|
| 254 |
+
" self.progress_display_box = VBox([\n",
|
| 255 |
+
" HBox([Label(\"Current File:\", layout=Layout(width='100px')), self.current_file_label]),\n",
|
| 256 |
+
" HBox([Label(\"File Count:\", layout=Layout(width='100px')), self.file_count_label]),\n",
|
| 257 |
+
" HBox([self.progress_bar, self.progress_percent_label], layout=Layout(align_items='center'))\n",
|
| 258 |
+
" ], layout=Layout(visibility='hidden', margin='10px 0', padding='10px', border='1px solid #ddd', width='98%'))\n",
|
| 259 |
+
"\n",
|
| 260 |
+
" def _bind_events(self):\n",
|
| 261 |
+
" self.directory_update_btn.on_click(self._update_directory_and_files)\n",
|
| 262 |
+
" self.upload_button.on_click(self._upload_files_handler)\n",
|
| 263 |
+
" self.clear_output_button.on_click(lambda _: self.output_area.clear_output(wait=True))\n",
|
| 264 |
+
" self.file_type_dropdown.observe(self._update_files, names='value')\n",
|
| 265 |
+
" self.sort_by_dropdown.observe(self._update_files, names='value')\n",
|
| 266 |
+
" self.recursive_search_checkbox.observe(self._update_files, names='value') # NEW BINDING\n",
|
| 267 |
+
"\n",
|
| 268 |
+
" def _update_directory_and_files(self, _):\n",
|
| 269 |
+
" new_dir = self.directory_text.value.strip()\n",
|
| 270 |
+
" if not new_dir:\n",
|
| 271 |
+
" with self.output_area:\n",
|
| 272 |
+
" clear_output(wait=True); print(f\"π Current directory remains: {self.current_directory}\")\n",
|
| 273 |
+
" self._update_files(None)\n",
|
| 274 |
+
" return\n",
|
| 275 |
+
"\n",
|
| 276 |
+
" if os.path.isdir(new_dir):\n",
|
| 277 |
+
" self.current_directory = os.path.abspath(new_dir)\n",
|
| 278 |
+
" self.directory_text.value = self.current_directory\n",
|
| 279 |
+
" self._update_files(None)\n",
|
| 280 |
+
" else:\n",
|
| 281 |
+
" with self.output_area:\n",
|
| 282 |
+
" clear_output(wait=True); print(f\"β Invalid Directory: {new_dir}\")\n",
|
| 283 |
+
"\n",
|
| 284 |
+
" def _update_files(self, _):\n",
|
| 285 |
+
" file_extension = self.file_type_dropdown.value\n",
|
| 286 |
+
" self.output_area.clear_output(wait=True)\n",
|
| 287 |
+
" try:\n",
|
| 288 |
+
" # ENHANCEMENT: Use glob '**/ ' for recursive search\n",
|
| 289 |
+
" glob_prefix = '**/' if self.recursive_search_checkbox.value else ''\n",
|
| 290 |
+
" glob_pattern = f\"{glob_prefix}*.{file_extension}\"\n",
|
| 291 |
+
" \n",
|
| 292 |
+
" if not os.path.isdir(self.current_directory):\n",
|
| 293 |
+
" with self.output_area: print(f\"β οΈ Source directory '{self.current_directory}' is not valid.\")\n",
|
| 294 |
+
" self.file_picker_selectmultiple.options = []\n",
|
| 295 |
+
" return\n",
|
| 296 |
+
"\n",
|
| 297 |
+
" # Use rglob for simplicity if recursive\n",
|
| 298 |
+
" source_path = Path(self.current_directory)\n",
|
| 299 |
+
" found_paths = list(source_path.rglob(f'*.{file_extension}')) if self.recursive_search_checkbox.value else list(source_path.glob(f'*.{file_extension}'))\n",
|
| 300 |
+
" \n",
|
| 301 |
+
" valid_files_info = []\n",
|
| 302 |
+
" for p in found_paths:\n",
|
| 303 |
+
" if p.is_symlink() or not p.is_file(): continue\n",
|
| 304 |
+
" sort_key = p.stat().st_mtime if self.sort_by_dropdown.value == 'date' else p.name.lower()\n",
|
| 305 |
+
" valid_files_info.append((str(p), sort_key))\n",
|
| 306 |
+
"\n",
|
| 307 |
+
" if self.sort_by_dropdown.value == 'date':\n",
|
| 308 |
+
" valid_files_info.sort(key=lambda item: item[1], reverse=True)\n",
|
| 309 |
+
" else:\n",
|
| 310 |
+
" valid_files_info.sort(key=lambda item: item[1])\n",
|
| 311 |
+
" \n",
|
| 312 |
+
" # Display relative paths in the picker for clarity, but store absolute paths\n",
|
| 313 |
+
" # self.file_picker_selectmultiple.options now a list of (display_name, value)\n",
|
| 314 |
+
" display_options = []\n",
|
| 315 |
+
" for abs_path_str, _ in valid_files_info:\n",
|
| 316 |
+
" display_name = os.path.relpath(abs_path_str, self.current_directory)\n",
|
| 317 |
+
" display_options.append((display_name, abs_path_str))\n",
|
| 318 |
+
" \n",
|
| 319 |
+
" self.file_picker_selectmultiple.options = display_options\n",
|
| 320 |
+
" \n",
|
| 321 |
+
" with self.output_area:\n",
|
| 322 |
+
" if not display_options:\n",
|
| 323 |
+
" print(f\"π€· No '.{file_extension}' files found in '{self.current_directory}'.\")\n",
|
| 324 |
+
" else:\n",
|
| 325 |
+
" print(f\"β¨ Found {len(display_options)} '.{file_extension}' files. Select files to upload.\")\n",
|
| 326 |
+
"\n",
|
| 327 |
+
" except Exception as e:\n",
|
| 328 |
+
" with self.output_area:\n",
|
| 329 |
+
" clear_output(wait=True); print(f\"β Error listing files: {e}\"); traceback.print_exc(file=self.output_area)\n",
|
| 330 |
+
"\n",
|
| 331 |
+
" def _format_size(self, size_bytes):\n",
|
| 332 |
+
" if size_bytes < 0: return \"Invalid size\"\n",
|
| 333 |
+
" if size_bytes == 0: return \"0 B\"\n",
|
| 334 |
+
" units = (\"B\", \"KB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\")\n",
|
| 335 |
+
" # CORRECTED: Ensure direct calls to math module, not Path.math\n",
|
| 336 |
+
" i = math.floor(math.log(size_bytes, 1024)) if size_bytes > 0 else 0\n",
|
| 337 |
+
" if i >= len(units): i = len(units) - 1\n",
|
| 338 |
+
" s = round(size_bytes / (1024 ** i), 2)\n",
|
| 339 |
+
" return f\"{s} {units[i]}\"\n",
|
| 340 |
+
"\n",
|
| 341 |
+
" def _print_file_info(self, file_path_str, index, total_files):\n",
|
| 342 |
+
" file_path = Path(file_path_str)\n",
|
| 343 |
+
" try:\n",
|
| 344 |
+
" file_size = file_path.stat().st_size\n",
|
| 345 |
+
" self.output_area.append_stdout(f\"π¦ Uploading {index}/{total_files}: {file_path.name} ({self._format_size(file_size)})\\n\")\n",
|
| 346 |
+
" except FileNotFoundError:\n",
|
| 347 |
+
" self.output_area.append_stdout(f\"β οΈ File not found: {file_path_str}\\n\")\n",
|
| 348 |
+
"\n",
|
| 349 |
+
" def _upload_files_handler(self, _):\n",
|
| 350 |
+
" org_or_user = self.org_name_text.value.strip()\n",
|
| 351 |
+
" repo_name = self.repo_name_text.value.strip()\n",
|
| 352 |
+
"\n",
|
| 353 |
+
" if not org_or_user or not repo_name:\n",
|
| 354 |
+
" with self.output_area: clear_output(wait=True); print(\"β Please fill in 'Owner' and 'Repo Name'.\")\n",
|
| 355 |
+
" return\n",
|
| 356 |
+
"\n",
|
| 357 |
+
" repo_id = f\"{org_or_user}/{repo_name}\"\n",
|
| 358 |
+
" selected_file_paths = list(self.file_picker_selectmultiple.value)\n",
|
| 359 |
+
"\n",
|
| 360 |
+
" if not selected_file_paths:\n",
|
| 361 |
+
" with self.output_area: clear_output(wait=True); print(\"π Nothing selected for upload.\")\n",
|
| 362 |
+
" return\n",
|
| 363 |
+
"\n",
|
| 364 |
+
" self.output_area.clear_output(wait=True)\n",
|
| 365 |
+
" self.output_area.append_stdout(f\"π― Preparing to upload to: https://huggingface.co/{repo_id}\\n\")\n",
|
| 366 |
+
" if self.hf_transfer_active: self.output_area.append_stdout(\"π HF_TRANSFER is enabled.\\n\")\n",
|
| 367 |
+
" else: self.output_area.append_stdout(\"βΉοΈ For faster uploads, run `%pip install -q hf_transfer` and restart kernel.\\n\")\n",
|
| 368 |
+
"\n",
|
| 369 |
+
" self.progress_display_box.layout.visibility = 'visible'\n",
|
| 370 |
+
" self.progress_bar.value = 0\n",
|
| 371 |
+
" self.progress_percent_label.value = \"0%\"\n",
|
| 372 |
+
" self.current_file_label.value = \"Initializing...\"\n",
|
| 373 |
+
" \n",
|
| 374 |
+
" total_files = len(selected_file_paths)\n",
|
| 375 |
+
" self.file_count_label.value = f\"File 0/{total_files}\"\n",
|
| 376 |
+
" \n",
|
| 377 |
+
" repo_type = self.repo_type_dropdown.value\n",
|
| 378 |
+
" repo_folder_prefix = self.repo_folder_text.value.strip().replace('\\\\', '/')\n",
|
| 379 |
+
" base_commit_msg = self.commit_msg_textarea.value or \"Upload via HuggingFaceUploader Widget π€\"\n",
|
| 380 |
+
"\n",
|
| 381 |
+
" success_count = 0\n",
|
| 382 |
+
" for idx, local_file_path_str in enumerate(selected_file_paths, 1):\n",
|
| 383 |
+
" try:\n",
|
| 384 |
+
" current_file_path = Path(local_file_path_str)\n",
|
| 385 |
+
" self.current_file_label.value = current_file_path.name\n",
|
| 386 |
+
" self.file_count_label.value = f\"File {idx}/{total_files}\"\n",
|
| 387 |
+
" self._print_file_info(local_file_path_str, idx, total_files)\n",
|
| 388 |
+
" \n",
|
| 389 |
+
" start_time = time.time()\n",
|
| 390 |
+
" \n",
|
| 391 |
+
" if not current_file_path.exists():\n",
|
| 392 |
+
" self.output_area.append_stdout(f\"β SKIPPED: File '{current_file_path.name}' not found.\\n\")\n",
|
| 393 |
+
" continue\n",
|
| 394 |
+
" \n",
|
| 395 |
+
" # This logic correctly handles recursive uploads\n",
|
| 396 |
+
" path_in_repo_base = os.path.relpath(current_file_path, self.current_directory).replace('\\\\', '/')\n",
|
| 397 |
+
" path_in_repo = f\"{repo_folder_prefix}/{path_in_repo_base}\" if repo_folder_prefix.strip('/') else path_in_repo_base\n",
|
| 398 |
+
" \n",
|
| 399 |
+
" commit_message_for_file = f\"{base_commit_msg} ({current_file_path.name})\"\n",
|
| 400 |
+
"\n",
|
| 401 |
+
" response_url = self.api.upload_file(\n",
|
| 402 |
+
" path_or_fileobj=str(current_file_path),\n",
|
| 403 |
+
" path_in_repo=path_in_repo,\n",
|
| 404 |
+
" repo_id=repo_id,\n",
|
| 405 |
+
" repo_type=repo_type,\n",
|
| 406 |
+
" create_pr=self.create_pr_checkbox.value,\n",
|
| 407 |
+
" commit_message=commit_message_for_file,\n",
|
| 408 |
+
" )\n",
|
| 409 |
+
" duration = time.time() - start_time\n",
|
| 410 |
+
" self.output_area.append_stdout(f\"β
Uploaded '{current_file_path.name}' to '{path_in_repo}' in {duration:.1f}s.\\n\")\n",
|
| 411 |
+
" self.output_area.append_stdout(f\" View at: {response_url}\\n\")\n",
|
| 412 |
+
" success_count += 1\n",
|
| 413 |
+
"\n",
|
| 414 |
+
" except Exception as e:\n",
|
| 415 |
+
" self.output_area.append_stdout(f\"β Error uploading {current_file_path.name}: {e}\\n\")\n",
|
| 416 |
+
" import traceback\n",
|
| 417 |
+
" with self.output_area: traceback.print_exc()\n",
|
| 418 |
+
" self.output_area.append_stdout(\"\\n\")\n",
|
| 419 |
+
" finally:\n",
|
| 420 |
+
" percentage = int((idx / total_files) * 100)\n",
|
| 421 |
+
" self.progress_bar.value = percentage\n",
|
| 422 |
+
" self.progress_percent_label.value = f\"{percentage}%\"\n",
|
| 423 |
+
"\n",
|
| 424 |
+
" self.output_area.append_stdout(f\"\\nβ¨ Upload complete. {success_count}/{total_files} files processed. β¨\\n\")\n",
|
| 425 |
+
" if self.create_pr_checkbox.value and success_count > 0:\n",
|
| 426 |
+
" self.output_area.append_stdout(f\"π View Pull Request: https://huggingface.co/{repo_id}/pulls\\n\")\n",
|
| 427 |
+
" elif success_count > 0 :\n",
|
| 428 |
+
" repo_tree_url = f\"https://huggingface.co/{repo_id}/tree/main/{repo_folder_prefix.strip('/')}\".rstrip('/')\n",
|
| 429 |
+
" self.output_area.append_stdout(f\"π View files at: {repo_tree_url}\\n\")\n",
|
| 430 |
+
"\n",
|
| 431 |
+
" self.current_file_label.value = \"Completed.\"\n",
|
| 432 |
+
" if self.clear_after_checkbox.value:\n",
|
| 433 |
+
" time.sleep(5)\n",
|
| 434 |
+
" self.output_area.clear_output(wait=True)\n",
|
| 435 |
+
" self.progress_display_box.layout.visibility = 'hidden'\n",
|
| 436 |
+
"\n",
|
| 437 |
+
" def display(self):\n",
|
| 438 |
+
" repo_box = HBox([self.org_name_text, self.repo_name_text, self.repo_type_dropdown], layout=Layout(flex_flow='wrap', justify_content='space-between'))\n",
|
| 439 |
+
" repo_folder_box = HBox([self.repo_folder_text], layout=Layout(width='100%'))\n",
|
| 440 |
+
" dir_select_box = HBox([self.directory_label, self.directory_text, self.directory_update_btn], layout=Layout(width='100%', align_items='center'))\n",
|
| 441 |
+
" file_opts_box = HBox([self.file_type_dropdown, self.sort_by_dropdown, self.recursive_search_checkbox], layout=Layout(flex_flow='wrap', justify_content='space-between', align_items='center'))\n",
|
| 442 |
+
" upload_opts_box = HBox([self.create_pr_checkbox, self.clear_after_checkbox], layout=Layout(margin='5px 0'))\n",
|
| 443 |
+
" action_buttons_box = HBox([self.upload_button, self.clear_output_button], layout=Layout(margin='10px 0 0 0', spacing='10px'))\n",
|
| 444 |
+
"\n",
|
| 445 |
+
" main_layout = VBox([\n",
|
| 446 |
+
" self.repo_info_html, repo_box, repo_folder_box,\n",
|
| 447 |
+
" HTML(\"<hr>\"),\n",
|
| 448 |
+
" self.file_section_html, file_opts_box, dir_select_box,\n",
|
| 449 |
+
" self.file_picker_selectmultiple,\n",
|
| 450 |
+
" HTML(\"<hr>\"),\n",
|
| 451 |
+
" self.commit_section_html, self.commit_msg_textarea,\n",
|
| 452 |
+
" HTML(\"<hr>\"),\n",
|
| 453 |
+
" self.upload_section_html, upload_opts_box,\n",
|
| 454 |
+
" action_buttons_box,\n",
|
| 455 |
+
" self.progress_display_box,\n",
|
| 456 |
+
" self.output_area\n",
|
| 457 |
+
" ], layout=Layout(width='700px', padding='10px', border='1px solid lightgray'))\n",
|
| 458 |
+
" \n",
|
| 459 |
+
" display(main_layout)"
|
| 460 |
+
]
|
| 461 |
+
},
|
| 462 |
+
{
|
| 463 |
+
"cell_type": "markdown",
|
| 464 |
+
"metadata": {},
|
| 465 |
+
"source": [
|
| 466 |
+
"# π <a href=\"https://emoji.gg/sticker/72567-doro\"><img src=\"https://cdn3.emoji.gg/stickers/72567-doro.png\" width=\"32px\" height=\"32px\" alt=\"Doro\"></a> Uploader Widget! \n",
|
| 467 |
+
"\n",
|
| 468 |
+
"**Run the next cell to initiate the uploader widget!**\n"
|
| 469 |
+
]
|
| 470 |
+
},
|
| 471 |
+
{
|
| 472 |
+
"cell_type": "code",
|
| 473 |
+
"execution_count": 5,
|
| 474 |
+
"metadata": {},
|
| 475 |
+
"outputs": [
|
| 476 |
+
{
|
| 477 |
+
"name": "stdout",
|
| 478 |
+
"output_type": "stream",
|
| 479 |
+
"text": [
|
| 480 |
+
"π Initializing Hugging Face Uploader...\n"
|
| 481 |
+
]
|
| 482 |
+
},
|
| 483 |
+
{
|
| 484 |
+
"data": {
|
| 485 |
+
"application/vnd.jupyter.widget-view+json": {
|
| 486 |
+
"model_id": "1e5ee45c1f814f75a8a55436d3b7b448",
|
| 487 |
+
"version_major": 2,
|
| 488 |
+
"version_minor": 0
|
| 489 |
+
},
|
| 490 |
+
"text/plain": [
|
| 491 |
+
"VBox(children=(HTML(value='<b>π Repository Details</b>'), HBox(children=(Text(value='', description='Owner:', β¦"
|
| 492 |
+
]
|
| 493 |
+
},
|
| 494 |
+
"metadata": {},
|
| 495 |
+
"output_type": "display_data"
|
| 496 |
+
},
|
| 497 |
+
{
|
| 498 |
+
"name": "stdout",
|
| 499 |
+
"output_type": "stream",
|
| 500 |
+
"text": [
|
| 501 |
+
"β
Uploader interface is ready. You can now select files and upload.\n"
|
| 502 |
+
]
|
| 503 |
+
}
|
| 504 |
+
],
|
| 505 |
+
"source": [
|
| 506 |
+
"# Uploader Widget Code\n",
|
| 507 |
+
"print(\"π Initializing Hugging Face Uploader...\")\n",
|
| 508 |
+
"uploader = HuggingFaceUploader()\n",
|
| 509 |
+
"uploader.display()\n",
|
| 510 |
+
"print(\"β
Uploader interface is ready. You can now select files and upload.\")"
|
| 511 |
+
]
|
| 512 |
+
},
|
| 513 |
+
{
|
| 514 |
+
"cell_type": "code",
|
| 515 |
+
"execution_count": null,
|
| 516 |
+
"metadata": {},
|
| 517 |
+
"outputs": [],
|
| 518 |
+
"source": []
|
| 519 |
+
}
|
| 520 |
+
],
|
| 521 |
+
"metadata": {
|
| 522 |
+
"colab": {
|
| 523 |
+
"collapsed_sections": [
|
| 524 |
+
"IZ_JYwvBLrg-",
|
| 525 |
+
"PNF2kdyeO3Dn"
|
| 526 |
+
],
|
| 527 |
+
"private_outputs": true,
|
| 528 |
+
"provenance": []
|
| 529 |
+
},
|
| 530 |
+
"kernelspec": {
|
| 531 |
+
"display_name": "Python3 (ipykernel)",
|
| 532 |
+
"language": "python",
|
| 533 |
+
"name": "python3"
|
| 534 |
+
},
|
| 535 |
+
"language_info": {
|
| 536 |
+
"codemirror_mode": {
|
| 537 |
+
"name": "ipython",
|
| 538 |
+
"version": 3
|
| 539 |
+
},
|
| 540 |
+
"file_extension": ".py",
|
| 541 |
+
"mimetype": "text/x-python",
|
| 542 |
+
"name": "python",
|
| 543 |
+
"nbconvert_exporter": "python",
|
| 544 |
+
"pygments_lexer": "ipython3",
|
| 545 |
+
"version": "3.10.12"
|
| 546 |
+
}
|
| 547 |
+
},
|
| 548 |
+
"nbformat": 4,
|
| 549 |
+
"nbformat_minor": 4
|
| 550 |
+
}
|