Create app.py

app.py

@@ -0,0 +1,220 @@
+import streamlit as st
+import requests
+import base64
+import os
+import asyncio
+from huggingface_hub import HfApi, snapshot_download
+import plotly.express as px
+import zipfile
+import tempfile
+import shutil
+
+# Initialize the Hugging Face API
+api = HfApi()
+
+# Directories for saving files
+HTML_DIR = "generated_html_pages"
+ZIP_DIR = "generated_zips"
+SNAPSHOT_DIR = "snapshot_downloads"
+
+for directory in [HTML_DIR, ZIP_DIR, SNAPSHOT_DIR]:
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+# Default list of Hugging Face usernames
+default_users = {
+    "users": [
+        "awacke1", "rogerxavier", "jonatasgrosman", "kenshinn", "Csplk", "DavidVivancos",
+        "cdminix", "Jaward", "TuringsSolutions", "Severian", "Wauplin",
+        "phosseini", "Malikeh1375", "gokaygokay", "MoritzLaurer", "mrm8488",
+        "TheBloke", "lhoestq", "xw-eric", "Paul", "Muennighoff",
+        "ccdv", "haonan-li", "chansung", "lukaemon", "hails",
+        "pharmapsychotic", "KingNish", "merve", "ameerazam08", "ashleykleynhans"
+    ]
+}
+
+async def fetch_user_content(username):
+    try:
+        models = list(await asyncio.to_thread(api.list_models, author=username))
+        datasets = list(await asyncio.to_thread(api.list_datasets, author=username))
+        return {
+            "username": username,
+            "models": models,
+            "datasets": datasets
+        }
+    except Exception as e:
+        return {"username": username, "error": str(e)}
+
+def download_user_page(username):
+    url = f"https://huggingface.co/{username}"
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        html_content = response.text
+        html_file_path = os.path.join(HTML_DIR, f"{username}.html")
+        with open(html_file_path, "w", encoding='utf-8') as html_file:
+            html_file.write(html_content)
+        return html_file_path, None
+    except Exception as e:
+        return None, str(e)
+
+@st.cache_resource
+def create_zip_of_files(files, zip_name):
+    zip_file_path = os.path.join(ZIP_DIR, zip_name)
+    with zipfile.ZipFile(zip_file_path, 'w') as zipf:
+        for file in files:
+            zipf.write(file, arcname=os.path.basename(file))
+    return zip_file_path
+
+@st.cache_resource
+def get_download_link(file_path, link_text):
+    with open(file_path, 'rb') as f:
+        data = f.read()
+    b64 = base64.b64encode(data).decode()
+    return f'<a href="data:application/zip;base64,{b64}" download="{os.path.basename(file_path)}">{link_text}</a>'
+
+async def fetch_all_users(usernames):
+    tasks = [fetch_user_content(username) for username in usernames]
+    return await asyncio.gather(*tasks)
+
+def get_all_html_files(usernames):
+    html_files = []
+    errors = {}
+    for username in usernames:
+        html_file, error = download_user_page(username)
+        if html_file:
+            html_files.append(html_file)
+        else:
+            errors[username] = error
+    return html_files, errors
+
+def perform_snapshot_download(repo_id, repo_type):
+    try:
+        temp_dir = tempfile.mkdtemp()
+        snapshot_download(repo_id=repo_id, repo_type=repo_type, local_dir=temp_dir)
+        zip_name = f"{repo_id.replace('/', '_')}_{repo_type}.zip"
+        zip_path = os.path.join(SNAPSHOT_DIR, zip_name)
+        shutil.make_archive(zip_path[:-4], 'zip', temp_dir)
+        shutil.rmtree(temp_dir)
+        return zip_path
+    except Exception as e:
+        return str(e)
+
+st.title("Hugging Face User Page Downloader & Zipper 📄➕📦")
+
+user_input = st.text_area(
+    "Enter Hugging Face usernames (one per line):",
+    value="\n".join(default_users["users"]),
+    height=300
+)
+
+if st.button("Show User Content and Download Snapshots"):
+    if user_input:
+        username_list = [username.strip() for username in user_input.split('\n') if username.strip()]
+        
+        user_data_list = asyncio.run(fetch_all_users(username_list))
+        
+        stats = {"username": [], "models_count": [], "datasets_count": []}
+        successful_html_files = []
+        snapshot_downloads = []
+        
+        st.markdown("### User Content Overview")
+        for user_data in user_data_list:
+            username = user_data["username"]
+            with st.container():
+                st.markdown(f"**{username}** [🔗 Profile](https://huggingface.co/{username})")
+                
+                if "error" in user_data:
+                    st.warning(f"{username}: {user_data['error']} - Something went wrong! ⚠️")
+                else:
+                    models = user_data["models"]
+                    datasets = user_data["datasets"]
+                    
+                    html_file_path, download_error = download_user_page(username)
+                    if html_file_path:
+                        successful_html_files.append(html_file_path)
+                        st.success(f"✅ Successfully downloaded {username}'s page.")
+                    else:
+                        st.error(f"❌ Failed to download {username}'s page: {download_error}")
+                    
+                    stats["username"].append(username)
+                    stats["models_count"].append(len(models))
+                    stats["datasets_count"].append(len(datasets))
+                    
+                    with st.expander(f"🧠 Models ({len(models)})", expanded=False):
+                        if models:
+                            for model in models:
+                                model_name = model.modelId.split("/")[-1]
+                                st.markdown(f"- [{model_name}](https://huggingface.co/{model.modelId})")
+                                if st.button(f"Download Snapshot: {model_name}", key=f"model_{model.modelId}"):
+                                    with st.spinner(f"Downloading snapshot for {model_name}..."):
+                                        result = perform_snapshot_download(model.modelId, "model")
+                                        if isinstance(result, str):
+                                            st.error(f"Failed to download {model_name}: {result}")
+                                        else:
+                                            snapshot_downloads.append(result)
+                                            st.success(f"Successfully downloaded snapshot for {model_name}")
+                        else:
+                            st.markdown("No models found. 🤷‍♂️")
+                    
+                    with st.expander(f"📚 Datasets ({len(datasets)})", expanded=False):
+                        if datasets:
+                            for dataset in datasets:
+                                dataset_name = dataset.id.split("/")[-1]
+                                st.markdown(f"- [{dataset_name}](https://huggingface.co/datasets/{dataset.id})")
+                                if st.button(f"Download Snapshot: {dataset_name}", key=f"dataset_{dataset.id}"):
+                                    with st.spinner(f"Downloading snapshot for {dataset_name}..."):
+                                        result = perform_snapshot_download(dataset.id, "dataset")
+                                        if isinstance(result, str):
+                                            st.error(f"Failed to download {dataset_name}: {result}")
+                                        else:
+                                            snapshot_downloads.append(result)
+                                            st.success(f"Successfully downloaded snapshot for {dataset_name}")
+                        else:
+                            st.markdown("No datasets found. 🤷‍♀️")
+                
+                st.markdown("---")
+        
+        if successful_html_files:
+            html_zip_path = create_zip_of_files(successful_html_files, "HuggingFace_User_Pages.zip")
+            html_download_link = get_download_link(html_zip_path, "📥 Download All HTML Pages as ZIP")
+            st.markdown(html_download_link, unsafe_allow_html=True)
+        else:
+            st.warning("No HTML files were successfully downloaded to create a ZIP archive.")
+        
+        if snapshot_downloads:
+            snapshot_zip_path = create_zip_of_files(snapshot_downloads, "HuggingFace_Snapshots.zip")
+            snapshot_download_link = get_download_link(snapshot_zip_path, "📥 Download All Snapshots as ZIP")
+            st.markdown(snapshot_download_link, unsafe_allow_html=True)
+        
+        if stats["username"]:
+            st.markdown("### User Content Statistics")
+            
+            fig_models = px.bar(
+                x=stats["username"],
+                y=stats["models_count"],
+                labels={'x': 'Username', 'y': 'Number of Models'},
+                title="Number of Models per User"
+            )
+            st.plotly_chart(fig_models)
+            
+            fig_datasets = px.bar(
+                x=stats["username"],
+                y=stats["datasets_count"],
+                labels={'x': 'Username', 'y': 'Number of Datasets'},
+                title="Number of Datasets per User"
+            )
+            st.plotly_chart(fig_datasets)
+        
+    else:
+        st.warning("Please enter at least one username. Don't be shy! 😅")
+
+st.sidebar.markdown("""
+## How to use:
+1. The text area is pre-filled with a list of Hugging Face usernames. You can edit this list or add more usernames.
+2. Click **'Show User Content and Download Snapshots'**.
+3. View each user's models and datasets along with a link to their Hugging Face profile.
+4. For each model or dataset, you can click the "Download Snapshot" button to download a snapshot.
+5. **Download ZIP archives** containing all the HTML pages and snapshots by clicking the download links.
+6. Check out the statistics visualizations below!
+""")