upload.py

import os
from huggingface_hub import HfApi
import time

def upload_with_retry(folder_path, repo_id, max_retries=10):
    api = HfApi()
    
    # 获取所有文件
    files = []
    for root, _, filenames in os.walk(folder_path):
        for filename in filenames:
            if not any(pattern in filename for pattern in [".git"]):
                full_path = os.path.join(root, filename)
                relative_path = os.path.relpath(full_path, folder_path)
                files.append((full_path, relative_path))
    
    print(f"Found {len(files)} files to upload")
    
    for file_path, path_in_repo in files:
        for attempt in range(max_retries):
            try:
                print(f"Uploading {path_in_repo} (Attempt {attempt + 1} of {max_retries})")
                api.upload_file(
                    path_or_fileobj=file_path,
                    path_in_repo=path_in_repo,
                    repo_id=repo_id,
                    repo_type="model"
                )
                print(f"Successfully uploaded {path_in_repo}")
                break
            except Exception as e:
                print(f"Error uploading {path_in_repo}: {e}")
                if attempt < max_retries - 1:
                    wait_time = 10 * (attempt + 1)
                    print(f"Retrying in {wait_time} seconds...")
                    time.sleep(wait_time)
                else:
                    print(f"Max retries reached. Failed to upload {path_in_repo}")

# 使用
upload_with_retry(".", "Neph0s/CoSER-Llama-3.1-70B")