Update serenityai.py

serenityai.py

@@ -8,243 +8,14 @@ Original file is located at
 """
 
 # %% Cell 1 - Install Dependencies
-!pip install -q torch==2.2.1 torchvision torchaudio xformers
-!pip install -q fastapi nest-asyncio pyngrok uvicorn python-multipart streamlit langchain chromadb pydub whisper openai-whisper
 
-!pip install -q torch==2.2.1 torchvision torchaudio xformers
 
-!pip install --upgrade unsloth
 
-!pip install optuna
 
 # %% Cell 1 - Setup and Configuration
 import os
 import torch
-import optuna  # Hyperparameter tuning
-from datasets import load_dataset
-from unsloth import FastLanguageModel
-from transformers import TrainingArguments, Trainer
-import wandb
-
-# Initialize wandb
-wandb.init(project="serenity-ai-advanced")
-
-# Hardware configuration
-print(f"Available GPUs: {torch.cuda.device_count()}")
-print(f"CUDA version: {torch.version.cuda}")
-torch.cuda.empty_cache()
-
-# Model configuration
-model_name = "unsloth/llama-3-8B-bnb-4bit"
-max_seq_length = 2048  # Max for Llama-3
-dtype = torch.float16
-checkpoint_dir = "./serenity-advanced-checkpoints"
-
-# %% Cell 2 - Dataset Processing & Tokenization
-def format_chat_template(example):
-    """
-    Formats the dataset into a chat-like structure.
-    Uses 'utterance' and a placeholder for response
-    since the dataset doesn't have a dedicated 'response' column.
-    """
-    return {
-        "text": f"User: {example['utterance']}\nAI: [Placeholder Response]" # Placeholder response for now
-    }
-
-
-# Load and process dataset
-dataset = load_dataset("empathetic_dialogues", split="train[:2000]")  # First 2000 samples
-dataset = dataset.map(format_chat_template, remove_columns=dataset.column_names)
-
-# Split dataset
-split_dataset = dataset.train_test_split(test_size=0.1)
-train_dataset = split_dataset["train"]
-eval_dataset = split_dataset["test"]
-
-# Tokenization function
-def tokenize_function(examples):
-    return tokenizer(
-        examples["text"],
-        padding="max_length",
-        truncation=True,
-        max_length=max_seq_length,
-        return_tensors="pt",
-    )
-
-# Apply tokenization
-tokenized_train = train_dataset.map(
-    tokenize_function,
-    batched=True,
-    remove_columns=["text"]
-)
-
-tokenized_eval = eval_dataset.map(
-    tokenize_function,
-    batched=True,
-    remove_columns=["text"]
-)
-
-# Create data collator
-data_collator = DataCollatorForLanguageModeling(
-    tokenizer=tokenizer,
-    mlm=False
-)
-
-# %% Cell 3 - Resume Training from Checkpoint if Available
-latest_checkpoint = None
-if os.path.exists(checkpoint_dir):
-    checkpoints = sorted([ckpt for ckpt in os.listdir(checkpoint_dir) if ckpt.startswith("checkpoint-")])
-    if checkpoints:
-        latest_checkpoint = os.path.join(checkpoint_dir, checkpoints[-1])
-        print(f"Resuming training from checkpoint: {latest_checkpoint}")
-    else:
-        print("No checkpoint found, starting fresh.")
-
-# %% Cell 4 - Define Hyperparameter Search Function
-def hyperparameter_objective(trial):
-    """Optuna function to find the best hyperparameters."""
-    # Hyperparameter search space
-    learning_rate = trial.suggest_float("learning_rate", 1e-5, 5e-4, log=True)
-    batch_size = trial.suggest_categorical("batch_size", [2, 4, 8])
-    weight_decay = trial.suggest_float("weight_decay", 0.01, 0.1)
-    lora_rank = trial.suggest_int("lora_rank", 16, 64, step=16)
-    lora_dropout = trial.suggest_float("lora_dropout", 0.05, 0.2)
-
-    # %% Load Model with Optimized LoRA
-    model, tokenizer = FastLanguageModel.from_pretrained(
-        model_name=model_name,
-        max_seq_length=max_seq_length,
-        dtype=dtype,
-        load_in_4bit=True,
-        token=None,
-        device_map="auto",
-        rope_scaling={"type": "dynamic", "factor": 2.0},
-        attn_implementation="flash_attention_2",
-    )
-
-    model = FastLanguageModel.get_peft_model(
-        model,
-        r=lora_rank,
-        target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
-                        "gate_proj", "up_proj", "down_proj"],
-        lora_alpha=64,
-        lora_dropout=lora_dropout,
-        bias="none",
-        use_gradient_checkpointing="unsloth",
-        random_state=3407,
-        max_seq_length=max_seq_length,
-        use_rslora=True,
-        loftq_config={},
-    )
-
-    # Training Arguments
-    training_args = TrainingArguments(
-        output_dir=checkpoint_dir,
-        num_train_epochs=2,
-        per_device_train_batch_size=batch_size,
-        gradient_accumulation_steps=4,
-        learning_rate=learning_rate,
-        weight_decay=weight_decay,
-        warmup_ratio=0.1,
-        lr_scheduler_type="cosine",
-        evaluation_strategy="steps",
-        eval_steps=5000,
-        logging_steps=10,
-        fp16=torch.cuda.is_available(),
-        seed=42,
-        report_to="wandb",
-        save_strategy="steps",
-        save_steps=5000,
-        save_total_limit=3,
-        load_best_model_at_end=True,
-        gradient_checkpointing=True,
-        push_to_hub=False,
-    )
-
-    #trainer = Trainer(
-        model=model,
-        args=training_args,
-        train_dataset=tokenized_train,
-        eval_dataset=tokenized_eval,
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-    )
-
-    #trainer.train(resume_from_checkpoint=latest_checkpoint)
-
-    # Return validation loss for optimization
-    eval_results = trainer.evaluate()
-    return eval_results["eval_loss"]
-
-# %% Cell 5 - Run Optuna Hyperparameter Optimization
-study = optuna.create_study(direction="minimize")
-study.optimize(hyperparameter_objective, n_trials=10)  # Run 10 trials
-
-# %% Cell 6 - Train with Best Hyperparameters
-best_params = study.best_params
-print(f"Best Hyperparameters: {best_params}")
-
-# Load best model
-model, tokenizer = FastLanguageModel.from_pretrained(
-    model_name=model_name,
-    max_seq_length=max_seq_length,
-    dtype=dtype,
-    load_in_4bit=True,
-    token=None,
-    device_map="auto",
-    rope_scaling={"type": "dynamic", "factor": 2.0},
-    attn_implementation="flash_attention_2",
-)
-
-model = FastLanguageModel.get_peft_model(
-    model,
-    r=best_params["lora_rank"],
-    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
-                    "gate_proj", "up_proj", "down_proj"],
-    lora_alpha=64,
-    lora_dropout=best_params["lora_dropout"],
-    bias="none",
-    use_gradient_checkpointing="unsloth",
-    random_state=3407,
-    max_seq_length=max_seq_length,
-    use_rslora=True,
-    loftq_config={},
-)
-
-# Final Training
-training_args = TrainingArguments(
-    output_dir=checkpoint_dir,
-    num_train_epochs=2,
-    per_device_train_batch_size=best_params["batch_size"],
-    gradient_accumulation_steps=4,
-    learning_rate=best_params["learning_rate"],
-    weight_decay=best_params["weight_decay"],
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    evaluation_strategy="steps",
-    eval_steps=5000,
-    logging_steps=10,
-    fp16=torch.cuda.is_available(),
-    seed=42,
-    report_to="wandb",
-    save_strategy="steps",
-    save_steps=5000,
-    save_total_limit=3,
-    load_best_model_at_end=True,
-    gradient_checkpointing=True,
-    push_to_hub=False,
-)
-
-#trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=tokenized_train,
-    eval_dataset=tokenized_eval,
-    data_collator=data_collator,
-    tokenizer=tokenizer,
-)
 
-#trainer.train(resume_from_checkpoint=latest_checkpoint)
 
 # %% Save the Optimized Model
 model.save_pretrained_merged("serenity-advanced", tokenizer, save_method="merged_16bit", push_to_hub=False)