himu1780 commited on
Commit
610b4ec
Β·
verified Β·
1 Parent(s): b1a6adc

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +392 -380
app.py CHANGED
@@ -1,380 +1,392 @@
1
- """
2
- AI Python Code Model Trainer
3
- Hugging Face Space for continuous training with auto-resume
4
- Username: himu1780 | Model: ai-python-model
5
-
6
- FINAL VERSION - All optimizations applied
7
- """
8
-
9
- import os
10
- import gc
11
- import gradio as gr
12
- import threading
13
- import time
14
- from datetime import datetime
15
- from huggingface_hub import HfApi, login
16
- from transformers import (
17
- AutoModelForCausalLM,
18
- AutoTokenizer,
19
- TrainingArguments,
20
- Trainer,
21
- DataCollatorForLanguageModeling,
22
- )
23
- from datasets import load_dataset, Dataset
24
-
25
- # Try to import torch for memory cleanup
26
- try:
27
- import torch
28
- TORCH_AVAILABLE = True
29
- except ImportError:
30
- TORCH_AVAILABLE = False
31
-
32
- # ============ CONFIGURATION ============
33
- HF_USERNAME = "himu1780"
34
- MODEL_REPO = f"{HF_USERNAME}/ai-python-model"
35
- DATASET_NAME = "jtatman/python-code-dataset-500k"
36
- BASE_MODEL = "gpt2"
37
-
38
- # Training hyperparameters (Memory optimized)
39
- BATCH_SIZE = 1
40
- GRADIENT_ACCUMULATION = 8
41
- SAVE_STEPS = 500
42
- LOGGING_STEPS = 50
43
- MAX_LENGTH = 256
44
- LEARNING_RATE = 5e-5
45
- MAX_STEPS_PER_SESSION = 10000
46
- EXAMPLES_PER_SESSION = 50000
47
-
48
- # Continuous training settings
49
- CONTINUOUS_TRAINING = True # Set False to stop after one session
50
- WAIT_BETWEEN_SESSIONS = 60 # Seconds to wait before next session
51
-
52
- # ============ GLOBAL STATE ============
53
- training_status = {
54
- "is_training": False,
55
- "current_step": 0,
56
- "total_loss": 0,
57
- "last_save": "Never",
58
- "start_time": None,
59
- "message": "Initializing...",
60
- "session_count": 0,
61
- }
62
-
63
- stop_requested = False
64
-
65
- # ============ MEMORY CLEANUP ============
66
- def cleanup_memory():
67
- """Free up memory after training"""
68
- gc.collect()
69
- if TORCH_AVAILABLE and torch.cuda.is_available():
70
- torch.cuda.empty_cache()
71
- print("[INFO] Memory cleaned up")
72
-
73
- # ============ AUTHENTICATION ============
74
- def authenticate():
75
- """Login to Hugging Face Hub"""
76
- token = os.environ.get("HF_TOKEN")
77
- if token:
78
- login(token=token)
79
- training_status["message"] = "βœ… Authenticated with Hugging Face"
80
- return True
81
- else:
82
- training_status["message"] = "❌ HF_TOKEN not found in secrets!"
83
- return False
84
-
85
- # ============ MODEL LOADING ============
86
- def load_model_and_tokenizer():
87
- """Load model from Hub (resume) or start fresh from base model"""
88
- global training_status
89
-
90
- tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
91
- tokenizer.pad_token = tokenizer.eos_token
92
-
93
- try:
94
- training_status["message"] = f"πŸ”„ Attempting to resume from {MODEL_REPO}..."
95
- model = AutoModelForCausalLM.from_pretrained(MODEL_REPO)
96
- training_status["message"] = f"βœ… Resumed from {MODEL_REPO}"
97
- print(f"[INFO] Resumed training from {MODEL_REPO}")
98
- except Exception as e:
99
- training_status["message"] = f"πŸ†• Starting fresh from {BASE_MODEL}"
100
- model = AutoModelForCausalLM.from_pretrained(BASE_MODEL)
101
- print(f"[INFO] Starting fresh from {BASE_MODEL}: {e}")
102
-
103
- return model, tokenizer
104
-
105
- # ============ DATASET PROCESSING ============
106
- def prepare_dataset(tokenizer):
107
- """Load and prepare dataset"""
108
- global training_status
109
- training_status["message"] = "πŸ“₯ Loading dataset (streaming mode)..."
110
-
111
- try:
112
- dataset = load_dataset(DATASET_NAME, split="train", streaming=True)
113
- dataset = dataset.take(EXAMPLES_PER_SESSION)
114
-
115
- def tokenize_function(examples):
116
- texts = []
117
- instructions = examples.get("instruction", [])
118
- outputs = examples.get("output", [])
119
-
120
- for instruction, output in zip(instructions, outputs):
121
- if instruction and output:
122
- text = f"### Instruction:\n{instruction}\n\n### Response:\n{output}"
123
- texts.append(text)
124
-
125
- if not texts:
126
- texts = [""]
127
-
128
- result = tokenizer(
129
- texts,
130
- truncation=True,
131
- max_length=MAX_LENGTH,
132
- padding="max_length",
133
- return_tensors=None,
134
- )
135
- result["labels"] = result["input_ids"].copy()
136
- return result
137
-
138
- tokenized_dataset = dataset.map(
139
- tokenize_function,
140
- batched=True,
141
- batch_size=100,
142
- remove_columns=["instruction", "output"],
143
- )
144
-
145
- training_status["message"] = "πŸ”„ Converting dataset for Trainer..."
146
-
147
- all_examples = []
148
- for i, example in enumerate(tokenized_dataset):
149
- all_examples.append(example)
150
- # Progress every 5000 (IMPROVED)
151
- if i % 5000 == 0:
152
- training_status["message"] = f"πŸ“₯ Loaded {i:,}/{EXAMPLES_PER_SESSION:,} examples..."
153
- if i >= EXAMPLES_PER_SESSION - 1:
154
- break
155
-
156
- train_dataset = Dataset.from_list(all_examples)
157
-
158
- training_status["message"] = f"βœ… Dataset ready: {len(train_dataset):,} examples"
159
- return train_dataset
160
-
161
- except Exception as e:
162
- training_status["message"] = f"❌ Dataset error: {str(e)}"
163
- print(f"[ERROR] Dataset preparation failed: {e}")
164
- raise e
165
-
166
- # ============ CUSTOM TRAINER ============
167
- class StatusTrainer(Trainer):
168
- """Custom trainer with status updates and stop support"""
169
-
170
- def training_step(self, model, inputs):
171
- global stop_requested
172
- if stop_requested:
173
- raise KeyboardInterrupt("Stop requested by user")
174
- return super().training_step(model, inputs)
175
-
176
- def log(self, logs):
177
- super().log(logs)
178
- if "loss" in logs:
179
- training_status["total_loss"] = logs["loss"]
180
- training_status["current_step"] = self.state.global_step
181
-
182
- def save_model(self, output_dir=None, _internal_call=False):
183
- super().save_model(output_dir, _internal_call)
184
- training_status["last_save"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
185
-
186
- # ============ SINGLE TRAINING SESSION ============
187
- def run_training_session():
188
- """Run a single training session"""
189
- global training_status, stop_requested
190
-
191
- model = None
192
- trainer = None
193
-
194
- try:
195
- if not authenticate():
196
- return False
197
-
198
- model, tokenizer = load_model_and_tokenizer()
199
- train_dataset = prepare_dataset(tokenizer)
200
-
201
- data_collator = DataCollatorForLanguageModeling(
202
- tokenizer=tokenizer,
203
- mlm=False,
204
- )
205
-
206
- training_args = TrainingArguments(
207
- output_dir="./temp_checkpoints",
208
- overwrite_output_dir=True,
209
- per_device_train_batch_size=BATCH_SIZE,
210
- gradient_accumulation_steps=GRADIENT_ACCUMULATION,
211
- learning_rate=LEARNING_RATE,
212
- warmup_steps=100,
213
- weight_decay=0.01,
214
- logging_steps=LOGGING_STEPS,
215
- save_steps=SAVE_STEPS,
216
- save_total_limit=1,
217
- push_to_hub=True,
218
- hub_model_id=MODEL_REPO,
219
- hub_strategy="every_save",
220
- report_to="none",
221
- max_steps=MAX_STEPS_PER_SESSION,
222
- fp16=False,
223
- dataloader_num_workers=0,
224
- remove_unused_columns=False,
225
- )
226
-
227
- trainer = StatusTrainer(
228
- model=model,
229
- args=training_args,
230
- train_dataset=train_dataset,
231
- data_collator=data_collator,
232
- tokenizer=tokenizer,
233
- )
234
-
235
- training_status["message"] = "πŸƒ Training in progress..."
236
- trainer.train()
237
- trainer.push_to_hub()
238
-
239
- training_status["session_count"] += 1
240
- training_status["message"] = f"βœ… Session {training_status['session_count']} completed!"
241
- return True
242
-
243
- except KeyboardInterrupt:
244
- training_status["message"] = "⏹️ Training stopped by user"
245
- return False
246
- except Exception as e:
247
- training_status["message"] = f"❌ Error: {str(e)}"
248
- print(f"[ERROR] Training failed: {e}")
249
- import traceback
250
- traceback.print_exc()
251
- return False
252
- finally:
253
- # MEMORY CLEANUP (IMPROVED)
254
- del model, trainer
255
- cleanup_memory()
256
-
257
- # ============ MAIN TRAINING LOOP ============
258
- def start_training():
259
- """Main training function with continuous loop"""
260
- global training_status, stop_requested
261
-
262
- if training_status["is_training"]:
263
- return "Training already in progress!"
264
-
265
- training_status["is_training"] = True
266
- training_status["start_time"] = datetime.now()
267
- stop_requested = False
268
-
269
- # CONTINUOUS TRAINING LOOP (IMPROVED)
270
- while not stop_requested:
271
- training_status["message"] = f"πŸš€ Starting session {training_status['session_count'] + 1}..."
272
-
273
- success = run_training_session()
274
-
275
- if stop_requested:
276
- break
277
-
278
- if not CONTINUOUS_TRAINING:
279
- break
280
-
281
- if success:
282
- training_status["message"] = f"⏳ Waiting {WAIT_BETWEEN_SESSIONS}s before next session..."
283
- time.sleep(WAIT_BETWEEN_SESSIONS)
284
- else:
285
- training_status["message"] = "⚠️ Session failed, retrying in 60s..."
286
- time.sleep(60)
287
-
288
- training_status["is_training"] = False
289
- stop_requested = False
290
- training_status["message"] = f"βœ… Training finished! Total sessions: {training_status['session_count']}"
291
- return training_status["message"]
292
-
293
- # ============ GRADIO INTERFACE ============
294
- def get_status():
295
- """Get current training status"""
296
- elapsed = ""
297
- if training_status["start_time"]:
298
- delta = datetime.now() - training_status["start_time"]
299
- hours, remainder = divmod(int(delta.total_seconds()), 3600)
300
- minutes, seconds = divmod(remainder, 60)
301
- elapsed = f"{hours}h {minutes}m {seconds}s"
302
-
303
- return f"""
304
- ## πŸ€– AI Python Model Trainer
305
-
306
- ### Status
307
- | Item | Value |
308
- |------|-------|
309
- | **State** | {"🟒 Training" if training_status["is_training"] else "πŸ”΄ Stopped"} |
310
- | **Message** | {training_status["message"]} |
311
- | **Sessions Completed** | {training_status["session_count"]} |
312
-
313
- ### Progress
314
- | Metric | Value |
315
- |--------|-------|
316
- | **Current Step** | {training_status["current_step"]:,} / {MAX_STEPS_PER_SESSION:,} |
317
- | **Current Loss** | {training_status["total_loss"]:.4f if training_status["total_loss"] else "N/A"} |
318
- | **Last Checkpoint** | {training_status["last_save"]} |
319
- | **Elapsed Time** | {elapsed if elapsed else "N/A"} |
320
-
321
- ### Configuration
322
- | Setting | Value |
323
- |---------|-------|
324
- | **Model Repo** | [{MODEL_REPO}](https://huggingface.co/{MODEL_REPO}) |
325
- | **Continuous Mode** | {"βœ… Enabled" if CONTINUOUS_TRAINING else "❌ Disabled"} |
326
- | **Batch Size** | {BATCH_SIZE} (effective: {BATCH_SIZE * GRADIENT_ACCUMULATION}) |
327
- | **Max Steps/Session** | {MAX_STEPS_PER_SESSION:,} |
328
- """
329
-
330
- def start_training_async():
331
- """Start training in background"""
332
- if training_status["is_training"]:
333
- return "⚠️ Training already in progress!"
334
- thread = threading.Thread(target=start_training, daemon=True)
335
- thread.start()
336
- return "πŸš€ Training started in background!"
337
-
338
- def stop_training():
339
- """Stop training"""
340
- global stop_requested
341
- if not training_status["is_training"]:
342
- return "⚠️ No training in progress"
343
- stop_requested = True
344
- training_status["message"] = "⏹️ Stopping after current step..."
345
- return "⏹️ Stop requested"
346
-
347
- # ============ AUTO-START ============
348
- def auto_start():
349
- """Auto-start continuous training on Space launch"""
350
- time.sleep(10)
351
- while True:
352
- if not training_status["is_training"] and not stop_requested:
353
- print("[INFO] Auto-starting training session...")
354
- start_training()
355
- time.sleep(WAIT_BETWEEN_SESSIONS)
356
-
357
- auto_thread = threading.Thread(target=auto_start, daemon=True)
358
- auto_thread.start()
359
-
360
- # ============ GRADIO APP ============
361
- with gr.Blocks(title="AI Python Trainer", theme=gr.themes.Soft()) as demo:
362
- gr.Markdown("# 🐍 AI Python Code Model Trainer")
363
- gr.Markdown(f"**Continuous training** on `{DATASET_NAME}` with auto-checkpoint")
364
-
365
- status_display = gr.Markdown(get_status)
366
-
367
- with gr.Row():
368
- start_btn = gr.Button("▢️ Start Training", variant="primary")
369
- stop_btn = gr.Button("⏹️ Stop Training", variant="stop")
370
- refresh_btn = gr.Button("πŸ”„ Refresh Status")
371
-
372
- output = gr.Textbox(label="Output", interactive=False)
373
-
374
- start_btn.click(start_training_async, outputs=output)
375
- stop_btn.click(stop_training, outputs=output)
376
- refresh_btn.click(get_status, outputs=status_display)
377
-
378
- demo.load(get_status, outputs=status_display, every=30)
379
-
380
- demo.launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ AI Python Code Model Trainer
3
+ Hugging Face Space for continuous training with auto-resume
4
+ Username: himu1780 | Model: ai-python-model
5
+
6
+ FINAL VERSION - All optimizations applied
7
+ """
8
+
9
+ import os
10
+ import gc
11
+ import gradio as gr
12
+ import threading
13
+ import time
14
+ from datetime import datetime
15
+ from huggingface_hub import HfApi, login
16
+ from transformers import (
17
+ AutoModelForCausalLM,
18
+ AutoTokenizer,
19
+ TrainingArguments,
20
+ Trainer,
21
+ DataCollatorForLanguageModeling,
22
+ )
23
+ from datasets import load_dataset, Dataset
24
+
25
+ # Try to import torch for memory cleanup
26
+ try:
27
+ import torch
28
+ TORCH_AVAILABLE = True
29
+ except ImportError:
30
+ TORCH_AVAILABLE = False
31
+
32
+ # ============ CONFIGURATION ============
33
+ HF_USERNAME = "himu1780"
34
+ MODEL_REPO = f"{HF_USERNAME}/ai-python-model"
35
+ DATASET_NAME = "jtatman/python-code-dataset-500k"
36
+ BASE_MODEL = "gpt2"
37
+
38
+ # Training hyperparameters (Memory optimized)
39
+ BATCH_SIZE = 1
40
+ GRADIENT_ACCUMULATION = 8
41
+ SAVE_STEPS = 500
42
+ LOGGING_STEPS = 50
43
+ MAX_LENGTH = 256
44
+ LEARNING_RATE = 5e-5
45
+ MAX_STEPS_PER_SESSION = 10000
46
+ EXAMPLES_PER_SESSION = 50000
47
+
48
+ # Continuous training settings
49
+ CONTINUOUS_TRAINING = True # Set False to stop after one session
50
+ WAIT_BETWEEN_SESSIONS = 60 # Seconds to wait before next session
51
+
52
+ # ============ GLOBAL STATE ============
53
+ training_status = {
54
+ "is_training": False,
55
+ "current_step": 0,
56
+ "total_loss": 0,
57
+ "last_save": "Never",
58
+ "start_time": None,
59
+ "message": "Initializing...",
60
+ "session_count": 0,
61
+ }
62
+
63
+ stop_requested = False
64
+
65
+ # ============ MEMORY CLEANUP ============
66
+ def cleanup_memory():
67
+ """Free up memory after training"""
68
+ gc.collect()
69
+ if TORCH_AVAILABLE and torch.cuda.is_available():
70
+ torch.cuda.empty_cache()
71
+ print("[INFO] Memory cleaned up")
72
+
73
+ # ============ AUTHENTICATION ============
74
+ def authenticate():
75
+ """Login to Hugging Face Hub"""
76
+ token = os.environ.get("HF_TOKEN")
77
+ if token:
78
+ login(token=token)
79
+ training_status["message"] = "βœ… Authenticated with Hugging Face"
80
+ return True
81
+ else:
82
+ training_status["message"] = "❌ HF_TOKEN not found in secrets!"
83
+ return False
84
+
85
+ # ============ MODEL LOADING ============
86
+ def load_model_and_tokenizer():
87
+ """Load model from Hub (resume) or start fresh from base model"""
88
+ global training_status
89
+
90
+ tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
91
+ tokenizer.pad_token = tokenizer.eos_token
92
+
93
+ try:
94
+ training_status["message"] = f"πŸ”„ Attempting to resume from {MODEL_REPO}..."
95
+ model = AutoModelForCausalLM.from_pretrained(MODEL_REPO)
96
+ training_status["message"] = f"βœ… Resumed from {MODEL_REPO}"
97
+ print(f"[INFO] Resumed training from {MODEL_REPO}")
98
+ except Exception as e:
99
+ training_status["message"] = f"πŸ†• Starting fresh from {BASE_MODEL}"
100
+ model = AutoModelForCausalLM.from_pretrained(BASE_MODEL)
101
+ print(f"[INFO] Starting fresh from {BASE_MODEL}: {e}")
102
+
103
+ return model, tokenizer
104
+
105
+ # ============ DATASET PROCESSING ============
106
+ def prepare_dataset(tokenizer):
107
+ """Load and prepare dataset"""
108
+ global training_status
109
+ training_status["message"] = "πŸ“₯ Loading dataset (streaming mode)..."
110
+
111
+ try:
112
+ dataset = load_dataset(DATASET_NAME, split="train", streaming=True)
113
+ dataset = dataset.take(EXAMPLES_PER_SESSION)
114
+
115
+ def tokenize_function(examples):
116
+ texts = []
117
+ instructions = examples.get("instruction", [])
118
+ outputs = examples.get("output", [])
119
+
120
+ for instruction, output in zip(instructions, outputs):
121
+ if instruction and output:
122
+ text = f"### Instruction:\n{instruction}\n\n### Response:\n{output}"
123
+ texts.append(text)
124
+
125
+ if not texts:
126
+ texts = [""]
127
+
128
+ result = tokenizer(
129
+ texts,
130
+ truncation=True,
131
+ max_length=MAX_LENGTH,
132
+ padding="max_length",
133
+ return_tensors=None,
134
+ )
135
+ result["labels"] = result["input_ids"].copy()
136
+ return result
137
+
138
+ tokenized_dataset = dataset.map(
139
+ tokenize_function,
140
+ batched=True,
141
+ batch_size=100,
142
+ remove_columns=["instruction", "output"],
143
+ )
144
+
145
+ training_status["message"] = "πŸ”„ Converting dataset for Trainer..."
146
+
147
+ all_examples = []
148
+ for i, example in enumerate(tokenized_dataset):
149
+ all_examples.append(example)
150
+ # Progress every 5000 (IMPROVED)
151
+ if i % 5000 == 0:
152
+ training_status["message"] = f"πŸ“₯ Loaded {i:,}/{EXAMPLES_PER_SESSION:,} examples..."
153
+ if i >= EXAMPLES_PER_SESSION - 1:
154
+ break
155
+
156
+ train_dataset = Dataset.from_list(all_examples)
157
+
158
+ training_status["message"] = f"βœ… Dataset ready: {len(train_dataset):,} examples"
159
+ return train_dataset
160
+
161
+ except Exception as e:
162
+ training_status["message"] = f"❌ Dataset error: {str(e)}"
163
+ print(f"[ERROR] Dataset preparation failed: {e}")
164
+ raise e
165
+
166
+ # ============ CUSTOM TRAINER ============
167
+ class StatusTrainer(Trainer):
168
+ """Custom trainer with status updates and stop support"""
169
+
170
+ def training_step(self, model, inputs):
171
+ global stop_requested
172
+ if stop_requested:
173
+ raise KeyboardInterrupt("Stop requested by user")
174
+ return super().training_step(model, inputs)
175
+
176
+ def log(self, logs):
177
+ super().log(logs)
178
+ if "loss" in logs:
179
+ training_status["total_loss"] = logs["loss"]
180
+ training_status["current_step"] = self.state.global_step
181
+
182
+ def save_model(self, output_dir=None, _internal_call=False):
183
+ super().save_model(output_dir, _internal_call)
184
+ training_status["last_save"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
185
+
186
+ # ============ SINGLE TRAINING SESSION ============
187
+ def run_training_session():
188
+ """Run a single training session"""
189
+ global training_status, stop_requested
190
+
191
+ model = None
192
+ trainer = None
193
+
194
+ try:
195
+ if not authenticate():
196
+ return False
197
+
198
+ model, tokenizer = load_model_and_tokenizer()
199
+ train_dataset = prepare_dataset(tokenizer)
200
+
201
+ data_collator = DataCollatorForLanguageModeling(
202
+ tokenizer=tokenizer,
203
+ mlm=False,
204
+ )
205
+
206
+ training_args = TrainingArguments(
207
+ output_dir="./temp_checkpoints",
208
+ overwrite_output_dir=True,
209
+ per_device_train_batch_size=BATCH_SIZE,
210
+ gradient_accumulation_steps=GRADIENT_ACCUMULATION,
211
+ learning_rate=LEARNING_RATE,
212
+ warmup_steps=100,
213
+ weight_decay=0.01,
214
+ logging_steps=LOGGING_STEPS,
215
+ save_steps=SAVE_STEPS,
216
+ save_total_limit=1,
217
+ push_to_hub=True,
218
+ hub_model_id=MODEL_REPO,
219
+ hub_strategy="every_save",
220
+ report_to="none",
221
+ max_steps=MAX_STEPS_PER_SESSION,
222
+ fp16=False,
223
+ dataloader_num_workers=0,
224
+ remove_unused_columns=False,
225
+ )
226
+
227
+ trainer = StatusTrainer(
228
+ model=model,
229
+ args=training_args,
230
+ train_dataset=train_dataset,
231
+ data_collator=data_collator,
232
+ tokenizer=tokenizer,
233
+ )
234
+
235
+ training_status["message"] = "πŸƒ Training in progress..."
236
+ trainer.train()
237
+ trainer.push_to_hub()
238
+
239
+ training_status["session_count"] += 1
240
+ training_status["message"] = f"βœ… Session {training_status['session_count']} completed!"
241
+ return True
242
+
243
+ except KeyboardInterrupt:
244
+ training_status["message"] = "⏹️ Training stopped by user"
245
+ return False
246
+ except Exception as e:
247
+ training_status["message"] = f"❌ Error: {str(e)}"
248
+ print(f"[ERROR] Training failed: {e}")
249
+ import traceback
250
+ traceback.print_exc()
251
+ return False
252
+ finally:
253
+ # MEMORY CLEANUP (IMPROVED)
254
+ del model, trainer
255
+ cleanup_memory()
256
+
257
+ # ============ MAIN TRAINING LOOP ============
258
+ def start_training():
259
+ """Main training function with continuous loop"""
260
+ global training_status, stop_requested
261
+
262
+ if training_status["is_training"]:
263
+ return "Training already in progress!"
264
+
265
+ training_status["is_training"] = True
266
+ training_status["start_time"] = datetime.now()
267
+ stop_requested = False
268
+
269
+ # CONTINUOUS TRAINING LOOP (IMPROVED)
270
+ while not stop_requested:
271
+ training_status["message"] = f"πŸš€ Starting session {training_status['session_count'] + 1}..."
272
+
273
+ success = run_training_session()
274
+
275
+ if stop_requested:
276
+ break
277
+
278
+ if not CONTINUOUS_TRAINING:
279
+ break
280
+
281
+ if success:
282
+ training_status["message"] = f"⏳ Waiting {WAIT_BETWEEN_SESSIONS}s before next session..."
283
+ time.sleep(WAIT_BETWEEN_SESSIONS)
284
+ else:
285
+ training_status["message"] = "⚠️ Session failed, retrying in 60s..."
286
+ time.sleep(60)
287
+
288
+ training_status["is_training"] = False
289
+ stop_requested = False
290
+ training_status["message"] = f"βœ… Training finished! Total sessions: {training_status['session_count']}"
291
+ return training_status["message"]
292
+
293
+ # ============ GRADIO INTERFACE ============
294
+ def get_status():
295
+ """Get current training status"""
296
+ elapsed = ""
297
+ if training_status["start_time"]:
298
+ delta = datetime.now() - training_status["start_time"]
299
+ hours, remainder = divmod(int(delta.total_seconds()), 3600)
300
+ minutes, seconds = divmod(remainder, 60)
301
+ elapsed = f"{hours}h {minutes}m {seconds}s"
302
+
303
+ # Pre-compute formatted loss string to avoid f-string format specifier issues
304
+ if training_status["total_loss"]:
305
+ loss_str = f"{training_status['total_loss']:.4f}"
306
+ else:
307
+ loss_str = "N/A"
308
+
309
+ # Pre-compute other conditional strings
310
+ state_str = "🟒 Training" if training_status["is_training"] else "πŸ”΄ Stopped"
311
+ continuous_str = "βœ… Enabled" if CONTINUOUS_TRAINING else "❌ Disabled"
312
+ elapsed_str = elapsed if elapsed else "N/A"
313
+ effective_batch = BATCH_SIZE * GRADIENT_ACCUMULATION
314
+
315
+ return f"""
316
+ ## πŸ€– AI Python Model Trainer
317
+
318
+ ### Status
319
+ | Item | Value |
320
+ |------|-------|
321
+ | **State** | {state_str} |
322
+ | **Message** | {training_status["message"]} |
323
+ | **Sessions Completed** | {training_status["session_count"]} |
324
+
325
+ ### Progress
326
+ | Metric | Value |
327
+ |--------|-------|
328
+ | **Current Step** | {training_status["current_step"]:,} / {MAX_STEPS_PER_SESSION:,} |
329
+ | **Current Loss** | {loss_str} |
330
+ | **Last Checkpoint** | {training_status["last_save"]} |
331
+ | **Elapsed Time** | {elapsed_str} |
332
+
333
+ ### Configuration
334
+ | Setting | Value |
335
+ |---------|-------|
336
+ | **Model Repo** | [{MODEL_REPO}](https://huggingface.co/{MODEL_REPO}) |
337
+ | **Continuous Mode** | {continuous_str} |
338
+ | **Batch Size** | {BATCH_SIZE} (effective: {effective_batch}) |
339
+ | **Max Steps/Session** | {MAX_STEPS_PER_SESSION:,} |
340
+ """
341
+
342
+ def start_training_async():
343
+ """Start training in background"""
344
+ if training_status["is_training"]:
345
+ return "⚠️ Training already in progress!"
346
+ thread = threading.Thread(target=start_training, daemon=True)
347
+ thread.start()
348
+ return "πŸš€ Training started in background!"
349
+
350
+ def stop_training():
351
+ """Stop training"""
352
+ global stop_requested
353
+ if not training_status["is_training"]:
354
+ return "⚠️ No training in progress"
355
+ stop_requested = True
356
+ training_status["message"] = "⏹️ Stopping after current step..."
357
+ return "⏹️ Stop requested"
358
+
359
+ # ============ AUTO-START ============
360
+ def auto_start():
361
+ """Auto-start continuous training on Space launch"""
362
+ time.sleep(10)
363
+ while True:
364
+ if not training_status["is_training"] and not stop_requested:
365
+ print("[INFO] Auto-starting training session...")
366
+ start_training()
367
+ time.sleep(WAIT_BETWEEN_SESSIONS)
368
+
369
+ auto_thread = threading.Thread(target=auto_start, daemon=True)
370
+ auto_thread.start()
371
+
372
+ # ============ GRADIO APP ============
373
+ with gr.Blocks(title="AI Python Trainer") as demo:
374
+ gr.Markdown("# 🐍 AI Python Code Model Trainer")
375
+ gr.Markdown(f"**Continuous training** on `{DATASET_NAME}` with auto-checkpoint")
376
+
377
+ status_display = gr.Markdown(get_status)
378
+
379
+ with gr.Row():
380
+ start_btn = gr.Button("▢️ Start Training", variant="primary")
381
+ stop_btn = gr.Button("⏹️ Stop Training", variant="stop")
382
+ refresh_btn = gr.Button("πŸ”„ Refresh Status")
383
+
384
+ output = gr.Textbox(label="Output", interactive=False)
385
+
386
+ start_btn.click(start_training_async, outputs=output)
387
+ stop_btn.click(stop_training, outputs=output)
388
+ refresh_btn.click(get_status, outputs=status_display)
389
+
390
+ demo.load(get_status, outputs=status_display, every=30)
391
+
392
+ demo.launch()