Add capacity check

ui/src/app/api/hf-jobs/route.ts

@@ -10,6 +10,19 @@ export async function POST(request: NextRequest) {
     const { action, token, hardware, namespace, jobConfig, datasetRepo, participateHackathon } = body;
 
     switch (action) {
+      case 'checkCapacity':
+        try {
+          if (!token) {
+            return NextResponse.json({ error: 'Token required' }, { status: 400 });
+          }
+
+          const capacityStatus = await checkHFJobsCapacity(token);
+          return NextResponse.json(capacityStatus);
+        } catch (error: any) {
+          console.error('Capacity check error:', error);
+          return NextResponse.json({ error: error.message }, { status: 500 });
+        }
+
       case 'checkStatus':
         try {
           if (!token || !jobConfig?.hf_job_id) {
@@ -966,9 +979,9 @@ async function checkHFJobStatus(token: string, jobId: string, jobNamespace?: str
     args.push(jobId);
 
     const childProcess = spawn('hf', args, {
-      env: { 
-        ...process.env, 
-        HF_TOKEN: token 
+      env: {
+        ...process.env,
+        HF_TOKEN: token
       }
     });
 
@@ -1018,3 +1031,66 @@ async function checkHFJobStatus(token: string, jobId: string, jobNamespace?: str
     });
   });
 }
+
+async function checkHFJobsCapacity(token: string): Promise<any> {
+  return new Promise((resolve, reject) => {
+    console.log('Checking HF Jobs capacity for namespace: lora-training-frenzi');
+    const args = ['jobs', 'ps', '--namespace', 'lora-training-frenzi'];
+
+    const childProcess = spawn('hf', args, {
+      env: {
+        ...process.env,
+        HF_TOKEN: token
+      }
+    });
+
+    let output = '';
+    let error = '';
+
+    childProcess.stdout.on('data', (data) => {
+      const text = data.toString();
+      output += text;
+    });
+
+    childProcess.stderr.on('data', (data) => {
+      const text = data.toString();
+      error += text;
+    });
+
+    childProcess.on('close', (code) => {
+      if (code === 0) {
+        try {
+          // Count RUNNING jobs in the output
+          const lines = output.split('\n');
+          let runningCount = 0;
+
+          for (const line of lines) {
+            if (line.includes('RUNNING')) {
+              runningCount++;
+            }
+          }
+
+          const atCapacity = runningCount > 35;
+
+          console.log(`Found ${runningCount} RUNNING jobs. At capacity: ${atCapacity}`);
+
+          resolve({
+            runningJobs: runningCount,
+            atCapacity,
+            capacityLimit: 35,
+          });
+        } catch (parseError: any) {
+          console.error('Failed to parse jobs ps output:', parseError);
+          reject(new Error('Failed to parse capacity status'));
+        }
+      } else {
+        reject(new Error(error || output || 'Failed to check capacity'));
+      }
+    });
+
+    childProcess.on('error', (err) => {
+      console.error('HF Jobs ps process error:', err);
+      reject(new Error(`Process error: ${err.message}`));
+    });
+  });
+}

ui/src/components/HFJobsWorkflow.tsx

@@ -213,7 +213,7 @@ export default function HFJobsWorkflow({ jobConfig, onComplete, hackathonEligibl
   const validateToken = async () => {
     setLoading(true);
     setError(null);
-    
+
     const effectiveToken = authToken || settings.HF_TOKEN;
 
     try {
@@ -221,6 +221,16 @@ export default function HFJobsWorkflow({ jobConfig, onComplete, hackathonEligibl
         throw new Error('A valid Hugging Face token is required to continue.');
       }
 
+      // Check capacity first
+      const capacityResponse = await apiClient.post('/api/hf-jobs', {
+        action: 'checkCapacity',
+        token: effectiveToken,
+      });
+
+      if (capacityResponse.data.atCapacity) {
+        throw new Error('Whoa, our GPUs are going brr 🔥, we are at capacity right now. Try again soon');
+      }
+
       const response = await apiClient.post('/api/hf-hub', {
         action: 'whoami',
         token: effectiveToken,
@@ -236,7 +246,7 @@ export default function HFJobsWorkflow({ jobConfig, onComplete, hackathonEligibl
         setCurrentStep('upload');
       }
     } catch (err: any) {
-      setError(err.response?.data?.error || 'Failed to validate token');
+      setError(err.response?.data?.error || err.message || 'Failed to validate token');
     } finally {
       setLoading(false);
     }