attempt to save in file

ui/src/app/api/hf-jobs/route.ts

@@ -1,6 +1,6 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { spawn } from 'child_process';
-import { writeFile } from 'fs/promises';
+import { writeFile, readFile, unlink } from 'fs/promises';
 import path from 'path';
 import { tmpdir } from 'os';
 
@@ -1033,85 +1033,75 @@ async function checkHFJobStatus(token: string, jobId: string, jobNamespace?: str
 }
 
 async function checkHFJobsCapacity(token: string): Promise<any> {
-  return new Promise((resolve, reject) => {
+  return new Promise(async (resolve, reject) => {
     console.log('Checking HF Jobs capacity for namespace: lora-training-frenzi');
-    const args = [
-      'jobs', 'ps',
-      '--namespace', 'lora-training-frenzi',
-      '--token', token
-    ];
 
-    const childProcess = spawn('hf', args, {
+    // Create a temporary file to store the output
+    const tempFile = path.join(tmpdir(), `hf_jobs_ps_${Date.now()}.txt`);
+    console.log(`Writing output to temp file: ${tempFile}`);
+
+    // Use shell redirection to write to file
+    const command = `hf jobs ps --namespace lora-training-frenzi --token "${token}" > "${tempFile}" 2>&1`;
+
+    const childProcess = spawn('sh', ['-c', command], {
       env: {
         ...process.env,
         HF_TOKEN: token,
-        TERM: 'dumb',  // Disable fancy terminal features
-        NO_COLOR: '1',  // Disable color output
-      },
-      stdio: ['ignore', 'pipe', 'pipe']  // Explicitly set stdio to avoid TTY issues
+        TERM: 'dumb',
+        NO_COLOR: '1',
+      }
     });
 
-    let output = '';
-    let error = '';
-
-    childProcess.stdout.on('data', (data) => {
-      const text = data.toString();
-      output += text;
-    });
+    childProcess.on('close', async (code) => {
+      console.log(`hf jobs ps process exited with code: ${code}`);
 
-    childProcess.stderr.on('data', (data) => {
-      const text = data.toString();
-      error += text;
-    });
+      try {
+        // Read the output from the temporary file
+        const output = await readFile(tempFile, 'utf-8');
 
-    childProcess.on('close', (code) => {
-      console.log(`hf jobs ps process exited with code: ${code}`);
-      console.log('=== RAW OUTPUT START ===');
-      console.log(output);
-      console.log('=== RAW OUTPUT END ===');
-      console.log('=== RAW ERROR START ===');
-      console.log(error);
-      console.log('=== RAW ERROR END ===');
+        console.log('=== RAW OUTPUT START ===');
+        console.log(output);
+        console.log('=== RAW OUTPUT END ===');
 
-      if (code === 0) {
-        try {
-          // Count RUNNING jobs in the output
-          // Split by newline and filter out empty lines
-          const lines = output.split(/\r?\n/).filter(line => line.trim().length > 0);
-          let runningCount = 0;
+        // Count RUNNING jobs in the output
+        // Split by newline and filter out empty lines
+        const lines = output.split(/\r?\n/).filter(line => line.trim().length > 0);
+        let runningCount = 0;
 
-          console.log(`Total non-empty lines in output: ${lines.length}`);
+        console.log(`Total non-empty lines in output: ${lines.length}`);
 
-          for (let i = 0; i < lines.length; i++) {
-            const line = lines[i];
-            console.log(`Line ${i}: "${line}"`);
+        for (let i = 0; i < lines.length; i++) {
+          const line = lines[i];
+          console.log(`Line ${i}: "${line}"`);
 
-            // Check if line contains RUNNING (case-sensitive as shown in your output)
-            if (line.includes('RUNNING')) {
-              runningCount++;
-              console.log(`  ✓ Line ${i} contains RUNNING (count: ${runningCount})`);
-            }
+          // Check if line contains RUNNING (case-sensitive as shown in your output)
+          if (line.includes('RUNNING')) {
+            runningCount++;
+            console.log(`  ✓ Line ${i} contains RUNNING (count: ${runningCount})`);
           }
+        }
 
-          const atCapacity = runningCount >= 32;
+        const atCapacity = runningCount >= 32;
 
-          console.log(`\n=== FINAL COUNT ===`);
-          console.log(`Found ${runningCount} RUNNING jobs. At capacity: ${atCapacity}`);
-          console.log(`==================\n`);
+        console.log(`\n=== FINAL COUNT ===`);
+        console.log(`Found ${runningCount} RUNNING jobs. At capacity: ${atCapacity}`);
+        console.log(`==================\n`);
 
-          resolve({
-            runningJobs: runningCount,
-            atCapacity,
-            capacityLimit: 32,
-          });
-        } catch (parseError: any) {
-          console.error('Failed to parse jobs ps output:', parseError);
-          reject(new Error('Failed to parse capacity status'));
+        // Clean up temp file
+        try {
+          await unlink(tempFile);
+        } catch (unlinkError) {
+          console.warn('Failed to delete temp file:', unlinkError);
         }
-      } else {
-        console.error('hf jobs ps failed with code:', code);
-        console.error('Error output:', error);
-        reject(new Error(error || output || 'Failed to check capacity'));
+
+        resolve({
+          runningJobs: runningCount,
+          atCapacity,
+          capacityLimit: 32,
+        });
+      } catch (parseError: any) {
+        console.error('Failed to read or parse jobs ps output:', parseError);
+        reject(new Error('Failed to parse capacity status'));
       }
     });