Streaming interview feedback

api/llm.py

@@ -13,21 +13,74 @@ class LLMManager:
         self.is_demo = os.getenv("IS_DEMO")
         self.demo_word_limit = os.getenv("DEMO_WORD_LIMIT")
 
-    def test_connection(self):
+        self.status = self.test_llm()
+        if self.status:
+            self.streaming = self.test_llm_stream()
+        else:
+            self.streaming = False
+
+        if self.streaming:
+            self.end_interview = self.end_interview_stream
+        else:
+            self.end_interview = self.end_interview_full
+
+    def text_processor(self):
+        def ans_full(response):
+            return response
+
+        def ans_stream(response):
+            yield from response
+
+        if self.streaming:
+            return ans_full
+        else:
+            return ans_stream
+
+    def get_text(self, messages):
+        try:
+            response = self.client.chat.completions.create(model=self.config.llm.name, messages=messages, temperature=1)
+            if not response.choices:
+                raise APIError("LLM Get Text Error", details="No choices in response")
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            raise APIError(f"LLM Get Text Error: Unexpected error: {e}")
+
+    def get_text_stream(self, messages):
         try:
             response = self.client.chat.completions.create(
                 model=self.config.llm.name,
-                messages=[
-                    {"role": "system", "content": "You just help me test the connection."},
-                    {"role": "user", "content": "Hi!"},
-                    {"role": "user", "content": "Ping!"},
-                ],
+                messages=messages,
+                temperature=1,
+                stream=True,
             )
-            if not response.choices:
-                raise APIError("LLM Test Connection Error", details="No choices in response")
-            return response.choices[0].message.content.strip()
         except Exception as e:
-            raise APIError(f"LLM Test Connection Error: Unexpected error: {e}")
+            raise APIError(f"LLM End Interview Error: Unexpected error: {e}")
+        text = ""
+        for chunk in response:
+            if chunk.choices[0].delta.content:
+                text += chunk.choices[0].delta.content
+            yield text
+
+    test_messages = [
+        {"role": "system", "content": "You just help me test the connection."},
+        {"role": "user", "content": "Hi!"},
+        {"role": "user", "content": "Ping!"},
+    ]
+
+    def test_llm(self):
+        try:
+            self.get_text(self.test_messages)
+            return True
+        except:
+            return False
+
+    def test_llm_stream(self):
+        try:
+            for _ in self.get_text_stream(self.test_messages):
+                pass
+            return True
+        except:
+            return False
 
     def init_bot(self, problem=""):
         system_prompt = self.prompts["coding_interviewer_prompt"]
@@ -50,20 +103,12 @@ class LLMManager:
         if self.is_demo:
             full_prompt += f" Keep your response very short and simple, no more than {self.demo_word_limit} words."
 
-        try:
-            response = self.client.chat.completions.create(
-                model=self.config.llm.name,
-                messages=[
-                    {"role": "system", "content": self.prompts["problem_generation_prompt"]},
-                    {"role": "user", "content": full_prompt},
-                ],
-                temperature=1.0,
-            )
-            if not response.choices:
-                raise APIError("LLM Problem Generation Error", details="No choices in response")
-            question = response.choices[0].message.content.strip()
-        except Exception as e:
-            raise APIError(f"LLM Problem Generation Error: Unexpected error: {e}")
+        question = self.get_text(
+            [
+                {"role": "system", "content": self.prompts["problem_generation_prompt"]},
+                {"role": "user", "content": full_prompt},
+            ]
+        )
 
         chat_history = self.init_bot(question)
         return question, chat_history
@@ -73,14 +118,7 @@ class LLMManager:
             chat_history.append({"role": "user", "content": f"My latest code:\n{code}"})
         chat_history.append({"role": "user", "content": message})
 
-        try:
-            response = self.client.chat.completions.create(model=self.config.llm.name, messages=chat_history)
-            if not response.choices:
-                raise APIError("LLM Send Request Error", details="No choices in response")
-            reply = response.choices[0].message.content.strip()
-        except Exception as e:
-            raise APIError(f"LLM Send Request Error: Unexpected error: {e}")
-
+        reply = self.get_text(chat_history)
         chat_history.append({"role": "assistant", "content": reply})
 
         if chat_display:
@@ -90,11 +128,8 @@ class LLMManager:
 
         return chat_history, chat_display, "", code
 
-    def end_interview(self, problem_description, chat_history):
-
-        if not chat_history or len(chat_history) <= 2:
-            yield "No interview content available to review."
-
+    # TODO: implement both streaming and non-streaming versions
+    def end_interview_prepare_messages(self, problem_description, chat_history):
         transcript = [f"{message['role'].capitalize()}: {message['content']}" for message in chat_history[1:]]
 
         system_prompt = self.prompts["grading_feedback_prompt"]
@@ -108,27 +143,18 @@ class LLMManager:
             {"role": "user", "content": "Grade the interview based on the transcript provided and give feedback."},
         ]
 
-        if os.getenv("STREAMING", False):
-            try:
-                response = self.client.chat.completions.create(
-                    model=self.config.llm.name,
-                    messages=messages,
-                    temperature=0.5,
-                    stream=True,
-                )
-            except Exception as e:
-                raise APIError(f"LLM End Interview Error: Unexpected error: {e}")
-
-            feedback = ""
-            for chunk in response:
-                if chunk.choices[0].delta.content:
-                    feedback += chunk.choices[0].delta.content
-                yield feedback
-        #     else:
-        #         response = self.client.chat.completions.create(
-        #             model=self.config.llm.name,
-        #             messages=messages,
-        #             temperature=0.5,
-        #         )
-        #         feedback = response.choices[0].message.content.strip()
-        #         return feedback
+        return messages
+
+    def end_interview_full(self, problem_description, chat_history):
+        if len(chat_history) <= 2:
+            return "No interview history available"
+        else:
+            messages = self.end_interview_prepare_messages(problem_description, chat_history)
+            return self.get_text_stream(messages)
+
+    def end_interview_stream(self, problem_description, chat_history):
+        if len(chat_history) <= 2:
+            yield "No interview history available"
+        else:
+            messages = self.end_interview_prepare_messages(problem_description, chat_history)
+            yield from self.get_text_stream(messages)

app.py

@@ -78,11 +78,8 @@ with gr.Blocks(title="AI Interviewer") as demo:
                 except:
                     gr.Markdown(f"STT status: 🔴{space} {config.stt.name}")
 
-                try:
-                    llm.test_connection()
-                    gr.Markdown(f"LLM status: 🟢{space} {config.llm.name}")
-                except:
-                    gr.Markdown(f"LLM status: 🔴{space} {config.llm.name}")
+                llm_status = get_status_color(llm)
+                gr.Markdown(f"LLM status: {llm_status}{space}{config.llm.name}")
 
         gr.Markdown(instruction["quick_start"])
         with gr.Row():