Update app.py

app.py

@@ -11,7 +11,10 @@ client = AsyncInferenceClient(api_url)
 
 
 system_message = """
+### Instruction:
 Refactor the provided Python code to improve its maintainability and efficiency and reduce complexity. Include the refactored code along with the comments on the changes made for improving the metrics.
+### Input:
+
 """
 title = "Python Refactoring"
 description = """
@@ -87,12 +90,34 @@ def analyze_sales_data(sales_records):
 """]]
 
 
-# query client using streaming mode
-def inference(message, history):
+# Stream text - stream tokens with InferenceClient from TGI
+async def predict(message, chatbot,  temperature=0.9, max_new_tokens=4096, top_p=0.6, repetition_penalty=1.0,):
+    
+    if system_prompt != "":
+        input_prompt = f"{system_prompt}"
+    
+    temperature = float(temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(top_p)
+    
+  
+    input_prompt = input_prompt + str(message) + " [/INST] "
+
     partial_message = ""
-    for token in client.text_generation(message, max_new_tokens=4096, stream=True):
-        partial_message += token
+    async for token in await client.text_generation(prompt=input_prompt, 
+                                    max_new_tokens=max_new_tokens, 
+                                    stream=True, 
+                                    best_of=1, 
+                                    temperature=temperature, 
+                                    top_p=top_p, 
+                                    do_sample=True, 
+                                    repetition_penalty=repetition_penalty):
+        partial_message = partial_message + token 
         yield partial_message
+        
+
+
 
 gr.ChatInterface(
     inference,