Add streaming again

app.py

@@ -30,8 +30,7 @@ decode_kwargs = dict(skip_special_tokens=True)
 streamer = TextIteratorStreamer(tokenizer, decode_kwargs=decode_kwargs)
 
 cplusplus = None
-def translate(python, progress=gr.Progress()):
-    progress(0, desc="Starting")
+def translate(python):
     formatted_prompt = tokenizer.apply_chat_template(
                         messages_for(python),
                         tokenize=False,
@@ -41,7 +40,7 @@ def translate(python, progress=gr.Progress()):
     attention_mask = inputs.attention_mask
     input_ids = inputs.input_ids
 
-    outputs = model.generate(
+    generation_kwargs = dict(
         input_ids,
         attention_mask=attention_mask,
         max_new_tokens=1024,
@@ -49,24 +48,13 @@ def translate(python, progress=gr.Progress()):
         pad_token_id=tokenizer.eos_token_id,
         eos_token_id=tokenizer.eos_token_id,
     )
-    progress(1, desc="Finished")
-    return tokenizer.decode(outputs[0][len(input_ids[0]):], skip_special_tokens=True)
-    '''
-    generation_kwargs = dict(
-                            inputs,
-                            streamer=streamer,
-                            max_new_tokens=1024,
-                            do_sample=False,
-                            pad_token_id=tokenizer.eos_token_id,
-                            eos_token_id=tokenizer.eos_token_id
-                        )
+    
     thread = Thread(target=model.generate, kwargs=generation_kwargs)
     thread.start()
     cplusplus = ""
     for chunk in streamer:
         cplusplus += chunk
         yield cplusplus
-    '''
 
 demo = gr.Interface(fn=translate, inputs="code", outputs="markdown")
 demo.launch()