feat: longer max tokens

app.py

@@ -12,17 +12,14 @@ def main(
     device="cpu",
     port=8070,
 ):
-    tokenizer = AutoTokenizer.from_pretrained(base_model)
     pipeline = transformers.pipeline(
-        "text-generation",
-        model=base_model,
-        torch_dtype=torch.float16,
-        device=device
+        "text-generation", model=base_model, torch_dtype=torch.bfloat16, device=device
     )
+
     def evaluate_magicoder(
         instruction,
         temperature=1,
-        max_new_tokens=2048,
+        max_length=8192,
     ):
         MAGICODER_PROMPT = """You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.
 
@@ -30,7 +27,7 @@ def main(
 {instruction}
 
 @@ Response
-""" 
+"""
         prompt = MAGICODER_PROMPT.format(instruction=instruction)
 
         if temperature > 0:
@@ -38,26 +35,28 @@ def main(
                 prompt,
                 do_sample=True,
                 temperature=temperature,
-                max_new_tokens=max_new_tokens,
+                max_length=max_length,
             )
         else:
             sequences = pipeline(
                 prompt,
-                max_new_tokens=max_new_tokens,
+                max_length=max_length,
             )
         for seq in sequences:
-            generated_text = seq['generated_text'].replace(prompt, "")
+            generated_text = seq["generated_text"].replace(prompt, "")
             return generated_text
 
     gr.Interface(
         fn=evaluate_magicoder,
         inputs=[
             gr.components.Textbox(
-                lines=3, label="Instruction", placeholder="Anything you want to ask Magicoder ?"
+                lines=3,
+                label="Instruction",
+                placeholder="Anything you want to ask Magicoder ?",
             ),
             gr.components.Slider(minimum=0, maximum=1, value=0, label="Temperature"),
             gr.components.Slider(
-                minimum=1, maximum=2048, step=1, value=1024, label="Max tokens"
+                minimum=1, maximum=8192, step=1, value=1024, label="Max tokens"
             ),
         ],
         outputs=[
@@ -67,8 +66,9 @@ def main(
             )
         ],
         title="Magicoder",
-        description="This is a LLM playground for Magicoder! Follow us on Github: https://github.com/ise-uiuc/magicoder and Huggingface: https://huggingface.co/ise-uiuc."
+        description="This is a playground for Magicoder-S-DS-6.7B! Follow us on Github: https://github.com/ise-uiuc/magicoder and Huggingface: https://huggingface.co/ise-uiuc.",
     ).queue().launch(share=True, server_port=port)
 
+
 if __name__ == "__main__":
     fire.Fire(main)