demo buildout 13

app.py

@@ -50,13 +50,12 @@ def infer(
         max_new_tokens: int
 ) -> str:
     inputs = processor(text=text, images=resize_and_pad(image, 448), return_tensors="pt", padding="longest", do_convert_rgb=True).to(device).to(dtype=model.dtype)
-    with torch.inference_mode():
-        generated_ids = model.generate(
-            **inputs,
-            max_length=2048
-        )
-    result = processor.batch_decode(generated_ids, skip_special_tokens=True)
-    return result[0][len(text):].lstrip("\n")
+    generated_ids = model.generate(
+        **inputs,
+        max_length=2048
+    )
+    result = processor.decode(generated_ids[0], skip_special_tokens=True)
+    return result
 
 ######## Demo
 
@@ -81,14 +80,6 @@ with gr.Blocks(css="style.css") as demo:
 
             text_output = gr.Text(label="Text Output")
             chat_btn = gr.Button()
-            tokens = gr.Slider(
-                label="Max New Tokens",
-                info="Set to larger for longer generation.",
-                minimum=10,
-                maximum=100,
-                value=20,
-                step=10,
-            )
 
         chat_inputs = [
             image,