Create app.py

app.py

@@ -0,0 +1,93 @@
+import gradio as gr
+import torch
+import time
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from threading import Thread
+print("Loading model and tokenizer...")
+model_name = "large-traversaal/Phi-4-Hindi"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    torch_dtype=torch.float16,
+    load_in_4bit=True,
+    device_map="auto"
+)
+print("Model and tokenizer loaded successfully!")
+def generate_response(message, temperature, max_new_tokens, top_p):
+    print(f"Input: {message}")
+    start_time = time.time()
+    inputs = tokenizer(message, return_tensors="pt").to(model.device)
+    streamer = TextIteratorStreamer(tokenizer, skip_special_tokens=True)
+    gen_kwargs = {
+        "input_ids": inputs["input_ids"],
+        "streamer": streamer,
+        "temperature": temperature,
+        "max_new_tokens": max_new_tokens,
+        "top_p": top_p,
+        "do_sample": True if temperature > 0 else False,
+    }
+    thread = Thread(target=model.generate, kwargs=gen_kwargs)
+    thread.start()
+    result = []
+    for text in streamer:
+        result.append(text)
+        yield "".join(result)
+    end_time = time.time()
+    time_taken = end_time - start_time
+    output_text = "".join(result)
+    print(f"Output: {output_text}")
+    print(f"Time taken: {time_taken:.2f} seconds")
+with gr.Blocks() as demo:
+    gr.Markdown("# Phi-4-Hindi Demo")
+    with gr.Row():
+        with gr.Column():
+            input_text = gr.Textbox(
+                label="Input",
+                placeholder="Enter your text here...",
+                lines=5
+            )
+            with gr.Row():
+                with gr.Column():
+                    temperature = gr.Slider(
+                        minimum=0.0,
+                        maximum=1.0,
+                        value=0.1,
+                        step=0.01,
+                        label="Temperature"
+                    )
+                with gr.Column():
+                    max_new_tokens = gr.Slider(
+                        minimum=50,
+                        maximum=1000,
+                        value=400,
+                        step=10,
+                        label="Max New Tokens"
+                    )
+                with gr.Column():
+                    top_p = gr.Slider(
+                        minimum=0.0,
+                        maximum=1.0,
+                        value=0.1,
+                        step=0.01,
+                        label="Top P"
+                    )
+            with gr.Row():
+                clear_btn = gr.Button("Clear")
+                send_btn = gr.Button("Send", variant="primary")
+        with gr.Column():
+            output_text = gr.Textbox(
+                label="Output",
+                lines=15
+            )
+    send_btn.click(
+        fn=generate_response,
+        inputs=[input_text, temperature, max_new_tokens, top_p],
+        outputs=output_text
+    )
+    clear_btn.click(
+        fn=lambda: ("", "", "", ""),
+        inputs=None,
+        outputs=[input_text, output_text]
+    )
+if __name__ == "__main__":
+    demo.queue().launch()