Create app.py

app.py

@@ -0,0 +1,18 @@
+import torch
+from transformers import AutoTokenizer
+from petals import AutoDistributedModelForCausalLM
+
+model_name = "petals-team/StableBeluga2"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, add_bos_token=False)
+model = AutoDistributedModelForCausalLM.from_pretrained(model_name)
+
+import gradio as gr
+
+def generate(input, params):
+    tokenized = tokenizer(input, return_tensors="pt")["input_ids"]
+    outputs = model.generate(inputs, max_new_tokens=80, do_sample=True, temperature=0.9)
+    return tokenizer.decode(outputs[0]).replace("</s>", "");
+
+iface = gr.Interface(fn=generate, inputs="text", outputs="text")
+iface.launch()