Update app.py

app.py

@@ -9,6 +9,7 @@ from threading import Thread
 import base64
 import shutil
 import re
+from io import BytesIO
 
 import gradio as gr
 import spaces
@@ -38,8 +39,8 @@ from diffusers.utils import export_to_ply
 # Additional import for Phi-4 multimodality (audio support)
 import soundfile as sf
 
-
 os.system('pip install backoff')
+
 # Global constants and helper functions
 
 MAX_SEED = np.iinfo(np.int32).max
@@ -59,6 +60,17 @@ def glb_to_data_url(glb_path: str) -> str:
     b64_data = base64.b64encode(data).decode("utf-8")
     return f"data:model/gltf-binary;base64,{b64_data}"
 
+def load_audio_file(file):
+    """
+    Loads an audio file. If file is a string path, it reads directly.
+    Otherwise, it assumes file is a file-like object.
+    """
+    if isinstance(file, str):
+        audio, samplerate = sf.read(file)
+    else:
+        audio, samplerate = sf.read(BytesIO(file.read()))
+    return audio, samplerate
+
 # Model class for Text-to-3D Generation (ShapE)
 
 class Model:
@@ -458,11 +470,13 @@ def process_phi4(input_type: str, file, question: str, max_new_tokens: int = 200
 
     if input_type.lower() == "image":
         prompt = f'{user_prompt}<|image_1|>{question}{prompt_suffix}{assistant_prompt}'
-        image = Image.open(file)
+        # Use load_image (as in Qwen2-VL-OCR-2B-Instruct) to handle image file input
+        image = load_image(file)
         inputs = phi4_processor(text=prompt, images=image, return_tensors='pt').to(phi4_model.device)
     elif input_type.lower() == "audio":
         prompt = f'{user_prompt}<|audio_1|>{question}{prompt_suffix}{assistant_prompt}'
-        audio, samplerate = sf.read(file)
+        # Use load_audio_file to handle audio file input
+        audio, samplerate = load_audio_file(file)
         inputs = phi4_processor(text=prompt, audios=[(audio, samplerate)], return_tensors='pt').to(phi4_model.device)
     else:
         yield "Invalid input type selected."
@@ -719,7 +733,7 @@ demo = gr.ChatInterface(
     description=DESCRIPTION,
     css=css,
     fill_height=True,
-    textbox=gr.MultimodalTextbox(label="Query Input", file_types=["image", "audio"], file_count="multiple", placeholder="@tts1, @tts2, @image, @3d, @ragent, @web, @yolo, @phi4, or plain text"),
+    textbox=gr.MultimodalTextbox(label="Query Input", file_types=["image", "audio"], file_count="multiple", placeholder="@tts1, @tts2, @image, @3d, @ragent, @web, @yolo, @phi4 - audio, image, or plain text"),
     stop_btn="Stop Generation",
     multimodal=True,
 )