Upload 2 files

app.py

@@ -0,0 +1,210 @@
+import os
+import io
+import json
+import torch
+import requests
+from PIL import Image
+import soundfile as sf
+import gradio as gr
+from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
+
+# -------------------------------
+# 模型與處理器載入設定
+# -------------------------------
+model_path = "microsoft/Phi-4-multimodal-instruct"
+
+processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    model_path,
+    device_map="auto",
+    torch_dtype="auto",
+    trust_remote_code=True,
+    _attn_implementation="eager",
+)
+
+generation_config = GenerationConfig.from_pretrained(model_path)
+
+# -------------------------------
+# 根據任務模式組合 prompt 並調用模型生成結果
+# -------------------------------
+def process_task(mode, system_msg, user_msg, image_multi, audio, vs_images, vs_audio):
+    """
+    根據不同任務模式組合 prompt，並使用 processor 與 model 進行生成
+    """
+    # -------------------------------
+    # 依據不同模式構建 prompt 與處理輸入資料
+    # -------------------------------
+    if mode == "Text Chat":
+        prompt = f"<|system|>{system_msg}<|end|><|user|>{user_msg}<|end|><|assistant|>"
+        inputs = processor(text=prompt, return_tensors='pt').to(model.device)
+        
+    elif mode == "Tool-enabled Function Calling":
+        tools = [{
+            "name": "get_weather_updates",
+            "description": "Fetches weather updates for a given city using the RapidAPI Weather API.",
+            "parameters": {
+                "city": {
+                    "description": "The name of the city for which to retrieve weather information.",
+                    "type": "str",
+                    "default": "London"
+                }
+            }
+        }]
+        tools_json = json.dumps(tools, ensure_ascii=False)
+        prompt = f"<|system|>{system_msg}<|tool|>{tools_json}<|/tool|><|end|><|user|>{user_msg}<|end|><|assistant|>"
+        inputs = processor(text=prompt, return_tensors='pt').to(model.device)
+        
+    elif mode == "Vision-Language":
+        # 優先判斷單一圖片上傳；若無則檢查多圖上傳
+        if image_multi is not None and len(image_multi) > 0:
+            num = len(image_multi)
+            image_tags = ''.join([f"<|image_{i+1}|>" for i in range(num)])
+            prompt = f"<|user|>{image_tags}{user_msg}<|end|><|assistant|>"
+            images = []
+            for file in image_multi:
+                images.append(Image.open(file))
+            inputs = processor(text=prompt, images=images, return_tensors='pt').to(model.device)
+        else:
+            return "No image provided."
+        
+    elif mode == "Speech-Language":
+        prompt = f"<|user|><|audio_1|>{user_msg}<|end|><|assistant|>"
+        if audio is None:
+            return "No audio provided."
+        # 若 audio 為 tuple，則直接取出取樣率與音訊資料
+        if isinstance(audio, tuple):
+            sample_rate, audio_data = audio
+        else:
+            audio_data, sample_rate = sf.read(audio)
+        inputs = processor(text=prompt, audios=[(audio_data, sample_rate)], return_tensors='pt').to(model.device)
+        
+    elif mode == "Vision-Speech":
+        prompt = f"<|user|>"
+        images = []
+        if vs_images is not None and len(vs_images) > 0:
+            num = len(vs_images)
+            image_tags = ''.join([f"<|image_{i+1}|>" for i in range(num)])
+            prompt += image_tags
+            for file in vs_images:
+                images.append(Image.open(file))
+        if vs_audio is None:
+            return "No audio provided for vision-speech."
+        prompt += "<|audio_1|><|end|><|assistant|>"
+        audio_data, samplerate = sf.read(vs_audio)
+        inputs = processor(text=prompt, images=images, audios=[(audio_data, samplerate)], return_tensors='pt').to(model.device)
+        
+    else:
+        return "Invalid mode."
+
+    # -------------------------------
+    # 調用模型生成回應
+    # -------------------------------
+    generate_ids = model.generate(
+        **inputs,
+        max_new_tokens=1000,
+        generation_config=generation_config,
+    )
+    # 裁剪掉輸入部分的 token
+    generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]
+    response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+    return response
+
+# -------------------------------
+# 更新介面元件顯示 (根據任務模式決定顯示哪些輸入區塊)
+# -------------------------------
+def update_visibility(mode):
+    if mode == "Text Chat":
+        return (gr.update(visible=True),    # system_msg
+                gr.update(visible=True),    # user_msg
+                gr.update(visible=False),   # image_upload_multi (多圖)
+                gr.update(visible=False),   # audio_upload
+                gr.update(visible=False),   # vs_image_upload
+                gr.update(visible=False))   # vs_audio_upload
+    elif mode == "Tool-enabled Function Calling":
+        return (gr.update(visible=True),
+                gr.update(visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False))
+    elif mode == "Vision-Language":
+        return (gr.update(visible=False),
+                gr.update(visible=True),
+                gr.update(visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False))
+    elif mode == "Speech-Language":
+        return (gr.update(visible=False),
+                gr.update(visible=True),
+                gr.update(visible=False),
+                gr.update(visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False))
+    elif mode == "Vision-Speech":
+        return (gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=True),
+                gr.update(visible=True))
+    else:
+        return (gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update())
+
+# -------------------------------
+# 建立 Gradio Blocks 介面
+# -------------------------------
+with gr.Blocks() as demo:
+    gr.Markdown("## Multi-Modal Prompt Builder & Model Inference")
+    # 任務模式選單
+    mode_radio = gr.Radio(
+        choices=["Text Chat", "Vision-Language", "Speech-Language", "Vision-Speech"], #, "Tool-enabled Function Calling"
+        label="Select Task Mode",
+        value="Text Chat"
+    )
+    
+    # 文字輸入區塊 (Text Chat 與 Tool-enabled 都需要)
+    system_text = gr.Textbox(label="System Message", value="You are a helpful assistant.", visible=True)
+    user_text = gr.Textbox(label="User Message", visible=True)
+    
+    # 圖片上傳區塊 (Vision-Language)
+    # image_upload = gr.Image(label="Upload Image (Single)", type="pil", visible=False)
+    image_upload_multi = gr.File(label="Upload Image(s) (Multiple)", file_count="multiple", visible=False)
+    
+    # 音檔上傳區塊 (Speech-Language)
+    audio_upload = gr.Audio(label="Upload Audio (wav, mp3, flac)", visible=False)
+    
+    # Vision-Speech 區塊：圖片上傳 (多張) 與音檔上傳
+    vs_image_upload = gr.File(label="Upload Image(s) for Vision-Speech", file_count="multiple", visible=False)
+    vs_audio_upload = gr.Audio(label="Upload Audio for Vision-Speech", visible=False)
+    
+    # 送出按鈕與結果輸出區塊
+    submit_btn = gr.Button("Submit")
+    output_text = gr.Textbox(label="Result", lines=6)
+    
+    # gr.Examples 區塊，提供部份任務的文字範例（其他任務請自行上傳圖片或音檔）
+    examples = gr.Examples(
+        examples=[
+            ["Text Chat", "hi who are you?"],
+            # ["Tool-enabled Function Calling", "You are a helpful assistant with some tools.", "What is the weather like in Paris today?"],
+            ["Vision-Language", "Describe the image in detail."],
+            ["Speech-Language", "Transcribe the audio to text."],
+            ["Vision-Speech", ""]
+        ],
+        inputs=[mode_radio, user_text],
+        label="Examples"
+    )
+    
+    # 當任務模式改變時，更新介面各元件顯示狀態
+    mode_radio.change(fn=update_visibility, 
+                      inputs=mode_radio, 
+                      outputs=[system_text, user_text, image_upload_multi, audio_upload, vs_image_upload, vs_audio_upload])
+    
+    # 點擊送出按鈕時根據選擇的模式與輸入內容生成 prompt 並調用模型生成回答
+    submit_btn.click(
+        fn=process_task,
+        inputs=[mode_radio, system_text, user_text, image_upload_multi, audio_upload, vs_image_upload, vs_audio_upload],
+        outputs=output_text
+    )
+
+demo.launch()

requirements.txt

@@ -0,0 +1,11 @@
+transformers
+torchvision
+scipy
+peft
+backoff
+gradio
+spaces
+requests
+torch
+pillow
+soundfile