Update app.py

app.py

@@ -46,29 +46,35 @@ def encode_video(video_path):
     return [Image.fromarray(f.astype("uint8")) for f in frames]
 
 @spaces.GPU(duration=60)
-def generate(image, video, audio, instruction=DEFAULT_INPUT, sampling=False, temperature=0.7, top_p=0.8, top_k=100, repetition_penalty=1.05, max_tokens=512):
+def generate(input=[], instruction=DEFAULT_INPUT, sampling=False, temperature=0.7, top_p=0.8, top_k=100, repetition_penalty=1.05, max_tokens=512):
 
-    print(image)
-    print(video)
-    print(audio)
+    print(input)
     print(instruction)
 
-    if image is not None:
-        img = Image.fromarray(image.astype("uint8"), "RGB")
-        inputs = [{"role": "user", "content": [img, instruction]}]
-    elif video is not None:
-        vid = encode_video(video)
-        inputs = [{"role": "user", "content": vid + [instruction]}]
-    elif audio is not None:
-        aud, _ = librosa.load(audio, sr=16000, mono=True)
-        inputs = [{"role": "user", "content": [aud, instruction]}]
-    else:
+    content = []
+    if not files:
         return "No input provided."
 
+    for file_path in files:
+        ext = os.path.splitext(file_path)[1].lower()
+        if ext in [".jpg", ".jpeg", ".png", ".bmp", ".gif"]:
+            img = Image.open(file_path).convert("RGB")
+            content.append(img)
+        elif ext in [".mp4", ".mov", ".avi", ".mkv"]:
+            frames = encode_video(file_path)
+            content.extend(frames)
+        elif ext in [".wav", ".mp3", ".flac", ".aac"]:
+            aud, _ = librosa.load(file_path, sr=16000, mono=True)
+            content.append(aud)
+        else:
+            continue
+
+    content.append(instruction)
+    inputs_payload = [{"role": "user", "content": content}]
+
     params = {
-        "msgs": inputs,
+        "msgs": inputs_payload,
         "tokenizer": tokenizer,
-        #"processor": processor,
         "sampling": sampling,
         "temperature": temperature,
         "top_p": top_p,
@@ -78,7 +84,7 @@ def generate(image, video, audio, instruction=DEFAULT_INPUT, sampling=False, tem
     }
 
     output = repo.chat(**params)
-
+    
     print(output)
 
     return output
@@ -89,9 +95,7 @@ def cloud():
 # Initialize
 with gr.Blocks(css=css) as main:
     with gr.Column():
-        input = gr.Image(label="Image")
-        input_2 = gr.Video(label="Video")
-        input_3 = gr.Audio(label="Audio", type="filepath")
+        input = gr.File(label="Input", file_count="multiple", file_types=["image", "video", "audio"], type="filepath", allow_reordering=True)
         instruction = gr.Textbox(lines=1, value=DEFAULT_INPUT, label="Instruction")
         sampling = gr.Checkbox(value=False, label="Sampling")
         temperature = gr.Slider(minimum=0.01, maximum=1.99, step=0.01, value=0.7, label="Temperature")
@@ -101,11 +105,11 @@ with gr.Blocks(css=css) as main:
         max_tokens = gr.Slider(minimum=1, maximum=4096, step=1, value=512, label="Max Tokens")
         submit = gr.Button("▶")
         maintain = gr.Button("☁️")
-        
+
     with gr.Column():
         output = gr.Textbox(lines=1, value="", label="Output")
 
-    submit.click(fn=generate, inputs=[input, input_2, input_3, instruction, sampling, temperature, top_p, top_k, repetition_penalty, max_tokens], outputs=[output], queue=False)
+    submit.click(fn=generate, inputs=[input, instruction, sampling, temperature, top_p, top_k, repetition_penalty, max_tokens], outputs=[output], queue=False)
     maintain.click(cloud, inputs=[], outputs=[], queue=False)
 
 main.launch(show_api=True)