media handling brainstorming

app.py

@@ -98,8 +98,8 @@ def generate_story_and_timeline(include_media=False):
     # Include media-related items if specified
     if include_media:
         media_files = generate_media_file_list(5)
-        rendered_media = render_media_with_dropdowns(media_files)
-        media_timeline = generate_timeline(rendered_media, "Media")
+        #rendered_media = render_media_with_dropdowns(media_files)
+        media_timeline = generate_timeline(media_files, "Media")
         merged_timeline += media_timeline
 
     # Sort the merged timeline based on the random numbers
@@ -118,63 +118,94 @@ media_file_types = ["image", "video", "audio"]
 def generate_media_file_list(n):
     return [random.choice(media_file_types) for _ in range(n)]
 
-def render_media_with_dropdowns(media_files):
-    rendered_output = []
-    for index, media in enumerate(media_files):
-        iframe = f'<iframe src="{media}_source_{index}.html"></iframe>'
-        dropdown = f'''
-        <select>
-            <option value="{media}_source_1">Source 1</option>
-            <option value="{media}_source_2">Source 2</option>
-            <option value="{media}_source_3">Source 3</option>
-        </select>
-        '''
-        rendered_output.append(f"{iframe}{dropdown}")
-    return rendered_output
-
-def save_to_placeholder(rendered_media):
-    placeholder = []
-    for item in rendered_media:
-        placeholder.append(item)
-    return placeholder
+# def render_media_with_dropdowns(media_files):
+#     rendered_output = []
+#     for index, media in enumerate(media_files):
+#         iframe = f'<iframe src="{media}_source_{index}.html"></iframe>'
+#         dropdown = f'''
+#         <select>
+#             <option value="{media}_source_1">Source 1</option>
+#             <option value="{media}_source_2">Source 2</option>
+#             <option value="{media}_source_3">Source 3</option>
+#         </select>
+#         '''
+#         rendered_output.append(f"{iframe}{dropdown}")
+#     return rendered_output
+
+# def save_to_placeholder(rendered_media):
+#     placeholder = []
+#     for item in rendered_media:
+#         placeholder.append(item)
+#     return placeholder
 
 # Example usage with and without media-related items
 timeline_with_media, story_with_media = generate_story_and_timeline(include_media=True)
 timeline_without_media, story_without_media = generate_story_and_timeline(include_media=False)
 
-def split_content_into_blocks(timeline):
-    blocks = []
-    current_text_block = []
+# def split_content_into_blocks(timeline):
+#     blocks = []
+#     current_text_block = []
     
-    for entry in timeline.split('\n'):
-        if 'Media -' in entry:
-            # If we have accumulated text, add it as a block
-            if current_text_block:
-                blocks.append((' '.join(current_text_block), 'text'))
-                current_text_block = []
+#     for entry in timeline.split('\n'):
+#         if 'Media -' in entry:
+#             # If we have accumulated text, add it as a block
+#             if current_text_block:
+#                 blocks.append((' '.join(current_text_block), 'text'))
+#                 current_text_block = []
             
-            # Extract the iframe and dropdown
-            match = re.search(r'<iframe.*?</select>', entry, re.DOTALL)
-            if match:
-                blocks.append((match.group(), 'iframe'))
-        else:
-            current_text_block.append(entry)
+#             # Extract the iframe and dropdown
+#             match = re.search(r'<iframe.*?</select>', entry, re.DOTALL)
+#             if match:
+#                 blocks.append((match.group(), 'iframe'))
+#         else:
+#             current_text_block.append(entry)
+    
+#     # Add any remaining text as a final block
+#     if current_text_block:
+#         blocks.append((' '.join(current_text_block), 'text'))
+    
+#     return blocks
+
+# def show_elements(text):
+#     blocks = split_content_into_blocks(text)
+#     outputs = []
     
-    # Add any remaining text as a final block
-    if current_text_block:
-        blocks.append((' '.join(current_text_block), 'text'))
+#     for content, block_type in blocks:
+#         if block_type == 'text':
+#             outputs.append(gr.Markdown(content))
+#         elif block_type == 'iframe':
+#             outputs.append(gr.HTML(content))
     
-    return blocks
+#     return outputs
 
 def show_elements(text):
-    blocks = split_content_into_blocks(text)
+    # Parse the input text
+    pattern = r'(\d+): (UI|Story|Media) - (.+)'
+    blocks = re.findall(pattern, text)
+    
+    # Sort blocks by their timestamp
+    blocks.sort(key=lambda x: int(x[0]))
+    
     outputs = []
     
-    for content, block_type in blocks:
-        if block_type == 'text':
-            outputs.append(gr.Markdown(content))
-        elif block_type == 'iframe':
-            outputs.append(gr.HTML(content))
+    for timestamp, block_type, content in blocks:
+        if block_type == 'UI':
+            # Create HTML for UI elements
+            ui_html = f'<div class="ui-element">{content}</div>'
+            outputs.append(gr.HTML(ui_html))
+        elif block_type == 'Story':
+            # Display story elements as Markdown
+            outputs.append(gr.Markdown(f"**{content}**"))
+        elif block_type == 'Media':
+            if content.lower() == 'audio':
+                # Placeholder for audio element
+                outputs.append(gr.Audio(label=f"Audio at {timestamp} in the order"))
+            elif content.lower() == 'video':
+                # Placeholder for video element
+                outputs.append(gr.Video(label=f"Video at {timestamp} in the order"))
+            elif content.lower() == 'image':
+                # Placeholder for image element
+                outputs.append(gr.Image(label=f"Image at {timestamp} in the order"))
     
     return outputs
 
@@ -520,7 +551,7 @@ with gr.Blocks() as demo:
                         def update(text):
                             return show_elements(text)
                     with gr.Tab("Conversion of game version"):
-                        gr.HTML("Can copy in the next tab - only linear path for now")
+                        gr.HTML("Can copy in the Test Example State Machine tab - only linear path for now")
                         media_checkbox = gr.Checkbox(label="Include Media")
                         output_text = gr.Code(language="json")
                         
@@ -539,34 +570,64 @@ with gr.Blocks() as demo:
 
                     generate_button = gr.Button("Generate Story and Timeline")
                     generate_button.click(generate_story_and_timeline, inputs=[], outputs=[timeline_output, story_output])
-            with gr.Tab("Test Example State Machine"): 
-                with gr.Row():
-                    with gr.Column(scale=2):
-                        gr.Markdown("# Text-based Adventure Game")
-
-                        description = gr.Textbox(label="Current Situation", lines=4, value=initgameinfo[0])
-                        media = gr.HTML("Placeholder to load all media tests")
-                        choices = gr.Radio(label="Your Choices", choices=initgameinfo[1])
-                        submit_btn = gr.Button("Make Choice")
-                        game_log = gr.Textbox(label="Game Log", lines=20, value=initgameinfo[2])
-                        game_session = gr.State(value=initgameinfo[3])
-                        submit_btn.click(
-                            make_choice,
-                            inputs=[choices, game_session],
-                            outputs=[description, choices, game_log, game_session]
+    with gr.Tab("Test Example State Machine"): 
+        with gr.Tab("Config Without Assets"):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    gr.Markdown("# Text-based Adventure Game")
+
+                    description = gr.Textbox(label="Current Situation", lines=4, value=initgameinfo[0])
+                    media = gr.HTML("Placeholder to load all media tests")
+                    choices = gr.Radio(label="Your Choices", choices=initgameinfo[1])
+                    submit_btn = gr.Button("Make Choice")
+                    game_log = gr.Textbox(label="Game Log", lines=20, value=initgameinfo[2])
+                    game_session = gr.State(value=initgameinfo[3])
+                    submit_btn.click(
+                        make_choice,
+                        inputs=[choices, game_session],
+                        outputs=[description, choices, game_log, game_session]
+                    )
+                with gr.Column(scale=1):
+                    gr.Markdown("# Debugging")
+                    error_box = gr.Textbox(label="Path Errors", lines=4, value=path_errors)
+                    with gr.Accordion("Config (Game Spoiler and Example for llm to remix)", open=False):
+                        custom_config = gr.Textbox(label="Custom Configuration (JSON)", value=json.dumps(all_states, default=lambda o: o.__dict__, indent=2), lines=8)
+                        custom_configbtn = gr.Button("Load Custom Config")
+
+                        custom_configbtn.click(
+                            load_game,
+                            inputs=[custom_config],
+                            outputs=[error_box, game_log, description, choices, game_session, custom_config]
+                        )
+        with gr.Tab("Config With Assets"):
+            gr.HTML("Placeholder as not complete yet (still only text)")
+            with gr.Row():
+                with gr.Column(scale=2):
+                    gr.Markdown("# Text-based Adventure Game")
+
+                    wadescription = gr.Textbox(label="Current Situation", lines=4, value=initgameinfo[0])
+                    wamedia = gr.HTML("Placeholder to load all media tests")
+                    wachoices = gr.Radio(label="Your Choices", choices=initgameinfo[1])
+                    wasubmit_btn = gr.Button("Make Choice")
+                    wagame_log = gr.Textbox(label="Game Log", lines=20, value=initgameinfo[2])
+                    wagame_session = gr.State(value=initgameinfo[3])
+                    wasubmit_btn.click(
+                        make_choice,
+                        inputs=[wachoices, wagame_session],
+                        outputs=[wadescription, wachoices, wagame_log, wagame_session]
+                    )
+                with gr.Column(scale=1):
+                    gr.Markdown("# Debugging")
+                    waerror_box = gr.Textbox(label="Path Errors", lines=4, value=path_errors)
+                    with gr.Accordion("Config (Game Spoiler and Example for llm to remix)", open=False):
+                        wacustom_config = gr.Textbox(label="Custom Configuration (JSON)", value=json.dumps(all_states, default=lambda o: o.__dict__, indent=2), lines=8)
+                        wacustom_configbtn = gr.Button("Load Custom Config")
+
+                        wacustom_configbtn.click(
+                            load_game,
+                            inputs=[wacustom_config],
+                            outputs=[waerror_box, wagame_log, wadescription, wachoices, wagame_session, wacustom_config]
                         )
-                    with gr.Column(scale=1):
-                        gr.Markdown("# Debugging")
-                        error_box = gr.Textbox(label="Path Errors", lines=4, value=path_errors)
-                        with gr.Accordion("Config (Game Spoiler)", open=False):
-                            custom_config = gr.Textbox(label="Custom Configuration (JSON)", value=json.dumps(all_states, default=lambda o: o.__dict__, indent=2), lines=8)
-                            custom_configbtn = gr.Button("Load Custom Config")
-
-                            custom_configbtn.click(
-                                load_game,
-                                inputs=[custom_config],
-                                outputs=[error_box, game_log, description, choices, game_session, custom_config]
-                            )
     with gr.Tab("Custom JS Config Creator"):
         gr.HTML("Companion Space for zerogpu / client api workflow planning for a way to send a zip to the Basic Game Engine at the bottom of https://huggingface.co/spaces/KwabsHug/TestSvelteStatic (Also to test how much can be done majority on cpu)")
         with gr.Tab("Simple Config Creator"):
@@ -688,9 +749,10 @@ with gr.Blocks() as demo:
             gr.HTML("Placeholder for models small enough to run on cpu here in this space that can assist")
         
         with gr.Tab("Shaders and related"):
+            gr.HTML("Any output that is not understood by the common person can be used as special effects eg. depth map filters on images etc.")
             gr.HTML("Post-processing Effects, material effects, Particle systems, visual feedback")
             gr.HTML("Visual Effects - eg. explosion can turn all items white for a moment, losing conciousness blurs whole screen")
-            gr.HTML("Placeholder for huggingface spaces that can assist")
+            gr.HTML("Placeholder for huggingface spaces that can assist - https://huggingface.co/spaces/EPFL-VILAB/4M ")
             gr.HTML("Placeholder for models small enough to run on cpu here in this space that can assist")
         
     with gr.Tab("Other Considerations"):
@@ -714,6 +776,27 @@ with gr.Blocks() as demo:
             gr.Image(value="testmedia/Flash scribble SDXL - random squiggles as roads.webp")
             gr.Video(value="testmedia/SVD - random squiggles as roads video 004484.mp4")
             gr.Audio(value="testmedia/Stable Audio - Raindrops, output.wav")
-        gr.HTML("Placeholder to test use as all media loader for ease of prototyping")
+        gr.HTML("""
+    <div style="display: flex; justify-content: space-between; margin-bottom: 20px;">
+        <div style="width: 30%;">
+            <img src="testmedia/Flash scribble SDXL - random squiggles as roads.webp" alt="Random squiggles as roads" style="width: 100%; height: auto;">
+        </div>
+        <div style="width: 30%;">
+            <video width="100%" height="auto" controls>
+                <source src="testmedia/SVD - random squiggles as roads video 004484.mp4" type="video/mp4">
+                Your browser does not support the video tag.
+            </video>
+        </div>
+        <div style="width: 30%;">
+            <audio controls style="width: 100%;">
+                <source src="testmedia/Stable Audio - Raindrops, output.wav" type="audio/wav">
+                Your browser does not support the audio element.
+            </audio>
+        </div>
+    </div>
+    <div>
+        <p>This is a placeholder to test use as an all-media loader for ease of prototyping.</p>
+    </div>
+    """)
 
 demo.launch()