with ref image

App/Generate/Schema.py

@@ -3,11 +3,23 @@ from pydantic import BaseModel, HttpUrl
 from pydantic import validator
 
 
+class Scene(BaseModel):
+    narration: str
+    image_prompts: List[str]
+    voice: str
+
+
+class Story(BaseModel):
+    scenes: List[Scene]
+    reference_image_url: Optional[str] = None
+    ip_adapter_weight: Optional[float] = 0.4  # Add this line
+
+
 class GeneratorRequest(BaseModel):
     prompt: str
-    grok: Optional[bool] = True
     batch_size: int = 4
-    model: str = "llama3-70b-8192"
+    grok: bool = False
+    model: str = "command"
 
 
 class GeneratorBulkRequest(BaseModel):

App/Generate/database/Model.py

@@ -227,9 +227,13 @@ class Scene(orm.Model):
         transcript = await self.tts._make_transcript(links=links, text=text)
         return transform_alignment_data(data=transcript, offset=offset)
 
-    async def generate_scene_data(self):
+    async def generate_scene_data(
+        self, reference_image_url: str = None, ip_adapter_weight: float = 0.4
+    ):
         # Run narrate() and generate_images() concurrently
-        await asyncio.gather(self.narrate(), self.generate_images())
+        await asyncio.gather(
+            self.narrate(), self.generate_images(reference_image_url, ip_adapter_weight)
+        )
         self.calculate_durations()
 
     async def narrate(self):
@@ -257,16 +261,18 @@ class Scene(orm.Model):
         self.narration_duration = int(len(audio_file) / 1000)
         self.image_duration = self.narration_duration / len(self.image_prompts)
 
-    async def generate_images(self):
+    async def generate_images(
+        self,
+        reference_image_url: str = "https://image.lexica.art/full_webp/d6ddd5c5-060c-4aba-b9d0-cf0e02dc65bd",
+        ip_adapter_weight: float = 0.4,
+    ):
         self.images = []
         async with aiohttp.ClientSession() as session:
             image_generator = ModalImageGenerator(session)
             for prompt in self.image_prompts:
-                # You may want to define a method to get a suitable reference image URL
-                reference_image_url = "https://image.lexica.art/full_webp/11caada0-c3c3-4b70-b7a5-d3172c07fc14"
                 try:
-                    image_url = await image_generator.generate_and_wait(
-                        prompt, reference_image_url
+                    image_url = await image_generator.generate_image(
+                        prompt, reference_image_url, ip_adapter_weight
                     )
                     self.images.append(image_url)
                 except Exception as e:
@@ -309,7 +315,7 @@ class BackgroundMusic(orm.Model):
 #         pass
 
 
-# # # Create the tables
+# # Create the tables
 # async def create_tables():
 #     datas = {
 #         "narration": "Welcome to a journey through some of history's strangest moments! Get ready to explore the bizarre, the unusual, and the downright weird.",

App/Generate/generatorRoutes.py

@@ -20,8 +20,8 @@ from App.Editor.editorRoutes import celery_task, EditorRequest
 import uuid
 
 
-async def update_scene(model_scene):
-    await model_scene.generate_scene_data()
+async def update_scene(model_scene, reference_image_url=None, ip_adapter_weight=0.4):
+    await model_scene.generate_scene_data(reference_image_url, ip_adapter_weight)
     await model_scene.update(**model_scene.__dict__)
 
 
@@ -53,7 +53,11 @@ async def generate_assets(generated_story: Story, batch_size=4, threeD=True):
                 await model_scene.update(**model_scene.__dict__)
                 all_scenes.append(model_scene)
                 batch_updates.append(
-                    update_scene(model_scene)
+                    update_scene(
+                        model_scene,
+                        generated_story.reference_image_url,
+                        generated_story.ip_adapter_weight,
+                    )
                 )  # Append update coroutine to batch_updates
             # pause per batch
             await asyncio.gather(
@@ -127,7 +131,7 @@ async def generate_video(
 
 @generator_router.post("/generate_video_from_json")
 async def generate_video_from_json(jsonReq: Story, background_task: BackgroundTasks):
-    background_task.add_task(from_dict_generate, jsonReq)
+    background_task.add_task(generate_assets, jsonReq)
     return {"task_id": "started"}