deformes4D_engine.py

# deformes4D_engine.py
# Copyright (C) 4 de Agosto de 2025  Carlos Rodrigues dos Santos
#
# MODIFICATIONS FOR ADUC-SDR:
# Copyright (C) 2025 Carlos Rodrigues dos Santos. All rights reserved.
#
# This file is part of the ADUC-SDR project. It contains the core logic for
# video fragment generation, latent manipulation, and dynamic editing, 
# governed by the ADUC orchestrator.
# This component is licensed under the GNU Affero General Public License v3.0.

import os
import time
import imageio
import numpy as np
import torch
import logging
from PIL import Image, ImageOps
from dataclasses import dataclass
import gradio as gr
import subprocess
import random
import gc

from ltx_manager_helpers import ltx_manager_singleton
from gemini_helpers import gemini_singleton 
from ltx_video.models.autoencoders.vae_encode import vae_encode, vae_decode

logger = logging.getLogger(__name__)

@dataclass
class LatentConditioningItem:
    latent_tensor: torch.Tensor
    media_frame_number: int
    conditioning_strength: float

class Deformes4DEngine:
    def __init__(self, ltx_manager, workspace_dir="deformes_workspace"):
        self.ltx_manager = ltx_manager
        self.workspace_dir = workspace_dir
        self._vae = None
        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
        logger.info("Especialista Deformes4D (SDR Executor) inicializado.")

    @property
    def vae(self):
        if self._vae is None:
            self._vae = self.ltx_manager.workers[0].pipeline.vae
        self._vae.to(self.device); self._vae.eval()
        return self._vae

    def save_latent_tensor(self, tensor: torch.Tensor, path: str):
        torch.save(tensor.cpu(), path)
        logger.info(f"Tensor latente salvo em: {path}")

    def load_latent_tensor(self, path: str) -> torch.Tensor:
        tensor = torch.load(path, map_location=self.device)
        logger.info(f"Tensor latente carregado de: {path} para o dispositivo {self.device}")
        return tensor

    @torch.no_grad()
    def pixels_to_latents(self, tensor: torch.Tensor) -> torch.Tensor:
        tensor = tensor.to(self.device, dtype=self.vae.dtype)
        return vae_encode(tensor, self.vae, vae_per_channel_normalize=True)

    @torch.no_grad()
    def latents_to_pixels(self, latent_tensor: torch.Tensor, decode_timestep: float = 0.05) -> torch.Tensor:
        latent_tensor = latent_tensor.to(self.device, dtype=self.vae.dtype)
        timestep_tensor = torch.tensor([decode_timestep] * latent_tensor.shape[0], device=self.device, dtype=latent_tensor.dtype)
        return vae_decode(latent_tensor, self.vae, is_video=True, timestep=timestep_tensor, vae_per_channel_normalize=True)

    def save_video_from_tensor(self, video_tensor: torch.Tensor, path: str, fps: int = 24):
        if video_tensor is None or video_tensor.ndim != 5 or video_tensor.shape[2] == 0:
            logger.warning("Tentativa de salvar um tensor de vídeo inválido. Abortando.")
            return
        video_tensor = video_tensor.squeeze(0).permute(1, 2, 3, 0)
        video_tensor = (video_tensor.clamp(-1, 1) + 1) / 2.0
        video_np = (video_tensor.detach().cpu().float().numpy() * 255).astype(np.uint8)
        with imageio.get_writer(path, fps=fps, codec='libx264', quality=8) as writer:
            for frame in video_np: writer.append_data(frame)
        logger.info(f"Vídeo salvo em: {path}")

    def _preprocess_image_for_latent_conversion(self, image: Image.Image, target_resolution: tuple) -> Image.Image:
        if image.size != target_resolution:
            logger.info(f"  - AÇÃO: Redimensionando imagem de {image.size} para {target_resolution} antes da conversão para latente.")
            return ImageOps.fit(image, target_resolution, Image.Resampling.LANCZOS)
        return image

    def pil_to_latent(self, pil_image: Image.Image) -> torch.Tensor:
        image_np = np.array(pil_image).astype(np.float32) / 255.0
        tensor = torch.from_numpy(image_np).permute(2, 0, 1).unsqueeze(0).unsqueeze(2)
        tensor = (tensor * 2.0) - 1.0
        return self.pixels_to_latents(tensor)
        
    def _generate_video_from_latents(self, latent_tensor, base_name):
        silent_video_path = os.path.join(self.workspace_dir, f"{base_name}_silent.mp4")
        pixel_tensor = self.latents_to_pixels(latent_tensor)
        self.save_video_from_tensor(pixel_tensor, silent_video_path, fps=24)
        del pixel_tensor; gc.collect()
        return silent_video_path

    def _generate_latent_tensor_internal(self, conditioning_items, ltx_params, target_resolution, total_frames_to_generate):
        final_ltx_params = {**ltx_params, 'width': target_resolution[0], 'height': target_resolution[1], 'video_total_frames': total_frames_to_generate, 'video_fps': 24, 'current_fragment_index': int(time.time()), 'conditioning_items_data': conditioning_items}
        new_full_latents, _ = self.ltx_manager.generate_latent_fragment(**final_ltx_params)
        return new_full_latents

    def concatenate_videos_ffmpeg(self, video_paths: list[str], output_path: str) -> str:
        if not video_paths: raise gr.Error("Nenhum fragmento de vídeo para montar.")
        list_file_path = os.path.join(self.workspace_dir, "concat_list.txt")
        with open(list_file_path, 'w', encoding='utf-8') as f:
            for path in video_paths: f.write(f"file '{os.path.abspath(path)}'\n")
        cmd_list = ['ffmpeg', '-y', '-f', 'concat', '-safe', '0', '-i', list_file_path, '-c', 'copy', output_path]
        logger.info("Executando concatenação FFmpeg...")
        try:
            subprocess.run(cmd_list, check=True, capture_output=True, text=True)
        except subprocess.CalledProcessError as e:
            logger.error(f"Erro no FFmpeg: {e.stderr}")
            raise gr.Error(f"Falha na montagem final do vídeo. Detalhes: {e.stderr}")
        return output_path
    
    def generate_full_movie(self, keyframes: list, global_prompt: str, storyboard: list, seconds_per_fragment: float, 
                            trim_chunks: int, echo_chunks: int,
                            handler_strength: float, destination_convergence_strength: float, video_resolution: int, 
                            use_continuity_director: bool, progress: gr.Progress = gr.Progress()):
        
        base_ltx_params = {"guidance_scale": 1.0, "stg_scale": 0.0, "rescaling_scale": 0.15, "num_inference_steps": 20}
        keyframe_paths = [item[0] if isinstance(item, tuple) else item for item in keyframes]
        video_clips_paths, story_history = [], ""
        target_resolution_tuple = (video_resolution, video_resolution) 
        
        total_frames_base = self._quantize_to_multiple(round(seconds_per_fragment * 24), 8)
        if total_frames_base == 0: total_frames_base = 8
        
        logger.info("="*50)
        logger.info("CÁLCULOS DE GERAÇÃO E GUIAS (BASEADO EM CHUNKS):")
        logger.info(f"  - Duração Base Solicitada: {total_frames_base} frames ({total_frames_base // 8} chunks)")
        logger.info(f"  - N_Corte para guias: {trim_chunks} chunks")
        logger.info(f"  - N_Eco (Dejavu) para guias: {echo_chunks} chunks")
        logger.info("="*50)

        dejavu_latent, evo_latent, last_eco_chunk = None, None, None
        
        if len(keyframe_paths) < 3:
            raise gr.Error(f"O modelo de geração requer no mínimo 3 keyframes (Passado, Presente, Futuro). Você forneceu {len(keyframe_paths)}.")
        
        num_transitions_to_generate = len(keyframe_paths) - 2
        logger.info(f"Modelo 'K-2' ativado: {len(keyframe_paths)} keyframes resultarão em {num_transitions_to_generate} fragmentos de vídeo.")
        
        for i in range(num_transitions_to_generate):
            start_keyframe_index = i + 1
            
            logger.info(f"--- INICIANDO FRAGMENTO {i+1}/{num_transitions_to_generate} (índice de loop i={i}) ---")
            progress((i + 1) / num_transitions_to_generate, desc=f"Produzindo Transição {i+1}/{num_transitions_to_generate}")
            
            past_keyframe_path = keyframe_paths[start_keyframe_index - 1]
            start_keyframe_path = keyframe_paths[start_keyframe_index]
            destination_keyframe_path = keyframe_paths[start_keyframe_index + 1]
            future_story_prompt = storyboard[start_keyframe_index + 1] if (start_keyframe_index + 1) < len(storyboard) else "A cena final."

            
            decision = gemini_singleton.get_cinematic_decision(
                global_prompt,
                story_history,
                past_keyframe_path,
                start_keyframe_path,
                destination_keyframe_path,
                storyboard[start_keyframe_index - 1], # Story para o Keyframe Passado
                storyboard[start_keyframe_index],    # Story para o Keyframe de Início (Presente)
                future_story_prompt
            )
            transition_type, motion_prompt = decision["transition_type"], decision["motion_prompt"]
            
            story_history += f"\n- Ato {i+1} ({transition_type}): {motion_prompt}"

            total_frames_to_generate = total_frames_base
            
            conditioning_items = []
            logger.info(f"  [0. PREPARAÇÃO] Montando itens de condicionamento para K{start_keyframe_index} -> K{start_keyframe_index+1}.")
                        
            
            if last_eco_chunk is None:
               # Nenhum eco → sempre trata como "primeiro fragmento"
               img_start = self._preprocess_image_for_latent_conversion(
               Image.open(start_keyframe_path).convert("RGB"),
                   target_resolution_tuple
               )
               conditioning_items.append(
                   LatentConditioningItem(self.pil_to_latent(img_start), 0, 1.0)
               )
            else:
               # Usa eco + handler do fragmeto anterior 
               conditioning_items.append(LatentConditioningItem(last_eco_chunk, 0, 1.0))
               handler_frame = (echo_chunks + trim_chunks) * 8
               conditioning_items.append(LatentConditioningItem(handler_Chunk, handler_frame, handler_strength))
              
            
            img_dest = self._preprocess_image_for_latent_conversion(Image.open(destination_keyframe_path).convert("RGB"), target_resolution_tuple)
            conditioning_items.append(LatentConditioningItem(self.pil_to_latent(img_dest), total_frames_base, destination_convergence_strength))

            current_ltx_params = {**base_ltx_params, "motion_prompt": motion_prompt}
            new_full_latents = self._generate_latent_tensor_internal(conditioning_items, current_ltx_params, target_resolution_tuple, total_frames_to_generate)


            # --- [INÍCIO] Bloco de Verificação de Frames por Chunk ---
            logger.info("--- [VERIFICAÇÃO DE CHUNKS INDIVIDUAIS] ---")
            total_chunks_verificados = new_full_latents.shape[2]
            for chunk_idx in range(total_chunks_verificados):
                try:
                    # Isola o chunk atual
                    single_chunk_latent = new_full_latents[:, :, chunk_idx:chunk_idx+1, :, :]
                    
                    # Gera um nome de arquivo temporário para o vídeo do chunk
                    temp_video_base_name = f"debug_chunk_{chunk_idx}"
                    
                    # Converte o latente do chunk em um vídeo MP4
                    temp_video_path = self._generate_video_from_latents(single_chunk_latent, temp_video_base_name)
                    
                    # Conta os frames no vídeo gerado
                    if os.path.exists(temp_video_path):
                        with imageio.get_reader(temp_video_path) as reader:
                            frame_count = reader.count_frames()
                        logger.info(f"  - VERIFICADO: Chunk {chunk_idx} gerou um vídeo com {frame_count} frames.")
                        # Apaga o vídeo de debug
                        os.remove(temp_video_path)
                    else:
                        logger.warning(f"  - FALHA: Não foi possível gerar o vídeo para o Chunk {chunk_idx}.")
                
                except Exception as e:
                    logger.error(f"  - ERRO ao verificar Chunk {chunk_idx}: {e}")
            logger.info("--- [FIM DA VERIFICAÇÃO] ---")
            # --- [FIM] Bloco de Verificação ---
            
            logger.info(f"  [1. GERAÇÃO] Tensor latente bruto gerado com shape: {new_full_latents.shape}.")
            
            
            total_chunks = new_full_latents.shape[2]
            logger.info(f"  [GUIAS] Extraindo guias de continuidade para a PRÓXIMA iteração (Total: {total_chunks} chunks).")
                
            handler_Chunk = new_full_latents[:, :, -1:, :, :].clone()
            logger.info(f"    - 'handler_Chunk' (guia de evolução) extraído do chunk final (índice {total_chunks-1}).")
            
            index_of_last_usable_chunk = total_chunks
            end_chunk_index = index_of_last_usable_chunk - trim_chunks
            start_chunk_index = end_chunk_index - echo_chunks
            
            logger.info(f"    - Fatia de chunks para 'Dejavu' (guia de memória) será: [{start_chunk_index}:{end_chunk_index}].")
            last_eco_chunk = new_full_latents[:, :, start_chunk_index:end_chunk_index, :, :].clone()
            
            logger.info(f"  [2. EDIÇÃO] Realizando a montagem do clipe de vídeo a partir do tensor bruto.")
            latents_for_video = new_full_latents

            latents_for_video = latents_for_video[:, :, :1, :, :]

            latents_for_video = latents_for_video[:, :, echo_chunks:-(trim_chunks), :, :]
              
            base_name = f"fragment_{i}_{int(time.time())}"
            
            logger.info(f"  [3. DECODIFICAÇÃO] Tensor final para o clipe tem {latents_for_video.shape[2]} chunks. Enviando para gerar vídeo.")
            video_path = self._generate_video_from_latents(latents_for_video, base_name)
            video_clips_paths.append(video_path)
            yield {"fragment_path": video_path}
                 
        final_movie_path = os.path.join(self.workspace_dir, f"final_movie_silent_{int(time.time())}.mp4")
        self.concatenate_videos_ffmpeg(video_clips_paths, final_movie_path)
        
        logger.info(f"Filme completo salvo em: {final_movie_path}")
        yield {"final_path": final_movie_path}

    def _quantize_to_multiple(self, n, m):
        if m == 0: return n
        quantized = int(round(n / m) * m)
        return m if n > 0 and quantized == 0 else quantized