MogensR commited on
Commit
00a34b8
·
1 Parent(s): ca7243a

Rename video_processor.py to api/video_processor.py

Browse files
Files changed (2) hide show
  1. api/video_processor.py +785 -0
  2. video_processor.py +0 -1209
api/video_processor.py ADDED
@@ -0,0 +1,785 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Video processing API module for BackgroundFX Pro.
3
+ Wraps CoreVideoProcessor with additional API features for streaming, batching, and real-time processing.
4
+ """
5
+
6
+ import cv2
7
+ import numpy as np
8
+ import torch
9
+ from typing import Dict, List, Optional, Tuple, Union, Callable, Generator, Any
10
+ from dataclasses import dataclass, field
11
+ from enum import Enum
12
+ from pathlib import Path
13
+ import time
14
+ import threading
15
+ from queue import Queue, Empty
16
+ import tempfile
17
+ import shutil
18
+ from concurrent.futures import ThreadPoolExecutor, as_completed
19
+ import subprocess
20
+ import json
21
+ import os
22
+ import asyncio
23
+ from datetime import datetime
24
+
25
+ from ..utils.logger import setup_logger
26
+ from ..utils.device import DeviceManager
27
+ from ..utils import TimeEstimator, MemoryMonitor
28
+ from ..core.temporal import TemporalCoherence
29
+ from .pipeline import ProcessingPipeline, PipelineConfig, PipelineResult, ProcessingMode
30
+
31
+ # Import your existing CoreVideoProcessor
32
+ from core_video import CoreVideoProcessor
33
+
34
+ logger = setup_logger(__name__)
35
+
36
+
37
+ class VideoStreamMode(Enum):
38
+ """Video streaming modes."""
39
+ FILE = "file"
40
+ WEBCAM = "webcam"
41
+ RTSP = "rtsp"
42
+ HTTP = "http"
43
+ VIRTUAL = "virtual"
44
+ SCREEN = "screen"
45
+
46
+
47
+ class OutputFormat(Enum):
48
+ """Output format options."""
49
+ MP4 = "mp4"
50
+ AVI = "avi"
51
+ MOV = "mov"
52
+ WEBM = "webm"
53
+ HLS = "hls"
54
+ DASH = "dash"
55
+ FRAMES = "frames"
56
+
57
+
58
+ @dataclass
59
+ class StreamConfig:
60
+ """Configuration for video streaming."""
61
+ # Input configuration
62
+ source: Union[str, int] = 0 # File path, camera index, or URL
63
+ stream_mode: VideoStreamMode = VideoStreamMode.FILE
64
+
65
+ # Output configuration
66
+ output_path: Optional[str] = None
67
+ output_format: OutputFormat = OutputFormat.MP4
68
+ output_codec: str = "h264"
69
+ output_bitrate: str = "5M"
70
+ output_fps: Optional[float] = None
71
+
72
+ # Streaming settings
73
+ buffer_size: int = 30
74
+ chunk_duration: float = 2.0 # For HLS/DASH
75
+ enable_adaptive_bitrate: bool = False
76
+
77
+ # Real-time settings
78
+ enable_preview: bool = False
79
+ preview_scale: float = 0.5
80
+ low_latency: bool = False
81
+
82
+ # Performance
83
+ hardware_acceleration: bool = True
84
+ num_threads: int = 4
85
+
86
+
87
+ @dataclass
88
+ class VideoStats:
89
+ """Enhanced video processing statistics."""
90
+ # Timing
91
+ start_time: float = 0.0
92
+ total_duration: float = 0.0
93
+ processing_fps: float = 0.0
94
+
95
+ # Frame stats
96
+ frames_total: int = 0
97
+ frames_processed: int = 0
98
+ frames_dropped: int = 0
99
+ frames_cached: int = 0
100
+
101
+ # Quality metrics
102
+ avg_quality_score: float = 0.0
103
+ min_quality_score: float = 1.0
104
+ max_quality_score: float = 0.0
105
+
106
+ # Performance
107
+ cpu_usage: float = 0.0
108
+ gpu_usage: float = 0.0
109
+ memory_usage_mb: float = 0.0
110
+
111
+ # Errors
112
+ error_count: int = 0
113
+ warnings: List[str] = field(default_factory=list)
114
+
115
+
116
+ class VideoProcessorAPI:
117
+ """
118
+ API wrapper for video processing with streaming and real-time capabilities.
119
+ Extends CoreVideoProcessor with additional features.
120
+ """
121
+
122
+ def __init__(self, core_processor: Optional[CoreVideoProcessor] = None):
123
+ """
124
+ Initialize Video Processor API.
125
+
126
+ Args:
127
+ core_processor: Optional existing CoreVideoProcessor instance
128
+ """
129
+ self.logger = setup_logger(f"{__name__}.VideoProcessorAPI")
130
+
131
+ # Use provided core processor or create pipeline-based one
132
+ self.core_processor = core_processor
133
+ self.pipeline = ProcessingPipeline(PipelineConfig(mode=ProcessingMode.VIDEO))
134
+
135
+ # State management
136
+ self.is_processing = False
137
+ self.is_streaming = False
138
+ self.should_stop = False
139
+
140
+ # Statistics
141
+ self.stats = VideoStats()
142
+
143
+ # Streaming components
144
+ self.input_queue = Queue(maxsize=100)
145
+ self.output_queue = Queue(maxsize=100)
146
+ self.preview_queue = Queue(maxsize=10)
147
+
148
+ # Thread pool
149
+ self.executor = ThreadPoolExecutor(max_workers=8)
150
+ self.stream_thread = None
151
+ self.process_threads = []
152
+
153
+ # FFmpeg process for advanced streaming
154
+ self.ffmpeg_process = None
155
+
156
+ # WebRTC support
157
+ self.webrtc_peers = {}
158
+
159
+ self.logger.info("VideoProcessorAPI initialized")
160
+
161
+ async def process_video_async(self,
162
+ input_path: str,
163
+ output_path: str,
164
+ background: Optional[Union[str, np.ndarray]] = None,
165
+ progress_callback: Optional[Callable] = None) -> VideoStats:
166
+ """
167
+ Asynchronously process a video file.
168
+
169
+ Args:
170
+ input_path: Path to input video
171
+ output_path: Path to output video
172
+ background: Background image or path
173
+ progress_callback: Progress callback function
174
+
175
+ Returns:
176
+ Processing statistics
177
+ """
178
+ return await asyncio.get_event_loop().run_in_executor(
179
+ None,
180
+ self.process_video,
181
+ input_path,
182
+ output_path,
183
+ background,
184
+ progress_callback
185
+ )
186
+
187
+ def process_video(self,
188
+ input_path: str,
189
+ output_path: str,
190
+ background: Optional[Union[str, np.ndarray]] = None,
191
+ progress_callback: Optional[Callable] = None) -> VideoStats:
192
+ """
193
+ Process a video file using either CoreVideoProcessor or Pipeline.
194
+
195
+ Args:
196
+ input_path: Path to input video
197
+ output_path: Path to output video
198
+ background: Background image or path
199
+ progress_callback: Progress callback function
200
+
201
+ Returns:
202
+ Processing statistics
203
+ """
204
+ self.stats = VideoStats(start_time=time.time())
205
+ self.is_processing = True
206
+
207
+ try:
208
+ # If we have CoreVideoProcessor, use it
209
+ if self.core_processor:
210
+ return self._process_with_core(
211
+ input_path, output_path, background, progress_callback
212
+ )
213
+ else:
214
+ # Use pipeline-based processing
215
+ return self._process_with_pipeline(
216
+ input_path, output_path, background, progress_callback
217
+ )
218
+
219
+ finally:
220
+ self.is_processing = False
221
+ self.stats.total_duration = time.time() - self.stats.start_time
222
+
223
+ def _process_with_pipeline(self,
224
+ input_path: str,
225
+ output_path: str,
226
+ background: Optional[Union[str, np.ndarray]],
227
+ progress_callback: Optional[Callable]) -> VideoStats:
228
+ """Process video using the Pipeline system."""
229
+
230
+ cap = cv2.VideoCapture(input_path)
231
+ if not cap.isOpened():
232
+ raise ValueError(f"Cannot open video: {input_path}")
233
+
234
+ # Get video properties
235
+ fps = cap.get(cv2.CAP_PROP_FPS)
236
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
237
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
238
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
239
+
240
+ self.stats.frames_total = total_frames
241
+
242
+ # Setup output writer
243
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
244
+ out = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
245
+
246
+ frame_idx = 0
247
+
248
+ try:
249
+ while True:
250
+ ret, frame = cap.read()
251
+ if not ret:
252
+ break
253
+
254
+ # Process frame through pipeline
255
+ result = self.pipeline.process_image(frame, background)
256
+
257
+ if result.success and result.output_image is not None:
258
+ out.write(result.output_image)
259
+ self.stats.frames_processed += 1
260
+
261
+ # Update quality metrics
262
+ self._update_quality_stats(result.quality_score)
263
+ else:
264
+ # Write original frame on failure
265
+ out.write(frame)
266
+ self.stats.frames_dropped += 1
267
+
268
+ frame_idx += 1
269
+
270
+ # Progress callback
271
+ if progress_callback:
272
+ progress = frame_idx / total_frames
273
+ progress_callback(progress, {
274
+ 'current_frame': frame_idx,
275
+ 'total_frames': total_frames,
276
+ 'fps': self.stats.frames_processed / (time.time() - self.stats.start_time)
277
+ })
278
+
279
+ # Check if should stop
280
+ if self.should_stop:
281
+ break
282
+
283
+ finally:
284
+ cap.release()
285
+ out.release()
286
+
287
+ self.stats.processing_fps = self.stats.frames_processed / (time.time() - self.stats.start_time)
288
+ return self.stats
289
+
290
+ def _process_with_core(self,
291
+ input_path: str,
292
+ output_path: str,
293
+ background: Optional[Union[str, np.ndarray]],
294
+ progress_callback: Optional[Callable]) -> VideoStats:
295
+ """Process video using CoreVideoProcessor."""
296
+
297
+ # Determine background choice
298
+ if isinstance(background, str):
299
+ if os.path.exists(background):
300
+ bg_choice = "custom"
301
+ custom_bg = background
302
+ else:
303
+ bg_choice = background
304
+ custom_bg = None
305
+ elif isinstance(background, np.ndarray):
306
+ # Save background to temp file
307
+ temp_bg = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
308
+ cv2.imwrite(temp_bg.name, background)
309
+ bg_choice = "custom"
310
+ custom_bg = temp_bg.name
311
+ else:
312
+ bg_choice = "blur"
313
+ custom_bg = None
314
+
315
+ # Process with CoreVideoProcessor
316
+ output, message = self.core_processor.process_video(
317
+ input_path,
318
+ bg_choice,
319
+ custom_bg,
320
+ progress_callback
321
+ )
322
+
323
+ if output:
324
+ # Move output to desired location
325
+ shutil.move(output, output_path)
326
+
327
+ # Extract stats from core processor
328
+ core_stats = self.core_processor.stats
329
+ self.stats.frames_processed = core_stats.get('successful_frames', 0)
330
+ self.stats.frames_dropped = core_stats.get('failed_frames', 0)
331
+ self.stats.processing_fps = core_stats.get('average_fps', 0)
332
+
333
+ return self.stats
334
+
335
+ def start_stream_processing(self,
336
+ config: StreamConfig,
337
+ background: Optional[Union[str, np.ndarray]] = None) -> bool:
338
+ """
339
+ Start real-time stream processing.
340
+
341
+ Args:
342
+ config: Stream configuration
343
+ background: Background for replacement
344
+
345
+ Returns:
346
+ True if stream started successfully
347
+ """
348
+ if self.is_streaming:
349
+ self.logger.warning("Stream already active")
350
+ return False
351
+
352
+ self.is_streaming = True
353
+ self.should_stop = False
354
+
355
+ # Start input stream thread
356
+ self.stream_thread = threading.Thread(
357
+ target=self._stream_input_handler,
358
+ args=(config,)
359
+ )
360
+ self.stream_thread.start()
361
+
362
+ # Start processing threads
363
+ for i in range(config.num_threads):
364
+ thread = threading.Thread(
365
+ target=self._stream_processor,
366
+ args=(background,)
367
+ )
368
+ thread.start()
369
+ self.process_threads.append(thread)
370
+
371
+ # Start output handler
372
+ if config.output_format in [OutputFormat.HLS, OutputFormat.DASH]:
373
+ self._start_adaptive_streaming(config)
374
+ else:
375
+ self._start_output_handler(config)
376
+
377
+ self.logger.info(f"Stream processing started: {config.stream_mode.value}")
378
+ return True
379
+
380
+ def _stream_input_handler(self, config: StreamConfig):
381
+ """Handle input stream capture."""
382
+ try:
383
+ # Open input stream
384
+ if config.stream_mode == VideoStreamMode.FILE:
385
+ cap = cv2.VideoCapture(config.source)
386
+ elif config.stream_mode == VideoStreamMode.WEBCAM:
387
+ cap = cv2.VideoCapture(int(config.source))
388
+ elif config.stream_mode in [VideoStreamMode.RTSP, VideoStreamMode.HTTP]:
389
+ cap = cv2.VideoCapture(config.source)
390
+ elif config.stream_mode == VideoStreamMode.SCREEN:
391
+ # Screen capture (platform-specific)
392
+ cap = self._setup_screen_capture()
393
+ else:
394
+ raise ValueError(f"Unsupported stream mode: {config.stream_mode}")
395
+
396
+ if not cap.isOpened():
397
+ raise ValueError("Failed to open stream")
398
+
399
+ frame_count = 0
400
+
401
+ while self.is_streaming and not self.should_stop:
402
+ ret, frame = cap.read()
403
+ if not ret:
404
+ if config.stream_mode == VideoStreamMode.FILE:
405
+ # End of file
406
+ break
407
+ else:
408
+ # Retry for live streams
409
+ time.sleep(0.1)
410
+ continue
411
+
412
+ # Add frame to processing queue
413
+ try:
414
+ self.input_queue.put(frame, timeout=0.1)
415
+ frame_count += 1
416
+ except:
417
+ # Queue full, drop frame
418
+ self.stats.frames_dropped += 1
419
+
420
+ # Control frame rate for live streams
421
+ if config.stream_mode != VideoStreamMode.FILE:
422
+ time.sleep(1.0 / 30) # 30 FPS limit
423
+
424
+ cap.release()
425
+
426
+ except Exception as e:
427
+ self.logger.error(f"Stream input handler error: {e}")
428
+ finally:
429
+ self.is_streaming = False
430
+
431
+ def _stream_processor(self, background: Optional[Union[str, np.ndarray]]):
432
+ """Process frames from input queue."""
433
+ while self.is_streaming or not self.input_queue.empty():
434
+ try:
435
+ frame = self.input_queue.get(timeout=0.5)
436
+
437
+ # Process frame
438
+ result = self.pipeline.process_image(frame, background)
439
+
440
+ if result.success and result.output_image is not None:
441
+ # Add to output queue
442
+ self.output_queue.put(result.output_image)
443
+
444
+ # Update stats
445
+ self.stats.frames_processed += 1
446
+ self._update_quality_stats(result.quality_score)
447
+
448
+ # Add to preview queue if enabled
449
+ if not self.preview_queue.full():
450
+ preview = cv2.resize(result.output_image, None, fx=0.5, fy=0.5)
451
+ try:
452
+ self.preview_queue.put_nowait(preview)
453
+ except:
454
+ pass
455
+
456
+ except Empty:
457
+ continue
458
+ except Exception as e:
459
+ self.logger.error(f"Stream processor error: {e}")
460
+ self.stats.error_count += 1
461
+
462
+ def _start_output_handler(self, config: StreamConfig):
463
+ """Start output stream handler."""
464
+ output_thread = threading.Thread(
465
+ target=self._output_handler,
466
+ args=(config,)
467
+ )
468
+ output_thread.start()
469
+ self.process_threads.append(output_thread)
470
+
471
+ def _output_handler(self, config: StreamConfig):
472
+ """Handle output stream writing."""
473
+ try:
474
+ if config.output_format == OutputFormat.FRAMES:
475
+ # Save individual frames
476
+ self._save_frames_output(config)
477
+ else:
478
+ # Video file output
479
+ self._save_video_output(config)
480
+
481
+ except Exception as e:
482
+ self.logger.error(f"Output handler error: {e}")
483
+
484
+ def _save_video_output(self, config: StreamConfig):
485
+ """Save processed frames to video file."""
486
+ out = None
487
+ frame_count = 0
488
+
489
+ try:
490
+ while self.is_streaming or not self.output_queue.empty():
491
+ try:
492
+ frame = self.output_queue.get(timeout=0.5)
493
+
494
+ # Initialize writer on first frame
495
+ if out is None:
496
+ h, w = frame.shape[:2]
497
+ fps = config.output_fps or 30.0
498
+
499
+ if config.output_format == OutputFormat.MP4:
500
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
501
+ elif config.output_format == OutputFormat.AVI:
502
+ fourcc = cv2.VideoWriter_fourcc(*'XVID')
503
+ else:
504
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
505
+
506
+ out = cv2.VideoWriter(
507
+ config.output_path,
508
+ fourcc,
509
+ fps,
510
+ (w, h)
511
+ )
512
+
513
+ out.write(frame)
514
+ frame_count += 1
515
+
516
+ except Empty:
517
+ continue
518
+
519
+ finally:
520
+ if out:
521
+ out.release()
522
+ self.logger.info(f"Saved {frame_count} frames to {config.output_path}")
523
+
524
+ def _save_frames_output(self, config: StreamConfig):
525
+ """Save processed frames as individual images."""
526
+ output_dir = Path(config.output_path)
527
+ output_dir.mkdir(parents=True, exist_ok=True)
528
+
529
+ frame_count = 0
530
+
531
+ while self.is_streaming or not self.output_queue.empty():
532
+ try:
533
+ frame = self.output_queue.get(timeout=0.5)
534
+
535
+ # Save frame
536
+ frame_path = output_dir / f"frame_{frame_count:06d}.png"
537
+ cv2.imwrite(str(frame_path), frame)
538
+ frame_count += 1
539
+
540
+ except Empty:
541
+ continue
542
+
543
+ def _start_adaptive_streaming(self, config: StreamConfig):
544
+ """Start HLS or DASH adaptive streaming."""
545
+ try:
546
+ # Prepare FFmpeg command for streaming
547
+ if config.output_format == OutputFormat.HLS:
548
+ self._start_hls_streaming(config)
549
+ elif config.output_format == OutputFormat.DASH:
550
+ self._start_dash_streaming(config)
551
+
552
+ except Exception as e:
553
+ self.logger.error(f"Adaptive streaming setup failed: {e}")
554
+
555
+ def _start_hls_streaming(self, config: StreamConfig):
556
+ """Start HLS streaming with FFmpeg."""
557
+ output_dir = Path(config.output_path)
558
+ output_dir.mkdir(parents=True, exist_ok=True)
559
+
560
+ # FFmpeg command for HLS
561
+ cmd = [
562
+ 'ffmpeg',
563
+ '-f', 'rawvideo',
564
+ '-pix_fmt', 'bgr24',
565
+ '-s', '1920x1080', # Will be updated with actual size
566
+ '-r', '30',
567
+ '-i', '-', # Input from pipe
568
+ '-c:v', 'libx264',
569
+ '-preset', 'ultrafast',
570
+ '-tune', 'zerolatency',
571
+ '-f', 'hls',
572
+ '-hls_time', str(config.chunk_duration),
573
+ '-hls_list_size', '10',
574
+ '-hls_flags', 'delete_segments',
575
+ str(output_dir / 'stream.m3u8')
576
+ ]
577
+
578
+ # Start FFmpeg process
579
+ self.ffmpeg_process = subprocess.Popen(
580
+ cmd,
581
+ stdin=subprocess.PIPE,
582
+ stdout=subprocess.PIPE,
583
+ stderr=subprocess.PIPE
584
+ )
585
+
586
+ # Start thread to pipe frames to FFmpeg
587
+ ffmpeg_thread = threading.Thread(
588
+ target=self._pipe_to_ffmpeg
589
+ )
590
+ ffmpeg_thread.start()
591
+ self.process_threads.append(ffmpeg_thread)
592
+
593
+ self.logger.info(f"HLS streaming started: {output_dir / 'stream.m3u8'}")
594
+
595
+ def _pipe_to_ffmpeg(self):
596
+ """Pipe processed frames to FFmpeg."""
597
+ while self.is_streaming or not self.output_queue.empty():
598
+ try:
599
+ frame = self.output_queue.get(timeout=0.5)
600
+
601
+ if self.ffmpeg_process and self.ffmpeg_process.stdin:
602
+ self.ffmpeg_process.stdin.write(frame.tobytes())
603
+
604
+ except Empty:
605
+ continue
606
+ except Exception as e:
607
+ self.logger.error(f"FFmpeg pipe error: {e}")
608
+ break
609
+
610
+ def _setup_screen_capture(self) -> cv2.VideoCapture:
611
+ """Setup screen capture (platform-specific)."""
612
+ # This would need platform-specific implementation
613
+ # For now, return a dummy capture
614
+ return cv2.VideoCapture(0)
615
+
616
+ def _update_quality_stats(self, quality_score: float):
617
+ """Update quality statistics."""
618
+ n = self.stats.frames_processed
619
+ if n == 0:
620
+ self.stats.avg_quality_score = quality_score
621
+ else:
622
+ self.stats.avg_quality_score = (
623
+ (self.stats.avg_quality_score * n + quality_score) / (n + 1)
624
+ )
625
+
626
+ self.stats.min_quality_score = min(self.stats.min_quality_score, quality_score)
627
+ self.stats.max_quality_score = max(self.stats.max_quality_score, quality_score)
628
+
629
+ def stop_stream_processing(self):
630
+ """Stop stream processing."""
631
+ self.should_stop = True
632
+ self.is_streaming = False
633
+
634
+ # Wait for threads to finish
635
+ if self.stream_thread:
636
+ self.stream_thread.join(timeout=5)
637
+
638
+ for thread in self.process_threads:
639
+ thread.join(timeout=5)
640
+
641
+ # Stop FFmpeg if running
642
+ if self.ffmpeg_process:
643
+ self.ffmpeg_process.terminate()
644
+ self.ffmpeg_process.wait(timeout=5)
645
+
646
+ self.logger.info("Stream processing stopped")
647
+
648
+ def get_preview_frame(self) -> Optional[np.ndarray]:
649
+ """Get a preview frame from the preview queue."""
650
+ try:
651
+ return self.preview_queue.get_nowait()
652
+ except Empty:
653
+ return None
654
+
655
+ def get_stats(self) -> VideoStats:
656
+ """Get current processing statistics."""
657
+ if self.is_processing or self.is_streaming:
658
+ self.stats.processing_fps = (
659
+ self.stats.frames_processed /
660
+ (time.time() - self.stats.start_time)
661
+ )
662
+ return self.stats
663
+
664
+ def process_video_batch(self,
665
+ input_paths: List[str],
666
+ output_dir: str,
667
+ background: Optional[Union[str, np.ndarray]] = None,
668
+ parallel: bool = True) -> List[VideoStats]:
669
+ """
670
+ Process multiple videos in batch.
671
+
672
+ Args:
673
+ input_paths: List of input video paths
674
+ output_dir: Output directory
675
+ background: Background for all videos
676
+ parallel: Process in parallel
677
+
678
+ Returns:
679
+ List of processing statistics
680
+ """
681
+ output_dir = Path(output_dir)
682
+ output_dir.mkdir(parents=True, exist_ok=True)
683
+
684
+ results = []
685
+
686
+ if parallel:
687
+ # Process in parallel
688
+ futures = []
689
+
690
+ for input_path in input_paths:
691
+ input_name = Path(input_path).stem
692
+ output_path = output_dir / f"{input_name}_processed.mp4"
693
+
694
+ future = self.executor.submit(
695
+ self.process_video,
696
+ input_path,
697
+ str(output_path),
698
+ background
699
+ )
700
+ futures.append(future)
701
+
702
+ # Collect results
703
+ for future in as_completed(futures):
704
+ try:
705
+ stats = future.result(timeout=3600) # 1 hour timeout
706
+ results.append(stats)
707
+ except Exception as e:
708
+ self.logger.error(f"Batch processing error: {e}")
709
+ results.append(VideoStats(error_count=1))
710
+ else:
711
+ # Process sequentially
712
+ for input_path in input_paths:
713
+ input_name = Path(input_path).stem
714
+ output_path = output_dir / f"{input_name}_processed.mp4"
715
+
716
+ stats = self.process_video(
717
+ input_path,
718
+ str(output_path),
719
+ background
720
+ )
721
+ results.append(stats)
722
+
723
+ return results
724
+
725
+ def export_to_format(self,
726
+ input_path: str,
727
+ output_path: str,
728
+ format: OutputFormat,
729
+ **kwargs) -> bool:
730
+ """
731
+ Export processed video to specific format.
732
+
733
+ Args:
734
+ input_path: Input video path
735
+ output_path: Output path
736
+ format: Target format
737
+ **kwargs: Format-specific options
738
+
739
+ Returns:
740
+ True if successful
741
+ """
742
+ try:
743
+ if format == OutputFormat.WEBM:
744
+ cmd = [
745
+ 'ffmpeg', '-i', input_path,
746
+ '-c:v', 'libvpx-vp9',
747
+ '-crf', '30',
748
+ '-b:v', '0',
749
+ output_path
750
+ ]
751
+ elif format == OutputFormat.HLS:
752
+ cmd = [
753
+ 'ffmpeg', '-i', input_path,
754
+ '-c:v', 'libx264',
755
+ '-hls_time', '10',
756
+ '-hls_list_size', '0',
757
+ '-f', 'hls',
758
+ output_path
759
+ ]
760
+ else:
761
+ # Default MP4 conversion
762
+ cmd = [
763
+ 'ffmpeg', '-i', input_path,
764
+ '-c:v', 'libx264',
765
+ '-preset', 'medium',
766
+ '-crf', '23',
767
+ output_path
768
+ ]
769
+
770
+ result = subprocess.run(cmd, capture_output=True, text=True)
771
+ return result.returncode == 0
772
+
773
+ except Exception as e:
774
+ self.logger.error(f"Export failed: {e}")
775
+ return False
776
+
777
+ def cleanup(self):
778
+ """Cleanup resources."""
779
+ self.stop_stream_processing()
780
+ self.executor.shutdown(wait=True)
781
+
782
+ if self.core_processor:
783
+ self.core_processor.cleanup()
784
+
785
+ self.logger.info("VideoProcessorAPI cleanup complete")
video_processor.py DELETED
@@ -1,1209 +0,0 @@
1
- """
2
- Core Video Processing Module - Enhanced with Temporal Consistency
3
- VERSION: 2.0-temporal-enhanced
4
- ROLLBACK: Set USE_TEMPORAL_ENHANCEMENT = False to revert to original behavior
5
- """
6
-
7
- import os
8
- import cv2
9
- import numpy as np
10
- import time
11
- import logging
12
- import threading
13
- from typing import Optional, Tuple, Dict, Any, Callable, List
14
- from pathlib import Path
15
-
16
- # Import modular components
17
- import app_config
18
- import memory_manager
19
- import progress_tracker
20
- import exceptions
21
-
22
- # Import utilities
23
- from utilities import (
24
- segment_person_hq,
25
- refine_mask_hq,
26
- replace_background_hq,
27
- create_professional_background,
28
- PROFESSIONAL_BACKGROUNDS,
29
- validate_video_file
30
- )
31
-
32
- # ============================================================================
33
- # VERSION CONTROL AND FEATURE FLAGS - EASY ROLLBACK
34
- # ============================================================================
35
-
36
- # ROLLBACK CONTROL: Set to False to use original functions
37
- USE_TEMPORAL_ENHANCEMENT = True
38
- USE_HAIR_DETECTION = True
39
- USE_OPTICAL_FLOW_TRACKING = True
40
- USE_ADAPTIVE_REFINEMENT = True
41
-
42
- logger = logging.getLogger(__name__)
43
-
44
- class CoreVideoProcessor:
45
- """
46
- ENHANCED: Core video processing pipeline with temporal consistency and fine-detail handling
47
- """
48
-
49
- def __init__(self, sam2_predictor: Any, matanyone_model: Any,
50
- config: app_config.ProcessingConfig, memory_mgr: memory_manager.MemoryManager):
51
- self.sam2_predictor = sam2_predictor
52
- self.matanyone_model = matanyone_model
53
- self.config = config
54
- self.memory_manager = memory_mgr
55
-
56
- # Processing state
57
- self.processing_active = False
58
- self.last_refined_mask = None
59
- self.frame_cache = {}
60
-
61
- # ENHANCED: Temporal consistency state
62
- self.mask_history = [] # Store recent masks for temporal smoothing
63
- self.optical_flow_data = None # Previous frame for optical flow
64
- self.hair_regions_cache = {} # Cache detected hair regions
65
- self.quality_scores_history = [] # Track quality over time
66
-
67
- # Statistics
68
- self.stats = {
69
- 'videos_processed': 0,
70
- 'total_frames_processed': 0,
71
- 'total_processing_time': 0.0,
72
- 'average_fps': 0.0,
73
- 'failed_frames': 0,
74
- 'successful_frames': 0,
75
- 'cache_hits': 0,
76
- 'segmentation_errors': 0,
77
- 'refinement_errors': 0,
78
- 'temporal_corrections': 0, # NEW: Track temporal fixes
79
- 'hair_detections': 0, # NEW: Track hair detection success
80
- 'flow_tracking_failures': 0 # NEW: Track optical flow issues
81
- }
82
-
83
- # Quality settings based on config
84
- self.quality_settings = config.get_quality_settings()
85
-
86
- logger.info("CoreVideoProcessor initialized")
87
- logger.info(f"Quality preset: {config.quality_preset}")
88
- logger.info(f"Quality settings: {self.quality_settings}")
89
-
90
- if USE_TEMPORAL_ENHANCEMENT:
91
- logger.info("ENHANCED: Temporal consistency enabled")
92
- if USE_HAIR_DETECTION:
93
- logger.info("ENHANCED: Hair detection enabled")
94
-
95
- def process_video(
96
- self,
97
- video_path: str,
98
- background_choice: str,
99
- custom_background_path: Optional[str] = None,
100
- progress_callback: Optional[Callable] = None,
101
- cancel_event: Optional[threading.Event] = None,
102
- preview_mask: bool = False,
103
- preview_greenscreen: bool = False
104
- ) -> Tuple[Optional[str], str]:
105
- """
106
- ENHANCED: Process video with temporal consistency and fine-detail handling
107
- """
108
- if self.processing_active:
109
- return None, "Processing already in progress"
110
-
111
- self.processing_active = True
112
- start_time = time.time()
113
-
114
- # ENHANCED: Reset temporal state for new video
115
- self._reset_temporal_state()
116
-
117
- try:
118
- # Validate input video
119
- is_valid, validation_msg = validate_video_file(video_path)
120
- if not is_valid:
121
- return None, f"Invalid video file: {validation_msg}"
122
-
123
- # Open video file
124
- cap = cv2.VideoCapture(video_path)
125
- if not cap.isOpened():
126
- return None, "Could not open video file"
127
-
128
- # Get video properties
129
- video_info = self._get_video_info(cap)
130
- logger.info(f"Processing video: {video_info}")
131
-
132
- # Check memory requirements
133
- memory_check = self.memory_manager.can_process_video(
134
- video_info['width'], video_info['height']
135
- )
136
-
137
- if not memory_check['can_process']:
138
- cap.release()
139
- return None, f"Insufficient memory: {memory_check['recommendations']}"
140
-
141
- # Prepare background
142
- background = self.prepare_background(
143
- background_choice, custom_background_path,
144
- video_info['width'], video_info['height']
145
- )
146
-
147
- if background is None:
148
- cap.release()
149
- return None, "Failed to prepare background"
150
-
151
- # Setup output video
152
- output_path = self._setup_output_video(video_info, preview_mask, preview_greenscreen)
153
- out = self._create_video_writer(output_path, video_info)
154
-
155
- if out is None:
156
- cap.release()
157
- return None, "Could not create output video writer"
158
-
159
- # ENHANCED: Process video frames with temporal consistency
160
- result = self._process_video_frames_enhanced(
161
- cap, out, background, video_info,
162
- progress_callback, cancel_event,
163
- preview_mask, preview_greenscreen
164
- )
165
-
166
- # Cleanup
167
- cap.release()
168
- out.release()
169
-
170
- if result['success']:
171
- # Update statistics
172
- processing_time = time.time() - start_time
173
- self._update_processing_stats(video_info, processing_time, result)
174
-
175
- success_msg = (
176
- f"Processing completed successfully!\n"
177
- f"Processed: {result['successful_frames']}/{result['total_frames']} frames\n"
178
- f"Time: {processing_time:.1f}s\n"
179
- f"Average FPS: {result['total_frames'] / processing_time:.1f}\n"
180
- f"Temporal corrections: {self.stats['temporal_corrections']}\n"
181
- f"Hair detections: {self.stats['hair_detections']}\n"
182
- f"Background: {background_choice}"
183
- )
184
-
185
- return output_path, success_msg
186
- else:
187
- # Clean up failed output
188
- try:
189
- os.remove(output_path)
190
- except:
191
- pass
192
- return None, result['error_message']
193
-
194
- except Exception as e:
195
- logger.error(f"Video processing failed: {e}")
196
- return None, f"Processing failed: {str(e)}"
197
-
198
- finally:
199
- self.processing_active = False
200
-
201
- def _reset_temporal_state(self):
202
- """ENHANCED: Reset temporal consistency state"""
203
- self.mask_history.clear()
204
- self.optical_flow_data = None
205
- self.hair_regions_cache.clear()
206
- self.quality_scores_history.clear()
207
- self.last_refined_mask = None
208
- self.stats['temporal_corrections'] = 0
209
- self.stats['hair_detections'] = 0
210
- self.stats['flow_tracking_failures'] = 0
211
-
212
- def _get_video_info(self, cap: cv2.VideoCapture) -> Dict[str, Any]:
213
- """Extract comprehensive video information"""
214
- return {
215
- 'fps': cap.get(cv2.CAP_PROP_FPS),
216
- 'total_frames': int(cap.get(cv2.CAP_PROP_FRAME_COUNT)),
217
- 'width': int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
218
- 'height': int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
219
- 'duration': cap.get(cv2.CAP_PROP_FRAME_COUNT) / cap.get(cv2.CAP_PROP_FPS),
220
- 'codec': int(cap.get(cv2.CAP_PROP_FOURCC))
221
- }
222
-
223
- def _setup_output_video(self, video_info: Dict[str, Any],
224
- preview_mask: bool, preview_greenscreen: bool) -> str:
225
- """Setup output video path"""
226
- timestamp = int(time.time())
227
-
228
- if preview_mask:
229
- filename = f"mask_preview_{timestamp}.mp4"
230
- elif preview_greenscreen:
231
- filename = f"greenscreen_preview_{timestamp}.mp4"
232
- else:
233
- filename = f"processed_video_{timestamp}.mp4"
234
-
235
- return os.path.join(self.config.temp_dir, filename)
236
-
237
- def _create_video_writer(self, output_path: str,
238
- video_info: Dict[str, Any]) -> Optional[cv2.VideoWriter]:
239
- """Create video writer with optimal settings"""
240
- try:
241
- # Choose codec based on quality settings
242
- if self.config.output_quality == 'high':
243
- fourcc = cv2.VideoWriter_fourcc(*'mp4v')
244
- else:
245
- fourcc = cv2.VideoWriter_fourcc(*'XVID')
246
-
247
- writer = cv2.VideoWriter(
248
- output_path,
249
- fourcc,
250
- video_info['fps'],
251
- (video_info['width'], video_info['height'])
252
- )
253
-
254
- if not writer.isOpened():
255
- logger.error("Failed to open video writer")
256
- return None
257
-
258
- return writer
259
-
260
- except Exception as e:
261
- logger.error(f"Error creating video writer: {e}")
262
- return None
263
-
264
- def _process_video_frames_enhanced(
265
- self,
266
- cap: cv2.VideoCapture,
267
- out: cv2.VideoWriter,
268
- background: np.ndarray,
269
- video_info: Dict[str, Any],
270
- progress_callback: Optional[Callable],
271
- cancel_event: Optional[threading.Event],
272
- preview_mask: bool,
273
- preview_greenscreen: bool
274
- ) -> Dict[str, Any]:
275
- """ENHANCED: Process all video frames with temporal consistency"""
276
-
277
- # Initialize progress tracking
278
- prog_tracker = progress_tracker.ProgressTracker(
279
- total_frames=video_info['total_frames'],
280
- callback=progress_callback,
281
- track_performance=True
282
- )
283
-
284
- frame_count = 0
285
- successful_frames = 0
286
- failed_frames = 0
287
-
288
- # Reset enhanced state
289
- self._reset_temporal_state()
290
-
291
- try:
292
- prog_tracker.set_stage("Processing frames with temporal enhancement")
293
-
294
- while True:
295
- # Check for cancellation
296
- if cancel_event and cancel_event.is_set():
297
- return {
298
- 'success': False,
299
- 'error_message': 'Processing cancelled by user',
300
- 'total_frames': frame_count,
301
- 'successful_frames': successful_frames,
302
- 'failed_frames': failed_frames
303
- }
304
-
305
- # Read frame
306
- ret, frame = cap.read()
307
- if not ret:
308
- break
309
-
310
- try:
311
- # Update progress
312
- prog_tracker.update(frame_count, "Processing frame with temporal consistency")
313
-
314
- # ENHANCED: Process frame with temporal consistency
315
- if USE_TEMPORAL_ENHANCEMENT:
316
- processed_frame = self._process_single_frame_enhanced(
317
- frame, background, frame_count,
318
- preview_mask, preview_greenscreen
319
- )
320
- else:
321
- processed_frame = self._process_single_frame_original(
322
- frame, background, frame_count,
323
- preview_mask, preview_greenscreen
324
- )
325
-
326
- # Write processed frame
327
- out.write(processed_frame)
328
- successful_frames += 1
329
-
330
- # Memory management
331
- if frame_count % self.config.memory_cleanup_interval == 0:
332
- self.memory_manager.auto_cleanup_if_needed()
333
-
334
- except Exception as frame_error:
335
- logger.warning(f"Frame {frame_count} processing failed: {frame_error}")
336
-
337
- # Write original frame as fallback
338
- out.write(frame)
339
- failed_frames += 1
340
- self.stats['failed_frames'] += 1
341
-
342
- frame_count += 1
343
-
344
- # Skip frames if configured (for performance)
345
- if self.config.frame_skip > 1:
346
- for _ in range(self.config.frame_skip - 1):
347
- ret, _ = cap.read()
348
- if not ret:
349
- break
350
- frame_count += 1
351
-
352
- # Finalize progress tracking
353
- final_stats = prog_tracker.finalize()
354
-
355
- return {
356
- 'success': successful_frames > 0,
357
- 'error_message': f'No frames processed successfully' if successful_frames == 0 else '',
358
- 'total_frames': frame_count,
359
- 'successful_frames': successful_frames,
360
- 'failed_frames': failed_frames,
361
- 'processing_stats': final_stats
362
- }
363
-
364
- except Exception as e:
365
- logger.error(f"Frame processing loop failed: {e}")
366
- return {
367
- 'success': False,
368
- 'error_message': f'Frame processing failed: {str(e)}',
369
- 'total_frames': frame_count,
370
- 'successful_frames': successful_frames,
371
- 'failed_frames': failed_frames
372
- }
373
-
374
- def _process_single_frame_enhanced(
375
- self,
376
- frame: np.ndarray,
377
- background: np.ndarray,
378
- frame_number: int,
379
- preview_mask: bool,
380
- preview_greenscreen: bool
381
- ) -> np.ndarray:
382
- """ENHANCED: Process a single video frame with temporal consistency"""
383
-
384
- try:
385
- # Person segmentation
386
- mask = self._segment_person_enhanced(frame, frame_number)
387
-
388
- # ENHANCED: Detect hair and fine details
389
- if USE_HAIR_DETECTION:
390
- hair_regions = self._detect_hair_regions(frame, mask, frame_number)
391
- else:
392
- hair_regions = None
393
-
394
- # ENHANCED: Apply temporal consistency
395
- if USE_TEMPORAL_ENHANCEMENT and len(self.mask_history) > 0:
396
- mask = self._apply_temporal_consistency_enhanced(frame, mask, frame_number)
397
-
398
- # ENHANCED: Adaptive mask refinement based on frame content
399
- if USE_ADAPTIVE_REFINEMENT:
400
- refined_mask = self._adaptive_mask_refinement(frame, mask, frame_number, hair_regions)
401
- else:
402
- refined_mask = self._refine_mask_original(frame, mask, frame_number)
403
-
404
- # Store mask in history for temporal consistency
405
- self._update_mask_history(refined_mask)
406
-
407
- # Generate output based on mode
408
- if preview_mask:
409
- return self._create_mask_preview_enhanced(frame, refined_mask, hair_regions)
410
- elif preview_greenscreen:
411
- return self._create_greenscreen_preview(frame, refined_mask)
412
- else:
413
- return self._replace_background_enhanced(frame, refined_mask, background, hair_regions)
414
-
415
- except Exception as e:
416
- logger.warning(f"Enhanced single frame processing failed: {e}")
417
- # Fallback to original processing
418
- return self._process_single_frame_original(frame, background, frame_number, preview_mask, preview_greenscreen)
419
-
420
- def _detect_hair_regions(self, frame: np.ndarray, mask: np.ndarray, frame_number: int) -> Optional[np.ndarray]:
421
- """ENHANCED: Detect hair and fine detail regions automatically"""
422
- try:
423
- # Check cache first
424
- if frame_number in self.hair_regions_cache:
425
- self.stats['cache_hits'] += 1
426
- return self.hair_regions_cache[frame_number]
427
-
428
- # Convert frame to different color spaces for better hair detection
429
- hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
430
- gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
431
-
432
- # Method 1: Texture-based hair detection
433
- # Hair typically has high frequency texture
434
- laplacian = cv2.Laplacian(gray, cv2.CV_64F)
435
- texture_strength = np.abs(laplacian)
436
-
437
- # Method 2: Color-based hair detection
438
- # Hair is typically in darker hue ranges
439
- hair_hue_mask = ((hsv[:,:,0] >= 0) & (hsv[:,:,0] <= 30)) | \
440
- ((hsv[:,:,0] >= 150) & (hsv[:,:,0] <= 180))
441
- hair_value_mask = hsv[:,:,2] < 100 # Darker regions
442
-
443
- # Combine texture and color information
444
- hair_probability = np.zeros_like(gray, dtype=np.float32)
445
-
446
- # High texture regions
447
- texture_norm = (texture_strength - texture_strength.min()) / (texture_strength.max() - texture_strength.min() + 1e-8)
448
- hair_probability += texture_norm * 0.6
449
-
450
- # Color-based probability
451
- color_prob = (hair_hue_mask.astype(np.float32) * hair_value_mask.astype(np.float32))
452
- hair_probability += color_prob * 0.4
453
-
454
- # Only consider regions near the mask boundary (where hair typically is)
455
- mask_boundary = self._get_mask_boundary_region(mask, boundary_width=20)
456
- hair_probability *= mask_boundary
457
-
458
- # Threshold to get hair regions
459
- hair_threshold = np.percentile(hair_probability[hair_probability > 0], 75)
460
- hair_regions = (hair_probability > hair_threshold).astype(np.uint8)
461
-
462
- # Clean up hair regions
463
- kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
464
- hair_regions = cv2.morphologyEx(hair_regions, cv2.MORPH_CLOSE, kernel)
465
-
466
- # Cache the result
467
- self.hair_regions_cache[frame_number] = hair_regions
468
-
469
- # Update stats if hair was detected
470
- if np.any(hair_regions):
471
- self.stats['hair_detections'] += 1
472
- logger.debug(f"Hair regions detected in frame {frame_number}")
473
-
474
- return hair_regions
475
-
476
- except Exception as e:
477
- logger.warning(f"Hair detection failed for frame {frame_number}: {e}")
478
- return None
479
-
480
- def _get_mask_boundary_region(self, mask: np.ndarray, boundary_width: int = 20) -> np.ndarray:
481
- """Get region around mask boundary where hair/fine details are likely"""
482
- try:
483
- # Create dilated and eroded versions of mask
484
- kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (boundary_width, boundary_width))
485
- dilated = cv2.dilate(mask, kernel, iterations=1)
486
- eroded = cv2.erode(mask, kernel, iterations=1)
487
-
488
- # Boundary region is dilated minus eroded
489
- boundary_region = ((dilated > 0) & (eroded == 0)).astype(np.float32)
490
-
491
- return boundary_region
492
-
493
- except Exception as e:
494
- logger.warning(f"Boundary region detection failed: {e}")
495
- return np.ones_like(mask, dtype=np.float32)
496
-
497
- def _apply_temporal_consistency_enhanced(self, frame: np.ndarray, current_mask: np.ndarray, frame_number: int) -> np.ndarray:
498
- """ENHANCED: Apply temporal consistency using optical flow and history"""
499
- try:
500
- if len(self.mask_history) == 0:
501
- return current_mask
502
-
503
- previous_mask = self.mask_history[-1]
504
-
505
- # Method 1: Optical flow-based consistency
506
- if USE_OPTICAL_FLOW_TRACKING and self.optical_flow_data is not None:
507
- try:
508
- flow_corrected_mask = self._apply_optical_flow_consistency(
509
- frame, current_mask, previous_mask
510
- )
511
-
512
- # Blend flow-corrected with current mask
513
- alpha = 0.7 # Weight for current mask
514
- beta = 0.3 # Weight for flow-corrected mask
515
-
516
- blended_mask = cv2.addWeighted(
517
- current_mask.astype(np.float32), alpha,
518
- flow_corrected_mask.astype(np.float32), beta, 0
519
- ).astype(np.uint8)
520
-
521
- self.stats['temporal_corrections'] += 1
522
-
523
- except Exception as e:
524
- logger.debug(f"Optical flow consistency failed: {e}")
525
- self.stats['flow_tracking_failures'] += 1
526
- blended_mask = current_mask
527
- else:
528
- blended_mask = current_mask
529
-
530
- # Method 2: Multi-frame temporal smoothing
531
- if len(self.mask_history) >= 3:
532
- # Use weighted average of recent masks
533
- weights = [0.5, 0.3, 0.2] # Current, previous, before previous
534
- masks_to_blend = [blended_mask] + self.mask_history[-2:]
535
-
536
- temporal_mask = np.zeros_like(blended_mask, dtype=np.float32)
537
- for mask, weight in zip(masks_to_blend, weights):
538
- temporal_mask += mask.astype(np.float32) * weight
539
-
540
- blended_mask = np.clip(temporal_mask, 0, 255).astype(np.uint8)
541
-
542
- # Method 3: Edge-aware temporal filtering
543
- blended_mask = self._temporal_edge_filtering(frame, blended_mask, current_mask)
544
-
545
- return blended_mask
546
-
547
- except Exception as e:
548
- logger.warning(f"Temporal consistency failed: {e}")
549
- return current_mask
550
-
551
- def _apply_optical_flow_consistency(self, current_frame: np.ndarray,
552
- current_mask: np.ndarray, previous_mask: np.ndarray) -> np.ndarray:
553
- """Apply optical flow to warp previous mask to current frame"""
554
- try:
555
- # Convert frames to grayscale for optical flow
556
- current_gray = cv2.cvtColor(current_frame, cv2.COLOR_BGR2GRAY)
557
- previous_gray = self.optical_flow_data
558
-
559
- # Calculate dense optical flow
560
- flow = cv2.calcOpticalFlowPyrLK(previous_gray, current_gray, None, None)
561
-
562
- # Warp previous mask using optical flow
563
- h, w = previous_mask.shape
564
- flow_map = np.zeros((h, w, 2), dtype=np.float32)
565
-
566
- # Create flow field
567
- y_coords, x_coords = np.mgrid[0:h, 0:w]
568
- flow_map[:, :, 0] = x_coords + flow[0] if flow[0] is not None else x_coords
569
- flow_map[:, :, 1] = y_coords + flow[1] if flow[1] is not None else y_coords
570
-
571
- # Warp previous mask
572
- warped_mask = cv2.remap(previous_mask, flow_map, None, cv2.INTER_LINEAR)
573
-
574
- return warped_mask
575
-
576
- except Exception as e:
577
- logger.debug(f"Optical flow warping failed: {e}")
578
- return previous_mask
579
-
580
- def _temporal_edge_filtering(self, frame: np.ndarray, blended_mask: np.ndarray, current_mask: np.ndarray) -> np.ndarray:
581
- """Apply edge-aware temporal filtering"""
582
- try:
583
- # Detect edges in current frame
584
- gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
585
- edges = cv2.Canny(gray, 50, 150)
586
-
587
- # In edge regions, favor the current mask (more responsive)
588
- # In smooth regions, favor the blended mask (more stable)
589
- edge_weight = cv2.GaussianBlur(edges.astype(np.float32), (5, 5), 1.0) / 255.0
590
-
591
- filtered_mask = (current_mask.astype(np.float32) * edge_weight +
592
- blended_mask.astype(np.float32) * (1 - edge_weight))
593
-
594
- return np.clip(filtered_mask, 0, 255).astype(np.uint8)
595
-
596
- except Exception as e:
597
- logger.warning(f"Temporal edge filtering failed: {e}")
598
- return blended_mask
599
-
600
- def _adaptive_mask_refinement(self, frame: np.ndarray, mask: np.ndarray,
601
- frame_number: int, hair_regions: Optional[np.ndarray]) -> np.ndarray:
602
- """ENHANCED: Adaptive mask refinement based on content analysis"""
603
- try:
604
- # Determine refinement strategy based on frame content
605
- refinement_needed = self._assess_refinement_needs(frame, mask, hair_regions)
606
-
607
- if refinement_needed['hair_refinement'] and hair_regions is not None:
608
- # Special handling for hair regions
609
- mask = self._refine_hair_regions(frame, mask, hair_regions)
610
-
611
- if refinement_needed['edge_refinement']:
612
- # Enhanced edge refinement
613
- mask = self._enhanced_edge_refinement(frame, mask)
614
-
615
- if refinement_needed['temporal_refinement']:
616
- # Apply temporal-aware refinement
617
- mask = self._temporal_aware_refinement(frame, mask, frame_number)
618
-
619
- # Standard refinement if needed
620
- if self._should_refine_mask(frame_number):
621
- if self.matanyone_model is not None and self.quality_settings.get('edge_refinement', True):
622
- mask = refine_mask_hq(frame, mask, self.matanyone_model)
623
- else:
624
- mask = self._fallback_mask_refinement_enhanced(mask)
625
-
626
- return mask
627
-
628
- except Exception as e:
629
- logger.warning(f"Adaptive mask refinement failed: {e}")
630
- return self._refine_mask_original(frame, mask, frame_number)
631
-
632
- def _assess_refinement_needs(self, frame: np.ndarray, mask: np.ndarray,
633
- hair_regions: Optional[np.ndarray]) -> Dict[str, bool]:
634
- """Assess what type of refinement is needed for this frame"""
635
- try:
636
- needs = {
637
- 'hair_refinement': False,
638
- 'edge_refinement': False,
639
- 'temporal_refinement': False
640
- }
641
-
642
- # Check if hair refinement is needed
643
- if hair_regions is not None and np.any(hair_regions):
644
- needs['hair_refinement'] = True
645
-
646
- # Check edge quality
647
- edges = cv2.Canny(mask, 50, 150)
648
- edge_density = np.sum(edges > 0) / (mask.shape[0] * mask.shape[1])
649
- if edge_density > 0.1: # High edge density suggests rough boundaries
650
- needs['edge_refinement'] = True
651
-
652
- # Check temporal consistency needs
653
- if len(self.mask_history) > 0:
654
- prev_mask = self.mask_history[-1]
655
- diff = cv2.absdiff(mask, prev_mask)
656
- change_ratio = np.sum(diff > 50) / (mask.shape[0] * mask.shape[1])
657
- if change_ratio > 0.15: # High change suggests temporal inconsistency
658
- needs['temporal_refinement'] = True
659
-
660
- return needs
661
-
662
- except Exception as e:
663
- logger.warning(f"Refinement assessment failed: {e}")
664
- return {'hair_refinement': False, 'edge_refinement': True, 'temporal_refinement': False}
665
-
666
- def _refine_hair_regions(self, frame: np.ndarray, mask: np.ndarray, hair_regions: np.ndarray) -> np.ndarray:
667
- """Special refinement for hair and fine detail regions"""
668
- try:
669
- # Create a more aggressive mask for hair regions
670
- hair_mask = hair_regions > 0
671
-
672
- # Use different thresholding for hair areas
673
- refined_mask = mask.copy()
674
-
675
- # In hair regions, use lower threshold (include more pixels)
676
- hair_area_values = mask[hair_mask]
677
- if len(hair_area_values) > 0:
678
- hair_threshold = max(100, np.percentile(hair_area_values, 25)) # Lower threshold for hair
679
- refined_mask[hair_mask] = np.where(mask[hair_mask] > hair_threshold, 255, 0)
680
-
681
- # Apply morphological closing to connect hair strands
682
- kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2, 2))
683
- refined_mask = cv2.morphologyEx(refined_mask, cv2.MORPH_CLOSE, kernel)
684
-
685
- return refined_mask
686
-
687
- except Exception as e:
688
- logger.warning(f"Hair region refinement failed: {e}")
689
- return mask
690
-
691
- def _enhanced_edge_refinement(self, frame: np.ndarray, mask: np.ndarray) -> np.ndarray:
692
- """Enhanced edge refinement using image gradients"""
693
- try:
694
- # Use bilateral filter to preserve edges while smoothing
695
- refined = cv2.bilateralFilter(mask, 9, 75, 75)
696
-
697
- # Edge-guided smoothing
698
- gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
699
- edges = cv2.Canny(gray, 50, 150)
700
-
701
- # In edge areas, preserve original mask more
702
- edge_weight = cv2.GaussianBlur(edges.astype(np.float32), (3, 3), 1.0) / 255.0
703
- edge_weight = np.clip(edge_weight * 2, 0, 1) # Amplify edge influence
704
-
705
- final_mask = (mask.astype(np.float32) * edge_weight +
706
- refined.astype(np.float32) * (1 - edge_weight))
707
-
708
- return np.clip(final_mask, 0, 255).astype(np.uint8)
709
-
710
- except Exception as e:
711
- logger.warning(f"Enhanced edge refinement failed: {e}")
712
- return mask
713
-
714
- def _temporal_aware_refinement(self, frame: np.ndarray, mask: np.ndarray, frame_number: int) -> np.ndarray:
715
- """Temporal-aware refinement considering motion and stability"""
716
- try:
717
- if len(self.mask_history) == 0:
718
- return mask
719
-
720
- # Calculate motion between frames
721
- if self.optical_flow_data is not None:
722
- current_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
723
- motion_magnitude = cv2.absdiff(current_gray, self.optical_flow_data)
724
- motion_mask = motion_magnitude > 10 # Areas with motion
725
-
726
- # In high-motion areas, trust current mask more
727
- # In low-motion areas, use temporal smoothing
728
- prev_mask = self.mask_history[-1]
729
-
730
- motion_weight = cv2.GaussianBlur(motion_mask.astype(np.float32), (5, 5), 1.0)
731
- motion_weight = np.clip(motion_weight, 0.3, 1.0) # Don't completely ignore temporal info
732
-
733
- temporal_mask = (mask.astype(np.float32) * motion_weight +
734
- prev_mask.astype(np.float32) * (1 - motion_weight))
735
-
736
- return np.clip(temporal_mask, 0, 255).astype(np.uint8)
737
-
738
- return mask
739
-
740
- except Exception as e:
741
- logger.warning(f"Temporal-aware refinement failed: {e}")
742
- return mask
743
-
744
- def _update_mask_history(self, mask: np.ndarray):
745
- """Update mask history for temporal consistency"""
746
- self.mask_history.append(mask.copy())
747
-
748
- # Keep only recent history (limit memory usage)
749
- max_history = 5
750
- if len(self.mask_history) > max_history:
751
- self.mask_history.pop(0)
752
-
753
- def _create_mask_preview_enhanced(self, frame: np.ndarray, mask: np.ndarray,
754
- hair_regions: Optional[np.ndarray]) -> np.ndarray:
755
- """ENHANCED: Create mask visualization with hair regions highlighted"""
756
- try:
757
- # Create colored mask overlay
758
- mask_colored = np.zeros_like(frame)
759
- mask_colored[:, :, 1] = mask # Green channel for person
760
-
761
- # Highlight hair regions in blue if available
762
- if hair_regions is not None:
763
- mask_colored[:, :, 2] = np.maximum(mask_colored[:, :, 2], hair_regions * 255)
764
-
765
- # Blend with original frame
766
- alpha = 0.6
767
- preview = cv2.addWeighted(frame, 1-alpha, mask_colored, alpha, 0)
768
-
769
- return preview
770
-
771
- except Exception as e:
772
- logger.warning(f"Enhanced mask preview creation failed: {e}")
773
- return self._create_mask_preview_original(frame, mask)
774
-
775
- def _replace_background_enhanced(self, frame: np.ndarray, mask: np.ndarray,
776
- background: np.ndarray, hair_regions: Optional[np.ndarray]) -> np.ndarray:
777
- """ENHANCED: Replace background with special handling for hair regions"""
778
- try:
779
- # Standard background replacement
780
- result = replace_background_hq(frame, mask, background)
781
-
782
- # If hair regions detected, apply additional processing
783
- if hair_regions is not None and np.any(hair_regions):
784
- result = self._enhance_hair_compositing(frame, mask, background, hair_regions, result)
785
-
786
- return result
787
-
788
- except Exception as e:
789
- logger.warning(f"Enhanced background replacement failed: {e}")
790
- return replace_background_hq(frame, mask, background)
791
-
792
- def _enhance_hair_compositing(self, frame: np.ndarray, mask: np.ndarray,
793
- background: np.ndarray, hair_regions: np.ndarray,
794
- initial_result: np.ndarray) -> np.ndarray:
795
- """Enhanced compositing specifically for hair regions"""
796
- try:
797
- # In hair regions, use softer alpha blending
798
- hair_mask = hair_regions > 0
799
-
800
- if np.any(hair_mask):
801
- # Create soft alpha for hair regions
802
- hair_alpha = cv2.GaussianBlur((hair_regions * mask / 255.0).astype(np.float32), (3, 3), 1.0)
803
- hair_alpha = np.clip(hair_alpha, 0, 1)
804
-
805
- # Apply softer blending only in hair regions
806
- for c in range(3):
807
- channel_blend = (frame[:, :, c].astype(np.float32) * hair_alpha +
808
- background[:, :, c].astype(np.float32) * (1 - hair_alpha))
809
-
810
- initial_result[:, :, c] = np.where(
811
- hair_mask,
812
- np.clip(channel_blend, 0, 255).astype(np.uint8),
813
- initial_result[:, :, c]
814
- )
815
-
816
- return initial_result
817
-
818
- except Exception as e:
819
- logger.warning(f"Hair compositing enhancement failed: {e}")
820
- return initial_result
821
-
822
- # ============================================================================
823
- # ORIGINAL FUNCTIONS PRESERVED FOR ROLLBACK
824
- # ============================================================================
825
-
826
- def _process_single_frame_original(
827
- self,
828
- frame: np.ndarray,
829
- background: np.ndarray,
830
- frame_number: int,
831
- preview_mask: bool,
832
- preview_greenscreen: bool
833
- ) -> np.ndarray:
834
- """ORIGINAL: Process a single video frame (preserved for rollback)"""
835
-
836
- try:
837
- # Person segmentation
838
- mask = self._segment_person(frame, frame_number)
839
-
840
- # Mask refinement (keyframe-based for performance)
841
- if self._should_refine_mask(frame_number):
842
- refined_mask = self._refine_mask_original(frame, mask, frame_number)
843
- self.last_refined_mask = refined_mask.copy()
844
- else:
845
- # Use temporal consistency with previous refined mask
846
- refined_mask = self._apply_temporal_consistency_original(mask, frame_number)
847
-
848
- # Generate output based on mode
849
- if preview_mask:
850
- return self._create_mask_preview_original(frame, refined_mask)
851
- elif preview_greenscreen:
852
- return self._create_greenscreen_preview(frame, refined_mask)
853
- else:
854
- return self._replace_background(frame, refined_mask, background)
855
-
856
- except Exception as e:
857
- logger.warning(f"Single frame processing failed: {e}")
858
- raise
859
-
860
- def _segment_person(self, frame: np.ndarray, frame_number: int) -> np.ndarray:
861
- """Perform person segmentation"""
862
- try:
863
- mask = segment_person_hq(frame, self.sam2_predictor)
864
-
865
- if mask is None or mask.size == 0:
866
- raise exceptions.SegmentationError(frame_number, "Segmentation returned empty mask")
867
-
868
- # Store current frame for optical flow (if enhanced mode enabled)
869
- if USE_OPTICAL_FLOW_TRACKING:
870
- current_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
871
- self.optical_flow_data = current_gray
872
-
873
- return mask
874
-
875
- except Exception as e:
876
- self.stats['segmentation_errors'] += 1
877
- raise exceptions.SegmentationError(frame_number, f"Segmentation failed: {str(e)}")
878
-
879
- def _segment_person_enhanced(self, frame: np.ndarray, frame_number: int) -> np.ndarray:
880
- """ENHANCED: Perform person segmentation with improvements"""
881
- try:
882
- mask = segment_person_hq(frame, self.sam2_predictor)
883
-
884
- if mask is None or mask.size == 0:
885
- raise exceptions.SegmentationError(frame_number, "Segmentation returned empty mask")
886
-
887
- # Store current frame for optical flow
888
- current_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
889
- self.optical_flow_data = current_gray
890
-
891
- return mask
892
-
893
- except Exception as e:
894
- self.stats['segmentation_errors'] += 1
895
- raise exceptions.SegmentationError(frame_number, f"Enhanced segmentation failed: {str(e)}")
896
-
897
- def _should_refine_mask(self, frame_number: int) -> bool:
898
- """Determine if mask should be refined for this frame"""
899
- # Refine on keyframes or if no previous refined mask exists
900
- return (
901
- frame_number % self.quality_settings['keyframe_interval'] == 0 or
902
- self.last_refined_mask is None or
903
- not self.quality_settings.get('temporal_consistency', True)
904
- )
905
-
906
- def _refine_mask_original(self, frame: np.ndarray, mask: np.ndarray, frame_number: int) -> np.ndarray:
907
- """ORIGINAL: Refine mask using MatAnyone or fallback methods"""
908
- try:
909
- if self.matanyone_model is not None and self.quality_settings.get('edge_refinement', True):
910
- refined_mask = refine_mask_hq(frame, mask, self.matanyone_model)
911
- else:
912
- # Fallback refinement using OpenCV operations
913
- refined_mask = self._fallback_mask_refinement(mask)
914
-
915
- return refined_mask
916
-
917
- except Exception as e:
918
- self.stats['refinement_errors'] += 1
919
- logger.warning(f"Mask refinement failed for frame {frame_number}: {e}")
920
- # Return original mask as fallback
921
- return mask
922
-
923
- def _fallback_mask_refinement(self, mask: np.ndarray) -> np.ndarray:
924
- """ORIGINAL: Fallback mask refinement using basic OpenCV operations"""
925
- try:
926
- # Morphological operations to clean up mask
927
- kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
928
- refined = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
929
- refined = cv2.morphologyEx(refined, cv2.MORPH_OPEN, kernel)
930
-
931
- # Smooth edges
932
- refined = cv2.GaussianBlur(refined, (3, 3), 1.0)
933
-
934
- return refined
935
-
936
- except Exception as e:
937
- logger.warning(f"Fallback mask refinement failed: {e}")
938
- return mask
939
-
940
- def _fallback_mask_refinement_enhanced(self, mask: np.ndarray) -> np.ndarray:
941
- """ENHANCED: Improved fallback mask refinement"""
942
- try:
943
- # More aggressive morphological operations
944
- kernel_small = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2, 2))
945
- kernel_large = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
946
-
947
- # Remove small noise
948
- refined = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel_small)
949
- # Fill gaps
950
- refined = cv2.morphologyEx(refined, cv2.MORPH_CLOSE, kernel_large)
951
-
952
- # Edge smoothing with bilateral filter instead of Gaussian
953
- refined = cv2.bilateralFilter(refined, 9, 75, 75)
954
-
955
- return refined
956
-
957
- except Exception as e:
958
- logger.warning(f"Enhanced fallback mask refinement failed: {e}")
959
- return mask
960
-
961
- def _apply_temporal_consistency_original(self, current_mask: np.ndarray, frame_number: int) -> np.ndarray:
962
- """ORIGINAL: Apply temporal consistency using previous refined mask"""
963
- if self.last_refined_mask is None or not self.quality_settings.get('temporal_consistency', True):
964
- return current_mask
965
-
966
- try:
967
- # Blend current mask with previous refined mask
968
- alpha = 0.7 # Weight for current mask
969
- beta = 0.3 # Weight for previous mask
970
-
971
- # Ensure masks have same shape
972
- if current_mask.shape != self.last_refined_mask.shape:
973
- last_mask = cv2.resize(self.last_refined_mask,
974
- (current_mask.shape[1], current_mask.shape[0]))
975
- else:
976
- last_mask = self.last_refined_mask
977
-
978
- # Weighted blend
979
- blended_mask = cv2.addWeighted(current_mask, alpha, last_mask, beta, 0)
980
-
981
- # Apply slight smoothing for temporal stability
982
- blended_mask = cv2.GaussianBlur(blended_mask, (3, 3), 0.5)
983
-
984
- return blended_mask
985
-
986
- except Exception as e:
987
- logger.warning(f"Temporal consistency application failed: {e}")
988
- return current_mask
989
-
990
- def _create_mask_preview_original(self, frame: np.ndarray, mask: np.ndarray) -> np.ndarray:
991
- """ORIGINAL: Create mask visualization preview"""
992
- try:
993
- # Create colored mask overlay
994
- mask_colored = np.zeros_like(frame)
995
- mask_colored[:, :, 1] = mask # Green channel for person
996
-
997
- # Blend with original frame
998
- alpha = 0.6
999
- preview = cv2.addWeighted(frame, 1-alpha, mask_colored, alpha, 0)
1000
-
1001
- return preview
1002
-
1003
- except Exception as e:
1004
- logger.warning(f"Mask preview creation failed: {e}")
1005
- return frame
1006
-
1007
- def _create_greenscreen_preview(self, frame: np.ndarray, mask: np.ndarray) -> np.ndarray:
1008
- """Create green screen preview"""
1009
- try:
1010
- # Create pure green background
1011
- green_bg = np.zeros_like(frame)
1012
- green_bg[:, :] = [0, 255, 0] # Pure green in BGR
1013
-
1014
- # Apply mask
1015
- mask_3ch = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) if len(mask.shape) == 2 else mask
1016
- mask_norm = mask_3ch.astype(np.float32) / 255.0
1017
-
1018
- result = (frame * mask_norm + green_bg * (1 - mask_norm)).astype(np.uint8)
1019
-
1020
- return result
1021
-
1022
- except Exception as e:
1023
- logger.warning(f"Greenscreen preview creation failed: {e}")
1024
- return frame
1025
-
1026
- def _replace_background(self, frame: np.ndarray, mask: np.ndarray, background: np.ndarray) -> np.ndarray:
1027
- """Replace background using the refined mask"""
1028
- try:
1029
- result = replace_background_hq(frame, mask, background)
1030
- return result
1031
-
1032
- except Exception as e:
1033
- logger.warning(f"Background replacement failed: {e}")
1034
- return frame
1035
-
1036
- def prepare_background(
1037
- self,
1038
- background_choice: str,
1039
- custom_background_path: Optional[str],
1040
- width: int,
1041
- height: int
1042
- ) -> Optional[np.ndarray]:
1043
- """Prepare background image for processing (unchanged)"""
1044
- try:
1045
- if background_choice == "custom" and custom_background_path:
1046
- if not os.path.exists(custom_background_path):
1047
- raise exceptions.BackgroundProcessingError("custom", f"File not found: {custom_background_path}")
1048
-
1049
- background = cv2.imread(custom_background_path)
1050
- if background is None:
1051
- raise exceptions.BackgroundProcessingError("custom", "Could not read custom background image")
1052
-
1053
- logger.info(f"Loaded custom background: {custom_background_path}")
1054
-
1055
- else:
1056
- # Use professional background
1057
- if background_choice not in PROFESSIONAL_BACKGROUNDS:
1058
- raise exceptions.BackgroundProcessingError(background_choice, "Unknown professional background")
1059
-
1060
- bg_config = PROFESSIONAL_BACKGROUNDS[background_choice]
1061
- background = create_professional_background(bg_config, width, height)
1062
-
1063
- logger.info(f"Generated professional background: {background_choice}")
1064
-
1065
- # Resize to match video dimensions
1066
- if background.shape[:2] != (height, width):
1067
- background = cv2.resize(background, (width, height), interpolation=cv2.INTER_LANCZOS4)
1068
-
1069
- # Validate background
1070
- if background is None or background.size == 0:
1071
- raise exceptions.BackgroundProcessingError(background_choice, "Background image is empty")
1072
-
1073
- return background
1074
-
1075
- except Exception as e:
1076
- if isinstance(e, exceptions.BackgroundProcessingError):
1077
- logger.error(str(e))
1078
- return None
1079
- else:
1080
- logger.error(f"Unexpected error preparing background: {e}")
1081
- return None
1082
-
1083
- def _update_processing_stats(self, video_info: Dict[str, Any],
1084
- processing_time: float, result: Dict[str, Any]):
1085
- """Update processing statistics"""
1086
- self.stats['videos_processed'] += 1
1087
- self.stats['total_frames_processed'] += result['successful_frames']
1088
- self.stats['total_processing_time'] += processing_time
1089
- self.stats['successful_frames'] += result['successful_frames']
1090
- self.stats['failed_frames'] += result['failed_frames']
1091
-
1092
- # Calculate average FPS across all processing
1093
- if self.stats['total_processing_time'] > 0:
1094
- self.stats['average_fps'] = self.stats['total_frames_processed'] / self.stats['total_processing_time']
1095
-
1096
- def get_processing_capabilities(self) -> Dict[str, Any]:
1097
- """Get current processing capabilities"""
1098
- capabilities = {
1099
- 'sam2_available': self.sam2_predictor is not None,
1100
- 'matanyone_available': self.matanyone_model is not None,
1101
- 'quality_preset': self.config.quality_preset,
1102
- 'supports_temporal_consistency': self.quality_settings.get('temporal_consistency', False),
1103
- 'supports_edge_refinement': self.quality_settings.get('edge_refinement', False),
1104
- 'keyframe_interval': self.quality_settings['keyframe_interval'],
1105
- 'max_resolution': self.config.get_resolution_limits(),
1106
- 'supported_formats': ['.mp4', '.avi', '.mov', '.mkv'],
1107
- 'memory_limit_gb': self.memory_manager.memory_limit_gb
1108
- }
1109
-
1110
- # Add enhanced capabilities
1111
- if USE_TEMPORAL_ENHANCEMENT:
1112
- capabilities.update({
1113
- 'temporal_enhancement': True,
1114
- 'hair_detection': USE_HAIR_DETECTION,
1115
- 'optical_flow_tracking': USE_OPTICAL_FLOW_TRACKING,
1116
- 'adaptive_refinement': USE_ADAPTIVE_REFINEMENT
1117
- })
1118
-
1119
- return capabilities
1120
-
1121
- def get_status(self) -> Dict[str, Any]:
1122
- """Get current processor status"""
1123
- status = {
1124
- 'processing_active': self.processing_active,
1125
- 'models_available': {
1126
- 'sam2': self.sam2_predictor is not None,
1127
- 'matanyone': self.matanyone_model is not None
1128
- },
1129
- 'quality_settings': self.quality_settings,
1130
- 'statistics': self.stats.copy(),
1131
- 'cache_size': len(self.frame_cache),
1132
- 'memory_usage': self.memory_manager.get_memory_usage(),
1133
- 'capabilities': self.get_processing_capabilities()
1134
- }
1135
-
1136
- # Add enhanced status
1137
- if USE_TEMPORAL_ENHANCEMENT:
1138
- status.update({
1139
- 'mask_history_length': len(self.mask_history),
1140
- 'hair_cache_size': len(self.hair_regions_cache),
1141
- 'optical_flow_active': self.optical_flow_data is not None
1142
- })
1143
-
1144
- return status
1145
-
1146
- def optimize_for_video(self, video_info: Dict[str, Any]) -> Dict[str, Any]:
1147
- """Optimize settings for specific video characteristics"""
1148
- optimizations = {
1149
- 'original_settings': self.quality_settings.copy(),
1150
- 'optimizations_applied': []
1151
- }
1152
-
1153
- try:
1154
- # High resolution video optimizations
1155
- if video_info['width'] * video_info['height'] > 1920 * 1080:
1156
- if self.quality_settings['keyframe_interval'] < 10:
1157
- self.quality_settings['keyframe_interval'] = 10
1158
- optimizations['optimizations_applied'].append('increased_keyframe_interval_for_high_res')
1159
-
1160
- # Long video optimizations
1161
- if video_info['duration'] > 300: # 5 minutes
1162
- if self.config.memory_cleanup_interval > 20:
1163
- self.config.memory_cleanup_interval = 20
1164
- optimizations['optimizations_applied'].append('increased_memory_cleanup_frequency')
1165
-
1166
- # Low FPS video optimizations
1167
- if video_info['fps'] < 15:
1168
- self.quality_settings['temporal_consistency'] = False
1169
- optimizations['optimizations_applied'].append('disabled_temporal_consistency_for_low_fps')
1170
-
1171
- # Memory-constrained optimizations
1172
- memory_usage = self.memory_manager.get_memory_usage()
1173
- memory_pressure = self.memory_manager.check_memory_pressure()
1174
-
1175
- if memory_pressure['under_pressure']:
1176
- self.quality_settings['edge_refinement'] = False
1177
- self.quality_settings['keyframe_interval'] = max(self.quality_settings['keyframe_interval'], 15)
1178
- optimizations['optimizations_applied'].extend([
1179
- 'disabled_edge_refinement_for_memory',
1180
- 'increased_keyframe_interval_for_memory'
1181
- ])
1182
-
1183
- optimizations['final_settings'] = self.quality_settings.copy()
1184
-
1185
- if optimizations['optimizations_applied']:
1186
- logger.info(f"Applied video optimizations: {optimizations['optimizations_applied']}")
1187
-
1188
- return optimizations
1189
-
1190
- except Exception as e:
1191
- logger.warning(f"Video optimization failed: {e}")
1192
- return optimizations
1193
-
1194
- def reset_cache(self):
1195
- """Reset frame cache and temporal state"""
1196
- self.frame_cache.clear()
1197
- self.last_refined_mask = None
1198
- self.stats['cache_hits'] = 0
1199
- self._reset_temporal_state()
1200
- logger.debug("Frame cache and temporal state reset")
1201
-
1202
- def cleanup(self):
1203
- """Clean up processor resources"""
1204
- try:
1205
- self.reset_cache()
1206
- self.processing_active = False
1207
- logger.info("CoreVideoProcessor cleanup completed")
1208
- except Exception as e:
1209
- logger.warning(f"Error during cleanup: {e}")