source/utils_preprocess.py

# This file contains function for video or image collection preprocessing.
# For video we do the preprocessing and select k sharpest frames.
# Afterwards scene is constructed 
import cv2
import numpy as np
from tqdm import tqdm
import pycolmap
import os
import time
import tempfile
from moviepy import VideoFileClip
from matplotlib import pyplot as plt
from PIL import Image
import cv2
from tqdm import tqdm

WORKDIR = "../outputs/"


def get_rotation_moviepy(video_path):
    clip = VideoFileClip(video_path)
    rotation = 0

    try:
        displaymatrix = clip.reader.infos['inputs'][0]['streams'][2]['metadata'].get('displaymatrix', '')
        if 'rotation of' in displaymatrix:
            angle = float(displaymatrix.strip().split('rotation of')[-1].split('degrees')[0])
            rotation = int(angle) % 360
            
    except Exception as e:
        print(f"No displaymatrix rotation found: {e}")

    clip.reader.close()
    #if clip.audio:
    #    clip.audio.reader.close_proc()

    return rotation

def resize_max_side(frame, max_size):
    h, w = frame.shape[:2]
    scale = max_size / max(h, w)
    if scale < 1:
        frame = cv2.resize(frame, (int(w * scale), int(h * scale)))
    return frame

def read_video_frames(video_input, k=1, max_size=1024):
    """
    Extracts every k-th frame from a video or list of images, resizes to max size, and returns frames as list.

    Parameters:
        video_input (str, file-like, or list): Path to video file, file-like object, or list of image files.
        k (int): Interval for frame extraction (every k-th frame).
        max_size (int): Maximum size for width or height after resizing.

    Returns:
        frames (list): List of resized frames (numpy arrays).
    """
    # Handle list of image files (not single video in a list)
    if isinstance(video_input, list):
        # If it's a single video in a list, treat it as video
        if len(video_input) == 1 and video_input[0].name.endswith(('.mp4', '.avi', '.mov')):
            video_input = video_input[0]  # unwrap single video file
        else:
            # Treat as list of images
            frames = []
            for img_file in video_input:
                img = Image.open(img_file.name).convert("RGB")
                img.thumbnail((max_size, max_size))
                frames.append(np.array(img)[...,::-1])
            return frames

    # Handle file-like or path
    if hasattr(video_input, 'name'):
        video_path = video_input.name
    elif isinstance(video_input, (str, os.PathLike)):
        video_path = str(video_input)
    else:
        raise ValueError("Unsupported video input type. Must be a filepath, file-like object, or list of images.")

    
    cap = cv2.VideoCapture(video_path)
    if not cap.isOpened():
        raise ValueError(f"Error: Could not open video {video_path}.")

    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    frame_count = 0
    frames = []

    with tqdm(total=total_frames // k, desc="Processing Video", unit="frame") as pbar:
        while True:
            ret, frame = cap.read()
            if not ret:
                break
            if frame_count % k == 0:
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                h, w = frame.shape[:2]
                scale = max(h, w) / max_size
                if scale > 1:
                    frame = cv2.resize(frame, (int(w / scale), int(h / scale)))
                frames.append(frame[...,[2,1,0]])
                pbar.update(1)
            frame_count += 1

    cap.release()
    return frames

def resize_max_side(frame, max_size):
    """
    Resizes the frame so that its largest side equals max_size, maintaining aspect ratio.
    """
    height, width = frame.shape[:2]
    max_dim = max(height, width)
    
    if max_dim <= max_size:
        return frame  # No need to resize

    scale = max_size / max_dim
    new_width = int(width * scale)
    new_height = int(height * scale)

    resized_frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA)
    return resized_frame



def variance_of_laplacian(image):
	# compute the Laplacian of the image and then return the focus
	# measure, which is simply the variance of the Laplacian
	return cv2.Laplacian(image, cv2.CV_64F).var()
    
def process_all_frames(IMG_FOLDER = '/scratch/datasets/hq_data/night2_all_frames',
                       to_visualize=False,
                       save_images=True):
    dict_scores = {}
    for idx, img_name in tqdm(enumerate(sorted([x for x in os.listdir(IMG_FOLDER) if '.png' in x]))):
        
        img = cv2.imread(os.path.join(IMG_FOLDER, img_name))#[250:, 100:]
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        fm = variance_of_laplacian(gray) + \
                variance_of_laplacian(cv2.resize(gray, (0,0), fx=0.75, fy=0.75)) + \
                variance_of_laplacian(cv2.resize(gray, (0,0), fx=0.5, fy=0.5)) + \
                variance_of_laplacian(cv2.resize(gray, (0,0), fx=0.25, fy=0.25))
        if to_visualize:
            plt.figure()
            plt.title(f"Laplacian score: {fm:.2f}")
            plt.imshow(img[..., [2,1,0]])
            plt.show()
        dict_scores[idx] = {"idx" : idx, 
                            "img_name" : img_name,
                            "score" : fm}
        if save_images:
            dict_scores[idx]["img"] = img
        
    return dict_scores

def select_optimal_frames(scores, k):
    """
    Selects a minimal subset of frames while ensuring no gaps exceed k.

    Args:
        scores (list of float): List of scores where index represents frame number.
        k (int): Maximum allowed gap between selected frames.

    Returns:
        list of int: Indices of selected frames.
    """
    n = len(scores)
    selected = [0, n-1]
    i = 0  # Start at the first frame

    while i < n:
        # Find the best frame to select within the next k frames
        best_idx = max(range(i, min(i + k + 1, n)), key=lambda x: scores[x], default=None)

        if best_idx is None:
            break  # No more frames left

        selected.append(best_idx)
        i = best_idx + k + 1  # Move forward, ensuring gaps stay within k

    return sorted(selected)


def variance_of_laplacian(image):
    """
    Compute the variance of Laplacian as a focus measure.
    """
    return cv2.Laplacian(image, cv2.CV_64F).var()

def preprocess_frames(frames, verbose=False):
    """
    Compute sharpness scores for a list of frames using multi-scale Laplacian variance.

    Args:
        frames (list of np.ndarray): List of frames (BGR images).
        verbose (bool): If True, print scores.

    Returns:
        list of float: Sharpness scores for each frame.
    """
    scores = []

    for idx, frame in enumerate(tqdm(frames, desc="Scoring frames")):
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

        fm = (
            variance_of_laplacian(gray) +
            variance_of_laplacian(cv2.resize(gray, (0, 0), fx=0.75, fy=0.75)) +
            variance_of_laplacian(cv2.resize(gray, (0, 0), fx=0.5, fy=0.5)) +
            variance_of_laplacian(cv2.resize(gray, (0, 0), fx=0.25, fy=0.25))
        )
        
        if verbose:
            print(f"Frame {idx}: Sharpness Score = {fm:.2f}")

        scores.append(fm)

    return scores

def select_optimal_frames(scores, k):
    """
    Selects k frames by splitting into k segments and picking the sharpest frame from each.

    Args:
        scores (list of float): List of sharpness scores.
        k (int): Number of frames to select.

    Returns:
        list of int: Indices of selected frames.  
    """
    n = len(scores)
    selected_indices = []
    segment_size = n // k

    for i in range(k):
        start = i * segment_size
        end = (i + 1) * segment_size if i < k - 1 else n  # Last chunk may be larger
        segment_scores = scores[start:end]
        
        if len(segment_scores) == 0:
            continue  # Safety check if some segment is empty
        
        best_in_segment = start + np.argmax(segment_scores)
        selected_indices.append(best_in_segment)

    return sorted(selected_indices)

def save_frames_to_scene_dir(frames, scene_dir):
    """
    Saves a list of frames into the target scene directory under 'images/' subfolder.

    Args:
        frames (list of np.ndarray): List of frames (BGR images) to save.
        scene_dir (str): Target path where 'images/' subfolder will be created.
    """
    images_dir = os.path.join(scene_dir, "images")
    os.makedirs(images_dir, exist_ok=True)

    for idx, frame in enumerate(frames):
        filename = os.path.join(images_dir, f"{idx:08d}.png")  # 00000000.png, 00000001.png, etc.
        cv2.imwrite(filename, frame)

    print(f"Saved {len(frames)} frames to {images_dir}")


def run_colmap_on_scene(scene_dir):
    """
    Runs feature extraction, matching, and mapping on all images inside scene_dir/images using pycolmap.

    Args:
        scene_dir (str): Path to scene directory containing 'images' folder.
    
    TODO: if the function hasn't managed to match all the frames either increase image size,
    increase number of features or just remove those frames from the folder scene_dir/images
    """
    start_time = time.time()
    print(f"Running COLMAP pipeline on all images inside {scene_dir}")

    # Setup paths
    database_path = os.path.join(scene_dir, "database.db")
    sparse_path = os.path.join(scene_dir, "sparse")
    image_dir = os.path.join(scene_dir, "images")
    
    # Make sure output directories exist
    os.makedirs(sparse_path, exist_ok=True)

    # Step 1: Feature Extraction
    pycolmap.extract_features(
        database_path,
        image_dir,
        sift_options={
            "max_num_features": 512 * 2,
            "max_image_size": 512 * 1,
        }
    )
    print(f"Finished feature extraction in {(time.time() - start_time):.2f}s.")

    # Step 2: Feature Matching
    pycolmap.match_exhaustive(database_path)
    print(f"Finished feature matching in {(time.time() - start_time):.2f}s.")

    # Step 3: Mapping
    pipeline_options = pycolmap.IncrementalPipelineOptions()
    pipeline_options.min_num_matches = 15
    pipeline_options.multiple_models = True
    pipeline_options.max_num_models = 50
    pipeline_options.max_model_overlap = 20
    pipeline_options.min_model_size = 10
    pipeline_options.extract_colors = True
    pipeline_options.num_threads = 8
    pipeline_options.mapper.init_min_num_inliers = 30
    pipeline_options.mapper.init_max_error = 8.0
    pipeline_options.mapper.init_min_tri_angle = 5.0

    reconstruction = pycolmap.incremental_mapping(
        database_path=database_path,
        image_path=image_dir,
        output_path=sparse_path,
        options=pipeline_options,
    )
    print(f"Finished incremental mapping in {(time.time() - start_time):.2f}s.")

    # Step 4: Post-process Cameras to SIMPLE_PINHOLE
    recon_path = os.path.join(sparse_path, "0")
    reconstruction = pycolmap.Reconstruction(recon_path)

    for cam in reconstruction.cameras.values():
        cam.model = 'SIMPLE_PINHOLE'
        cam.params = cam.params[:3]  # Keep only [f, cx, cy]

    reconstruction.write(recon_path)

    print(f"Total pipeline time: {(time.time() - start_time):.2f}s.")