app.py

import torch
from PIL import Image, ImageDraw, ImageFont
from transformers import GroundingDinoProcessor
from modeling_grounding_dino import GroundingDinoForObjectDetection

from PIL import Image, ImageDraw, ImageFont
from itertools import cycle

import gradio as gr

import spaces

# Load model and processor
model_id = "fushh7/llmdet_swin_large_hf"
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

print(f"[INFO] Using device: {DEVICE}")
print(f"[INFO] Loading model from {model_id}...")

processor = GroundingDinoProcessor.from_pretrained(model_id)
model = GroundingDinoForObjectDetection.from_pretrained(model_id).to(DEVICE)
model.eval();

print("[INFO] Model loaded successfully.")

# Pre-defined palette (extend or tweak as you like)
BOX_COLORS = [
    "deepskyblue", "red", "lime", "dodgerblue",
    "cyan", "magenta", "yellow",
    "orange", "chartreuse"
]

def draw_boxes(image, boxes, labels, scores, colors=BOX_COLORS, font_path="arial.ttf", font_size=16):
    """
    Draw bounding boxes and labels on a PIL Image.

    :param image: PIL Image object
    :param boxes: Iterable of [x_min, y_min, x_max, y_max]
    :param labels: Iterable of label strings
    :param scores: Iterable of scalar confidences (0-1)
    :param colors: List/tuple of colour names or RGB tuples
    :param font_path: Path to a TTF font for labels
    :param font_size: Int size of font to use, default 16
    :return: PIL Image with drawn boxes
    """
    # Ensure we can iterate colours indefinitely
    colour_cycle = cycle(colors)
    draw = ImageDraw.Draw(image)

    # Pick a font (fallback to default if missing)
    try:
        font = ImageFont.truetype(font_path, size=font_size)
    except IOError:
        font = ImageFont.load_default(size=font_size)

    # Assign a consistent colour per label (optional)
    label_to_colour = {}

    for box, label, score in zip(boxes, labels, scores):
        # Reuse colour if label seen before, else take next from cycle
        colour = label_to_colour.setdefault(label, next(colour_cycle))

        x_min, y_min, x_max, y_max = map(int, box)

        # Draw rectangle
        draw.rectangle([x_min, y_min, x_max, y_max], outline=colour, width=2)

        # Compose text
        text = f"{label} ({score:.3f})"
        text_size = draw.textbbox((0, 0), text, font=font)[2:]

        # Draw text background for legibility
        bg_coords = [x_min, y_min - text_size[1] - 4,
                     x_min + text_size[0] + 4, y_min]
        draw.rectangle(bg_coords, fill=colour)

        # Draw text
        draw.text((x_min + 2, y_min - text_size[1] - 2),
                  text, fill="black", font=font)

    return image

def resize_image_max_dimension(image, max_size=1024):
    """
    Resize an image so that the longest side is at most max_size pixels,
    while maintaining the aspect ratio.
    
    :param image: PIL Image object
    :param max_size: Maximum dimension in pixels (default: 1024)
    :return: PIL Image object (resized)
    """
    width, height = image.size
    
    # Check if resizing is needed
    if max(width, height) <= max_size:
        return image
    
    # Calculate new dimensions maintaining aspect ratio
    ratio = max_size / max(width, height)
    new_width = int(width * ratio)
    new_height = int(height * ratio)
    
    # Resize the image using high-quality resampling
    return image.resize((new_width, new_height), Image.Resampling.LANCZOS)

@spaces.GPU(duration=120)
def detect_and_draw(
    img: Image.Image,
    text_query: str,
    box_threshold: float = 0.4,
    text_threshold: float = 0.3
) -> Image.Image:
    """
    Detect objects described in `text_query`, draw boxes, return the image.
    Note: `text_query` must be lowercase and each concept ends with a dot
          (e.g. 'a cat. a remote control.')
    """

    # Make sure text is lowered
    text_query = text_query.lower()

    # If the image size is too large, we make it smaller
    img = resize_image_max_dimension(img, max_size=1024)

    # Preprocess the image
    inputs = processor(images=img, text=text_query, return_tensors="pt").to(DEVICE)

    with torch.no_grad():
        outputs = model(**inputs)

    results = processor.post_process_grounded_object_detection(
        outputs,
        inputs.input_ids,
        box_threshold=box_threshold,
        text_threshold=text_threshold,
        target_sizes=[img.size[::-1]]
    )[0]

    img_out = img.copy()
    img_out = draw_boxes(
        img_out,
        boxes  = results["boxes"].cpu().numpy(),
        labels = results.get("text_labels", results.get("labels", [])),
        scores = results["scores"]
    )
    return img_out

# Create example list
examples = [
    ["examples/IMG_8920.jpeg", "bin. water bottle. hand. shoe.", 0.4, 0.3],
    ["examples/IMG_9435.jpeg", "lettuce. orange slices (group). eggs (group). cheese (group). red cabbage. pear slices (group).", 0.4, 0.3],
]

# Create Gradio demo
demo = gr.Interface(
    fn      = detect_and_draw,
    inputs  = [
        gr.Image(type="pil", label="Image"),
        gr.Textbox(value="",
                   label="Text Query (lowercase, end each with '.', for example 'a bird. a tree.')"),
        gr.Slider(0.0, 1.0, 0.4, 0.05, label="Box Threshold"),
        gr.Slider(0.0, 1.0, 0.3, 0.05, label="Text Threshold")
    ],
    outputs = gr.Image(type="pil", label="Detections"),
    title   = "LLMDet Demo: Open-Vocabulary Grounded Object Detection",
    description = f"""Upload an image, enter text queries, and adjust thresholds to see detections.
    
    Adapted from LLMDet GitHub repo [Hugging Face demo](https://github.com/iSEE-Laboratory/LLMDet/tree/main/hf_model).
    
    This space uses: {model_id}
    
    See original:
    
    * [LLMDet GitHub](https://github.com/iSEE-Laboratory/LLMDet/tree/main?tab=readme-ov-file)
    * [LLMDet Paper](https://arxiv.org/abs/2501.18954) - LLMDet: Learning Strong Open-Vocabulary Object Detectors under the Supervision of Large Language Models
    * LLMDet model checkpoints:
        * [Tiny](https://huggingface.co/fushh7/llmdet_swin_tiny_hf) (173M params) 
        * [Base](https://huggingface.co/fushh7/llmdet_swin_base_hf) (233M params)
        * [Large](https://huggingface.co/fushh7/llmdet_swin_large_hf) (344M params)
    """,
    examples = examples,
    cache_examples = True,
)

demo.launch()