Upload app.py

app.py

@@ -5,7 +5,7 @@ import base64
 from PIL import Image, ImageDraw, ImageFont
 import io
 
-def process_with_openrouter(image, prompt, api_key, model="openai/gpt-5-chat", temperature=0.5):
+def process_with_openrouter(image, prompt, api_key, model="qwen/qwen2.5-vl-32b-instruct", temperature=0.5):
     """Process image with OpenRouter API for object detection"""
     if not api_key:
         return "Please enter your OpenRouter API key", "error"
@@ -64,7 +64,7 @@ def process_with_openrouter(image, prompt, api_key, model="openai/gpt-5-chat", t
         return f"Error processing request: {str(e)}", "error"
 
 def draw_bounding_boxes(image, detections):
-    """Draw bounding boxes with detailed labels on the image"""
+    """Draw bounding boxes with class names only, same color per class"""
     if not detections or len(detections) == 0:
         return image
     
@@ -72,13 +72,26 @@ def draw_bounding_boxes(image, detections):
     draw = ImageDraw.Draw(annotated_image)
     
     try:
-        font = ImageFont.truetype("/System/Library/Fonts/Arial.ttf", 14)
-        small_font = ImageFont.truetype("/System/Library/Fonts/Arial.ttf", 12)
+        font = ImageFont.truetype("/System/Library/Fonts/Arial.ttf", 16)
     except:
         font = ImageFont.load_default()
-        small_font = ImageFont.load_default()
     
-    colors = ["#FF0000", "#00FF00", "#0000FF", "#FFFF00", "#FF00FF", "#00FFFF", "#FFA500", "#800080"]
+    # Predefined colors for different classes
+    class_colors = {
+        "Class I": "#FF0000",    # Red
+        "Class II": "#00FF00",   # Green
+        "Class III": "#0000FF",  # Blue
+        "Class IV": "#FFFF00",   # Yellow
+        "Class V": "#FF00FF",    # Magenta
+        "Class VI": "#00FFFF",   # Cyan
+        "Class VII": "#FFA500",  # Orange
+        "Class VIII": "#800080", # Purple
+        "Class IX": "#008000",   # Dark Green
+        "Class X": "#FF1493",    # Deep Pink
+    }
+    
+    # Fallback colors if more than 10 classes
+    fallback_colors = ["#8B4513", "#2F4F4F", "#DC143C", "#00CED1", "#FF4500", "#DA70D6", "#32CD32", "#FF6347"]
     
     for i, detection in enumerate(detections):
         if all(key in detection for key in ['x', 'y', 'width', 'height']):
@@ -87,12 +100,8 @@ def draw_bounding_boxes(image, detections):
             width = detection['width'] * image.width
             height = detection['height'] * image.height
             
-            # Get detection information
-            label = detection.get('label', f'Detection {i+1}')
-            class_name = detection.get('class', 'unknown')
-            details = detection.get('details', '')
-            criteria_match = detection.get('criteria_match', '')
-            confidence = detection.get('confidence', 1.0)
+            # Get class name - this is what we'll display
+            class_name = detection.get('class', f'Class {i+1}')
             
             x1, y1 = int(x), int(y)
             x2, y2 = int(x + width), int(y + height)
@@ -102,114 +111,114 @@ def draw_bounding_boxes(image, detections):
             x2 = max(0, min(x2, image.width))
             y2 = max(0, min(y2, image.height))
             
-            color = colors[i % len(colors)]
+            # Get consistent color for this class
+            if class_name in class_colors:
+                color = class_colors[class_name]
+            else:
+                # Use hash of class name to get consistent color
+                color_index = hash(class_name) % len(fallback_colors)
+                color = fallback_colors[color_index]
             
-            # Draw bounding box with thicker line for better visibility
+            # Draw bounding box
             draw.rectangle([x1, y1, x2, y2], outline=color, width=4)
             
-            # Create multi-line label with detailed information
-            display_lines = []
-            display_lines.append(f"{class_name} ({confidence:.2f})")
-            
-            if details:
-                # Truncate details if too long
-                details_short = details[:40] + "..." if len(details) > 40 else details
-                display_lines.append(details_short)
-            
-            if criteria_match:
-                display_lines.append(f"Criteria: {criteria_match}")
-            
-            # Calculate total label size
-            max_width = 0
-            total_height = 0
-            line_heights = []
-            
-            for line in display_lines:
-                text_bbox = draw.textbbox((0, 0), line, font=small_font)
-                line_width = text_bbox[2] - text_bbox[0]
-                line_height = text_bbox[3] - text_bbox[1]
-                max_width = max(max_width, line_width)
-                total_height += line_height + 2
-                line_heights.append(line_height)
+            # Calculate label size
+            text_bbox = draw.textbbox((0, 0), class_name, font=font)
+            text_width = text_bbox[2] - text_bbox[0]
+            text_height = text_bbox[3] - text_bbox[1]
             
             # Position label above the box, or below if no space above
-            if y1 - total_height - 4 >= 0:
-                label_y = y1 - total_height - 4
+            if y1 - text_height - 6 >= 0:
+                label_y = y1 - text_height - 6
             else:
-                label_y = y2 + 2
+                label_y = y2 + 4
             
             label_x = x1
             
             # Ensure label stays within image bounds
-            if label_x + max_width > image.width:
-                label_x = image.width - max_width - 4
+            if label_x + text_width + 4 > image.width:
+                label_x = image.width - text_width - 4
             
             # Draw label background
             draw.rectangle(
-                [label_x - 2, label_y, label_x + max_width + 4, label_y + total_height + 2],
+                [label_x - 2, label_y - 2, label_x + text_width + 2, label_y + text_height + 2],
                 fill=color,
                 outline=color
             )
             
-            # Draw each line of text
-            current_y = label_y + 2
-            for j, line in enumerate(display_lines):
-                draw.text((label_x + 2, current_y), line, fill="white", font=small_font)
-                current_y += line_heights[j] + 2
+            # Draw class name
+            draw.text((label_x, label_y), class_name, fill="white", font=font)
     
     return annotated_image
 
-def create_detection_prompt(detailed_classes, confidence_threshold=0.5, detection_mode="specific"):
-    """Create a detection prompt for detailed class specifications with different modes"""
-    if isinstance(detailed_classes, str):
-        detailed_classes = [cls.strip() for cls in detailed_classes.split('\n') if cls.strip()]
+def create_detection_prompt(class_descriptions, confidence_threshold=0.5, detection_mode="specific"):
+    """Create a detection prompt for class descriptions with condition checking"""
+    if isinstance(class_descriptions, str):
+        class_descriptions = [cls.strip() for cls in class_descriptions.split('\n') if cls.strip()]
     
-    # Build detailed detection instructions
+    # Build detection instructions
     if detection_mode == "specific":
-        condition_text = "ONLY detect objects that match these specific detailed criteria. Ignore all other objects:"
+        condition_text = "ONLY detect objects that match these class descriptions and their conditions. Ignore all other objects:"
     elif detection_mode == "include":
-        condition_text = "Detect objects matching these detailed criteria AND any other objects you can identify:"
+        condition_text = "Detect objects matching these class descriptions AND any other objects you can identify:"
     else:  # "exclude"
-        condition_text = "Detect all objects EXCEPT those matching these detailed criteria. Avoid detecting:"
+        condition_text = "Detect all objects EXCEPT those matching these class descriptions. Avoid detecting:"
     
-    # Format each detailed class specification
-    detailed_specs = []
-    for i, spec in enumerate(detailed_classes, 1):
-        detailed_specs.append(f"{i}. {spec}")
+    # Format each class description
+    class_specs = []
+    for i, description in enumerate(class_descriptions, 1):
+        # Parse class name and description if formatted as "Class Name: description"
+        if ':' in description:
+            class_name, class_desc = description.split(':', 1)
+            class_name = class_name.strip()
+            class_desc = class_desc.strip()
+            class_specs.append(f"Class {i} ({class_name}): {class_desc}")
+        else:
+            class_specs.append(f"Class {i}: {description}")
     
-    classes_text = "\n".join(detailed_specs) if detailed_specs else "No specific criteria provided"
+    classes_text = "\n".join(class_specs) if class_specs else "No class descriptions provided"
     
     prompt = f"""{condition_text}
 
 {classes_text}
 
 Detection Instructions:
-- Carefully analyze each object against the detailed specifications above
+- Analyze each object against the class descriptions above
+- Check if objects meet the specified conditions for each class
 - Only include detections with confidence above {confidence_threshold}
-- For each detection, provide specific measurements, characteristics, or details when possible
-- Be precise about the criteria matching (e.g., actual crack width, size measurements, specific conditions)
+- Assign objects to the most appropriate class based on the descriptions
+
+SCALE/RULER DETECTION FOR CRACK MEASUREMENT:
+- First look for scales, rulers, measurement tools, or reference objects in the image
+- If found, identify the scale markings and determine the measurement reference
+- Use the scale to calculate actual crack widths in millimeters or appropriate units
+- For crack classifications, measure crack width using the identified scale
+- Include actual measurements in your analysis (e.g., "2.5mm crack width based on ruler scale")
+- If no scale is visible, estimate crack width relative to common objects or provide qualitative assessment
 
 Output a JSON list where each entry contains:
 - "x": normalized x coordinate (0-1) of top-left corner
 - "y": normalized y coordinate (0-1) of top-left corner  
 - "width": normalized width (0-1) of the bounding box
 - "height": normalized height (0-1) of the bounding box
-- "label": detailed description with measurements/characteristics and confidence score
+- "label": brief description with confidence score
 - "confidence": confidence score (0-1)
-- "class": the general category name
-- "details": specific measurements, characteristics, or conditions observed
-- "criteria_match": which detailed criteria this detection matches (reference number from list above)
+- "class": the assigned class name (e.g., "Class I", "Class II", etc.)
+- "description": why this object matches the class criteria
+- "class_number": the class number from the list above (1, 2, 3, etc.)
+- "measured_width": actual crack width measurement if scale is available (e.g., "2.5mm", "1.2cm")
+- "measurement_method": how the measurement was obtained (e.g., "ruler scale", "coin reference", "estimated")
 
-Example format for crack detection:
-[{{"x": 0.1, "y": 0.2, "width": 0.3, "height": 0.4, "label": "crack width ~3mm, length ~15cm (0.92)", "confidence": 0.92, "class": "crack", "details": "width: 3mm, length: 15cm, surface: concrete", "criteria_match": 1}}]"""
+Example format:
+[{{"x": 0.1, "y": 0.2, "width": 0.3, "height": 0.4, "label": "Structural crack (0.92)", "confidence": 0.92, "class": "Class I", "description": "Crack width exceeds 2mm threshold based on ruler measurement", "class_number": 1, "measured_width": "2.5mm", "measurement_method": "ruler scale"}}]"""
     
     return prompt
 
 def create_interface():
     """Create the Gradio interface for object detection"""
-    with gr.Blocks(title="Detailed Object Detection", theme=gr.themes.Soft()) as demo:
-        gr.Markdown("# 🔍 Detailed Object Detection with Custom Specifications")
-        gr.Markdown("Detect objects with detailed specifications (e.g., 'crack width more than 2mm', 'rust spots larger than 5cm')")
+    with gr.Blocks(title="Class-Based Object Detection", theme=gr.themes.Soft()) as demo:
+        gr.Markdown("# 🔍 Class-Based Object Detection with Descriptions")
+        gr.Markdown("Define classes with descriptions and conditions. Objects will be classified and annotated with class names only.")
         
         with gr.Row():
             with gr.Column(scale=1):
@@ -230,6 +239,8 @@ def create_interface():
                 
                 model_preset = gr.Dropdown(
                     choices=[
+                        "qwen/qwen2.5-vl-32b-instruct",
+                        "qwen/qwen-vl-max",
                         "openai/gpt-5-chat",
                         "openai/gpt-5-mini",
                         "anthropic/claude-opus-4.1",
@@ -241,7 +252,7 @@ def create_interface():
                         "openai/gpt-4o",
                         "openai/gpt-4o-mini"
                     ],
-                    value="openai/gpt-5-chat",
+                    value="qwen/qwen2.5-vl-32b-instruct",
                     label="Preset Models",
                     info="Select from popular OpenRouter models",
                     visible=True
@@ -256,28 +267,28 @@ def create_interface():
                 
                 detection_mode = gr.Radio(
                     choices=[
-                        ("Detect Only These Specifications", "specific"),
-                        ("Include These + Others", "include"), 
-                        ("Exclude These Specifications", "exclude")
+                        ("Detect Only These Classes", "specific"),
+                        ("Include These Classes + Others", "include"), 
+                        ("Exclude These Classes", "exclude")
                     ],
                     value="specific",
                     label="Detection Mode",
-                    info="How to handle the specified detailed criteria"
+                    info="How to handle the specified class descriptions"
                 )
                 
-                detailed_specifications = gr.Textbox(
-                    label="Detailed Detection Specifications",
-                    placeholder="""Enter each specification on a new line, e.g.:
-crack width more than 2mm
-rust spots larger than 5cm in diameter
-concrete spalling deeper than 1cm
-structural damage with visible deformation
-paint peeling areas greater than 10cm²""",
-                    value="""crack width more than 2mm
-rust spots larger than 5cm in diameter
-concrete spalling deeper than 1cm""",
+                class_descriptions = gr.Textbox(
+                    label="Class Descriptions",
+                    placeholder="""Define each class with its description and conditions, e.g.:
+Severe Cracks: Crack width more than 2mm (use ruler/scale if present for measurement)
+Minor Cracks: Crack width 0.5-2mm (measure using visible scale)
+Rust Damage: Rust spots larger than 5cm in diameter
+Concrete Spalling: Concrete spalling deeper than 1cm
+Paint Defects: Paint peeling areas greater than 10cm²""",
+                    value="""Severe Cracks: Crack width more than 2mm (use ruler/scale if present for measurement)
+Minor Cracks: Crack width 0.5-2mm (measure using visible scale)
+Rust Damage: Rust spots larger than 5cm in diameter""",
                     lines=8,
-                    info="Enter detailed specifications, one per line"
+                    info="Enter class descriptions, one per line. Format: 'Class Name: Description' or just 'Description'"
                 )
                 
                 confidence_threshold = gr.Slider(
@@ -337,15 +348,15 @@ concrete spalling deeper than 1cm""",
             outputs=[model_preset, custom_model_input]
         )
         
-        def process_detection(image, detailed_specs, conf_threshold, api_key_val, use_preset_val, model_preset_val, custom_model_val, temp_val, mode_val):
+        def process_detection(image, class_desc, conf_threshold, api_key_val, use_preset_val, model_preset_val, custom_model_val, temp_val, mode_val):
             if not api_key_val:
                 return None, "❌ Please enter your OpenRouter API key", "No API key provided"
             
             if image is None:
                 return None, "❌ Please upload an image", "No image uploaded"
             
-            if not detailed_specs or not detailed_specs.strip():
-                return None, "❌ Please enter at least one detailed specification", "No specifications provided"
+            if not class_desc or not class_desc.strip():
+                return None, "❌ Please enter at least one class description", "No class descriptions provided"
             
             # Determine which model to use
             if use_preset_val == "Custom Model":
@@ -356,7 +367,7 @@ concrete spalling deeper than 1cm""",
                 final_model = model_preset_val
             
             try:
-                prompt = create_detection_prompt(detailed_specs, conf_threshold, mode_val)
+                prompt = create_detection_prompt(class_desc, conf_threshold, mode_val)
                 
                 result, error = process_with_openrouter(image, prompt, api_key_val, final_model, temp_val)
                 
@@ -371,43 +382,41 @@ concrete spalling deeper than 1cm""",
                     filtered_detections = [d for d in detections if d.get('confidence', 1.0) >= conf_threshold]
                     
                     mode_descriptions = {
-                        "specific": "Detecting only objects matching detailed specifications",
-                        "include": "Including specified detailed criteria + other objects", 
-                        "exclude": "Excluding objects matching detailed specifications"
+                        "specific": "Detecting only objects matching class descriptions",
+                        "include": "Including specified classes + other objects", 
+                        "exclude": "Excluding objects matching class descriptions"
                     }
                     
                     summary_text = f"✅ {mode_descriptions.get(mode_val, 'Detection')} - Found {len(filtered_detections)} objects\n🤖 Model: {final_model}"
                     
                     if filtered_detections:
-                        # Group by class and show details
-                        class_details = {}
+                        # Group by class and show counts
+                        class_counts = {}
                         for det in filtered_detections:
                             class_name = det.get('class', 'unknown')
-                            details = det.get('details', '')
-                            criteria_match = det.get('criteria_match', '')
+                            description = det.get('description', '')
+                            confidence = det.get('confidence', 1.0)
                             
-                            if class_name not in class_details:
-                                class_details[class_name] = []
+                            if class_name not in class_counts:
+                                class_counts[class_name] = {
+                                    'count': 0,
+                                    'avg_confidence': 0,
+                                    'descriptions': []
+                                }
                             
-                            class_details[class_name].append({
-                                'details': details,
-                                'criteria': criteria_match,
-                                'confidence': det.get('confidence', 1.0)
-                            })
+                            class_counts[class_name]['count'] += 1
+                            class_counts[class_name]['avg_confidence'] += confidence
+                            if description and description not in class_counts[class_name]['descriptions']:
+                                class_counts[class_name]['descriptions'].append(description)
                         
-                        summary_text += "\n\nDetailed Results:"
-                        for class_name, items in class_details.items():
-                            summary_text += f"\n• {class_name} ({len(items)} found):"
-                            for item in items[:3]:  # Show first 3 items
-                                summary_text += f"\n  - {item['details']} (conf: {item['confidence']:.2f})"
-                                if item['criteria']:
-                                    summary_text += f" [criteria: {item['criteria']}]"
-                            if len(items) > 3:
-                                summary_text += f"\n  ... and {len(items)-3} more"
+                        summary_text += "\n\nClass Detection Results:"
+                        for class_name, data in class_counts.items():
+                            avg_conf = data['avg_confidence'] / data['count']
+                            summary_text += f"\n• {class_name}: {data['count']} detected (avg conf: {avg_conf:.2f})"
                     
                     return annotated_img, json.dumps(filtered_detections, indent=2), summary_text
                 else:
-                    return image, "No objects detected matching detailed specifications", "No detections matching criteria above confidence threshold"
+                    return image, "No objects detected matching class descriptions", "No detections matching criteria above confidence threshold"
                     
             except json.JSONDecodeError:
                 return None, f"❌ Invalid JSON response: {result}", "JSON parsing failed"
@@ -416,41 +425,57 @@ concrete spalling deeper than 1cm""",
         
         detect_btn.click(
             process_detection,
-            inputs=[image_input, detailed_specifications, confidence_threshold, api_key, use_preset, model_preset, custom_model_input, temperature, detection_mode],
+            inputs=[image_input, class_descriptions, confidence_threshold, api_key, use_preset, model_preset, custom_model_input, temperature, detection_mode],
             outputs=[annotated_image, detection_results, detection_summary]
         )
         
         gr.Markdown("""
         ## 💡 Usage Tips
-        - **Specific Mode**: Only detect objects matching your detailed specifications
-        - **Include Mode**: Detect your specified criteria plus any other objects found
-        - **Exclude Mode**: Detect everything except objects matching your specifications
+        - **Specific Mode**: Only detect objects matching your class descriptions
+        - **Include Mode**: Detect your specified classes plus any other objects found
+        - **Exclude Mode**: Detect everything except objects matching your class descriptions
+        
+        ### 🏷️ Class Definition
+        **Format Options:**
+        1. `Class Name: Description` - e.g., "Severe Cracks: Crack width more than 2mm"
+        2. `Description only` - Will be automatically assigned as "Class I", "Class II", etc.
+        
+        **Annotation Behavior:**
+        - Images show only class names (e.g., "Class I", "Class II")
+        - Same class = same color throughout the image
+        - Clean, simple visual identification
         
         ### 🤖 Model Selection
         **Default Models (Recommended):**
-        - `openai/gpt-5-chat` - Latest GPT-5 with advanced vision capabilities (Default)
+        - `qwen/qwen2.5-vl-32b-instruct` - Advanced Qwen vision model optimized for detailed analysis (Default)
+        - `qwen/qwen-vl-max` - Premium Qwen vision model with maximum capabilities
+        - `openai/gpt-5-chat` - Latest GPT-5 with advanced vision capabilities
         - `openai/gpt-5-mini` - Faster, efficient GPT-5 variant
         - `anthropic/claude-opus-4.1` - Next-gen Claude with superior reasoning
         - `x-ai/grok-4` - Advanced Grok model with detailed analysis
         
         **Custom Models**: Enter any OpenRouter model ID from [openrouter.ai/models](https://openrouter.ai/models)
         
-        ### Example Detailed Specifications:
+        ### Example Class Descriptions:
         ```
-        crack width more than 2mm
-        rust spots larger than 5cm in diameter
-        concrete spalling deeper than 1cm
-        structural damage with visible deformation
-        paint peeling areas greater than 10cm²
-        corrosion affecting more than 20% of surface area
-        missing bolts or fasteners
-        water damage stains larger than 15cm
+        Severe Cracks: Crack width more than 2mm (use ruler/scale for measurement)
+        Minor Cracks: Crack width 0.5-2mm (measure using visible scale)
+        Rust Damage: Rust spots larger than 5cm in diameter
+        Concrete Spalling: Concrete spalling deeper than 1cm
+        Paint Defects: Paint peeling areas greater than 10cm²
+        Water Damage: Water damage stains larger than 15cm
         ```
         
-        - Enter one detailed specification per line
-        - Be specific about measurements, sizes, conditions
+        ### 📏 Scale-Based Measurement:
+        - **Automatic Scale Detection**: The system looks for rulers, measuring tools, or reference objects
+        - **Precise Measurements**: When scales are found, actual crack widths are calculated
+        - **Measurement Methods**: Supports rulers, crack gauges, coins, or other reference objects
+        - **Enhanced Classification**: More accurate class assignment based on measured dimensions
+        
+        - Enter one class description per line
+        - Be specific about conditions and measurements
+        - Objects will be classified and labeled with class names only
         - Adjust confidence threshold to filter weak detections
-        - Use lower temperature values for consistent results
         - Get your API key from [openrouter.ai](https://openrouter.ai/)
         """)