make_leaderboard

.env.template

@@ -1,6 +0,0 @@
-HF_TOKEN="your_huggingface_write_token"
-OWNER="your_huggingface_username_or_org"
-RESULTS_DATASET_ID="your_username/guardbench-results"
-SUBMITTER_TOKEN="your_secret_submission_token"
-ADMIN_USERNAME="admin"
-ADMIN_PASSWORD="password" # Change this!

.gitignore

@@ -1,52 +1,13 @@
-# Python
+auto_evals/
+venv/
 __pycache__/
-*.py[cod]
-*$py.class
-*.so
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-.venv/
-*.egg-info/
-.installed.cfg
-*.egg
-.gradio/
-
-# Environment variables
 .env
-
-# Virtual Environment
-venv/
-ENV/
-
-# IDE
-.idea/
+.ipynb_checkpoints
+*ipynb
 .vscode/
-*.swp
-*.swo
-
-# OS
-.DS_Store
-Thumbs.db
 
-# Hugging Face cache
 eval-queue/
 eval-results/
 eval-queue-bk/
 eval-results-bk/
-
-# Data files
-data/
-
-# Versioned leaderboard files
-data/leaderboard_v*.json
+logs/

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "guard-bench-submodule"]
-	path = guard-bench-submodule
-	url = https://github.com/whitecircle-ai/circle-guard-bench.git

.gradio/certificate.pem

@@ -1,31 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
-TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
-cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
-WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
-ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
-MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
-h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
-0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
-A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
-T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
-B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
-B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
-KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
-OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
-jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
-qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
-rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
-HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
-hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
-ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
-3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
-NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
-ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
-TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
-jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
-oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
-4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
-mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
-emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
------END CERTIFICATE-----

README.md

@@ -1,227 +1,46 @@
 ---
 title: CodeReviewBench
-emoji: 😎
-colorFrom: gray
+emoji: 🥇
+colorFrom: green
 colorTo: indigo
 sdk: gradio
-
-sdk_version: 4.44.1
 app_file: app.py
 pinned: true
-short_description: A comprehensive benchmark for codereview.
-models:
-- openai/gpt-4o-mini
-- openai/gpt-4o
-- claude-3-7-sonnet
-- deepseek/deepseek-r1
-
+license: mit
+short_description: Result of benchmark presented in paper CodeReviewBench
+sdk_version: 5.19.0
 ---
 
-# CodeReview Bench Leaderboard
-
-A comprehensive benchmark and leaderboard for code review generation models, inspired by [CodeReviewBench](https://huggingface.co/spaces/your-org/CodeReviewBench).
-## Features
-
-- **Multi-Language Support**: Evaluates models across 17+ programming languages including Python, JavaScript, Java, C++, TypeScript, Go, Rust, and more
-- **Dual Language Comments**: Supports both Russian and English comment languages
-- **Comprehensive Metrics**:
-  - LLM-based multimetric evaluation (readability, relevance, explanation clarity, problem identification, actionability, completeness, specificity, contextual adequacy, consistency, brevity)
-  - Exact-match metrics (pass@1, pass@5, pass@10, BLEU@10)
-- **Interactive Visualization**: Compare model performance across categories with radar plots
-- **Easy Submission**: Submit your model results via web interface
-
-## Metrics
-
-### LLM-based Multimetric
-
-- **Readability**: How easy the review is to understand
-- **Relevance**: How relevant the review is to the code
-- **Explanation Clarity**: How clear the explanations are
-- **Problem Identification**: How well problems are identified
-- **Actionability**: How actionable the suggestions are
-- **Completeness**: How complete the review is
-- **Specificity**: How specific the feedback is
-- **Contextual Adequacy**: How well the review fits the context
-- **Consistency**: How consistent the review style is
-- **Brevity**: How concise the review is
-
-### Exact-Match Metrics
-
-- **Pass@1**: Percentage of correct reviews on first attempt
-- **Pass@5**: Percentage of correct reviews in top 5 attempts
-- **Pass@10**: Percentage of correct reviews in top 10 attempts
-- **BLEU@10**: BLEU score for top 10 review candidates
-
-## Programming Languages Supported
-
-- Python
-- JavaScript
-- Java
-- C++
-- C#
-- TypeScript
-- Go
-- Rust
-- Swift
-- Kotlin
-- Ruby
-- PHP
-- C
-- Scala
-- R
-- Dart
-- Other
-
-## Comment Languages
-
-- Russian (ru)
-- English (en)
+# Start the configuration
 
-## Example Categories
-
-- Bug Fix
-- Code Style
-- Performance
-- Security
-- Refactoring
-- Documentation
-- Testing
-- Architecture
-- Other
-
-## Installation
-
-```bash
-pip install -r requirements.txt
-```
-
-## Usage
-
-```bash
-python app.py
-```
-
-## Submission Format
-
-Submit your results as a JSONL file where each line contains:
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
 
+Results files should have the following format and be stored as json files:
 ```json
 {
-  "model_name": "your-model-name",
-  "programming_language": "python",
-  "comment_language": "en",
-  "readability": 8.5,
-  "relevance": 9.0,
-  "explanation_clarity": 7.8,
-  "problem_identification": 8.2,
-  "actionability": 8.7,
-  "completeness": 8.0,
-  "specificity": 7.5,
-  "contextual_adequacy": 8.3,
-  "consistency": 8.8,
-  "brevity": 7.2,
-  "pass_at_1": 0.75,
-  "pass_at_5": 0.88,
-  "pass_at_10": 0.92,
-  "bleu_at_10": 0.65,
-  "total_evaluations": 100
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
 }
 ```
 
-## Environment Variables
-
-Set the following environment variables:
-
-
-## Citation
+Request files are created automatically by this tool.
 
-<<<<<<< HEAD
-- **Multi-tab Interface**: Organized navigation with dedicated sections
-- **Advanced Filtering**: Real-time filtering by multiple criteria
-- **Dark Theme**: Modern, GitHub-inspired dark interface
-- **IP-based Submissions**: Secure submission tracking
-- **Comprehensive Analytics**: Detailed performance insights
-- **Data Export**: Multiple export formats
-- **Rate Limiting**: Anti-spam protection
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
 
-### 🔧 Technical Improvements
-
-- **Modular Architecture**: Clean separation of concerns
-- **Type Safety**: Full type annotations throughout
-- **Error Handling**: Comprehensive error handling and logging
-- **Data Validation**: Multi-layer validation with Pydantic
-- **Performance**: Optimized data processing and display
-
-## 📈 Metrics & Evaluation
-
-### Performance Metrics
-
-- **BLEU**: Text similarity score (0.0-1.0)
-- **Pass@1**: Success rate in single attempt (0.0-1.0)
-- **Pass@5**: Success rate in 5 attempts (0.0-1.0)
-- **Pass@10**: Success rate in 10 attempts (0.0-1.0)
-
-### Quality Dimensions
-
-1. **Readability**: How clear and readable are the reviews?
-2. **Relevance**: How relevant to the code changes?
-3. **Explanation Clarity**: How well does it explain issues?
-4. **Problem Identification**: How effectively does it identify problems?
-5. **Actionability**: How actionable are the suggestions?
-6. **Completeness**: How thorough are the reviews?
-7. **Specificity**: How specific are the comments?
-8. **Contextual Adequacy**: How well does it understand context?
-9. **Consistency**: How consistent across different reviews?
-10. **Brevity**: How concise without losing important information?
-
-## 🔒 Security Features
-
-### Rate Limiting
-
-- **5 submissions per IP per 24 hours**
-- **Automatic IP tracking and logging**
-- **Graceful error handling for rate limits**
-
-### Data Validation
-
-- **Model name format validation**
-- **Score range validation (0.0-1.0 for performance, 0-10 for quality)**
-- **Logical consistency checks (Pass@1 ≤ Pass@5 ≤ Pass@10)**
-- **Required field validation**
-
-### Audit Trail
-
-- **Complete submission logging**
-- **IP address tracking (partially masked for privacy)**
-- **Timestamp recording**
-- **Data integrity checks**
-
-## 🤝 Contributing
-
-1. Fork the repository
-2. Create a feature branch
-3. Make your changes
-4. Add tests if applicable
-5. Submit a pull request
-
-## 📄 License
-
-This project is licensed under the MIT License - see the LICENSE file for details.
-
-## 🙏 Acknowledgments
-
-- Inspired by [CodeReviewBench](https://huggingface.co/spaces/your-org/CodeReviewBench)
-- Built with [Gradio](https://gradio.app/) for the web interface
-- Thanks to the open-source community for tools and inspiration
-
-## 📞 Support
-
-For questions, issues, or contributions:
-
-- Open an issue on GitHub
-- Check the documentation
-- Contact the maintainers
-
----
+# Code logic for more complex edits
 
-**Built with ❤️ for the code review research community**
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

SUBMISSION_EXAMPLE.md

@@ -1,266 +0,0 @@
-# 📝 Model Submission Example
-
-This guide shows you exactly how to submit your code review model to the leaderboard.
-
-## 🚀 Step-by-Step Submission Process
-
-### 1. **Access the Submission Form**
-
-- Open the CodeReview Leaderboard in your browser
-- Navigate to the **📝 Submit Model** tab
-- Click on the "📝 Submit New Model Results" accordion to expand the form
-
-### 2. **Fill in Basic Information**
-
-#### **Model Name** ✨
-
-```
-Example: microsoft/CodeT5-base
-Format: organization/model-name
-```
-
-#### **Programming Language** 🔍
-
-```
-Select: Python
-(or Java, JavaScript, C++, Go, Rust, etc.)
-```
-
-#### **Comment Language** 🌍
-
-```
-Select: English  
-(or Chinese, Spanish, French, German, etc.)
-```
-
-#### **Taxonomy Category** 🏷️
-
-```
-Select: Bug Detection
-(or Security, Performance, Code Style, etc.)
-```
-
-### 3. **Performance Scores** (0.0 - 1.0)
-
-#### **BLEU Score**
-
-```
-Example: 0.742
-Range: 0.0 to 1.0
-Description: Measures similarity between generated and reference reviews
-```
-
-#### **Pass@1**
-
-```
-Example: 0.685
-Range: 0.0 to 1.0  
-Description: Success rate when model gets 1 attempt
-```
-
-#### **Pass@5**
-
-```
-Example: 0.834
-Range: 0.0 to 1.0
-Description: Success rate when model gets 5 attempts  
-```
-
-#### **Pass@10**
-
-```
-Example: 0.901
-Range: 0.0 to 1.0
-Description: Success rate when model gets 10 attempts
-```
-
-### 4. **Quality Metrics** (0 - 10)
-
-Rate your model across these 10 dimensions:
-
-#### **Readability: 8**
-
-```
-How clear and readable are the generated code reviews?
-Scale: 0 (unreadable) to 10 (very clear)
-```
-
-#### **Relevance: 7**
-
-```  
-How relevant are the reviews to the actual code changes?
-Scale: 0 (irrelevant) to 10 (highly relevant)
-```
-
-#### **Explanation Clarity: 8**
-
-```
-How well does the model explain identified issues?
-Scale: 0 (unclear) to 10 (very clear explanations)
-```
-
-#### **Problem Identification: 7**
-
-```
-How effectively does it identify real code problems?
-Scale: 0 (misses issues) to 10 (finds all problems)
-```
-
-#### **Actionability: 6**
-
-```
-How actionable and useful are the suggestions?
-Scale: 0 (not actionable) to 10 (very actionable)
-```
-
-#### **Completeness: 7**
-
-```
-How thorough and complete are the reviews?
-Scale: 0 (incomplete) to 10 (comprehensive)
-```
-
-#### **Specificity: 6**
-
-```
-How specific are the comments and suggestions?
-Scale: 0 (too generic) to 10 (very specific)
-```
-
-#### **Contextual Adequacy: 7**
-
-```
-How well does it understand the code context?
-Scale: 0 (ignores context) to 10 (perfect context understanding)
-```
-
-#### **Consistency: 6**
-
-```
-How consistent is the model across different code reviews?
-Scale: 0 (inconsistent) to 10 (very consistent)
-```
-
-#### **Brevity: 5**
-
-```
-How concise are the reviews without losing important information?
-Scale: 0 (too verbose/too brief) to 10 (perfect length)
-```
-
-### 5. **Submit Your Model**
-
-- Click the **🚀 Submit Model** button
-- Wait for validation and processing
-- Check for success/error message
-
-## 📋 Complete Example Submission
-
-Here's a real example of submitting the CodeT5-base model:
-
-```yaml
-Model Information:
-  Model Name: "microsoft/CodeT5-base"
-  Programming Language: "Python"
-  Comment Language: "English"
-  Taxonomy Category: "Bug Detection"
-
-Performance Scores:
-  BLEU Score: 0.742
-  Pass@1: 0.685
-  Pass@5: 0.834
-  Pass@10: 0.901
-
-Quality Metrics:
-  Readability: 8
-  Relevance: 7  
-  Explanation Clarity: 8
-  Problem Identification: 7
-  Actionability: 6
-  Completeness: 7
-  Specificity: 6
-  Contextual Adequacy: 7
-  Consistency: 6
-  Brevity: 5
-```
-
-## 🔒 Security & Rate Limiting
-
-### **IP-based Rate Limiting**
-
-- **5 submissions per IP address per 24 hours**
-- Submissions are tracked by your IP address
-- Rate limit resets every 24 hours
-
-### **Validation Rules**
-
-- Model name must follow `organization/model` format
-- All performance scores must be between 0.0 and 1.0
-- All quality metrics must be between 0 and 10
-- Pass@1 ≤ Pass@5 ≤ Pass@10 (logical consistency)
-
-## ✅ After Submission
-
-### **Immediate Feedback**
-
-You'll see one of these messages:
-
-#### **Success ✅**
-
-```
-✅ Submission recorded successfully!
-```
-
-#### **Error Examples ❌**
-
-```
-❌ Rate limit exceeded: 5/5 submissions in 24 hours
-❌ Model name contains invalid characters
-❌ Pass@1 score cannot be higher than Pass@5
-❌ Score BLEU out of range: 1.2 (must be between 0 and 1)
-```
-
-### **View Your Results**
-
-- Your model will appear in the **🏆 Leaderboard** tab
-- Use filters to find your specific submission
-- Check the **📈 Analytics** tab for submission history
-
-## 🎯 Tips for Better Submissions
-
-### **Model Naming**
-
-```
-✅ Good: "microsoft/CodeT5-base"
-✅ Good: "facebook/bart-large"  
-✅ Good: "my-org/custom-model-v2"
-❌ Bad: "my model"
-❌ Bad: "model@v1.0"
-```
-
-### **Performance Scores**
-
-- Be honest and accurate with your evaluations
-- Use proper evaluation methodology  
-- Ensure Pass@k scores are logically consistent
-- Document your evaluation process
-
-### **Quality Metrics**
-
-- Rate based on actual model performance
-- Consider multiple test cases
-- Be objective in your assessment
-- Document your rating criteria
-
-## 🤝 Need Help?
-
-If you encounter issues:
-
-1. Check the error message for specific guidance
-2. Verify all fields are filled correctly  
-3. Ensure you haven't exceeded rate limits
-4. Contact maintainers if problems persist
-
----
-
-**Ready to submit your model? Head to the 📝 Submit Model tab and follow this guide!** 🚀

app.py

@@ -1,20 +1,8 @@
-"""
-CodeReview Leaderboard - Inspired by CodeReviewBench
-A comprehensive leaderboard for code review generation models
-"""
-
-import os
-import json
-import tempfile
-import logging
 import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
-import plotly.express as px
-import plotly.graph_objects as go
 from apscheduler.schedulers.background import BackgroundScheduler
-import numpy as np
-from gradio.themes.utils import fonts, colors
-from dataclasses import fields, dataclass
+from huggingface_hub import snapshot_download
 
 from src.about import (
     CITATION_BUTTON_LABEL,
@@ -26,1091 +14,191 @@ from src.about import (
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
-    CODEREVIEW_COLUMN,
-    DISPLAY_COLS,
-    METRIC_COLS,
-    HIDDEN_COLS,
-    NEVER_HIDDEN_COLS,
-    CATEGORIES,
-    COMMENT_LANGUAGES,
-    EXAMPLE_CATEGORIES,
-    TOPICS,
+    BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    AutoEvalColumn,
     ModelType,
-    Mode,
-    Precision,
+    fields,
     WeightType,
-    ReviewModelType,
-    get_all_column_choices,
-    get_default_visible_columns,
-)
-from src.display.formatting import styled_message, styled_error, styled_warning
-from src.envs import (
-    ADMIN_USERNAME,
-    ADMIN_PASSWORD,
-    RESULTS_DATASET_ID,
-    SUBMITTER_TOKEN,
-    TOKEN,
-    DATA_PATH,
-)
-from src.populate import get_leaderboard_df, get_category_leaderboard_df
-from src.submission.submit import process_submission
-
-# Configure logging
-logging.basicConfig(
-    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+    Precision
 )
-logger = logging.getLogger(__name__)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
 
-# Ensure data directory exists
-os.makedirs(DATA_PATH, exist_ok=True)
 
-# Available benchmark versions
-BENCHMARK_VERSIONS = ["v0"]
-CURRENT_VERSION = "v0"
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
 
-# Initialize leaderboard data
+### Space initialisation
 try:
-    logger.info("Initializing leaderboard data...")
-    LEADERBOARD_DF = get_leaderboard_df(version=CURRENT_VERSION)
-    logger.info(f"Loaded leaderboard with {len(LEADERBOARD_DF)} entries")
-except Exception as e:
-    logger.error(f"Error loading leaderboard data: {e}")
-    LEADERBOARD_DF = pd.DataFrame()
-
-custom_theme = gr.themes.Default(
-    primary_hue=colors.slate,
-    secondary_hue=colors.slate,
-    neutral_hue=colors.neutral,
-    font=(fonts.GoogleFont("Inter"), "sans-serif"),
-).set(
-    # font_size="16px",
-    body_background_fill="#0f0f10",
-    body_background_fill_dark="#0f0f10",
-    body_text_color="#f4f4f5",
-    body_text_color_subdued="#a1a1aa",
-    block_background_fill="#1e1e1e",  # Cooler Grey
-    block_border_color="#333333",  # Cooler Grey
-    block_shadow="none",
-    # Swapped primary and secondary button styles
-    button_primary_background_fill="#121212",  # Changed to specific color for Refresh button
-    button_primary_text_color="#f4f4f5",
-    button_primary_border_color="#333333",  # Keep border grey or change to #121212?
-    button_secondary_background_fill="#f4f4f5",
-    button_secondary_text_color="#0f0f10",
-    button_secondary_border_color="#f4f4f5",
-    input_background_fill="#1e1e1e",  # Cooler Grey
-    input_border_color="#333333",  # Cooler Grey
-    input_placeholder_color="#71717a",
-    table_border_color="#333333",  # Cooler Grey
-    table_even_background_fill="#2d2d2d",  # Cooler Grey (Slightly lighter)
-    table_odd_background_fill="#1e1e1e",  # Cooler Grey
-    table_text_color="#f4f4f5",
-    link_text_color="#ffffff",
-    border_color_primary="#333333",  # Cooler Grey
-    background_fill_secondary="#333333",  # Cooler Grey
-    color_accent="#f4f4f5",
-    border_color_accent="#333333",  # Cooler Grey
-    button_primary_background_fill_hover="#424242",  # Cooler Grey
-    block_title_text_color="#f4f4f5",
-    accordion_text_color="#f4f4f5",
-    panel_background_fill="#1e1e1e",  # Cooler Grey
-    panel_border_color="#333333",  # Cooler Grey
-    # Explicitly setting primary/secondary/accent colors/borders
-    background_fill_primary="#0f0f10",
-    background_fill_primary_dark="#0f0f10",
-    background_fill_secondary_dark="#333333",  # Cooler Grey
-    border_color_primary_dark="#333333",  # Cooler Grey
-    border_color_accent_dark="#333333",  # Cooler Grey
-    border_color_accent_subdued="#424242",  # Cooler Grey
-    border_color_accent_subdued_dark="#424242",  # Cooler Grey
-    color_accent_soft="#a1a1aa",
-    color_accent_soft_dark="#a1a1aa",
-    # Explicitly setting input hover/focus states
-    input_background_fill_dark="#1e1e1e",  # Cooler Grey
-    input_background_fill_focus="#424242",  # Cooler Grey
-    input_background_fill_focus_dark="#424242",  # Cooler Grey
-    input_background_fill_hover="#2d2d2d",  # Cooler Grey
-    input_background_fill_hover_dark="#2d2d2d",  # Cooler Grey
-    input_border_color_dark="#333333",  # Cooler Grey
-    input_border_color_focus="#f4f4f5",
-    input_border_color_focus_dark="#f4f4f5",
-    input_border_color_hover="#424242",  # Cooler Grey
-    input_border_color_hover_dark="#424242",  # Cooler Grey
-    input_placeholder_color_dark="#71717a",
-    # Explicitly set dark variants for table backgrounds
-    table_even_background_fill_dark="#2d2d2d",  # Cooler Grey
-    table_odd_background_fill_dark="#1e1e1e",  # Cooler Grey
-    # Explicitly set dark text variants
-    body_text_color_dark="#f4f4f5",
-    body_text_color_subdued_dark="#a1a1aa",
-    block_title_text_color_dark="#f4f4f5",
-    accordion_text_color_dark="#f4f4f5",
-    table_text_color_dark="#f4f4f5",
-    # Explicitly set dark panel/block variants
-    panel_background_fill_dark="#1e1e1e",  # Cooler Grey
-    panel_border_color_dark="#333333",  # Cooler Grey
-    block_background_fill_dark="#1e1e1e",  # Cooler Grey
-    block_border_color_dark="#333333",  # Cooler Grey
-)
-
-
-@dataclass
-class ColumnInfo:
-    """Information about a column in the leaderboard."""
-
-    name: str
-    display_name: str
-    type: str = "text"
-    hidden: bool = False
-    never_hidden: bool = False
-    displayed_by_default: bool = True
-
-
-def update_column_choices(df):
-    """Update column choices based on what's actually in the dataframe"""
-    if df is None or df.empty:
-        return get_all_column_choices()
-
-    # Get columns that actually exist in the dataframe
-    existing_columns = list(df.columns)
-
-    # Get all possible columns with their display names
-    all_columns = get_all_column_choices()
-
-    # Filter to only include columns that exist in the dataframe
-    valid_columns = [
-        (col_name, display_name)
-        for col_name, display_name in all_columns
-        if col_name in existing_columns
-    ]
-
-    # Return default if there are no valid columns
-    if not valid_columns:
-        return get_all_column_choices()
-
-    return valid_columns
-
-
-# Update the column_selector initialization
-def get_initial_columns():
-    """Get initial columns to show in the dropdown"""
-    try:
-        # Get available columns in the main dataframe
-        available_cols = list(LEADERBOARD_DF.columns)
-        logger.info(f"Available columns in LEADERBOARD_DF: {available_cols}")
-
-        # If dataframe is empty, use default visible columns
-        if not available_cols:
-            return get_default_visible_columns()
-
-        # Get default visible columns that actually exist in the dataframe
-        valid_defaults = [
-            col for col in get_default_visible_columns() if col in available_cols
-        ]
-
-        # If none of the defaults exist, return all available columns
-        if not valid_defaults:
-            return available_cols
-
-        return valid_defaults
-    except Exception as e:
-        logger.error(f"Error getting initial columns: {e}")
-        return get_default_visible_columns()
-
-
-def init_leaderboard(dataframe, visible_columns=None):
-    """
-    Initialize a standard Gradio Dataframe component for the leaderboard.
-    """
-    if dataframe is None or dataframe.empty:
-        # Create an empty dataframe with the right columns
-        columns = [getattr(CODEREVIEW_COLUMN, col).name for col in DISPLAY_COLS]
-        dataframe = pd.DataFrame(columns=columns)
-        logger.warning("Initializing empty leaderboard")
-
-    # Lowercase model_name for display
-    if "model_name" in dataframe.columns:
-        dataframe = dataframe.copy()
-        dataframe["model_name"] = dataframe["model_name"].str.lower()
-
-    if "model_type" in dataframe.columns:
-        dataframe = dataframe.copy()
-        dataframe["model_type"] = dataframe["model_type"].str.replace(" : ", "-")
-
-    if "review_model_type" in dataframe.columns:
-        dataframe = dataframe.copy()
-        dataframe["review_model_type"] = dataframe["review_model_type"].str.replace("custom", "custom")
-
-    # print("\n\n", "dataframe", dataframe, "--------------------------------\n\n")
-
-    # Determine which columns to display
-    display_column_names = [
-        getattr(CODEREVIEW_COLUMN, col).name for col in DISPLAY_COLS
-    ]
-    hidden_column_names = [getattr(CODEREVIEW_COLUMN, col).name for col in HIDDEN_COLS]
-
-    # Columns that should always be shown
-    always_visible = [getattr(CODEREVIEW_COLUMN, col).name for col in NEVER_HIDDEN_COLS]
-
-    # Use provided visible columns if specified, otherwise use default
-    if visible_columns is None:
-        # Determine which columns to show initially
-        visible_columns = [
-            col for col in display_column_names if col not in hidden_column_names
-        ]
-
-    # Always include the never-hidden columns
-    for col in always_visible:
-        if col not in visible_columns and col in dataframe.columns:
-            visible_columns.append(col)
-
-    # Make sure we only include columns that actually exist in the dataframe
-    visible_columns = [col for col in visible_columns if col in dataframe.columns]
-
-    # Map GuardBench column types to Gradio's expected datatype strings
-    # Valid Gradio datatypes are: 'str', 'number', 'bool', 'date', 'markdown', 'html', 'image'
-    type_mapping = {
-        "text": "str",
-        "number": "number",
-        "bool": "bool",
-        "date": "date",
-        "markdown": "markdown",
-        "html": "html",
-        "image": "image",
-    }
-
-    # Create a list of datatypes in the format Gradio expects
-    datatypes = []
-    for col in visible_columns:
-        # Find the corresponding CODEREVIEW_COLUMN entry
-        col_type = None
-        for display_col in DISPLAY_COLS:
-            if getattr(CODEREVIEW_COLUMN, display_col).name == col:
-                orig_type = getattr(CODEREVIEW_COLUMN, display_col).type
-                # Map to Gradio's expected types
-                col_type = type_mapping.get(orig_type, "str")
-                break
-
-        # Default to 'str' if type not found or not mappable
-        if col_type is None:
-            col_type = "str"
-
-        datatypes.append(col_type)
-
-    # Create a dummy column for search functionality if it doesn't exist
-    if "search_dummy" not in dataframe.columns:
-        dataframe["search_dummy"] = dataframe.apply(
-            lambda row: " ".join(str(val) for val in row.values if pd.notna(val)),
-            axis=1,
-        )
-
-    # Select only the visible columns for display
-    visible_columns.remove("model_name")
-
-    visible_columns = ["model_name"] + visible_columns
-    display_df = dataframe[visible_columns].copy()
-
-    # print(f"--- DataFrame inside init_leaderboard (before rounding) ---")
-    # print(display_df[['model_name', 'macro_accuracy', 'macro_recall', 'total_evals_count']].head() if all(c in display_df.columns for c in ['model_name', 'macro_accuracy', 'macro_recall', 'total_evals_count']) else "Relevant columns not present")
-    # print(f"-------------------------------------------------------------")
-
-    # Round numeric columns to 3 decimal places for display
-    numeric_cols = display_df.select_dtypes(include=np.number).columns
-    for col in numeric_cols:
-        # Avoid rounding integer columns like counts
-        if not pd.api.types.is_integer_dtype(display_df[col]):
-            # Format floats to exactly 3 decimal places, preserving trailing zeros
-            display_df[col] = display_df[col].apply(
-                lambda x: f"{x:.3f}" if pd.notna(x) else None
-            )
-
-    column_info_map = {
-        f.name: getattr(CODEREVIEW_COLUMN, f.name) for f in fields(CODEREVIEW_COLUMN)
-    }
-    column_mapping = {
-        col: column_info_map.get(col, ColumnInfo(col, col)).display_name
-        for col in visible_columns
-    }
-
-    # Rename columns in the DataFrame
-    display_df.rename(columns=column_mapping, inplace=True)
-
-    # Apply styling - note: styling might need adjustment if it relies on column names
-    styler = display_df.style.set_properties(**{"text-align": "right"}).set_properties(
-        subset=["Model"], **{"width": "200px"}
+    print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
     )
-
-    return gr.Dataframe(
-        value=styler,
-        datatype=datatypes,
-        interactive=False,
-        wrap=True,
-        height=2500,
-        elem_id="leaderboard-table",
-        row_count=len(display_df),
+except Exception:
+    restart_space()
+try:
+    print(EVAL_RESULTS_PATH)
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
     )
+except Exception:
+    restart_space()
 
 
-def search_filter_leaderboard(
-    df, search_query="", comment_languages=None, version=CURRENT_VERSION
-):
-    """
-    Filter the leaderboard based on search query and comment languages.
-    """
-    if df is None or df.empty:
-        return df
-
-    filtered_df = df.copy()
-
-    # Add search dummy column if it doesn't exist
-    if "search_dummy" not in filtered_df.columns:
-        filtered_df["search_dummy"] = filtered_df.apply(
-            lambda row: " ".join(str(val) for val in row.values if pd.notna(val)),
-            axis=1,
-        )
-
-    # Apply comment language filter (assuming there's a comment_language column in the data)
-    if comment_languages and len(comment_languages) > 0:
-        # Look for a comment language column in the dataframe
-        comment_lang_cols = [col for col in filtered_df.columns if 'comment_language' in col.lower()]
-        if comment_lang_cols:
-            filtered_df = filtered_df[
-                filtered_df[comment_lang_cols[0]].isin(comment_languages)
-            ]
-
-    # Apply search query
-    if search_query:
-        search_terms = [
-            term.strip() for term in search_query.split(";") if term.strip()
-        ]
-        if search_terms:
-            combined_mask = None
-            for term in search_terms:
-                mask = filtered_df["search_dummy"].str.contains(
-                    term, case=False, na=False
-                )
-                if combined_mask is None:
-                    combined_mask = mask
-                else:
-                    combined_mask = combined_mask | mask
-
-            if combined_mask is not None:
-                filtered_df = filtered_df[combined_mask]
-
-    # Drop the search dummy column before returning
-    visible_columns = [col for col in filtered_df.columns if col != "search_dummy"]
-    return filtered_df[visible_columns]
-
-
-def refresh_data_with_filters(
-    version=CURRENT_VERSION, search_query="", comment_languages=None, selected_columns=None
-):
-    """
-    Refresh the leaderboard data and update all components with filtering.
-    Ensures we handle cases where dataframes might have limited columns.
-    """
-    global LEADERBOARD_DF
-    try:
-        logger.info(f"Performing refresh of leaderboard data with filters...")
-        # Get new data
-        main_df = get_leaderboard_df(version=version)
-        LEADERBOARD_DF = main_df
-        category_dfs = [
-            get_category_leaderboard_df(category, version=version)
-            for category in CATEGORIES
-        ]
-        selected_columns = [
-            x.lower()
-            .replace(" ", "_")
-            .replace("(", "")
-            .replace(")", "")
-            .replace("_recall", "_recall_binary")
-            .replace("_precision", "_precision_binary")
-            for x in selected_columns
-        ]
-
-        # Log the actual columns we have
-        logger.info(f"Main dataframe columns: {list(main_df.columns)}")
-
-        # Apply filters to each dataframe
-        filtered_main_df = search_filter_leaderboard(
-            main_df, search_query, comment_languages, version
-        )
-        filtered_category_dfs = [
-            search_filter_leaderboard(df, search_query, comment_languages, version)
-            for df in category_dfs
-        ]
-
-        # Get available columns from the dataframe
-        available_columns = list(filtered_main_df.columns)
-
-        # Filter selected columns to only those available in the data
-        if selected_columns:
-            # Convert display names to internal names first
-            internal_selected_columns = [
-                x.lower()
-                .replace(" ", "_")
-                .replace("(", "")
-                .replace(")", "")
-                .replace("_recall", "_recall_binary")
-                .replace("_precision", "_precision_binary")
-                for x in selected_columns
-            ]
-            valid_selected_columns = [
-                col for col in internal_selected_columns if col in available_columns
-            ]
-            if not valid_selected_columns and "model_name" in available_columns:
-                # Fallback if conversion/filtering leads to empty selection
-                valid_selected_columns = ["model_name"] + [
-                    col
-                    for col in get_default_visible_columns()
-                    if col in available_columns
-                ]
-        else:
-            # If no columns were selected in the dropdown, use default visible columns that exist
-            valid_selected_columns = [
-                col for col in get_default_visible_columns() if col in available_columns
-            ]
-
-        # Initialize dataframes for display with valid selected columns
-        main_dataframe = init_leaderboard(filtered_main_df, valid_selected_columns)
-
-        # For category dataframes, get columns that actually exist in each one
-        category_dataframes = []
-        for df in filtered_category_dfs:
-            df_columns = list(df.columns)
-            df_valid_columns = [
-                col for col in valid_selected_columns if col in df_columns
-            ]
-            if not df_valid_columns and "model_name" in df_columns:
-                df_valid_columns = ["model_name"] + get_default_visible_columns()
-            category_dataframes.append(init_leaderboard(df, df_valid_columns))
-
-        return main_dataframe, *category_dataframes
-
-    except Exception as e:
-        logger.error(f"Error in refresh with filters: {e}")
-        # Return the current leaderboards on error
-        return leaderboard, *[
-            tab.children[0] for tab in category_tabs.children[1 : len(CATEGORIES) + 1]
-        ]
-
-
-def submit_results(
-    model_name: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-    mode: str,
-    submission_file: tempfile._TemporaryFileWrapper,
-    version: str,
-    review_model_type: ReviewModelType,
-    programming_language: str,
-    comment_language: str,
-):
-    """
-    Handle submission of results with model metadata.
-    """
-    if submission_file is None:
-        return styled_error("No submission file provided")
-
-    if not model_name:
-        return styled_error("Model name is required")
-
-    if not model_type:
-        return styled_error("Please select a model type")
-
-    if not mode:
-        return styled_error("Please select an inference mode")
-
-    file_path = submission_file.name
-    logger.info(f"Received submission for model {model_name}: {file_path}")
-
-    # Add metadata to the submission
-    metadata = {
-        "model_name": model_name,
-        "base_model": base_model,
-        "revision": revision if revision else "main",
-        "precision": precision,
-        "weight_type": weight_type,
-        "model_type": model_type,
-        "mode": mode,
-        "version": version,
-        "review_model_type": review_model_type,
-        "programming_language": programming_language,
-        "comment_language": comment_language,
-    }
-
-    # Process the submission
-    result = process_submission(file_path, metadata, version=version)
-
-    # Refresh the leaderboard data
-    global LEADERBOARD_DF
-    try:
-        logger.info(
-            f"Refreshing leaderboard data after submission for version {version}..."
-        )
-        LEADERBOARD_DF = get_leaderboard_df(version=version)
-        logger.info("Refreshed leaderboard data after submission")
-    except Exception as e:
-        logger.error(f"Error refreshing leaderboard data: {e}")
-
-    return result
-
-
-def refresh_data(version=CURRENT_VERSION):
-    """
-    Refresh the leaderboard data and update all components.
-    """
-    try:
-        logger.info(f"Performing scheduled refresh of leaderboard data...")
-        # Get new data
-        main_df = get_leaderboard_df(version=version)
-        category_dfs = [
-            get_category_leaderboard_df(category, version=version)
-            for category in CATEGORIES
-        ]
-
-        # For gr.Dataframe, we return the actual dataframes
-        return main_df, *category_dfs
-
-    except Exception as e:
-        logger.error(f"Error in scheduled refresh: {e}")
-        return None, *[None for _ in CATEGORIES]
-
+LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 
-def update_leaderboards(version):
-    """
-    Update all leaderboard components with data for the selected version.
-    """
-    try:
-        new_df = get_leaderboard_df(version=version)
-        category_dfs = [
-            get_category_leaderboard_df(category, version=version)
-            for category in CATEGORIES
-        ]
-        return new_df, *category_dfs
-    except Exception as e:
-        logger.error(f"Error updating leaderboards for version {version}: {e}")
-        return None, *[None for _ in CATEGORIES]
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-
-def create_performance_plot(
-    selected_models, category, metric="f1_binary", version=CURRENT_VERSION
-):
-    """
-    Create a radar plot comparing model performance for selected models.
-    """
-    if category == "All Results":
-        df = get_leaderboard_df(version=version)
-    else:
-        df = get_category_leaderboard_df(category, version=version)
-
-    if df.empty:
-        return go.Figure()
-
-    # Lowercase model_name in df and selected_models
-    df = df.copy()
-    df["model_name"] = df["model_name"].str.lower()
-    selected_models = [m.lower() for m in selected_models]
-    df = df[df["model_name"].isin(selected_models)]
-    metric_cols = [col for col in df.columns if metric in col]
-    fig = go.Figure()
-    colors = ["#8FCCCC", "#C2A4B6", "#98B4A6", "#B68F7C"]
-    for idx, model in enumerate(selected_models):
-        model_data = df[df["model_name"] == model]
-        if not model_data.empty:
-            values = model_data[metric_cols].values[0].tolist()
-            values = values + [values[0]]
-            categories = [col.replace(f"_{metric}", "") for col in metric_cols]
-            # Replace 'jailbreaked' with 'jailbroken' in categories
-            categories = [cat.replace('jailbreaked', 'jailbroken') for cat in categories]
-            categories = categories + [categories[0]]
-            fig.add_trace(
-                go.Scatterpolar(
-                    r=values,
-                    theta=categories,
-                    name=model,
-                    line_color=colors[idx % len(colors)],
-                    fill="toself",
-                )
-            )
-    fig.update_layout(
-        paper_bgcolor="#000000",
-        plot_bgcolor="#000000",
-        font={"color": "#ffffff"},
-        title={
-            "text": f"{category} - {metric.upper()} Score Comparison",
-            "font": {"color": "#ffffff", "size": 24},
-        },
-        polar=dict(
-            bgcolor="#000000",
-            radialaxis=dict(
-                visible=True,
-                range=[0, 1],
-                gridcolor="#333333",
-                linecolor="#333333",
-                tickfont={"color": "#ffffff"},
+def init_leaderboard(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(
+                AutoEvalColumn.params.name,
+                type="slider",
+                min=0.01,
+                max=150,
+                label="Select the number of parameters (B)",
             ),
-            angularaxis=dict(
-                gridcolor="#333333",
-                linecolor="#333333",
-                tickfont={"color": "#ffffff"},
+            ColumnFilter(
+                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
             ),
-        ),
-        height=600,
-        showlegend=True,
-        legend=dict(
-            yanchor="top",
-            y=0.99,
-            xanchor="right",
-            x=0.99,
-            bgcolor="rgba(0,0,0,0.5)",
-            font={"color": "#ffffff"},
-        ),
-    )
-    return fig
-
-
-def update_model_choices(version):
-    """
-    Update the list of available models for the given version.
-    """
-    df = get_leaderboard_df(version=version)
-    if df.empty:
-        return []
-    return sorted(df["model_name"].str.lower().unique().tolist())
-
-
-def update_visualization(selected_models, selected_category, selected_metric, version):
-    """
-    Update the visualization based on user selections.
-    """
-    if not selected_models:
-        return go.Figure()
-    return create_performance_plot(
-        selected_models, selected_category, selected_metric, version
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
     )
 
 
-# Create Gradio app
-demo = gr.Blocks(css=custom_css, theme=custom_theme)
-
-CATEGORY_DISPLAY_MAP = {
-    "Python": "Python",
-    "Java": "Java",
-    "Scala": "Scala",
-    "Go": "Go"
-}
-# Create reverse mapping for lookups
-CATEGORY_REVERSE_MAP = {v: k for k, v in CATEGORY_DISPLAY_MAP.items()}
-
+demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
-    # gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-    gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-    with gr.Row():
-        tabs = gr.Tabs(elem_classes="tab-buttons")
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with tabs:
-            with gr.TabItem("Leaderboard", elem_id="codereview-leaderboard-tab", id=0):
-                with gr.Row():
-                    version_selector = gr.Dropdown(
-                        choices=BENCHMARK_VERSIONS,
-                        label="Benchmark Version",
-                        value=CURRENT_VERSION,
-                        interactive=True,
-                        elem_classes="version-selector",
-                        scale=1,
-                        visible=False,
-                    )
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+            with gr.Column():
                 with gr.Row():
-                    search_input = gr.Textbox(
-                        placeholder="Search by models (use ; to split)",
-                        label="Search",
-                        elem_id="search-bar",
-                        scale=2,
-                    )
-                    comment_language_filter = gr.Dropdown(
-                        choices=["en", "ru"],
-                        label="Comment Language",
-                        multiselect=True,
-                        value=[],
-                        interactive=True,
-                        scale=1,
-                    )
-                    programming_language_filter = gr.Dropdown(
-                        choices=["Python", "Java", "Scala", "Go"],
-                        label="Programming Language",
-                        multiselect=True,
-                        value=[],
-                        interactive=True,
-                        scale=1,
-                    )
-                with gr.Row():
-                    topic_filter = gr.Dropdown(
-                        choices=TOPICS,
-                        label="Topic",
-                        multiselect=True,
-                        value=[],
-                        interactive=True,
-                        scale=2,
-                    )
-                    column_selector = gr.Dropdown(
-                        choices=get_all_column_choices(),
-                        label="Columns",
-                        multiselect=True,
-                        value=get_initial_columns(),
-                        interactive=True,
-                        visible=False,
-                        scale=1,
-                    )
-                with gr.Row():
-                    refresh_button = gr.Button(
-                        "Refresh", scale=0, elem_id="refresh-button"
-                    )
-
-                # Create tabs for each category
-                with gr.Tabs(elem_classes="category-tabs") as category_tabs:
-                    # First tab for average metrics across all categories
-                    with gr.TabItem("All Results", elem_id="overall-tab"):
-                        leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-                    # Create a tab for each category using display names
-                    for category in CATEGORIES:
-                        display_name = CATEGORY_DISPLAY_MAP.get(category, category)
-                        elem_id = f"category-{display_name.lower().replace(' ', '-').replace('&', 'and')}-tab"
-                        with gr.TabItem(display_name, elem_id=elem_id):
-                            category_df = get_category_leaderboard_df(
-                                category, version=CURRENT_VERSION
-                            )
-                            category_leaderboard = init_leaderboard(category_df)
-
-                # Connect search and filter inputs to update function
-                def update_with_search_filters(
-                    version=CURRENT_VERSION,
-                    search_query="",
-                    comment_languages=None,
-                    selected_columns=None,
-                ):
-                    """
-                    Update the leaderboards with search and filter settings.
-                    """
-                    return refresh_data_with_filters(
-                        version, search_query, comment_languages, selected_columns
-                    )
-
-                # Refresh button functionality
-                def refresh_and_update(
-                    version, search_query, comment_languages, selected_columns
-                ):
-                    """
-                    Refresh data, update LEADERBOARD_DF, and return updated components.
-                    """
-                    global LEADERBOARD_DF
-                    main_df = get_leaderboard_df(version=version)
-                    LEADERBOARD_DF = main_df  # Update the global DataFrame
-                    return refresh_data_with_filters(
-                        version, search_query, comment_languages, selected_columns
-                    )
-
-                refresh_button.click(
-                    fn=refresh_and_update,
-                    inputs=[
-                        version_selector,
-                        search_input,
-                        comment_language_filter,
-                        column_selector,
-                    ],
-                    outputs=[leaderboard]
-                    + [
-                        category_tabs.children[i].children[0]
-                        for i in range(1, len(CATEGORIES) + 1)
-                    ],
-                )
-                # Search input functionality
-                search_input.change(
-                    fn=refresh_data_with_filters,
-                    inputs=[
-                        version_selector,
-                        search_input,
-                        comment_language_filter,
-                        column_selector,
-                    ],
-                    outputs=[leaderboard]
-                    + [
-                        category_tabs.children[i].children[0]
-                        for i in range(1, len(CATEGORIES) + 1)
-                    ],
-                )
-
-                # Comment language filter functionality
-                comment_language_filter.change(
-                    fn=refresh_data_with_filters,
-                    inputs=[
-                        version_selector,
-                        search_input,
-                        comment_language_filter,
-                        column_selector,
-                    ],
-                    outputs=[leaderboard]
-                    + [
-                        category_tabs.children[i].children[0]
-                        for i in range(1, len(CATEGORIES) + 1)
-                    ],
-                )
-
-                # Version selector functionality
-                version_selector.change(
-                    fn=refresh_data_with_filters,
-                    inputs=[
-                        version_selector,
-                        search_input,
-                        comment_language_filter,
-                        column_selector,
-                    ],
-                    outputs=[leaderboard]
-                    + [
-                        category_tabs.children[i].children[0]
-                        for i in range(1, len(CATEGORIES) + 1)
-                    ],
-                )
-
-                # Update the update_columns function to handle updating all tabs at once
-                def update_columns(selected_columns):
-                    """
-                    Update all leaderboards to show the selected columns.
-                    Ensures all selected columns are preserved in the update.
-
-                    """
-
-                    try:
-                        logger.info(f"Updating columns to show: {selected_columns}")
-
-                        # If no columns are selected, use default visible columns
-                        if not selected_columns or len(selected_columns) == 0:
-                            selected_columns = get_default_visible_columns()
-                            logger.info(
-                                f"No columns selected, using defaults: {selected_columns}"
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+                with gr.Column():
+                    with gr.Accordion(
+                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            finished_eval_table = gr.components.Dataframe(
+                                value=finished_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
                             )
-
-                        # Convert display names to internal names
-                        internal_selected_columns = [
-                            x.lower()
-                            .replace(" ", "_")
-                            .replace("(", "")
-                            .replace(")", "")
-                            .replace("_recall", "_recall_binary")
-                            .replace("_precision", "_precision_binary")
-                            for x in selected_columns
-                        ]
-
-                        # Get the current data with ALL columns preserved
-                        main_df = get_leaderboard_df(version=version_selector.value)
-
-                        # Get category dataframes with ALL columns preserved
-                        category_dfs = [
-                            get_category_leaderboard_df(
-                                category, version=version_selector.value
+                    with gr.Accordion(
+                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            running_eval_table = gr.components.Dataframe(
+                                value=running_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
                             )
-                            for category in CATEGORIES
-                        ]
 
-                        # Log columns for debugging
-                        logger.info(f"Main dataframe columns: {list(main_df.columns)}")
-                        logger.info(
-                            f"Selected columns (internal): {internal_selected_columns}"
-                        )
-
-                        # IMPORTANT: Make sure model_name is always included
-                        if (
-                            "model_name" in main_df.columns
-                            and "model_name" not in internal_selected_columns
-                        ):
-                            internal_selected_columns = [
-                                "model_name"
-                            ] + internal_selected_columns
-
-                        # Initialize the main leaderboard with the selected columns
-                        # We're passing the internal_selected_columns directly to preserve the selection
-                        main_leaderboard = init_leaderboard(
-                            main_df, internal_selected_columns
-                        )
-
-                        # Initialize category dataframes with the same selected columns
-                        # This ensures consistency across all tabs
-                        category_leaderboards = []
-                        for df in category_dfs:
-                            # Use the same selected columns for each category
-                            # init_leaderboard will automatically handle filtering to columns that exist
-                            category_leaderboards.append(
-                                init_leaderboard(df, internal_selected_columns)
+                    with gr.Accordion(
+                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            pending_eval_table = gr.components.Dataframe(
+                                value=pending_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
                             )
-
-                        return main_leaderboard, *category_leaderboards
-
-                    except Exception as e:
-                        logger.error(f"Error updating columns: {e}")
-                        import traceback
-
-                        logger.error(traceback.format_exc())
-                        return leaderboard, *[
-                            tab.children[0]
-                            for tab in category_tabs.children[1 : len(CATEGORIES) + 1]
-                        ]
-
-                # Connect column selector to update function
-                column_selector.change(
-                    fn=update_columns,
-                    inputs=[column_selector],
-                    outputs=[leaderboard]
-                    + [
-                        category_tabs.children[i].children[0]
-                        for i in range(1, len(CATEGORIES) + 1)
-                    ],
-                )
-
-            # with gr.TabItem("About", elem_id="codereview-about-tab", id=2):
-            #     gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-            with gr.TabItem("Submit", elem_id="codereview-submit-tab", id=1):
-                gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Row():
-                    # with gr.Column(scale=3):
-                    #     gr.Markdown("# ✉️✨ Submit your results here!", elem_classes="markdown-text")
-                    with gr.Column(scale=1):
-                        # Add version selector specifically for the submission tab
-                        submission_version_selector = gr.Dropdown(
-                            choices=BENCHMARK_VERSIONS,
-                            label="Benchmark Version",
-                            value=CURRENT_VERSION,
-                            interactive=True,
-                            elem_classes="version-selector",
-                            visible=False,
-                        )
-
-                with gr.Row():
-                    with gr.Column():
-                        model_name_textbox = gr.Textbox(label="Model name")
-                        mode_selector = gr.Dropdown(
-                            choices=[m.name for m in Mode],
-                            label="Mode",
-                            multiselect=False,
-                            value=None,
-                            interactive=True,
-                        )
-                        revision_name_textbox = gr.Textbox(
-                            label="Revision commit", placeholder="main"
-                        )
-                        model_type = gr.Dropdown(
-                            choices=[
-                                t.to_str("-")
-                                for t in ModelType
-                                if t != ModelType.Unknown and t != ModelType.ClosedSource
-                            ],
-                            label="Model type",
-                            multiselect=False,
-                            value=None,
-                            interactive=True,
-                        )
-                        review_model_type = gr.Dropdown(
-                            choices=[t.name for t in ReviewModelType],
-                            label="Review model type",
-                            multiselect=False,
-                            value=ReviewModelType.CUSTOM.name,
-                            interactive=True,
-                        )
-                        programming_language_selector = gr.Dropdown(
-                            choices=["Python", "Java", "Scala", "Go"],
-                            label="Programming Language",
-                            multiselect=False,
-                            value=None,
-                            interactive=True,
-                        )
-                        comment_language_selector = gr.Dropdown(
-                            choices=["en", "ru"],
-                            label="Comment Language",
-                            multiselect=False,
-                            value="en",
-                            interactive=True,
-                        )
-
-                    with gr.Column():
-                        precision = gr.Dropdown(
-                            choices=[
-                                i.name for i in Precision if i != Precision.Unknown
-                            ],
-                            label="Precision",
-                            multiselect=False,
-                            value="float16",
-                            interactive=True,
-                        )
-                        weight_type = gr.Dropdown(
-                            choices=[i.name for i in WeightType],
-                            label="Weights type",
-                            multiselect=False,
-                            value="Original",
-                            interactive=True,
-                        )
-                        base_model_name_textbox = gr.Textbox(
-                            label="Base model (for delta or adapter weights)"
-                        )
-
-                with gr.Row():
-                    file_input = gr.File(
-                        label="Upload JSONL Results File", file_types=[".jsonl"]
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name")
+                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    model_type = gr.Dropdown(
+                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        label="Model type",
+                        multiselect=False,
+                        value=None,
+                        interactive=True,
                     )
 
-                submit_button = gr.Button("Submit Results")
-                result_output = gr.Markdown()
-
-                submit_button.click(
-                    fn=submit_results,
-                    inputs=[
-                        model_name_textbox,
-                        base_model_name_textbox,
-                        revision_name_textbox,
-                        precision,
-                        weight_type,
-                        model_type,
-                        mode_selector,
-                        file_input,
-                        submission_version_selector,
-                        review_model_type,
-                        programming_language_selector,
-                        comment_language_selector,
-                    ],
-                    outputs=result_output,
-                )
-
-    # Version selector functionality
-    version_selector.change(
-        fn=update_leaderboards,
-        inputs=[version_selector],
-        outputs=[leaderboard]
-        + [
-            category_tabs.children[i].children[0] for i in range(1, len(CATEGORIES) + 1)
-        ],
-    ).then(
-        lambda version: refresh_data_with_filters(version),
-        inputs=[version_selector],
-        outputs=[leaderboard]
-        + [
-            category_tabs.children[i].children[0] for i in range(1, len(CATEGORIES) + 1)
-        ],
-    )
+                with gr.Column():
+                    precision = gr.Dropdown(
+                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                        label="Precision",
+                        multiselect=False,
+                        value="float16",
+                        interactive=True,
+                    )
+                    weight_type = gr.Dropdown(
+                        choices=[i.value.name for i in WeightType],
+                        label="Weights type",
+                        multiselect=False,
+                        value="Original",
+                        interactive=True,
+                    )
+                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+            submit_button.click(
+                add_new_eval,
+                [
+                    model_name_textbox,
+                    base_model_name_textbox,
+                    revision_name_textbox,
+                    precision,
+                    weight_type,
+                    model_type,
+                ],
+                submission_result,
+            )
 
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
 
-# Set up the scheduler to refresh data periodically
 scheduler = BackgroundScheduler()
-scheduler.add_job(refresh_data, "interval", minutes=30)
+scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-
-# Launch the app
-if __name__ == "__main__":
-    demo.launch()
+demo.queue(default_concurrency_limit=40).launch()

data/.gitkeep

@@ -1 +0,0 @@
-# Keep this directory in git 

data/leaderboard_data.json

@@ -1,30 +0,0 @@
-{
-  "leaderboard": [
-    {
-      "model_name": "example/model",
-      "programming_language": "Python",
-      "comment_language": "English",
-      "taxonomy_category": "Bug Detection",
-      "bleu": 0.5,
-      "llm_pass_1": 0.5,
-      "llm_pass_5": 0.5,
-      "llm_pass_10": 0.5,
-      "metrics": {
-        "readability": 5,
-        "relevance": 5,
-        "explanation_clarity": 5,
-        "problem_identification": 5,
-        "actionability": 5,
-        "completeness": 5,
-        "specificity": 5,
-        "contextual_adequacy": 5,
-        "consistency": 5,
-        "brevity": 5
-      },
-      "submission_ip": "127.0.0.1",
-      "submission_date": "2024-01-01T00:00:00Z"
-    }
-  ],
-  "last_updated": "2025-07-03T13:10:47.434623+00:00",
-  "total_entries": 1
-}

data/submissions.json

@@ -1,5 +0,0 @@
-{
-  "submissions": [],
-  "last_updated": "2025-07-03T13:10:47.435548+00:00",
-  "total_submissions": 0
-}

example_submission.jsonl

@@ -1,4 +0,0 @@
-{"model_name": "GPT-4-CodeReview", "programming_language": "Python", "comment_language": "en", "topic": "Code Reliability", "observation_id": "obs_001", "code_snippet": "def calculate_sum(a, b):\n    return a + b", "review_text": "This function is simple and correct, but consider adding type hints and docstring for better documentation.", "readability": 8.5, "relevance": 9.0, "explanation_clarity": 7.8, "problem_identification": 8.2, "actionability": 8.7, "completeness": 8.0, "specificity": 7.5, "contextual_adequacy": 8.3, "consistency": 8.8, "brevity": 7.2, "pass_at_1": 0.75, "pass_at_5": 0.88, "pass_at_10": 0.92, "bleu_at_10": 0.65, "total_evaluations": 100}
-{"model_name": "GPT-4-CodeReview", "programming_language": "Java", "comment_language": "en", "topic": "Coding Standards", "observation_id": "obs_002", "code_snippet": "public class Calculator {\n    public int add(int a, int b) {\n        return a + b;\n    }\n}", "review_text": "Consider following Java naming conventions and adding JavaDoc comments. The method is functionally correct.", "readability": 8.2, "relevance": 8.8, "explanation_clarity": 7.5, "problem_identification": 8.0, "actionability": 8.5, "completeness": 7.8, "specificity": 7.2, "contextual_adequacy": 8.1, "consistency": 8.6, "brevity": 7.0, "pass_at_1": 0.72, "pass_at_5": 0.85, "pass_at_10": 0.90, "bleu_at_10": 0.62, "total_evaluations": 100}
-{"model_name": "Claude-3-CodeReview", "programming_language": "Scala", "comment_language": "ru", "topic": "Performance Issues", "observation_id": "obs_003", "code_snippet": "def fibonacci(n: Int): Int = {\n  if (n <= 1) n\n  else fibonacci(n-1) + fibonacci(n-2)\n}", "review_text": "Эта реализация неэффективна из-за экспоненциальной сложности. Рекомендуется использовать мемоизацию или итеративный подход.", "readability": 8.8, "relevance": 8.5, "explanation_clarity": 8.2, "problem_identification": 9.2, "actionability": 8.3, "completeness": 8.5, "specificity": 8.0, "contextual_adequacy": 8.6, "consistency": 8.2, "brevity": 8.8, "pass_at_1": 0.78, "pass_at_5": 0.89, "pass_at_10": 0.93, "bleu_at_10": 0.68, "total_evaluations": 100}
-{"model_name": "Llama-CodeReview", "programming_language": "Go", "comment_language": "en", "topic": "Variables", "observation_id": "obs_004", "code_snippet": "package main\n\nimport \"fmt\"\n\nfunc main() {\n    var x int = 5\n    var y int = 10\n    fmt.Println(x + y)\n}", "review_text": "Consider using short variable declarations (:=) for local variables. Also, the variable names could be more descriptive.", "readability": 7.5, "relevance": 7.8, "explanation_clarity": 7.0, "problem_identification": 7.5, "actionability": 7.2, "completeness": 7.8, "specificity": 6.8, "contextual_adequacy": 7.3, "consistency": 7.6, "brevity": 6.5, "pass_at_1": 0.65, "pass_at_5": 0.78, "pass_at_10": 0.85, "bleu_at_10": 0.55, "total_evaluations": 100} 

gradio_test.ipynb

@@ -1,32 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "agent_env",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.13.2"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

leaderboard_data.json

@@ -1,32 +0,0 @@
-{
-    "entries": [
-        {
-            "model_name": "GPT-4-CodeReview",
-            "model_type": "LLM",
-            "mode": "Strict",
-            "review_model_type": "gpt-4",
-            "programming_language": "Python",
-            "comment_language": "en",
-            "topic": "Code Reliability",
-            "submission_date": "2024-10-06T12:00:00Z",
-            "version": "v0",
-            "readability": 8.5,
-            "relevance": 9.0,
-            "explanation_clarity": 7.8,
-            "problem_identification": 8.2,
-            "actionability": 8.7,
-            "completeness": 8.0,
-            "specificity": 7.5,
-            "contextual_adequacy": 8.3,
-            "consistency": 8.8,
-            "brevity": 7.2,
-            "pass_at_1": 0.75,
-            "pass_at_5": 0.88,
-            "pass_at_10": 0.92,
-            "bleu_at_10": 0.65,
-            "total_evaluations": 100
-        }
-    ],
-    "last_updated": "2024-10-06T12:00:00Z",
-    "version": "v0"
-}

requirements.txt

@@ -1,8 +1,16 @@
-gradio==4.44.1
-pandas>=2.0.0
-huggingface_hub>=0.20.0
-datasets>=2.0.0
-apscheduler>=3.10.0
-python-dotenv>=1.0.0
-plotly>=5.18.0
-pydantic==2.10.6
+APScheduler
+black
+datasets
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.13
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece

src/__init__.py

@@ -1 +0,0 @@
-# CodeReview Leaderboard - Source Module 

src/about.py

@@ -1,59 +1,72 @@
-"""
-Text content for the CodeReview Bench Leaderboard.
-"""
+from dataclasses import dataclass
+from enum import Enum
 
-TITLE = """
-<div style="text-align: center; margin-bottom: 1rem">
-    <h1>CodeReview Bench Leaderboard</h1>
-</div>
-"""
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
 
-INTRODUCTION_TEXT = """
-## Introduction
 
-CodeReview Bench is a comprehensive benchmark for evaluating the quality and effectiveness of automated code review systems.
-This leaderboard tracks model performance across various programming languages and review criteria,
-including readability, relevance, explanation clarity, and actionability.
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    task0 = Task("anli_r1", "acc", "ANLI")
+    task1 = Task("logiqa", "acc_norm", "LogiQA")
+
+NUM_FEWSHOT = 0 # Change with your few shot
+# ---------------------------------------------------
 
-Models are evaluated on their ability to provide high-quality code reviews that are helpful,
-accurate, and actionable across multiple programming languages and review categories.
-"""
 
-LLM_BENCHMARKS_TEXT = """
-CodeReview Bench is a comprehensive benchmark for evaluating automated code review systems across programming languages and review quality dimensions.
 
-It evaluates models on their ability to provide high-quality code reviews using both LLM-based multimetric evaluation (readability, relevance, explanation clarity, problem identification, actionability, completeness, specificity, contextual adequacy, consistency, brevity) and exact-match metrics (pass@1, pass@5, pass@10) presented in our paper.
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
 
-The benchmark supports both Russian and English comment languages across 4 programming languages including Python, Java, Go, Scala
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = """
+Intro text
+"""
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = f"""
+## How it works
+
+## Reproducibility
+To reproduce our results, here is the commands you can run:
 
 """
 
 EVALUATION_QUEUE_TEXT = """
-## Submit Your Model
+## Some good practices before submitting a model
+
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
 
-To add your model to the CodeReview Bench leaderboard:
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
 
-1. Run your evaluation using the CodeReview Bench framework
-2. Upload your results in .jsonl format using this form.
-3. Once validated, your model will appear on the leaderboard.
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
 
-### Requirements:
-- Results must include all required metrics: LLM-based multimetric scores and exact-match metrics
-- Submissions should cover multiple programming languages where applicable
-- Both Russian and English comment languages are supported
+### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
 
-### ✉️✨ Ready? Upload your results below!
+## In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
+If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
 """
 
-CITATION_BUTTON_LABEL = "Cite CodeReview Bench"
-
-CITATION_BUTTON_TEXT = """
-@misc{codereviewbench2025,
-  author = {CodeReview Bench Team},
-  title = {CodeReview Bench: Comprehensive Benchmark for Automated Code Review Systems},
-  year = {2025},
-  publisher = {GitHub},
-  journal = {GitHub repository},
-  howpublished = {\\url{https://github.com/your-org/codereview-bench}}
-}
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
 """

src/display/__init__.py

@@ -1 +0,0 @@
-# Display utilities module 

src/display/css_html_js.py

@@ -1,97 +1,105 @@
-"""
-CSS and styling for the CodeReview Bench Leaderboard.
-"""
-
 custom_css = """
+
 .markdown-text {
     font-size: 16px !important;
-    text-align: justify !important;
-    line-height: 1.0 !important;
-    margin-top: 10px !important;
-    margin-bottom: 10px !important;
 }
 
-.tab-buttons button.selected {
-    border-color: #f4f4f5 !important;
-    background: #3f3f46 !important;
-    color: #f4f4f5 !important;
+#models-to-add-text {
+    font-size: 18px !important;
 }
 
-#citation-button textarea {
-    font-family: monospace !important;
+#citation-button span {
+    font-size: 16px !important;
 }
 
-.leaderboard-container {
-    margin-top: 20px;
+#citation-button textarea {
+    font-size: 16px !important;
 }
 
-.category-header {
-    font-weight: bold;
-    background-color: #f5f5f5;
-    padding: 10px;
-    margin-top: 15px;
-    border-radius: 5px;
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
 }
 
-.metric-name {
-    font-weight: bold;
-    color: #a1a1aa !important;
+#leaderboard-table {
+    margin-top: 15px
 }
 
-.model-name {
-    font-weight: bold;
+#leaderboard-table-lite {
+    margin-top: 15px
 }
 
-.model-link:hover {
-    text-decoration: underline;
-    color: #ffffff !important;
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
 }
-
-.version-selector {
-    margin: 0 !important;
-    padding: 5px;
-    border-radius: 5px;
+ 
+#search-bar {
+    padding: 0px;
 }
 
-.version-selector label {
-    font-weight: bold;
-    color: #f4f4f5 !important;
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+#leaderboard-table td:nth-child(2),
+#leaderboard-table th:nth-child(2) {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
 }
 
-.version-selector select {
-    border-color: #3f3f46 !important;
-    border-radius: 5px;
+.tab-buttons button {
+    font-size: 20px;
 }
 
-/* Make sure the version selector is properly aligned with refresh button */
-.version-selector > .block {
-    padding: 0 !important;
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
 }
 
-.version-selector > .block > .wrap {
-    position: relative;
-    top: -5px;
+#scale-logo .download {
+    display: none;
 }
-
-/* Force background/border for common layout containers */
-.gradio-row > .block,
-.gradio-column > .block,
-.form,
-.panel {
-    /* background: #18181b !important; */ /* Removed background override */
-    border-color: #27272a80 !important; /* Made border color semi-transparent */
-    border-width: 1px !important; /* Ensure border is visible */
-    border-style: solid !important;
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
 }
-
-/* Target the specific file upload component area */
-.gradio-file .wrap {
-     /* background: #18181b !important; */ /* Removed background override */
-     border-color: #27272a !important;
+#filter_type label {
+    display: flex;
 }
-
-#refresh-button {
-    margin-top: 5px !important;
-    margin-bottom: 5px !important;
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
 }
 """
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -1,71 +1,27 @@
-"""
-Formatting utilities for the GuardBench Leaderboard.
-"""
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
-import pandas as pd
-import numpy as np
 
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
 
-def make_clickable_model(model_name: str) -> str:
-    """
-    Create a clickable link for a model name.
-    """
-    return f'<a href="https://huggingface.co/{model_name}" target="_blank">{model_name}</a>'
 
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
 
-def has_no_nan_values(df: pd.DataFrame, columns: list) -> pd.Series:
-    """
-    Check if a row has no NaN values in the specified columns.
-    """
-    return ~df[columns].isna().any(axis=1)
 
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
 
-def format_percentage(value: float) -> str:
-    """
-    Format a value as a percentage.
-    """
-    if pd.isna(value):
-        return "N/A"
-    return f"{value * 100:.2f}%"
 
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
 
-def format_number(value: float, precision: int = 2) -> str:
-    """
-    Format a number with specified precision.
-    """
-    if pd.isna(value):
-        return "N/A"
-    return f"{value:.{precision}f}"
 
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
 
-def styled_message(message: str) -> str:
-    """
-    Format a success message with styling.
-    """
-    return f"""
-    <div style="padding: 10px; border-radius: 5px; background-color: #e6f7e6; color: #2e7d32; border: 1px solid #2e7d32;">
-        ✅ {message}
-    </div>
-    """
 
-
-def styled_warning(message: str) -> str:
-    """
-    Format a warning message with styling.
-    """
-    return f"""
-    <div style="padding: 10px; border-radius: 5px; background-color: #fff8e1; color: #ff8f00; border: 1px solid #ff8f00;">
-        ⚠️ {message}
-    </div>
-    """
-
-
-def styled_error(message: str) -> str:
-    """
-    Format an error message with styling.
-    """
-    return f"""
-    <div style="padding: 10px; border-radius: 5px; background-color: #ffebee; color: #c62828; border: 1px solid #c62828;">
-        ❌ {message}
-    </div>
-    """
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,417 +1,110 @@
-"""
-Utility classes and functions for the CodeReview Bench Leaderboard display.
-"""
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
 
-from dataclasses import dataclass, field, fields
-from enum import Enum, auto
-from typing import List, Optional
+import pandas as pd
 
+from src.about import Tasks
 
-class Mode(Enum):
-    """Inference mode for the review model."""
-    CoT = auto()  # Chain of Thought
-    Strict = auto()
-
-    def __str__(self):
-        """String representation of the mode."""
-        return self.name
-
-
-class ModelType(Enum):
-    """Model types for the leaderboard."""
-    Unknown = auto()
-    OpenSource = auto()
-    ClosedSource = auto()
-    API = auto()
-
-    def to_str(self, separator: str = "-") -> str:
-        """Convert enum to string with separator."""
-        if self == ModelType.Unknown:
-            return "Unknown"
-        elif self == ModelType.OpenSource:
-            return f"Open{separator}Source"
-        elif self == ModelType.ClosedSource:
-            return f"Closed{separator}Source"
-        elif self == ModelType.API:
-            return "API"
-        return "Unknown"
-
-
-class ReviewModelType(str, Enum):
-    """Review model types for the leaderboard."""
-    GPT_4 = "gpt-4"
-    GPT_3_5 = "gpt-3.5-turbo"
-    CLAUDE = "claude"
-    LLAMA = "llama"
-    GEMINI = "gemini"
-    CUSTOM = "custom"
-
-    def __str__(self):
-        """String representation of the review model type."""
-        return self.value
-
-
-class Precision(Enum):
-    """Model precision types."""
-    Unknown = auto()
-    float16 = auto()
-    bfloat16 = auto()
-    float32 = auto()
-    int8 = auto()
-    int4 = auto()
-    NA = auto()
-
-    def __str__(self):
-        """String representation of the precision type."""
-        return self.name
-
-
-class WeightType(Enum):
-    """Model weight types."""
-    Original = auto()
-    Delta = auto()
-    Adapter = auto()
-    
-    def __str__(self):
-        """String representation of the weight type."""
-        return self.name
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
 @dataclass
-class ColumnInfo:
-    """Information about a column in the leaderboard."""
+class ColumnContent:
     name: str
-    display_name: str
-    type: str = "text"
+    type: str
+    displayed_by_default: bool
     hidden: bool = False
     never_hidden: bool = False
-    displayed_by_default: bool = True
-
 
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+## All the model information that we might need
 @dataclass
-class CodeReviewBenchColumn:
-    """Columns for the CodeReview Bench leaderboard."""
-    # Core metadata
-    model_name: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="model_name",
-        display_name="Model",
-        never_hidden=True,
-        displayed_by_default=True
-    ))
-    mode: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="mode",
-        display_name="Mode",
-        displayed_by_default=True
-    ))
-    model_type: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="model_type",
-        display_name="Access_Type",
-        displayed_by_default=True
-    ))
-    submission_date: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="submission_date",
-        display_name="Submission_Date",
-        displayed_by_default=False
-    ))
-    version: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="version",
-        display_name="Version",
-        displayed_by_default=False
-    ))
-    review_model_type: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="review_model_type",
-        display_name="Type",
-        displayed_by_default=False
-    ))
-    base_model: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="base_model",
-        display_name="Base Model",
-        displayed_by_default=False
-    ))
-    revision: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="revision",
-        display_name="Revision",
-        displayed_by_default=False
-    ))
-    precision: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="precision",
-        display_name="Precision",
-        displayed_by_default=False
-    ))
-    weight_type: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="weight_type",
-        display_name="Weight Type",
-        displayed_by_default=False
-    ))
-    topic: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="topic",
-        display_name="Topic",
-        displayed_by_default=True
-    ))
-
-    # LLM-based multimetric scores
-    readability: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="readability",
-        display_name="Readability",
-        type="number",
-        displayed_by_default=True
-    ))
-    relevance: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="relevance",
-        display_name="Relevance",
-        type="number",
-        displayed_by_default=True
-    ))
-    explanation_clarity: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="explanation_clarity",
-        display_name="Explanation_Clarity",
-        type="number",
-        displayed_by_default=True
-    ))
-    problem_identification: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="problem_identification",
-        display_name="Problem_Identification",
-        type="number",
-        displayed_by_default=True
-    ))
-    actionability: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="actionability",
-        display_name="Actionability",
-        type="number",
-        displayed_by_default=True
-    ))
-    completeness: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="completeness",
-        display_name="Completeness",
-        type="number",
-        displayed_by_default=True
-    ))
-    specificity: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="specificity",
-        display_name="Specificity",
-        type="number",
-        displayed_by_default=True
-    ))
-    contextual_adequacy: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="contextual_adequacy",
-        display_name="Contextual_Adequacy",
-        type="number",
-        displayed_by_default=True
-    ))
-    consistency: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="consistency",
-        display_name="Consistency",
-        type="number",
-        displayed_by_default=True
-    ))
-    brevity: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="brevity",
-        display_name="Brevity",
-        type="number",
-        displayed_by_default=True
-    ))
-
-    # LLM-based-exact-match metrics
-    pass_at_1: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="pass_at_1",
-        display_name="Pass@1",
-        type="number",
-        displayed_by_default=True
-    ))
-    pass_at_5: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="pass_at_5",
-        display_name="Pass@5",
-        type="number",
-        displayed_by_default=True
-    ))
-    pass_at_10: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="pass_at_10",
-        display_name="Pass@10",
-        type="number",
-        displayed_by_default=True
-    ))
-    bleu_at_10: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="bleu_at_10",
-        display_name="BLEU@10",
-        type="number",
-        displayed_by_default=True
-    ))
-
-    # Overall aggregated metrics
-    overall_score: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="overall_score",
-        display_name="Overall_Score",
-        type="number",
-        displayed_by_default=True
-    ))
-    multimetric_average: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="multimetric_average",
-        display_name="Multimetric_Average",
-        type="number",
-        displayed_by_default=True
-    ))
-    exact_match_average: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="exact_match_average",
-        display_name="Exact_Match_Average",
-        type="number",
-        displayed_by_default=True
-    ))
-    total_evaluations: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="total_evaluations",
-        display_name="Total_Evaluations",
-        type="number",
-        displayed_by_default=True
-    ))
-
-    # Language-specific metrics (Russian)
-    ru_readability: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="ru_readability",
-        display_name="RU_Readability",
-        type="number",
-        displayed_by_default=False
-    ))
-    ru_relevance: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="ru_relevance",
-        display_name="RU_Relevance",
-        type="number",
-        displayed_by_default=False
-    ))
-    ru_overall_score: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="ru_overall_score",
-        display_name="RU_Overall_Score",
-        type="number",
-        displayed_by_default=False
-    ))
-
-    # Language-specific metrics (English)
-    en_readability: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="en_readability",
-        display_name="EN_Readability",
-        type="number",
-        displayed_by_default=False
-    ))
-    en_relevance: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="en_relevance",
-        display_name="EN_Relevance",
-        type="number",
-        displayed_by_default=False
-    ))
-    en_overall_score: ColumnInfo = field(default_factory=lambda: ColumnInfo(
-        name="en_overall_score",
-        display_name="EN_Overall_Score",
-        type="number",
-        displayed_by_default=False
-    ))
-
-
-# Create instances for easy access
-CODEREVIEW_COLUMN = CodeReviewBenchColumn()
-
-# Extract column lists for different views
-COLS = [f.name for f in fields(CODEREVIEW_COLUMN)]
-DISPLAY_COLS = [getattr(CODEREVIEW_COLUMN, f.name).name for f in fields(CODEREVIEW_COLUMN)
-                if getattr(CODEREVIEW_COLUMN, f.name).displayed_by_default]
-
-# Manually reorder DISPLAY_COLS to put 'mode' after 'model_name'
-def reorder_display_cols():
-    cols = DISPLAY_COLS
-    if 'model_name' in cols and 'mode' in cols:
-        cols.remove('mode')
-        model_name_index = cols.index('model_name')
-        cols.insert(model_name_index + 1, 'mode')
-    return cols
-DISPLAY_COLS = reorder_display_cols()
-
-METRIC_COLS = [getattr(CODEREVIEW_COLUMN, f.name).name for f in fields(CODEREVIEW_COLUMN)
-               if getattr(CODEREVIEW_COLUMN, f.name).type == "number"]
-HIDDEN_COLS = [getattr(CODEREVIEW_COLUMN, f.name).name for f in fields(CODEREVIEW_COLUMN)
-               if getattr(CODEREVIEW_COLUMN, f.name).hidden]
-NEVER_HIDDEN_COLS = [getattr(CODEREVIEW_COLUMN, f.name).name for f in fields(CODEREVIEW_COLUMN)
-                     if getattr(CODEREVIEW_COLUMN, f.name).never_hidden]
-
-# Categories for CodeReview Bench (Programming Languages)
-CATEGORIES = [
-    'Python',
-    'Java', 
-    'Scala',
-    'Go'
-]
-
-# Language taxonomies for CodeReview Bench
-COMMENT_LANGUAGES = [
-    'ru',  # Russian
-    'en'   # English
-]
-
-# Topics for CodeReview Bench
-TOPICS = [
-    'Code Reliability',
-    'Coding Standards', 
-    'Code Organization',
-    'Performance Issues',
-    'Validation',
-    'Variables'
-]
-
-# Example categories
-EXAMPLE_CATEGORIES = [
-    'Bug_Fix',
-    'Code_Style',
-    'Performance',
-    'Security',
-    'Refactoring',
-    'Documentation',
-    'Testing',
-    'Architecture',
-    'Other'
-]
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = "" # emoji
 
-# Metrics for CodeReview Bench
-MULTIMETRIC_METRICS = [
-    "readability",
-    "relevance", 
-    "explanation_clarity",
-    "problem_identification",
-    "actionability",
-    "completeness",
-    "specificity",
-    "contextual_adequacy",
-    "consistency",
-    "brevity"
-]
 
-EXACT_MATCH_METRICS = [
-    "pass_at_1",
-    "pass_at_5",
-    "pass_at_10",
-    "bleu_at_10"
-]
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
 
-def get_all_column_choices():
-    """
-    Get all available column choices for the multiselect dropdown.
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
 
-    Returns:
-        List of tuples with (column_name, display_name) for all columns.
-    """
-    column_choices = []
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    Unknown = ModelDetails("?")
 
-    default_visible_columns = get_default_visible_columns()
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        return Precision.Unknown
 
-    for f in fields(CODEREVIEW_COLUMN):
-        column_info = getattr(CODEREVIEW_COLUMN, f.name)
-        # Create a tuple with both the internal name and display name
-        if column_info.name not in default_visible_columns:
-            column_choices.append((column_info.name, column_info.display_name))
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 
-    return column_choices
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-def get_default_visible_columns():
-    """
-    Get the list of column names that should be visible by default.
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]
 
-    Returns:
-        List of column names that are displayed by default.
-    """
-    return [getattr(CODEREVIEW_COLUMN, f.name).name for f in fields(CODEREVIEW_COLUMN)
-            if getattr(CODEREVIEW_COLUMN, f.name).displayed_by_default]

src/envs.py

@@ -1,27 +1,25 @@
 import os
+
 from huggingface_hub import HfApi
-from dotenv import load_dotenv
 
-# Load environment variables
-load_dotenv()
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-# Hugging Face configuration
-TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
-OWNER = os.environ.get("OWNER", "codereview-bench")  # Change to your org
-SUBMITTER_TOKEN = os.environ.get("SUBMITTER_TOKEN")
-ADMIN_USERNAME = os.environ.get("ADMIN_USERNAME")
-ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD")
+OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
 
-# Repository IDs
-REPO_ID = f"{OWNER}/codereview-bench"
-RESULTS_DATASET_ID = os.environ.get("RESULTS_DATASET_ID", f"{OWNER}/codereview-bench-results")
+REPO_ID = f"{OWNER}/leaderboard"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
 
-# Cache paths
-CACHE_PATH = os.getenv("HF_HOME", ".")
-DATA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
 
-# Local data paths
-LEADERBOARD_FILE = os.path.join(DATA_PATH, "leaderboard.json")
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
-# HF API instance
 API = HfApi(token=TOKEN)

src/leaderboard/__init__.py

@@ -1 +0,0 @@
-# Leaderboard processing module 

src/leaderboard/processor.py

@@ -1,271 +0,0 @@
-"""
-Process CodeReview Bench leaderboard data and submissions.
-"""
-
-import json
-import os
-import pandas as pd
-from datetime import datetime
-from typing import Dict, List, Tuple, Optional
-import numpy as np
-
-from src.display.utils import (
-    CODEREVIEW_COLUMN, DISPLAY_COLS, CATEGORIES, COMMENT_LANGUAGES, EXAMPLE_CATEGORIES,
-    MULTIMETRIC_METRICS, EXACT_MATCH_METRICS
-)
-
-
-def process_jsonl_submission(file_path: str) -> Tuple[List[Dict], str]:
-    """
-    Process a JSONL submission file for CodeReview Bench.
-    
-    Args:
-        file_path: Path to the JSONL submission file
-        
-    Returns:
-        Tuple of (entries_list, message)
-    """
-    try:
-        entries = []
-        with open(file_path, 'r', encoding='utf-8') as f:
-            for line_num, line in enumerate(f, 1):
-                line = line.strip()
-                if not line:
-                    continue
-                    
-                try:
-                    entry = json.loads(line)
-                    
-                    # Validate required fields
-                    required_fields = ['model_name', 'programming_language', 'comment_language']
-                    missing_fields = [field for field in required_fields if field not in entry]
-                    if missing_fields:
-                        return [], f"Missing required fields {missing_fields} in line {line_num}"
-                    
-                    # Validate metrics exist
-                    has_multimetric = any(metric in entry for metric in MULTIMETRIC_METRICS)
-                    has_exact_match = any(metric in entry for metric in EXACT_MATCH_METRICS)
-                    
-                    if not has_multimetric and not has_exact_match:
-                        return [], f"No valid metrics found in line {line_num}. Required: {MULTIMETRIC_METRICS + EXACT_MATCH_METRICS}"
-                    
-                    entries.append(entry)
-                    
-                except json.JSONDecodeError as e:
-                    return [], f"Invalid JSON in line {line_num}: {e}"
-                    
-        if not entries:
-            return [], "No valid entries found in submission file"
-            
-        return entries, f"Successfully processed {len(entries)} entries"
-        
-    except Exception as e:
-        return [], f"Error processing submission: {e}"
-
-
-def calculate_overall_score(entry: Dict) -> float:
-    """
-    Calculate overall score for a CodeReview Bench entry.
-    
-    Args:
-        entry: Dictionary containing model evaluation results
-        
-    Returns:
-        Overall score as float
-    """
-    # Calculate multimetric average
-    multimetric_scores = []
-    for metric in MULTIMETRIC_METRICS:
-        if metric in entry and isinstance(entry[metric], (int, float)):
-            multimetric_scores.append(entry[metric])
-    
-    multimetric_avg = np.mean(multimetric_scores) if multimetric_scores else 0
-    
-    # Calculate exact match average
-    exact_match_scores = []
-    for metric in EXACT_MATCH_METRICS:
-        if metric in entry and isinstance(entry[metric], (int, float)):
-            exact_match_scores.append(entry[metric])
-    
-    exact_match_avg = np.mean(exact_match_scores) if exact_match_scores else 0
-    
-    # Weighted combination (can be adjusted based on requirements)
-    overall_score = (multimetric_avg * 0.7) + (exact_match_avg * 0.3)
-    
-    return overall_score
-
-
-def load_leaderboard_data(file_path: str) -> Dict:
-    """
-    Load the leaderboard data from a JSON file.
-    """
-    if not os.path.exists(file_path):
-        version = "v0"
-        if "_v" in file_path:
-            version = file_path.split("_")[-1].split(".")[0]
-        return {"entries": [], "last_updated": datetime.now().isoformat(), "version": version}
-
-    with open(file_path, 'r') as f:
-        data = json.load(f)
-
-    # Ensure version field exists
-    if "version" not in data:
-        version = "v0"
-        if "_v" in file_path:
-            version = file_path.split("_")[-1].split(".")[0]
-        data["version"] = version
-
-    return data
-
-
-def save_leaderboard_data(data: Dict, file_path: str) -> None:
-    """
-    Save the leaderboard data to a JSON file.
-    """
-    # Ensure the directory exists
-    os.makedirs(os.path.dirname(file_path), exist_ok=True)
-
-    # Update the last_updated timestamp
-    data["last_updated"] = datetime.now().isoformat()
-
-    # Ensure version is set
-    if "version" not in data:
-        version = "v0"
-        if "_v" in file_path:
-            version = file_path.split("_")[-1].split(".")[0]
-        data["version"] = version
-
-    with open(file_path, 'w') as f:
-        json.dump(data, f, indent=2)
-
-
-def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
-    """
-    Convert leaderboard data to a pandas DataFrame for display.
-    """
-    rows = []
-
-    for entry in leaderboard_data.get("entries", []):
-        model_name = entry.get("model_name", "Unknown Model")
-
-        # Extract basic metadata
-        row = {
-            "model_name": model_name,
-            "model_type": entry.get("model_type", "Unknown"),
-            "mode": entry.get("mode", "Strict"),
-            "submission_date": entry.get("submission_date", ""),
-            "version": entry.get("version", "v0"),
-            "review_model_type": entry.get("review_model_type", "custom").lower()
-        }
-
-        # Add additional metadata fields if present
-        for key in ["base_model", "revision", "precision", "weight_type", "topic", "programming_language", "comment_language"]:
-            if key in entry:
-                row[key] = entry[key]
-
-        # Add multimetric scores
-        for metric in MULTIMETRIC_METRICS:
-            if metric in entry:
-                row[metric] = entry[metric]
-            else:
-                row[metric] = pd.NA
-
-        # Add exact match metrics
-        for metric in EXACT_MATCH_METRICS:
-            if metric in entry:
-                row[metric] = entry[metric]
-            else:
-                row[metric] = pd.NA
-
-        # Calculate aggregated metrics
-        multimetric_scores = [entry.get(metric, 0) for metric in MULTIMETRIC_METRICS if metric in entry and pd.notna(entry[metric])]
-        exact_match_scores = [entry.get(metric, 0) for metric in EXACT_MATCH_METRICS if metric in entry and pd.notna(entry[metric])]
-
-        if multimetric_scores:
-            row["multimetric_average"] = np.mean(multimetric_scores)
-        else:
-            row["multimetric_average"] = pd.NA
-
-        if exact_match_scores:
-            row["exact_match_average"] = np.mean(exact_match_scores)
-        else:
-            row["exact_match_average"] = pd.NA
-
-        # Calculate overall score
-        row["overall_score"] = calculate_overall_score(entry)
-
-        # Add language-specific metrics if available
-        for lang in COMMENT_LANGUAGES:
-            for metric in ["readability", "relevance", "overall_score"]:
-                lang_key = f"{lang}_{metric}"
-                if lang_key in entry:
-                    row[lang_key] = entry[lang_key]
-                else:
-                    row[lang_key] = pd.NA
-
-        # Add evaluation count
-        row["total_evaluations"] = entry.get("total_evaluations", entry.get("evaluation_count", pd.NA))
-
-        rows.append(row)
-
-    # Create DataFrame and sort by overall score
-    df = pd.DataFrame(rows)
-
-    # Ensure all expected columns exist
-    for metric in MULTIMETRIC_METRICS + EXACT_MATCH_METRICS:
-        if metric not in df.columns:
-            df[metric] = pd.NA
-
-    # Sort by overall score (descending)
-    if not df.empty:
-        df = df.sort_values(by="overall_score", ascending=False, na_position='last')
-
-    # Ensure summary columns exist
-    summary_cols = ["overall_score", "multimetric_average", "exact_match_average", "total_evaluations"]
-    for col in summary_cols:
-        if col not in df.columns:
-            df[col] = pd.NA
-
-    return df
-
-
-def add_entries_to_leaderboard(leaderboard_data: Dict, new_entries: List[Dict]) -> Dict:
-    """
-    Add new entries to the leaderboard, replacing any with the same model name.
-    """
-    # Create a mapping of existing entries by model name and version
-    existing_entries = {
-        (entry["model_name"], entry.get("version", "v0")): i
-        for i, entry in enumerate(leaderboard_data.get("entries", []))
-    }
-
-    # Process each new entry
-    for new_entry in new_entries:
-        model_name = new_entry.get("model_name")
-        version = new_entry.get("version", "v0")
-
-        # Add calculated metrics
-        new_entry["overall_score"] = calculate_overall_score(new_entry)
-        
-        # Calculate averages
-        multimetric_scores = [new_entry.get(metric) for metric in MULTIMETRIC_METRICS if metric in new_entry and pd.notna(new_entry[metric])]
-        exact_match_scores = [new_entry.get(metric) for metric in EXACT_MATCH_METRICS if metric in new_entry and pd.notna(new_entry[metric])]
-        
-        if multimetric_scores:
-            new_entry["multimetric_average"] = np.mean(multimetric_scores)
-        if exact_match_scores:
-            new_entry["exact_match_average"] = np.mean(exact_match_scores)
-
-        if (model_name, version) in existing_entries:
-            # Replace existing entry
-            leaderboard_data["entries"][existing_entries[(model_name, version)]] = new_entry
-        else:
-            # Add new entry
-            if "entries" not in leaderboard_data:
-                leaderboard_data["entries"] = []
-            leaderboard_data["entries"].append(new_entry)
-
-    # Update the last_updated timestamp
-    leaderboard_data["last_updated"] = datetime.now().isoformat()
-
-    return leaderboard_data

src/leaderboard/read_evals.py

@@ -0,0 +1,196 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # org_model_precision (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    revision: str # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original # Original or Adapter
+    architecture: str = "Unknown" 
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = "" # submission date of request file
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision.value.name}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision.value.name}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,  
+            revision= config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/populate.py

@@ -1,188 +1,58 @@
-"""
-Populate the CodeReview Bench leaderboard from HuggingFace datasets.
-"""
-
 import json
 import os
-import pandas as pd
-import tempfile
-from typing import Dict, List, Optional
-from datetime import datetime
-import numpy as np
-
-from huggingface_hub import hf_hub_download, HfApi
-from datasets import load_dataset
-
-from src.display.utils import CODEREVIEW_COLUMN, DISPLAY_COLS, CATEGORIES
-from src.envs import RESULTS_DATASET_ID, TOKEN, CACHE_PATH
-from src.leaderboard.processor import leaderboard_to_dataframe
-
-
-def get_latest_leaderboard(version="v0") -> Optional[Dict]:
-    """
-    Get the latest leaderboard data from HuggingFace dataset.
-    Fallback to local JSON file if HF download fails or is unavailable.
-    """
-    # First try to fetch from HuggingFace Hub
-    try:
-        leaderboard_path = hf_hub_download(
-            repo_id=RESULTS_DATASET_ID,
-            filename=f"leaderboards/leaderboard_{version}.json",
-            repo_type="dataset",
-            token=TOKEN
-        )
-        with open(leaderboard_path, 'r') as f:
-            return json.load(f)
-    except Exception as hf_err:
-        print(f"HF download failed or unavailable: {hf_err}. Trying local fallback...")
-
-    # Fallback: attempt to load a local leaderboard_data.json located at the project root
-    project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    local_path_candidates = [
-        os.path.join(project_root, "leaderboard_data.json"),  # legacy path in root
-        os.path.join(project_root, "data", "leaderboard.json"),  # path defined in envs.py
-    ]
-
-    for local_path in local_path_candidates:
-        if os.path.exists(local_path):
-            try:
-                with open(local_path, 'r') as f:
-                    return json.load(f)
-            except Exception as local_err:
-                print(f"Error loading local leaderboard file {local_path}: {local_err}")
-
-    # If nothing found, return None
-    return None
-
-
-def get_model_entry(model_name: str, mode: str, version="v0") -> Optional[Dict]:
-    """
-    Get a specific model's entry from the entries folder, uniquely identified by model_name, mode, and version.
-    """
-    try:
-        model_name_safe = model_name.replace("/", "_").replace(" ", "_")
-        mode_safe = str(mode).replace("/", "_").replace(" ", "_").lower()
-        entry_path = hf_hub_download(
-            repo_id=RESULTS_DATASET_ID,
-            filename=f"entries/entry_{model_name_safe}_{mode_safe}_{version}.json",
-            repo_type="dataset",
-            token=TOKEN
-        )
-        with open(entry_path, 'r') as f:
-            return json.load(f)
-    except Exception as e:
-        print(f"Error downloading model entry: {e}")
-        return None
-
-
-def get_all_entries(version="v0") -> List[Dict]:
-    """
-    Get all entries from the HuggingFace dataset.
-    """
-    try:
-        api = HfApi(token=TOKEN)
-        files = api.list_repo_files(repo_id=RESULTS_DATASET_ID, repo_type="dataset")
-        entry_files = [f for f in files if f.startswith("entries/") and f.endswith(f"_{version}.json")]
-
-        all_entries = []
-        for entry_file in entry_files:
-            try:
-                entry_path = hf_hub_download(
-                    repo_id=RESULTS_DATASET_ID,
-                    filename=entry_file,
-                    repo_type="dataset",
-                    token=TOKEN
-                )
-                with open(entry_path, 'r') as f:
-                    entry_data = json.load(f)
-                    all_entries.append(entry_data)
-            except Exception as e:
-                print(f"Error loading entry {entry_file}: {e}")
-
-        return all_entries
-    except Exception as e:
-        print(f"Error getting all entries: {e}")
-        return []
-
-
-def get_leaderboard_df(version="v0") -> pd.DataFrame:
-    """
-    Get the leaderboard data as a DataFrame.
-    """
-    # Get latest leaderboard data
-    leaderboard_data = get_latest_leaderboard(version)
-
-    if not leaderboard_data:
-        # If no leaderboard exists, try to build it from entries
-        entries = get_all_entries(version)
-        if entries:
-            leaderboard_data = {
-                "entries": entries,
-                "last_updated": datetime.now().isoformat(),
-                "version": version
-            }
-        else:
-            # Return empty DataFrame if no data available
-            return pd.DataFrame(columns=DISPLAY_COLS)
-
-    # Convert to DataFrame
-    return leaderboard_to_dataframe(leaderboard_data)
-
-
-def get_category_leaderboard_df(category: str, version="v0") -> pd.DataFrame:
-    """
-    Get the leaderboard data filtered by a specific programming language category.
-    """
-    # Get latest leaderboard data
-    leaderboard_data = get_latest_leaderboard(version)
-
-    if not leaderboard_data:
-        # If no leaderboard exists, try to build it from entries
-        entries = get_all_entries(version)
-        if entries:
-            leaderboard_data = {
-                "entries": entries,
-                "last_updated": datetime.now().isoformat(),
-                "version": version
-            }
-        else:
-            # Return empty DataFrame if no data available
-            return pd.DataFrame(columns=DISPLAY_COLS)
-
-    # Filter entries to only include those with data for the specified programming language
-    filtered_entries = []
-    for entry in leaderboard_data.get("entries", []):
-        # Check if entry has data for this programming language
-        programming_language = entry.get("programming_language", "").lower()
-        if programming_language == category.lower() or category.lower() == "other":
-            # For "other" category, include entries that don't match any specific language
-            if category.lower() == "other":
-                if programming_language not in [cat.lower() for cat in CATEGORIES[:-1]]:  # Exclude "Other" from check
-                    filtered_entries.append(entry)
-            else:
-                filtered_entries.append(entry)
-
-    # Create a new leaderboard data structure with the filtered entries
-    filtered_leaderboard = {
-        "entries": filtered_entries,
-        "last_updated": leaderboard_data.get("last_updated", datetime.now().isoformat()),
-        "version": version
-    }
-
-    # Convert to DataFrame
-    return leaderboard_to_dataframe(filtered_leaderboard)
 
+import pandas as pd
 
-def get_detailed_model_data(model_name: str, mode: str, version="v0") -> Dict:
-    """
-    Get detailed data for a specific model and mode.
-    """
-    entry = get_model_entry(model_name, mode, version)
-    if entry:
-        return entry
-    leaderboard_data = get_latest_leaderboard(version)
-    if leaderboard_data:
-        for entry in leaderboard_data.get("entries", []):
-            if entry.get("model_name") == model_name and str(entry.get("mode")).lower() == str(mode).lower():
-                return entry
-    return {}
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/__init__.py

@@ -1 +0,0 @@
-# Submission handling module 

src/submission/check_validity.py

@@ -0,0 +1,99 @@
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        if test_tokenizer:
+            try:
+                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return (
+                    False,
+                    f"uses a tokenizer which is not in a transformers release: {e}",
+                    None
+                )
+            except Exception as e:
+                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None
+        )
+
+    except Exception as e:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,184 +1,119 @@
-"""
-Handle submissions to the CodeReview Bench leaderboard.
-"""
-
 import json
 import os
-import tempfile
-from datetime import datetime
-from typing import Dict, List, Tuple
-
-from huggingface_hub import HfApi
-from datasets import load_dataset
-
-from src.display.formatting import styled_error, styled_message
-from src.envs import RESULTS_DATASET_ID, TOKEN, REPO_ID
-from src.leaderboard.processor import process_jsonl_submission, add_entries_to_leaderboard
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Does the model actually exist?
+    if revision == "":
+        revision = "main"
+
+    # Is the model on the hub?
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
 
+    model_size = get_model_size(model_info=model_info, precision=precision)
 
-def validate_submission(file_path: str) -> Tuple[bool, str]:
-    """
-    Validate a submission file.
-    """
-    try:
-        entries, message = process_jsonl_submission(file_path)
-        if not entries:
-            return False, message
-        return True, "Submission is valid"
-    except Exception as e:
-        return False, f"Error validating submission: {e}"
-
-
-def submit_entry_to_hub(entry: Dict, model_name: str, mode: str, version="v0") -> Tuple[bool, str]:
-    """
-    Submit a model's evaluation entry to the HuggingFace dataset. The entry is uniquely identified by model_name, mode, and version.
-    """
-    try:
-        # Create safe model name for file path
-        model_name_safe = model_name.replace("/", "_").replace(" ", "_")
-        mode_safe = str(mode).replace("/", "_").replace(" ", "_").lower()
-
-        # Create entry path in entries folder
-        entry_path = f"entries/entry_{model_name_safe}_{mode_safe}_{version}.json"
-
-        # Save entry to temporary file
-        with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as temp_file:
-            json.dump(entry, temp_file, indent=2)
-            temp_path = temp_file.name
-
-        # Upload file
-        api = HfApi(token=TOKEN)
-        api.upload_file(
-            path_or_fileobj=temp_path,
-            path_in_repo=entry_path,
-            repo_id=RESULTS_DATASET_ID,
-            repo_type="dataset",
-            commit_message=f"Add evaluation entry for {model_name} (mode {mode}, version {version})"
-        )
-
-        os.unlink(temp_path)
-        return True, f"Successfully uploaded evaluation entry for {model_name} (mode {mode})"
-    except Exception as e:
-        return False, f"Error submitting entry to dataset: {e}"
-
-
-def submit_leaderboard_to_hub(entries: List[Dict], version="v0") -> Tuple[bool, str]:
-    """
-    Submit updated leaderboard to the HuggingFace dataset.
-    """
-    try:
-        # Create leaderboard data
-        leaderboard_data = {
-            "entries": entries,
-            "last_updated": datetime.now().isoformat(),
-            "version": version
-        }
-
-        # Save to temporary file
-        with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as temp_file:
-            json.dump(leaderboard_data, temp_file, indent=2)
-            temp_path = temp_file.name
-
-        # Upload file
-        api = HfApi(token=TOKEN)
-        api.upload_file(
-            path_or_fileobj=temp_path,
-            path_in_repo=f"leaderboards/leaderboard_{version}.json",
-            repo_id=RESULTS_DATASET_ID,
-            repo_type="dataset",
-            commit_message=f"Update leaderboard for version {version}"
-        )
-
-        os.unlink(temp_path)
-        return True, "Leaderboard updated successfully"
-    except Exception as e:
-        return False, f"Error updating leaderboard: {e}"
-
-
-def process_submission(file_path: str, metadata: Dict, version="v0") -> str:
-    """
-    Process a submission to the CodeReview Bench leaderboard.
-    """
+    # Were the model card and license filled?
     try:
-        # Validate submission
-        is_valid, validation_message = validate_submission(file_path)
-        if not is_valid:
-            return styled_error(validation_message)
-
-        # Process the submission entries
-        entries, message = process_jsonl_submission(file_path)
-        if not entries:
-            return styled_error(f"Failed to process submission: {message}")
-
-        # Upload raw submission file
-        model_name = metadata.get("model_name", "unknown")
-        model_name_safe = model_name.replace("/", "_").replace(" ", "_")
-        
-        api = HfApi(token=TOKEN)
-        submission_path = f"submissions_{version}/{model_name_safe}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.jsonl"
-        api.upload_file(
-            path_or_fileobj=file_path,
-            path_in_repo=submission_path,
-            repo_id=RESULTS_DATASET_ID,
-            repo_type="dataset",
-            commit_message=f"Add raw submission for {model_name}"
-        )
-
-        # Process entries and add metadata
-        processed_entries = []
-        for entry in entries:
-            # Add metadata to entry
-            entry.update({
-                "model_name": metadata.get("model_name"),
-                "model_type": metadata.get("model_type"),
-                "review_model_type": str(metadata.get("review_model_type", "custom")).lower(),
-                "mode": metadata.get("mode"),
-                "base_model": metadata.get("base_model"),
-                "revision": metadata.get("revision"),
-                "precision": metadata.get("precision"),
-                "weight_type": metadata.get("weight_type"),
-                "version": version,
-                "submission_date": datetime.now().isoformat()
-            })
-            processed_entries.append(entry)
-
-        # Submit entries to entries folder
-        for entry in processed_entries:
-            success, message = submit_entry_to_hub(entry, model_name, metadata.get("mode"), version)
-            if not success:
-                return styled_error(message)
-
-        # Get all entries from HF dataset and update leaderboard
-        files = api.list_repo_files(repo_id=RESULTS_DATASET_ID, repo_type="dataset")
-        entry_files = [f for f in files if f.startswith("entries/") and f.endswith(f"_{version}.json")]
-
-        all_entries = []
-        for entry_file in entry_files:
-            try:
-                entry_path = api.hf_hub_download(
-                    repo_id=RESULTS_DATASET_ID,
-                    filename=entry_file,
-                    repo_type="dataset",
-                )
-                with open(entry_path, 'r') as f:
-                    entry_data = json.load(f)
-                    all_entries.append(entry_data)
-            except Exception as e:
-                print(f"Error loading entry {entry_file}: {e}")
-
-        # Update leaderboard with all entries
-        success, message = submit_leaderboard_to_hub(all_entries, version)
-        if not success:
-            return styled_error(message)
-
-        return styled_message("Submission successful! Model evaluated and leaderboard updated.")
-
-    except Exception as e:
-        return styled_error(f"Error processing submission: {e}")
-    finally:
-        # Clean up temporary files if they exist
-        try:
-            if os.path.exists(file_path):
-                os.remove(file_path)
-        except:
-            pass
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+        "private": False,
+    }
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )