zalupa5

app.py

@@ -33,6 +33,7 @@ from src.display.utils import (
     CATEGORIES,
     COMMENT_LANGUAGES,
     EXAMPLE_CATEGORIES,
+    TOPICS,
     ModelType,
     Mode,
     Precision,
@@ -350,10 +351,10 @@ def init_leaderboard(dataframe, visible_columns=None):
 
 
 def search_filter_leaderboard(
-    df, search_query="", model_types=None, version=CURRENT_VERSION
+    df, search_query="", comment_languages=None, version=CURRENT_VERSION
 ):
     """
-    Filter the leaderboard based on search query and model types.
+    Filter the leaderboard based on search query and comment languages.
     """
     if df is None or df.empty:
         return df
@@ -367,11 +368,14 @@ def search_filter_leaderboard(
             axis=1,
         )
 
-    # Apply model type filter
-    if model_types and len(model_types) > 0:
-        filtered_df = filtered_df[
-            filtered_df[GUARDBENCH_COLUMN.model_type.name].isin(model_types)
-        ]
+    # Apply comment language filter (assuming there's a comment_language column in the data)
+    if comment_languages and len(comment_languages) > 0:
+        # Look for a comment language column in the dataframe
+        comment_lang_cols = [col for col in filtered_df.columns if 'comment_language' in col.lower()]
+        if comment_lang_cols:
+            filtered_df = filtered_df[
+                filtered_df[comment_lang_cols[0]].isin(comment_languages)
+            ]
 
     # Apply search query
     if search_query:
@@ -398,7 +402,7 @@ def search_filter_leaderboard(
 
 
 def refresh_data_with_filters(
-    version=CURRENT_VERSION, search_query="", model_types=None, selected_columns=None
+    version=CURRENT_VERSION, search_query="", comment_languages=None, selected_columns=None
 ):
     """
     Refresh the leaderboard data and update all components with filtering.
@@ -429,10 +433,10 @@ def refresh_data_with_filters(
 
         # Apply filters to each dataframe
         filtered_main_df = search_filter_leaderboard(
-            main_df, search_query, model_types, version
+            main_df, search_query, comment_languages, version
         )
         filtered_category_dfs = [
-            search_filter_leaderboard(df, search_query, model_types, version)
+            search_filter_leaderboard(df, search_query, comment_languages, version)
             for df in category_dfs
         ]
 
@@ -502,6 +506,8 @@ def submit_results(
     submission_file: tempfile._TemporaryFileWrapper,
     version: str,
     review_model_type: ReviewModelType,
+    programming_language: str,
+    comment_language: str,
 ):
     """
     Handle submission of results with model metadata.
@@ -532,6 +538,8 @@ def submit_results(
         "mode": mode,
         "version": version,
         "review_model_type": review_model_type,
+        "programming_language": programming_language,
+        "comment_language": comment_language,
     }
 
     # Process the submission
@@ -691,22 +699,9 @@ demo = gr.Blocks(css=custom_css, theme=custom_theme)
 
 CATEGORY_DISPLAY_MAP = {
     "Python": "Python",
-    "JavaScript": "JavaScript",
     "Java": "Java",
-    "C++": "C++",
-    "C#": "C#",
-    "TypeScript": "TypeScript",
-    "Go": "Go",
-    "Rust": "Rust",
-    "Swift": "Swift",
-    "Kotlin": "Kotlin",
-    "Ruby": "Ruby",
-    "PHP": "PHP",
-    "C": "C",
     "Scala": "Scala",
-    "R": "R",
-    "Dart": "Dart",
-    "Other": "Other"
+    "Go": "Go"
 }
 # Create reverse mapping for lookups
 CATEGORY_REVERSE_MAP = {v: k for k, v in CATEGORY_DISPLAY_MAP.items()}
@@ -739,16 +734,31 @@ with demo:
                         elem_id="search-bar",
                         scale=2,
                     )
-                    model_type_filter = gr.Dropdown(
-                        choices=[
-                            t.to_str("-") for t in ModelType if t != ModelType.Unknown and t != ModelType.ClosedSource
-                        ],
-                        label="Access Type",
+                    comment_language_filter = gr.Dropdown(
+                        choices=["en", "ru"],
+                        label="Comment Language",
+                        multiselect=True,
+                        value=[],
+                        interactive=True,
+                        scale=1,
+                    )
+                    programming_language_filter = gr.Dropdown(
+                        choices=["Python", "Java", "Scala", "Go"],
+                        label="Programming Language",
                         multiselect=True,
                         value=[],
                         interactive=True,
                         scale=1,
                     )
+                with gr.Row():
+                    topic_filter = gr.Dropdown(
+                        choices=TOPICS,
+                        label="Topic",
+                        multiselect=True,
+                        value=[],
+                        interactive=True,
+                        scale=2,
+                    )
                     column_selector = gr.Dropdown(
                         choices=get_all_column_choices(),
                         label="Columns",
@@ -783,19 +793,19 @@ with demo:
                 def update_with_search_filters(
                     version=CURRENT_VERSION,
                     search_query="",
-                    model_types=None,
+                    comment_languages=None,
                     selected_columns=None,
                 ):
                     """
                     Update the leaderboards with search and filter settings.
                     """
                     return refresh_data_with_filters(
-                        version, search_query, model_types, selected_columns
+                        version, search_query, comment_languages, selected_columns
                     )
 
                 # Refresh button functionality
                 def refresh_and_update(
-                    version, search_query, model_types, selected_columns
+                    version, search_query, comment_languages, selected_columns
                 ):
                     """
                     Refresh data, update LEADERBOARD_DF, and return updated components.
@@ -804,7 +814,7 @@ with demo:
                     main_df = get_leaderboard_df(version=version)
                     LEADERBOARD_DF = main_df  # Update the global DataFrame
                     return refresh_data_with_filters(
-                        version, search_query, model_types, selected_columns
+                        version, search_query, comment_languages, selected_columns
                     )
 
                 refresh_button.click(
@@ -812,7 +822,7 @@ with demo:
                     inputs=[
                         version_selector,
                         search_input,
-                        model_type_filter,
+                        comment_language_filter,
                         column_selector,
                     ],
                     outputs=[leaderboard]
@@ -827,7 +837,7 @@ with demo:
                     inputs=[
                         version_selector,
                         search_input,
-                        model_type_filter,
+                        comment_language_filter,
                         column_selector,
                     ],
                     outputs=[leaderboard]
@@ -837,13 +847,13 @@ with demo:
                     ],
                 )
 
-                # Model type filter functionality
-                model_type_filter.change(
+                # Comment language filter functionality
+                comment_language_filter.change(
                     fn=refresh_data_with_filters,
                     inputs=[
                         version_selector,
                         search_input,
-                        model_type_filter,
+                        comment_language_filter,
                         column_selector,
                     ],
                     outputs=[leaderboard]
@@ -859,7 +869,7 @@ with demo:
                     inputs=[
                         version_selector,
                         search_input,
-                        model_type_filter,
+                        comment_language_filter,
                         column_selector,
                     ],
                     outputs=[leaderboard]
@@ -963,175 +973,10 @@ with demo:
                     ],
                 )
 
-            with gr.TabItem("Visualize", elem_id="codereview-viz-tab", id=1):
-                with gr.Row():
-                    with gr.Column():
-                        viz_version_selector = gr.Dropdown(
-                            choices=BENCHMARK_VERSIONS,
-                            label="Benchmark Version",
-                            value=CURRENT_VERSION,
-                            interactive=True,
-                            visible=False,
-                        )
-
-                        # New: Mode selector
-                        def get_model_mode_choices(version):
-                            df = get_leaderboard_df(version=version)
-                            if df.empty:
-                                return []
-                            return sorted([
-                                f"{str(row['model_name']).lower()} [{row['mode']}]"
-                                for _, row in df.drop_duplicates(subset=["model_name", "mode"]).iterrows()
-                            ])
-
-                        model_mode_selector = gr.Dropdown(
-                            choices=get_model_mode_choices(CURRENT_VERSION),
-                            label="Select Model(s) [Mode] to Compare",
-                            multiselect=True,
-                            interactive=True,
-                        )
-                    with gr.Column():
-                        # Add Overall Performance to categories, use display names
-                        viz_categories_display = ["All Results"] + [
-                            CATEGORY_DISPLAY_MAP.get(cat, cat) for cat in CATEGORIES
-                        ]
-                        category_selector = gr.Dropdown(
-                            choices=viz_categories_display,
-                            label="Select Category",
-                            value=viz_categories_display[0],
-                            interactive=True,
-                        )
-                        metric_selector = gr.Dropdown(
-                            choices=[
-                                "accuracy",
-                                "f1_binary",
-                                "precision_binary",
-                                "recall_binary",
-                                "error_ratio",
-                            ],
-                            label="Select Metric",
-                            value="accuracy",
-                            interactive=True,
-                        )
-
-                plot_output = gr.Plot()
-
-                # Update visualization when any selector changes
-                def update_visualization_with_mode(
-                    selected_model_modes, selected_category, selected_metric, version
-                ):
-                    if not selected_model_modes:
-                        return go.Figure()
-                    df = (
-                        get_leaderboard_df(version=version)
-                        if selected_category == "All Results"
-                        else get_category_leaderboard_df(selected_category, version=version)
-                    )
-                    if df.empty:
-                        return go.Figure()
-                    df = df.copy()
-                    df["model_name"] = df["model_name"].str.lower()
-                    selected_pairs = [s.rsplit(" [", 1) for s in selected_model_modes]
-                    selected_pairs = [
-                        (name.strip().lower(), mode.strip("] "))
-                        for name, mode in selected_pairs
-                    ]
-                    mask = df.apply(
-                        lambda row: (row["model_name"], str(row["mode"])) in selected_pairs,
-                        axis=1,
-                    )
-                    filtered_df = df[mask]
-                    metric_cols = [col for col in filtered_df.columns if selected_metric in col]
-                    fig = go.Figure()
-                    colors = ["#8FCCCC", "#C2A4B6", "#98B4A6", "#B68F7C"]
-                    for idx, (model_name, mode) in enumerate(selected_pairs):
-                        model_data = filtered_df[
-                            (filtered_df["model_name"] == model_name)
-                            & (filtered_df["mode"] == mode)
-                        ]
-                        if not model_data.empty:
-                            values = model_data[metric_cols].values[0].tolist()
-                            values = values + [values[0]]
-                            categories = [col.replace(f"_{selected_metric}", "") for col in metric_cols]
-                            # Replace 'jailbreaked' with 'jailbroken' in categories
-                            categories = [cat.replace('jailbreaked', 'jailbroken') for cat in categories]
-                            categories = categories + [categories[0]]
-                            fig.add_trace(
-                                go.Scatterpolar(
-                                    r=values,
-                                    theta=categories,
-                                    name=f"{model_name} [{mode}]",
-                                    line_color=colors[idx % len(colors)],
-                                    fill="toself",
-                                )
-                            )
-                    fig.update_layout(
-                        paper_bgcolor="#000000",
-                        plot_bgcolor="#000000",
-                        font={"color": "#ffffff"},
-                        title={
-                            "text": f"{selected_category} - {selected_metric.upper()} Score Comparison",
-                            "font": {"color": "#ffffff", "size": 24},
-                        },
-                        polar=dict(
-                            bgcolor="#000000",
-                            radialaxis=dict(
-                                visible=True,
-                                range=[0, 1],
-                                gridcolor="#333333",
-                                linecolor="#333333",
-                                tickfont={"color": "#ffffff"},
-                            ),
-                            angularaxis=dict(
-                                gridcolor="#333333",
-                                linecolor="#333333",
-                                tickfont={"color": "#ffffff"},
-                            ),
-                        ),
-                        height=600,
-                        showlegend=True,
-                        legend=dict(
-                            yanchor="top",
-                            y=0.99,
-                            xanchor="right",
-                            x=0.99,
-                            bgcolor="rgba(0,0,0,0.5)",
-                            font={"color": "#ffffff"},
-                        ),
-                    )
-                    return fig
-
-                # Connect selectors to update function
-                for control in [
-                    viz_version_selector,
-                    model_mode_selector,
-                    category_selector,
-                    metric_selector,
-                ]:
-                    control.change(
-                        fn=lambda smm, sc, s_metric, v: update_visualization_with_mode(
-                            smm, CATEGORY_REVERSE_MAP.get(sc, sc), s_metric, v
-                        ),
-                        inputs=[
-                            model_mode_selector,
-                            category_selector,
-                            metric_selector,
-                            viz_version_selector,
-                        ],
-                        outputs=plot_output,
-                    )
-
-                # Update model_mode_selector choices when version changes
-                viz_version_selector.change(
-                    fn=get_model_mode_choices,
-                    inputs=[viz_version_selector],
-                    outputs=[model_mode_selector],
-                )
-
             # with gr.TabItem("About", elem_id="codereview-about-tab", id=2):
             #     gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-            with gr.TabItem("Submit", elem_id="codereview-submit-tab", id=3):
+            with gr.TabItem("Submit", elem_id="codereview-submit-tab", id=1):
                 gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
                 with gr.Row():
@@ -1179,6 +1024,20 @@ with demo:
                             value=ReviewModelType.CUSTOM.name,
                             interactive=True,
                         )
+                        programming_language_selector = gr.Dropdown(
+                            choices=["Python", "Java", "Scala", "Go"],
+                            label="Programming Language",
+                            multiselect=False,
+                            value=None,
+                            interactive=True,
+                        )
+                        comment_language_selector = gr.Dropdown(
+                            choices=["en", "ru"],
+                            label="Comment Language",
+                            multiselect=False,
+                            value="en",
+                            interactive=True,
+                        )
 
                     with gr.Column():
                         precision = gr.Dropdown(
@@ -1222,6 +1081,8 @@ with demo:
                         file_input,
                         submission_version_selector,
                         review_model_type,
+                        programming_language_selector,
+                        comment_language_selector,
                     ],
                     outputs=result_output,
                 )

example_submission.jsonl

@@ -1,4 +1,4 @@
-{"model_name": "GPT-4-CodeReview", "programming_language": "python", "comment_language": "en", "readability": 8.5, "relevance": 9.0, "explanation_clarity": 7.8, "problem_identification": 8.2, "actionability": 8.7, "completeness": 8.0, "specificity": 7.5, "contextual_adequacy": 8.3, "consistency": 8.8, "brevity": 7.2, "pass_at_1": 0.75, "pass_at_5": 0.88, "pass_at_10": 0.92, "bleu_at_10": 0.65, "total_evaluations": 100}
-{"model_name": "GPT-4-CodeReview", "programming_language": "javascript", "comment_language": "en", "readability": 8.2, "relevance": 8.8, "explanation_clarity": 7.5, "problem_identification": 8.0, "actionability": 8.5, "completeness": 7.8, "specificity": 7.2, "contextual_adequacy": 8.1, "consistency": 8.6, "brevity": 7.0, "pass_at_1": 0.72, "pass_at_5": 0.85, "pass_at_10": 0.90, "bleu_at_10": 0.62, "total_evaluations": 100}
-{"model_name": "Claude-3-CodeReview", "programming_language": "python", "comment_language": "en", "readability": 8.8, "relevance": 8.5, "explanation_clarity": 8.2, "problem_identification": 8.0, "actionability": 8.3, "completeness": 8.5, "specificity": 8.0, "contextual_adequacy": 8.6, "consistency": 8.2, "brevity": 8.8, "pass_at_1": 0.78, "pass_at_5": 0.89, "pass_at_10": 0.93, "bleu_at_10": 0.68, "total_evaluations": 100}
-{"model_name": "Llama-CodeReview", "programming_language": "java", "comment_language": "en", "readability": 7.5, "relevance": 7.8, "explanation_clarity": 7.0, "problem_identification": 7.5, "actionability": 7.2, "completeness": 7.8, "specificity": 6.8, "contextual_adequacy": 7.3, "consistency": 7.6, "brevity": 6.5, "pass_at_1": 0.65, "pass_at_5": 0.78, "pass_at_10": 0.85, "bleu_at_10": 0.55, "total_evaluations": 100} 
+{"model_name": "GPT-4-CodeReview", "programming_language": "Python", "comment_language": "en", "topic": "Code Reliability", "observation_id": "obs_001", "code_snippet": "def calculate_sum(a, b):\n    return a + b", "review_text": "This function is simple and correct, but consider adding type hints and docstring for better documentation.", "readability": 8.5, "relevance": 9.0, "explanation_clarity": 7.8, "problem_identification": 8.2, "actionability": 8.7, "completeness": 8.0, "specificity": 7.5, "contextual_adequacy": 8.3, "consistency": 8.8, "brevity": 7.2, "pass_at_1": 0.75, "pass_at_5": 0.88, "pass_at_10": 0.92, "bleu_at_10": 0.65, "total_evaluations": 100}
+{"model_name": "GPT-4-CodeReview", "programming_language": "Java", "comment_language": "en", "topic": "Coding Standards", "observation_id": "obs_002", "code_snippet": "public class Calculator {\n    public int add(int a, int b) {\n        return a + b;\n    }\n}", "review_text": "Consider following Java naming conventions and adding JavaDoc comments. The method is functionally correct.", "readability": 8.2, "relevance": 8.8, "explanation_clarity": 7.5, "problem_identification": 8.0, "actionability": 8.5, "completeness": 7.8, "specificity": 7.2, "contextual_adequacy": 8.1, "consistency": 8.6, "brevity": 7.0, "pass_at_1": 0.72, "pass_at_5": 0.85, "pass_at_10": 0.90, "bleu_at_10": 0.62, "total_evaluations": 100}
+{"model_name": "Claude-3-CodeReview", "programming_language": "Scala", "comment_language": "ru", "topic": "Performance Issues", "observation_id": "obs_003", "code_snippet": "def fibonacci(n: Int): Int = {\n  if (n <= 1) n\n  else fibonacci(n-1) + fibonacci(n-2)\n}", "review_text": "Эта реализация неэффективна из-за экспоненциальной сложности. Рекомендуется использовать мемоизацию или итеративный подход.", "readability": 8.8, "relevance": 8.5, "explanation_clarity": 8.2, "problem_identification": 9.2, "actionability": 8.3, "completeness": 8.5, "specificity": 8.0, "contextual_adequacy": 8.6, "consistency": 8.2, "brevity": 8.8, "pass_at_1": 0.78, "pass_at_5": 0.89, "pass_at_10": 0.93, "bleu_at_10": 0.68, "total_evaluations": 100}
+{"model_name": "Llama-CodeReview", "programming_language": "Go", "comment_language": "en", "topic": "Variables", "observation_id": "obs_004", "code_snippet": "package main\n\nimport \"fmt\"\n\nfunc main() {\n    var x int = 5\n    var y int = 10\n    fmt.Println(x + y)\n}", "review_text": "Consider using short variable declarations (:=) for local variables. Also, the variable names could be more descriptive.", "readability": 7.5, "relevance": 7.8, "explanation_clarity": 7.0, "problem_identification": 7.5, "actionability": 7.2, "completeness": 7.8, "specificity": 6.8, "contextual_adequacy": 7.3, "consistency": 7.6, "brevity": 6.5, "pass_at_1": 0.65, "pass_at_5": 0.78, "pass_at_10": 0.85, "bleu_at_10": 0.55, "total_evaluations": 100} 

leaderboard_data.json

@@ -1,23 +1,32 @@
 {
-    "leaderboard": [
+    "entries": [
         {
-            "model_name": "example/model",
-            "bleu": 0.5,
-            "llm_pass_1": 0.5,
-            "llm_pass_5": 0.5,
-            "llm_pass_10": 0.5,
-            "metrics": {
-                "readability": 5,
-                "relevance": 5,
-                "explanation_clarity": 5,
-                "problem_identification": 5,
-                "actionability": 5,
-                "completeness": 5,
-                "specificity": 5,
-                "contextual_adequacy": 5,
-                "consistency": 5,
-                "brevity": 5
-            }
+            "model_name": "GPT-4-CodeReview",
+            "model_type": "LLM",
+            "mode": "Strict",
+            "review_model_type": "gpt-4",
+            "programming_language": "Python",
+            "comment_language": "en",
+            "topic": "Code Reliability",
+            "submission_date": "2024-10-06T12:00:00Z",
+            "version": "v0",
+            "readability": 8.5,
+            "relevance": 9.0,
+            "explanation_clarity": 7.8,
+            "problem_identification": 8.2,
+            "actionability": 8.7,
+            "completeness": 8.0,
+            "specificity": 7.5,
+            "contextual_adequacy": 8.3,
+            "consistency": 8.8,
+            "brevity": 7.2,
+            "pass_at_1": 0.75,
+            "pass_at_5": 0.88,
+            "pass_at_10": 0.92,
+            "bleu_at_10": 0.65,
+            "total_evaluations": 100
         }
-    ]
+    ],
+    "last_updated": "2024-10-06T12:00:00Z",
+    "version": "v0"
 }

src/display/utils.py

@@ -327,22 +327,9 @@ NEVER_HIDDEN_COLS = [getattr(CODEREVIEW_COLUMN, f.name).name for f in fields(COD
 # Categories for CodeReview Bench (Programming Languages)
 CATEGORIES = [
     'Python',
-    'JavaScript',
-    'Java',
-    'C++',
-    'C#',
-    'TypeScript',
-    'Go',
-    'Rust',
-    'Swift',
-    'Kotlin',
-    'Ruby',
-    'PHP',
-    'C',
+    'Java', 
     'Scala',
-    'R',
-    'Dart',
-    'Other'
+    'Go'
 ]
 
 # Language taxonomies for CodeReview Bench
@@ -351,6 +338,16 @@ COMMENT_LANGUAGES = [
     'en'   # English
 ]
 
+# Topics for CodeReview Bench
+TOPICS = [
+    'Code Reliability',
+    'Coding Standards', 
+    'Code Organization',
+    'Performance Issues',
+    'Validation',
+    'Variables'
+]
+
 # Example categories
 EXAMPLE_CATEGORIES = [
     'Bug_Fix',

src/populate.py

@@ -21,21 +21,38 @@ from src.leaderboard.processor import leaderboard_to_dataframe
 def get_latest_leaderboard(version="v0") -> Optional[Dict]:
     """
     Get the latest leaderboard data from HuggingFace dataset.
+    Fallback to local JSON file if HF download fails or is unavailable.
     """
+    # First try to fetch from HuggingFace Hub
     try:
-        # Try to download the leaderboard file
         leaderboard_path = hf_hub_download(
             repo_id=RESULTS_DATASET_ID,
             filename=f"leaderboards/leaderboard_{version}.json",
             repo_type="dataset",
             token=TOKEN
         )
-
         with open(leaderboard_path, 'r') as f:
             return json.load(f)
-    except Exception as e:
-        print(f"Error downloading leaderboard: {e}")
-        return None
+    except Exception as hf_err:
+        print(f"HF download failed or unavailable: {hf_err}. Trying local fallback...")
+
+    # Fallback: attempt to load a local leaderboard_data.json located at the project root
+    project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    local_path_candidates = [
+        os.path.join(project_root, "leaderboard_data.json"),  # legacy path in root
+        os.path.join(project_root, "data", "leaderboard.json"),  # path defined in envs.py
+    ]
+
+    for local_path in local_path_candidates:
+        if os.path.exists(local_path):
+            try:
+                with open(local_path, 'r') as f:
+                    return json.load(f)
+            except Exception as local_err:
+                print(f"Error loading local leaderboard file {local_path}: {local_err}")
+
+    # If nothing found, return None
+    return None
 
 
 def get_model_entry(model_name: str, mode: str, version="v0") -> Optional[Dict]: