update mj-bench

app.py

@@ -1,8 +1,14 @@
-import subprocess
+import os
+import json
 import gradio as gr
 import pandas as pd
+import numpy as np
+
+from pathlib import Path
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
+from datasets import load_dataset
+
 
 from src.about import (
     CITATION_BUTTON_LABEL,
@@ -11,6 +17,7 @@ from src.about import (
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
+    ABOUT_TEXT
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
@@ -27,319 +34,277 @@ from src.display.utils import (
     Precision
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+
+SUBSET_COUNTS = {
+    "Alignment-Object": 250,
+    "Alignment-Attribute": 229,
+    "Alignment-Action": 115,
+    "Alignment-Count": 55,
+    "Alignment-Location": 75,
+    "Safety-Toxicity-Crime": 29,
+    "Safety-Toxicity-Shocking": 31,
+    "Safety-Toxicity-Disgust": 42,
+    "Safety-Nsfw-Evident": 197,
+    "Safety-Nsfw-Evasive": 177,
+    "Safety-Nsfw-Subtle": 98,
+    "Quality-Distortion-Human_face": 169,
+    "Quality-Distortion-Human_limb": 152,
+    "Quality-Distortion-Object": 100,
+    "Quality-Blurry-Defocused": 350,
+    "Quality-Blurry-Motion": 350,
+    "Bias-Age": 80,
+    "Bias-Gender": 140,
+    "Bias-Race": 140,
+    "Bias-Nationality": 120,
+    "Bias-Religion": 60,
+}
+
+PERSPECTIVE_COUNTS= {
+    "Alignment": 724,
+    "Safety": 574,
+    "Quality": 1121,
+    "Bias": 540
+}
+
+
+
+META_DATA = ['Model', 'Model Type', 'Input Type', 'Organization']
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-
-# Searching and filtering
-def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
-    query: str,
-):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
+color_map = {
+    "Score Model": "#7497db",
+    "Opensource VLM": "#E8ECF2",
+    "Closesource VLM": "#ffcd75",
+    "Others": "#75809c",
+
+    # #7497db #E8ECF2 #ffcd75 #75809c
+}
+def color_model_type_column(df, color_map):
+    """
+    Apply color to the 'Model Type' column of the DataFrame based on a given color mapping.
+
+    Parameters:
+    df (pd.DataFrame): The DataFrame containing the 'Model Type' column.
+    color_map (dict): A dictionary mapping model types to colors.
+
+    Returns:
+    pd.Styler: The styled DataFrame.
+    """
+    # Function to apply color based on the model type
+    def apply_color(val):
+        color = color_map.get(val, "default")  # Default color if not specified in color_map
+        return f'background-color: {color}'
+    
+    # Format for different columns
+    format_dict = {col: "{:.1f}" for col in df.columns if col not in META_DATA}
+    format_dict['Overall Score'] = "{:.2f}"
+    format_dict[''] = "{:d}"
+
+    return df.style.applymap(apply_color, subset=['Model Type']).format(format_dict, na_rep='')
+
+def regex_table(dataframe, regex, filter_button, style=True):
+    """
+    Takes a model name as a regex, then returns only the rows that has that in it.
+    """
+    # Split regex statement by comma and trim whitespace around regexes
+    regex_list = [x.strip() for x in regex.split(",")]
+    # Join the list into a single regex pattern with '|' acting as OR
+    combined_regex = '|'.join(regex_list)
+    
+    # if filter_button, remove all rows with "ai2" in the model name
+    update_scores = False
+    if isinstance(filter_button, list) or isinstance(filter_button, str):
+        if "Score Model" not in filter_button:
+            dataframe = dataframe[~dataframe["Model Type"].str.contains("Score Model", case=False, na=False)]
+        if "Opensource VLM" not in filter_button:
+            dataframe = dataframe[~dataframe["Model Type"].str.contains("Opensource VLM", case=False, na=False)]
+        if "Closesource VLM" not in filter_button:
+            dataframe = dataframe[~dataframe["Model Type"].str.contains("Closesource VLM", case=False, na=False)]
+        if "Others" not in filter_button:
+            dataframe = dataframe[~dataframe["Model Type"].str.contains("Others", case=False, na=False)]
+    # Filter the dataframe such that 'model' contains any of the regex patterns
+    data = dataframe[dataframe["Model"].str.contains(combined_regex, case=False, na=False)]
+
+    data.reset_index(drop=True, inplace=True)
+
+    # replace column '' with count/rank
+    data.insert(0, '', range(1, 1 + len(data)))
+    
+    if style:
+        # apply color
+        data = color_model_type_column(data, color_map)
+
+    return data
+
+def get_leaderboard_results(results_path):
+    data_dir = Path(results_path)
+    files = [d for d in os.listdir(data_dir)] # TODO check if "Path(data_dir) / d" is a dir
+
+    df = pd.DataFrame()
+    for file in files:
+        if not file.endswith(".json"):
+            continue
+        with open(results_path / file) as rf:
+            result = json.load(rf)
+            result = pd.DataFrame(result)
+            df = pd.concat([result, df])
+    df.reset_index(drop=True, inplace=True)
     return df
 
-
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
-    ]
-    # We use COLS to maintain sorting
-    filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
-    ]
-    return filtered_df
-
-
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-    final_df = []
-    if query != "":
-        queries = [q.strip() for q in query.split(";")]
-        for _q in queries:
-            _q = _q.strip()
-            if _q != "":
-                temp_filtered_df = search_table(filtered_df, _q)
-                if len(temp_filtered_df) > 0:
-                    final_df.append(temp_filtered_df)
-        if len(final_df) > 0:
-            filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-            )
-
-    return filtered_df
-
-
-def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
-    # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
-
-    return filtered_df
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
+def avg_all_subset(orig_df: pd.DataFrame, columns_name: list, meta_data=META_DATA, subset_counts=SUBSET_COUNTS):
+    new_df = orig_df.copy()[meta_data + columns_name]
+    
+    # Filter the dictionary to include only the counts relevant to the specified columns
+    new_subset_counts = {col: subset_counts[col] for col in columns_name}
+    
+    # Calculate the weights for each subset
+    total_count = sum(new_subset_counts.values())
+    weights = {subset: count / total_count for subset, count in new_subset_counts.items()}
+    
+    # Calculate the weight_avg value for each row
+    def calculate_weighted_avg(row):
+        weighted_sum = sum(row[col] * weights[col] for col in columns_name)
+        return weighted_sum
+
+    new_df["Overall Score"] = new_df.apply(calculate_weighted_avg, axis=1)
+
+    cols = meta_data + ["Overall Score"]  + columns_name
+    new_df = new_df[cols].sort_values(by="Overall Score", ascending=False).reset_index(drop=True)
+    return new_df
+
+
+def avg_all_perspective(orig_df: pd.DataFrame, columns_name: list, meta_data=META_DATA, perspective_counts=PERSPECTIVE_COUNTS):
+    new_df = orig_df[meta_data + columns_name]
+    new_perspective_counts = {col: perspective_counts[col] for col in columns_name}
+    total_count = sum(perspective_counts.values())
+    weights = {perspective: count / total_count for perspective, count in perspective_counts.items()}
+    def calculate_weighted_avg(row):
+        weighted_sum = sum(row[col] * weights[col] for col in columns_name)
+        return weighted_sum
+    new_df["Overall Score"] = new_df.apply(calculate_weighted_avg, axis=1)
+
+    cols = meta_data + ["Overall Score"]  + columns_name
+    new_df = new_df[cols].sort_values(by="Overall Score", ascending=False).reset_index(drop=True)
+    return new_df
+    
+
+results_path = Path("./evals/mjbench/eval-results")
+orig_df = get_leaderboard_results(results_path)
+colmuns_name = list(SUBSET_COUNTS.keys())
+detailed_df = avg_all_subset(orig_df, colmuns_name).round(2)
+
+results_path = Path("./evals/mjbench/overall-results")
+orig_df = get_leaderboard_results(results_path)
+colmuns_name = list(PERSPECTIVE_COUNTS.keys())
+perspective_df = avg_all_perspective(orig_df, colmuns_name).round(2)
+
+total_models = len(detailed_df)
+with gr.Blocks(css=custom_css) as app:
+    with gr.Row():
+        with gr.Column(scale=6):
+            gr.Markdown(INTRODUCTION_TEXT.format(str(total_models)))
+        with gr.Column(scale=4): 
+            gr.Markdown("![](./src/mj-bench-logo.png)")
+            # gr.HTML(BGB_LOGO, elem_classes="logo")
+    
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏆 MJ-Bench Leaderboard"):
             with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden
-                            ],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-            leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                    + shown_columns.value
-                ],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS],
-                headers=COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-            )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
-                        search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
+                search_overall = gr.Textbox(
+                    label="Model Search (delimit with , )", 
+                    placeholder="🔍 Search model (separate multiple queries with ``) and press ENTER...",
+                    show_label=False
+                )
+                model_type_overall = gr.CheckboxGroup(
+                    choices=["Score Model", "Opensource VLM", "Closesource VLM", "Others"], 
+                    value=["Score Model", "Opensource VLM", "Closesource VLM", "Others"], 
+                    label="Model Types", 
+                    show_label=False, 
+                    interactive=True,
                 )
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
+                mjbench_table_overall_hidden = gr.Dataframe(
+                    perspective_df,
+                    headers=perspective_df.columns.tolist(),
+                    elem_id="mjbench_leadboard_overall_hidden",
+                    wrap=True,
+                    visible=False,
+                )
+                mjbench_table_overall = gr.Dataframe(
+                    regex_table(
+                        perspective_df.copy(), 
+                        "", 
+                        ["Score Model", "Opensource VLM", "Closesource VLM", "Others"]
+                     ),
+                    headers=perspective_df.columns.tolist(),
+                    elem_id="mjbench_leadboard_overall",
+                    wrap=True,
+                    height=1000,
+                )
+        # with gr.TabItem("🔍 MJ-Bench Detailed Results"):
+        #     with gr.Row():
+        #         search_detail = gr.Textbox(
+        #             label="Model Search (delimit with , )", 
+        #             placeholder="🔍 Search model (separate multiple queries with ``) and press ENTER...",
+        #             show_label=False
+        #         )
+        #         model_type_detail = gr.CheckboxGroup(
+        #             choices=["Score Model", "Opensource VLM", "Closesource VLM", "Others"], 
+        #             value=["Score Model", "Opensource VLM", "Closesource VLM", "Others"], 
+        #             label="Model Types", 
+        #             show_label=False, 
+        #             interactive=True,
+        #         )
+        #     with gr.Row():
+        #         mjbench_table_detail_hidden = gr.Dataframe(
+        #             detailed_df,
+        #             headers=detailed_df.columns.tolist(),
+        #             elem_id="mjbench_detailed_hidden",
+        #             # column_widths = ["500px", "500px"],
+        #             wrap=True,
+        #             visible=False,
+        #         )
+        #         mjbench_table_detail = gr.Dataframe(
+        #             regex_table(
+        #                 detailed_df.copy(), 
+        #                 "", 
+        #                 ["Score Model", "Opensource VLM", "Closesource VLM", "Others"]
+        #              ),
+        #             headers=detailed_df.columns.tolist(),
+        #             elem_id="mjbench_detailed",
+        #             column_widths = ["40px", "200px", "180px", "130px", "150px"] + ["130px"]*50,
+        #             wrap=True,
+        #             height=1000,
+        #         )
+        with gr.TabItem("About"):
             with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
+                gr.Markdown(ABOUT_TEXT)
+    
+    with gr.Accordion("📚 Citation", open=False):
             citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
+                value=r"""@misc{mjbench2024mjbench,
+  title={MJ-BENCH: Is Your Multimodal Reward Model Really a Good Judge?},
+  author={Chen*, Zhaorun and Du*, Yichao and Wen, Zichen and Zhou, Yiyang and Cui, Chenhang and Weng, Zhenzhen and Tu, Haoqin and Wang, Chaoqi and Tong, Zhengwei and HUANG, Leria and Chen, Canyu and Ye Qinghao and Zhu, Zhihong and Zhang, Yuqing and Zhou, Jiawei and Zhao, Zhuokai and Rafailov, Rafael and Finn, Chelsea and Yao, Huaxiu},
+  year={2024}
+}""",
+                lines=7,
+                label="Copy the following to cite these results.",
                 elem_id="citation-button",
                 show_copy_button=True,
             )
-
+        
+    search_overall.change(regex_table, inputs=[mjbench_table_overall_hidden, search_overall, model_type_overall], outputs=mjbench_table_overall)
+    model_type_overall.change(regex_table, inputs=[mjbench_table_overall_hidden, search_overall, model_type_overall], outputs=mjbench_table_overall)
+    
+    # search_detail.change(regex_table, inputs=[mjbench_table_detail_hidden, search_detail, model_type_detail], outputs=mjbench_table_detail)
+    # model_type_detail.change(regex_table, inputs=[mjbench_table_detail_hidden, search_detail, model_type_detail], outputs=mjbench_table_detail)
+    
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(restart_space, "interval", seconds=18000) # restarted every 3h
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+# app.queue(default_concurrency_limit=40).launch()
+app.launch(allowed_paths=['./', "./src", "./evals"])

evals/mjbench/detailed-results/AestheticsPredictor.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "AestheticsPredictor",
+        "Model Type": "Score Model",
+        "Input Type": "Single Image",
+        "Organization": "LAION",
+        "Alignment-Object": 35.9,
+        "Alignment-Attribute": 38.4,
+        "Alignment-Action": 43.6,
+        "Alignment-Location": 31.6,
+        "Alignment-Count": 35.7,
+        "Alignment-Avg": 34.8,
+        "Safety-Toxicity-Crime": 51.7,
+        "Safety-Toxicity-Shocking": 58.6,
+        "Safety-Toxicity-Disgust": 64.3,
+        "Safety-Toxicity-Avg": 57.3,
+        "Safety-Nsfw-Evident": 14.6,
+        "Safety-Nsfw-Evasive": 55.2,
+        "Safety-Nsfw-Subtle": 14.2,
+        "Safety-Nsfw-Avg": 37.5,
+        "Quality-Distortion-Human_face": 78.7,
+        "Quality-Distortion-Human_limb": 57.1,
+        "Quality-Distortion-Object": 51.3,
+        "Quality-Distortion-Avg": 52.1,
+        "Quality-Blurry-Defocused": 90.1,
+        "Quality-Blurry-Motion": 93.4,
+        "Quality-Blurry-Avg": 91.6,
+        "Bias-Age": 59.4,
+        "Bias-Gender": 62.0,
+        "Bias-Race": 64.2,
+        "Bias-Nationality": 62.4,
+        "Bias-Religion": 61.0,
+        "Bias-Avg": 62.0,
+        "Bias-Age-NDS": 85.3,
+        "Bias-Gender-NDS": 85.9,
+        "Bias-Race-NDS": 86.3,
+        "Bias-Nationality-NDS": 85.8,
+        "Bias-Religion-NDS": 86.2,
+        "Bias-Avg-NDS": 85.9,
+        "Bias-Age-GES": 91.9,
+        "Bias-Gender-GES": 92.1,
+        "Bias-Race-GES": 92.4,
+        "Bias-Nationality-GES": 92.1,
+        "Bias-Religion-GES": 92.3,
+        "Bias-Avg-GES": 92.1
+    }
+]

evals/mjbench/detailed-results/BLIP-v2.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "BLIP-v2",
+        "Model Type": "Score Model",
+        "Input Type": "Single Image",
+        "Organization": "Salesforce",
+        "Alignment-Object": 23.5,
+        "Alignment-Attribute": 22.7,
+        "Alignment-Action": 24.8,
+        "Alignment-Location": 19.7,
+        "Alignment-Count": 16.1,
+        "Alignment-Avg": 21.5,
+        "Safety-Toxicity-Crime": 6.9,
+        "Safety-Toxicity-Shocking": 0.0,
+        "Safety-Toxicity-Disgust": 4.8,
+        "Safety-Toxicity-Avg": 4.5,
+        "Safety-Nsfw-Evident": 58.4,
+        "Safety-Nsfw-Evasive": 51.1,
+        "Safety-Nsfw-Subtle": 35.7,
+        "Safety-Nsfw-Avg": 49.1,
+        "Quality-Distortion-Human_face": 3.6,
+        "Quality-Distortion-Human_limb": 2.0,
+        "Quality-Distortion-Object": 1.1,
+        "Quality-Distortion-Avg": 1.9,
+        "Quality-Blurry-Defocused": 8.3,
+        "Quality-Blurry-Motion": 47.2,
+        "Quality-Blurry-Avg": 15.0,
+        "Bias-Age": 69.6,
+        "Bias-Gender": 68.5,
+        "Bias-Race": 65.9,
+        "Bias-Nationality": 68.6,
+        "Bias-Religion": 74.7,
+        "Bias-Avg": 68.5,
+        "Bias-Age-NDS": 85.3,
+        "Bias-Gender-NDS": 83.6,
+        "Bias-Race-NDS": 82.7,
+        "Bias-Nationality-NDS": 81.8,
+        "Bias-Religion-NDS": 87.5,
+        "Bias-Avg-NDS": 83.6,
+        "Bias-Age-GES": 92.2,
+        "Bias-Gender-GES": 91.3,
+        "Bias-Race-GES": 90.7,
+        "Bias-Nationality-GES": 90.4,
+        "Bias-Religion-GES": 93.1,
+        "Bias-Avg-GES": 91.3
+    }
+]

evals/mjbench/detailed-results/CLIP-v2.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "CLIP-v2",
+        "Model Type": "Score Model",
+        "Input Type": "Single Image",
+        "Organization": "LAION",
+        "Alignment-Object": 42.2,
+        "Alignment-Attribute": 45.9,
+        "Alignment-Action": 45.3,
+        "Alignment-Location": 43.4,
+        "Alignment-Count": 55.4,
+        "Alignment-Avg": 44.0,
+        "Safety-Toxicity-Crime": 89.7,
+        "Safety-Toxicity-Shocking": 96.6,
+        "Safety-Toxicity-Disgust": 97.6,
+        "Safety-Toxicity-Avg": 94.4,
+        "Safety-Nsfw-Evident": 20.8,
+        "Safety-Nsfw-Evasive": 4.5,
+        "Safety-Nsfw-Subtle": 16.6,
+        "Safety-Nsfw-Avg": 7.9,
+        "Quality-Distortion-Human_face": 26.6,
+        "Quality-Distortion-Human_limb": 17.2,
+        "Quality-Distortion-Object": 34.0,
+        "Quality-Distortion-Avg": 19.3,
+        "Quality-Blurry-Defocused": 50.6,
+        "Quality-Blurry-Motion": 63.7,
+        "Quality-Blurry-Avg": 56.7,
+        "Bias-Age": 57.2,
+        "Bias-Gender": 57.8,
+        "Bias-Race": 55.5,
+        "Bias-Nationality": 59.5,
+        "Bias-Religion": 60.8,
+        "Bias-Avg": 57.7,
+        "Bias-Age-NDS": 73.6,
+        "Bias-Gender-NDS": 75.2,
+        "Bias-Race-NDS": 73.1,
+        "Bias-Nationality-NDS": 79.1,
+        "Bias-Religion-NDS": 78.4,
+        "Bias-Avg-NDS": 75.2,
+        "Bias-Age-GES": 73.6,
+        "Bias-Gender-GES": 75.2,
+        "Bias-Race-GES": 73.1,
+        "Bias-Nationality-GES": 79.1,
+        "Bias-Religion-GES": 78.4,
+        "Bias-Avg-GES": 75.2
+    }
+]

evals/mjbench/detailed-results/Claude 3 Opus.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "Claude 3 Opus",
+        "Model Type": "Closesource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "Anthropic",
+        "Alignment-Object": 64.9,
+        "Alignment-Attribute": 38.9,
+        "Alignment-Action": 44.4,
+        "Alignment-Location": 55.3,
+        "Alignment-Count": 55.4,
+        "Alignment-Avg": 57.1,
+        "Safety-Toxicity-Crime": 62.1,
+        "Safety-Toxicity-Shocking": 37.9,
+        "Safety-Toxicity-Disgust": 50.0,
+        "Safety-Toxicity-Avg": 50.6,
+        "Safety-Nsfw-Evident": 10.5,
+        "Safety-Nsfw-Evasive": 6.2,
+        "Safety-Nsfw-Subtle": 3.6,
+        "Safety-Nsfw-Avg": 8.3,
+        "Quality-Distortion-Human_face": 26.6,
+        "Quality-Distortion-Human_limb": 19.3,
+        "Quality-Distortion-Object": 10.7,
+        "Quality-Distortion-Avg": 17.6,
+        "Quality-Blurry-Defocused": 89.6,
+        "Quality-Blurry-Motion": 93.3,
+        "Quality-Blurry-Avg": 92.7,
+        "Bias-Age": 53.9,
+        "Bias-Gender": 58.2,
+        "Bias-Race": 62.1,
+        "Bias-Nationality": 59.0,
+        "Bias-Religion": 54.0,
+        "Bias-Avg": 58.2,
+        "Bias-Age-NDS": 63.3,
+        "Bias-Gender-NDS": 66.1,
+        "Bias-Race-NDS": 67.5,
+        "Bias-Nationality-NDS": 66.9,
+        "Bias-Religion-NDS": 66.8,
+        "Bias-Avg-NDS": 66.1,
+        "Bias-Age-GES": 83.2,
+        "Bias-Gender-GES": 85.2,
+        "Bias-Race-GES": 86.5,
+        "Bias-Nationality-GES": 85.8,
+        "Bias-Religion-GES": 84.8,
+        "Bias-Avg-GES": 85.2
+    }
+]

evals/mjbench/detailed-results/GPT-4-vision.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "GPT-4-vision",
+        "Model Type": "Closesource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "OpenAI",
+        "Alignment-Object": 68.1,
+        "Alignment-Attribute": 62.9,
+        "Alignment-Action": 64.1,
+        "Alignment-Location": 67.1,
+        "Alignment-Count": 73.2,
+        "Alignment-Avg": 66.1,
+        "Safety-Toxicity-Crime": 75.9,
+        "Safety-Toxicity-Shocking": 69.0,
+        "Safety-Toxicity-Disgust": 81.0,
+        "Safety-Toxicity-Avg": 76.4,
+        "Safety-Nsfw-Evident": 69.5,
+        "Safety-Nsfw-Evasive": 43.2,
+        "Safety-Nsfw-Subtle": 32.5,
+        "Safety-Nsfw-Avg": 44.1,
+        "Quality-Distortion-Human_face": 87.6,
+        "Quality-Distortion-Human_limb": 57.6,
+        "Quality-Distortion-Object": 83.1,
+        "Quality-Distortion-Avg": 75.7,
+        "Quality-Blurry-Defocused": 98.8,
+        "Quality-Blurry-Motion": 99.3,
+        "Quality-Blurry-Avg": 99.2,
+        "Bias-Age": 76.7,
+        "Bias-Gender": 79.1,
+        "Bias-Race": 77.4,
+        "Bias-Nationality": 81.0,
+        "Bias-Religion": 86.5,
+        "Bias-Avg": 79.1,
+        "Bias-Age-NDS": 81.2,
+        "Bias-Gender-NDS": 80.2,
+        "Bias-Race-NDS": 77.6,
+        "Bias-Nationality-NDS": 79.9,
+        "Bias-Religion-NDS": 88.2,
+        "Bias-Avg-NDS": 80.2,
+        "Bias-Age-GES": 93.0,
+        "Bias-Gender-GES": 93.2,
+        "Bias-Race-GES": 92.2,
+        "Bias-Nationality-GES": 93.4,
+        "Bias-Religion-GES": 96.4,
+        "Bias-Avg-GES": 93.2
+    }
+]

evals/mjbench/detailed-results/GPT-4o.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "GPT-4o",
+        "Model Type": "Closesource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "OpenAI",
+        "Alignment-Object": 62.2,
+        "Alignment-Attribute": 57.2,
+        "Alignment-Action": 64.1,
+        "Alignment-Location": 63.2,
+        "Alignment-Count": 67.9,
+        "Alignment-Avg": 61.5,
+        "Safety-Toxicity-Crime": 86.2,
+        "Safety-Toxicity-Shocking": 96.6,
+        "Safety-Toxicity-Disgust": 95.2,
+        "Safety-Toxicity-Avg": 92.1,
+        "Safety-Nsfw-Evident": 72.3,
+        "Safety-Nsfw-Evasive": 51.7,
+        "Safety-Nsfw-Subtle": 38.9,
+        "Safety-Nsfw-Avg": 54.3,
+        "Quality-Distortion-Human_face": 99.4,
+        "Quality-Distortion-Human_limb": 78.2,
+        "Quality-Distortion-Object": 100.0,
+        "Quality-Distortion-Avg": 93.8,
+        "Quality-Blurry-Defocused": 100.0,
+        "Quality-Blurry-Motion": 100.0,
+        "Quality-Blurry-Avg": 100.0,
+        "Bias-Age": 60.9,
+        "Bias-Gender": 66.6,
+        "Bias-Race": 69.1,
+        "Bias-Nationality": 68.2,
+        "Bias-Religion": 69.6,
+        "Bias-Avg": 66.6,
+        "Bias-Age-NDS": 81.2,
+        "Bias-Gender-NDS": 82.7,
+        "Bias-Race-NDS": 82.8,
+        "Bias-Nationality-NDS": 83.2,
+        "Bias-Religion-NDS": 86.1,
+        "Bias-Avg-NDS": 82.7,
+        "Bias-Age-GES": 91.8,
+        "Bias-Gender-GES": 92.9,
+        "Bias-Race-GES": 93.1,
+        "Bias-Nationality-GES": 93.3,
+        "Bias-Religion-GES": 94.4,
+        "Bias-Avg-GES": 92.9
+    }
+]

evals/mjbench/detailed-results/Gemini Ultra.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "Gemini Ultra",
+        "Model Type": "Closesource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "Google",
+        "Alignment-Object": 71.7,
+        "Alignment-Attribute": 65.1,
+        "Alignment-Action": 63.2,
+        "Alignment-Location": 64.5,
+        "Alignment-Count": 67.8,
+        "Alignment-Avg": 67.2,
+        "Safety-Toxicity-Crime": 65.5,
+        "Safety-Toxicity-Shocking": 41.4,
+        "Safety-Toxicity-Disgust": 78.6,
+        "Safety-Toxicity-Avg": 64.0,
+        "Safety-Nsfw-Evident": 31.6,
+        "Safety-Nsfw-Evasive": 19.1,
+        "Safety-Nsfw-Subtle": 10.3,
+        "Safety-Nsfw-Avg": 22.7,
+        "Quality-Distortion-Human_face": 73.4,
+        "Quality-Distortion-Human_limb": 32.5,
+        "Quality-Distortion-Object": 61.0,
+        "Quality-Distortion-Avg": 55.7,
+        "Quality-Blurry-Defocused": 86.5,
+        "Quality-Blurry-Motion": 97.3,
+        "Quality-Blurry-Avg": 93.9,
+        "Bias-Age": 48.7,
+        "Bias-Gender": 56.9,
+        "Bias-Race": 62.9,
+        "Bias-Nationality": 60.0,
+        "Bias-Religion": 49.9,
+        "Bias-Avg": 56.9,
+        "Bias-Age-NDS": 72.6,
+        "Bias-Gender-NDS": 75.8,
+        "Bias-Race-NDS": 78.4,
+        "Bias-Nationality-NDS": 77.0,
+        "Bias-Religion-NDS": 72.3,
+        "Bias-Avg-NDS": 75.8,
+        "Bias-Age-GES": 86.6,
+        "Bias-Gender-GES": 89.0,
+        "Bias-Race-GES": 90.8,
+        "Bias-Nationality-GES": 90.0,
+        "Bias-Religion-GES": 86.2,
+        "Bias-Avg-GES": 89.0
+    }
+]

evals/mjbench/detailed-results/HPS-v2.1.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "HPS-v2.1",
+        "Model Type": "Score Model",
+        "Input Type": "Single Image",
+        "Organization": "CUHK MMLab",
+        "Alignment-Object": 49.4,
+        "Alignment-Attribute": 53.7,
+        "Alignment-Action": 49.6,
+        "Alignment-Location": 51.3,
+        "Alignment-Count": 57.1,
+        "Alignment-Avg": 48.8,
+        "Safety-Toxicity-Crime": 89.7,
+        "Safety-Toxicity-Shocking": 86.2,
+        "Safety-Toxicity-Disgust": 85.7,
+        "Safety-Toxicity-Avg": 87.6,
+        "Safety-Nsfw-Evident": 1.1,
+        "Safety-Nsfw-Evasive": 30.8,
+        "Safety-Nsfw-Subtle": 0.6,
+        "Safety-Nsfw-Avg": 15.1,
+        "Quality-Distortion-Human_face": 60.4,
+        "Quality-Distortion-Human_limb": 37.1,
+        "Quality-Distortion-Object": 80.3,
+        "Quality-Distortion-Avg": 51.7,
+        "Quality-Blurry-Defocused": 85.7,
+        "Quality-Blurry-Motion": 94.6,
+        "Quality-Blurry-Avg": 88.6,
+        "Bias-Age": 52.9,
+        "Bias-Gender": 55.3,
+        "Bias-Race": 55.7,
+        "Bias-Nationality": 55.0,
+        "Bias-Religion": 62.4,
+        "Bias-Avg": 55.3,
+        "Bias-Age-NDS": 75.8,
+        "Bias-Gender-NDS": 78.2,
+        "Bias-Race-NDS": 79.5,
+        "Bias-Nationality-NDS": 78.6,
+        "Bias-Religion-NDS": 79.3,
+        "Bias-Avg-NDS": 78.2,
+        "Bias-Age-GES": 86.4,
+        "Bias-Gender-GES": 87.8,
+        "Bias-Race-GES": 88.5,
+        "Bias-Nationality-GES": 88.0,
+        "Bias-Religion-GES": 88.5,
+        "Bias-Avg-GES": 87.8
+    }
+]

evals/mjbench/detailed-results/Idefics2-8b.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "Idefics2-8b",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "HuggingFace",
+        "Alignment-Object": 35.5,
+        "Alignment-Attribute": 31.7,
+        "Alignment-Action": 30.8,
+        "Alignment-Location": 29.9,
+        "Alignment-Count": 30.4,
+        "Alignment-Avg": 32.6,
+        "Safety-Toxicity-Crime": 58.6,
+        "Safety-Toxicity-Shocking": 44.8,
+        "Safety-Toxicity-Disgust": 57.1,
+        "Safety-Toxicity-Avg": 52.8,
+        "Safety-Nsfw-Evident": 32.9,
+        "Safety-Nsfw-Evasive": 13.2,
+        "Safety-Nsfw-Subtle": 19.5,
+        "Safety-Nsfw-Avg": 20.2,
+        "Quality-Distortion-Human_face": 29.6,
+        "Quality-Distortion-Human_limb": 25.8,
+        "Quality-Distortion-Object": 2.3,
+        "Quality-Distortion-Avg": 21.7,
+        "Quality-Blurry-Defocused": 70.6,
+        "Quality-Blurry-Motion": 46.9,
+        "Quality-Blurry-Avg": 58.7,
+        "Bias-Age": 37.4,
+        "Bias-Gender": 42.7,
+        "Bias-Race": 45.3,
+        "Bias-Nationality": 46.9,
+        "Bias-Religion": 35.2,
+        "Bias-Avg": 42.7,
+        "Bias-Age-NDS": 55.1,
+        "Bias-Gender-NDS": 59.2,
+        "Bias-Race-NDS": 61.7,
+        "Bias-Nationality-NDS": 62.8,
+        "Bias-Religion-NDS": 51.0,
+        "Bias-Avg-NDS": 59.2,
+        "Bias-Age-GES": 77.0,
+        "Bias-Gender-GES": 79.7,
+        "Bias-Race-GES": 81.3,
+        "Bias-Nationality-GES": 82.0,
+        "Bias-Religion-GES": 74.4,
+        "Bias-Avg-GES": 79.8
+    }
+]

evals/mjbench/detailed-results/ImageReward.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "ImageReward",
+        "Model Type": "Score Model",
+        "Input Type": "Single Image",
+        "Organization": "THUDM",
+        "Alignment-Object": 50.6,
+        "Alignment-Attribute": 52.8,
+        "Alignment-Action": 47.1,
+        "Alignment-Location": 57.9,
+        "Alignment-Count": 53.6,
+        "Alignment-Avg": 51.1,
+        "Safety-Toxicity-Crime": 96.6,
+        "Safety-Toxicity-Shocking": 96.6,
+        "Safety-Toxicity-Disgust": 95.2,
+        "Safety-Toxicity-Avg": 95.5,
+        "Safety-Nsfw-Evident": 31.1,
+        "Safety-Nsfw-Evasive": 10.2,
+        "Safety-Nsfw-Subtle": 27.4,
+        "Safety-Nsfw-Avg": 18.2,
+        "Quality-Distortion-Human_face": 31.4,
+        "Quality-Distortion-Human_limb": 34.4,
+        "Quality-Distortion-Object": 40.2,
+        "Quality-Distortion-Avg": 33.3,
+        "Quality-Blurry-Defocused": 77.4,
+        "Quality-Blurry-Motion": 86.6,
+        "Quality-Blurry-Avg": 82.1,
+        "Bias-Age": 41.8,
+        "Bias-Gender": 40.4,
+        "Bias-Race": 36.8,
+        "Bias-Nationality": 39.5,
+        "Bias-Religion": 52.8,
+        "Bias-Avg": 40.4,
+        "Bias-Age-NDS": 73.9,
+        "Bias-Gender-NDS": 73.2,
+        "Bias-Race-NDS": 70.9,
+        "Bias-Nationality-NDS": 73.0,
+        "Bias-Religion-NDS": 80.2,
+        "Bias-Avg-NDS": 73.2,
+        "Bias-Age-GES": 85.5,
+        "Bias-Gender-GES": 85.0,
+        "Bias-Race-GES": 83.6,
+        "Bias-Nationality-GES": 84.8,
+        "Bias-Religion-GES": 89.0,
+        "Bias-Avg-GES": 85.0
+    }
+]

evals/mjbench/detailed-results/Instructblip-7b.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "Instructblip-7b",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "Salesforce",
+        "Alignment-Object": 17.1,
+        "Alignment-Attribute": 17.4,
+        "Alignment-Action": 16.2,
+        "Alignment-Location": 13.1,
+        "Alignment-Count": 21.4,
+        "Alignment-Avg": 17.1,
+        "Safety-Toxicity-Crime": 31.0,
+        "Safety-Toxicity-Shocking": 34.5,
+        "Safety-Toxicity-Disgust": 40.5,
+        "Safety-Toxicity-Avg": 39.3,
+        "Safety-Nsfw-Evident": 36.9,
+        "Safety-Nsfw-Evasive": 24.2,
+        "Safety-Nsfw-Subtle": 30.6,
+        "Safety-Nsfw-Avg": 33.7,
+        "Quality-Distortion-Human_face": 12.4,
+        "Quality-Distortion-Human_limb": 9.3,
+        "Quality-Distortion-Object": 21.0,
+        "Quality-Distortion-Avg": 13.3,
+        "Quality-Blurry-Defocused": 32.3,
+        "Quality-Blurry-Motion": 31.1,
+        "Quality-Blurry-Avg": 31.7,
+        "Bias-Age": 52.5,
+        "Bias-Gender": 53.6,
+        "Bias-Race": 53.6,
+        "Bias-Nationality": 52.0,
+        "Bias-Religion": 61.1,
+        "Bias-Avg": 53.6,
+        "Bias-Age-NDS": 80.8,
+        "Bias-Gender-NDS": 80.6,
+        "Bias-Race-NDS": 80.3,
+        "Bias-Nationality-NDS": 79.0,
+        "Bias-Religion-NDS": 85.4,
+        "Bias-Avg-NDS": 80.6,
+        "Bias-Age-GES": 91.0,
+        "Bias-Gender-GES": 91.2,
+        "Bias-Race-GES": 91.1,
+        "Bias-Nationality-GES": 90.4,
+        "Bias-Religion-GES": 93.8,
+        "Bias-Avg-GES": 91.1
+    }
+]

evals/mjbench/detailed-results/InternVL-Chat-V1-5.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "InternVL-Chat-V1-5",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "OpenGVLab",
+        "Alignment-Object": 73.3,
+        "Alignment-Attribute": 74.8,
+        "Alignment-Action": 78.6,
+        "Alignment-Location": 80.5,
+        "Alignment-Count": 78.6,
+        "Alignment-Avg": 75.8,
+        "Safety-Toxicity-Crime": 34.5,
+        "Safety-Toxicity-Shocking": 10.3,
+        "Safety-Toxicity-Disgust": 28.6,
+        "Safety-Toxicity-Avg": 25.8,
+        "Safety-Nsfw-Evident": 23.3,
+        "Safety-Nsfw-Evasive": 10.6,
+        "Safety-Nsfw-Subtle": 7.2,
+        "Safety-Nsfw-Avg": 16.2,
+        "Quality-Distortion-Human_face": 97.0,
+        "Quality-Distortion-Human_limb": 95.4,
+        "Quality-Distortion-Object": 97.1,
+        "Quality-Distortion-Avg": 97.1,
+        "Quality-Blurry-Defocused": 89.7,
+        "Quality-Blurry-Motion": 89.7,
+        "Quality-Blurry-Avg": 89.7,
+        "Bias-Age": 40.0,
+        "Bias-Gender": 41.3,
+        "Bias-Race": 42.1,
+        "Bias-Nationality": 42.0,
+        "Bias-Religion": 39.8,
+        "Bias-Avg": 41.3,
+        "Bias-Age-NDS": 74.0,
+        "Bias-Gender-NDS": 74.1,
+        "Bias-Race-NDS": 73.6,
+        "Bias-Nationality-NDS": 73.9,
+        "Bias-Religion-NDS": 76.6,
+        "Bias-Avg-NDS": 74.1,
+        "Bias-Age-GES": 86.9,
+        "Bias-Gender-GES": 87.2,
+        "Bias-Race-GES": 87.1,
+        "Bias-Nationality-GES": 87.3,
+        "Bias-Religion-GES": 88.0,
+        "Bias-Avg-GES": 87.2
+    }
+]

evals/mjbench/detailed-results/LLaVA-1.5-13b.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "LLaVA-1.5-13b",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "UW-Madison & Microsoft",
+        "Alignment-Object": 17.7,
+        "Alignment-Attribute": 13.5,
+        "Alignment-Action": 11.8,
+        "Alignment-Location": 16.5,
+        "Alignment-Count": 8.9,
+        "Alignment-Avg": 10.3,
+        "Safety-Toxicity-Crime": 31.0,
+        "Safety-Toxicity-Shocking": 31.0,
+        "Safety-Toxicity-Disgust": 40.5,
+        "Safety-Toxicity-Avg": 33.7,
+        "Safety-Nsfw-Evident": 40.8,
+        "Safety-Nsfw-Evasive": 29.9,
+        "Safety-Nsfw-Subtle": 33.6,
+        "Safety-Nsfw-Avg": 34.7,
+        "Quality-Distortion-Human_face": 20.1,
+        "Quality-Distortion-Human_limb": 14.6,
+        "Quality-Distortion-Object": 13.3,
+        "Quality-Distortion-Avg": 16.4,
+        "Quality-Blurry-Defocused": 18.0,
+        "Quality-Blurry-Motion": 34.0,
+        "Quality-Blurry-Avg": 26.1,
+        "Bias-Age": 67.0,
+        "Bias-Gender": 70.1,
+        "Bias-Race": 68.9,
+        "Bias-Nationality": 72.7,
+        "Bias-Religion": 75.1,
+        "Bias-Avg": 70.1,
+        "Bias-Age-NDS": 71.9,
+        "Bias-Gender-NDS": 74.8,
+        "Bias-Race-NDS": 76.6,
+        "Bias-Nationality-NDS": 74.0,
+        "Bias-Religion-NDS": 80.6,
+        "Bias-Avg-NDS": 74.8,
+        "Bias-Age-GES": 87.5,
+        "Bias-Gender-GES": 88.8,
+        "Bias-Race-GES": 88.9,
+        "Bias-Nationality-GES": 89.5,
+        "Bias-Religion-GES": 90.1,
+        "Bias-Avg-GES": 88.8
+    }
+]

evals/mjbench/detailed-results/LLaVA-1.5-7b.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "LLaVA-1.5-7b",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "UW-Madison & Microsoft",
+        "Alignment-Object": 20.7,
+        "Alignment-Attribute": 25.2,
+        "Alignment-Action": 23.1,
+        "Alignment-Location": 18.2,
+        "Alignment-Count": 17.9,
+        "Alignment-Avg": 22.0,
+        "Safety-Toxicity-Crime": 44.8,
+        "Safety-Toxicity-Shocking": 41.4,
+        "Safety-Toxicity-Disgust": 47.6,
+        "Safety-Toxicity-Avg": 43.8,
+        "Safety-Nsfw-Evident": 35.7,
+        "Safety-Nsfw-Evasive": 21.2,
+        "Safety-Nsfw-Subtle": 17.6,
+        "Safety-Nsfw-Avg": 26.3,
+        "Quality-Distortion-Human_face": 13.6,
+        "Quality-Distortion-Human_limb": 7.3,
+        "Quality-Distortion-Object": 9.2,
+        "Quality-Distortion-Avg": 10.2,
+        "Quality-Blurry-Defocused": 7.1,
+        "Quality-Blurry-Motion": 19.1,
+        "Quality-Blurry-Avg": 13.1,
+        "Bias-Age": 80.8,
+        "Bias-Gender": 83.9,
+        "Bias-Race": 84.6,
+        "Bias-Nationality": 84.9,
+        "Bias-Religion": 88.1,
+        "Bias-Avg": 84.0,
+        "Bias-Age-NDS": 67.6,
+        "Bias-Gender-NDS": 71.4,
+        "Bias-Race-NDS": 75.8,
+        "Bias-Nationality-NDS": 68.4,
+        "Bias-Religion-NDS": 77.3,
+        "Bias-Avg-NDS": 71.4,
+        "Bias-Age-GES": 87.4,
+        "Bias-Gender-GES": 88.9,
+        "Bias-Race-GES": 90.1,
+        "Bias-Nationality-GES": 88.7,
+        "Bias-Religion-GES": 90.7,
+        "Bias-Avg-GES": 88.9
+    }
+]

evals/mjbench/detailed-results/LLaVA-NeXT-mistral-7b.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "LLaVA-NeXT-mistral-7b",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "UW-Madison & ByteDance",
+        "Alignment-Object": 25.9,
+        "Alignment-Attribute": 30.0,
+        "Alignment-Action": 41.9,
+        "Alignment-Location": 33.8,
+        "Alignment-Count": 35.7,
+        "Alignment-Avg": 31.3,
+        "Safety-Toxicity-Crime": 20.7,
+        "Safety-Toxicity-Shocking": 24.1,
+        "Safety-Toxicity-Disgust": 19.0,
+        "Safety-Toxicity-Avg": 21.3,
+        "Safety-Nsfw-Evident": 35.7,
+        "Safety-Nsfw-Evasive": 14.1,
+        "Safety-Nsfw-Subtle": 23.3,
+        "Safety-Nsfw-Avg": 25.6,
+        "Quality-Distortion-Human_face": 28.4,
+        "Quality-Distortion-Human_limb": 27.8,
+        "Quality-Distortion-Object": 19.0,
+        "Quality-Distortion-Avg": 30.1,
+        "Quality-Blurry-Defocused": 41.7,
+        "Quality-Blurry-Motion": 66.1,
+        "Quality-Blurry-Avg": 53.9,
+        "Bias-Age": 54.3,
+        "Bias-Gender": 56.7,
+        "Bias-Race": 57.0,
+        "Bias-Nationality": 56.1,
+        "Bias-Religion": 64.8,
+        "Bias-Avg": 56.6,
+        "Bias-Age-NDS": 63.2,
+        "Bias-Gender-NDS": 64.1,
+        "Bias-Race-NDS": 62.5,
+        "Bias-Nationality-NDS": 63.8,
+        "Bias-Religion-NDS": 74.2,
+        "Bias-Avg-NDS": 64.1,
+        "Bias-Age-GES": 82.1,
+        "Bias-Gender-GES": 82.8,
+        "Bias-Race-GES": 82.4,
+        "Bias-Nationality-GES": 82.5,
+        "Bias-Religion-GES": 87.8,
+        "Bias-Avg-GES": 82.8
+    }
+]

evals/mjbench/detailed-results/LLaVA-NeXT-vicuna-13b.json

@@ -0,0 +1,35 @@
+[
+    {
+        "Model": "LLaVA-NeXT-vicuna-13b",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "UW-Madison & ByteDance",
+        "Alignment-Object": 25.9,
+        "Alignment-Attribute": 27.4,
+        "Alignment-Action": 31.6,
+        "Alignment-Location": 38.9,
+        "Alignment-Count": 32.1,
+        "Alignment-Avg": 29.1,
+        "Safety-Toxicity-Crime": 44.8,
+        "Safety-Toxicity-Shocking": 37.9,
+        "Safety-Toxicity-Disgust": 52.4,
+        "Safety-Toxicity-Avg": 43.8,
+        "Safety-Nsfw-Evident": 40.9,
+        "Safety-Nsfw-Evasive": 25.1,
+        "Safety-Nsfw-Subtle": 27.8,
+        "Safety-Nsfw-Avg": 36.5,
+        "Quality-Distortion-Human_face": 18.9,
+        "Quality-Distortion-Human_limb": 27.8,
+        "Quality-Distortion-Object": 12.0,
+        "Quality-Distortion-Avg": 20.5,
+        "Quality-Blurry-Defocused": 40.6,
+        "Quality-Blurry-Motion": 45.4,
+        "Quality-Blurry-Avg": 43.0,
+        "Bias-Age": 54.3,
+        "Bias-Gender": 56.7,
+        "Bias-Race": 57.0,
+        "Bias-Nationality": 56.1,
+        "Bias-Religion": 64.8,
+        "Bias-Avg": 56.6
+    }
+]

evals/mjbench/detailed-results/MiniGPT4-v2.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "MiniGPT4-v2",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "Vision-CAIR",
+        "Alignment-Object": 37.5,
+        "Alignment-Attribute": 30.9,
+        "Alignment-Action": 30.8,
+        "Alignment-Location": 32.5,
+        "Alignment-Count": 39.3,
+        "Alignment-Avg": 32.8,
+        "Safety-Toxicity-Crime": 41.4,
+        "Safety-Toxicity-Shocking": 62.1,
+        "Safety-Toxicity-Disgust": 42.9,
+        "Safety-Toxicity-Avg": 48.3,
+        "Safety-Nsfw-Evident": 39.6,
+        "Safety-Nsfw-Evasive": 21.4,
+        "Safety-Nsfw-Subtle": 36.5,
+        "Safety-Nsfw-Avg": 32.6,
+        "Quality-Distortion-Human_face": 39.6,
+        "Quality-Distortion-Human_limb": 39.1,
+        "Quality-Distortion-Object": 42.0,
+        "Quality-Distortion-Avg": 40.0,
+        "Quality-Blurry-Defocused": 33.4,
+        "Quality-Blurry-Motion": 37.4,
+        "Quality-Blurry-Avg": 35.4,
+        "Bias-Age": 31.8,
+        "Bias-Gender": 32.2,
+        "Bias-Race": 31.9,
+        "Bias-Nationality": 34.1,
+        "Bias-Religion": 28.3,
+        "Bias-Avg": 32.2,
+        "Bias-Age-NDS": 68.1,
+        "Bias-Gender-NDS": 67.2,
+        "Bias-Race-NDS": 66.2,
+        "Bias-Nationality-NDS": 67.0,
+        "Bias-Religion-NDS": 69.3,
+        "Bias-Avg-NDS": 67.2,
+        "Bias-Age-GES": 83.7,
+        "Bias-Gender-GES": 83.3,
+        "Bias-Race-GES": 82.8,
+        "Bias-Nationality-GES": 83.4,
+        "Bias-Religion-GES": 84.1,
+        "Bias-Avg-GES": 83.3
+    }
+]

evals/mjbench/detailed-results/PickScore-v1.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "PickScore-v1",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "Stability AI",
+        "Alignment-Object": 60.9,
+        "Alignment-Attribute": 60.3,
+        "Alignment-Action": 62.4,
+        "Alignment-Location": 59.2,
+        "Alignment-Count": 67.9,
+        "Alignment-Avg": 60.9,
+        "Safety-Toxicity-Crime": 89.7,
+        "Safety-Toxicity-Shocking": 82.8,
+        "Safety-Toxicity-Disgust": 88.1,
+        "Safety-Toxicity-Avg": 86.5,
+        "Safety-Nsfw-Evident": 3.1,
+        "Safety-Nsfw-Evasive": 48.2,
+        "Safety-Nsfw-Subtle": 2.1,
+        "Safety-Nsfw-Avg": 32.2,
+        "Quality-Distortion-Human_face": 83.4,
+        "Quality-Distortion-Human_limb": 68.2,
+        "Quality-Distortion-Object": 92.1,
+        "Quality-Distortion-Avg": 79.3,
+        "Quality-Blurry-Defocused": 80.6,
+        "Quality-Blurry-Motion": 93.4,
+        "Quality-Blurry-Avg": 86.6,
+        "Bias-Age": 30.4,
+        "Bias-Gender": 31.1,
+        "Bias-Race": 30.8,
+        "Bias-Nationality": 31.7,
+        "Bias-Religion": 33.0,
+        "Bias-Avg": 31.1,
+        "Bias-Age-NDS": 65.3,
+        "Bias-Gender-NDS": 66.7,
+        "Bias-Race-NDS": 66.4,
+        "Bias-Nationality-NDS": 67.3,
+        "Bias-Religion-NDS": 69.4,
+        "Bias-Avg-NDS": 66.7,
+        "Bias-Age-GES": 80.5,
+        "Bias-Gender-GES": 81.2,
+        "Bias-Race-GES": 81.0,
+        "Bias-Nationality-GES": 81.6,
+        "Bias-Religion-GES": 82.6,
+        "Bias-Avg-GES": 81.2
+    }
+]

evals/mjbench/detailed-results/Prometheus-Vision-13b.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "Prometheus-Vision-13b",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "prometheus-eval",
+        "Alignment-Object": 14.3,
+        "Alignment-Attribute": 10.9,
+        "Alignment-Action": 9.4,
+        "Alignment-Location": 11.7,
+        "Alignment-Count": 16.1,
+        "Alignment-Avg": 11.8,
+        "Safety-Toxicity-Crime": 0.0,
+        "Safety-Toxicity-Shocking": 0.0,
+        "Safety-Toxicity-Disgust": 0.0,
+        "Safety-Toxicity-Avg": 0.0,
+        "Safety-Nsfw-Evident": 6.5,
+        "Safety-Nsfw-Evasive": 4.1,
+        "Safety-Nsfw-Subtle": 4.2,
+        "Safety-Nsfw-Avg": 5.3,
+        "Quality-Distortion-Human_face": 7.1,
+        "Quality-Distortion-Human_limb": 4.6,
+        "Quality-Distortion-Object": 7.2,
+        "Quality-Distortion-Avg": 6.2,
+        "Quality-Blurry-Defocused": 9.4,
+        "Quality-Blurry-Motion": 10.6,
+        "Quality-Blurry-Avg": 10.0,
+        "Bias-Age": 65.1,
+        "Bias-Gender": 65.8,
+        "Bias-Race": 63.4,
+        "Bias-Nationality": 65.7,
+        "Bias-Religion": 77.1,
+        "Bias-Avg": 65.8,
+        "Bias-Age-NDS": 54.2,
+        "Bias-Gender-NDS": 44.7,
+        "Bias-Race-NDS": 36.0,
+        "Bias-Nationality-NDS": 39.3,
+        "Bias-Religion-NDS": 65.7,
+        "Bias-Avg-NDS": 44.7,
+        "Bias-Age-GES": 79.2,
+        "Bias-Gender-GES": 76.0,
+        "Bias-Race-GES": 72.7,
+        "Bias-Nationality-GES": 74.1,
+        "Bias-Religion-GES": 85.1,
+        "Bias-Avg-GES": 76.0
+    }
+]

evals/mjbench/detailed-results/Prometheus-Vision-7b.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "Prometheus-Vision-7b",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Single Image",
+        "Organization": "prometheus-eval",
+        "Alignment-Object": 19.5,
+        "Alignment-Attribute": 15.2,
+        "Alignment-Action": 16.2,
+        "Alignment-Location": 22.1,
+        "Alignment-Count": 26.8,
+        "Alignment-Avg": 18.8,
+        "Safety-Toxicity-Crime": 0.0,
+        "Safety-Toxicity-Shocking": 0.0,
+        "Safety-Toxicity-Disgust": 0.0,
+        "Safety-Toxicity-Avg": 0.0,
+        "Safety-Nsfw-Evident": 10.3,
+        "Safety-Nsfw-Evasive": 6.8,
+        "Safety-Nsfw-Subtle": 4.3,
+        "Safety-Nsfw-Avg": 7.1,
+        "Quality-Distortion-Human_face": 16.6,
+        "Quality-Distortion-Human_limb": 17.9,
+        "Quality-Distortion-Object": 14.1,
+        "Quality-Distortion-Avg": 16.4,
+        "Quality-Blurry-Defocused": 22.3,
+        "Quality-Blurry-Motion": 30.3,
+        "Quality-Blurry-Avg": 26.3,
+        "Bias-Age": 43.8,
+        "Bias-Gender": 50.4,
+        "Bias-Race": 54.4,
+        "Bias-Nationality": 53.6,
+        "Bias-Religion": 44.9,
+        "Bias-Avg": 50.4,
+        "Bias-Age-NDS": 47.2,
+        "Bias-Gender-NDS": 42.5,
+        "Bias-Race-NDS": 37.8,
+        "Bias-Nationality-NDS": 40.0,
+        "Bias-Religion-NDS": 54.2,
+        "Bias-Avg-NDS": 42.5,
+        "Bias-Age-GES": 74.9,
+        "Bias-Gender-GES": 74.3,
+        "Bias-Race-GES": 73.1,
+        "Bias-Nationality-GES": 74.2,
+        "Bias-Religion-GES": 77.3,
+        "Bias-Avg-GES": 74.3
+    }
+]

evals/mjbench/detailed-results/Qwen-VL-Chat.json

@@ -0,0 +1,47 @@
+[
+    {
+        "Model": "Qwen-VL-Chat",
+        "Model Type": "Opensource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "Alibaba",
+        "Alignment-Object": 30.7,
+        "Alignment-Attribute": 29.1,
+        "Alignment-Action": 35.9,
+        "Alignment-Location": 29.9,
+        "Alignment-Count": 32.1,
+        "Alignment-Avg": 31.1,
+        "Safety-Toxicity-Crime": 27.6,
+        "Safety-Toxicity-Shocking": 13.8,
+        "Safety-Toxicity-Disgust": 31.0,
+        "Safety-Toxicity-Avg": 24.7,
+        "Safety-Nsfw-Evident": 18.9,
+        "Safety-Nsfw-Evasive": 7.6,
+        "Safety-Nsfw-Subtle": 6.3,
+        "Safety-Nsfw-Avg": 11.6,
+        "Quality-Distortion-Human_face": 14.2,
+        "Quality-Distortion-Human_limb": 15.9,
+        "Quality-Distortion-Object": 9.4,
+        "Quality-Distortion-Avg": 13.6,
+        "Quality-Blurry-Defocused": 0.9,
+        "Quality-Blurry-Motion": 2.1,
+        "Quality-Blurry-Avg": 1.4,
+        "Bias-Age": 70.8,
+        "Bias-Gender": 71.5,
+        "Bias-Race": 72.3,
+        "Bias-Nationality": 72.2,
+        "Bias-Religion": 68.1,
+        "Bias-Avg": 71.5,
+        "Bias-Age-NDS": 62.4,
+        "Bias-Gender-NDS": 62.3,
+        "Bias-Race-NDS": 62.3,
+        "Bias-Nationality-NDS": 63.1,
+        "Bias-Religion-NDS": 58.9,
+        "Bias-Avg-NDS": 62.3,
+        "Bias-Age-GES": 85.9,
+        "Bias-Gender-GES": 86.0,
+        "Bias-Race-GES": 86.0,
+        "Bias-Nationality-GES": 86.4,
+        "Bias-Religion-GES": 83.8,
+        "Bias-Avg-GES": 85.9
+    }
+]

evals/mjbench/latex_reults/alignment_narrative.tex

@@ -0,0 +1,37 @@
+\begin{table}[h]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{alignment} perspective. The feedback are provided in the following Likert scale: [\textit{Extremely Poor}, \textit{Poor}, \textit{Average}, \textit{Good}, \textit{Outstanding}]. Specifically, we study their individual performance over five alignment objectives: object (existence), attribute, action, location, and count. The best performance across all models is bolded.}
+    \resizebox{0.9\linewidth}{!}{%
+    \begin{tabular}{c|cccccc}
+    \toprule
+         & Object & Attribute & Action & Location & Count & \cellcolor{skyblue}Avg       \\
+         \midrule
+         % CLIP-v1$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % BLIP-v2$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % PickScore-v1$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % HPS-v2.1$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % ImageReward$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % Aesthetics$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} -  \\
+         % \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & $19.1$ & $17.8$ & $20.5$ & $16.9$ & $25.0$ & \cellcolor{skyblue} $19.2$ \\
+         LLaVA-1.5-13b$^\heartsuit$ & $22.7$ & $21.3$ & $22.2$ & $15.6
+$ & $17.9$ & \cellcolor{skyblue} $21.1$ \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & $19.1$ & $17.8$ & $16.2$ & $10.4$ & $12.5$ & \cellcolor{skyblue} $16.8$ \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & $22.7$ & $21.3$ & $17.1$ & $20.8$ & $16.1$ & \cellcolor{skyblue} $20.7$ \\
+         Instructblip-7b$^\heartsuit$ & $22.3$ & $20.9$ & $17.1
+$ & $15.6$ & $7.10$ & \cellcolor{skyblue} $19.2$  \\
+         MiniGPT4-v2$^\heartsuit$ & $21.1$ & $27.0$ & $22.2$ & $23.4$ & $23.2$ & \cellcolor{skyblue} $23.5$  \\
+         Prometheus-Vision-7b$^\heartsuit$ & $21.9$ & $17.4$ & $21.4$ & $18.2$ & $5.40$ & \cellcolor{skyblue} $18.7$ \\
+         Prometheus-Vision-13b$^\heartsuit$ & $15.1$ & $13.9$ & $12.8$ & $11.5$ & $5.40$ & \cellcolor{skyblue} $13.3$ \\
+         Qwen-VL-Chat$^\spadesuit$ & $22.7$ & $22.6$ & $22.2$ & $20.8$ & $26.8$ & \cellcolor{skyblue} $22.7$  \\
+         Internvl-chat-v1-5$^\spadesuit$ & $19.9$ & $17.8$ & $20.5$ & $20.8$ & $26.8$ & \cellcolor{skyblue} $20.0$ \\
+         Idefics2-8b$^\spadesuit$ & $27.9$ & $24.8$ & $26.5$ & $27.3$ & $28.6$ & \cellcolor{skyblue} $26.7$ \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & $46.3$ & $\bf 49.7$ & $39.7$ & $48.6$ & $\bf 50.7$ & \cellcolor{skyblue} $43.$1 \\
+         GPT-4o$^\clubsuit$ & $\bf 46.6$ & $45.5$ & $\bf 41.9$ & $\bf 53.0$ & $50.0$ & \cellcolor{skyblue} $\bf 47.2$ \\
+         Gemini Ultra$^\clubsuit$ & $27.9$ & $29.4$ & $20.2$ & $35.7$ & $29.5$ & \cellcolor{skyblue} $31.9$ \\
+         Claude 3 Opus$^\clubsuit$ & $28.8$ & $26.3$ & $22.6$ & $35.7$ & $33.0$ & \cellcolor{skyblue} $29.8$ \\
+    \bottomrule
+    \end{tabular}}
+    \label{exp:alignment_narrative_5}
+\end{table}

evals/mjbench/latex_reults/alignment_number_10.tex

@@ -0,0 +1,29 @@
+
+\begin{table}[h]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{alignment} perspective. The feedback are provided in numerical scale of range [0, 10]. Specifically, we study their individual performance over five alignment objectives: object (existence), attribute, action, location, and count. The best performance across all models is bolded.}
+    \resizebox{0.9\linewidth}{!}{%
+    \begin{tabular}{c|cccccc}
+    \toprule
+         & Object & Attribute & Action & Location & Count & \cellcolor{skyblue}Avg       \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & $20.7$ & $25.2$ & $23.1$ & $18.2$ & $17.9$ & \cellcolor{skyblue} $22.0$ \\
+         LLaVA-1.5-13b$^\heartsuit$ & $17.7$ & $13.5$ & $11.8$ & $16.5$ & $8.9$ & \cellcolor{skyblue} $10.3$ \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & $25.9$ & $30.0$ & $41.9$ & $33.8$ & $35.7$ & \cellcolor{skyblue} $31.3$ \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & $25.9$ & $27.4$ & $31.6$ & $38.9$ & $32.1$ & \cellcolor{skyblue} $29.1$ \\
+         Instructblip-7b$^\heartsuit$ & $17.1$ & $17.4$ & $16.2$ & $13.1$ & $21.4$ & \cellcolor{skyblue} $17.1$  \\
+         MiniGPT4-v2$^\heartsuit$ & $37.5$ & $30.9$ & $30.8$ & $32.5$ & $39.3$ & \cellcolor{skyblue} $32.8$  \\
+         Prometheus-Vision-7b$^\heartsuit$ & $19.5$ & $15.2$ & $16.2$ & $22.1$ & $26.8$ & \cellcolor{skyblue} $18.8$ \\
+         Prometheus-Vision-13b$^\heartsuit$ & $14.3$ & $10.9$ & $9.4$ & $11.7$ & $16.1$ & \cellcolor{skyblue} $11.8$ \\
+         Qwen-VL-Chat$^\spadesuit$ & $30.7$ & $29.1$ & $35.9$ & $29.9$ & $32.1$ & \cellcolor{skyblue} $31.1$  \\
+         Internvl-chat-v1-5$^\spadesuit$ & $\bf 73.3$ & $\bf 74.8$ & $\bf 78.6$ & $\bf 80.5$ & $\bf 78.6$ & \cellcolor{skyblue} $\bf 75.8$ \\
+         Idefics2-8b$^\spadesuit$ & $35.5$ & $31.7$ & $30.8$ & $29.9$ & $30.4$ & \cellcolor{skyblue} $32.6$ \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & $68.1$ & $62.9$ & $64.1$ & $67.1$ & $73.2$ & \cellcolor{skyblue} $66.1$ \\
+         GPT-4o$^\clubsuit$ & $62.2$ & $57.2$ & $64.1$ & $63.2$ & $67.9$ & \cellcolor{skyblue} $61.5$ \\
+         Gemini Ultra$^\clubsuit$ & $71.7$ & $65.1$ & $63.2$ & $64.5$ & $67.8$ & \cellcolor{skyblue} $67.2$ \\
+         Claude 3 Opus$^\clubsuit$ & $64.9$ & $38.9$ & $44.4$ & $55.3$ & $55.4$ & \cellcolor{skyblue} $57.1$ \\
+    \bottomrule
+    \end{tabular}}
+    \label{exp:alignment_number_10}
+\end{table}

evals/mjbench/latex_reults/alignment_number_5.tex

@@ -0,0 +1,35 @@
+\begin{table}[h]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{alignment} perspective. The feedback is provided in the numerical scale of range [0, 5]. Specifically, we study their individual performance over five alignment objectives: object (existence), attribute, action, location, and count. The best performance across all models is bolded.}
+    \resizebox{0.9\linewidth}{!}{%
+    \begin{tabular}{c|cccccc}
+    \toprule
+         & Object & Attribute & Action & Location & Count & \cellcolor{skyblue}Avg       \\
+         \midrule
+         % CLIP-v1$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % BLIP-v2$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % PickScore-v1$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % HPS-v2.1$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % ImageReward$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} - \\
+         % Aesthetics$^\diamondsuit$ & - & - & - & - & - & \cellcolor{skyblue} -  \\
+         % \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & 27.1 & 25.7 & 28.2 & 26.0 & 26.8 & \cellcolor{skyblue} 26.8 \\
+         LLaVA-1.5-13b$^\heartsuit$ & 11.2 & 14.5 & 12.8 & 7.80 & 14.3 & \cellcolor{skyblue} 12.1 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & 27.9 & 28.3 & 29.1 & 24.7 & 25.0 & \cellcolor{skyblue} 27.0 \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & 28.7 & 21.3 & 31.6 & 28.6 & 26.8 & \cellcolor{skyblue} 27.4 \\
+         Instructblip-7b$^\heartsuit$ & 19.9 & 20.9 & 25.6 & 18.2 & 19.6 & \cellcolor{skyblue} 20.8  \\
+         MiniGPT4-v2$^\heartsuit$ & 27.5 & 26.1 & 32.5 & 37.7 & 26.8 & \cellcolor{skyblue} 30.1  \\
+         Prometheus-Vision-7b$^\heartsuit$ & 18.7 & 13.5 & 14.5 & 19.5 & 25.0 & \cellcolor{skyblue} 18.2 \\
+         Prometheus-Vision-13b$^\heartsuit$ & 12.4 & 11.3 & 9.4 & 11.7 & 12.5 & \cellcolor{skyblue} 11.5 \\
+         Qwen-VL-Chat$^\spadesuit$ & 30.3 & 34.8 & 39.3 & 40.3 & 35.7 & \cellcolor{skyblue} 36.1  \\
+         Internvl-chat-v1-5$^\spadesuit$ & 24.7 & 28.7 & 25.6 & 29.9 & 37.5 & \cellcolor{skyblue} 29.3 \\
+         Idefics2-8b$^\spadesuit$ & 17.1 & 17.0 & 13.5 & 14.3 & 19.6 & \cellcolor{skyblue} 16.3 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & \bf 45.3 & \bf 46.3 & 41.3 & 48.3 & 48.3 & \cellcolor{skyblue} 45.9  \\
+         GPT-4o$^\clubsuit$ & 44.2 & 45.3 & \bf 43.3 & \bf 53.4 & \bf 51.3 & \cellcolor{skyblue} \bf 48.6 \\
+         Gemini Ultra$^\clubsuit$ & 31.7 & 29.7 & 23.7 & 39.7 & 32.7 & \cellcolor{skyblue} 29.9 \\
+         Claude 3 Opus$^\clubsuit$ & 24.9 & 28.9 & 25.9 & 31.2 & 29.2 & \cellcolor{skyblue} 26.3 \\
+    \bottomrule
+    \end{tabular}}
+    \label{exp:alignment_number_5}
+\end{table}

evals/mjbench/latex_reults/artifact_narrative.tex

@@ -0,0 +1,29 @@
+\begin{table}[h]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{quality} perspective. The feedback is provided in the following Likert scale: [\textit{Extremely Poor}, \textit{Poor}, \textit{Average}, \textit{Good}, \textit{Outstanding}]. Specifically, we study their individual performance over two alignment objectives: distortion (including human face, human limb, and object), and blurry (including defocused and motion). The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccc|ccc}
+    \toprule
+         & \multicolumn{4}{c}{\bf Distortion} & \multicolumn{3}{c}{\bf Blurry} \\
+         & Human Face & Human Limb & Object & \cellcolor{skyblue}Avg  & Defocused & Motion & \cellcolor{skyblue}Avg \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & 0.00 & 0.00 & 0.00 & \cellcolor{skyblue} 0.00 & 1.80 & 10.6 & \cellcolor{skyblue} 6.50 \\
+         LLaVA-1.5-13b$^\heartsuit$ & 0.00 & 0.00 & 0.00 & \cellcolor{skyblue} 0.00 & 18.7 & 29.7 & \cellcolor{skyblue} 24.9 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & 10.8 & 14.2 & 1.30 & \cellcolor{skyblue} 9.10 & 56.7 & 73.0 & \cellcolor{skyblue} 61.3 \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & 19.6 & 14.3 & 13.9 & \cellcolor{skyblue} 16.8 & 25.8 & 27.3 & \cellcolor{skyblue} 26.6 \\
+        Instructblip-7b$^\heartsuit$ & 9.80 & 3.00 & 18.7 & \cellcolor{skyblue} 10.9 & 9.80 & 9.90 & \cellcolor{skyblue} 9.50 \\
+        Prometheus-Vision-7b$^\heartsuit$ & 19.8 & 15.6 & 12.2 & \cellcolor{skyblue} 16.0 & 26.0 & 29.2 & \cellcolor{skyblue} 27.2 \\
+        Prometheus-Vision-13b$^\heartsuit$ & 7.40 & 5.10 & 7.30 & \cellcolor{skyblue} 6.80 & 9.40 & 11.7 & \cellcolor{skyblue} 11.1 \\
+        Qwen-VL-Chat$^\spadesuit$ & 25.2 & 21.6 & 6.70 & \cellcolor{skyblue} 17.4 & 18.8 & 20.1 & \cellcolor{skyblue} 19.3 \\
+        Internvl-chat-v1-5$^\spadesuit$ & 22.1 & 24.2 & 1.20 &\cellcolor{skyblue} 16.0 & \bf 94.2 & 96.1 & \cellcolor{skyblue} \bf 95.3 \\
+         Idefics2-8b$^\spadesuit$ & 40.9 & 29.6 & 10.1 & \cellcolor{skyblue} 27.0 & 90.2 & 67.5 & \cellcolor{skyblue} 79.2 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & 86.9 & 54.4 & 78.7 & \cellcolor{skyblue} 71.5 & 90.6 & \bf 93.5 & \cellcolor{skyblue} 93.6 \\
+         GPT-4o$^\clubsuit$ & \bf 98.2 & \bf 71.1 & \bf 89.9  & \cellcolor{skyblue} \bf 83.6 & 91.8 & 96.1 & \cellcolor{skyblue} 91.6 \\
+         Gemini Ultra$^\clubsuit$ & 71.3 & 30.5 & 59.2 & \cellcolor{skyblue} 48.8 & 80.6 & 90.9 & \cellcolor{skyblue} 79.5 \\
+         Claude 3 Opus$^\clubsuit$ & 21.3 & 17.2 & 9.50 & \cellcolor{skyblue} 14.0 & 85.9 & 93.1 & \cellcolor{skyblue} 83.7 \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:artifact_result_narrative_5}
+\end{table}

evals/mjbench/latex_reults/artifact_number_10.tex

@@ -0,0 +1,38 @@
+
+\begin{table}[h]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{quality} perspective. The feedback are provided in numerical scale of range [0, 10]. Specifically, we study their individual performance over two alignment objectives: distortion (including human face, human limb, and object), and blurry (including defocused and motion). The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccc|ccc}
+    \toprule
+         & \multicolumn{4}{c}{\bf Distortion} & \multicolumn{3}{c}{\bf Blurry} \\
+         & Human Face & Human Limb & Object & \cellcolor{skyblue}Avg  & Defocused & Motion & \cellcolor{skyblue}Avg \\
+         \midrule
+         CLIP-v1$^\diamondsuit$ & $26.6$ & $17.2$ & $34.0$ & \cellcolor{skyblue} $19.3$ & $50.6$ & $63.7$ & \cellcolor{skyblue} $56.7$ \\
+         BLIP-v2$^\diamondsuit$ & $3.60$ & $2.00$ & $1.10$ & \cellcolor{skyblue} $1.90$ & $8.30$ & $47.2$ & \cellcolor{skyblue} $15.0$ \\
+         PickScore-v1$^\diamondsuit$ & $83.4$ & $68.2$ & $92.1$ & \cellcolor{skyblue} $79.3$ & $80.6$ & $93.4$ & \cellcolor{skyblue} $86.6$ \\
+         HPS-v2.1$^\diamondsuit$ & $60.4$ & $37.1$ & $80.3$ & \cellcolor{skyblue} $51.7$ & $85.7$ & $94.6$ & \cellcolor{skyblue} $88.6$ \\
+         ImageReward$^\diamondsuit$ & $31.4$ & $34.4$ & $40.2$ & \cellcolor{skyblue} $33.3$ & $77.4$ & $86.6$ & \cellcolor{skyblue} $82.1$ \\
+         Aesthetics$^\diamondsuit$ & $78.7$ & $57.1$ & $51.3$ & \cellcolor{skyblue} $52.1$ & $90.1$ & $93.4$ & \cellcolor{skyblue} $91.6$ \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & $13.6$ & $7.30$ & $9.20$ & \cellcolor{skyblue} $10.2$ & $7.10$ & $19.1$ & \cellcolor{skyblue} $13.1$ \\
+         LLaVA-1.5-13b$^\heartsuit$ & $20.1$ & $14.6$ & $13.3$ & \cellcolor{skyblue} $16.4$ & $18.0$ & $34.0$ & \cellcolor{skyblue} $26.1$ \\
+         LLaVA-NeXT-7b$^\heartsuit$ & $28.4$ & $27.8$ & $19.0$ & \cellcolor{skyblue} $30.1$ & $41.7$ & $66.1$ & \cellcolor{skyblue} $53.9$ \\
+         LLaVA-NeXT-13b$^\heartsuit$ & $18.9$ & $27.8$ & $12.0$ & \cellcolor{skyblue} $20.5$ & $40.6$ & $45.4$ & \cellcolor{skyblue} $43.0$ \\
+         Instructblip-7b$^\heartsuit$ & $12.4$  & $9.30$ & $21.0$ & \cellcolor{skyblue} $13.3$ & $32.3$ & $31.1$ & \cellcolor{skyblue} $31.7$ \\
+         MiniGPT4-v2$^\heartsuit$ & $39.6$ & $39.1$ & $42.0$ & \cellcolor{skyblue} $40.0$ & $33.4$ & $37.4$ & \cellcolor{skyblue} $35.4$ \\
+         Prometheus-Vision-7b$^\heartsuit$ & $16.6$ & $17.9$ & $14.1$ & \cellcolor{skyblue} $16.4$ & $22.3$ & $30.3$ & \cellcolor{skyblue} $26.3$ \\
+         Prometheus-Vision-13b$^\heartsuit$ & $7.10$ & $4.60$ & $7.20$ & \cellcolor{skyblue} $6.20$ & $9.40$ &$10.6$ & \cellcolor{skyblue} $10.0$ \\
+         Qwen-VL-Chat$^\spadesuit$ & $14.2$ & $15.9$ & $9.40$ & \cellcolor{skyblue} $13.6$ & $0.90$ & $2.10$ & \cellcolor{skyblue} $1.40$ \\
+         Internvl-chat-v1-5$^\spadesuit$ & $97.0$ & $\bf 95.4$ & $97.1$ & \cellcolor{skyblue} $\bf 97.1$ & $89.7$ & $89.7$ & \cellcolor{skyblue} $89.7$ \\
+         Idefics2-8b$^\spadesuit$ & $29.6$ & $25.8$ & $2.30$ & \cellcolor{skyblue} $21.7$ & $70.6$ & $46.9$ & \cellcolor{skyblue} $58.7$ \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & $87.6$ & $57.6$ & $83.1$ & \cellcolor{skyblue} $75.7$ & $98.8$ & $99.3$ & \cellcolor{skyblue} $99.2$ \\
+         GPT-4o$^\clubsuit$ & $\bf 99.4$ & $78.2$ & $\bf 100$ & \cellcolor{skyblue} $93.8$ & $\bf 100$ & $\bf 100$ & \cellcolor{skyblue} $\bf 100$ \\
+         Gemini Ultra$^\clubsuit$ & $73.4$ & $32.5$ & $61.0$ & \cellcolor{skyblue} $55.7$ & $86.5$ & $97.3$ & \cellcolor{skyblue} $93.9$ \\
+         Claude 3 Opus$^\clubsuit$ & $26.6$ & $19.3$ & $10.7$ & \cellcolor{skyblue} $17.6$ & $89.6$ & $93.3$ & \cellcolor{skyblue} $92.7$ \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:artifact_result_number_10}
+\end{table}

evals/mjbench/latex_reults/artifact_number_5.tex

@@ -0,0 +1,29 @@
+\begin{table}[h]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{quality} perspective. The feedback are provided in numerical scale of range [0, 5]. Specifically, we study their individual performance over two alignment objectives: distortion (including human face, human limb, and object), and blurry (including defocused and motion). The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccc|ccc}
+    \toprule
+         & \multicolumn{4}{c}{\bf Distortion} & \multicolumn{3}{c}{\bf Blurry} \\
+         & Human Face & Human Limb & Object & \cellcolor{skyblue}Avg  & Defocused & Motion & \cellcolor{skyblue}Avg \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & 0.00 & 0.00 & 0.00 & \cellcolor{skyblue} 0.00 & 2.90 & 11.3 & \cellcolor{skyblue} 7.80 \\
+         LLaVA-1.5-13b$^\heartsuit$ & 0.00 & 0.00 & 0.00 & \cellcolor{skyblue} 0.00 & 24.9 & 36.9 & \cellcolor{skyblue} 32.9 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & 11.2 & 13.9 & 1.00 & \cellcolor{skyblue} 8.70 & 56.3 & 73.2 & \cellcolor{skyblue} 61.1 \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & 18.3 & 17.9 & 17.0 & \cellcolor{skyblue} 17.7 & 27.7 & 34.3 & \cellcolor{skyblue} 28.8 \\
+         Instructblip-7b$^\heartsuit$ & 9.50 & 3.30 & 19.0 & \cellcolor{skyblue} 10.6 & 10.0 & 10.2 & \cellcolor{skyblue} 9.60 \\
+         Prometheus-Vision-7b$^\heartsuit$ & 20.1 & 15.2 & 12.0 & \cellcolor{skyblue} 15.8 & 26.3 & 29.5 & \cellcolor{skyblue} 27.5 \\
+         Prometheus-Vision-13b$^\heartsuit$ & 7.10 & 5.30 & 7.00 & \cellcolor{skyblue} 6.50 & 9.70 & 11.5 & \cellcolor{skyblue} 10.9 \\
+         Qwen-VL-Chat$^\spadesuit$ & 24.9 & 21.2 & 7.00 & \cellcolor{skyblue} 17.7 & 18.3 & 19.6 & \cellcolor{skyblue} 18.9 \\
+         Internvl-chat-v1-5$^\spadesuit$ & 21.9 & 24.5 & 1.00 &\cellcolor{skyblue} 15.8 & \bf 93.7 & 96.6 & \cellcolor{skyblue} \bf 95.7 \\
+         Idefics2-8b$^\spadesuit$ & 44.4 & 33.1 & 9.0 & \cellcolor{skyblue} 28.8 & 88.3 & 68.6 & \cellcolor{skyblue} 75.9 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & 86.3 & 54.1 & 79.2 & \cellcolor{skyblue} 72.4 & 90.8 & 93.3 & \cellcolor{skyblue} 91.2 \\
+         GPT-4o$^\clubsuit$ & \bf 98.6 & \bf 73.5 & \bf 100 & \cellcolor{skyblue} \bf 90.4 & 91.6 & \bf 96.7 & \cellcolor{skyblue} 93.0 \\
+         Gemini Ultra$^\clubsuit$ & 71.6 & 29.9 & 59.8 & \cellcolor{skyblue} 50.7 & 80.7 & 90.8 & \cellcolor{skyblue} 83.9 \\
+         Claude 3 Opus$^\clubsuit$ & 21.6 & 16.9 & 9.30 & \cellcolor{skyblue} 16.6 & 85.3 & 93.3 & \cellcolor{skyblue} 87.7 \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:artifact_result_number_5}
+\end{table}

evals/mjbench/latex_reults/bias_acc.tex

@@ -0,0 +1,39 @@
+
+\begin{table}[t]
+    \centering
+    \caption{The detailed evaluation result in terms of ACC (accuracy) for all multimodal judges on \textbf{bias} perspective. The feedback is provided in numerical scale with range [0, 10]. Specifically, we separately report the bias w.r.t. different demographic identifications, i.e. age, gender, race, nationality, and religion. The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccccc}
+    \toprule
+         % & \multicolumn{6}{c}{\bf Occupation} & \multicolumn{4}{c}{\bf Education} \\
+         & Age & Gender & Race & Nationality & Religion & \cellcolor{skyblue}Avg \\
+         \midrule
+         CLIP-v1$^\diamondsuit$ & 57.2 & 57.8 & 55.5 & 59.5 & 60.8 & \cellcolor{skyblue} 57.7  \\
+         BLIP-v2$^\diamondsuit$ & 69.6 & 68.5 & 65.9 & 68.6 & 74.7 & \cellcolor{skyblue} 68.5 \\
+         PickScore-v1$^\diamondsuit$ & 30.4 & 31.1 & 30.8 & 31.7 & 33.0 & \cellcolor{skyblue} 31.1 \\
+         HPS-v2.1$^\diamondsuit$ & 52.9 & 55.3 & 55.7 & 55.0 & 62.4 & \cellcolor{skyblue} 55.3  \\
+         ImageReward$^\diamondsuit$ & 41.8 & 40.4 & 36.8 & 39.5 & 52.8 & \cellcolor{skyblue} 40.4 \\
+         Aesthetics$^\diamondsuit$ & 59.4 & 62.0 & 64.2 & 62.4 & 61.0 & \cellcolor{skyblue} 62.0  \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & \bf 80.8 & \bf 83.9 & \bf 84.6 & \bf 84.9 & \bf 88.1 & \cellcolor{skyblue} \bf 84.0 \\
+         LLaVA-1.5-13b$^\heartsuit$ & 67.0 & 70.1 & 68.9 & 72.7 & 75.1 & \cellcolor{skyblue} 70.1 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & 71.8 & 70.8 & 70.8 & 67.8 & 78.3 & \cellcolor{skyblue} 70.8 \\
+         LLaVA-NeXT-vicuna-7b$^\heartsuit$ & 54.3 & 56.7 & 57.0 & 56.1 & 64.8 & \cellcolor{skyblue} 56.6 \\
+         Instructblip-7b$^\heartsuit$ & 52.5 & 53.6 & 53.6 & 52.0 & 61.1 & \cellcolor{skyblue} 53.6 \\
+         MiniGPT4-v2$^\heartsuit$ & 31.8 & 32.2 & 31.9 & 34.1 & 28.3 & \cellcolor{skyblue} 32.2 \\
+         Prometheus-Vision-7b$^\heartsuit$ & 43.8 & 50.4 & 54.4 & 53.6 & 44.9 & \cellcolor{skyblue} 50.4 \\
+         Prometheus-Vision-13b$^\heartsuit$ & 65.1 & 65.8 & 63.4 & 65.7 & 77.1 & \cellcolor{skyblue} 65.8 \\
+         Qwen-VL-Chat$^\spadesuit$ & 70.8 & 71.5 & 72.3 & 72.2 & 68.1 & \cellcolor{skyblue} 71.5 \\
+         Internvl-chat-v1-5$^\spadesuit$ & 40.0 & 41.3 & 42.1 & 42.0 & 39.8 & \cellcolor{skyblue} 41.3 \\
+         Idefics2-8b$^\spadesuit$ & 37.4 & 42.7 & 45.3 & 46.9 & 35.2 & \cellcolor{skyblue} 42.7 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & 76.7 & 79.1 & 77.4 & 81.0 & 86.5 & \cellcolor{skyblue} 79.1 \\
+         GPT-4o$^\clubsuit$ & 60.9 & 66.6 & 69.1 & 68.2 & 69.6 & \cellcolor{skyblue} 66.6 \\
+         Gemini Ultra$^\clubsuit$ & 48.7 & 56.9 & 62.9 & 60.0 & 49.9 & \cellcolor{skyblue} 56.9 \\
+         Claude 3 Opus$^\clubsuit$ & 53.9 & 58.2 & 62.1 & 59.0 & 54.0 & \cellcolor{skyblue} 58.2 \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:bias_acc}
+\end{table}
+

evals/mjbench/latex_reults/bias_ges.tex

@@ -0,0 +1,37 @@
+\begin{table}[t]
+    \centering
+    \caption{The detailed evaluation result in terms of Gini-based Equality Score (GES) for all multimodal judges on \textbf{bias} perspective. The feedback is provided in numerical scale with range [0, 10]. Specifically, we separately report the bias w.r.t. different demographic identifications, i.e. age, gender, race, nationality, and religion. The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccccc}
+    \toprule
+         % & \multicolumn{6}{c}{\bf Occupation} & \multicolumn{4}{c}{\bf Education} \\
+         & Age & Gender & Race & Nationality & Religion & \cellcolor{skyblue}Avg \\
+         \midrule
+         CLIP-v1$^\diamondsuit$ & 73.6 & 75.2 & 73.1 & 79.1 & 78.4 & \cellcolor{skyblue} 75.2  \\
+         BLIP-v2$^\diamondsuit$ & 92.2 & 91.3 & 90.7 & 90.4 & 93.1 & \cellcolor{skyblue} 91.3 \\
+         PickScore-v1$^\diamondsuit$ & 80.5 & 81.2 & 81.0 & 81.6 & 82.6 & \cellcolor{skyblue} 81.2 \\
+         HPS-v2.1$^\diamondsuit$ & 86.4 & 87.8 & 88.5 & 88.0 & 88.5 & \cellcolor{skyblue} 87.8  \\
+         ImageReward$^\diamondsuit$ & 85.5 & 85.0 & 83.6 & 84.8 & 89.0 & \cellcolor{skyblue} 85.0 \\
+         Aesthetics$^\diamondsuit$ & 91.9 & 92.1 & 92.4 & 92.1 & 92.3 & \cellcolor{skyblue} 92.1  \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & 87.4 & 88.9 & 90.1 & 88.7 & 90.7 & \cellcolor{skyblue} 88.9 \\
+         LLaVA-1.5-13b$^\heartsuit$ & 87.5 & 88.8 & 88.9 & 89.5 & 90.1 & \cellcolor{skyblue} 88.8 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & 86.4 & 85.8 & 85.8 & 84.1 & 90.2 & \cellcolor{skyblue} 85.8 \\
+         LLaVA-NeXT-vicuna-7b$^\heartsuit$ & 82.1 & 82.8 & 82.4 & 82.5 & 87.8 & \cellcolor{skyblue} 82.8\\
+         Instructblip-7b$^\heartsuit$ & 91.0 & 91.2 & 91.1 & 90.4 & 93.8 & \cellcolor{skyblue} 91.1 \\
+         MiniGPT4-v2$^\heartsuit$ & 83.7 & 83.3 & 82.8 & 83.4 & 84.1 & \cellcolor{skyblue} 83.3 \\
+         Prometheus-Vision-7b$^\heartsuit$ & 74.9 & 74.3 & 73.1 & 74.2 & 77.3 & \cellcolor{skyblue} 74.3 \\
+         Prometheus-Vision-13b$^\heartsuit$ & 79.2 & 76.0 & 72.7 & 74.1 & 85.1 & \cellcolor{skyblue} 76.0 \\
+         Qwen-VL-Chat$^\spadesuit$ & 85.9 & 86.0 & 86.0 & 86.4 & 83.8 & \cellcolor{skyblue} 85.9 \\
+         Internvl-chat-v1-5$^\spadesuit$ & 86.9 & 87.2 & 87.1 & 87.3 & 88.0 & \cellcolor{skyblue} 87.2 \\
+         Idefics2-8b$^\spadesuit$ & 77.0 & 79.7 & 81.3 & 82.0 & 74.4 & \cellcolor{skyblue} 79.8 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & \bf 93.0 & \bf 93.2 & 92.2 & \bf 93.4 & \bf 96.4 & \cellcolor{skyblue} \bf 93.2 \\
+         GPT-4o$^\clubsuit$ & 91.8 & 92.9 & \bf 93.1 & 93.3 & 94.4 & \cellcolor{skyblue} 92.9 \\
+         Gemini Ultra$^\clubsuit$ & 86.6 & 89.0 & 90.8 & 90.0 & 86.2 & \cellcolor{skyblue} 89.0 \\
+         Claude 3 Opus$^\clubsuit$ & 83.2 & 85.2 & 86.5 & 85.8 & 84.8 & \cellcolor{skyblue} 85.2 \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:bias_ges}
+\end{table}

evals/mjbench/latex_reults/bias_nds.tex

@@ -0,0 +1,39 @@
+\begin{table}[t]
+    \centering
+    \caption{The detailed evaluation result in terms of Normalized Dispersion Score (NDS) for all multimodal judges on \textbf{bias} perspective. The feedback is provided in numerical scale with range [0, 10]. Specifically, we separately report the bias w.r.t. different demographic identifications, i.e. age, gender, race, nationality, and religion. The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccccc}
+    \toprule
+         % & \multicolumn{6}{c}{\bf Occupation} & \multicolumn{4}{c}{\bf Education} \\
+         & Age & Gender & Race & Nationality & Religion & \cellcolor{skyblue}Avg \\
+         \midrule
+         CLIP-v1$^\diamondsuit$ & 73.6 & 75.2 & 73.1 & 79.1 & 78.4 & \cellcolor{skyblue} 75.2  \\
+         BLIP-v2$^\diamondsuit$ & 85.3 & 83.6 & 82.7 & 81.8 & 87.5 & \cellcolor{skyblue} 83.6 \\
+         PickScore-v1$^\diamondsuit$ & 65.3 & 66.7 & 66.4 & 67.3 & 69.4 & \cellcolor{skyblue} 66.7  \\
+         HPS-v2.1$^\diamondsuit$ & 75.8 & 78.2 & 79.5 & 78.6 & 79.3 & \cellcolor{skyblue} 78.2  \\
+         ImageReward$^\diamondsuit$ & 73.9 & 73.2 & 70.9 & 73.0 & 80.2 & \cellcolor{skyblue} 73.2 \\
+         Aesthetics$^\diamondsuit$ & \bf 85.3 & \bf 85.9 & \bf 86.3 & \bf 85.8 & 86.2 & \cellcolor{skyblue} \bf 85.9  \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & 67.6 & 71.4 & 75.8 & 68.4 & 77.3 & \cellcolor{skyblue} 71.4 \\
+         LLaVA-1.5-13b$^\heartsuit$ & 71.9 & 74.8 & 76.6 & 74.0 & 80.6 & \cellcolor{skyblue} 74.8 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & 68.4 & 64.6 & 62.4 & 59.7 & 78.1 & \cellcolor{skyblue} 64.6 \\
+         LLaVA-NeXT-vicuna-7b$^\heartsuit$ & 63.2 & 64.1 & 62.5 & 63.8 & 74.2 & \cellcolor{skyblue} 64.1\\
+         Instructblip-7b$^\heartsuit$ & 80.8 & 80.6 & 80.3 & 79.0 & 85.4 & \cellcolor{skyblue} 80.6 \\
+         MiniGPT4-v2$^\heartsuit$ & 68.1 & 67.2 & 66.2 & 67.0 & 69.3 & \cellcolor{skyblue} 67.2 \\
+         Prometheus-Vision-7b$^\heartsuit$ & 47.2 & 42.5 & 37.8 & 40.0 & 54.2 & \cellcolor{skyblue} 42.5 \\
+         Prometheus-Vision-13b$^\heartsuit$ & 54.2 & 44.7 & 36.0 & 39.3 & 65.7 & \cellcolor{skyblue} 44.7 \\
+         Qwen-VL-Chat$^\spadesuit$ & 62.4 & 62.3 & 62.3 & 63.1 & 58.9 & \cellcolor{skyblue} 62.3 \\
+         Internvl-chat-v1-5$^\spadesuit$ & 74.0 & 74.1 & 73.6 & 73.9 & 76.6 & \cellcolor{skyblue} 74.1 \\
+         Idefics2-8b$^\spadesuit$ & 55.1 & 59.2 & 61.7 & 62.8 & 51.0 & \cellcolor{skyblue} 59.2 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & 81.2 & 80.2 & 77.6 & 79.9 & \bf 88.2 & \cellcolor{skyblue} 80.2 \\
+         GPT-4o$^\clubsuit$ & 81.2 & 82.7 & 82.8 & 83.2 & 86.1 & \cellcolor{skyblue} 82.7 \\
+         Gemini Ultra$^\clubsuit$ & 72.6 & 75.8 & 78.4 & 77.0 & 72.3 & \cellcolor{skyblue} 75.8 \\
+         Claude 3 Opus$^\clubsuit$ & 63.3 & 66.1 & 67.5 & 66.9 & 66.8 & \cellcolor{skyblue} 66.1 \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:bias_nds}
+\end{table}
+
+

evals/mjbench/latex_reults/bias_scale.tex

@@ -0,0 +1,30 @@
+\begin{table}[t]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{bias} perspective. The feedback are provided in different scales including numerical scales ([0-5], and [0-10]) and Likert scale: [\textit{Extremely Poor}, \textit{Poor}, \textit{Average}, \textit{Good}, \textit{Outstanding}]. We study the average ACC, NDS, and GES score for each model across all occupations/educations. The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|ccc|ccc|ccc}
+    \toprule
+         & \multicolumn{3}{c}{\bf Numerical [0-5]} & \multicolumn{3}{c}{\bf Numerical [0-10]} & \multicolumn{3}{c}{\bf Likert scale}\\
+         & ACC & NDS & GES & ACC & NDS & GES & ACC & NDS & GES \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & \bf 80.8 & 64.6 & 87.7 & 47.1 & 77.3 & 90.1 & \bf 81.5 & 82.4 & \bf 94.2 \\
+         LLaVA-1.5-13b$^\heartsuit$ & 55.5 & 77.5 & 90.0 & 37.8 & 78.7 & 89.4 & 61.2 & 78.4 & 91.0 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & 72.1 & 71.2 & 88.3 & 58.6 & 65.4 & 84.1 & 59.1 & 68.3 & 86.1 \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & 49.3 & 68.1 & 85.2 & 42.6 & 69.6 & 84.9 & 53.5 & 73.1 & 87.6\\
+         Instructblip-7b$^\heartsuit$ & 58.7 & \bf 85.3 & 91.5 & 53.6 & 80.6 & 91.1 & 71.5 & 84.5 & 94.3 \\
+         MiniGPT4-v2$^\heartsuit$ & 35.6 & 69.2 & 79.5 & 32.6 & 67.0 & 83.3 & 38.5 & 39.3 & 68.9 \\
+          Prometheus-Vision-7b$^\heartsuit$ & 49.5 & 43.4 & 74.4 & 52.1 & 37.9 & 73.0 & 47.4 & 25.3 & 64.6 \\
+          Prometheus-Vision-13b$^\heartsuit$ & 66.3 & 46.3 & 76.8 & \bf 68.2 & 23.3 & 69.4 & 67.6 & 47.4 & 77.6 \\
+         Qwen-VL-Chat$^\spadesuit$ & 71.8 & 76.3 & 91.3 & 30.1 & 70.6 & 85.7 & 45.9 & 74.9 & 88.0 \\
+         Internvl-chat-v1-5$^\spadesuit$ & 41.0 & 74.1 & 87.2 & 25.4 & 69.6 & 84.3 & 59.2 & 83.6 & 92.6\\
+         Idefics2-8b$^\spadesuit$ & 41.9 & 68.7 & 84.4 & 42.1 & 66.7 & 83.4 & 61.6 & \bf 86.5 & 93.9 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & 79.1 & 80.2 & \bf 93.2 & 41.5 & \bf 86.4 & \bf 93.7 & 58.7 & 69.8 & 87.1 \\
+         GPT-4o$^\clubsuit$ & 66.6 & 82.7 & 92.9 & 26.2 & 74.2 & 86.5 & 74.3 & 79.2 & 92.2 \\
+         Gemini Ultra$^\clubsuit$ & 56.9 & 75.8 & 89.0 & 36.2 & 72.4 & 85.6 & 74.5 & 78.4 & 91.6 \\
+         Claude 3 Opus$^\clubsuit$ & 58.2 & 66.1 & 85.2 & 52.1 & 59.5 & 82.1 & 57.4 & 83.6 & 92.5 \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:bias_scale}
+\end{table}

evals/mjbench/latex_reults/consitient_analysis.tex

@@ -0,0 +1,26 @@
+\begin{table}[htb]
+    \vspace{-5pt}
+    \centering
+    \small
+    \caption{Comparison of open-source judges w.r.t. different input modes. Specifically, we study VLMs with single image input, pairwise image input (pair-f), and pairwise image input in reverse order (pair-r). The best performance is in bold.}
+    
+    \resizebox{0.92\linewidth}{!}{%
+    \begin{tabular}{l|ccc|ccc|cccccc}
+    \toprule
+         & \multicolumn{3}{c}{\bf Alignment} & \multicolumn{3}{c}{\bf Safety} & \multicolumn{3}{c}{\bf Artifact}  \\
+         & single & pair-f  & pair-r  & single & pair-f & pair-r  & single & pair-f & pair-r  \\
+         \midrule
+          Qwen-VL-Chat$^\spadesuit$       & $29.1$ & $31.1$ & $\textbf{73.0}$ & $\textbf{33.5}$ & $6.8$  & $\textbf{60.1}$  & $19.8$ & $5.7$   & $41.5$  \\
+          Internvl-chat-v1-5$^\spadesuit$ & $\textbf{32.8}$ & $\textbf{75.8}$ & $34.8$ & $20.1$ & $5.9$  & $4.6$   & $38.8$ & $\textbf{91.8}$  & $40.7$  \\
+          Idefics2-8b$^\spadesuit$        & $30.2$ & $32.6$ & $32.6$ & $27.3$ & $\textbf{13.7}$ & $32.6$  & $\textbf{40.2}$ & $49.0$  & $\textbf{43.2}$  \\
+          % \midrule
+          % GPT-4-vision$^\clubsuit$        & - & - & - & - & - & - & 80.4 & 93.2  \\
+          % GPT-4o$^\clubsuit$              & - & - & - & - & - & - & 82.5 & 92.8 \\
+          % Gemini Ultra$^\clubsuit$        & - & - & - & - & - & - & 75.3 & 88.6 \\
+          % Claude 3 Opus$^\clubsuit$       & - & - & - & - & - & - & 65.6 & 85.0  \\
+    \bottomrule
+    \end{tabular}%
+    }
+    
+    \label{exp:judge_consitiency}
+\end{table}

evals/mjbench/latex_reults/dataset.text

@@ -0,0 +1,69 @@
+\begin{table}[h!]
+    \centering
+    \caption{Summary of the dataset proposed in \algname.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{lllrl}
+        \toprule
+        \textbf{Category} & \textbf{Scenario} & \textbf{Subset} & \textbf{N} & \textbf{Description} \\ 
+        \midrule
+        \multirow{5}{*}{\textbf{Alignment}} 
+        & \multirow{1}{*}{Object}      & -  & 250 & Ensures the correct objects are present in the image as specified by the text \\ 
+        \cmidrule{2-5}
+        & \multirow{1}{*}{Attributes}  & -  & 229 & Verifies correct association of attributes such as color, shape, size, and texture \\ 
+        \cmidrule{2-5}
+        & \multirow{1}{*}{Actions}     & -  & 115 & Ensures actions specified in the text are accurately depicted in the image \\ 
+        \cmidrule{2-5}
+        & \multirow{1}{*}{Counting}    & -  & 55 & Verifies the correct number of objects as specified by the text \\ 
+        \cmidrule{2-5}
+        & \multirow{1}{*}{Spatial}     & -  & 75 & Ensures correct spatial relationships and positions of objects in the image \\ 
+        
+        \midrule
+        \multirow{8}{*}{\textbf{Safety}} 
+        & \multirow{3}{*}{Toxicity} & Crime & 29 & Evaluates the presence of crime-related content in images \\ 
+        \cmidrule{3-5}
+        & & Shocking & 31 & Evaluates the presence of shocking or disturbing content in images \\ 
+        \cmidrule{3-5}
+        & & Disgust & 42 & Evaluates the presence of disgusting or offensive content in images \\ 
+        \cmidrule{2-5}
+        & \multirow{4}{*}{NSFW} & Evident & 197 & Images with clear and obvious NSFW content \\ 
+        \cmidrule{3-5}
+        & & Evasive & 177 & Images with attempts to subtly include NSFW content \\ 
+        \cmidrule{3-5}
+        & & Subtle & 98 & Images with low-level, hard-to-detect NSFW content \\ 
+        
+        \midrule
+        \multirow{8}{*}{\textbf{Quality}} 
+        & \multirow{3}{*}{Distortion} & Human Face & 169 & Prefers images without distortions in human faces \\ 
+        \cmidrule{3-5}
+        & & Human Limbs & 152 & Prefers images without distortions in human limbs \\ 
+        \cmidrule{3-5}
+        & & Object & 100 & Prefers images without distortions in objects \\ 
+        \cmidrule{2-5}
+        & \multirow{2}{*}{Blurry} & Defocused blur & 350 & Evaluates resistance to defocused blur in images \\ 
+        \cmidrule{3-5}
+        &  & Motion blur & 350 & Evaluates resistance to motion blur in images \\ 
+        
+        \midrule
+        \multirow{10}{*}{\textbf{Bias}} 
+        & \multirow{5}{*}{Occupation} & Age & 80 & Evaluates bias across different age groups (young, adult, old) \\ 
+        \cmidrule{3-5}
+        & & Gender & 80 & Evaluates bias across different genders (male, female, non-binary) \\ 
+        \cmidrule{3-5}
+        & & Race & 80 & Evaluates bias across different races (Asian, Black, Latino, Middle Eastern, Indian, White) \\ 
+        \cmidrule{3-5}
+        & & Nationality & 60 & Evaluates bias across different nationalities \\ 
+        \cmidrule{3-5}
+        & & Nationality (continued) & 60 & (American, Mexican, European, Spanish, British, Russian, Chinese, Japanese, Korean) \\ 
+        \cmidrule{3-5}
+        & & Religion & 60 & Evaluates bias across different religions (Christian, Muslim, Jewish, Hindu) \\ 
+        \cmidrule{2-5}
+        & \multirow{3}{*}{Education} & Gender & 60 & Evaluates bias in educational contexts across different genders \\ 
+        \cmidrule{3-5}
+        & & Race & 60 & Evaluates bias in educational contexts across different races \\ 
+        \cmidrule{3-5}
+        & & Nationality & 60 & Evaluates bias in educational contexts across different nationalities \\ 
+        \bottomrule
+    \end{tabular}
+    }
+    \label{tab:dataset_detail}
+\end{table}

evals/mjbench/latex_reults/human_eval.tex

@@ -0,0 +1,22 @@
+\begin{table}[t]
+    \centering
+    \caption{Human evaluation result on the generated images from six fine-tuned SD-v1.5 model using the feedback from six multimodal judges, i.e. GPT-4o, GPT-4-vision, Gemini Ultra, Claude 3 Opus, Internvl-chat-v1-5, and HPS-v2.1. Specifically, we consider the following four metrics: ranking over fixed seed (\textbf{FR}), ranking over random seed (\textbf{RR}), average ranking (\textbf{AR}), and average voting (\textbf{AV}). The best performance across all models are bolded.}
+    \setlength{\tabcolsep}{2pt}
+    \renewcommand{\arraystretch}{0.9}
+\resizebox{1.0\linewidth}{!}{%
+\begin{tabular}{l|cccc|cccc|cccc}
+\toprule
+     & \multicolumn{4}{c}{\bf Alignment} & \multicolumn{4}{c}{\bf Safety} & \multicolumn{4}{c}{\bf Bias}  \\
+     & FR $\downarrow$ & RR $\downarrow$ & \cellcolor{skyblue}{AR $\downarrow$} & \cellcolor{skyblue}{AV $\uparrow$} & FR $\downarrow$ & RR $\downarrow$ & \cellcolor{skyblue}{AR $\downarrow$} & \cellcolor{skyblue}{AV $\uparrow$} & FR $\downarrow$ & RR $\downarrow$ & \cellcolor{skyblue}{AR $\downarrow$} & \cellcolor{skyblue}{AV $\uparrow$} \\
+     \midrule
+      GPT-4o$^\clubsuit$       & \bf 2.16 & \bf 2.66 & \cellcolor{skyblue}{\bf 2.50} & \cellcolor{skyblue}{\bf 17.21\%} & 1.91 & \bf 1.88 & \cellcolor{skyblue}{\bf 1.89} & \cellcolor{skyblue}{\bf 17.37\%} & \bf 1.72 & \bf 2.48 & \cellcolor{skyblue}{\bf 2.10} & \cellcolor{skyblue}{\bf 21.58\%} \\
+      GPT-4-vision$^\clubsuit$ & 2.43 & 2.81 & \cellcolor{skyblue}{2.68} & \cellcolor{skyblue}{15.96\%} & \bf 1.84 & 1.98 & \cellcolor{skyblue}{1.94} & \cellcolor{skyblue}{16.81\%} & 1.99 & 3.14 & \cellcolor{skyblue}{2.57} & \cellcolor{skyblue}{16.80\%} \\
+      Gemini Ultra$^\clubsuit$ & \bf 2.15 & 2.72 & \cellcolor{skyblue}{2.54} & \cellcolor{skyblue}{14.87\%} & \bf 1.55 & \bf 1.69 & \cellcolor{skyblue}{\bf 1.64} & \cellcolor{skyblue}{\bf 18.98\%} & 2.23 & \bf 2.65 & \cellcolor{skyblue}{2.44} & \cellcolor{skyblue}{16.18\%} \\
+      Claude 3 Opus$^\clubsuit$ & 2.25 & 2.80 & \cellcolor{skyblue}{2.62} & \cellcolor{skyblue}{15.34\%} & 2.07 & 2.12 & \cellcolor{skyblue}{2.10} & \cellcolor{skyblue}{16.15\%} & 2.29 & 3.43 & \cellcolor{skyblue}{2.86} & \cellcolor{skyblue}{11.62\%} \\
+      Internvl-chat-v1-5$^\spadesuit$ & 3.16 & 2.99 & \cellcolor{skyblue}{3.05} & \cellcolor{skyblue}{16.90\%} & 2.49 & 2.28 & \cellcolor{skyblue}{2.35} & \cellcolor{skyblue}{15.30\%} & 1.97 & 3.43 & \cellcolor{skyblue}{2.70} & \cellcolor{skyblue}{14.52\%} \\
+      HPS-v2.1$^\diamondsuit$ & 2.21 & \bf 2.42 & \cellcolor{skyblue}{\bf 2.35} & \cellcolor{skyblue}{\bf 19.72\%} & 2.42 & 2.37 & \cellcolor{skyblue}{2.39} & \cellcolor{skyblue}{15.39\%} & \bf 1.78 & \bf 2.65 & \cellcolor{skyblue}{\bf 2.21} & \cellcolor{skyblue}{\bf 19.29\%} \\
+\bottomrule
+\end{tabular}%
+}
+\label{exp:human_eval}
+\end{table}

evals/mjbench/latex_reults/main_result.tex

@@ -0,0 +1,49 @@
+
+\begin{table}[t]
+    \centering
+    \caption{Evaluation of three types of multimodal judges across four perspectives on \algname dataset. The average accuracy (\%) with and without ties are provided for alignment, safety, and artifact. We evaluate preference biases over three metrics, i.e. accuracy (ACC), normalized dispersion score (NDS), Gini-based equality score (GES). The best performance across all models is bolded.}
+    \setlength{\tabcolsep}{2pt}
+    \renewcommand{\arraystretch}{0.9}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{l|cc|cc|cc|ccc}
+    \toprule
+         & \multicolumn{2}{c}{\bf Alignment} & \multicolumn{2}{c}{\bf Safety} & \multicolumn{2}{c}{\bf Artifact} & \multicolumn{3}{c}{\bf Bias} \\
+         & Avg w/ tie & Avg w/o Tie & Avg w/ tie & Avg w/o Tie & Avg w/ tie & Avg w/o Tie & ACC & NDS & GES \\
+    \midrule
+         CLIP-v1$^\diamondsuit$      & $38.1$ & $59.5$ & $12.7$ & $33.3$ & $34.4$ & $68.4$ & $57.4$ & $76.3$ & $86.9$ \\
+         BLIP-v2$^\diamondsuit$      & $17.3$ & $38.8$ & $44.0$ & $65.6$ & $7.5$  & $36.5$ & $68.7$ & $83.7$ & $91.3$ \\
+         PickScore-v1$^\diamondsuit$ & $58.8$ & $64.6$ & \bf 37.2 & $42.2$ & $83.8$ & $89.6$ & $31.0$ & $66.5$ & $81.1$ \\
+         HPS-v2.1$^\diamondsuit$     & $47.3$ & \bf 70.1 & $18.8$ & $41.3$ & $67.3$ & $93.5$ & $55.0$ & $77.9$ & $87.6$ \\
+         ImageReward$^\diamondsuit$  & $50.9$ & $64.7$ & $24.9$ & $38.7$ & $63.5$ & $81.8$ & $40.9$ & $73.7$ & $85.3$ \\
+         Aesthetics$^\diamondsuit$   & $32.4$ & $52.7$ & $27.0$ & $53.6$ & $69.6$ & $92.5$ & $61.4$ & $85.7$ & $92.1$ \\
+         
+         
+    \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & $22.0$ & $50.8$ & $24.8$ & $50.2$ & $12.4$ & $51.6$ & 83.7 & 70.4 & 88.7 \\
+         LLaVA-1.5-13b$^\heartsuit$ & $10.3$ & $51.9$ & $30.7$ & $60.7$ & $23.3$ & $61.2$ & 69.7 & 74.3 & 88.6 \\
+         LLaVA-1.6-mistral-7b$^\heartsuit$ & $31.3$ & $62.7$ & $15.2$ & $40.9$ & $45.8$ & $73.2$ & 69.9 & 64.3 & 85.4 \\
+         LLaVA-1.6-vicuna-13b$^\heartsuit$ & $29.1$ & $60.3$ & $27.9$ & $45.6$ & $36.8$ & $62.5$ & 56.3 & 64.0 & 82.7 \\
+         Instructblip-7b$^\heartsuit$ & $17.1$ & $49.8$ & $26.4$ & $46.9$ & $25.2$ & $64.1$ & 53.1 & 80.8 & 91.2 \\
+         MiniGPT4-v2$^\heartsuit$ & $32.8$ & $51.2$ & $25.7$ & $60.1$ & $36.7$ & $47.8$ & 32.6 & 67.0 & 83.3 \\
+         Prometheus-Vision-7b$^\heartsuit$ & $18.8$ & $63.9$ & $7.1$ & $58.8$ & $23.4$ & $67.7$ & 49.5 & 43.4 & 74.4 \\
+         Prometheus-Vision-13b$^\heartsuit$ & $11.8$ & $64.3$ & $3.6$ & $71.4$ & $8.7$ & $67.9$ & 66.3 & 46.3 & 76.8 \\
+         % Qwen-VL-Chat$^\spadesuit$ & $31.1$ & $31.6$ & $6.8$ & $7.1$ & $5.7$ & $7.1$ & 71.9 & 62.8 & 86.2 \\
+         % Internvl-chat-v1-5$^\spadesuit$ & $75.8$ & $77.6$ & $5.9$ & $6.0$ & $91.8$ & $92.7$ & 25.4 & 69.6 & 84.3 \\
+         % Idefics2-8b$^\spadesuit$ & $32.6$ & $43.5$ & $13.7$ & $52.0$ & $49.0$ & $74.7$ & 42.1 & 58.7 & 79.4 \\
+         Qwen-VL-Chat$^\spadesuit$       & $52.1$ & $31.6$ & $26.8$  & $7.1$  & $23.6$ & $24.6$ & 71.9 & 62.8 & 86.2 \\
+         Internvl-chat-v1-5$^\spadesuit$ & $55.3$ & $67.6$ & $6.3$   & $60.0$ & $66.3$ & $65.1$ & 25.4 & 69.6 & 84.3 \\
+         Idefics2-8b$^\spadesuit$        & $32.6$ & $43.5$ & $13.6$  & $52.0$ & $46.1$ & $68.9$ & 42.1 & 58.7 & 79.4 \\
+    \midrule
+         GPT-4-vision$^\clubsuit$ & $66.1$ & $67.0$ & $26.5$ & $97.6$ & $90.4$ & $96.5$ & \bf 79.0 & 80.4 & \bf 93.2 \\
+         GPT-4o$^\clubsuit$ & $61.5$ & $62.5$ & $35.3$ & \bf 100.0 & \bf 97.6 & \bf 98.7 & 65.8 & \bf 82.5 & 92.8 \\
+         Gemini Ultra$^\clubsuit$ & \bf 67.2 & $69.0$ & $13.1$ & $95.1$ & $55.7$ & $96.7$ & 55.6 & 75.3 & 88.6 \\
+         Claude 3 Opus$^\clubsuit$ & $57.1$ & $55.9$ & $13.4$ & $78.9$ & $11.9$ & $70.4$ & 57.7 & 65.6 & 85.0 \\
+    % \midrule
+    %     Random & 33.3 & 50.0 & 33.3 & 50.0 & 33.3 & 50.0 & 33.3 & 50.0 & 50.0 \\
+    \bottomrule
+    \end{tabular}%
+    \vspace{-0.2cm}
+    }
+    \label{exp:main_result}
+\end{table}
+

evals/mjbench/latex_reults/original_scale_study.tex

@@ -0,0 +1,29 @@
+\begin{table}[t]
+    \centering
+    \caption{Result with different scale.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cc|cc|cc|cc}
+    \toprule
+         & \multicolumn{2}{c}{\bf Alignment} & \multicolumn{2}{c}{\bf Safety} & \multicolumn{2}{c}{\bf Artifact} & \multicolumn{2}{c}{\bf Bias} \\
+         & numeric & likert  & numeric & likert  & numeric & likert & numeric & likert\\
+          \midrule
+          LLaVA-1.5-7b$^\heartsuit$ & - & - & - & - & - & - & - & - \\
+          LLaVA-1.5-13b$^\heartsuit$ & - & - & - & - & - & - & - & -  \\
+          LLaVA-NeXT-mistral-7b$^\heartsuit$ & - & - & - & - & - & - & - & -  \\
+          LLaVA-NeXT-vicuna-7b$^\heartsuit$ & - & - & - & - & - & - & - & -\\
+          Instructblip-7b$^\heartsuit$ & - & - & - & - & - & - & 57.4 & 85.8  \\
+          MiniGPT4-v2$^\heartsuit$ & - & - & - & - & - & - & - & -\\
+          Prometheus-Vision-13b$^\heartsuit$ & - & - & - & - & - & - & - & -  \\
+          Qwen-VL-Chat$^\spadesuit$ & - & - & - & - & - & - & - & - \\
+          Internvl-chat-v1-5$^\spadesuit$ & - & - & - & - & - & - & 65.3 & 83.5  \\
+          Idefics2-8b$^\spadesuit$ & - & - & - & - & - & - & 52.7 & 77.6 \\
+          \midrule
+          GPT-4-vision$^\clubsuit$ & - & - & - & - & - & - & 80.4 & 93.2  \\
+          GPT-4o$^\clubsuit$ & - & - & - & - & - & - & 82.5 & 92.8 \\
+          Gemini Ultra$^\clubsuit$ & - & - & - & - & - & - & 75.3 & 88.6 \\
+          Claude 3 Opus$^\clubsuit$ & - & - & - & - & - & - & 65.6 & 85.0  \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:numeric_likert}
+\end{table}

evals/mjbench/latex_reults/safety_narrative.tex

@@ -0,0 +1,29 @@
+\begin{table}[t]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{safety} perspective. The feedback is provided in the following Likert scale: [\textit{Extremely Poor}, \textit{Poor}, \textit{Average}, \textit{Good}, \textit{Outstanding}]. Specifically, we study their individual performance over two alignment objectives: toxicity (crime, shocking, and disgust) and NSFW (evident, evasive, and subtle). The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccc|cccc}
+    \toprule
+         & \multicolumn{4}{c}{\bf Toxicity} & \multicolumn{4}{c}{\bf NSFW} \\
+         & Crime & Shocking & Disgust & \cellcolor{skyblue}Avg & Evident & Evasive & Subtle & \cellcolor{skyblue}Avg \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & $10.3$ & $31.0$ & $26.2$ & \cellcolor{skyblue} $20.2$ & 14.2 & 9.90 & 6.80 & \cellcolor{skyblue} 9.70 \\
+         LLaVA-1.5-13b$^\heartsuit$ & $13.8$ & $24.1$ & $23.8$ & \cellcolor{skyblue} $18.0$ & 16.9 & 10.5 & 9.60 & \cellcolor{skyblue} 15.6 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & $27.6$ & $17.2$ & $21.4$ & \cellcolor{skyblue} $21.3$ & 26.9 & 9.30 & 6.70 & \cellcolor{skyblue} 19.5 \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & $34.5$ & $27.6$ & $40.5$ & \cellcolor{skyblue} $32.6$ & 26.8 & 13.9 & 11.5 & \cellcolor{skyblue} 19.7 \\
+         Instructblip-7b$^\heartsuit$ & $34.5$ & $20.7$ & $31.0$ & \cellcolor{skyblue} $29.2$ & 23.9 & 12.6 & 5.90 & \cellcolor{skyblue} 16.8 \\
+          Prometheus-Vision-7b$^\heartsuit$ & $27.6$ & $20.7$ & $28.6$ & \cellcolor{skyblue} $24.7$ & 10.4 & 4.90 & 2.70 & \cellcolor{skyblue} 25.6 \\
+          Prometheus-Vision-13b$^\heartsuit$ & $0.00$ & $0.00$ & $4.80$ & \cellcolor{skyblue} $2.20$ & 9.80 & 3.00 & 1.50 & \cellcolor{skyblue} 5.60 \\
+         Qwen-VL-Chat$^\spadesuit$ & $34.5$ & $41.4$ & $42.9$ & \cellcolor{skyblue} $38.2$ & 32.2 & 24.0 & 16.6 & \cellcolor{skyblue} 30.1 \\
+         Internvl-chat-v1-5$^\spadesuit$ & $0.00$ & $3.40$ & $2.40$ & \cellcolor{skyblue} $2.20$ & 2.80 & 1.00 & 0.70 & \cellcolor{skyblue} 1.30 \\
+         Idefics2-8b$^\spadesuit$ & $37.9$ & $10.3$ & $38.1$ & \cellcolor{skyblue} $29.2$ & 20.2 & 10.0 & 7.10 & \cellcolor{skyblue} 16.7 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & $10.3$ & $24.1$ & $31.0$ & \cellcolor{skyblue} $22.5$ & 64.0 & 50.1 & 34.4 & \cellcolor{skyblue} \bf 54.4 \\
+         GPT-4o$^\clubsuit$ & $34.5$ & $\bf 48.3$ & $50.0$ & \cellcolor{skyblue} $46.1$ & \bf 69.6 & \bf 50.9 & \bf 35.9 & \cellcolor{skyblue} 50.3 \\
+         Gemini Ultra$^\clubsuit$ & $\bf 41.4$ & $44.8$ & $\bf 66.7$ & \cellcolor{skyblue} $\bf 52.8$ & 53.5 & 45.6 & 31.9 & \cellcolor{skyblue} 51.5 \\
+         Claude 3 Opus$^\clubsuit$ & $10.3$ & $3.40$ & $4.80$ & \cellcolor{skyblue} $5.60$ & 45.6 & 32.4 & 27.0 & \cellcolor{skyblue} 35.2 \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:safety_result_narrative_5}
+\end{table}

evals/mjbench/latex_reults/safety_number_10.tex

@@ -0,0 +1,38 @@
+
+\begin{table}[t]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{safety} perspective. The feedback are provided in numerical scale of range [0, 10]. Specifically, we study their individual performance over two alignment objectives: toxicity (crime, shocking, and disgust) and NSFW (evident, evasive, and subtle). The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccc|cccc}
+    \toprule
+         & \multicolumn{4}{c}{\bf Toxicity} & \multicolumn{4}{c}{\bf NSFW} \\
+         & Crime & Shocking & Disgust & \cellcolor{skyblue}Avg & Evident & Evasive & Subtle & \cellcolor{skyblue}Avg \\
+         \midrule
+         CLIP-v1$^\diamondsuit$ & $\bf 89.7$ & $\bf 96.6$ & $\bf 97.6$ & \cellcolor{skyblue} $\bf 94.4$ & $20.8$ & $4.50$ & $16.6$ & \cellcolor{skyblue} $7.90$  \\
+         BLIP-v2$^\diamondsuit$ & $6.90$ & $0.00$ & $4.80$ &  \cellcolor{skyblue} $4.50$ & $58.4$ & $51.1$ & $35.7$ & \cellcolor{skyblue} $49.1$  \\
+         PickScore-v1$^\diamondsuit$ & $89.7$ & $82.8$ & $88.1$ & \cellcolor{skyblue} $86.5$ & $3.10$ & $48.2$ & $2.10$ & \cellcolor{skyblue} $32.2$  \\
+         HPS-v2.1$^\diamondsuit$ & $89.7$ & $86.2$ & $85.7$ & \cellcolor{skyblue} $87.6$ & $1.10$ & $30.8$ & $0.6$ & \cellcolor{skyblue} $15.1$   \\
+         ImageReward$^\diamondsuit$ & $96.6$ & $96.6$ & $95.2$ & \cellcolor{skyblue} $95.5$ & $31.1$ & $10.2$ & $27.4$ & \cellcolor{skyblue} $18.2$  \\
+         Aesthetics$^\diamondsuit$ & $51.7$ & $58.6$ & $64.3$ & \cellcolor{skyblue}  $57.3$& $14.6$ & $\bf 55.2$ & $14.2$ & \cellcolor{skyblue} $37.5$  \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & $44.8$ & $41.4$ & $47.6$ & \cellcolor{skyblue} $43.8$ & $35.7$ & $21.2$ & $17.6$ & \cellcolor{skyblue} $26.3$ \\
+         LLaVA-1.5-13b$^\heartsuit$ & $31.0$ & $31.0$ & $40.5$ & \cellcolor{skyblue} $33.7$ & $40.8$ & $29.9$ & $33.6$ & \cellcolor{skyblue} $34.7$ \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & $20.7$ & $24.1$ & $19.0$ & \cellcolor{skyblue} $21.3$ & $35.7$ & $14.1$ & $23.3$ & \cellcolor{skyblue} $25.6$ \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & $44.8$ & $37.9$ & $52.4$ & \cellcolor{skyblue} $43.8$ & $40.9$ & $25.1$ & $27.8$ & \cellcolor{skyblue} $36.5$ \\
+         Instructblip-7b$^\heartsuit$ & $31.0$ & $34.5$ & $40.5$ & \cellcolor{skyblue} $39.3$ & $36.9$ & $24.2$ & $30.6$ & \cellcolor{skyblue} $33.7$ \\
+         MiniGPT4-v2$^\heartsuit$ & $41.4$ & $62.1$ & $42.9$ & \cellcolor{skyblue} $48.3$ & $39.6$ & $21.4$ & $36.5$ & \cellcolor{skyblue} $32.6$ \\
+          Prometheus-Vision-7b$^\heartsuit$ & $0.00$ & $0.00$ & $0.00$ & \cellcolor{skyblue} $0.00$ & $10.3$ & $6.80$ & $4.30$ & \cellcolor{skyblue} $7.10$ \\
+          Prometheus-Vision-13b$^\heartsuit$ & $0.00$ & $0.00$ & $0.00$ & \cellcolor{skyblue} $0.00$ & $6.50$ & $4.10$ & $4.20$ & \cellcolor{skyblue} $5.30$ \\
+         Qwen-VL-Chat$^\spadesuit$ & $27.6$ & $13.8$ & $31.0$ & \cellcolor{skyblue} $24.7$ & $18.9$ & $7.60$ & $6.30$ & \cellcolor{skyblue} $11.6$ \\
+         Internvl-chat-v1-5$^\spadesuit$ & $34.5$ & $10.3$ & $28.6$ & \cellcolor{skyblue} $25.8$ & $23.3$ & $10.6$ & $7.20$ & \cellcolor{skyblue} $16.2$ \\
+         Idefics2-8b$^\spadesuit$ & $58.6$ & $44.8$ & $57.1$ & \cellcolor{skyblue} $52.8$ & $32.9$ & $13.2$ & $19.5$ & \cellcolor{skyblue} $20.2$ \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & $75.9$ & $69.0$ & $81.0$ & \cellcolor{skyblue} $76.4$ & $69.5$ & $43.2$ & $32.5$ & \cellcolor{skyblue} $44.1$ \\
+         GPT-4o$^\clubsuit$ & $86.2$ & $\bf 96.6$ & $95.2$ & \cellcolor{skyblue} $92.1$ & $\bf 72.3$ & $51.7$ & $\bf 38.9$ & \cellcolor{skyblue} $\bf 54.3$ \\
+         Gemini Ultra$^\clubsuit$ & $65.5$ & $41.4$ & $78.6$ & \cellcolor{skyblue} $64.0$ & $31.6$ & $19.1$ & $10.3$ & \cellcolor{skyblue} $22.7$ \\
+         Claude 3 Opus$^\clubsuit$ & $62.1$ & $37.9$ & $50.0$ & \cellcolor{skyblue} $50.6$ & $10.5$ & $6.20$ & $3.60$ & \cellcolor{skyblue} $8.30$ \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:safety_result_number_10}
+\end{table}

evals/mjbench/latex_reults/safety_number_5.tex

@@ -0,0 +1,30 @@
+
+\begin{table}[t]
+    \centering
+    \caption{The detailed evaluation result of all multimodal judges on \textbf{safety} perspective. The feedback is provided in numerical scale of range [0, 5]. Specifically, we study their individual performance over two alignment objectives: toxicity (crime, shocking, and disgust) and NSFW (evident, evasive, and subtle). The best performance across all models is bolded.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cccc|cccc}
+    \toprule
+         & \multicolumn{4}{c}{\bf Toxicity} & \multicolumn{4}{c}{\bf NSFW} \\
+         & Crime & Shocking & Disgust & \cellcolor{skyblue}Avg & Evident & Evasive & Subtle & \cellcolor{skyblue}Avg \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & $10.3$ & $20.7$ & $19.0$ & \cellcolor{skyblue} $15.7$ & 13.5 & 11.2 & 5.10 & \cellcolor{skyblue} 7.60 \\
+         LLaVA-1.5-13b$^\heartsuit$ & $13.8$ & $10.3$ & $23.8$ & \cellcolor{skyblue} $16.9$ & 16.9 & 11.2 & 8.90 & \cellcolor{skyblue} 12.7 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & $20.7$ & $17.2$ & $16.7$ & \cellcolor{skyblue} $16.9$ & 15.6 & 8.70 & 5.30 & \cellcolor{skyblue} 9.30 \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & $31.0$ & $27.6$ & $31.0$ & \cellcolor{skyblue} $27.0$ & 19.2 & 14.3 & 10.7 & \cellcolor{skyblue} 15.5 \\
+         Instructblip-7b$^\heartsuit$ & $20.7$ & $31.0$ & $16.7$ & \cellcolor{skyblue} $24.7$ & 16.8 & 12.4 & 5.60 & \cellcolor{skyblue} 13.0 \\
+          Prometheus-Vision-7b$^\heartsuit$ & $6.90$ & $0.00$ & $7.10$ & \cellcolor{skyblue} $4.50$ & 10.9 & 4.30 & 2.10 & \cellcolor{skyblue} 5.90 \\
+          Prometheus-Vision-13b$^\heartsuit$ & $0.00$ & $0.00$ & $0.00$ & \cellcolor{skyblue} $0.00$ & 9.30 & 2.50 & 1.30 & \cellcolor{skyblue} 4.90 \\
+         Qwen-VL-Chat$^\spadesuit$ & $31.0$ & $34.5$ & $21.4$ & \cellcolor{skyblue} $30.3$ & 31.6 & 24.9 & 16.3 & \cellcolor{skyblue} 25.3 \\
+         Internvl-chat-v1-5$^\spadesuit$ & $24.1$ & $6.90$ & $23.8$ & \cellcolor{skyblue} $19.1$ & 19.5 & 10.3 & 6.80 & \cellcolor{skyblue} 13.0 \\
+         Idefics2-8b$^\spadesuit$ & $44.8$ & $41.4$ & $54.8$ & \cellcolor{skyblue} $47.2$ & 29.1 & 10.6 & 8.60 & \cellcolor{skyblue} 16.8 \\
+         \midrule
+         GPT-4-vision$^\clubsuit$ & $69.0$ & $72.4$ & $73.8$ & \cellcolor{skyblue} $70.8$ & 63.5 & 49.6 & 33.8 & \cellcolor{skyblue} $52.3$ \\
+         GPT-4o$^\clubsuit$ & $\bf 75.9$ & $\bf 82.8$ & $\bf 92.9$ & \cellcolor{skyblue} $\bf 84.3$ & $\bf 70.1$ & $\bf 50.6$ & $\bf 36.2$ & \cellcolor{skyblue} $\bf 54.3$ \\
+         Gemini Ultra$^\clubsuit$ & $48.3$ & $69.0$ & $73.8$ & \cellcolor{skyblue} $65.2$ & 53.9 & 45.2 & 31.2 & \cellcolor{skyblue} $47.7$ \\
+         Claude 3 Opus$^\clubsuit$ & $13.8$ & $6.90$ & $7.10$ & \cellcolor{skyblue} $10.1$ & 45.9 & 32.6 & 26.8 & \cellcolor{skyblue} $38.3$ \\
+    \bottomrule
+    \end{tabular}%
+    }
+    \label{exp:safety_result_number_5}
+\end{table}

evals/mjbench/latex_reults/scale_study.tex

@@ -0,0 +1,63 @@
+\begin{table}[t]
+    \centering
+    \small
+    \caption{Performance comparison of multimodal judges w.r.t. different ranges of numerical scale and likert range. The results are evaluated on alignment perspective, where we consider four numerical ranges, i.e. [0, 1], [0, 5], [0, 10], [0, 100]. The best performance across all models is bolded.}
+    \resizebox{0.7\linewidth}{!}{%
+    \begin{tabular}{l|cccc|cc}
+    \toprule
+         & \multicolumn{4}{c|}{\bf Numerical} & \multicolumn{2}{c}{\bf Likert} \\
+         & [0, 1] & [0, 5] & [0, 10] & [0, 100]  &  5-likert & 10-likert     \\
+         \midrule
+         LLaVA-1.5-7b$^\heartsuit$           & $15.0$  & $26.7$ & $22.0$ & $18.3$ & $ 5.3$ & $10.3$  \\
+         LLaVA-1.5-13b$^\heartsuit$          & $ 9.7$  & $12.0$ & $10.3$ & $20.5$ & $ 2.6$ & $ 6.8$ \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$  & $20.8$  & $27.1$ & $31.3$ & $29.3$ & $36.0$ & $38.6$ \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$  & $18.3$  & $26.7$ & $29.1$ & $17.2$ & $28.7$ & $17.2$  \\
+         Instructblip-7b$^\heartsuit$        & $15.0$  & $20.9$ & $17.1$ & $17.6$ & $11.9$ & $16.8$ \\
+         MiniGPT4-v2$^\heartsuit$            & $20.4$  & $28.9$ & $32.8$ & $20.9$ & $16.0$ & $28.7$ \\
+         Prometheus-Vision-7b$^\heartsuit$   & $3.8 $  & $16.7$ & $18.4$ & $15.7$ & $28.7$ & $31.3$ \\
+         Prometheus-Vision-13b$^\heartsuit$  & $19.7$  & $11.5$ & $11.8$ & $11.2$ & $11.0$ & $6.9$  \\
+         \midrule
+         Qwen-VL-Chat$^\spadesuit$           & $26.7$  & $34.6$  & $31.1$ & $26.9$ & $55.5$ & $30.6$  \\
+         Internvl-chat-v1-5$^\spadesuit$     & $33.0$  & $27.6$  & $75.8$ & $35.3$ & $73.3$ & $18.9$  \\
+         Idefics2-8b$^\spadesuit$            & $14.6$  & $16.6$  & $32.6$ & $32.6$ & $41.2$ & $25.6$  \\
+         \midrule
+         GPT-4-vision$^\clubsuit$            & $63.2$  & $61.2$  & $66.1$ & \bf 67.2 & $\textbf{60.2}$ & $\textbf{63.0}$  \\
+         GPT-4o$^\clubsuit$                  & \bf 63.9  & $61.3$  & $61.5$ & $62.8$ & $56.3$ & $60.3$  \\
+         Gemini Ultra$^\clubsuit$            & $59.3$  & $\textbf{67.3}$  & \bf 67.2 & $60.1$ & $51.4$ & $57.8$  \\
+         Claude 3 Opus$^\clubsuit$           & $60.7$  & $45.5$  & $57.1$ & $49.4$ & $56.1$ & $62.4$  \\
+         \midrule
+         \cellcolor{skyblue} Overall & \cellcolor{skyblue}30.3  & \cellcolor{skyblue}32.3 & \cellcolor{skyblue} 37.6 & \cellcolor{skyblue}32.33 & \cellcolor{skyblue}35.6 & \cellcolor{skyblue}31.7 \\
+    \bottomrule
+    \end{tabular}
+    \label{exp:scale_study}
+    }
+    \vspace{-1em}
+\end{table}
+
+% \begin{table}[t]
+%     \centering
+%     \caption{Performance comparison of these multimodal judges w.r.t. different ranges of numerical scale. The results are evaluated on alignment perspective, where we consider four numerical ranges, i.e. [0, 1], [0, 5], [0, 10], and [0, 100]. The best performance across all models is bolded.}
+%     \resizebox{0.7\linewidth}{!}{%
+%     \begin{tabular}{c|cccccc}
+%     \toprule
+%          & [0, 1] & [0, 5] & [0, 10] & [0, 100] & \cellcolor{skyblue}Avg       \\
+%          \midrule
+%          LLaVA-1.5-7b$^\heartsuit$ & - &  - & - & - & \cellcolor{skyblue} \\
+%          LLaVA-1.5-13b$^\heartsuit$ &  - & - & - & - & \cellcolor{skyblue} \\
+%          LLaVA-NeXT-mistral-7b$^\heartsuit$ &  - & - & - & - & \cellcolor{skyblue}  \\
+%          LLaVA-NeXT-vicuna-13b$^\heartsuit$ &  - & - & - & - & \cellcolor{skyblue} \\
+%          Instructblip-7b$^\heartsuit$ & - &  - & - & - & \cellcolor{skyblue} -  \\
+%          MiniGPT4-v2$^\heartsuit$ & - &  - & - & - & \cellcolor{skyblue} -  \\
+%          Qwen-VL-Chat$^\spadesuit$ &  - & - & - & - & \cellcolor{skyblue} -  \\
+%          Internvl-chat-v1-5$^\spadesuit$ &  - & - & - & - & \cellcolor{skyblue} - \\
+%          Idefics2-8b$^\spadesuit$ & - & - & - & - & \cellcolor{skyblue} - \\
+%          Prometheus-Vision-13b$^\spadesuit$ & -  & - & - & - & \cellcolor{skyblue} - \\
+%          \midrule
+%          GPT-4-vision$^\clubsuit$ & -  & - & - & - & \cellcolor{skyblue} - \\
+%          GPT-4o$^\clubsuit$ & - & -  & - & - & \cellcolor{skyblue}  - \\
+%          Gemini Ultra$^\clubsuit$ & -  & - & - & - & \cellcolor{skyblue} - \\
+%          Claude 3 Opus$^\clubsuit$ & - & - & - & - & \cellcolor{skyblue} - \\
+%     \bottomrule
+%     \end{tabular}}
+%     \label{exp:scale_study}
+% \end{table}

evals/mjbench/latex_reults/summary.tex

@@ -0,0 +1,69 @@
+\begin{table}[h!]
+    \centering
+    \caption{Summary of the dataset proposed in \algname.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{lllrl}
+        \toprule
+        \textbf{Category} & \textbf{Scenario} & \textbf{Subset} & \textbf{N} & \textbf{Description} \\ 
+        \midrule
+        \multirow{5}{*}{\textbf{Alignment}} 
+        & \multirow{1}{*}{Object}      & -  & 250 & Ensures the correct objects are present in the image as specified by the text \\ 
+        \cmidrule{2-5}
+        & \multirow{1}{*}{Attributes}  & -  & 229 & Verifies correct association of attributes such as color, shape, size, and texture \\ 
+        \cmidrule{2-5}
+        & \multirow{1}{*}{Actions}     & -  & 115 & Ensures actions specified in the text are accurately depicted in the image \\ 
+        \cmidrule{2-5}
+        & \multirow{1}{*}{Counting}    & -  & 55 & Verifies the correct number of objects as specified by the text \\ 
+        \cmidrule{2-5}
+        & \multirow{1}{*}{Spatial}     & -  & 75 & Ensures correct spatial relationships and positions of objects in the image \\ 
+        
+        \midrule
+        \multirow{8}{*}{\textbf{Safety}} 
+        & \multirow{3}{*}{Toxicity} & Crime & 29 & Evaluates the presence of crime-related content in images \\ 
+        \cmidrule{3-5}
+        & & Shocking & 31 & Evaluates the presence of shocking or disturbing content in images \\ 
+        \cmidrule{3-5}
+        & & Disgust & 42 & Evaluates the presence of disgusting or offensive content in images \\ 
+        \cmidrule{2-5}
+        & \multirow{4}{*}{NSFW} & Evident & 197 & Images with clear and obvious NSFW content \\ 
+        \cmidrule{3-5}
+        & & Evasive & 177 & Images with attempts to subtly include NSFW content \\ 
+        \cmidrule{3-5}
+        & & Subtle & 98 & Images with low-level, hard-to-detect NSFW content \\ 
+        
+        \midrule
+        \multirow{8}{*}{\textbf{Quality}} 
+        & \multirow{3}{*}{Distortion} & Human Face & 169 & Prefers images without distortions in human faces \\ 
+        \cmidrule{3-5}
+        & & Human Limbs & 152 & Prefers images without distortions in human limbs \\ 
+        \cmidrule{3-5}
+        & & Object & 100 & Prefers images without distortions in objects \\ 
+        \cmidrule{2-5}
+        & \multirow{2}{*}{Blurry} & Defocused blur & 350 & Evaluates resistance to defocused blur in images \\ 
+        \cmidrule{3-5}
+        &  & Motion blur & 350 & Evaluates resistance to motion blur in images \\ 
+        
+        \midrule
+        \multirow{10}{*}{\textbf{Bias}} 
+        & \multirow{5}{*}{Occupation} & Age & 80 & Evaluates bias across different age groups (young, adult, old) \\ 
+        \cmidrule{3-5}
+        & & Gender & 80 & Evaluates bias across different genders (male, female, non-binary) \\ 
+        \cmidrule{3-5}
+        & & Race & 80 & Evaluates bias across different races (Asian, Black, Latino, Middle Eastern, Indian, White) \\ 
+        \cmidrule{3-5}
+        & & Nationality & 60 & Evaluates bias across different nationalities \\ 
+        \cmidrule{3-5}
+        & & Nationality (continued) & 60 & (American, Mexican, European, Spanish, British, Russian, Chinese, Japanese, Korean) \\ 
+        \cmidrule{3-5}
+        & & Religion & 60 & Evaluates bias across different religions (Christian, Muslim, Jewish, Hindu) \\ 
+        \cmidrule{2-5}
+        & \multirow{3}{*}{Education} & Gender & 60 & Evaluates bias in educational contexts across different genders \\ 
+        \cmidrule{3-5}
+        & & Race & 60 & Evaluates bias in educational contexts across different races \\ 
+        \cmidrule{3-5}
+        & & Nationality & 60 & Evaluates bias in educational contexts across different nationalities \\ 
+        \bottomrule
+    \end{tabular}
+    }
+    \label{tab:dataset_detail}
+\end{table}

evals/mjbench/latex_reults/temp_table.tex

@@ -0,0 +1,40 @@
+\begin{table}[t]
+    \centering
+    \caption{Main result.}
+    \resizebox{1.0\linewidth}{!}{%
+    \begin{tabular}{c|cc|cc|cc|ccc}
+    \toprule
+         & \multicolumn{2}{c}{\bf Alignment} & \multicolumn{2}{c}{\bf Safety} & \multicolumn{2}{c}{\bf Artifact} & \multicolumn{3}{c}{\bf Bias} \\
+         & Avg w. tie & Avg w.o. Tie & Avg w. tie & Avg w.o. Tie & Avg w. tie & Avg w.o. Tie & ACC & NDS & GES \\
+    \midrule
+         CLIP-v1$^\diamondsuit$ & $44.0$ & $60.7$ & $13.1$ & $25.7$ & $41.9$ & $82.7$ & 57.4 & 76.3 & 86.9 \\
+         BLIP-v2$^\diamondsuit$ & $21.5$ & $34.1$ & $44.3$ & $75.3$ & $7.8$ & $24.4$ & 68.7 & 83.7 & 91.3 \\
+         PickScore-v1$^\diamondsuit$ & $60.9$ & $65.9$ & $37.3$ & $41.3$ & $83.9$ & $92.2$ & 31.0 & 66.5 & 81.1 \\
+         HPS-v2.1$^\diamondsuit$ & $48.8$ & $73.6$ & $20.8$ & $35.7$ & $69.6$ & $99.1$ & 55.0 & 77.9 & 87.6 \\
+         ImageReward$^\diamondsuit$ & $51.1$ & $67.9$ & $24.9$ & $35.9$ & $63.5$ & $91.7$ & 40.9 & 73.7 & 85.3 \\
+         Aesthetics$^\diamondsuit$ & $34.8$ & $56.7$ & $31.6$ & $54.7$ & $70.8$ & $98.5$ & 61.4 & 85.7 & 92.1 \\
+    \midrule
+         LLaVA-1.5-7b$^\heartsuit$ & $22.0$ & $50.8$ & - & - & - & - & 83.7 & 70.4 & 88.7 \\
+         LLaVA-1.5-13b$^\heartsuit$ & $10.3$ & $51.9$ & - & - & - & - & 69.7 & 74.3 & 88.6 \\
+         LLaVA-NeXT-mistral-7b$^\heartsuit$ & - & - & - & - & - & - & 69.9 & 64.3 & 85.4 \\
+         LLaVA-NeXT-vicuna-13b$^\heartsuit$ & - & - & - & - & - & - & 56.3 & 64.0 & 82.7 \\
+         Instructblip-7b$^\heartsuit$ & - & - & - & - & - & - & 53.1 & 80.8 & 91.2 \\
+         MiniGPT4-v2$^\heartsuit$ & - & - & - & - & - & - & 32.6 & 67.0 & 83.3 \\
+         Prometheus-Vision-7b$^\heartsuit$ & - & - & - & - & - & - & 49.5 & 43.4 & 74.4 \\
+         Prometheus-Vision-13b$^\heartsuit$ & - & - & - & - & - & - & 66.3 & 46.3 & 76.8 \\
+         Qwen-VL-Chat$^\heartsuit$ & $31.1$ & $31.6$ & - & - & - & - & 71.9 & 62.8 & 86.2 \\
+         Internvl-chat-v1-5$^\heartsuit$ & $75.8$ & $77.6$ & - & - & - & - & 25.4 & 69.6 & 84.3 \\
+         Idefics2-8b$^\heartsuit$ & $32.6$ & $43.5$ & - & - & - & - & 42.1 & 58.7 & 79.4 \\
+     \midrule
+         Qwen-VL-Chat$^\spadesuit$ & $31.1$ & $31.6$ & - & - & - & - & 71.9 & 62.8 & 86.2 \\
+         Internvl-chat-v1-5$^\spadesuit$ & $75.8$ & $77.6$ & - & - & - & - & 25.4 & 69.6 & 84.3 \\
+         Idefics2-8b$^\spadesuit$ & $32.6$ & $43.5$ & - & - & - & - & 42.1 & 58.7 & 79.4 \\
+         GPT-4-vision$^\clubsuit$ & - & - & - & - & - & - & 79.0 & 80.4 & 93.2 \\
+         GPT-4o$^\clubsuit$ & - & - & - & - & - & - & 65.8 & 82.5 & 92.8 \\
+         Gemini Ultra$^\clubsuit$ & - & - & - & - & - & - & 55.6 & 75.3 & 88.6 \\
+         Claude 3 Opus$^\clubsuit$ & - & - & - & - & - & - & 57.7 & 65.6 & 85.0 \\
+    \bottomrule
+    \end{tabular}%
+    }
+    % \label{exp:main_result}
+\end{table}

evals/mjbench/overall-results/AestheticsPredictor.json

@@ -0,0 +1,12 @@
+[
+    {
+        "Model": "AestheticsPredictor",
+        "Model Type": "Score Model",
+        "Input Type": "Single Image",
+        "Organization": "LAION",
+        "Alignment": 32.4,
+        "Safety": 27.0,
+        "Quality": 69.6,
+        "Bias": 61.4
+    }
+]

evals/mjbench/overall-results/BLIP-v2.json

@@ -0,0 +1,12 @@
+[
+    {
+        "Model": "BLIP-v2",
+        "Model Type": "Score Model",
+        "Input Type": "Single Image",
+        "Organization": "Salesforce",
+        "Alignment": 17.3,
+        "Safety": 44.0,
+        "Quality": 7.5,
+        "Bias": 68.7
+    }
+]

evals/mjbench/overall-results/CLIP-v2.json

@@ -0,0 +1,12 @@
+[
+    {
+        "Model": "CLIP-v2",
+        "Model Type": "Score Model",
+        "Input Type": "Single Image",
+        "Organization": "LAION",
+        "Alignment": 38.1,
+        "Safety": 12.7,
+        "Quality": 34.4,
+        "Bias": 57.4
+    }
+]

evals/mjbench/overall-results/Claude 3 Opus.json

@@ -0,0 +1,12 @@
+[
+    {
+        "Model": "Claude 3 Opus",
+        "Model Type": "Closesource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "Anthropic",
+        "Alignment": 57.1,
+        "Safety": 13.4,
+        "Quality": 11.9,
+        "Bias": 57.7
+    }
+]

evals/mjbench/overall-results/GPT-4-vision.json

@@ -0,0 +1,12 @@
+[
+    {
+        "Model": "GPT-4-vision",
+        "Model Type": "Closesource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "OpenAI",
+        "Alignment": 66.1,
+        "Safety": 26.5,
+        "Quality": 90.4,
+        "Bias": 79.0
+    }
+]

evals/mjbench/overall-results/GPT-4o.json

@@ -0,0 +1,12 @@
+[
+    {
+        "Model": "GPT-4o",
+        "Model Type": "Closesource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "OpenAI",
+        "Alignment": 61.5,
+        "Safety": 35.3,
+        "Quality": 97.6,
+        "Bias": 65.8
+    }
+]

evals/mjbench/overall-results/Gemini Ultra.json

@@ -0,0 +1,12 @@
+[
+    {
+        "Model": "Gemini Ultra",
+        "Model Type": "Closesource VLM",
+        "Input Type": "Multi Image",
+        "Organization": "Google",
+        "Alignment": 67.2,
+        "Safety": 13.1,
+        "Quality": 55.7,
+        "Bias": 55.6
+    }
+]