Update app.py

app.py

@@ -51,6 +51,7 @@ import plotly.graph_objects as go
 
 selected_indices = []
 selected_values = {}
+selected_dropdown_weight = 'All'
 
 # Start ephemeral Spaces on PRs (see config in README.md)
 #enable_space_ci()
@@ -59,52 +60,160 @@ precision_to_dtype = {
     "2bit": ["int2"],
     "3bit": ["int3"],
     "4bit": ["int4", "nf4", "fp4"],
-    "?": ["?"]
+    "8bit": ["int8"],
+    "16bit": ['float16', 'bfloat16'],
+    "32bit": ["float32"],
+    "?": ["?"],
 }
 
-current_weightDtype = ["All", "int2", "int3", "int4", "nf4", "fp4", "?"]
+dtype_to_precision = {
+    "int2": ["2bit"],
+    "int3": ["3bit"],
+    "int4": ["4bit"],
+    "nf4": ["4bit"],
+    "fp4": ["4bit"],
+    "int8": ["8bit"],
+    "float16": ["16bit"],
+    "bfloat16": ["16bit"],
+    "float32": ["32bit"],
+    "?": ["?"],
+}
 
-# Global variable to store the selected dtypes
-selected_dtypes = ["All"]
-init_select = False
+current_weightDtype = ["int2", "int3", "int4", "nf4", "fp4", "?"]
+current_computeDtype = ['int8', 'bfloat16', 'float16', 'float32']
+current_quant = [t.to_str() for t in QuantType if t != QuantType.QuantType_None]
+current_precision = ['2bit', '3bit', '4bit', '8bit', '?']
 
-def quant_update_Weight_Dtype(selected_precisions):
-    global current_weightDtype
-    if '✖ None' in selected_precisions:
-        if not any(dtype in ['float16', 'bfloat16', 'float32'] for dtype in current_weightDtype):
-            current_weightDtype += ['float16', 'bfloat16', 'float32']
-    else:
-        if any(dtype in ['float16', 'bfloat16', 'float32'] for dtype in current_weightDtype):
-            current_weightDtype = [dtype for dtype in current_weightDtype if dtype not in ['float16', 'bfloat16', 'float32']]
-    return gr.Dropdown(choices=current_weightDtype, value="All")
 
+def display_sort(key):
+    order = {"All": 0, "?": 1, "int2": 2, "int3": 3, "int4": 4, "fp4": 5, "nf4": 6, "float16": 7, "bfloat16": 8, "float32": 9}
+    return order.get(key, float('inf'))
 
-def update_Weight_Dtype(selected_precisions):
-    global selected_dtypes
+def comp_display_sort(key):
+    order = {"All": 0, "?": 1, "int8": 2, "float16": 3, "bfloat16": 4, "float32": 5}
+    return order.get(key, float('inf'))
+
+def update_quantization_types(selected_quant):
     global current_weightDtype
-    global init_select
-    init_select = True
+    global current_computeDtype
+    global current_quant
+    global current_precision
+
+    if set(current_quant) == set(selected_quant):
+        return [
+            gr.Dropdown(choices=current_weightDtype, value=selected_dropdown_weight),
+            gr.Dropdown(choices=current_computeDtype, value="All"),
+            gr.CheckboxGroup(value=current_precision),
+        ]
+     
+    print('update_quantization_types', selected_quant, current_quant)
+    if any(value != '✖ None' for value in selected_quant):
+        selected_weight = ['All', '?', 'int2', 'int3', 'int4', 'nf4', 'fp4', 'int8']
+        selected_compute = ['All', '?', 'int8', 'float16', 'bfloat16', 'float32']
+        selected_precision = ["2bit", "3bit", "4bit", "8bit", "?"]
+    
+    current_weightDtype = selected_weight
+    current_computeDtype = selected_compute
+    current_quant = selected_quant  
+    current_precision = selected_precision
+
+    return [
+        gr.Dropdown(choices=selected_weight, value="All"),
+        gr.Dropdown(choices=selected_compute, value="All"),
+        gr.CheckboxGroup(value=selected_precision),
+    ]
 
-    if not selected_precisions:  # If no precision is selected, return "All"
-        selected_dtypes = ["All"]
-        return gr.Dropdown(choices=["All"], value="All")
+def update_Weight_Precision(temp_precisions):
+    global current_weightDtype
+    global current_computeDtype
+    global current_quant
+    global current_precision
+    global selected_dropdown_weight
 
-    selected_dtypes_set = set()
-    for precision in selected_precisions:
-        if precision in precision_to_dtype:
-            selected_dtypes_set.update(precision_to_dtype[precision])
+    
+    if set(current_precision) == set(temp_precisions):
+        return [
+            gr.Dropdown(choices=current_weightDtype, value=selected_dropdown_weight),
+            gr.Dropdown(choices=current_computeDtype, value="All"),
+            gr.CheckboxGroup(value=current_precision),
+            gr.CheckboxGroup(value=current_quant),
+        ]   # No update needed
+    if len(temp_precisions) > 1:
+        selected_dropdown_weight = 'All'
+    precisions = [precision for precision in temp_precisions if precision not in current_precision]
+
+    selected_weight = []
+    selected_compute = ['All', '?', 'int8', 'float16', 'bfloat16', 'float32'] 
+    selected_quant = [t.to_str() for t in QuantType if t != QuantType.QuantType_None]
+
+    if set(temp_precisions) == {"16bit", "32bit"}:
+        selected_precisions = temp_precisions.copy()
+    elif (temp_precisions == ["16bit"] or temp_precisions == ["32bit"]) and current_precision == ["16bit", "32bit"]:
+        selected_precisions = temp_precisions
+    else:
+        if "16bit" in precisions or "32bit" in precisions:
+            selected_precisions = precisions
+        else:
+            filtered_precisions = [p for p in temp_precisions if p not in ["16bit", "32bit"]]
+            selected_precisions = filtered_precisions.copy()
+
+    current_precision = selected_precisions
 
+    # Map selected_precisions to corresponding weights
+    for precision in current_precision:
+        if precision in precision_to_dtype:
+            selected_weight.extend(precision_to_dtype[precision])
+    
+    # Special rules for 16bit and 32bit
+    if "16bit" in current_precision:
+        selected_weight = [option for option in selected_weight if option in ["All", "?", "float16", "bfloat16"]]
+        if "int8" in selected_compute:
+            selected_compute.remove("int8")
+                    
+    if "32bit" in current_precision:
+        selected_weight = [option for option in selected_weight if option in ["All", "?", "float32"]]
+        if "int8" in selected_compute:
+            selected_compute.remove("int8")
+
+    if "16bit" in current_precision or "32bit" in current_precision:
+        selected_quant = ['✖ None']
+    if "16bit" in current_precision and "32bit" in current_precision:
+        selected_weight = ["All", "?", "float16", "bfloat16", "float32"]        
+    # Ensure "All" and "?" options are included
+    selected_weight = ["All", "?"] + [opt for opt in selected_weight if opt not in ["All", "?"]]
+    selected_compute = ["All", "?"] + [opt for opt in selected_compute if opt not in ["All", "?"]]
     
-    # Convert set to sorted list to maintain order
-    selected_dtypes = sorted(selected_dtypes_set)
-    if any(dtype in ['float16', 'bfloat16', 'float32'] for dtype in current_weightDtype) and not any(dtype in ['float16', 'bfloat16', 'float32'] for dtype in selected_dtypes):
-        selected_dtypes += ['float16', 'bfloat16', 'float32']
-    # Add "All" to the beginning of the list for display purposes
-    display_choices = ["All"] + selected_dtypes
+    # Remove duplicates
+    selected_weight = list(set(selected_weight))
+    selected_compute = list(set(selected_compute))
+    
+    # Update global variables
+    current_weightDtype = selected_weight
+    current_computeDtype = selected_compute
+    current_quant = selected_quant          
+    
+    # Return updated components
+    return [
+        gr.Dropdown(choices=selected_weight, value=selected_dropdown_weight),
+        gr.Dropdown(choices=selected_compute, value="All"),
+        gr.CheckboxGroup(value=selected_precisions),
+        gr.CheckboxGroup(value=selected_quant),
+    ]
 
+def update_Weight_Dtype(weight):    
+    global selected_dropdown_weight
+    print('update_Weight_Dtype', weight)
+    # Initialize selected_precisions
+    if weight == selected_dropdown_weight or weight == 'All':
+        return current_precision
+    else:
+        selected_precisions = []        
+        selected_precisions.extend(dtype_to_precision[weight])
+    selected_dropdown_weight =  weight       
+    print('selected_precisions', selected_precisions)
+    # Return updated components
+    return selected_precisions
 
-    current_weightDtype = display_choices
-    return gr.Dropdown(choices=display_choices, value="All")
 
 
 
@@ -177,18 +286,15 @@ def update_table(
 ):
     global init_select
     global current_weightDtype
+    global current_computeDtype
 
-
-    if selected_dtypes == ['All']:
+    if weight_dtype == ['All'] or weight_dtype == 'All':
         weight_dtype = current_weightDtype
-    elif weight_dtype == ['All'] or weight_dtype == 'All' or init_select:
-        weight_dtype = selected_dtypes
-        init_select = False
     else:
         weight_dtype = [weight_dtype]
 
     if compute_dtype == 'All':
-        compute_dtype = ['bfloat16', 'float16', 'int8', 'float32']
+        compute_dtype = current_computeDtype
     else:
         compute_dtype = [compute_dtype]   
         
@@ -265,12 +371,20 @@ def filter_models(
         filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
 
     type_emoji = [t[0] for t in type_query]
+    if any(emoji != '✖' for emoji in type_emoji):
+        type_emoji = [emoji for emoji in type_emoji if emoji != '✖']
+    else:
+        type_emoji = ['✖']
+
     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-    
+
     filtered_df = filtered_df.loc[df[AutoEvalColumn.weight_dtype.name].isin(weight_dtype)]
+
     filtered_df = filtered_df.loc[df[AutoEvalColumn.compute_dtype.name].isin(compute_dtype)]
+
     filtered_df = filtered_df.loc[df[AutoEvalColumn.double_quant.name].isin(double_quant)]
+
     filtered_df = filtered_df.loc[df[AutoEvalColumn.group_size.name].isin(group_dtype)]
 
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
@@ -310,7 +424,6 @@ def select(df, data: gr.SelectData):
             text_content = match.group(1)
             selected_values[text_content] = value
 
-    print('selected_values', selected_values, selected_indices)
     return gr.CheckboxGroup(list(selected_values.keys()), value=list(selected_values.keys()))
 
 def init_comparison_data():
@@ -319,7 +432,6 @@ def init_comparison_data():
 
 def generate_spider_chart(df, selected_keys):
     global selected_values
-    print('generate_spider_chart', selected_values, selected_keys)
     current_selected_values = [selected_values[key] for key in selected_keys if key in selected_values]
     selected_rows = df[df.iloc[:, 1].isin(current_selected_values)]
 
@@ -344,7 +456,7 @@ def generate_spider_chart(df, selected_keys):
 
 leaderboard_df = filter_models(
     df=leaderboard_df, 
-    type_query=[t.to_str(" : ") for t in QuantType], 
+    type_query=[t.to_str(" : ") for t in QuantType if t != QuantType.QuantType_None], 
     size_query=list(NUMERIC_INTERVALS.keys()), 
     params_query=list(NUMERIC_MODELSIZE.keys()),
     precision_query=[i.value.name for i in Precision],
@@ -407,7 +519,7 @@ with demo:
                     #with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
                         label="Quantization types",
-                        choices=[t.to_str() for t in QuantType],
+                        choices=[t.to_str() for t in QuantType if t != QuantType.QuantType_None],
                         value=[t.to_str() for t in QuantType if t != QuantType.QuantType_None],
                         interactive=True,
                         elem_id="filter-columns-type",  
@@ -415,7 +527,7 @@ with demo:
                     filter_columns_precision = gr.CheckboxGroup(
                         label="Weight precision",
                         choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
+                        value=[i.value.name for i in Precision  if ( i.value.name != '16bit' and i.value.name != '32bit')],
                         interactive=True,
                         elem_id="filter-columns-precision",
                     )
@@ -430,7 +542,7 @@ with demo:
 
                     with gr.Row():
                         with gr.Column():
-                            model_comparison = gr.CheckboxGroup(label="Accuracy Comparison on Selected Models", choices=list(selected_values.keys()), value=list(selected_values.keys()), interactive=True, elem_id="model_comparison")
+                            model_comparison = gr.CheckboxGroup(label="Accuracy Comparison (Selected Models from Table)", choices=list(selected_values.keys()), value=list(selected_values.keys()), interactive=True, elem_id="model_comparison")
                         with gr.Column():
                             spider_btn = gr.Button("Compare")                 
       
@@ -514,21 +626,30 @@ with demo:
             demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
             
             """
+            filter_columns_type.change(
+                update_quantization_types,
+                [filter_columns_type],
+                [filter_columns_weightDtype, filter_columns_computeDtype, filter_columns_precision]
+            )
+
             filter_columns_precision.change(
-                update_Weight_Dtype,
+                update_Weight_Precision,
                 [filter_columns_precision],
-                [filter_columns_weightDtype]
+                [filter_columns_weightDtype, filter_columns_computeDtype, filter_columns_precision, filter_columns_type]
             )
 
-            filter_columns_type.change(
-                quant_update_Weight_Dtype,
-                [filter_columns_type],
-                [filter_columns_weightDtype]
+            filter_columns_weightDtype.change(
+                update_Weight_Dtype,
+                [filter_columns_weightDtype],
+                [filter_columns_precision]
             )
-
+            # filter_columns_computeDtype.change(
+            #     Compute_Dtype_update,
+            #     [filter_columns_computeDtype, filter_columns_precision],
+            #     [filter_columns_precision, filter_columns_type]
+            # )
             
 
-
     
             for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, filter_columns_parameters, hide_models, filter_columns_computeDtype, filter_columns_weightDtype, filter_columns_doubleQuant, filter_columns_groupDtype]:
                 selector.change(
@@ -680,4 +801,4 @@ scheduler.add_job(update_dynamic_files, "interval", hours=12) # launched every 2
 scheduler.start()
 
 demo.queue(default_concurrency_limit=40).launch()
-# demo.queue(concurrency_count=40).launch()
+# demo.queue(concurrency_count=40).launch()