Remove precision

app.py

@@ -21,7 +21,6 @@ from src.display.utils import (
     AutoEvalColumn,
     ModelType,
     fields,
-    Precision
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
@@ -77,8 +76,6 @@ def init_leaderboard(dataframe):
         filter_columns=[
             ColumnFilter(AutoEvalColumn.model_type.name,
                          type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name,
-                         type="checkboxgroup", label="Precision"),
             ColumnFilter(
                 AutoEvalColumn.params.name,
                 type="slider",
@@ -167,14 +164,6 @@ with demo:
                     )
 
                 with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i !=
-                                 Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
                     base_model_name_textbox = gr.Textbox(
                         label="Base model (for delta or adapter weights)")
 
@@ -186,7 +175,6 @@ with demo:
                     model_name_textbox,
                     base_model_name_textbox,
                     revision_name_textbox,
-                    precision,
                     model_type,
                 ],
                 submission_result,

src/display/utils.py

@@ -37,8 +37,6 @@ for task in Tasks:
 # Model information
 auto_eval_column_dict.append(
     ["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(
-    ["precision", ColumnContent, ColumnContent("Precision", "str", False)])
 auto_eval_column_dict.append(
     ["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(
@@ -56,7 +54,6 @@ class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
     status = ColumnContent("status", "str", True)
 
 # All the model information that we might need
@@ -86,19 +83,6 @@ class ModelType(Enum):
         return ModelType.Unknown
 
 
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 

src/leaderboard/read_evals.py

@@ -1,5 +1,5 @@
 from src.submission.check_validity import is_model_on_hub
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision
+from src.display.utils import AutoEvalColumn, ModelType, Tasks
 from src.display.formatting import make_clickable_model
 import numpy as np
 import dateutil
@@ -15,13 +15,12 @@ print("--- CONFIRMED: Running the modified version of read_evals.py ---")
 class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run.
     """
-    eval_name: str  # org_model_precision (uid)
+    eval_name: str  # org_model (uid)
     full_model: str  # org/model (path on hub)
     org: str
     model: str
     revision: str  # commit hash, "" if main
     results: dict
-    precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
     architecture: str = "Unknown"
     likes: int = 0
@@ -37,9 +36,6 @@ class EvalResult:
 
         config = data.get("config")
 
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
         # Get model and org
         org_and_model = config.get(
             "model_name", config.get("model_args", None))
@@ -48,11 +44,11 @@ class EvalResult:
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
+            result_key = f"{model}"
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
+            result_key = f"{org}"
         full_model = "/".join(org_and_model)
 
         still_on_hub, _, model_config = is_model_on_hub(
@@ -79,7 +75,6 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,
             revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
         )
@@ -87,7 +82,7 @@ class EvalResult:
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
         request_file = get_request_file_for_model(
-            requests_path, self.full_model, self.precision.value.name)
+            requests_path, self.full_model)
 
         try:
             with open(request_file, "r") as f:
@@ -97,13 +92,12 @@ class EvalResult:
             self.date = request.get("submitted_time", "")
         except Exception:
             print(
-                f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+                f"Could not find request file for {self.org}/{self.model}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
@@ -118,7 +112,7 @@ class EvalResult:
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision=None):
+def get_request_file_for_model(requests_path, model_name):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(
         requests_path,
@@ -126,7 +120,7 @@ def get_request_file_for_model(requests_path, model_name, precision=None):
     )
     request_files = glob.glob(request_files)
 
-    # Select correct request file (precision)
+    # Select correct request file
     request_file = ""
     request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:

src/submission/check_validity.py

@@ -55,15 +55,14 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
         return False, "was not found on hub!", None
 
 
-def get_model_size(model_info: ModelInfo, precision: str):
+def get_model_size(model_info: ModelInfo):
     """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
     except (AttributeError, TypeError):
         return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
 
-    size_factor = 8 if (
-        precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    size_factor = 8  # Default size factor for float32 models
     model_size = size_factor * model_size
     return model_size
 
@@ -88,7 +87,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
                     file_names.append(
-                        f"{info['model']}_{info['revision']}_{info['precision']}")
+                        f"{info['model']}_{info['revision']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -19,7 +19,6 @@ def add_new_eval(
     model: str,
     base_model: str,
     revision: str,
-    precision: str,
     model_type: str,
 ):
     global REQUESTED_MODELS
@@ -34,7 +33,6 @@ def add_new_eval(
         user_name = model.split("/")[0]
         model_path = model.split("/")[1]
 
-    precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
     if model_type is None or model_type == "":
@@ -50,7 +48,7 @@ def add_new_eval(
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
-    model_size = get_model_size(model_info=model_info, precision=precision)
+    model_size = get_model_size(model_info=model_info)
 
     # Were the model card and license filled?
     try:
@@ -69,7 +67,6 @@ def add_new_eval(
         "model": model,
         "base_model": base_model,
         "revision": revision,
-        "precision": precision,
         "status": "PENDING",
         "submitted_time": current_time,
         "model_type": model_type,
@@ -80,13 +77,13 @@ def add_new_eval(
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+    if f"{model}_{revision}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))