Add local dev

.env.example

@@ -1,4 +0,0 @@
-AUTOTRAIN_USERNAME=autoevaluator # The bot that authors evaluation jobs
-HF_TOKEN=hf_xxx # An API token of the `autoevaluator` user
-AUTOTRAIN_BACKEND_API=https://api-staging.autotrain.huggingface.co # The AutoTrain backend to send jobs to. Use https://api.autotrain.huggingface.co for prod
-DATASETS_PREVIEW_API=https://datasets-server.huggingface.co # The API to grab dataset information from

.env.template

@@ -0,0 +1,4 @@
+AUTOTRAIN_USERNAME=autoevaluator                                    # The bot or user that authors evaluation jobs
+HF_TOKEN=hf_xxx                                                     # An API token of the `autoevaluator` user
+AUTOTRAIN_BACKEND_API=https://api-staging.autotrain.huggingface.co  # The AutoTrain backend to send jobs to. Use https://api.autotrain.huggingface.co for prod or http://localhost:8000 for local development
+DATASETS_PREVIEW_API=https://datasets-server.huggingface.co         # The API to grab dataset information from

README.md

@@ -54,4 +54,10 @@ Models are evaluated by AutoTrain, with the payload sent to the `AUTOTRAIN_BACKE
 
 ```
 AUTOTRAIN_BACKEND_API=https://api.autotrain.huggingface.co
+```
+
+To evaluate models with a _local_ instance of AutoTrain, change the environment to:
+
+```
+AUTOTRAIN_BACKEND_API=http://localhost:8000
 ```

app.py

@@ -1,4 +1,5 @@
 import os
+import time
 from pathlib import Path
 
 import pandas as pd
@@ -515,47 +516,72 @@ with st.form(key="form"):
                             token=HF_TOKEN,
                             domain=AUTOTRAIN_BACKEND_API,
                         ).json()
-                        print(f"INFO -- AutoTrain job response: {train_json_resp}")
-                        if train_json_resp["success"]:
-                            train_eval_index = {
-                                "train-eval-index": [
-                                    {
-                                        "config": selected_config,
-                                        "task": AUTOTRAIN_TASK_TO_HUB_TASK[selected_task],
-                                        "task_id": selected_task,
-                                        "splits": {"eval_split": selected_split},
-                                        "col_mapping": col_mapping,
-                                    }
-                                ]
-                            }
-                            selected_metadata = yaml.dump(train_eval_index, sort_keys=False)
-                            dataset_card_url = get_dataset_card_url(selected_dataset)
-                            st.success("✅ Successfully submitted evaluation job!")
-                            st.markdown(
-                                f"""
-                            Evaluation can take up to 1 hour to complete, so grab a ☕️ or 🍵 while you wait:
-
-                            * 🔔 A [Hub pull request](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) with the evaluation results will be opened for each model you selected. Check your email for notifications.
-                            * 📊 Click [here](https://hf.co/spaces/autoevaluate/leaderboards?dataset={selected_dataset}) to view the results from your submission once the Hub pull request is merged.
-                            * 🥱 Tired of configuring evaluations? Add the following metadata to the [dataset card]({dataset_card_url}) to enable 1-click evaluations:
-                            """  # noqa
-                            )
-                            st.markdown(
-                                f"""
-                            ```yaml
-                            {selected_metadata}
-                            """
-                            )
-                            print("INFO -- Pushing evaluation job logs to the Hub")
-                            evaluation_log = {}
-                            evaluation_log["project_id"] = project_json_resp["id"]
-                            evaluation_log["is_evaluated"] = False
-                            evaluation_log["payload"] = project_payload
-                            evaluation_log["project_creation_response"] = project_json_resp
-                            evaluation_log["dataset_creation_response"] = data_json_resp
-                            evaluation_log["autotrain_job_response"] = train_json_resp
-                            commit_evaluation_log(evaluation_log, hf_access_token=HF_TOKEN)
+                        # For local development we process and approve projects on-the-fly
+                        if "localhost" in AUTOTRAIN_BACKEND_API:
+                            with st.spinner("⏳ Waiting for data processing to complete ..."):
+                                is_data_processing_success = False
+                                while is_data_processing_success is not True:
+                                    project_status = http_get(
+                                        path=f"/projects/{project_json_resp['id']}",
+                                        token=HF_TOKEN,
+                                        domain=AUTOTRAIN_BACKEND_API,
+                                    ).json()
+                                    if project_status["status"] == 3:
+                                        is_data_processing_success = True
+                                    time.sleep(10)
+
+                            # Approve training job
+                            train_job_resp = http_post(
+                                path=f"/projects/{project_json_resp['id']}/start_training",
+                                token=HF_TOKEN,
+                                domain=AUTOTRAIN_BACKEND_API,
+                            ).json()
+                            st.success("✅  Data processing and project approval complete - go forth and evaluate!")
                         else:
-                            st.error("🙈 Oh no, there was an error submitting your evaluation job!")
+                            # Prod/staging submissions are evaluated in a cron job via the run_evaluation_jobs.py script
+                            print(f"INFO -- AutoTrain job response: {train_json_resp}")
+                            if train_json_resp["success"]:
+                                train_eval_index = {
+                                    "train-eval-index": [
+                                        {
+                                            "config": selected_config,
+                                            "task": AUTOTRAIN_TASK_TO_HUB_TASK[selected_task],
+                                            "task_id": selected_task,
+                                            "splits": {"eval_split": selected_split},
+                                            "col_mapping": col_mapping,
+                                        }
+                                    ]
+                                }
+                                selected_metadata = yaml.dump(train_eval_index, sort_keys=False)
+                                dataset_card_url = get_dataset_card_url(selected_dataset)
+                                st.success("✅ Successfully submitted evaluation job!")
+                                st.markdown(
+                                    f"""
+                                Evaluation can take up to 1 hour to complete, so grab a ☕️ or 🍵 while you wait:
+
+                                * 🔔 A [Hub pull request](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) with the evaluation results will be opened for each model you selected. Check your email for notifications.
+                                * 📊 Click [here](https://hf.co/spaces/autoevaluate/leaderboards?dataset={selected_dataset}) to view the results from your submission once the Hub pull request is merged.
+                                * 🥱 Tired of configuring evaluations? Add the following metadata to the [dataset card]({dataset_card_url}) to enable 1-click evaluations:
+                                """  # noqa
+                                )
+                                st.markdown(
+                                    f"""
+                                ```yaml
+                                {selected_metadata}
+                                """
+                                )
+                                print("INFO -- Pushing evaluation job logs to the Hub")
+                                evaluation_log = {}
+                                evaluation_log["project_id"] = project_json_resp["id"]
+                                evaluation_log["autotrain_env"] = (
+                                    "staging" if "staging" in AUTOTRAIN_BACKEND_API else "prod"
+                                )
+                                evaluation_log["payload"] = project_payload
+                                evaluation_log["project_creation_response"] = project_json_resp
+                                evaluation_log["dataset_creation_response"] = data_json_resp
+                                evaluation_log["autotrain_job_response"] = train_json_resp
+                                commit_evaluation_log(evaluation_log, hf_access_token=HF_TOKEN)
+                            else:
+                                st.error("🙈 Oh no, there was an error submitting your evaluation job!")
             else:
                 st.warning("⚠️ No models left to evaluate! Please select other models and try again.")

run_evaluation_jobs.py

@@ -16,28 +16,39 @@ AUTOTRAIN_TOKEN = os.getenv("AUTOTRAIN_TOKEN")
 AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
 AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
 
+if "staging" in AUTOTRAIN_BACKEND_API:
+    AUTOTRAIN_ENV = "staging"
+else:
+    AUTOTRAIN_ENV = "prod"
+
 
 def main():
     logs_df = load_dataset("autoevaluate/evaluation-job-logs", use_auth_token=True, split="train").to_pandas()
-    evaluated_projects_ds = load_dataset("autoevaluate/evaluated-project-ids", use_auth_token=True, split="train")
-    projects_df = logs_df.copy()[(~logs_df["project_id"].isnull()) & (logs_df["is_evaluated"] == False)]
+    # Filter out
+    #   - legacy AutoTrain submissions prior to project approvals was implemented
+    #   - submissions for appropriate AutoTrain environment (staging vs prod)
+    projects_df = logs_df.copy()[
+        (~logs_df["project_id"].isnull()) & (logs_df.query(f"autotrain_env == '{AUTOTRAIN_ENV}'"))
+    ]
     projects_to_approve = projects_df["project_id"].astype(int).tolist()
 
     for project_id in projects_to_approve:
-        project_status = http_get(
-            path=f"/projects/{project_id}",
-            token=HF_TOKEN,
-            domain=AUTOTRAIN_BACKEND_API,
-        ).json()
-        if project_status["status"] == 3:
-            train_job_resp = http_post(
-                path=f"/projects/{project_id}/start_training",
+        try:
+            project_info = http_get(
+                path=f"/projects/{project_id}",
                 token=HF_TOKEN,
                 domain=AUTOTRAIN_BACKEND_API,
             ).json()
-            print(f"🏃‍♂️ Project {project_id} approval response: {train_job_resp}")
-            # if train_job_resp["approved"] == True:
-            #     # Update evaluation status
+            if project_info["status"] == 3 and project_info["training_status"] == "not_started":
+                train_job_resp = http_post(
+                    path=f"/projects/{project_id}/start_training",
+                    token=HF_TOKEN,
+                    domain=AUTOTRAIN_BACKEND_API,
+                ).json()
+                print(f"🏃‍♂️ Project {project_id} approval response: {train_job_resp}")
+        except:
+            print(f"There was a problem obtaining the project info for project ID {project_id}")
+            pass
 
 
 if __name__ == "__main__":