Merge pull request #51 from huggingface/add-nli

app.py

@@ -16,6 +16,7 @@ from utils import (
     create_autotrain_project_name,
     format_col_mapping,
     get_compatible_models,
+    get_config_metadata,
     get_dataset_card_url,
     get_key,
     get_metadata,
@@ -37,6 +38,7 @@ TASK_TO_ID = {
     "image_multi_class_classification": 18,
     "binary_classification": 1,
     "multi_class_classification": 2,
+    "natural_language_inference": 22,
     "entity_extraction": 4,
     "extractive_question_answering": 5,
     "translation": 6,
@@ -51,6 +53,7 @@ TASK_TO_DEFAULT_METRICS = {
         "recall",
         "accuracy",
     ],
+    "natural_language_inference": ["f1", "precision", "recall", "auc", "accuracy"],
     "entity_extraction": ["precision", "recall", "f1", "accuracy"],
     "extractive_question_answering": ["f1", "exact_match"],
     "translation": ["sacrebleu"],
@@ -72,7 +75,6 @@ AUTOTRAIN_TASK_TO_LANG = {
 
 
 SUPPORTED_TASKS = list(TASK_TO_ID.keys())
-UNSUPPORTED_TASKS = []
 
 # Extracted from utils.get_supported_metrics
 # Hardcoded for now due to speed / caching constraints
@@ -118,8 +120,6 @@ SUPPORTED_METRICS = [
     "jordyvl/ece",
     "lvwerra/ai4code",
     "lvwerra/amex",
-    "lvwerra/test",
-    "lvwerra/test_metric",
 ]
 
 
@@ -180,10 +180,6 @@ if metadata is None:
 
 with st.expander("Advanced configuration"):
     # Select task
-    # Hack to filter for unsupported tasks
-    # TODO(lewtun): remove this once we have SQuAD metrics support
-    if metadata is not None and metadata[0]["task_id"] in UNSUPPORTED_TASKS:
-        metadata = None
     selected_task = st.selectbox(
         "Select a task",
         SUPPORTED_TASKS,
@@ -201,6 +197,9 @@ with st.expander("Advanced configuration"):
             See the [docs](https://huggingface.co/docs/datasets/master/en/load_hub#configurations) for more details.
             """,
     )
+    # Some datasets have multiple metadata (one per config), so we grab the one associated with the selected config
+    config_metadata = get_config_metadata(selected_config, metadata)
+    print(f"INFO -- Config metadata: {config_metadata}")
 
     # Select splits
     splits_resp = http_get(
@@ -215,8 +214,8 @@ with st.expander("Advanced configuration"):
             if split["config"] == selected_config:
                 split_names.append(split["split"])
 
-        if metadata is not None:
-            eval_split = metadata[0]["splits"].get("eval_split", None)
+        if config_metadata is not None:
+            eval_split = config_metadata["splits"].get("eval_split", None)
         else:
             eval_split = None
         selected_split = st.selectbox(
@@ -260,16 +259,62 @@ with st.expander("Advanced configuration"):
             text_col = st.selectbox(
                 "This column should contain the text to be classified",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "text")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "text"))
+                if config_metadata is not None
+                else 0,
             )
             target_col = st.selectbox(
                 "This column should contain the labels associated with the text",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "target")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
+                if config_metadata is not None
+                else 0,
             )
             col_mapping[text_col] = "text"
             col_mapping[target_col] = "target"
 
+    if selected_task in ["natural_language_inference"]:
+        config_metadata = get_config_metadata(selected_config, metadata)
+        with col1:
+            st.markdown("`text1` column")
+            st.text("")
+            st.text("")
+            st.text("")
+            st.text("")
+            st.text("")
+            st.markdown("`text2` column")
+            st.text("")
+            st.text("")
+            st.text("")
+            st.text("")
+            st.text("")
+            st.markdown("`target` column")
+        with col2:
+            text1_col = st.selectbox(
+                "This column should contain the first text passage to be classified",
+                col_names,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "text1"))
+                if config_metadata is not None
+                else 0,
+            )
+            text2_col = st.selectbox(
+                "This column should contain the second text passage to be classified",
+                col_names,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "text2"))
+                if config_metadata is not None
+                else 0,
+            )
+            target_col = st.selectbox(
+                "This column should contain the labels associated with the text",
+                col_names,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
+                if config_metadata is not None
+                else 0,
+            )
+            col_mapping[text1_col] = "text1"
+            col_mapping[text2_col] = "text2"
+            col_mapping[target_col] = "target"
+
     elif selected_task == "entity_extraction":
         with col1:
             st.markdown("`tokens` column")
@@ -282,12 +327,16 @@ with st.expander("Advanced configuration"):
             tokens_col = st.selectbox(
                 "This column should contain the array of tokens to be classified",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "tokens")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "tokens"))
+                if config_metadata is not None
+                else 0,
             )
             tags_col = st.selectbox(
                 "This column should contain the labels associated with each part of the text",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "tags")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "tags"))
+                if config_metadata is not None
+                else 0,
             )
             col_mapping[tokens_col] = "tokens"
             col_mapping[tags_col] = "tags"
@@ -304,12 +353,16 @@ with st.expander("Advanced configuration"):
             text_col = st.selectbox(
                 "This column should contain the text to be translated",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "source")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "source"))
+                if config_metadata is not None
+                else 0,
             )
             target_col = st.selectbox(
                 "This column should contain the target translation",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "target")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
+                if config_metadata is not None
+                else 0,
             )
             col_mapping[text_col] = "source"
             col_mapping[target_col] = "target"
@@ -326,19 +379,23 @@ with st.expander("Advanced configuration"):
             text_col = st.selectbox(
                 "This column should contain the text to be summarized",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "text")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "text"))
+                if config_metadata is not None
+                else 0,
             )
             target_col = st.selectbox(
                 "This column should contain the target summary",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "target")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
+                if config_metadata is not None
+                else 0,
             )
             col_mapping[text_col] = "text"
             col_mapping[target_col] = "target"
 
     elif selected_task == "extractive_question_answering":
-        if metadata is not None:
-            col_mapping = metadata[0]["col_mapping"]
+        if config_metadata is not None:
+            col_mapping = config_metadata["col_mapping"]
             # Hub YAML parser converts periods to hyphens, so we remap them here
             col_mapping = format_col_mapping(col_mapping)
         with col1:
@@ -362,22 +419,24 @@ with st.expander("Advanced configuration"):
             context_col = st.selectbox(
                 "This column should contain the question's context",
                 col_names,
-                index=col_names.index(get_key(col_mapping, "context")) if metadata is not None else 0,
+                index=col_names.index(get_key(col_mapping, "context")) if config_metadata is not None else 0,
             )
             question_col = st.selectbox(
                 "This column should contain the question to be answered, given the context",
                 col_names,
-                index=col_names.index(get_key(col_mapping, "question")) if metadata is not None else 0,
+                index=col_names.index(get_key(col_mapping, "question")) if config_metadata is not None else 0,
             )
             answers_text_col = st.selectbox(
                 "This column should contain example answers to the question, extracted from the context",
                 col_names,
-                index=col_names.index(get_key(col_mapping, "answers.text")) if metadata is not None else 0,
+                index=col_names.index(get_key(col_mapping, "answers.text")) if config_metadata is not None else 0,
             )
             answers_start_col = st.selectbox(
                 "This column should contain the indices in the context of the first character of each `answers.text`",
                 col_names,
-                index=col_names.index(get_key(col_mapping, "answers.answer_start")) if metadata is not None else 0,
+                index=col_names.index(get_key(col_mapping, "answers.answer_start"))
+                if config_metadata is not None
+                else 0,
             )
             col_mapping[context_col] = "context"
             col_mapping[question_col] = "question"
@@ -395,12 +454,16 @@ with st.expander("Advanced configuration"):
             image_col = st.selectbox(
                 "This column should contain the images to be classified",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "image")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "image"))
+                if config_metadata is not None
+                else 0,
             )
             target_col = st.selectbox(
                 "This column should contain the labels associated with the images",
                 col_names,
-                index=col_names.index(get_key(metadata[0]["col_mapping"], "target")) if metadata is not None else 0,
+                index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
+                if config_metadata is not None
+                else 0,
             )
             col_mapping[image_col] = "image"
             col_mapping[target_col] = "target"

utils.py

@@ -12,6 +12,7 @@ from tqdm import tqdm
 AUTOTRAIN_TASK_TO_HUB_TASK = {
     "binary_classification": "text-classification",
     "multi_class_classification": "text-classification",
+    "natural_language_inference": "text-classification",
     "entity_extraction": "token-classification",
     "extractive_question_answering": "question-answering",
     "translation": "translation",
@@ -197,3 +198,14 @@ def create_autotrain_project_name(dataset_id: str) -> str:
     # Project names need to be unique, so we append a random string to guarantee this
     project_id = str(uuid.uuid4())[:8]
     return f"eval-project-{dataset_id_formatted}-{project_id}"
+
+
+def get_config_metadata(config: str, metadata: List[Dict] = None) -> Union[Dict, None]:
+    """Gets the dataset card metadata for the given config."""
+    if metadata is None:
+        return None
+    config_metadata = [m for m in metadata if m["config"] == config]
+    if len(config_metadata) >= 1:
+        return config_metadata[0]
+    else:
+        return None