Upload metrics.py with huggingface_hub

metrics.py

@@ -5,9 +5,7 @@ from dataclasses import field
 from typing import Any, Dict, Generator, List, Optional
 
 import evaluate
-import nltk
 import numpy
-from editdistance import eval
 
 from .dataclass import InternalField
 from .operator import (
@@ -19,8 +17,6 @@ from .operator import (
 from .operators import CopyFields
 from .stream import MultiStream, Stream
 
-nltk.download("punkt")
-
 
 def abstract_factory():
     return {}
@@ -65,7 +61,8 @@ class GlobalMetric(SingleStreamOperator, Metric):
             try:
                 instance_score = self._compute([refs], [pred])
             except:
-                instance_score = {"score": None}
+                instance_score = {"score": None, "score_name": self.main_score}
+
                 if isinstance(self.main_score, str) and self.main_score is not None:
                     instance_score[self.main_score] = None
 
@@ -86,6 +83,7 @@ class GlobalMetric(SingleStreamOperator, Metric):
     def _compute(self, references: List[List[str]], predictions: List[str]) -> dict:
         result = self.compute(references, predictions)
         result["score"] = result[self.main_score]
+        result["score_name"] = self.main_score
         return result
 
     @abstractmethod
@@ -131,6 +129,7 @@ class InstanceMetric(SingleStreamOperator, Metric):
                     global_score[field] = mean([instance["score"]["instance"][field] for instance in instances])
                     if field == self.main_score:
                         global_score["score"] = global_score[field]
+                        global_score["score_name"] = self.main_score
 
         for instance in instances:
             yield instance
@@ -138,6 +137,7 @@ class InstanceMetric(SingleStreamOperator, Metric):
     def _compute(self, references: List[List[str]], predictions: List[str]) -> dict:
         result = self.compute(references=references, predictions=predictions)
         result["score"] = result[self.main_score]
+        result["score_name"] = self.main_score
         return result
 
     @abstractmethod
@@ -147,7 +147,6 @@ class InstanceMetric(SingleStreamOperator, Metric):
 
 class Squad(GlobalMetric):
     _metric = None
-    reduction_map = {"mean": ["f1"]}
     main_score = "f1"
     metric = "squad"
 
@@ -172,6 +171,7 @@ class SingleReferenceInstanceMetric(InstanceMetric):
     def _compute(self, references: List[str], prediction: str) -> dict:
         result = self.compute(references[0], prediction)
         result["score"] = result[self.main_score]
+        result["score_name"] = self.main_score
         return result
 
     @abstractmethod
@@ -288,6 +288,7 @@ class F1MultiLabel(GlobalMetric):
     _metric = None
     main_score = "f1_macro"
     average = None  # Report per class then aggregate by mean
+    classes_to_ignore = ["none"]
 
     def prepare(self):
         super(F1MultiLabel, self).prepare()
@@ -314,17 +315,41 @@ class F1MultiLabel(GlobalMetric):
             len(reference) == 1 for reference in references
         ), "Only a single reference per prediction is allowed in F1 metric"
         references = [reference[0] for reference in references]
-        labels = list(set([label for reference in references for label in reference]))
+        labels = [
+            l
+            for l in set([label for reference in references for label in reference])
+            if l not in self.classes_to_ignore
+        ]
+        # if no classes are left then F1 is not defined
+        # (e.g. only "none" in references)
+        if len(labels) == 0:
+            return {self.main_score: float("nan")}
+
         for label in labels:
             self.add_str_to_id(label)
         formatted_references = [self.get_one_hot_vector(reference) for reference in references]
         formatted_predictions = [self.get_one_hot_vector(prediction) for prediction in predictions]
+
+        # There is odd behavior in scikit-learn that when passing a one-hot vector with a single
+        # element, it is treated a class identifier. Therefore, we add labels=[1] to limit to only
+        # to this class.
+        if len(labels) == 1:
+            labels_param = [1]
+        else:
+            labels_param = None
+
         result = self._metric.compute(
-            predictions=formatted_predictions, references=formatted_references, average=self.average
+            predictions=formatted_predictions,
+            references=formatted_references,
+            average=self.average,
+            labels=labels_param,
         )
         if isinstance(result["f1"], numpy.ndarray):
             from statistics import mean
 
+            assert len(result["f1"]) == len(
+                labels
+            ), f'F1 result ({result["f1"]}) has more entries than labels ({labels})'
             final_result = {self.main_score: mean(result["f1"])}
             for i, label in enumerate(labels):
                 final_result["f1_" + label] = result["f1"][i]
@@ -348,24 +373,36 @@ class Rouge(HuggingfaceMetric):
     main_score = "rougeL"
     scale = 1.0
 
+    def prepare(self):
+        super().prepare()
+        import nltk
+
+        nltk.download("punkt")
+        self.sent_tokenize = nltk.sent_tokenize
+
     def compute(self, references, predictions):
-        predictions = ["\n".join(nltk.sent_tokenize(prediction.strip())) for prediction in predictions]
-        references = [["\n".join(nltk.sent_tokenize(r.strip())) for r in reference] for reference in references]
+        predictions = ["\n".join(self.sent_tokenize(prediction.strip())) for prediction in predictions]
+        references = [["\n".join(self.sent_tokenize(r.strip())) for r in reference] for reference in references]
         return super().compute(references, predictions)
 
 
-# Computes chat edit distance, ignoring repeating whitespace
+# Computes chat edit distance, ignoring whitespace
 class CharEditDistanceAccuracy(SingleReferenceInstanceMetric):
     reduction_map = {"mean": ["char_edit_dist_accuracy"]}
     main_score = "char_edit_dist_accuracy"
 
+    def prepare(self):
+        import editdistance
+
+        self.eval = editdistance.eval
+
     def compute(self, reference, prediction: str) -> dict:
-        formatted_prediction = " ".join(prediction.split())
-        formatted_reference = " ".join(reference.split())
+        formatted_prediction = "".join(prediction.split())
+        formatted_reference = "".join(reference.split())
         max_length = max(len(formatted_reference), len(formatted_prediction))
         if max_length == 0:
             return 0
-        edit_dist = eval(formatted_reference, formatted_prediction)
+        edit_dist = self.eval(formatted_reference, formatted_prediction)
         return {"char_edit_dist_accuracy": (1 - edit_dist / max_length)}