Upload metrics.py with huggingface_hub

metrics.py

@@ -7,6 +7,7 @@ from typing import Any, Dict, Generator, List, Optional
 import evaluate
 import nltk
 import numpy
+from editdistance import eval
 
 from .dataclass import InternalField
 from .operator import (
@@ -21,12 +22,12 @@ from .stream import MultiStream, Stream
 nltk.download("punkt")
 
 
-def absrtact_factory():
+def abstract_factory():
     return {}
 
 
 def abstract_field():
-    return field(default_factory=absrtact_factory)
+    return field(default_factory=abstract_factory)
 
 
 class UpdateStream(StreamInstanceOperator):
@@ -253,7 +254,7 @@ class F1(GlobalMetric):
     def compute(self, references: List[List[str]], predictions: List[str]) -> dict:
         assert all(
             len(reference) == 1 for reference in references
-        ), "One single reference per predictition are allowed in F1 metric"
+        ), "Only a single reference per prediction is allowed in F1 metric"
         self.str_to_id = {}
         self.id_to_str = {}
         formatted_references = [self.get_str_id(reference[0]) for reference in references]
@@ -287,7 +288,6 @@ class F1MultiLabel(GlobalMetric):
     _metric = None
     main_score = "f1_macro"
     average = None  # Report per class then aggregate by mean
-    seperator = ","
 
     def prepare(self):
         super(F1MultiLabel, self).prepare()
@@ -310,17 +310,15 @@ class F1MultiLabel(GlobalMetric):
     def compute(self, references: List[List[str]], predictions: List[str]) -> dict:
         self.str_to_id = {}
         self.id_to_str = {}
+        assert all(
+            len(reference) == 1 for reference in references
+        ), "Only a single reference per prediction is allowed in F1 metric"
+        references = [reference[0] for reference in references]
         labels = list(set([label for reference in references for label in reference]))
         for label in labels:
-            assert (
-                not self.seperator in label
-            ), "Reference label (f{label}) can not contain multi label seperator (f{self.seperator}) "
             self.add_str_to_id(label)
         formatted_references = [self.get_one_hot_vector(reference) for reference in references]
-        split_predictions = [
-            [label.strip() for label in prediction.split(self.seperator)] for prediction in predictions
-        ]
-        formatted_predictions = [self.get_one_hot_vector(prediction) for prediction in split_predictions]
+        formatted_predictions = [self.get_one_hot_vector(prediction) for prediction in predictions]
         result = self._metric.compute(
             predictions=formatted_predictions, references=formatted_references, average=self.average
         )
@@ -356,6 +354,38 @@ class Rouge(HuggingfaceMetric):
         return super().compute(references, predictions)
 
 
+# Computes chat edit distance, ignoring repeating whitespace
+class CharEditDistanceAccuracy(SingleReferenceInstanceMetric):
+    reduction_map = {"mean": ["char_edit_dist_accuracy"]}
+    main_score = "char_edit_dist_accuracy"
+
+    def compute(self, reference, prediction: str) -> dict:
+        formatted_prediction = " ".join(prediction.split())
+        formatted_reference = " ".join(reference.split())
+        max_length = max(len(formatted_reference), len(formatted_prediction))
+        if max_length == 0:
+            return 0
+        edit_dist = eval(formatted_reference, formatted_prediction)
+        return {"char_edit_dist_accuracy": (1 - edit_dist / max_length)}
+
+
+class Wer(HuggingfaceMetric):
+    metric_name = "wer"
+    main_score = "wer"
+
+    def prepare(self):
+        super().prepare()
+        self.metric = evaluate.load(self.metric_name)
+
+    def compute(self, references: List[List[str]], predictions: List[str]) -> dict:
+        assert all(
+            len(reference) == 1 for reference in references
+        ), "Only single reference per prediction is allowed in wer metric"
+        formatted_references = [reference[0] for reference in references]
+        result = self.metric.compute(predictions=predictions, references=formatted_references)
+        return {self.main_score: result}
+
+
 class Bleu(HuggingfaceMetric):
     metric_name = "bleu"
     main_score = "bleu"