Upload 3 files

app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("negbleurt")
+launch_gradio_widget(module)

negbleurt.py

@@ -0,0 +1,72 @@
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import datasets
+import evaluate
+
+
+_CITATION = """\
+tba
+"""
+
+_DESCRIPTION = """\
+Negation-aware version of BLEURT metric.
+BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018) and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations and the CANNOT negation awareness dataset.
+"""
+
+_KWARGS_DESCRIPTION = """
+Calculates the NegBLEURT scores between references and predictions
+Args:
+    predictions: list of predictions to score. Each prediction should be a string.
+    references: single reference or list of references for each prediction. If only one reference is given, all predictions will be scored against the same reference
+    batch_size: batch_size for model inference. Default is 16
+Returns:
+    negBLEURT: List of NegBLEURT scores for all predictions
+Examples:
+    >>> negBLEURT = evaluate.load('negbleurt')
+    >>> predictions = ["Ray Charles is a legend.", "Ray Charles isn’t legendary."]
+    >>> reference = "Ray Charles is legendary."
+    >>> results = rouge.compute(predictions=predictions, references=reference)
+    >>> print(results)
+    {'negBLERUT': [0.8409, 0.2601]}
+"""
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class NegBLEURT(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=[
+                datasets.Features(
+                    {
+                        "predictions": datasets.Value("string", id="sequence"),
+                        "references": datasets.Sequence(datasets.Value("string", id="sequence")),
+                    }
+                ),
+                datasets.Features(
+                    {
+                        "predictions": datasets.Value("string", id="sequence"),
+                        "references": datasets.Value("string", id="sequence"),
+                    }
+                ),
+            ],
+            codebase_urls=["https://github.com/MiriUll/negation_aware_evaluation"]
+        )
+
+    def _download_and_prepare(self, dl_manager):
+        model_name = "tum-nlp/NegBLEURT"
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.model = AutoModelForSequenceClassification.from_pretrained(model_name)
+
+    def _compute(
+        self, predictions, references, batch_size=16
+    ):
+        single_ref = isinstance(references, str)
+        if single_ref:
+            references = [references] * len(predictions)
+
+        scores_negbleurt = []
+        for i in tqdm(range(0, len(references), batch_size)):
+            tokenized = self.tokenizer(references[i:i+batch_size], candidates[i:i+batch_size], return_tensors='pt', padding=True, max_length=512, truncation=True)
+            scores_negbleurt += self.model(**tokenized).logits.flatten().tolist()
+        return {'negBLEURT': scores_negbleurt}

requirements.txt

@@ -0,0 +1 @@
+transformers~=4.25.1