Upload 13 files

grewtse/__init__.py

@@ -0,0 +1,3 @@
+from .pipeline import GrewTSEPipe
+
+__all__ = ["GrewTSEPipe"]

grewtse/evaluators/__init__.py

@@ -0,0 +1,4 @@
+from .evaluator import GrewTSEvaluator
+from .evaluator import Evaluator
+
+__all__ = ["GrewTSEvaluator", "Evaluator"]

grewtse/evaluators/evaluator.py

@@ -0,0 +1,403 @@
+import ast
+
+from transformers import AutoModelForMaskedLM, AutoModelForCausalLM, AutoTokenizer
+from transformers import PreTrainedModel, PreTrainedTokenizerBase
+from typing import Tuple, NamedTuple, List, Any
+import torch.nn.functional as F
+from tqdm import tqdm
+import pandas as pd
+import itertools
+import logging
+import torch
+import math
+
+from grewtse.utils.validation import load_and_validate_mp_dataset
+from grewtse.evaluators.metrics import (
+    compute_normalised_surprisal_difference,
+    compute_average_surprisal_difference,
+    compute_entropy,
+    compute_surprisal,
+    compute_mean,
+    calculate_all_metrics
+)
+
+EVAL_TEMPLATE = {
+    "sentence_id": None,
+    "match_id": None,
+    "original_text": None,
+    "prompt_text": None,
+    "form_grammatical": None,
+    "p_grammatical": None,
+    "I_grammatical": None,
+    "form_ungrammatical": None,
+    "p_ungrammatical": None,
+    "I_ungrammatical": None,
+    "entropy": None,
+    "entropy_norm": None,
+}
+
+
+class TooManyMasksException(Exception):
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(f"TMM Exception: {message}")
+
+
+class Prediction(NamedTuple):
+    token: str
+    prob: float
+    surprisal: float
+
+
+class GrewTSEvaluator:
+    """
+    An evaluation class designed specifically for rapid syntactic evaluation of models available on the Hugging Face platform.
+    """
+
+    def __init__(self):
+        self.evaluator = Evaluator()
+        self.evaluation_dataset = None
+
+    def evaluate_model(
+        self,
+        mp_dataset: pd.DataFrame,
+        model_repo: str,
+        model_type: str,  # can be 'encoder' or 'decoder'
+        entropy_topk: int = 100,
+        row_limit: int = None,
+    ) -> pd.DataFrame:
+        """
+        Generic evaluation function for encoder or decoder models.
+        """
+
+        # --- Prepare dataset ---
+        mp_dataset_iter = mp_dataset.itertuples()
+        if row_limit:
+            mp_dataset_iter = itertools.islice(mp_dataset_iter, row_limit)
+        n = len(mp_dataset) if not row_limit else row_limit
+
+        # --- Load model & tokenizer ---
+        is_encoder = model_type == "encoder"
+        model, tokenizer = self.evaluator.setup_parameters(model_repo, is_encoder)
+        results = []
+
+        # --- Evaluate each row ---
+        for row in tqdm(mp_dataset_iter, ncols=n):
+            row_results = self._init_row_results(row)
+
+            try:
+                if is_encoder:
+                    self._evaluate_encoder_row(row, row_results)
+                else:
+                    self._evaluate_decoder_row(row, row_results)
+
+            except TooManyMasksException:
+                logging.error(f"Too many masks in {row.sentence_id}")
+                continue
+            except Exception as e:
+                raise RuntimeError(f"Model/tokeniser issue: {e}") from e
+
+            # --- Entropy ---
+            entropy, entropy_norm = self.evaluator.get_entropy(entropy_topk, True)
+            row_results["entropy"] = entropy
+            row_results["entropy_norm"] = entropy_norm
+
+            results.append(row_results)
+
+        results_df = pd.DataFrame(results, columns=EVAL_TEMPLATE.keys())
+        self.evaluation_dataset = results_df
+        return results_df
+
+    def evaluate_from_minimal_pairs(
+        self,
+        mp_dataset_filepath: str,
+        model_repo: str,
+        model_type: str,
+        entropy_topk: int = 100,
+        row_limit: int = None,
+    ) -> pd.DataFrame:
+        mp_dataset = load_and_validate_mp_dataset(mp_dataset_filepath)
+        self.mp_dataset = mp_dataset
+        return self.evaluate_model(model_repo, model_type, entropy_topk, row_limit)
+
+    # --- Helper functions ---
+    def _init_row_results(self, row):
+        row_results = EVAL_TEMPLATE.copy()
+        row_results.update(row._asdict())
+        return row_results
+
+    def _evaluate_encoder_row(self, row, row_results):
+        prob_gram, prob_ungram = self.evaluator.run_masked_prediction(
+            row.masked_text,
+            row.form_grammatical,
+            row.form_ungrammatical,
+        )
+
+        row_results["p_grammatical"] = prob_gram
+        row_results["p_ungrammatical"] = prob_ungram
+        row_results["I_grammatical"] = compute_surprisal(prob_gram)
+        row_results["I_ungrammatical"] = compute_surprisal(prob_ungram)
+
+        if "ood_minimal_pairs" in row:
+            ood_pairs_str = row.ood_pairs
+            ood_pairs = ast.literal_eval(ood_pairs_str)
+            all_ood_probs_gram = []
+            all_ood_probs_ungram = []
+            for pair in ood_pairs:
+                ood_prob_gram, ood_prob_ungram = self.evaluator.run_masked_prediction(
+                    row.masked_text, pair[0], pair[1]
+                )
+                all_ood_probs_gram.append(ood_prob_gram)
+                all_ood_probs_ungram.append(ood_prob_ungram)
+
+            avg_ood_prob_gram = compute_mean(all_ood_probs_gram)
+            avg_ood_prob_ungram = compute_mean(all_ood_probs_ungram)
+
+            row_results["ood_p_grammatical"] = avg_ood_prob_gram
+            row_results["ood_p_ungrammatical"] = avg_ood_prob_ungram
+            row_results["ood_I_grammatical"] = compute_surprisal(avg_ood_prob_gram)
+            row_results["ood_I_ungrammatical"] = compute_surprisal(avg_ood_prob_ungram)
+
+    def _evaluate_decoder_row(self, row, row_results):
+        prob_gram, prob_ungram = self.evaluator.run_next_word_prediction(
+            row.prompt_text, row.form_grammatical, row.form_ungrammatical
+        )
+        row_results["p_grammatical"] = prob_gram
+        row_results["p_ungrammatical"] = prob_ungram
+        row_results["I_grammatical"] = compute_surprisal(prob_gram)
+        row_results["I_ungrammatical"] = compute_surprisal(prob_ungram)
+
+        if "ood_minimal_pairs" in row:
+            ood_pairs_str = row.ood_pairs
+            ood_pairs = ast.literal_eval(ood_pairs_str)
+            all_ood_probs_gram = []
+            all_ood_probs_ungram = []
+            for pair in ood_pairs:
+                ood_prob_gram, ood_prob_ungram = (
+                    self.evaluator.run_next_word_prediction(
+                        row.masked_text, pair[0], pair[1]
+                    )
+                )
+                all_ood_probs_gram.append(ood_prob_gram)
+                all_ood_probs_ungram.append(ood_prob_ungram)
+
+            avg_ood_prob_gram = compute_mean(all_ood_probs_gram)
+            avg_ood_prob_ungram = compute_mean(all_ood_probs_ungram)
+
+            row_results["ood_p_grammatical"] = avg_ood_prob_gram
+            row_results["ood_p_ungrammatical"] = avg_ood_prob_ungram
+            row_results["ood_I_grammatical"] = compute_surprisal(avg_ood_prob_gram)
+            row_results["ood_I_ungrammatical"] = compute_surprisal(avg_ood_prob_ungram)
+
+    def get_norm_avg_surprisal_difference(self) -> float:
+        if not self.is_model_evaluated():
+            raise KeyError("Please evaluate a model first.")
+        return compute_normalised_surprisal_difference(
+            self.evaluation_dataset["p_grammatical"],
+            self.evaluation_dataset["p_ungrammatical"],
+        )
+
+    def get_avg_surprisal_difference(self, is_ood: bool = False) -> float:
+        p_grammatical_col = "p_grammatical" if not is_ood else "ood_p_grammatical"
+        p_ungrammatical_col = "p_ungrammatical" if not is_ood else "ood_p_ungrammatical"
+        if not self.is_model_evaluated():
+            raise KeyError("Please evaluate a model first.")
+        return compute_average_surprisal_difference(
+            self.evaluation_dataset[p_grammatical_col],
+            self.evaluation_dataset[p_ungrammatical_col],
+        )
+
+    def get_all_metrics(self):
+        if self.evaluation_dataset is not None:
+            print(self.evaluation_dataset.columns)
+            return calculate_all_metrics(self.evaluation_dataset)
+        else:
+            raise ValueError("Please evaluate a model first.")
+
+
+class Evaluator:
+    def __init__(self):
+        self.tokeniser: PreTrainedTokenizerBase = None
+        self.model: PreTrainedModel = None
+
+        self.mask_token_index: int = -1
+        self.mask_probs: torch.Tensor | None = None
+        self.logits: torch.Tensor = None
+
+    def setup_parameters(
+        self, model_name: str, is_mlm: bool = True
+    ) -> Tuple[PreTrainedTokenizerBase, PreTrainedModel]:
+        if is_mlm:
+            self.tokeniser = AutoTokenizer.from_pretrained(model_name)
+            self.model = AutoModelForMaskedLM.from_pretrained(model_name)
+        else:
+            self.tokeniser = AutoTokenizer.from_pretrained(model_name)
+            self.model = AutoModelForCausalLM.from_pretrained(model_name)
+
+        # set to eval mode, disabling things like dropout
+        self.model.eval()
+
+        return self.model, self.tokeniser
+
+    def run_masked_prediction(
+        self, sentence: str, grammatical_word: str, ungrammatical_word: str
+    ) -> Tuple[float, float]:
+        if not self.model or not self.tokeniser:
+            raise RuntimeError("Model and tokenizer must be loaded before prediction.")
+
+        mask_token = self.tokeniser.mask_token
+        sentence_masked = sentence.replace("[MASK]", mask_token)
+
+        if sentence_masked.count(mask_token) != 1:
+            raise TooManyMasksException("Only single-mask sentences are supported.")
+
+        masked_ids = self.tokeniser.encode(sentence_masked, add_special_tokens=False)
+        mask_index = masked_ids.index(self.tokeniser.mask_token_id)
+
+        device = next(self.model.parameters()).device
+        g_ids = self.tokeniser.encode(grammatical_word, add_special_tokens=False)
+        u_ids = self.tokeniser.encode(ungrammatical_word, add_special_tokens=False)
+
+        g_prob = self._compute_masked_joint_probability(
+            masked_ids, mask_index, g_ids, device
+        )
+        u_prob = self._compute_masked_joint_probability(
+            masked_ids, mask_index, u_ids, device
+        )
+
+        return g_prob, u_prob
+
+    def _compute_masked_joint_probability(
+        self, input_ids: List[int], mask_index: int, word_ids: List[int], device
+    ) -> float:
+        input_ids_tensor = torch.tensor([input_ids], device=device)
+        log_prob = 0.0
+        index = mask_index
+
+        for i, tid in enumerate(word_ids):
+            with torch.no_grad():
+                logits = self.model(input_ids_tensor).logits
+
+            probs = F.softmax(logits[:, index, :], dim=-1)
+            token_prob = probs[0, tid].item()
+            log_prob += math.log(token_prob + 1e-12)
+
+            if i == 0:
+                self.mask_probs = probs
+
+            # Replace mask with predicted token
+            input_ids_tensor[0, index] = tid
+
+            # Insert new mask if more tokens remain
+            if i < len(word_ids) - 1:
+                input_ids_tensor = torch.cat(
+                    [
+                        input_ids_tensor[:, : index + 1],
+                        torch.tensor([[self.tokeniser.mask_token_id]], device=device),
+                        input_ids_tensor[:, index + 1 :],
+                    ],
+                    dim=1,
+                )
+
+                # debugging
+                tokens_after_insertion = self.tokeniser.convert_ids_to_tokens(
+                    input_ids_tensor[0].tolist()
+                )
+
+                index += 1
+
+        return math.exp(log_prob)
+
+    def run_next_word_prediction(
+        self, context: str, grammatical_word: str, ungrammatical_word: str
+    ) -> Tuple[float, float]:
+        if not self.model or not self.tokeniser:
+            raise RuntimeError("Model and tokenizer must be loaded before prediction.")
+
+        context_ids = self.tokeniser.encode(context, add_special_tokens=False)
+        device = next(self.model.parameters()).device
+
+        g_ids = self.tokeniser.encode(grammatical_word, add_special_tokens=False)
+        u_ids = self.tokeniser.encode(ungrammatical_word, add_special_tokens=False)
+
+        g_prob = self._compute_next_word_joint_probability(context_ids, g_ids, device)
+        u_prob = self._compute_next_word_joint_probability(context_ids, u_ids, device)
+
+        return g_prob, u_prob
+
+    def _compute_next_word_joint_probability(
+        self, input_ids: List[int], word_ids: List[int], device
+    ) -> float:
+        input_ids_tensor = torch.tensor([input_ids], device=device)
+        # debugging
+        tokens_after_insertion = self.tokeniser.convert_ids_to_tokens(
+            input_ids_tensor[0].tolist()
+        )
+        log_prob = 0.0
+
+        for i, tid in enumerate(word_ids):
+            with torch.no_grad():
+                logits = self.model(input_ids_tensor).logits
+
+            index = input_ids_tensor.shape[1] - 1  # last token position
+            probs = F.softmax(logits[:, index, :], dim=-1)
+            token_prob = probs[0, tid].item()
+            log_prob += math.log(token_prob + 1e-12)
+
+            if i == 0:
+                self.mask_probs = probs
+
+            # Append predicted token to context
+            input_ids_tensor = torch.cat(
+                [input_ids_tensor, torch.tensor([[tid]], device=device)], dim=1
+            )
+
+            # debugging
+            tokens_after_insertion = self.tokeniser.convert_ids_to_tokens(
+                input_ids_tensor[0].tolist()
+            )
+
+        return math.exp(log_prob)
+
+    def get_entropy(self, k: int = 100, normalise: bool = False) -> float:
+        """Compute entropy over the prediction distribution.
+
+        Args:
+            k: Number of top tokens to consider.
+            normalise: Whether to normalise entropy.
+
+        Returns:
+            Entropy value.
+        Raises:
+            ValueError: If no probabilities are available.
+        """
+        if self.mask_probs is None:
+            raise ValueError("No output probabilities available. Run evaluation first.")
+        return compute_entropy(self.mask_probs, k, normalise)
+
+    def _get_mask_index(self, inputs: Any) -> int:
+        if "input_ids" not in inputs:
+            raise ValueError("Missing 'input_ids' in inputs.")
+        elif self.tokeniser.mask_token_id is None:
+            raise ValueError("The tokeniser does not have a defined mask_token_id.")
+
+        input_ids = inputs["input_ids"]
+        mask_positions = torch.where(input_ids == self.tokeniser.mask_token_id)
+
+        if len(mask_positions[0]) == 0:
+            raise ValueError("No mask token found in input_ids.")
+        elif len(mask_positions[0]) > 1:
+            raise ValueError("Multiple mask tokens found; expected only one.")
+
+        return (
+            mask_positions[1].item()
+            if len(mask_positions) > 1
+            else mask_positions[0].item()
+        )
+
+    def _get_mask_probabilities(
+        self, mask_token_index: int, logits: Any
+    ) -> torch.Tensor:
+        mask_logits = logits[0, mask_token_index, :]
+        probs = F.softmax(mask_logits, dim=-1)  # shape: (vocab_size, )
+        return probs

grewtse/evaluators/metrics.py

@@ -0,0 +1,104 @@
+import pandas as pd
+import numpy as np
+import math
+from typing import List
+
+
+def compute_mean(list_of_values: List[float]) -> float:
+    return sum(list_of_values) / len(list_of_values)
+
+
+def compute_surprisal(p: float) -> float:
+    return -math.log2(p) if p and p > 0 else float("inf")
+
+def compute_avg_surprisal(probs: pd.Series) -> float:
+    as_surprisal = probs.apply(compute_surprisal)
+    return as_surprisal.mean()
+
+
+def compute_average_surprisal_difference(
+    correct_form_probs: pd.Series, wrong_form_probs: pd.Series
+) -> float:
+    correct_form_avg_surp = compute_avg_surprisal(correct_form_probs)
+    wrong_form_avg_surp = compute_avg_surprisal(wrong_form_probs)
+    return wrong_form_avg_surp - correct_form_avg_surp
+
+
+def compute_normalised_surprisal_difference(
+    correct_form_probs: pd.Series, wrong_form_probs: pd.Series
+) -> float:
+    correct_form_avg_surp = compute_avg_surprisal(correct_form_probs)
+    wrong_form_avg_surp = compute_avg_surprisal(wrong_form_probs)
+    return (wrong_form_avg_surp - correct_form_avg_surp) / correct_form_avg_surp
+
+
+def compute_entropy(probs, k=None, normalise=False):
+    probs = np.array(probs, dtype=np.float64)
+
+    # remove zeros to avoid log(0)
+    probs = probs[probs > 0]
+
+    # get top-k probabilities
+    if k is not None:
+        probs = np.sort(probs)[::-1][:k]
+        probs = probs / probs.sum()  # renormalize to sum to 1
+
+    H = -np.sum(probs * np.log(probs))
+
+    if normalise:
+        n = len(probs)
+        return H, 1 - H / np.log(n)
+    else:
+        return H
+
+def get_predictions(df: pd.DataFrame) -> np.ndarray:
+    """
+    Convert probabilities to binary predictions.
+    Predicts grammatical (1) if p_grammatical > p_ungrammatical, else ungrammatical (0).
+    """
+    predictions = (df['p_grammatical'] > df['p_ungrammatical']).astype(int)
+    return predictions.values
+
+
+def calculate_accuracy(df: pd.DataFrame) -> float:
+    """
+    Calculate accuracy: proportion of correct predictions.
+    Assumes the model should always predict grammatical form (label = 1).
+    """
+    predictions = get_predictions(df)
+    # True labels: all should be grammatical (1)
+    true_labels = np.ones(len(df), dtype=int)
+
+    correct = np.sum(predictions == true_labels)
+    total = len(predictions)
+
+    return correct / total if total > 0 else 0.0
+
+def calculate_all_metrics(df: pd.DataFrame) -> dict:
+    predictions = get_predictions(df)
+    true_labels = np.ones(len(df), dtype=int)
+
+    # Calculate confusion matrix components
+    tp = np.sum((predictions == 1) & (true_labels == 1))
+    fp = np.sum((predictions == 1) & (true_labels == 0))
+    fn = np.sum((predictions == 0) & (true_labels == 1))
+    tn = np.sum((predictions == 0) & (true_labels == 0))
+
+    total = len(predictions)
+
+    # Calculate metrics
+    accuracy = (tp + tn) / total if total > 0 else 0.0
+    precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+    recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+    f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+
+    return {
+        'accuracy': round(accuracy,2),
+        'precision': round(precision, 2),
+        'recall': round(recall, 2),
+        'f1': round(f1, 2),
+        'true_positives': int(tp),
+        'false_positives': int(fp),
+        'false_negatives': int(fn),
+        'true_negatives': int(tn)
+    }

grewtse/pipeline.py

@@ -0,0 +1,295 @@
+import os
+import pandas as pd
+import random
+import logging
+from grewtse.preprocessing import ConlluParser
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.FileHandler("app.log"), logging.StreamHandler()],
+)
+
+
+class GrewTSEPipe:
+    """
+    Main pipeline controller for generating prompt- or masked-based minimal-pair datasets derived from UD treebanks.
+
+    This class acts as a high-level interface for the Grew-TSE workflow:
+
+    1. Parse treebanks to build lexical item datasets.
+    2. Generate masked or prompt-based datasets using GREW.
+    3. Create minimal pairs for syntactic evaluation.
+    """
+
+    def __init__(self):
+        self.parser = ConlluParser()
+
+        self.treebank_paths: list[str] = []
+        self.lexical_items: pd.DataFrame | None = None
+        self.grew_generated_dataset: pd.DataFrame | None = None
+        self.mp_dataset: pd.DataFrame | None = None
+        self.exception_dataset: pd.DataFrame | None = None
+        self.evaluation_results: pd.DataFrame | None = None
+
+    # 1. Initial step, parse a treebank from a .conllu file
+    def parse_treebank(
+        self, filepaths: str | list[str], reset: bool = False
+    ) -> pd.DataFrame:
+        """
+        Parse one or more treebanks and create a lexical item set.
+        A lexical item set is a dataset of words and their features.
+
+        Args:
+            filepaths: Path or list of paths to treebank files.
+            reset: If True, clears existing lexical_items before parsing.
+
+        """
+
+        if isinstance(filepaths, str):
+            filepaths = [filepaths]  # wrap single path in list
+
+        try:
+            if reset or self.lexical_items is None:
+                self.lexical_items = pd.DataFrame()
+                self.treebank_paths = []
+
+            self.lexical_items = self.parser.build_lexicon(filepaths)
+            self.treebank_paths = filepaths
+
+            return self.lexical_items
+        except Exception as e:
+            raise Exception(f"Issue parsing treebank: {e}")
+
+    def load_lexicon(self, filepath: str, treebank_paths: list[str]) -> None:
+        """
+        Load a previously generated lexicon (typically returned from the parse_treebank function) from disk and attach it to the pipeline.
+
+        This method is used when you want to resume processing using an existing LI_set that was
+        generated earlier and saved as a CSV. It loads the LI_set, validates the required columns,
+        sets the appropriate index, and updates the pipeline and parser with the loaded data.
+
+        Args:
+            filepath (str):
+                Path to the CSV file containing the LI_set to load. The file must contain the
+                columns ``"sentence_id"`` and ``"token_id"``.
+            treebank_paths (list[str]):
+                A list of paths to the treebanks associated with the LI_set. These paths are stored
+                so the pipeline can later reference the corresponding treebanks when generating or
+                analyzing data.
+
+        Raises:
+            FileNotFoundError:
+                If the CSV file cannot be found at the given ``filepath``.
+            ValueError:
+                If the required index columns (``"sentence_id"``, ``"token_id"``) are missing.
+
+        Example:
+            >>> pipe = GrewTSEPipe()
+            >>> pipe.load_lexicon("output/li_set.csv", ["treebank1.conllu", "treebank2.conllu"])
+
+        """
+        if not os.path.exists(filepath):
+            raise FileNotFoundError(f"LI_set file not found: {filepath}")
+
+        li_df = pd.read_csv(filepath)
+
+        required_cols = {"sentence_id", "token_id"}
+        missing = required_cols - set(li_df.columns)
+        if missing:
+            raise ValueError(
+                f"Missing required columns in LI_set: {', '.join(missing)}"
+            )
+
+        li_df.set_index(["sentence_id", "token_id"], inplace=True)
+
+        self.lexical_items = li_df
+        self.parser.lexicon = li_df
+        self.treebank_paths = treebank_paths
+
+    def generate_masked_dataset(
+        self, query: str, target_node: str, mask_token: str = "[MASK]"
+    ) -> pd.DataFrame:
+        """
+        Once a treebank has been parsed, if testing models on the task of masked language modelling (MLM) e.g. for encoder models, then you can generate a masked dataset with default token [MASK] by providing
+        a GREW query that isolates a particular construction and a target node that identifies the element
+        in that construction that you want to test.
+
+        :param query: the GREW query that specifies a construction. Test them over at https://universal.grew.fr/
+        :param target_node: the particular variable that you defined in your GREW query representing the target word
+        :return: a DataFrame consisting of the sentence ID in the given treebank, the index of the token to be masked in the set of tokens, the list of all tokens, the matched token itself, the original text, and lastly the masked text.
+
+        """
+
+        if not self.is_treebank_parsed():
+            raise ValueError(
+                "Cannot create masked dataset: no treebank or invalid treebank filepath provided."
+            )
+
+        results = self.parser.build_masked_dataset(
+            self.treebank_paths, query, target_node, mask_token
+        )
+        self.grew_generated_dataset = results["masked"]
+        self.exception_dataset = results["exception"]
+        return self.grew_generated_dataset
+
+    def generate_prompt_dataset(self, query: str, target_node: str) -> pd.DataFrame:
+        """
+        Once a treebank has been parsed, if testing models on the task of next-token prediction (NTP) e.g. for decoder models, then you can use this function to generate a prompt dataset by providing
+        a GREW query that isolates a particular construction and a target node that identifies the element
+        in that construction that you want to test.
+
+        :param query: the GREW query that specifies a construction. Test them over at https://universal.grew.fr/
+        :param target_node: the particular variable that you defined in your GREW query representing the target word
+        :return: a DataFrame consisting of the sentence ID in the given treebank, the index of the target token, the list of all tokens, the matched token itself, the original text, and the created prompt.
+
+        """
+
+        if not self.is_treebank_parsed():
+            raise ValueError(
+                "Cannot create prompt dataset: no treebank or invalid treebank filepath provided."
+            )
+
+        prompt_dataset = self.parser.build_prompt_dataset(
+            self.treebank_paths, query, target_node
+        )
+        self.grew_generated_dataset = prompt_dataset
+        return prompt_dataset
+
+    def generate_minimal_pair_dataset(
+        self,
+        morph_features: dict,
+        upos_features: dict | None,
+        ood_pairs: int | None = None,
+        has_leading_whitespace: bool = True,
+    ) -> pd.DataFrame:
+        """
+        After generating a masked or prompt dataset, that same dataset with minimal pairs can be created using this function by specifying the feature that you would like to change. You can also specify whether you want additional 'OOD' pairs to be created, as well as whether there should be a leading whitespace at the start of each minimal pair item.
+
+        NOTE: morph_features and upos_features expects lowercase keys, values remain as in the treebank.
+
+        :param morph_features: the morphological features from the UD treebank that you want to adjust for the second element of the minimal pair e.g. { 'case': 'Dat' } may convert the original target item e.g. German 'Hunde' (dog.PLUR.NOM / dog.PLUR.ACC) to the dative case e.g. 'Hunden' (dog.PLUR.DAT) to form the minimal pair (Hunde, Hunden). The exact keys and values will depend on the treebank that you're working with.
+        :param upos_features: the universal part-of-speech tags from the UD treebank that you want to adjust for the second element of the minimal pair e.g. { 'upos': 'VERB' } will only search for verbs
+        :param ood_pairs: a boolean argument that specifies whether you want alternative (likely semantically implausible) minimal pairs to be provided for each example. These may help in evaluating generalisation performance.
+        :param has_trailing_whitespace: a boolean argument that specifies whether an additional whitespace is included at the beginning of each element in the minimal pair e.g. (' is', ' are')
+        :return: a DataFrame containing the masked sentences or prompts as well as the minimal pairs
+
+        """
+
+        if self.grew_generated_dataset is None:
+            raise ValueError(
+                "Cannot generate minimal pairs: treebank must be parsed and masked first."
+            )
+
+        def convert_row_to_feature(row):
+            return self.parser.to_syntactic_feature(
+                row["sentence_id"],
+                row["match_id"] - 1,
+                row["match_token"],
+                morph_features,
+                upos_features
+            )
+
+        alternative_row = self.grew_generated_dataset.apply(
+            convert_row_to_feature, axis=1
+        )
+        self.mp_dataset = self.grew_generated_dataset
+        self.mp_dataset["form_ungrammatical"] = alternative_row
+
+        self.mp_dataset = self.mp_dataset.rename(
+            columns={"match_token": "form_grammatical"}
+        )
+
+        # rule 1: drop any rows where we don't find a minimal pair (henceforth MP)
+        self.mp_dataset = self.mp_dataset.dropna(subset=["form_ungrammatical"])
+
+        # rule 2: don't include MPs where the minimal pairs are the same string
+        self.mp_dataset = self.mp_dataset[
+            self.mp_dataset["form_grammatical"] != self.mp_dataset["form_ungrammatical"]
+        ]
+
+        # add leading whitespace if requested.
+        # this is useful for models that expect whitespace at the end such as many decoder models
+        if has_leading_whitespace:
+            self.mp_dataset["form_grammatical"] = (
+                " " + self.mp_dataset["form_grammatical"]
+            )
+            self.mp_dataset["form_ungrammatical"] = (
+                " " + self.mp_dataset["form_ungrammatical"]
+            )
+
+        # handle the assigning of the out-of-distribution pairs
+        if ood_pairs:
+            # assign additional pairs for OOD data
+            all_grammatical = self.mp_dataset["form_grammatical"].to_list()
+            all_ungrammatical = self.mp_dataset["form_ungrammatical"].to_list()
+
+            # combine both into one vocabulary
+            words = set(zip(all_grammatical, all_ungrammatical))
+
+            def pick_words(row):
+                excluded = (row["form_grammatical"], row["form_ungrammatical"])
+                available = list(words - {excluded})
+                return random.sample(list(available), ood_pairs)
+
+            # Apply function to each row
+            self.mp_dataset["ood_minimal_pairs"] = self.mp_dataset.apply(
+                pick_words, axis=1
+            )
+
+        return self.mp_dataset
+
+    def get_morphological_features(self) -> list:
+        """
+        Get a list of all available morphological features in a given treebank.
+        Similarly, you can go to the treebank's respective webpage to find this information.
+        A treebank must first be parsed in order to use this function.
+
+        :return: a list of strings with each morphological feature in the treebank.
+
+        """
+
+        if not self.is_treebank_parsed():
+            raise ValueError("Cannot get features: You must parse a treebank first.")
+
+        morph_df = self.lexical_items.copy()
+        morph_df.columns = [
+            col.replace("feats__", "") if col.startswith("feats__") else col
+            for col in morph_df.columns
+        ]
+
+        return morph_df
+
+    def is_treebank_parsed(self) -> bool:
+        return self.lexical_items is not None
+
+    def is_dataset_masked(self) -> bool:
+        return self.grew_generated_dataset is not None
+
+    def is_model_evaluated(self) -> bool:
+        return self.evaluation_dataset is not None
+
+    def get_lexical_items(self) -> pd.DataFrame:
+        return self.lexical_items
+
+    def get_masked_dataset(self) -> pd.DataFrame:
+        return self.grew_generated_dataset
+
+    def get_minimal_pair_dataset(self) -> pd.DataFrame:
+        return self.mp_dataset
+
+    def get_exceptions_dataset(self):
+        return self.exception_dataset
+
+    def get_num_exceptions(self):
+        if self.exception_dataset is not None:
+            return self.exception_dataset.shape[0]
+        else:
+            return -1
+
+    def are_minimal_pairs_generated(self) -> bool:
+        return (
+            self.is_treebank_parsed()
+            and self.is_dataset_masked()
+            and ("form_ungrammatical" in self.mp_dataset.columns)
+        )

grewtse/preprocessing/__init__.py

@@ -0,0 +1,3 @@
+from .conllu_parser import ConlluParser
+
+__all__ = ["ConlluParser"]

grewtse/preprocessing/conllu_parser.py

@@ -0,0 +1,348 @@
+from grewtse.preprocessing.grew_dependencies import match_dependencies
+from grewtse.preprocessing.reconstruction import (
+    perform_token_surgery,
+    recursive_match_token,
+)
+from conllu import parse_incr, Token
+from typing import Any
+import pandas as pd
+import numpy as np
+import logging
+
+
+class ConlluParser:
+    """
+    A class designed to parse .conllu files for Grew-TSE, that is, the standard format for UD treebanks.
+
+    """
+
+    def __init__(self) -> None:
+        self.lexicon: pd.DataFrame = None
+
+    def build_lexicon(self, filepaths: list[str] | str) -> pd.DataFrame:
+        """
+        Create a DataFrame that contains the set of all words with their features as generated from a UD treebank.
+        This is essential for the subsequent generation of minimal pairs.
+        This was not designed to handle treebanks that assign differing names to features, so please ensure multiple treebank files are all from the same treebank or treebank schema.
+
+        :param filepaths: a list of strings corresponding to the UD treebank files e.g. ["german_treebank_part_A.conllu", "german_treebank_part_B.conllu"].
+        :return: a DataFrame with all words and their features.
+
+        """
+        rows = []
+
+        if isinstance(filepaths, str):
+            filepaths = [filepaths]  # wrap single path in list
+
+        for conllu_path in filepaths:
+            with open(conllu_path, "r", encoding="utf-8") as f:
+                for tokenlist in parse_incr(f):
+                    # get the sentence ID in the dataset
+                    sent_id = tokenlist.metadata["sent_id"]
+                    logging.info(f"Parsing Sentence: {sent_id}")
+
+                    # iterate over each token
+                    for token in tokenlist:
+                        # check if it's worth saving to our lexical item dataset
+                        is_valid_token = is_valid_for_lexicon(token)
+                        if not is_valid_token:
+                            continue
+
+                        # from the token object create a dict and append
+                        row = build_token_row(token, sent_id)
+                        rows.append(row)
+
+            lexicon_df = pd.DataFrame(rows)
+
+            # make sure our nan values are interpreted as such
+            lexicon_df.replace("nan", np.nan, inplace=True)
+
+            # create the (Sentence ID, Token ID) primary key
+            lexicon_df.set_index(["sentence_id", "token_id"], inplace=True)
+
+            self.lexicon = lexicon_df
+
+        return lexicon_df
+
+    def to_syntactic_feature(
+        self,
+        sentence_id: str,
+        token_id: str,
+        token: str,
+        alt_morph_constraints: dict,
+        alt_universal_constraints: dict,
+    ) -> str | None:
+        """
+        The most important function for the finding of minimal pairs. Converts a given lexical item taken from a UD treebank sentence
+        to another lexical item of the same lemma but with the specified differing feature(s).
+
+        :param sentence_id: the ID in the treebank of the sentence.
+        :param token_id: the token index in the list of tokens corresponding to the isolated target word.
+        :param token: the token string itself that is the isolated target word.
+        :param alt_morph_constraints: the alternative morphological feature(s) for the target word.
+        :param alt_universal_constraints: the alternative UPOS feature(s) for the target word.
+        :return: a string representing the converted target word.
+        """
+
+        # distinguish morphological from universal features
+        # todo: find a better way to do this
+        prefix = "feats__"
+        # prefix = ''
+        alt_morph_constraints = {
+            prefix + key: value for key, value in alt_morph_constraints.items()
+        }
+
+        token_features = self.get_features(sentence_id, token_id)
+
+        token_features.update(alt_morph_constraints)
+        token_features.update(alt_universal_constraints)
+        lexical_items = self.lexicon
+
+        # get only those items which are the same lemma
+        lemma = self.get_lemma(sentence_id, token_id)
+        lemma_mask = lexical_items["lemma"] == lemma
+        lexical_items = lexical_items[lemma_mask]
+        logging.info(f"Looking for form {lemma}")
+
+        lexical_items = construct_candidate_set(lexical_items, token_features)
+        # ensure that it doesn't allow minimal pairs with different start cases e.g business, Business
+        filtered = lexical_items[
+            lexical_items["form"].apply(lambda w: is_same_start_case(w, token))
+        ]
+        if not filtered.empty:
+            return filtered["form"].iloc[0]
+        else:
+            return None
+
+    def get_lexicon(self) -> pd.DataFrame:
+        return self.lexicon
+
+    # this shouldn't be hard coded
+    def get_feature_names(self) -> list:
+        return self.lexicon.columns[4:].to_list()
+
+    # todo: add more safety
+    def get_features(self, sentence_id: str, token_id: int) -> dict:
+        return self.lexicon.loc[(sentence_id, token_id)][
+            self.get_feature_names()
+        ].to_dict()
+
+    def get_lemma(self, sentence_id: str, token_id: str) -> str:
+        return self.lexicon.loc[(sentence_id, token_id)]["lemma"]
+
+    def get_candidate_set(
+        self, universal_constraints: dict, morph_constraints: dict
+    ) -> pd.DataFrame:
+        has_parsed_conllu = self.lexicon is not None
+        if not has_parsed_conllu:
+            raise ValueError("Please parse a ConLLU file first.")
+
+        morph_constraints = {f"feats__{k}": v for k, v in morph_constraints.items()}
+        are_morph_features_valid = all(
+            f in self.lexicon.columns for f in morph_constraints.keys()
+        )
+        are_universal_features_valid = all(
+            f in self.lexicon.columns for f in universal_constraints.keys()
+        )
+        if not are_morph_features_valid or not are_universal_features_valid:
+            raise KeyError(
+                "Features provided for candidate set are not valid features in the dataset."
+            )
+
+        all_constraints = {**universal_constraints, **morph_constraints}
+        candidate_set = construct_candidate_set(self.lexicon, all_constraints)
+        return candidate_set
+
+    def build_prompt_dataset(
+        self,
+        filepaths: list[str],
+        grew_query: str,
+        dependency_node: str,
+        encoding: str = "utf-8",
+    ):
+        prompt_cutoff_token = "[PROMPT_CUTOFF]"
+        results = self.build_masked_dataset(
+            filepaths, grew_query, dependency_node, prompt_cutoff_token, encoding
+        )
+        prompt_dataset = results["masked"]
+
+        def substring_up_to_token(s: str, token: str) -> str:
+            idx = s.find(token)
+            return s[:idx].strip() if idx != -1 else s.strip()
+
+        prompt_dataset["prompt_text"] = prompt_dataset["masked_text"].apply(
+            lambda x: substring_up_to_token(x, prompt_cutoff_token)
+        )
+        prompt_dataset = prompt_dataset.drop(["masked_text"], axis=1)
+        return prompt_dataset
+
+    def build_masked_dataset(
+        self,
+        filepaths: list[str],
+        grew_query: str,
+        dependency_node: str,
+        mask_token: str,
+        encoding: str = "utf-8",
+    ):
+        masked_dataset = []
+        exception_dataset = []
+
+        try:
+            for filepath in filepaths:
+                get_tokens_to_mask = match_dependencies(
+                    filepath, grew_query, dependency_node
+                )
+
+                with open(filepath, "r", encoding=encoding) as data_file:
+                    for sentence in parse_incr(data_file):
+
+                        sentence_id = sentence.metadata["sent_id"]
+                        sentence_text = sentence.metadata["text"]
+
+                        if sentence_id in get_tokens_to_mask:
+                            for i in range(len(sentence)):
+                                sentence[i]["index"] = i
+
+                            token_to_mask_id = get_tokens_to_mask[sentence_id]
+
+                            try:
+                                t_match = [
+                                    tok
+                                    for tok in sentence
+                                    if tok.get("id") == token_to_mask_id
+                                ][0]
+                                t_match_form = t_match["form"]
+                                t_match_index = t_match["index"]
+                                sentence_as_str_list = [t["form"] for t in sentence]
+                            except KeyError:
+                                logging.info(
+                                    "There was a mismatch for the GREW-based ID and the Conllu ID."
+                                )
+                                exception_dataset.append(
+                                    {
+                                        "sentence_id": sentence_id,
+                                        "match_id": None,
+                                        "all_tokens": None,
+                                        "match_token": None,
+                                        "original_text": sentence_text,
+                                    }
+                                )
+                                continue
+
+                            try:
+                                matched_token_start_index = recursive_match_token(
+                                    sentence_text,  # the original string
+                                    sentence_as_str_list.copy(),  # the string as a list of tokens
+                                    t_match_index,  # the index of the token to be replaced
+                                    [
+                                        "_",
+                                        " ",
+                                    ],  # todo: skip lines where we don't encounter accounted for tokens
+                                )
+                            except ValueError:
+                                exception_dataset.append(
+                                    {
+                                        "sentence_id": sentence_id,
+                                        "match_id": token_to_mask_id,
+                                        "all_tokens": sentence_as_str_list,
+                                        "match_token": t_match_form,
+                                        "original_text": sentence_text,
+                                    }
+                                )
+                                continue
+
+                            # let's replace the matched token with a MASK token
+                            masked_sentence = perform_token_surgery(
+                                sentence_text,
+                                t_match_form,
+                                mask_token,
+                                matched_token_start_index,
+                            )
+
+                            # the sentence ID and match ID are together a primary key
+                            masked_dataset.append(
+                                {
+                                    "sentence_id": sentence_id,
+                                    "match_id": token_to_mask_id,
+                                    "match_token": t_match_form,
+                                    "original_text": sentence_text,
+                                    "masked_text": masked_sentence,
+                                }
+                            )
+        except Exception as e:
+            print(f"Issue building dataset: {e}")
+
+        masked_dataset_df = pd.DataFrame(masked_dataset)
+        exception_dataset_df = pd.DataFrame(exception_dataset)
+
+        return {"masked": masked_dataset_df, "exception": exception_dataset_df}
+
+
+def construct_candidate_set(
+    lexicon: pd.DataFrame, target_features: dict
+) -> pd.DataFrame:
+    """
+    This constructs a list of words which have the same feature set as the
+    target features which are passed as an argument. These resulting words are termed 'candidates'.
+
+    :param lexicon: the DataFrame consisting of all lexical items and their features
+    :param target_features: the differing features of the candidates.
+    :return: a DataFrame containing the candidate subset of the lexicon.
+
+    """
+
+    # optionally restrict search to a certain type of lexical item
+    subset = lexicon
+
+    # continuously filter the dataframe so as to be left
+    # only with those lexical items which match the target
+    # features
+    # this includes cases
+    for feat, value in target_features.items():
+        # ensure feature is a valid feature in feature set
+        if feat not in subset.columns:
+            print(subset.columns)
+            raise KeyError("Invalid feature provided to confound set: {}".format(feat))
+
+        # slim the mask down using each feature
+        # interesting edge case: np.nan == np.nan returns false!
+        mask = (subset[feat] == value) | (subset[feat].isna() & pd.isna(value))
+        subset = subset[mask]
+
+    return subset
+
+
+def is_same_start_case(s1, s2):
+    if not s1 or not s2:
+        return False
+    return s1[0].isupper() == s2[0].isupper()
+
+
+def is_valid_for_lexicon(token: Token) -> bool:
+    punctuation = [".", ",", "!", "?", "*"]
+
+    # skip multiword tokens, malformed entries and punctuation
+    is_punctuation = token.get("form") in punctuation
+    is_valid_type = isinstance(token, dict)
+    has_valid_id = isinstance(token.get("id"), int)
+    return is_valid_type and has_valid_id and not is_punctuation
+
+
+def build_token_row(token: Token, sentence_id: str) -> dict[str, Any]:
+    # get all token features such as Person, Mood, etc
+    feats = token.get("feats") or {}
+
+    row = {
+        "sentence_id": sentence_id,
+        "token_id": token.get("id") - 1,  # IDs are reduced by one to start at 0
+        "form": token.get("form"),
+        "lemma": token.get("lemma"),
+        "upos": token.get("upos"),
+        "xpos": token.get("xpos"),
+    }
+
+    # add each morphological feature as a column
+    for feat_name, feat_value in feats.items():
+        row["feats__" + feat_name.lower()] = feat_value
+
+    return row

grewtse/preprocessing/grew_dependencies.py

@@ -0,0 +1,83 @@
+from grewpy import Corpus, Request, set_config
+
+
+"""
+def match_dependencies(
+    filepaths: list[str] | str, grew_query: str, dependency_node: str
+) -> dict:
+    set_config("sud")  # ud or basic
+    dep_matches = {}
+
+    if isinstance(filepaths, str):
+        filepaths = [filepaths]  # wrap single path in list
+
+    try:
+        for corpus_path in filepaths:
+            # run the GREW request on the corpus
+            print("Corpus Path ", corpus_path)
+            corpus = Corpus(str(corpus_path))
+            request = Request(grew_query)
+            occurrences = corpus.search(request)
+
+            # step 2
+            for occ in occurrences:
+                sent_id = occ["sent_id"]
+
+                object_node_id = int(occ["matching"]["nodes"][dependency_node])
+
+                # one match per sentence
+                # todo: handle multiple matches per sentence
+                dep_matches[sent_id] = object_node_id
+    except KeyError:
+        raise KeyError(
+            "You must provide a dependency node name which exists in your GREW pattern."
+        )
+    except Exception as e:
+        raise ValueError(f"Invalid GREW query: {e}")
+
+    return dep_matches
+"""
+
+def match_dependencies(
+    filepaths: list[str] | str, grew_query: str, dependency_node: str
+) -> dict:
+    set_config("sud")  # ud or basic
+    dep_matches = {}
+
+    if isinstance(filepaths, str):
+        filepaths = [filepaths]  # wrap single path in list
+
+    try:
+        for corpus_path in filepaths:
+            # run the GREW request on the corpus
+            corpus = Corpus(str(corpus_path))
+            request = Request(grew_query)
+            occurrences = corpus.search(request)
+
+            # step 2
+            for occ in occurrences:
+                sent_id = occ["sent_id"]
+
+                # Handle both integer and float string representations
+                node_id_str = occ["matching"]["nodes"][dependency_node]
+                try:
+                    # Try converting to float first, then to int
+                    object_node_id = float(node_id_str)
+                except ValueError:
+                    raise ValueError("Cannot convert.")
+
+                # one match per sentence
+                # todo: handle multiple matches per sentence
+                dep_matches[sent_id] = object_node_id
+    except KeyError as e:
+        if dependency_node not in str(e):
+            raise KeyError(
+                f"You must provide a dependency node name which exists in your GREW pattern. Missing key: {e}"
+            )
+        raise KeyError(
+            "You must provide a dependency node name which exists in your GREW pattern."
+        )
+    except Exception as e:
+        raise ValueError(f"Invalid GREW query: {e}")
+
+    return dep_matches

grewtse/preprocessing/reconstruction.py

@@ -0,0 +1,71 @@
+def perform_token_surgery(
+    sentence: str,
+    original_token: str,
+    replacement_token: str,
+    start_index: int,
+) -> str:
+    t_len = len(original_token)
+
+    return sentence[:start_index] + replacement_token + sentence[start_index + t_len :]
+
+
+def recursive_match_token(
+    full_sentence: str,
+    token_list: list[str],
+    token_list_mask_index: int,
+    skippable_tokens: list[str],
+) -> int:
+    # ensure we can retrieve another token
+    n_remaining_tokens = len(token_list)
+    if n_remaining_tokens == 0:
+        raise ValueError(
+            "Mask index not reached but token list has been iterated for sentence: {}".format(
+                full_sentence
+            )
+        )
+    t = token_list[0]
+
+    # returns the index of the first occurrence
+    # of the token t
+    match_index = full_sentence.find(t)
+    is_match_found = match_index != -1
+    has_reached_mask_token = token_list_mask_index == 0
+
+    # BASE CASE
+    if has_reached_mask_token and is_match_found:
+        # we're at the end
+        return match_index
+    # RECURSIVE CASE
+    elif is_match_found:
+        sliced_sentence = full_sentence[match_index + len(t) :]
+        token_list.pop(0)
+
+        return (
+            match_index
+            + len(t)
+            + recursive_match_token(
+                sliced_sentence,
+                token_list,
+                token_list_mask_index - 1,
+                skippable_tokens,
+            )
+        )
+    else:
+        # no match found, is t irrelevant?
+        if t in skippable_tokens:
+            # need to watch out with the slicing here
+            # tests are important
+            sliced_sentence = full_sentence[len(t) - 1 :]
+            token_list.pop(0)
+            return recursive_match_token(
+                sliced_sentence,
+                token_list,
+                token_list_mask_index - 1,
+                skippable_tokens,
+            )
+        else:
+            raise ValueError(
+                "Token not found in string nor has it been specified as skippable: {}".format(
+                    t
+                )
+            )

grewtse/utils/validation.py

@@ -0,0 +1,24 @@
+import pandas as pd
+
+
+def load_and_validate_mp_dataset(filepath: str):
+    required_columns = {
+        "sentence_id",
+        "match_id",
+        "original_text",
+    }
+
+    try:
+        df = pd.read_csv(filepath)
+
+        missing = required_columns - set(df.columns)
+        if missing:
+            raise ValueError(f"Missing required column(s): {', '.join(missing)}")
+
+        return df
+
+    except FileNotFoundError:
+        raise FileNotFoundError(f"File not found: {filepath}")
+
+    except pd.errors.ParserError as e:
+        raise pd.errors.ParserError(f"Error parsing CSV file: {e}")

grewtse/visualise/__init__.py

@@ -0,0 +1,3 @@
+from .visualiser import GrewTSEVisualiser
+
+__all__ = ["GrewTSEVisualiser"]

grewtse/visualise/visualiser.py

@@ -0,0 +1,132 @@
+import pandas as pd
+from plotnine import (
+    labs,
+    theme,
+    theme_bw,
+    guides,
+    position_nudge,
+    aes,
+    geom_violin,
+    geom_line,
+    geom_jitter,
+    scale_x_discrete,
+    ggplot,
+)
+from pathlib import Path
+import math
+
+
+class GrewTSEVisualiser:
+    """
+    A basic visualisation class that creates a violin plot based on a syntactic evaluation.
+    """
+
+    def __init__(self) -> None:
+        self.data = None
+
+    def visualise_syntactic_performance(
+        self,
+        results: pd.DataFrame,
+        title: str,
+        target_x_label: str,
+        alt_x_label: str,
+        x_axis_label: str,
+        y_axis_label: str,
+        filename: str,
+    ) -> None:
+        """
+        Visualise a syntactic performance evaluation result.
+
+        :param results: pass the results DataFrame created by the GrewTSEEvaluator.
+        :param title: Give the diagram a main title.
+        :param target_x_label: Give the original target word and hence first word in the minimal pair a label e.g. 'Accusative'.
+        :param alt_x_label: Give the second element in the minimal pair a label e.g. 'Dative'.
+        :param x_axis_label: Give the X Axis a title.
+        :param y_axis_label: Give the Y Axis a title.
+        :param filename: A filename to save the visualisation.
+        :return:
+        """
+
+        visualise_slope(
+            filename,
+            results,
+            target_x_label,
+            alt_x_label,
+            x_axis_label,
+            y_axis_label,
+            title,
+        )
+
+
+def visualise_slope(
+    path: Path,
+    results: pd.DataFrame,
+    target_x_label: str,
+    alt_x_label: str,
+    x_axis_label: str,
+    y_axis_label: str,
+    title: str,
+):
+    lsize = 0.65
+    fill_alpha = 0.7
+
+    # X-axis: Acc, Gen
+    # Y-axis: surprisal
+    filtered_df = results[
+        results["form_ungrammatical"].notna()
+        & (results["form_ungrammatical"].str.strip() != "")
+    ]
+
+    filtered_df["subject_id"] = filtered_df.index
+
+    # Melt the dataframe
+    df_long = pd.melt(
+        filtered_df,
+        id_vars=["subject_id"],
+        value_vars=["p_grammatical", "p_ungrammatical"],
+        var_name="source",
+        value_name="log_prob",
+    )
+
+    # Map source to fixed x-axis labels
+    df_long["x_label"] = df_long["source"].map(
+        {"p_grammatical": target_x_label, "p_ungrammatical": alt_x_label}
+    )
+
+    def surprisal(p: float) -> float:
+        return -math.log2(p)
+
+    def confidence(p: float) -> float:
+        return math.log2(p)
+
+    df_long["surprisal"] = df_long["log_prob"].apply(surprisal)
+
+    p = (
+        ggplot(df_long, aes(x="x_label", y="surprisal", fill="x_label"))
+        + scale_x_discrete(limits=[target_x_label, alt_x_label])
+        + geom_jitter(aes(color="x_label"), width=0.01, alpha=0.7)
+        +
+        # geom_text(aes(label='label'), nudge_y=0.1) +
+        geom_line(aes(group="subject_id"), color="gray", alpha=0.7, size=0.2)
+        + geom_violin(
+            df_long[df_long["x_label"] == target_x_label],
+            aes(x="x_label", y="surprisal", group="x_label"),
+            position=position_nudge(x=-0.2),
+            style="left-right",
+            alpha=fill_alpha,
+            size=lsize,
+        )
+        + geom_violin(
+            df_long[df_long["x_label"] == alt_x_label],
+            aes(x="x_label", y="surprisal", group="x_label"),
+            position=position_nudge(x=0.2),
+            style="right-left",
+            alpha=fill_alpha,
+            size=lsize,
+        )
+        + guides(fill=False)
+        + theme_bw()
+        + theme(figure_size=(8, 4), legend_position="none")
+        + labs(x=x_axis_label, y=y_axis_label, title=title)
+    )
+    p.save(path, width=14, height=8, dpi=300)