[#5] literal2idiomatic:d-1-3 done (annotating with special tokens). Some of the data however are erroneous

config.yaml

@@ -15,7 +15,9 @@ idioms:
   ver: d-1-2
   description: the set of idioms in the traning set of literal2idiomatic_d-1-2.
 literal2idiomatic:
-  ver: d-1-2
-  description: PIE data split into train & test set (80 / 20 split). There is no validation set because I don't intend to do any hyperparameter tuning on this thing.
+  ver: d-1-3
+  description: The idioms are annotated with <idiom> & </idiom>.
   train_ratio: 0.8
-  seed: 104
+  seed: 104
+  boi_token: <idiom>
+  eoi_token: </idiom>

explore/explore_fetch_pie_annotate.py

@@ -0,0 +1,14 @@
+
+from idiomify.fetchers import fetch_pie
+from preprocess import annotate
+
+
+def main():
+    pie_df = fetch_pie()
+    pie_df = pie_df.pipe(annotate, boi_token="<idiom>", eoi_token="</idiom>")
+    for _, row in pie_df.iterrows():
+        print(row['Idiomatic_Sent'])
+
+
+if __name__ == '__main__':
+    main()

explore/explore_list_index.py

@@ -0,0 +1,13 @@
+
+
+def main():
+    labels = ["O", "O", "B", "O", "I", "I" "O", "I", "O", "O"]
+    boi_idx = labels.index("B")
+    eoi_idx = -1 * (list(reversed(labels)).index("I") + 1)
+    print(boi_idx, eoi_idx)
+    print(labels[boi_idx])
+    print(labels[eoi_idx])
+
+
+if __name__ == '__main__':
+    main()

idiomify/preprocess.py

@@ -17,6 +17,36 @@ def cleanse(df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
+def annotate(df: pd.DataFrame, boi_token: str, eoi_token: str) -> pd.DataFrame:
+    """
+    e.g.
+    given a row like this:
+    Idiom                                                 keep an eye on
+    Sense                   keep a watch on something or someone closely
+    Idiomatic_Sent     He had put on a lot of weight lately , so he started keeping an eye on what he ate .
+    Literal_Sent       He had put on a lot of weight lately , so he started to watch what he ate .
+    Idiomatic_Label            O O O O O O O O O O O O O B I I O O O O O
+    Literal_Label                  O O O O O O O O O O O O O B I O O O O
+
+    use Idiomatic_Label to replace Idiomatic_Sent with:
+    He had put on a lot of weight lately , so he started <idiom> keeping an eye on </idiom> what he ate .
+    """
+    for idx, row in df.iterrows():
+        tokens = row['Idiomatic_Sent'].split(" ")
+        labels = row["Idiomatic_Label"].split(" ")
+        if "B" in labels:
+            boi_idx = labels.index("B")
+            if "I" in labels:
+                eoi_idx = -1 * (list(reversed(labels)).index("I") + 1)
+                tokens[boi_idx] = f"{boi_token} {tokens[boi_idx]}"
+                tokens[eoi_idx] = f"{tokens[eoi_idx]} {eoi_token}"
+            else:
+                tokens[boi_idx] = f"{boi_token} {tokens[boi_idx]} {eoi_token}"
+            row['Idiomatic_Sent'] = " ".join(tokens)
+
+    return df
+
+
 def stratified_split(df: pd.DataFrame, ratio: float, seed: int) -> Tuple[pd.DataFrame, pd.DataFrame]:
     """
     stratified-split the given df into two df's.
@@ -29,3 +59,4 @@ def stratified_split(df: pd.DataFrame, ratio: float, seed: int) -> Tuple[pd.Data
                                           test_size=other_size, random_state=seed,
                                           shuffle=True)
     return ratio_df, other_df
+

main_upload_literal2idiomatic.py

@@ -4,7 +4,7 @@ literal2idiomatic ver: d-1-2
 import os
 from idiomify.paths import ROOT_DIR
 from idiomify.fetchers import fetch_pie, fetch_config
-from idiomify.preprocess import upsample, cleanse, stratified_split
+from idiomify.preprocess import upsample, cleanse, stratified_split, annotate
 import wandb
 
 
@@ -15,6 +15,7 @@ def main():
     config = fetch_config()['literal2idiomatic']
     train_df, test_df = pie_df.pipe(cleanse)\
                               .pipe(upsample, seed=config['seed'])\
+                              .pipe(annotate, boi_token=config['boi_token'], eoi_token=config['eoi_token'])\
                               .pipe(stratified_split, ratio=config['train_ratio'], seed=config['seed'])
     # why don't you just "select"  the columns? yeah, stop using csv library. just select them.
     train_df = train_df[["Idiom", "Literal_Sent", "Idiomatic_Sent"]]