add normalizer

app.py

@@ -1,6 +1,7 @@
 import streamlit as st
 from transformers import pipeline, set_seed
 from transformers import AutoTokenizer
+from normalizer import Normalizer
 import random
 
 import meta
@@ -120,12 +121,13 @@ def main():
 
     text = st.text_area("Enter text", prompt_box)
     generation_kwargs_ph = st.empty()
-
+    cleaner = Normalizer()
     if st.button("Generate !"):
         with st.spinner(text="Generating ..."):
             generation_kwargs_ph.markdown(", ".join([f"`{k}`: {v}" for k, v in generation_kwargs.items()]))
             if text:
                 generated_text = generator.generate(text, generation_kwargs)
+                generated_text = cleaner.clean_txt(generated_text)
                 st.markdown(
                     f'<p class="ltr ltr-box">'
                     f'<span class="result-text">{text} <span>'

normalizer.py

@@ -0,0 +1,17 @@
+class Normalizer:
+
+    def __init__(self):
+        pass
+
+    def remove_repetitions(self, text):
+        first_ocurrences = []
+        for sentence in text.split("."):
+            if sentence not in first_ocurrences:
+                first_ocurrences.append(sentence)
+        return '.'.join(first_ocurrences)
+
+    def trim_last_sentence(self, text):
+        return text[:text.rfind(".")+1]
+            
+    def clean_txt(self, text):
+        return self.trim_last_sentence(self.remove_repetitions(text))