Add postprocessed text

app.py

@@ -13,6 +13,13 @@ import re,os
 from io import BytesIO
 from datetime import datetime
 
+def remove_in_betweens(text):
+    removed_brackets = re.sub(r'\[.*?\]', ' ', text)
+    removed_parentheses = re.sub(r'\(.*?\)', ' ', removed_brackets)
+    return removed_parentheses
+
+def remove_punctuations(text):
+    return re.sub(r"[\,\;\:\?\!\'\’\"\(\)\{\}\[\]\/\\\*]", '', text)
 
 def receive_signal(signalNumber, frame):
     print('Received:', signalNumber)
@@ -95,14 +102,6 @@ async def extract_arxiv_pdf(document: DocumentID):
         if ref_pos is not None:
             pdf_text = pdf_text[:ref_pos - 10]
 
-        def remove_in_betweens(text):
-            removed_brackets = re.sub(r'\[.*?\]', ' ', text)
-            removed_parentheses = re.sub(r'\(.*?\)', ' ', removed_brackets)
-            return removed_parentheses
-
-        def remove_punctuations(text):
-            return re.sub(r"[\,\;\:\?\!\'\’\"\(\)\{\}\[\]\/\\\*]", '', text)
-
         postprocess_text = remove_in_betweens(pdf_text)
         postprocess_text = remove_punctuations(postprocess_text)
         postprocess_text = re.sub(r"\s+", " ", postprocess_text)
@@ -137,7 +136,13 @@ async def extract_pdf(pdf: WebPDF):
         doc = fitz.open(stream=pdf_data, filetype="pdf")
         pdf_text = " ".join([page.get_text("text") for page in doc])
         pdf_metadata = doc.metadata
-        return {"error": False, "title": pdf_metadata.get("title", "").strip(), "text": pdf_text}
+        print(pdf_metadata)
+
+        postprocess_text = remove_in_betweens(pdf_text)
+        postprocess_text = remove_punctuations(postprocess_text)
+        postprocess_text = re.sub(r"\s+", " ", postprocess_text)
+        postprocess_text = postprocess_text.strip()
+        return {"error": False, "title": pdf_metadata.get("title", "").strip(), "text": postprocess_text}
     else:
         print("URL: " + pdf.url)
         print("Status code: " + str(pdf_req.status_code))