Before branch dev

gradiobee/seg_text.py

@@ -0,0 +1,75 @@
+"""Split text to sentences.
+
+Use sentence_splitter if supported,
+else use polyglot.text.Text
+
+from hlm_texts
+
+!apt install libicu-dev
+!install pyicu pycld2 Morfessor
+!pip install polyglot sentence_splitter
+"""
+from typing import List, Optional
+
+from tqdm.auto import tqdm
+from polyglot.detect.base import logger as polyglot_logger
+from polyglot.text import Detector, Text
+from sentence_splitter import split_text_into_sentences
+
+from logzero import logger
+
+# turn of polyglot.text.Detector warning
+polyglot_logger.setLevel("ERROR")
+
+
+# fmt: off
+# use sentence_splitter if supported
+LANG_S = ["ca", "cs", "da", "nl", "en", "fi", "fr", "de",
+          "el", "hu", "is", "it", "lv", "lt", "no", "pl",
+          "pt", "ro", "ru", "sk", "sl", "es", "sv", "tr"]
+
+
+def seg_text(
+        text: str,
+        lang: Optional[str] = None,
+        qmode: bool = False,
+        maxlines: int = 1000
+) -> List[str]:
+    # fmt: on
+    """
+    Split text to sentences.
+
+    Use sentence_splitter if supported,
+    else use polyglot.text.Text.sentences
+
+    qmode: skip split_text_into_sentences if True, default False
+        vectors for all books are based on qmode=False.
+        qmode=True is for quick test purpose only
+
+    maxlines (default 1000), threhold for turn on tqdm progressbar
+        set to <1 or a large number to turn it off
+    """
+    if lang is None:
+        try:
+            lang = Detector(text).language.code
+        except Exception as exc:
+            logger.warning("polyglot.text.Detector exc: %s, setting to 'en'", exc)
+            lang = "en"
+
+    if not qmode and lang in LANG_S:
+        _ = []
+        lines = text.splitlines()
+        # if maxlines > 1 and len(lines) > maxlines:
+        if len(lines) > maxlines > 1:
+            for para in tqdm(lines):
+                if para.strip():
+                    _.extend(split_text_into_sentences(para, lang))
+        else:
+            for para in lines:
+                if para.strip():
+                    _.extend(split_text_into_sentences(para, lang))
+        return _
+
+        # return split_text_into_sentences(text, lang)
+
+    return [elm.string for elm in Text(text, lang).sentences]

requirements-save.txt

@@ -0,0 +1,11 @@
+chardet
+certifi
+charset-normalizer
+idna
+typing-extensions
+sklearn
+textacy
+logzero
+more_itertools
+psutil
+seaborn

requirements.in

@@ -0,0 +1,9 @@
+sklearn
+textacy
+logzero
+more_itertools
+psutil
+seaborn
+Morfessor
+# pyicu
+pycld2

requirements.txt

@@ -1,11 +1,182 @@
-chardet
-certifi
-charset-normalizer
-idna
-typing-extensions
-sklearn
-textacy
-logzero
-more_itertools
-psutil
-seaborn
+#
+# This file is autogenerated by pip-compile with python 3.7
+# To update, run:
+#
+#    pip-compile requirements.in
+#
+blis==0.7.5
+    # via
+    #   spacy
+    #   thinc
+cachetools==5.0.0
+    # via textacy
+catalogue==2.0.6
+    # via
+    #   spacy
+    #   srsly
+    #   thinc
+certifi==2021.10.8
+    # via requests
+charset-normalizer==2.0.9
+    # via requests
+click==8.0.3
+    # via typer
+colorama==0.4.4
+    # via
+    #   click
+    #   logzero
+    #   tqdm
+cycler==0.11.0
+    # via matplotlib
+cymem==2.0.6
+    # via
+    #   preshed
+    #   spacy
+    #   thinc
+cytoolz==0.11.2
+    # via textacy
+fonttools==4.28.5
+    # via matplotlib
+idna==3.3
+    # via requests
+importlib-metadata==4.10.0
+    # via click
+jellyfish==0.8.9
+    # via textacy
+jinja2==3.0.3
+    # via spacy
+joblib==1.1.0
+    # via
+    #   scikit-learn
+    #   textacy
+kiwisolver==1.3.2
+    # via matplotlib
+langcodes==3.3.0
+    # via spacy
+logzero==1.7.0
+    # via -r requirements.in
+markupsafe==2.0.1
+    # via jinja2
+matplotlib==3.5.1
+    # via seaborn
+more-itertools==8.12.0
+    # via -r requirements.in
+morfessor==2.0.6
+    # via -r requirements.in
+murmurhash==1.0.6
+    # via
+    #   preshed
+    #   spacy
+    #   thinc
+networkx==2.6.3
+    # via textacy
+numpy==1.21.5
+    # via
+    #   blis
+    #   matplotlib
+    #   pandas
+    #   scikit-learn
+    #   scipy
+    #   seaborn
+    #   spacy
+    #   textacy
+    #   thinc
+packaging==21.3
+    # via
+    #   matplotlib
+    #   spacy
+pandas==1.3.5
+    # via seaborn
+pathy==0.6.1
+    # via spacy
+pillow==8.4.0
+    # via matplotlib
+preshed==3.0.6
+    # via
+    #   spacy
+    #   thinc
+psutil==5.8.0
+    # via -r requirements.in
+pycld2==0.41
+    # via -r requirements.in
+pydantic==1.8.2
+    # via
+    #   spacy
+    #   thinc
+pyparsing==3.0.6
+    # via
+    #   matplotlib
+    #   packaging
+pyphen==0.12.0
+    # via textacy
+python-dateutil==2.8.2
+    # via
+    #   matplotlib
+    #   pandas
+pytz==2021.3
+    # via pandas
+requests==2.26.0
+    # via
+    #   spacy
+    #   textacy
+scikit-learn==1.0.2
+    # via
+    #   sklearn
+    #   textacy
+scipy==1.7.3
+    # via
+    #   scikit-learn
+    #   seaborn
+    #   textacy
+seaborn==0.11.2
+    # via -r requirements.in
+six==1.16.0
+    # via python-dateutil
+sklearn==0.0
+    # via -r requirements.in
+smart-open==5.2.1
+    # via pathy
+spacy==3.2.1
+    # via textacy
+spacy-legacy==3.0.8
+    # via spacy
+spacy-loggers==1.0.1
+    # via spacy
+srsly==2.4.2
+    # via
+    #   spacy
+    #   thinc
+textacy==0.11.0
+    # via -r requirements.in
+thinc==8.0.13
+    # via spacy
+threadpoolctl==3.0.0
+    # via scikit-learn
+toolz==0.11.2
+    # via cytoolz
+tqdm==4.62.3
+    # via
+    #   spacy
+    #   textacy
+typer==0.4.0
+    # via
+    #   pathy
+    #   spacy
+typing-extensions==3.10.0.2
+    # via
+    #   catalogue
+    #   pydantic
+    #   spacy
+    #   thinc
+urllib3==1.26.7
+    # via requests
+wasabi==0.9.0
+    # via
+    #   spacy
+    #   spacy-loggers
+    #   thinc
+zipp==3.6.0
+    # via catalogue
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools