add new transformers model for french + update entities

README.md

@@ -1,6 +1,6 @@
 ---
-title: Anonymizer demo
-emoji: 👁
+title: Aliae Anonymizer
+emoji: 😻
 colorFrom: gray
 colorTo: gray
 sdk: streamlit
@@ -9,4 +9,4 @@ app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -56,7 +56,7 @@ st_ta_key = st_ta_endpoint = ""
 
 model_list = [
     "spaCy/en_core_web_lg",
-    "spaCy/fr_core_news_md",
+    "spaCy/fr_core_news_lg",
 ]
 # "flair/ner-english-large",
 #
@@ -78,7 +78,7 @@ lang = st.sidebar.selectbox(
 
 # Extract model package.
 # st_model_package = st_model.split("/")[0]
-st_model_package = 'spaCy'
+
 
 # # Remove package prefix (if needed)
 # st_model = (
@@ -87,8 +87,14 @@ st_model_package = 'spaCy'
 #     else "/".join(st_model.split("/")[1:])
 # )
 st_model = 'en_core_web_lg'
-if lang =='en': st_model = 'en_core_web_lg'
-elif lang == 'fr' : st_model = 'fr_core_news_md'
+st_model_package = "spaCy"
+
+if lang =='en':
+    st_model_package = "spaCy"
+    st_model = 'en_core_web_lg'
+elif lang == 'fr' :
+    st_model_package = "HuggingFace"
+    st_model = 'fr_core_news_lg'
 
 # if st_model == "Other":
 #     st_model_package = st.sidebar.selectbox(

presidio_helpers.py

@@ -24,7 +24,7 @@ from presidio_anonymizer.entities import OperatorConfig
 from presidio_nlp_engine_config import (
     create_nlp_engine_with_spacy,
     # create_nlp_engine_with_flair,
-    # create_nlp_engine_with_transformers,
+    create_nlp_engine_with_transformers,
     # create_nlp_engine_with_azure_text_analytics,
 )
 
@@ -99,6 +99,7 @@ def get_supported_entities(
     #     model_family, model_path, ta_key, ta_endpoint
     # ).get_supported_entities() + ["GENERIC_PII"]
     return  ["PERSON", "IBAN_CODE", "PHONE_NUMBER", "CREDIT_CARD", "CRYPTO", "DATE_TIME", "EMAIL_ADDRESS",  "IP_ADDRESS", "NRP", "LOCATION", "URL", "FRENCH_SSN", "FRENCH_PASS", "FRENCH_NID"]
+             #
 
 
 @st.cache_data

presidio_nlp_engine_config.py

@@ -3,6 +3,7 @@ import logging
 import spacy
 from presidio_analyzer import RecognizerRegistry
 from presidio_analyzer.nlp_engine import NlpEngine, NlpEngineProvider
+from transformers_class import TransformerRecognizer
 
 logger = logging.getLogger("presidio-streamlit")
 
@@ -34,108 +35,96 @@ def create_nlp_engine_with_spacy(
     return nlp_engine, registry
 
 
-# def create_nlp_engine_with_transformers(
-#     model_path: str,
-# ) -> Tuple[NlpEngine, RecognizerRegistry]:
-#     """
-#     Instantiate an NlpEngine with a TransformersRecognizer and a small spaCy model.
-#     The TransformersRecognizer would return results from Transformers models, the spaCy model
-#     would return NlpArtifacts such as POS and lemmas.
-#     :param model_path: HuggingFace model path.
-#     """
-#
-#     from transformers_rec import (
-#         STANFORD_COFIGURATION,
-#         BERT_DEID_CONFIGURATION,
-#         TransformersRecognizer,
-#     )
-#
-#     registry = RecognizerRegistry()
-#     registry.load_predefined_recognizers()
-#
-#     if not spacy.util.is_package("en_core_web_sm"):
-#         spacy.cli.download("en_core_web_sm")
-#     # Using a small spaCy model + a HF NER model
-#     transformers_recognizer = TransformersRecognizer(model_path=model_path)
-#
-#     if model_path == "StanfordAIMI/stanford-deidentifier-base":
-#         transformers_recognizer.load_transformer(**STANFORD_COFIGURATION)
-#     elif model_path == "obi/deid_roberta_i2b2":
-#         transformers_recognizer.load_transformer(**BERT_DEID_CONFIGURATION)
-#     else:
-#         print(f"Warning: Model has no configuration, loading default.")
-#         transformers_recognizer.load_transformer(**BERT_DEID_CONFIGURATION)
-#
-#     # Use small spaCy model, no need for both spacy and HF models
-#     # The transformers model is used here as a recognizer, not as an NlpEngine
-#     nlp_configuration = {
-#         "nlp_engine_name": "spacy",
-#         "models": [{"lang_code": "en", "model_name": "en_core_web_sm"}],
-#     }
-#
-#     registry.add_recognizer(transformers_recognizer)
-#     registry.remove_recognizer("SpacyRecognizer")
-#
-#     nlp_engine = NlpEngineProvider(nlp_configuration=nlp_configuration).create_engine()
-#
-#     return nlp_engine, registry
-
-
-# def create_nlp_engine_with_flair(
-#     model_path: str,
-# ) -> Tuple[NlpEngine, RecognizerRegistry]:
-#     """
-#     Instantiate an NlpEngine with a FlairRecognizer and a small spaCy model.
-#     The FlairRecognizer would return results from Flair models, the spaCy model
-#     would return NlpArtifacts such as POS and lemmas.
-#     :param model_path: Flair model path.
-#     """
-#     from flair_recognizer import FlairRecognizer
-#
-#     registry = RecognizerRegistry()
-#     registry.load_predefined_recognizers()
-#
-#     if not spacy.util.is_package("en_core_web_sm"):
-#         spacy.cli.download("en_core_web_sm")
-#     # Using a small spaCy model + a Flair NER model
-#     flair_recognizer = FlairRecognizer(model_path=model_path)
-#     nlp_configuration = {
-#         "nlp_engine_name": "spacy",
-#         "models": [{"lang_code": "en", "model_name": "en_core_web_sm"}],
-#     }
-#     registry.add_recognizer(flair_recognizer)
-#     registry.remove_recognizer("SpacyRecognizer")
-#
-#     nlp_engine = NlpEngineProvider(nlp_configuration=nlp_configuration).create_engine()
-#
-#     return nlp_engine, registry
-
-
-# def create_nlp_engine_with_azure_text_analytics(ta_key: str, ta_endpoint: str):
-#     """
-#     Instantiate an NlpEngine with a TextAnalyticsWrapper and a small spaCy model.
-#     The TextAnalyticsWrapper would return results from calling Azure Text Analytics PII, the spaCy model
-#     would return NlpArtifacts such as POS and lemmas.
-#     :param ta_key: Azure Text Analytics key.
-#     :param ta_endpoint: Azure Text Analytics endpoint.
-#     """
-#     from text_analytics_wrapper import TextAnalyticsWrapper
-#
-#     if not ta_key or not ta_endpoint:
-#         raise RuntimeError("Please fill in the Text Analytics endpoint details")
-#
-#     registry = RecognizerRegistry()
-#     registry.load_predefined_recognizers()
-#
-#     ta_recognizer = TextAnalyticsWrapper(ta_endpoint=ta_endpoint, ta_key=ta_key)
-#     nlp_configuration = {
-#         "nlp_engine_name": "spacy",
-#         "models": [{"lang_code": "en", "model_name": "en_core_web_sm"}],
-#     }
-#
-#     nlp_engine = NlpEngineProvider(nlp_configuration=nlp_configuration).create_engine()
-#
-#     registry.add_recognizer(ta_recognizer)
-#     registry.remove_recognizer("SpacyRecognizer")
-#
-#     return nlp_engine, registry
+def create_nlp_engine_with_transformers(
+    model_path: str,
+) -> Tuple[NlpEngine, RecognizerRegistry]:
+    """
+    Instantiate an NlpEngine with a TransformersRecognizer and a small spaCy model.
+    The TransformersRecognizer would return results from Transformers models, the spaCy model
+    would return NlpArtifacts such as POS and lemmas.
+    :param model_path: HuggingFace model path.
+    """
+
+
+
+
+    # if not spacy.util.is_package("en_core_web_sm"):
+    #     spacy.cli.download("en_core_web_sm")
+    # # Using a small spaCy model + a HF NER model
+    # transformers_recognizer = TransformersRecognizer(model_path=model_path)
+    #
+    # if model_path == "StanfordAIMI/stanford-deidentifier-base":
+    #     transformers_recognizer.load_transformer(**STANFORD_COFIGURATION)
+    # elif model_path == "obi/deid_roberta_i2b2":
+    #     transformers_recognizer.load_transformer(**BERT_DEID_CONFIGURATION)
+    # else:
+    #     print(f"Warning: Model has no configuration, loading default.")
+    #     transformers_recognizer.load_transformer(**BERT_DEID_CONFIGURATION)
+
+    # Use small spaCy model, no need for both spacy and HF models
+    # The transformers model is used here as a recognizer, not as an NlpEngine
+    if not spacy.util.is_package(model_path):
+        spacy.cli.download(model_path)
+
+    nlp_configuration = {
+        "nlp_engine_name": "spacy",
+        "models": [{"lang_code": model_path.split('_')[0], "model_name": model_path}],
+    }
+
+    nlp_engine = NlpEngineProvider(nlp_configuration=nlp_configuration).create_engine()
+    registry = RecognizerRegistry()
+    registry = load_predefined_recognizers(registry)
+
+    mapping_labels = {"PER": "PERSON", 'LOC': 'LOCATION'}
+    model_name = "AliaeAI/camembert_anonymizer_production_v2"  # "Jean-Baptiste/camembert-ner" , "AliaeAI/camembert_anonymizer_production"
+    transformers_recognizer = TransformerRecognizer(model_name, mapping_labels)
+
+    registry.add_recognizer(transformers_recognizer)
+    registry.remove_recognizer("SpacyRecognizer")
+
+
+
+    return nlp_engine, registry
+
+
+
+from presidio_analyzer.predefined_recognizers import PhoneRecognizer, EmailRecognizer, CreditCardRecognizer, CryptoRecognizer, DateRecognizer, IpRecognizer, IbanRecognizer, UrlRecognizer
+import phonenumbers
+
+def load_predefined_recognizers(registry, lang='fr'):
+    # phone number
+    phone_recognizer_fr = PhoneRecognizer(supported_language=lang, supported_regions=phonenumbers.SUPPORTED_REGIONS,context=['téléphone'])
+    registry.add_recognizer(phone_recognizer_fr)
+
+    # email
+    email_recognizer_fr = EmailRecognizer(supported_language=lang, context=["email", "mail", "e-mail"])
+    registry.add_recognizer(email_recognizer_fr)
+
+    # credit card
+    creditcard_recognizer_fr = CreditCardRecognizer(supported_language=lang,context=["crédit", "carte", "carte de crédit"])
+    registry.add_recognizer(creditcard_recognizer_fr)
+
+    # crypto
+    crypto_recognizer_fr = CryptoRecognizer(supported_language=lang, context=["crypto"])
+    registry.add_recognizer(crypto_recognizer_fr)
+
+    # date time
+    date_recognizer_fr = DateRecognizer(supported_language=lang, context=["mois", "date", "jour", "année"])
+    registry.add_recognizer(date_recognizer_fr)
+
+    # ip address
+    ip_recognizer_fr = IpRecognizer(supported_language=lang, context=["IP", "ip"])
+    registry.add_recognizer(ip_recognizer_fr)
+
+    # iban
+    iban_recognizer_fr = IbanRecognizer(supported_language=lang, context = ["IBAN", "iban", "bancaire", "compte"])
+    registry.add_recognizer(iban_recognizer_fr)
+
+    # URL
+    url_recognizer_fr = UrlRecognizer(supported_language=lang, context = ["site", "web"])
+    registry.add_recognizer(url_recognizer_fr)
+
+    # load from yaml
+    registry.add_recognizers_from_yaml("recognizers.yaml")
+
+    return registry

recognizers.yaml

@@ -1,15 +1,15 @@
 recognizers:
-  -
-    name: "FRENCH_NID"
-    supported_language: "fr"
-    patterns:
-      -
-         name: "FRENCH_NID"
-         regex: "[0-9]{12}|([A-Z]|[0-9]){9}"
-         score: 0.5
-    context:
-      - national
-    supported_entity: "FRENCH_NID"
+#  -
+#    name: "FRENCH_NID"
+#    supported_language: "fr"
+#    patterns:
+#      -
+#         name: "FRENCH_NID"
+#         regex: "[0-9]{12}|([A-Z]|[0-9]){9}"
+#         score: 0.5
+#    context:
+#      - national
+#    supported_entity: "FRENCH_NID"
   -
     name: "FRENCH_NID"
     supported_language: "en"

requirements.txt

@@ -7,7 +7,5 @@ python-dotenv
 st-annotated-text
 torch
 transformers
-flair
-openai
 spacy
 azure-ai-textanalytics

transformers_class.py

@@ -0,0 +1,52 @@
+from transformers import pipeline
+from presidio_analyzer import (
+    RecognizerResult,
+    EntityRecognizer,
+    AnalysisExplanation,
+)
+from presidio_analyzer.nlp_engine import NlpEngine, SpacyNlpEngine, NlpArtifacts
+
+class TransformerRecognizer(EntityRecognizer):
+    def __init__(
+        self,
+        model_id_or_path,
+        mapping_labels,
+        aggregation_strategy="simple",
+        supported_language="fr",
+        ignore_labels=["O", "MISC"],
+    ):
+        # inits transformers pipeline for given mode or path
+        self.pipeline = pipeline(
+            "token-classification", model=model_id_or_path, aggregation_strategy=aggregation_strategy, ignore_labels=ignore_labels
+        )
+        # map labels to presidio labels
+        self.label2presidio = mapping_labels
+
+        # passes entities from model into parent class
+        super().__init__(supported_entities=list(self.label2presidio.values()), supported_language=supported_language)
+
+    def load(self) -> None:
+        """No loading is required."""
+        pass
+
+    def analyze(
+        self, text: str, entities = None, nlp_artifacts: NlpArtifacts = None
+    ):
+        """
+        Extracts entities using Transformers pipeline
+        """
+        results = []
+
+        predicted_entities = self.pipeline(text)
+        if len(predicted_entities) > 0:
+            for e in predicted_entities:
+                if(e['entity_group'] not in self.label2presidio):
+                    continue
+                converted_entity = self.label2presidio[e["entity_group"]]
+                if converted_entity in entities or entities is None:
+                    results.append(
+                        RecognizerResult(
+                            entity_type=converted_entity, start=e["start"], end=e["end"], score=e["score"]
+                        )
+                    )
+        return results