Merge pull request #198 from edenai/EAISCRUM-1458-privateai-text-anonymization-pii

[Feat] PrivateAI text PII

Author: Daggx

edenai_apis/apis/privateai/info.json

@@ -35,5 +35,66 @@
       },
       "version": "v3"
     }
+  },
+  "text": {
+    "anonymization": {
+      "constraints": {
+        "languages": [
+          "en",
+          "nl",
+          "fr",
+          "de",
+          "hi",
+          "it",
+          "ja",
+          "ko",
+          "pt",
+          "zh-Hans",
+          "ru",
+          "es",
+          "tl",
+          "uk",
+          "af",
+          "ar",
+          "bm",
+          "bn",
+          "be",
+          "bg",
+          "my",
+          "zh-Hant",
+          "ca",
+          "hr",
+          "cs",
+          "da",
+          "et",
+          "fi",
+          "ka",
+          "el",
+          "he",
+          "hu",
+          "is",
+          "id",
+          "km",
+          "lv",
+          "lt",
+          "lb",
+          "ms",
+          "ro",
+          "nb",
+          "fa",
+          "pl",
+          "pa",
+          "sk",
+          "sl",
+          "sw",
+          "sv",
+          "ta",
+          "th",
+          "tr",
+          "vi"
+        ]
+      },
+      "version": "v3"
+    }
   }
 }

edenai_apis/apis/privateai/outputs/text/anonymization_output.json

@@ -0,0 +1,66 @@
+{
+  "original_response": [
+    {
+      "processed_text": "The phone number of [NAME_GIVEN_1] is the [PHONE_NUMBER_1].",
+      "entities": [
+        {
+          "processed_text": "NAME_GIVEN_1",
+          "text": "Luc",
+          "location": {
+            "stt_idx": 20,
+            "end_idx": 23,
+            "stt_idx_processed": 20,
+            "end_idx_processed": 34
+          },
+          "best_label": "NAME_GIVEN",
+          "labels": {
+            "NAME_GIVEN": 0.9117,
+            "NAME": 0.9053
+          }
+        },
+        {
+          "processed_text": "PHONE_NUMBER_1",
+          "text": "06 21 32 43 54",
+          "location": {
+            "stt_idx": 31,
+            "end_idx": 45,
+            "stt_idx_processed": 42,
+            "end_idx_processed": 58
+          },
+          "best_label": "PHONE_NUMBER",
+          "labels": {
+            "PHONE_NUMBER": 0.9037
+          }
+        }
+      ],
+      "entities_present": true,
+      "characters_processed": 46,
+      "languages_detected": {
+        "en": 0.7309150695800781
+      }
+    }
+  ],
+  "standardized_response": {
+    "result": "The phone number of *** is the **************.",
+    "entities": [
+      {
+        "offset": 20,
+        "length": 3,
+        "category": "PersonalInformation",
+        "subcategory": null,
+        "original_label": "NAME_GIVEN",
+        "content": "Luc",
+        "confidence_score": 0.912
+      },
+      {
+        "offset": 31,
+        "length": 14,
+        "category": "PersonalInformation",
+        "subcategory": null,
+        "original_label": "PHONE_NUMBER",
+        "content": "06 21 32 43 54",
+        "confidence_score": 0.904
+      }
+    ]
+  }
+}

edenai_apis/apis/privateai/privateai_api.py

@@ -2,15 +2,20 @@
 import json
 import mimetypes
 import uuid
-from typing import Dict
+from typing import Dict, List
 from io import BytesIO
 
 import requests
 
-from edenai_apis.features import ProviderInterface, OcrInterface
+from edenai_apis.features import ProviderInterface, OcrInterface, TextInterface
 from edenai_apis.features.ocr.anonymization_async.anonymization_async_dataclass import (
     AnonymizationAsyncDataClass,
 )
+from edenai_apis.features.text.anonymization.anonymization_dataclass import (
+    AnonymizationDataClass,
+    AnonymizationEntity,
+)
+from edenai_apis.features.text.anonymization.category import CategoryType
 from edenai_apis.loaders.data_loader import ProviderDataEnum
 from edenai_apis.loaders.loaders import load_provider
 from apis.amazon.helpers import check_webhook_result
@@ -20,11 +25,13 @@
     AsyncLaunchJobResponseType,
     AsyncPendingResponseType,
     AsyncResponseType,
+    ResponseType,
 )
 from edenai_apis.utils.upload_s3 import upload_file_bytes_to_s3, USER_PROCESS
+from edenai_apis.utils.parsing import extract
 
 
-class PrivateaiApi(ProviderInterface, OcrInterface):
+class PrivateaiApi(ProviderInterface, OcrInterface, TextInterface):
     provider_name = "privateai"
 
     def __init__(self, api_keys: Dict = {}) -> None:
@@ -131,3 +138,52 @@ def ocr__anonymization_async__get_job_result(
             ),
             provider_job_id=provider_job_id,
         )
+
+    def text__anonymization(
+        self, text: str, language: str
+    ) -> ResponseType[AnonymizationDataClass]:
+        payload = {
+            "text": [text],
+            "entity_detection": {"accuracy": "high", "return_entity": True},
+        }
+        response = requests.post(
+            url=self.url + "v3/process/text", json=payload, headers=self.headers
+        )
+
+        if response.status_code != 200:
+            raise ProviderException(message=response.text, code=response.status_code)
+
+        try:
+            original_response = response.json()
+        except json.JSONDecodeError as exc:
+            raise ProviderException(message="Internal server error", code=500) from exc
+
+        entities: List[AnonymizationEntity] = []
+        new_text = text
+        for entity in original_response[0].get("entities", []):
+            start_index = extract(entity, ["location", "stt_idx"])
+            end_index = extract(entity, ["location", "end_idx"])
+            original_label = entity.get("best_label")
+            confidence = extract(entity, ["labels", original_label])
+            classification = CategoryType.choose_category_subcategory(original_label)
+            replacement = "*" * (end_index - start_index)
+            new_text = new_text[:start_index] + replacement + new_text[end_index:]
+            entities.append(
+                AnonymizationEntity(
+                    offset=start_index,
+                    length=end_index - start_index,
+                    category=classification["category"],
+                    subcategory=classification["subcategory"],
+                    original_label=original_label,
+                    content=entity.get("text"),
+                    confidence_score=confidence,
+                )
+            )
+
+        standardized_response = AnonymizationDataClass(
+            result=new_text, entities=entities
+        )
+        return ResponseType[AnonymizationDataClass](
+            original_response=original_response,
+            standardized_response=standardized_response,
+        )

edenai_apis/features/text/anonymization/pattern.py

@@ -23,11 +23,12 @@ class PersonnalInformation:
             PERSON_TYPE (list): List of all the patterns for the person type subcategory.
         """
 
-        NAME = ["name", "username", "person", "human"]
+        NAME = ["name", "username", "person", "human", "name_given", "name_family"]
         AGE = ["age"]
-        EMAIL = ["email"]
-        PHONE = ["phone", "phonenumber", "phone number"]
+        EMAIL = ["email", "email_address"]
+        PHONE = ["phone", "phonenumber", "phone number", "phone_number"]
         PERSON_TYPE = ["persontype"]
+        GENDER = ["gender_sexuality"]
 
     class FinancialInformation:
         """This class contains all the patterns for the subcategories of the anonymization category.
@@ -52,6 +53,7 @@ class FinancialInformation:
             "bank account number",
             "bank_account_number",
             "international_bank_account_number",
+            "account_number",
         ]
         BANK_ROUTING_NUMBER = ["bank_routing", "aba_routing_number", "iban"]
         SWIFT_CODE = ["swift code", "swift_code"]
@@ -172,10 +174,10 @@ class DateAndTime:
             DATE (list): List of all the patterns for the date subcategory.
         """
 
-        DURATION = ["duration", "timeduration"]
+        DURATION = ["duration", "timeduration", "date_interval"]
         DATE_TIME = ["date_time", "datetime"]
         TIME = ["time"]
-        DATE = ["date"]
+        DATE = ["date", "dob"]
 
     class LocationInformation:
         """This class contains all the patterns for the subcategories of the anonymization category.
@@ -185,8 +187,15 @@ class LocationInformation:
             ADDRESS (list): List of all the patterns for the address subcategory.
         """
 
-        ADDRESS = ["address"]
-        LOCATION = ["location"]
+        ADDRESS = ["address", "LOCATION_ADDRESS"]
+        LOCATION = [
+            "location",
+            "LOCATION_CITY",
+            "LOCATION_COORDINATE",
+            "LOCATION_COUNTRY",
+            "LOCATION_STATE",
+            "LOCATION_ZIP",
+        ]
 
     class Other:
         """This class contains all the patterns for the subcategories of the anonymization category.

edenai_apis/features/text/anonymization/subcategory.py

@@ -69,6 +69,7 @@ class PersonalInformationSubCategoryType(SubCategoryBase, Enum):
     Email = "Email"
     Phone = "Phone"
     PersonType = "PersonType"
+    Gender = "Gender"
 
     @classmethod
     def list_choices(cls) -> Dict["SubCategoryBase", List[str]]:
@@ -78,6 +79,7 @@ def list_choices(cls) -> Dict["SubCategoryBase", List[str]]:
             cls.Email: SubCategoryPattern.PersonnalInformation.EMAIL,
             cls.Phone: SubCategoryPattern.PersonnalInformation.PHONE,
             cls.PersonType: SubCategoryPattern.PersonnalInformation.PERSON_TYPE,
+            cls.Gender: SubCategoryPattern.PersonnalInformation.GENDER,
         }