From f7973577049c3e6f60633f39017f0e8a8e333708 Mon Sep 17 00:00:00 2001 From: ariannevjestic Date: Thu, 1 Aug 2024 22:24:49 +0800 Subject: [PATCH 1/2] adding in openai pii stripping support --- aisentry/facade/app.py | 14 +-- aisentry/utils/analyze_pii.py | 15 +-- aisentry/utils/analyze_pii_openai.py | 91 +++++++++++++++++++ .../worker/cosmos_logger/cosmos_logger.py | 41 +++++++-- aisentry/worker/requirements.txt | 1 + build/build-ai-sentry-containers.ps1 | 4 +- .../AI-Sentry-config-settings.md | 4 +- content/documentation/AKSDeployment.md | 3 +- .../documentation/Workload-identity-config.md | 7 +- content/documentation/ai-sentry-config.json | 8 +- infrastructure/APIM/ai-sentry-policy.xml | 32 +++++-- 11 files changed, 179 insertions(+), 41 deletions(-) create mode 100644 aisentry/utils/analyze_pii_openai.py diff --git a/aisentry/facade/app.py b/aisentry/facade/app.py index bead1bb..a2e7491 100644 --- a/aisentry/facade/app.py +++ b/aisentry/facade/app.py @@ -35,12 +35,6 @@ load_dotenv(".env", override=True) - -# os.environ["AZURE_CLIENT_ID"] = "your_client_id" -# os.environ["AZURE_TENANT_ID"] = "your_tenant_id" -# os.environ["AZURE_FEDERATED_TOKEN_FILE"] = "/var/run/secrets/tokens/azure-identity-token" - - logger.info("Starting Ai-Sentry Facade app") app = Quart(__name__) @@ -132,10 +126,10 @@ async def catch_all(path): client = endpoint_info["client"] - if openAI_request_headers.get('Api-Key') is not None: - logger.info("detected use of api-key header - will use this for authentication") - logger.debug(f"Swapping out api-key inside header with {endpoint_info['api-key']} value") - openAI_request_headers['Api-Key'] = endpoint_info['api-key'] + # if openAI_request_headers.get('Api-Key') is not None: + # logger.info("detected use of api-key header - will use this for authentication") + # logger.debug(f"Swapping out api-key inside header with {endpoint_info['api-key']} value") + # openAI_request_headers['Api-Key'] = endpoint_info['api-key'] if endpoint_info['api-key'] is not None: logger.info("No api-key header detected - will use the default api-key for authentication") diff --git a/aisentry/utils/analyze_pii.py b/aisentry/utils/analyze_pii.py index 25e23f6..06efa23 100644 --- a/aisentry/utils/analyze_pii.py +++ b/aisentry/utils/analyze_pii.py @@ -58,8 +58,6 @@ async def analyze_pii_async(input_text: List[str]) -> None: async for page in pages: document_results.append(page) - # doc="" - for doc, action_results in zip(chunk, document_results): for result in action_results: @@ -71,21 +69,18 @@ async def analyze_pii_async(input_text: List[str]) -> None: logger.debug(f".........Confidence Score: {pii_entity.confidence_score}") if pii_entity.confidence_score >= 0.8 and pii_entity.category != "DateTime": logger.debug(f"Removing PII entity: {pii_entity.text}, category: {pii_entity.category} from the logged payload") - # if pii_entity.text in "\},]": - # doc = doc.replace(pii_entity.text, "*PII*\"},") - # logger.info(f"PII-Processing: Replacing PII entity: {pii_entity.text} with extra escaping") - doc = doc.replace(pii_entity.text, "*PII*") + doc = doc.replace(pii_entity.text, "PII_REDACTED") elif result.is_error is True: logger.error(f'PII-Processing: An error with code {result.error.code} and message {result.error.message}') #UNTOCHED - if ": *PII*," in doc: - doc = doc.replace(": *PII*,", ":\"*PII*\",") + if ": PII_REDACTED," in doc: + doc = doc.replace(": PII_REDACTED,", ":\"PII_REDACTED\",") - if "*PII*," in doc: - doc = doc.replace("*PII*,", "*PII*\"") + if "PII_REDACTED," in doc: + doc = doc.replace("PII_REDACTED,", "PII_REDACTED\"") logger.info(f"PII stripping completed") return doc diff --git a/aisentry/utils/analyze_pii_openai.py b/aisentry/utils/analyze_pii_openai.py new file mode 100644 index 0000000..c2a424a --- /dev/null +++ b/aisentry/utils/analyze_pii_openai.py @@ -0,0 +1,91 @@ +import os +import logging +import asyncio +from dotenv import load_dotenv +from openai import AsyncAzureOpenAI + + + +# initial setup for logging / env variable loading +log_level = os.getenv('LOG-LEVEL', 'INFO').upper() + +# Set up the logger +logger = logging.getLogger(__name__) +logging.basicConfig(level=getattr(logging, log_level), + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%d-%m-%Y %H:%M:%S' + ) + +# Initialize the OpenAI client with your key and endpoint + +load_dotenv(".env", override=True) + +openai_key = os.environ.get("PII_STRIPPING_OPENAI_API_KEY") +openai_endpoint = os.environ.get("PII_STRIPPING_OPENAI_ENDPOINT") + + +client = AsyncAzureOpenAI( + api_key=openai_key, + api_version="2023-12-01-preview", + azure_endpoint=openai_endpoint +) + +pii_stripping_system_prompt = """Objective: Identify and flag any Personally Identifiable Information (PII) within text data to ensure data privacy and compliance with regulations such as GDPR, CCPA, etc. + +PII includes but is not limited to: +Full Names: First and last names +Addresses: Street address, city, state, zip code +Phone Numbers: Any format of telephone numbers +Email Addresses: Any format of email addresses +Social Security Numbers (SSNs): XXX-XX-XXXX or similar formats +Credit Card Numbers: Any format of credit/debit card numbers +Bank Account Numbers: Any format of bank account numbers +Driver's License Numbers: Any format of driver's license numbers +Passport Numbers: Any format of passport numbers +Date of Birth: Full date of birth (MM/DD/YYYY or similar formats) +IP Addresses: Any format of IPv4 or IPv6 addresses +API-KEY or Token: Any format of API keys or tokens +Medical Information: Any health-related information that can identify an individual +Biometric Data: Fingerprints, facial recognition data, etc. + +Instructions for the System: +Input: Accept text data for analysis. +Processing: +Use pattern matching, regular expressions, and machine learning algorithms to identify potential PII. +Cross-reference detected patterns with known PII formats. +Output: +Flag detected PII and categorize it. +Provide a confidence score for each detected PII item. +Highlight the specific text containing PII. + +Example: + +Input Text: + +John Doe lives at 123 Maple Street, Springfield, IL 62704. His email is john.doe@example.com, and his phone number is (555) 123-4567. He was born on 01/15/1985 and his SSN is 123-45-6789. + +Output: + +Keep the same text structure but replace the PII with placeholders: [PII-Redacted] + +Compliance Note: The system must handle all detected PII with strict confidentiality and in accordance with applicable data protection regulations.""" + + +async def get_chat_pii_stripped_completion(prompt): + # Send the request to Azure OpenAI + response = await client.chat.completions.create( + model="gpt-4", + messages=[ + {"role": "system", "content": pii_stripping_system_prompt}, + {"role": "user", "content": f"Rewrite the input and Strip out PII information as per the system message from following input: {prompt}"} + ] + ) + + # Extract the text from the response + #completion_text = response.completions[0].data.get("text", "") + message_content = response['choices'][0]['message']['content'] + + logger.info(f"PII Stripped Completion Text: {message_content}") + return message_content + + diff --git a/aisentry/worker/cosmos_logger/cosmos_logger.py b/aisentry/worker/cosmos_logger/cosmos_logger.py index 1add488..16e0d2f 100644 --- a/aisentry/worker/cosmos_logger/cosmos_logger.py +++ b/aisentry/worker/cosmos_logger/cosmos_logger.py @@ -1,10 +1,10 @@ from flask import Flask, request, jsonify from cloudevents.http import from_http from requests.exceptions import HTTPError -from azure.identity import DefaultAzureCredential from azure.core.exceptions import AzureError from dapr.clients import DaprClient from utils.analyze_pii import analyze_pii_async +from utils.analyze_pii_openai import get_chat_pii_stripped_completion import logging import datetime import asyncio @@ -16,6 +16,9 @@ import uuid from typing import List + +load_dotenv(".env", override=True) + # Get log level from environment variable log_level = os.getenv('LOG-LEVEL', 'INFO').upper() logger = logging.getLogger(__name__) @@ -29,10 +32,11 @@ -load_dotenv(".env", override=True) - app_port = os.getenv('APP_PORT', '7000') +# This can be either OPENAI or TEXTANALYTICS +pii_stripping_service = os.getenv('PII_STRIPPING_SERVICE', 'OPENAI') + # Register Dapr pub/sub subscriptions @app.route('/dapr/subscribe', methods=['GET']) def subscribe(): @@ -82,9 +86,34 @@ async def oairequests_subscriber(): input_data: List[str] = [] input_data.append(output_binding_data) - output_binding_data = await analyze_pii_async(input_data) - logger.debug(f"PII stripped data: {output_binding_data}") - + if pii_stripping_service == 'TEXTANALYTICS': + logger.debug(f"PII stripping service: {pii_stripping_service}") + output_binding_data = await analyze_pii_async(input_data) + + #OPENAI BASED PII Stripping + else: + logger.debug(f"PII stripping service: {pii_stripping_service}") + + # Ensure a new event loop is created if the current one is closed + try: + loop = asyncio.get_event_loop() + if loop.is_closed(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + if loop.is_running(): + task = loop.create_task(get_chat_pii_stripped_completion(input_data)) + result = loop.run_until_complete(task) + else: + result = loop.run_until_complete(get_chat_pii_stripped_completion(input_data)) + + print(result) + # output_binding_data = await get_chat_pii_stripped_completion(input_data) + + logger.debug(f"PII stripped data: {output_binding_data}") elif headers['ai-sentry-log-level'] == 'COMPLETE': diff --git a/aisentry/worker/requirements.txt b/aisentry/worker/requirements.txt index 42f67bc..d61dd17 100644 --- a/aisentry/worker/requirements.txt +++ b/aisentry/worker/requirements.txt @@ -46,6 +46,7 @@ msal==1.28.0 msal-extensions==1.1.0 multidict==6.0.5 openai==1.14.2 +nest_asyncio==1.6.0 packaging==24.0 portalocker==2.8.2 priority==2.0.0 diff --git a/build/build-ai-sentry-containers.ps1 b/build/build-ai-sentry-containers.ps1 index f899bd3..5ec55de 100644 --- a/build/build-ai-sentry-containers.ps1 +++ b/build/build-ai-sentry-containers.ps1 @@ -1,9 +1,9 @@ param( [string]$version = "0.1.1", - [string]$containerRegistry = "ariantestacr001.azurecr.io" + [string]$containerRegistry = "anevjesacrdevtest.azurecr.io" ) #Uncomment first line for very first time to set to the right acr context -# az acr login --name ariantestacr001 +# az acr login --name anevjesacrdevtest Write-Host "Building AI-Sentry Facade:$version" docker build --platform linux/amd64 -t ai-sentry-facadeapp:$version -f Dockerfile.facade ../aisentry/ docker tag ai-sentry-facadeapp:$version $containerRegistry/ai-sentry-facadeapp:$version diff --git a/content/documentation/AI-Sentry-config-settings.md b/content/documentation/AI-Sentry-config-settings.md index 176081e..c517612 100644 --- a/content/documentation/AI-Sentry-config-settings.md +++ b/content/documentation/AI-Sentry-config-settings.md @@ -6,4 +6,6 @@ Each pool is represented as an object within the "pools" array. Each pool object Each endpoint object has the following properties: - url: The URL of the OpenAI instance. You're expected to replace with the actual instance name. You need to append -- api-key: The API key for accessing the OpenAI instance. You're expected to replace your-api-key with the actual API key. \ No newline at end of file +- api-key: The API key for accessing the OpenAI instance. You're expected to replace your-api-key with the actual API key. + +Please note: We also support JWT auth to backend openAI instances. If you simply set "api-Key": null within the property bags inside the facade layer config; you will leverage aks workload identity to connect to openAi backends - however you will need worklaod identity federated out with managed identity stood up with your AKS cluster - and ofcourse grant the RBAC to the managed identity across all the required openAI instances in the backend. \ No newline at end of file diff --git a/content/documentation/AKSDeployment.md b/content/documentation/AKSDeployment.md index 6c5e6c2..d9b5415 100644 --- a/content/documentation/AKSDeployment.md +++ b/content/documentation/AKSDeployment.md @@ -48,7 +48,8 @@ helm repo add bitnami https://charts.bitnami.com/bitnami helm install sentry-redis bitnami/redis-cluster export REDIS_PASSWORD=$(kubectl get secret --namespace "default" sentry-redis-redis-cluster -o jsonpath="{.data.redis-password}" | base64 --decode) - kubectl create secret generic redis --from-literal=redis-password=$REDIS_PASSWORD -n sentry-ai +kubectl create namespace ai-sentry +kubectl create secret generic redis --from-literal=redis-password=$REDIS_PASSWORD -n ai-sentry ``` diff --git a/content/documentation/Workload-identity-config.md b/content/documentation/Workload-identity-config.md index 8c4ab6a..23fe3e8 100644 --- a/content/documentation/Workload-identity-config.md +++ b/content/documentation/Workload-identity-config.md @@ -1,5 +1,10 @@ # AKS Workload Identity setup +## Enable Workload identity against existing AKS cluster +```powershell +az aks update --resource-group "aks-devtest-rg" --name "anevjes-aks-dev" --enable-oidc-issuer --enable-workload-identity +``` + ## MI creation ```powershell az account set --subscription "subscriptionID" @@ -31,7 +36,7 @@ export SERVICE_ACCOUNT_NAMESPACE="ai-sentry" ## OIDC Issuer url ```bash -export AKS_OIDC_ISSUER="$(az aks show --name anevjes-aks --resource-group aks --query "oidcIssuerProfile.issuerUrl" -o tsv)" +export AKS_OIDC_ISSUER="$(az aks show --name anevjes-aks-dev --resource-group aks-devtest-rg --query "oidcIssuerProfile.issuerUrl" -o tsv)" ``` ## Create AKS Service Account diff --git a/content/documentation/ai-sentry-config.json b/content/documentation/ai-sentry-config.json index ee7f8f3..4e937f7 100644 --- a/content/documentation/ai-sentry-config.json +++ b/content/documentation/ai-sentry-config.json @@ -6,11 +6,11 @@ "endpoints": [ { "url": "https://.openai.azure.com/openai", - "api-key": "yourapi-key" + "api-key": "yourapi-key" //If you simply set "api-Key": null the facade layer will leverage aks workload identity to connect to openAi backends. }, { "url": "https://.openai.azure.com/openai", - "api-key": "your-api-key" + "api-key": "your-api-key" //If you simply set "api-Key": null the facade layer will leverage aks workload identity to connect to openAi backends. } ] }, @@ -20,11 +20,11 @@ "endpoints": [ { "url": "https://.openai.azure.com/openai", - "api-key": "your-api-key" + "api-key": "your-api-key" //If you simply set "api-Key": null the facade layer will leverage aks workload identity to connect to openAi backends. }, { "url": "https://.openai.azure.com/openai", - "api-key": "your-api-key" + "api-key": "your-api-key" //If you simply set "api-Key": null the facade layer will leverage aks workload identity to connect to openAi backends. } ] } diff --git a/infrastructure/APIM/ai-sentry-policy.xml b/infrastructure/APIM/ai-sentry-policy.xml index 6c4d112..142891b 100644 --- a/infrastructure/APIM/ai-sentry-policy.xml +++ b/infrastructure/APIM/ai-sentry-policy.xml @@ -1,20 +1,40 @@ + + + - + + + + + @((string)context.Variables.GetValueOrDefault("ai-sentry-callling-user")) + - aisentry + arian-test-caller COMPLETE - gpt-4 + pool1 + + + ["SampleApiRequestTransformer"] @@ -37,15 +57,15 @@ + + - - @((String)context.Variables["ai-sentry"]) - + From 744657b578cd250ea5e903bf5ee0bfe10c7e247b Mon Sep 17 00:00:00 2001 From: Arian Nevjestic Date: Thu, 1 Aug 2024 22:32:29 +0800 Subject: [PATCH 2/2] Update ai-sentry-deployment.yaml --- deploy/aks/ai-sentry-deployment.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/deploy/aks/ai-sentry-deployment.yaml b/deploy/aks/ai-sentry-deployment.yaml index c862d49..74b8752 100644 --- a/deploy/aks/ai-sentry-deployment.yaml +++ b/deploy/aks/ai-sentry-deployment.yaml @@ -64,6 +64,7 @@ spec: metadata: labels: app: facadeapp + azure.workload.identity/use: "true" annotations: dapr.io/enabled: "true" dapr.io/app-id: "facadeapp" @@ -75,6 +76,8 @@ spec: dapr.io/app-health-probe-interval: "3" dapr.io/app-health-probe-timeout: "200" dapr.io/app-health-threshold: "2" + azure.workload.identity/inject-proxy-sidecar: "true" + azure.workload.identity/proxy-sidecar-port: "8000" spec: containers: @@ -94,7 +97,7 @@ spec: failureThreshold: 3 env: - name: "AI-SENTRY-ENDPOINT-CONFIG" - value: "{\"pools\":[{\"name\":\"pool1\",\"description\":\"pool1 description\",\"endpoints\":[{\"url\":\"https://youropenaiendpoint.openai.azure.com/openai\",\"api-key\":\"yourkey\"}]},{\"name\":\"pool2\",\"description\":\"pool2 description\",\"endpoints\":[{\"url\":\"https://youropenai.openai.azure.com/openai\",\"api-key\":\"yourkey\"},{\"url\":\"https://youropenai.openai.azure.com/openai\",\"api-key\":\"yourkey\"}]}]}" + value: "{\"pools\":[{\"name\":\"pool1\",\"description\":\"pool1 description\",\"endpoints\":[{\"url\":\"https://youropenaiendpoint.openai.azure.com/openai\",\"api-key\":\"yourkey or simply null\"}]},{\"name\":\"pool2\",\"description\":\"pool2 description\",\"endpoints\":[{\"url\":\"https://youropenai.openai.azure.com/openai\",\"api-key or simply null\":\"yourkey or simply null\"},{\"url\":\"https://youropenai.openai.azure.com/openai\",\"api-key or simply null\":\"yourkey\"}]}]}" - name: "LOG-LEVEL" value: "INFO" --- @@ -141,6 +144,12 @@ spec: value: "your-key" - name: "LOG-LEVEL" value: "DEBUG" + - name: "PII_STRIPPING_SERVICE" + value: "OPENAI" + - name: PII_STRIPPING_OPENAI_ENDPOINT + value: "https://ptuopendeployment.openai.azure.com/" + - name: "PII_STRIPPING_OPENAI_API_KEY" + value: "yourapikeytoopenai / apim subscription key" --- #Summary Logger Logger kind: StatefulSet