From 7822b872d0e01693e707e7c706ad879abc52f449 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Tue, 1 Oct 2024 19:34:10 -0700 Subject: [PATCH 01/21] Disable cosmosdb access key from pctasks server core --- docker-compose.aux.yml | 1 - docker-compose.cosmosdb.yml | 1 - docker-compose.yml | 10 ++---- pctasks/core/pctasks/core/cosmos/settings.py | 33 ++++--------------- pctasks/server/Dockerfile | 1 + pctasks_funcs/Dockerfile | 2 ++ pctasks_funcs/PublishItemsCF/function.json | 2 +- pctasks_funcs/StorageEventsCF/function.json | 4 +-- .../StorageEventsQueue/function.json | 2 +- pctasks_funcs/WorkflowRunsCF/function.json | 4 +-- pctasks_funcs/WorkflowsCF/function.json | 4 +-- pctasks_funcs/host.json | 2 +- 12 files changed, 21 insertions(+), 45 deletions(-) diff --git a/docker-compose.aux.yml b/docker-compose.aux.yml index faab3d6dc..c848cf757 100644 --- a/docker-compose.aux.yml +++ b/docker-compose.aux.yml @@ -1,6 +1,5 @@ # Non-essential development enviroment servers -version: "2.1" services: stac-api: container_name: pctasks-stac-api diff --git a/docker-compose.cosmosdb.yml b/docker-compose.cosmosdb.yml index 98b1e396d..aafe0fe4a 100644 --- a/docker-compose.cosmosdb.yml +++ b/docker-compose.cosmosdb.yml @@ -1,4 +1,3 @@ -version: "2.1" services: cosmosdb: container_name: pctasks-cosmosdb diff --git a/docker-compose.yml b/docker-compose.yml index e8138aa51..82fd45872 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "2.1" services: azurite: container_name: pctasks-azurite @@ -55,10 +54,6 @@ services: - APP_PORT=8511 - WEB_CONCURRENCY=1 - - AZURE_TENANT_ID - - AZURE_CLIENT_ID - - AZURE_CLIENT_SECRET - - PCTASKS_SERVER__DEV=true - PCTASKS_SERVER__DEV_API_KEY=hunter2 - PCTASKS_SERVER__DEV_AUTH_TOKEN=Bearer hunter2 @@ -111,7 +106,6 @@ services: ### Cosmos DB - COSMOSDB_EMULATOR_HOST=cosmosdb - PCTASKS_COSMOSDB__URL=${PCTASKS_COSMOSDB__URL:-https://cosmosdb:8081/} - - PCTASKS_COSMOSDB__KEY=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==} - PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX ### Blobs @@ -121,6 +115,7 @@ services: volumes: - ./pctasks:/opt/src + - ~/.azure:/root/.azure ports: - "8511:8511" command: > @@ -177,6 +172,7 @@ services: volumes: - ./pctasks:/home/site/pctasks - ./pctasks_funcs:/home/site/wwwroot + - ~/.azure:/home/.azure:ro ports: - "7071:7071" # Functions environment: @@ -185,7 +181,7 @@ services: # Must use IP address to avoid SSL errors - PCTASKS_COSMOSDB__URL=${PCTASKS_COSMOSDB__URL:-https://172.16.238.246:8081/} - - PCTASKS_COSMOSDB__KEY=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==} + - PCTASKS_COSMOSDB__accountEndpoint=${PCTASKS_COSMOSDB__URL} # Set trigger app setting separately to avoid issues with __ in env var names - FUNC_COSMOSDB_CONN_STR=AccountEndpoint=${PCTASKS_COSMOSDB__URL:-https://172.16.238.246:8081/};AccountKey=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==}; - FUNC_STORAGE_ACCOUNT_CONN_STR=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1; diff --git a/pctasks/core/pctasks/core/cosmos/settings.py b/pctasks/core/pctasks/core/cosmos/settings.py index 9191ae9e9..66fa42125 100644 --- a/pctasks/core/pctasks/core/cosmos/settings.py +++ b/pctasks/core/pctasks/core/cosmos/settings.py @@ -90,28 +90,7 @@ def _validate_key(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str return v - def ensure_valid_connection_info(self) -> None: - if self.connection_string: - return - if not self.url: - raise CosmosDBSettingsError("Must set either connection_string or url") - if not self.key: - if not ( - os.environ.get("AZURE_CLIENT_ID") - and os.environ.get("AZURE_CLIENT_SECRET") - and os.environ.get("AZURE_TENANT_ID") - ): - # Validate that the Azure credentials are set - # Validation is here instead of pydantic validator - # because we may want to get container name settings - # without setting credentials. - raise CosmosDBSettingsError( - "Must set key or connection_string, account key or " - "provide Azure credentials to the environment" - ) - def get_cosmosdb_url(self) -> str: - self.ensure_valid_connection_info() if self.connection_string: m = re.search(r"AccountEndpoint=(.*?);", self.connection_string) assert m # Should be validated by pydantic @@ -143,9 +122,9 @@ def get_client(self) -> CosmosClient: self.connection_string, connection_verify=connection_verify ) else: - self.ensure_valid_connection_info() - assert self.url - credential = self.key or DefaultAzureCredential() + # If the connection string is not set, the credetials are + # automatically picked up from the environment/managed identity + credential = DefaultAzureCredential() return CosmosClient( self.url, credential=credential, connection_verify=connection_verify ) @@ -164,9 +143,9 @@ def get_async_client(self) -> AsyncCosmosClient: self.connection_string, connection_verify=connection_verify ) else: - self.ensure_valid_connection_info() - assert self.url - credential = self.key or azure.identity.aio.DefaultAzureCredential() + # If the connection string is not set, the credetials are + # automatically picked up from the environment/managed identity + credential = azure.identity.aio.DefaultAzureCredential() return AsyncCosmosClient( self.url, credential=credential, connection_verify=connection_verify ) diff --git a/pctasks/server/Dockerfile b/pctasks/server/Dockerfile index c9c1c6f15..34190d49f 100644 --- a/pctasks/server/Dockerfile +++ b/pctasks/server/Dockerfile @@ -36,6 +36,7 @@ RUN cd /opt/src/server && \ RUN if [ "${DEVELOPMENT}" = "TRUE" ]; then \ echo "INSTALLING DEVELOPMENT DEPENDENCIES"; \ + yum install -y azure-cli; \ pip install \ -e /opt/src/core \ -e /opt/src/cli \ diff --git a/pctasks_funcs/Dockerfile b/pctasks_funcs/Dockerfile index e1af23296..853428660 100644 --- a/pctasks_funcs/Dockerfile +++ b/pctasks_funcs/Dockerfile @@ -3,6 +3,8 @@ FROM mcr.microsoft.com/azure-functions/python:4-python3.10 ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ AzureFunctionsJobHost__Logging__Console__IsEnabled=true +RUN apt update && apt install -y azure-cli + # Copy pctasks packages COPY pctasks /home/site/pctasks diff --git a/pctasks_funcs/PublishItemsCF/function.json b/pctasks_funcs/PublishItemsCF/function.json index 53f8a659c..238888019 100644 --- a/pctasks_funcs/PublishItemsCF/function.json +++ b/pctasks_funcs/PublishItemsCF/function.json @@ -5,7 +5,7 @@ "name": "documents", "type": "cosmosDBTrigger", "direction": "in", - "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR", + "connection": "PCTASKS_COSMOSDB", "databaseName": "pctasks", "collectionName": "%FUNC_ITEMS_COLLECTION_NAME%" } diff --git a/pctasks_funcs/StorageEventsCF/function.json b/pctasks_funcs/StorageEventsCF/function.json index f2058d36a..6232a994b 100644 --- a/pctasks_funcs/StorageEventsCF/function.json +++ b/pctasks_funcs/StorageEventsCF/function.json @@ -5,9 +5,9 @@ "name": "documents", "type": "cosmosDBTrigger", "direction": "in", - "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR", + "connection": "PCTASKS_COSMOSDB", "databaseName": "pctasks", - "collectionName": "%FUNC_STORAGE_EVENTS_COLLECTION_NAME%" + "containerName": "%FUNC_STORAGE_EVENTS_COLLECTION_NAME%" } ] } \ No newline at end of file diff --git a/pctasks_funcs/StorageEventsQueue/function.json b/pctasks_funcs/StorageEventsQueue/function.json index ec86a6bce..775190842 100644 --- a/pctasks_funcs/StorageEventsQueue/function.json +++ b/pctasks_funcs/StorageEventsQueue/function.json @@ -6,7 +6,7 @@ "name": "msg", "direction": "in", "queueName": "storage-events", - "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR" + "connection": "PCTASKS_COSMOSDB" } ] } \ No newline at end of file diff --git a/pctasks_funcs/WorkflowRunsCF/function.json b/pctasks_funcs/WorkflowRunsCF/function.json index df5a44acc..4e990b722 100644 --- a/pctasks_funcs/WorkflowRunsCF/function.json +++ b/pctasks_funcs/WorkflowRunsCF/function.json @@ -5,9 +5,9 @@ "name": "container", "type": "cosmosDBTrigger", "direction": "in", - "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR", + "connection": "PCTASKS_COSMOSDB", "databaseName": "pctasks", - "collectionName": "%FUNC_WORKFLOW_RUNS_COLLECTION_NAME%" + "containerName": "%FUNC_WORKFLOW_RUNS_COLLECTION_NAME%" } ] } \ No newline at end of file diff --git a/pctasks_funcs/WorkflowsCF/function.json b/pctasks_funcs/WorkflowsCF/function.json index 477cfcef9..db260f85c 100644 --- a/pctasks_funcs/WorkflowsCF/function.json +++ b/pctasks_funcs/WorkflowsCF/function.json @@ -5,9 +5,9 @@ "name": "container", "type": "cosmosDBTrigger", "direction": "in", - "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR", + "connection": "PCTASKS_COSMOSDB", "databaseName": "pctasks", - "collectionName": "%FUNC_WORKFLOWS_COLLECTION_NAME%" + "containerName": "%FUNC_WORKFLOWS_COLLECTION_NAME%" } ] } \ No newline at end of file diff --git a/pctasks_funcs/host.json b/pctasks_funcs/host.json index 31fb5dc50..ca9ec67bd 100644 --- a/pctasks_funcs/host.json +++ b/pctasks_funcs/host.json @@ -10,7 +10,7 @@ }, "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[2.*, 3.0.0)" + "version": "[4.0.0, 5.0.0)" }, "extensions": { "queues": { From 0a407e207fdd72672f9cb0c2b6142a3a06608cec Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Wed, 2 Oct 2024 11:58:55 -0700 Subject: [PATCH 02/21] debugging --- scripts/setup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/setup b/scripts/setup index 63ea7289c..71a86acbe 100755 --- a/scripts/setup +++ b/scripts/setup @@ -75,6 +75,11 @@ while [[ $# -gt 0 ]]; do case $1 in source scripts/env +echo "===================== " +echo "AZURE_CLIENT_ID" +echo AZURE_CLIENT_ID +echo "===================== " + if [ "${BASH_SOURCE[0]}" = "${0}" ]; then if [ "${CLEAR_RECORDS}" ]; then echo "Clearing records from Azurite Tables..." From 9cc2018b2635b881820c6b6b2c01ba2a76e2cf08 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Wed, 2 Oct 2024 12:07:57 -0700 Subject: [PATCH 03/21] debugging --- scripts/setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup b/scripts/setup index 71a86acbe..431cfccbd 100755 --- a/scripts/setup +++ b/scripts/setup @@ -77,7 +77,7 @@ source scripts/env echo "===================== " echo "AZURE_CLIENT_ID" -echo AZURE_CLIENT_ID +echo $AZURE_CLIENT_ID echo "===================== " if [ "${BASH_SOURCE[0]}" = "${0}" ]; then From 43793991ab6d3ed251c3c2f41b461a2738060be8 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Wed, 2 Oct 2024 12:17:55 -0700 Subject: [PATCH 04/21] debugging --- .github/workflows/pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c7c115908..9a038cc0f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -7,6 +7,7 @@ env: PCTASKS_COSMOSDB__URL: ${{ secrets.COSMOSDB_URL }} PCTASKS_COSMOSDB__KEY: ${{ secrets.COSMOSDB_KEY }} PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX: ${{ github.run_id }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} jobs: pr-build-and-test: From 15b6d1b8c17175ff48018a63bc57e42ba2d2e6f2 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Wed, 2 Oct 2024 12:26:35 -0700 Subject: [PATCH 05/21] debugging --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9a038cc0f..a14bc4612 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -7,7 +7,7 @@ env: PCTASKS_COSMOSDB__URL: ${{ secrets.COSMOSDB_URL }} PCTASKS_COSMOSDB__KEY: ${{ secrets.COSMOSDB_KEY }} PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX: ${{ github.run_id }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} jobs: pr-build-and-test: From bee73cc8f6832b994971a887ba4349e64b196d40 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Wed, 2 Oct 2024 12:41:14 -0700 Subject: [PATCH 06/21] debugging --- .github/workflows/pr.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a14bc4612..3bdeb2a0e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -7,7 +7,7 @@ env: PCTASKS_COSMOSDB__URL: ${{ secrets.COSMOSDB_URL }} PCTASKS_COSMOSDB__KEY: ${{ secrets.COSMOSDB_KEY }} PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX: ${{ github.run_id }} - AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} +# AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} jobs: pr-build-and-test: @@ -20,6 +20,13 @@ jobs: with: python-version: "3.8" + - name: Log in with Azure + uses: azure/login@v1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: Install local dependencies run: ./scripts/install From 3ec736d5e265b357f74ee51ca98310a9173a0b39 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Wed, 2 Oct 2024 12:43:44 -0700 Subject: [PATCH 07/21] Adding logging in to azure and adding appropriate permissions to PR action. --- .github/workflows/pr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3bdeb2a0e..195fddbd4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -7,7 +7,9 @@ env: PCTASKS_COSMOSDB__URL: ${{ secrets.COSMOSDB_URL }} PCTASKS_COSMOSDB__KEY: ${{ secrets.COSMOSDB_KEY }} PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX: ${{ github.run_id }} -# AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} + +permissions: + id-token: write jobs: pr-build-and-test: From f4a9ae7669f806afb4133ea04c65371f6b6c313a Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Wed, 2 Oct 2024 21:41:05 -0700 Subject: [PATCH 08/21] bumping azurite version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 82fd45872..f1ed520b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: azurite: container_name: pctasks-azurite - image: mcr.microsoft.com/azure-storage/azurite:3.31.0 + image: mcr.microsoft.com/azure-storage/azurite:3.32.0 hostname: azurite command: "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 -l /workspace" From 26dc092bfc648fe70c7b1d7262d08afac2be2d73 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Wed, 2 Oct 2024 21:54:36 -0700 Subject: [PATCH 09/21] ensuring cosmos client has valid url --- pctasks/core/pctasks/core/cosmos/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pctasks/core/pctasks/core/cosmos/settings.py b/pctasks/core/pctasks/core/cosmos/settings.py index 66fa42125..4aa9a7955 100644 --- a/pctasks/core/pctasks/core/cosmos/settings.py +++ b/pctasks/core/pctasks/core/cosmos/settings.py @@ -124,6 +124,7 @@ def get_client(self) -> CosmosClient: else: # If the connection string is not set, the credetials are # automatically picked up from the environment/managed identity + assert self.url credential = DefaultAzureCredential() return CosmosClient( self.url, credential=credential, connection_verify=connection_verify @@ -145,6 +146,7 @@ def get_async_client(self) -> AsyncCosmosClient: else: # If the connection string is not set, the credetials are # automatically picked up from the environment/managed identity + assert self.url credential = azure.identity.aio.DefaultAzureCredential() return AsyncCosmosClient( self.url, credential=credential, connection_verify=connection_verify From 45ce2b252edbce80eadf68c9d83f41fb2be8559f Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Thu, 3 Oct 2024 10:26:50 -0700 Subject: [PATCH 10/21] test azure directory mount --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index f1ed520b8..02aed9394 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -162,6 +162,8 @@ services: image: registry:2 ports: - 5001:5000 + volumes: + - ~/.azure:/root/.azure functions: container_name: pctasks-functions From 2b8524bf380c9014fd46456d881e3e118ace1305 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Thu, 3 Oct 2024 11:22:33 -0700 Subject: [PATCH 11/21] reverting change in azure functions --- pctasks_funcs/PublishItemsCF/function.json | 2 +- pctasks_funcs/StorageEventsCF/function.json | 4 ++-- pctasks_funcs/StorageEventsQueue/function.json | 2 +- pctasks_funcs/WorkflowRunsCF/function.json | 4 ++-- pctasks_funcs/WorkflowsCF/function.json | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pctasks_funcs/PublishItemsCF/function.json b/pctasks_funcs/PublishItemsCF/function.json index 238888019..53f8a659c 100644 --- a/pctasks_funcs/PublishItemsCF/function.json +++ b/pctasks_funcs/PublishItemsCF/function.json @@ -5,7 +5,7 @@ "name": "documents", "type": "cosmosDBTrigger", "direction": "in", - "connection": "PCTASKS_COSMOSDB", + "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR", "databaseName": "pctasks", "collectionName": "%FUNC_ITEMS_COLLECTION_NAME%" } diff --git a/pctasks_funcs/StorageEventsCF/function.json b/pctasks_funcs/StorageEventsCF/function.json index 6232a994b..f2058d36a 100644 --- a/pctasks_funcs/StorageEventsCF/function.json +++ b/pctasks_funcs/StorageEventsCF/function.json @@ -5,9 +5,9 @@ "name": "documents", "type": "cosmosDBTrigger", "direction": "in", - "connection": "PCTASKS_COSMOSDB", + "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR", "databaseName": "pctasks", - "containerName": "%FUNC_STORAGE_EVENTS_COLLECTION_NAME%" + "collectionName": "%FUNC_STORAGE_EVENTS_COLLECTION_NAME%" } ] } \ No newline at end of file diff --git a/pctasks_funcs/StorageEventsQueue/function.json b/pctasks_funcs/StorageEventsQueue/function.json index 775190842..ec86a6bce 100644 --- a/pctasks_funcs/StorageEventsQueue/function.json +++ b/pctasks_funcs/StorageEventsQueue/function.json @@ -6,7 +6,7 @@ "name": "msg", "direction": "in", "queueName": "storage-events", - "connection": "PCTASKS_COSMOSDB" + "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR" } ] } \ No newline at end of file diff --git a/pctasks_funcs/WorkflowRunsCF/function.json b/pctasks_funcs/WorkflowRunsCF/function.json index 4e990b722..df5a44acc 100644 --- a/pctasks_funcs/WorkflowRunsCF/function.json +++ b/pctasks_funcs/WorkflowRunsCF/function.json @@ -5,9 +5,9 @@ "name": "container", "type": "cosmosDBTrigger", "direction": "in", - "connection": "PCTASKS_COSMOSDB", + "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR", "databaseName": "pctasks", - "containerName": "%FUNC_WORKFLOW_RUNS_COLLECTION_NAME%" + "collectionName": "%FUNC_WORKFLOW_RUNS_COLLECTION_NAME%" } ] } \ No newline at end of file diff --git a/pctasks_funcs/WorkflowsCF/function.json b/pctasks_funcs/WorkflowsCF/function.json index db260f85c..477cfcef9 100644 --- a/pctasks_funcs/WorkflowsCF/function.json +++ b/pctasks_funcs/WorkflowsCF/function.json @@ -5,9 +5,9 @@ "name": "container", "type": "cosmosDBTrigger", "direction": "in", - "connection": "PCTASKS_COSMOSDB", + "connectionStringSetting": "FUNC_COSMOSDB_CONN_STR", "databaseName": "pctasks", - "containerName": "%FUNC_WORKFLOWS_COLLECTION_NAME%" + "collectionName": "%FUNC_WORKFLOWS_COLLECTION_NAME%" } ] } \ No newline at end of file From f67afcb9bb39e7713557893322dfe36272bc1f73 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Thu, 3 Oct 2024 11:49:47 -0700 Subject: [PATCH 12/21] undo function host settings change --- pctasks_funcs/host.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pctasks_funcs/host.json b/pctasks_funcs/host.json index ca9ec67bd..31fb5dc50 100644 --- a/pctasks_funcs/host.json +++ b/pctasks_funcs/host.json @@ -10,7 +10,7 @@ }, "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[4.0.0, 5.0.0)" + "version": "[2.*, 3.0.0)" }, "extensions": { "queues": { From 81332916df47ea993d3469a4bb3b804072eb0f68 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Thu, 3 Oct 2024 12:16:19 -0700 Subject: [PATCH 13/21] removing readonly attribute from mount --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 02aed9394..9851fc194 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -174,7 +174,7 @@ services: volumes: - ./pctasks:/home/site/pctasks - ./pctasks_funcs:/home/site/wwwroot - - ~/.azure:/home/.azure:ro + - ~/.azure:/home/.azure ports: - "7071:7071" # Functions environment: From 8f282f07de6967e2b2d0b8a20f56a251cd8ea7e5 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Thu, 3 Oct 2024 13:23:34 -0700 Subject: [PATCH 14/21] debugging --- pctasks/client/pctasks/client/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pctasks/client/pctasks/client/client.py b/pctasks/client/pctasks/client/client.py index bef17fdf7..6f42cefaf 100644 --- a/pctasks/client/pctasks/client/client.py +++ b/pctasks/client/pctasks/client/client.py @@ -287,6 +287,10 @@ def get_workflow(self, workflow_id: str) -> Optional[WorkflowRecord]: ) return WorkflowRecordResponse.parse_obj(result).record except HTTPError as e: + logger.debug("$$$"+ e.request.url) + logger.debug("$$$"+ e.request.body) + logger.debug("$$$"+ e.response.text) + logger.debug("$$$"+ e.response.status_code) if e.response.status_code == 404: return None raise From 989543a0752a90ace9ff0bb04cf39f12666ca954 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Thu, 3 Oct 2024 13:36:14 -0700 Subject: [PATCH 15/21] debugging --- pctasks/client/pctasks/client/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pctasks/client/pctasks/client/client.py b/pctasks/client/pctasks/client/client.py index 6f42cefaf..abcc0b1b3 100644 --- a/pctasks/client/pctasks/client/client.py +++ b/pctasks/client/pctasks/client/client.py @@ -287,10 +287,10 @@ def get_workflow(self, workflow_id: str) -> Optional[WorkflowRecord]: ) return WorkflowRecordResponse.parse_obj(result).record except HTTPError as e: - logger.debug("$$$"+ e.request.url) - logger.debug("$$$"+ e.request.body) - logger.debug("$$$"+ e.response.text) - logger.debug("$$$"+ e.response.status_code) + logger.debug(e.request.url) + logger.debug(e.request.body) + logger.debug(e.response.text) + logger.debug(e.response.status_code) if e.response.status_code == 404: return None raise From 96d32258ca08e791096d31d3beecc7bfa65aeece Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Thu, 3 Oct 2024 13:52:42 -0700 Subject: [PATCH 16/21] debugging --- pctasks/client/pctasks/client/client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pctasks/client/pctasks/client/client.py b/pctasks/client/pctasks/client/client.py index abcc0b1b3..bef17fdf7 100644 --- a/pctasks/client/pctasks/client/client.py +++ b/pctasks/client/pctasks/client/client.py @@ -287,10 +287,6 @@ def get_workflow(self, workflow_id: str) -> Optional[WorkflowRecord]: ) return WorkflowRecordResponse.parse_obj(result).record except HTTPError as e: - logger.debug(e.request.url) - logger.debug(e.request.body) - logger.debug(e.response.text) - logger.debug(e.response.status_code) if e.response.status_code == 404: return None raise From 7be5c8028a67b25be92f2d2de6a140b2360bcd2e Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Thu, 3 Oct 2024 19:20:09 -0700 Subject: [PATCH 17/21] debugging --- pctasks/client/pctasks/client/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pctasks/client/pctasks/client/client.py b/pctasks/client/pctasks/client/client.py index bef17fdf7..64250cd4c 100644 --- a/pctasks/client/pctasks/client/client.py +++ b/pctasks/client/pctasks/client/client.py @@ -107,6 +107,9 @@ def do_request() -> requests.Response: params=params, **kwargs, ) + logger.debug(f"API request: {method} {url}") + logger.debug(f"API response: {resp.status_code}") + logger.debug(f"API response: {resp.text}") resp.raise_for_status() return resp From 9691aa393394e136dd01b7df439eb8d54fd3f802 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Sat, 5 Oct 2024 19:01:59 -0700 Subject: [PATCH 18/21] restore key for integration testsw --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9851fc194..dbecc52e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,6 +107,7 @@ services: - COSMOSDB_EMULATOR_HOST=cosmosdb - PCTASKS_COSMOSDB__URL=${PCTASKS_COSMOSDB__URL:-https://cosmosdb:8081/} - PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX + - PCTASKS_COSMOSDB__KEY=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==} ### Blobs - PCTASKS_RUN__BLOB_ACCOUNT_URL=http://azurite:10000/devstoreaccount1 From 6560ae007ff0865ab9e275ee54bec31cf9791ee0 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Sat, 5 Oct 2024 19:33:57 -0700 Subject: [PATCH 19/21] For integration tests, we still define key. Code uses key if defined. --- pctasks/core/pctasks/core/cosmos/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pctasks/core/pctasks/core/cosmos/settings.py b/pctasks/core/pctasks/core/cosmos/settings.py index 4aa9a7955..313d533cb 100644 --- a/pctasks/core/pctasks/core/cosmos/settings.py +++ b/pctasks/core/pctasks/core/cosmos/settings.py @@ -125,7 +125,7 @@ def get_client(self) -> CosmosClient: # If the connection string is not set, the credetials are # automatically picked up from the environment/managed identity assert self.url - credential = DefaultAzureCredential() + credential = self.key or DefaultAzureCredential() return CosmosClient( self.url, credential=credential, connection_verify=connection_verify ) @@ -147,7 +147,7 @@ def get_async_client(self) -> AsyncCosmosClient: # If the connection string is not set, the credetials are # automatically picked up from the environment/managed identity assert self.url - credential = azure.identity.aio.DefaultAzureCredential() + credential = self.key or azure.identity.aio.DefaultAzureCredential() return AsyncCosmosClient( self.url, credential=credential, connection_verify=connection_verify ) From e24121cc73a84fe0b02a82037deb4fc2ac492086 Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Mon, 7 Oct 2024 14:16:46 -0700 Subject: [PATCH 20/21] undo debugging changes --- scripts/setup | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/setup b/scripts/setup index 431cfccbd..63ea7289c 100755 --- a/scripts/setup +++ b/scripts/setup @@ -75,11 +75,6 @@ while [[ $# -gt 0 ]]; do case $1 in source scripts/env -echo "===================== " -echo "AZURE_CLIENT_ID" -echo $AZURE_CLIENT_ID -echo "===================== " - if [ "${BASH_SOURCE[0]}" = "${0}" ]; then if [ "${CLEAR_RECORDS}" ]; then echo "Clearing records from Azurite Tables..." From cc114bdf53c4ebc09d5b865f3771e71eaaba433b Mon Sep 17 00:00:00 2001 From: Manish Kumar Gupta Date: Mon, 7 Oct 2024 14:43:38 -0700 Subject: [PATCH 21/21] undo debugging changes --- pctasks/client/pctasks/client/client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pctasks/client/pctasks/client/client.py b/pctasks/client/pctasks/client/client.py index 64250cd4c..bef17fdf7 100644 --- a/pctasks/client/pctasks/client/client.py +++ b/pctasks/client/pctasks/client/client.py @@ -107,9 +107,6 @@ def do_request() -> requests.Response: params=params, **kwargs, ) - logger.debug(f"API request: {method} {url}") - logger.debug(f"API response: {resp.status_code}") - logger.debug(f"API response: {resp.text}") resp.raise_for_status() return resp