diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2b7f99d9..9bacdb2b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -8,6 +8,9 @@ env: PCTASKS_COSMOSDB__KEY: ${{ secrets.COSMOSDB_KEY }} PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX: ${{ github.run_id }} +permissions: + id-token: write + jobs: pr-build-and-test: runs-on: ubuntu-22.04 @@ -19,6 +22,13 @@ jobs: with: python-version: "3.8" + - name: Log in with Azure + uses: azure/login@v1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: Install local dependencies run: ./scripts/install diff --git a/docker-compose.aux.yml b/docker-compose.aux.yml index faab3d6d..c848cf75 100644 --- a/docker-compose.aux.yml +++ b/docker-compose.aux.yml @@ -1,6 +1,5 @@ # Non-essential development enviroment servers -version: "2.1" services: stac-api: container_name: pctasks-stac-api diff --git a/docker-compose.cosmosdb.yml b/docker-compose.cosmosdb.yml index 98b1e396..aafe0fe4 100644 --- a/docker-compose.cosmosdb.yml +++ b/docker-compose.cosmosdb.yml @@ -1,4 +1,3 @@ -version: "2.1" services: cosmosdb: container_name: pctasks-cosmosdb diff --git a/docker-compose.yml b/docker-compose.yml index 28a68fb1..99a09ec2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,7 @@ -version: "2.1" services: azurite: container_name: pctasks-azurite - image: mcr.microsoft.com/azure-storage/azurite:3.31.0 + image: mcr.microsoft.com/azure-storage/azurite:3.32.0 hostname: azurite command: "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 -l /workspace" @@ -55,10 +54,6 @@ services: - APP_PORT=8511 - WEB_CONCURRENCY=1 - - AZURE_TENANT_ID - - AZURE_CLIENT_ID - - AZURE_CLIENT_SECRET - - PCTASKS_SERVER__DEV=true - PCTASKS_SERVER__DEV_API_KEY=hunter2 - PCTASKS_SERVER__DEV_AUTH_TOKEN=Bearer hunter2 @@ -111,8 +106,8 @@ services: ### Cosmos DB - COSMOSDB_EMULATOR_HOST=cosmosdb - PCTASKS_COSMOSDB__URL=${PCTASKS_COSMOSDB__URL:-https://cosmosdb:8081/} - - PCTASKS_COSMOSDB__KEY=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==} - PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX + - PCTASKS_COSMOSDB__KEY=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==} ### Blobs - PCTASKS_RUN__BLOB_ACCOUNT_URL=http://azurite:10000/devstoreaccount1 @@ -121,6 +116,7 @@ services: volumes: - ./pctasks:/opt/src + - ~/.azure:/root/.azure ports: - "8511:8511" command: > @@ -167,6 +163,8 @@ services: image: registry:2 ports: - 5001:5000 + volumes: + - ~/.azure:/root/.azure functions: container_name: pctasks-functions @@ -177,6 +175,7 @@ services: volumes: - ./pctasks:/home/site/pctasks - ./pctasks_funcs:/home/site/wwwroot + - ~/.azure:/home/.azure ports: - "7071:7071" # Functions environment: @@ -185,7 +184,7 @@ services: # Must use IP address to avoid SSL errors - PCTASKS_COSMOSDB__URL=${PCTASKS_COSMOSDB__URL:-https://172.16.238.246:8081/} - - PCTASKS_COSMOSDB__KEY=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==} + - PCTASKS_COSMOSDB__accountEndpoint=${PCTASKS_COSMOSDB__URL} # Set trigger app setting separately to avoid issues with __ in env var names - FUNC_COSMOSDB_CONN_STR=AccountEndpoint=${PCTASKS_COSMOSDB__URL:-https://172.16.238.246:8081/};AccountKey=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==}; - FUNC_STORAGE_ACCOUNT_CONN_STR=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1; diff --git a/pctasks/core/pctasks/core/cosmos/settings.py b/pctasks/core/pctasks/core/cosmos/settings.py index 9191ae9e..313d533c 100644 --- a/pctasks/core/pctasks/core/cosmos/settings.py +++ b/pctasks/core/pctasks/core/cosmos/settings.py @@ -90,28 +90,7 @@ def _validate_key(cls, v: Optional[str], values: Dict[str, Any]) -> Optional[str return v - def ensure_valid_connection_info(self) -> None: - if self.connection_string: - return - if not self.url: - raise CosmosDBSettingsError("Must set either connection_string or url") - if not self.key: - if not ( - os.environ.get("AZURE_CLIENT_ID") - and os.environ.get("AZURE_CLIENT_SECRET") - and os.environ.get("AZURE_TENANT_ID") - ): - # Validate that the Azure credentials are set - # Validation is here instead of pydantic validator - # because we may want to get container name settings - # without setting credentials. - raise CosmosDBSettingsError( - "Must set key or connection_string, account key or " - "provide Azure credentials to the environment" - ) - def get_cosmosdb_url(self) -> str: - self.ensure_valid_connection_info() if self.connection_string: m = re.search(r"AccountEndpoint=(.*?);", self.connection_string) assert m # Should be validated by pydantic @@ -143,7 +122,8 @@ def get_client(self) -> CosmosClient: self.connection_string, connection_verify=connection_verify ) else: - self.ensure_valid_connection_info() + # If the connection string is not set, the credetials are + # automatically picked up from the environment/managed identity assert self.url credential = self.key or DefaultAzureCredential() return CosmosClient( @@ -164,7 +144,8 @@ def get_async_client(self) -> AsyncCosmosClient: self.connection_string, connection_verify=connection_verify ) else: - self.ensure_valid_connection_info() + # If the connection string is not set, the credetials are + # automatically picked up from the environment/managed identity assert self.url credential = self.key or azure.identity.aio.DefaultAzureCredential() return AsyncCosmosClient( diff --git a/pctasks/server/Dockerfile b/pctasks/server/Dockerfile index c9c1c6f1..34190d49 100644 --- a/pctasks/server/Dockerfile +++ b/pctasks/server/Dockerfile @@ -36,6 +36,7 @@ RUN cd /opt/src/server && \ RUN if [ "${DEVELOPMENT}" = "TRUE" ]; then \ echo "INSTALLING DEVELOPMENT DEPENDENCIES"; \ + yum install -y azure-cli; \ pip install \ -e /opt/src/core \ -e /opt/src/cli \ diff --git a/pctasks_funcs/Dockerfile b/pctasks_funcs/Dockerfile index e1af2329..85342866 100644 --- a/pctasks_funcs/Dockerfile +++ b/pctasks_funcs/Dockerfile @@ -3,6 +3,8 @@ FROM mcr.microsoft.com/azure-functions/python:4-python3.10 ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ AzureFunctionsJobHost__Logging__Console__IsEnabled=true +RUN apt update && apt install -y azure-cli + # Copy pctasks packages COPY pctasks /home/site/pctasks