From c36f05821b751c5d46047c9c58eea8fa4fdf6400 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:55:23 +0200 Subject: [PATCH] Add PCS prod environment (#4040) --- .vault-config/product-construction-int.yaml | 21 ------- .vault-config/product-construction-prod.yaml | 28 +++++++++ ...pipelines-product-construction-service.yml | 32 +++++++++- .../container-environment.bicep | 12 ++++ .../managed-identities.bicep | 11 ++++ .../production.bicepparam | 55 ++++++++++++++++ .../provision.bicep | 63 ++++++++++--------- .../ProductConstructionService/provision.ps1 | 7 ++- .../ProductConstructionService/redis.bicep | 2 +- .../staging.bicepparam | 57 +++++++++++++++++ eng/templates/jobs/e2e-pcs-tests.yml | 28 ++++++--- src/Maestro/Client/src/MaestroApiOptions.cs | 2 + .../appsettings.Production.json | 28 +++++++++ .../appsettings.Staging.json | 4 -- .../ProductConstructionServiceApiOptions.cs | 6 +- .../Deployer.cs | 39 +++++++----- .../appsettings.Production.json | 9 +++ .../appsettings.Production.json | 8 +++ .../appsettings.Production.json | 11 ++++ src/ProductConstructionService/Readme.md | 17 +++-- 20 files changed, 345 insertions(+), 95 deletions(-) create mode 100644 .vault-config/product-construction-prod.yaml create mode 100644 eng/service-templates/ProductConstructionService/production.bicepparam create mode 100644 eng/service-templates/ProductConstructionService/staging.bicepparam create mode 100644 src/ProductConstructionService/ProductConstructionService.Api/appsettings.Production.json create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Production.json create mode 100644 src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/appsettings.Production.json create mode 100644 src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Production.json diff --git a/.vault-config/product-construction-int.yaml b/.vault-config/product-construction-int.yaml index 7d597b7356..47f8f384a1 100644 --- a/.vault-config/product-construction-int.yaml +++ b/.vault-config/product-construction-int.yaml @@ -4,28 +4,7 @@ storageLocation: subscription: e6b5f9f5-0ca4-4351-879b-014d78400ec2 name: ProductConstructionInt -references: - helixkv: - type: azure-key-vault - parameters: - subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 - name: helixkv - - engkeyvault: - type: azure-key-vault - parameters: - subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 - name: engkeyvault - secrets: - BotAccount-dotnet-bot-repo-PAT: - type: github-access-token - parameters: - gitHubBotAccountSecret: - location: engkeyvault - name: BotAccount-dotnet-bot - gitHubBotAccountName: dotnet-bot - github: type: github-app-secret parameters: diff --git a/.vault-config/product-construction-prod.yaml b/.vault-config/product-construction-prod.yaml new file mode 100644 index 0000000000..52c2d7ac65 --- /dev/null +++ b/.vault-config/product-construction-prod.yaml @@ -0,0 +1,28 @@ +storageLocation: + type: azure-key-vault + parameters: + subscription: fbd6122a-9ad3-42e4-976e-bccb82486856 + name: ProductConstructionProd + +references: + engkeyvault: + type: azure-key-vault + parameters: + subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 + name: engkeyvault + +secrets: + BotAccount-dotnet-bot-repo-PAT: + type: github-access-token + parameters: + gitHubBotAccountSecret: + location: engkeyvault + name: BotAccount-dotnet-bot + gitHubBotAccountName: dotnet-bot + + github: + type: github-app-secret + parameters: + hasPrivateKey: true + hasWebhookSecret: false + hasOAuthSecret: true diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index a2c16709c6..2cb6a712a9 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -3,6 +3,7 @@ trigger: branches: include: - main + - production pr: branches: @@ -47,10 +48,35 @@ variables: value: "Darc: Maestro Staging" - name: MaestroAppId value: $(MaestroStagingAppClientId) - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - name: redisConnectionString + value: "product-construction-service-redis-int.redis.cache.windows.net:6380,ssl=true" +- ${{ else }}: + - name: subscriptionId + value: fbd6122a-9ad3-42e4-976e-bccb82486856 + - name: containerappName + value: product-construction-prod + - name: containerjobNames + value: sub-triggerer-twicedaily-prod,sub-triggerer-daily-prod,sub-triggerer-weekly-prod,longest-path-updater-job-prod,feed-cleaner-prod + - name: containerRegistryName + value: productconstructionprod + - name: containerappEnvironmentName + value: product-construction-service-env-prod + - name: containerappWorkspaceName + value: product-construction-service-workspace-prod + - name: dockerRegistryUrl + value: productconstructionprod.azurecr.io + - name: serviceConnectionName + value: ProductConstructionServiceDeploymentProd + - name: authServiceConnection + value: "Darc: Maestro Production" + - name: MaestroAppId + value: $(MaestroAppClientId) + - name: redisConnectionString + value: "product-construction-service-redis-prod.redis.cache.windows.net,ssl=true" +- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - name: devBranchSuffix value: - - ${{ else }}: +- ${{ else }}: - name: devBranchSuffix value: -dev @@ -199,7 +225,7 @@ stages: --azCliPath "$(azCliPath)" ` --isCi true ` --entraAppId $(MaestroAppId) ` - --redisConnectionString "product-construction-service-redis-int.redis.cache.windows.net:6380,ssl=true" + --redisConnectionString $(redisConnectionString) displayName: Deploy container app - task: AzureCLI@2 diff --git a/eng/service-templates/ProductConstructionService/container-environment.bicep b/eng/service-templates/ProductConstructionService/container-environment.bicep index 9bd3fd48f7..ade013c8c1 100644 --- a/eng/service-templates/ProductConstructionService/container-environment.bicep +++ b/eng/service-templates/ProductConstructionService/container-environment.bicep @@ -4,6 +4,8 @@ param containerEnvironmentName string param productConstructionServiceSubnetId string param infrastructureResourceGroupName string param applicationInsightsName string +param containerAppsManagedEnvironmentsContributor string +param deploymentIdentityPrincipalId string resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { name: logAnalyticsName @@ -57,5 +59,15 @@ resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { } } +resource deploymentSubscriptionTriggererContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerEnvironment + name: guid(subscription().id, resourceGroup().id, containerAppsManagedEnvironmentsContributor) + properties: { + roleDefinitionId: containerAppsManagedEnvironmentsContributor + principalType: 'ServicePrincipal' + principalId: deploymentIdentityPrincipalId + } + } + output applicationInsightsConnectionString string = applicationInsights.properties.ConnectionString output containerEnvironmentId string = containerEnvironment.id diff --git a/eng/service-templates/ProductConstructionService/managed-identities.bicep b/eng/service-templates/ProductConstructionService/managed-identities.bicep index e7fca31dca..3b3be0133e 100644 --- a/eng/service-templates/ProductConstructionService/managed-identities.bicep +++ b/eng/service-templates/ProductConstructionService/managed-identities.bicep @@ -4,6 +4,7 @@ param pcsIdentityName string param subscriptionTriggererIdentityName string param longestBuildPathUpdaterIdentityName string param feedCleanerIdentityName string +param contributorRole string resource deploymentIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { name: deploymentIdentityName @@ -30,6 +31,16 @@ resource feedCleanerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2 location: location } +resource pcsIdentityContributorRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: pcsIdentity + name: guid(subscription().id, resourceGroup().id, contributorRole) + properties: { + roleDefinitionId: contributorRole + principalType: 'ServicePrincipal' + principalId: deploymentIdentity.properties.principalId + } +} + output pcsIdentityPrincipalId string = pcsIdentity.properties.principalId output pcsIdentityId string = pcsIdentity.id output deploymentIdentityPrincipalId string = deploymentIdentity.properties.principalId diff --git a/eng/service-templates/ProductConstructionService/production.bicepparam b/eng/service-templates/ProductConstructionService/production.bicepparam new file mode 100644 index 0000000000..54245b02bd --- /dev/null +++ b/eng/service-templates/ProductConstructionService/production.bicepparam @@ -0,0 +1,55 @@ +using 'provision.bicep' + +param location = 'westus2' + +param containerRegistryName = 'productconstructionprod' + +param containerCpuCoreCount = '1.0' + +param containerMemory = '2Gi' + +param aspnetcoreEnvironment = 'Production' + +param applicationInsightsName = 'product-construction-service-ai-prod' + +param keyVaultName = 'ProductConstructionProd' + +param azureCacheRedisName = 'product-construction-service-redis-prod' + +param logAnalyticsName = 'product-construction-service-workspace-prod' + +param containerEnvironmentName = 'product-construction-service-env-prod' + +param productConstructionServiceName = 'product-construction-prod' + +param storageAccountName = 'productconstructionprod' + +param pcsIdentityName = 'ProductConstructionServiceProd' + +param deploymentIdentityName = 'ProductConstructionServiceDeploymentProd' + +param containerImageName = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' + +param virtualNetworkName = 'product-construction-service-vnet-prod' + +param productConstructionServiceSubnetName = 'product-construction-service-subnet' + +param subscriptionTriggererIdentityName = 'SubscriptionTriggererProd' + +param subscriptionTriggererWeeklyJobName = 'sub-triggerer-weekly-prod' + +param subscriptionTriggererTwiceDailyJobName = 'sub-triggerer-twicedaily-prod' + +param subscriptionTriggererDailyJobName = 'sub-triggerer-daily-prod' + +param longestBuildPathUpdaterIdentityName = 'LongestBuildPathUpdaterProd' + +param longestBuildPathUpdaterJobName = 'longest-path-updater-job-prod' + +param feedCleanerJobName = 'feed-cleaner-prod' + +param feedCleanerIdentityName = 'FeedCleanerProd' + +param networkSecurityGroupName = 'product-construction-service-nsg-prod' + +param infrastructureResourceGroupName = 'product-construction-service-ip-prod' diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 2dad5d0f7f..ecf2ecf22e 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -1,17 +1,17 @@ @minLength(1) @description('Primary location for all resources') -param location string = 'westus2' +param location string @minLength(5) @maxLength(50) @description('Name of the Azure Container Registry resource into which container images will be published') -param containerRegistryName string = 'productconstructionint' +param containerRegistryName string @description('CPU cores allocated to a single container instance') -param containerCpuCoreCount string = '1.0' +param containerCpuCoreCount string @description('Memory allocated to a single container instance') -param containerMemory string = '2Gi' +param containerMemory string @description('aspnetcore environment') @allowed([ @@ -19,76 +19,76 @@ param containerMemory string = '2Gi' 'Staging' 'Production' ]) -param aspnetcoreEnvironment string = 'Staging' +param aspnetcoreEnvironment string @description('Name of the application insights resource') -param applicationInsightsName string = 'product-construction-service-ai-int' +param applicationInsightsName string @description('Key Vault name') -param keyVaultName string = 'ProductConstructionInt' +param keyVaultName string @description('Dev Key Vault name') -param devKeyVaultName string = 'ProductConstructionDev' +param devKeyVaultName string = '' @description('Azure Cache for Redis name') -param azureCacheRedisName string = 'product-construction-service-redis-int' +param azureCacheRedisName string @description('Log analytics workspace name') -param logAnalyticsName string = 'product-construction-service-workspace-int' +param logAnalyticsName string @description('Name of the container apps environment') -param containerEnvironmentName string = 'product-construction-service-env-int' +param containerEnvironmentName string @description('Product construction service API name') -param productConstructionServiceName string = 'product-construction-int' +param productConstructionServiceName string @description('Storage account name') -param storageAccountName string = 'productconstructionint' +param storageAccountName string @description('Name of the MI used for the PCS container app') -param pcsIdentityName string = 'ProductConstructionServiceInt' +param pcsIdentityName string @description('Name of the identity used for the PCS deployment') -param deploymentIdentityName string = 'ProductConstructionServiceDeploymentInt' +param deploymentIdentityName string @description('Bicep requires an image when creating a containerapp. Using a dummy image for that.') -param containerImageName string = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' +param containerImageName string @description('Virtual network name') -param virtualNetworkName string = 'product-construction-service-vnet-int' +param virtualNetworkName string @description('Product construction service subnet name') -param productConstructionServiceSubnetName string = 'product-construction-service-subnet' +param productConstructionServiceSubnetName string @description('Subscription Triggerer Identity name') -param subscriptionTriggererIdentityName string = 'SubscriptionTriggererInt' +param subscriptionTriggererIdentityName string @description('Subscription Triggerer Weekly Job name') -param subscriptionTriggererWeeklyJobName string = 'sub-triggerer-weekly-int' +param subscriptionTriggererWeeklyJobName string @description('Subscription Triggerer Twice Daily Job name') -param subscriptionTriggererTwiceDailyJobName string = 'sub-triggerer-twicedaily-int' +param subscriptionTriggererTwiceDailyJobName string @description('Subscription Triggerer Daily Job name') -param subscriptionTriggererDailyJobName string = 'sub-triggerer-daily-int' +param subscriptionTriggererDailyJobName string @description('Longest Build Path Updater Identity Name') -param longestBuildPathUpdaterIdentityName string = 'LongestBuildPathUpdaterInt' +param longestBuildPathUpdaterIdentityName string @description('Longest Build Path Updater Job Name') -param longestBuildPathUpdaterJobName string = 'longest-path-updater-job-int' +param longestBuildPathUpdaterJobName string @description('Feed Cleaner Job name') -param feedCleanerJobName string = 'feed-cleaner-int' +param feedCleanerJobName string @description('Feed Cleaner Identity name') -param feedCleanerIdentityName string = 'FeedCleanerInt' +param feedCleanerIdentityName string @description('Network security group name') -param networkSecurityGroupName string = 'product-construction-service-nsg-int' +param networkSecurityGroupName string @description('Resource group where PCS IP resources will be created') -param infrastructureResourceGroupName string = 'product-construction-service-ip-int' +param infrastructureResourceGroupName string // azure system role for setting up acr pull access var acrPullRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') @@ -104,6 +104,10 @@ var contributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefini var blobContributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') // Key Vault Crypto User role var kvCryptoUserRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') +// Reader role +var readerRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') +// Container Apps ManagedEnvironments Contributor Role +var containerAppsManagedEnvironmentsContributor = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '57cc5028-e6a7-4284-868d-0611c5923f8d') module networkSecurityGroupModule 'nsg.bicep' = { name: 'networkSecurityGroupModule' @@ -132,6 +136,8 @@ module containerEnvironmentModule 'container-environment.bicep' = { productConstructionServiceSubnetId: virtualNetworkModule.outputs.productConstructionServiceSubnetId infrastructureResourceGroupName: infrastructureResourceGroupName applicationInsightsName: applicationInsightsName + containerAppsManagedEnvironmentsContributor: containerAppsManagedEnvironmentsContributor + deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId } } @@ -144,6 +150,7 @@ module managedIdentitiesModule 'managed-identities.bicep' = { subscriptionTriggererIdentityName: subscriptionTriggererIdentityName longestBuildPathUpdaterIdentityName: longestBuildPathUpdaterIdentityName feedCleanerIdentityName: feedCleanerIdentityName + contributorRole: contributorRole } } diff --git a/eng/service-templates/ProductConstructionService/provision.ps1 b/eng/service-templates/ProductConstructionService/provision.ps1 index c75893135e..ff4b2ac6fb 100644 --- a/eng/service-templates/ProductConstructionService/provision.ps1 +++ b/eng/service-templates/ProductConstructionService/provision.ps1 @@ -1,5 +1,6 @@ param( - [Parameter(Mandatory=$true)][string]$subscriptionName + [Parameter(Mandatory=$true)][string]$subscriptionName, + [Parameter(Mandatory=$true)][string]$bicepparamFileName ) az account set --subscription $subscriptionName @@ -7,5 +8,5 @@ az account set --subscription $subscriptionName # creates a resource group `product-construction-service` in West US 2 az group create --name product-construction-service --location "West US 2" -$provisionFilePath = Join-Path -Path $PSScriptRoot -ChildPath "provision.bicep" -az deployment group create --resource-group product-construction-service --template-file $provisionFilePath --name deploy \ No newline at end of file +$paramFile = Join-Path -Path $PSScriptRoot -ChildPath $bicepparamFileName +az deployment group create --resource-group product-construction-service --parameters $paramFile --name deploy \ No newline at end of file diff --git a/eng/service-templates/ProductConstructionService/redis.bicep b/eng/service-templates/ProductConstructionService/redis.bicep index 214dcf80a9..b36f9100c0 100644 --- a/eng/service-templates/ProductConstructionService/redis.bicep +++ b/eng/service-templates/ProductConstructionService/redis.bicep @@ -34,7 +34,7 @@ resource pcsRedisDataContributorRoleAssignment 'Microsoft.Cache/redis/accessPoli // allow redis cache read / write access to the service's identity resource deploymentRedisDataContributorRoleAssignment 'Microsoft.Cache/redis/accessPolicyAssignments@2024-03-01' = { - name: guid(subscription().id, resourceGroup().id, 'pcsDataContributor') + name: guid(subscription().id, resourceGroup().id, 'deploymentDataContributor') parent: redisCache properties: { accessPolicyName: 'Data Contributor' diff --git a/eng/service-templates/ProductConstructionService/staging.bicepparam b/eng/service-templates/ProductConstructionService/staging.bicepparam new file mode 100644 index 0000000000..d0cfeb23af --- /dev/null +++ b/eng/service-templates/ProductConstructionService/staging.bicepparam @@ -0,0 +1,57 @@ +using 'provision.bicep' + +param location = 'westus2' + +param containerRegistryName = 'productconstructionint' + +param containerCpuCoreCount = '1.0' + +param containerMemory = '2Gi' + +param aspnetcoreEnvironment = 'Staging' + +param applicationInsightsName = 'product-construction-service-ai-int' + +param keyVaultName = 'ProductConstructionInt' + +param devKeyVaultName = 'ProductConstructionDev' + +param azureCacheRedisName = 'product-construction-service-redis-int' + +param logAnalyticsName = 'product-construction-service-workspace-int' + +param containerEnvironmentName = 'product-construction-service-env-int' + +param productConstructionServiceName = 'product-construction-int' + +param storageAccountName = 'productconstructionint' + +param pcsIdentityName = 'ProductConstructionServiceInt' + +param deploymentIdentityName = 'ProductConstructionServiceDeploymentInt' + +param containerImageName = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' + +param virtualNetworkName = 'product-construction-service-vnet-int' + +param productConstructionServiceSubnetName = 'product-construction-service-subnet' + +param subscriptionTriggererIdentityName = 'SubscriptionTriggererInt' + +param subscriptionTriggererWeeklyJobName = 'sub-triggerer-weekly-int' + +param subscriptionTriggererTwiceDailyJobName = 'sub-triggerer-twicedaily-int' + +param subscriptionTriggererDailyJobName = 'sub-triggerer-daily-int' + +param longestBuildPathUpdaterIdentityName = 'LongestBuildPathUpdaterInt' + +param longestBuildPathUpdaterJobName = 'longest-path-updater-job-int' + +param feedCleanerJobName = 'feed-cleaner-int' + +param feedCleanerIdentityName = 'FeedCleanerInt' + +param networkSecurityGroupName = 'product-construction-service-nsg-int' + +param infrastructureResourceGroupName = 'product-construction-service-ip-int' diff --git a/eng/templates/jobs/e2e-pcs-tests.yml b/eng/templates/jobs/e2e-pcs-tests.yml index 1750a1750b..05ead7ae11 100644 --- a/eng/templates/jobs/e2e-pcs-tests.yml +++ b/eng/templates/jobs/e2e-pcs-tests.yml @@ -17,13 +17,22 @@ jobs: # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=20&path=Publish-Build-Assets # Required for MaestroAppClientId, MaestroStagingAppClientId - group: Publish-Build-Assets - - group: MaestroInt KeyVault - - name: PcsTestEndpoint - value: https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io - - name: ScenarioTestSubscription - value: "Darc: Maestro Staging" - - name: MaestroAppId - value: $(MaestroStagingAppClientId) + - ${{ if ne(variables['Build.SourceBranch'], 'refs/heads/production') }}: + - group: MaestroInt KeyVault + - name: PcsTestEndpoint + value: https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io + - name: ScenarioTestSubscription + value: "Darc: Maestro Staging" + - name: MaestroAppId + value: $(MaestroStagingAppClientId) + - ${{ else }}: + - group: MaestroProd KeyVault + - name: PcsTestEndpoint + value: https://product-construction-prod.wittysky-0c79e3cc.westus2.azurecontainerapps.io + - name: ScenarioTestSubscription + value: "Darc: Maestro Production" + - name: MaestroAppId + value: $(MaestroAppClientId) steps: - download: current displayName: Download Darc @@ -49,8 +58,11 @@ jobs: displayName: Install .NET - powershell: | + $darcNupkg = Get-ChildItem -Path $(Pipeline.Workspace)\PackageArtifacts -Filter Microsoft.DotNet.Darc* + $darcNupkg.Name -match "Microsoft.DotNet.Darc.(.*).nupkg" + $darcVersion = $Matches[1] mkdir darc - .\.dotnet\dotnet tool install Microsoft.DotNet.Darc --prerelease --tool-path .\darc --add-source $(Pipeline.Workspace)\PackageArtifacts + .\.dotnet\dotnet tool install Microsoft.DotNet.Darc --tool-path .\darc --add-source $(Pipeline.Workspace)\PackageArtifacts --version $darcVersion displayName: Install Darc - task: AzureCLI@2 diff --git a/src/Maestro/Client/src/MaestroApiOptions.cs b/src/Maestro/Client/src/MaestroApiOptions.cs index d6d549c4ba..e7385abe87 100644 --- a/src/Maestro/Client/src/MaestroApiOptions.cs +++ b/src/Maestro/Client/src/MaestroApiOptions.cs @@ -22,6 +22,7 @@ public partial class MaestroApiOptions public const string StagingMaestroUri = "https://maestro.int-dot.net/"; public const string OldPcsStagingUri = "https://maestro-int.westus2.cloudapp.azure.com/"; + public const string PcsProdUri = "https://product-construction-prod.wittysky-0c79e3cc.westus2.azurecontainerapps.io/"; public const string PcsStagingUri = "https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"; public const string PcsLocalUri = "https://localhost:53180/"; @@ -34,6 +35,7 @@ public partial class MaestroApiOptions [PcsStagingUri.TrimEnd('/')] = MaestroStagingAppId, [PcsLocalUri.TrimEnd('/')] = MaestroStagingAppId, + [PcsProdUri.TrimEnd('/')] = MaestroProductionAppId, [ProductionMaestroUri.TrimEnd('/')] = MaestroProductionAppId, [OldProductionMaestroUri.TrimEnd('/')] = MaestroProductionAppId, }; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Production.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Production.json new file mode 100644 index 0000000000..3291ac6785 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Production.json @@ -0,0 +1,28 @@ +{ + "KeyVaultName": "ProductConstructionProd", + "ConnectionStrings": { + "queues": "https://productconstructionprod.queue.core.windows.net", + "redis": "product-construction-service-redis-prod.redis.cache.windows.net:6380,ssl=true" + }, + "ManagedIdentityClientId": "e49bf24a-ec75-490b-803b-6fad99d19159", + "VmrUri": "https://github.com/maestro-auth-test/dnceng-vmr", + "BuildAssetRegistrySqlConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False; User Id=USER_ID_PLACEHOLDER", + "Kusto": { + "Database": "engineeringdata", + "KustoClusterUri": "https://engsrvprod.westus.kusto.windows.net" + }, + "DataProtection": { + "KeyBlobUri": "https://productconstructionprod.blob.core.windows.net/dataprotection/keys.xml", + "DataProtectionKeyUri": "https://productconstructionprod.vault.azure.net/keys/data-protection-encryption-key/" + }, + "EntraAuthentication": { + // https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ManagedAppMenuBlade/~/Overview/objectId/caf36d9b-2940-4270-9a1d-c494eda6ea18/appId/54c17f3d-7325-4eca-9db7-f090bfc765a8/preferredSingleSignOnMode~/null/servicePrincipalType/Application/fromNav/ + "ClientId": "54c17f3d-7325-4eca-9db7-f090bfc765a8", + "Scope": [ "api://54c17f3d-7325-4eca-9db7-f090bfc765a8/Maestro.User" ] + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "e49bf24a-ec75-490b-803b-6fad99d19159" + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json index 4c3b8bbf6d..4ece3ccc3e 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json @@ -6,10 +6,6 @@ }, "ManagedIdentityClientId": "1d43ba8a-c2a6-4fad-b064-6d8c16fc0745", "VmrUri": "https://github.com/maestro-auth-test/dnceng-vmr", - "Maestro": { - "Uri": "https://maestro.int-dot.net/", - "Token": "[vault(product-construction-service-int-token)]" - }, "BuildAssetRegistrySqlConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False; User Id=USER_ID_PLACEHOLDER", "Kusto": { "Database": "engineeringdata", diff --git a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs index f4259736cb..ca4f0a75a5 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs @@ -21,7 +21,8 @@ public partial class ProductConstructionServiceApiOptions public const string OldProductionMaestroUri = "https://maestro-prod.westus2.cloudapp.azure.com/"; public const string StagingMaestroUri = "https://maestro.int-dot.net/"; - public const string OldPcsStagingUri = "https://maestro-int.westus2.cloudapp.azure.com/"; + public const string OldStagingMaestroUri = "https://maestro-int.westus2.cloudapp.azure.com/"; + public const string PcsProdUri = "https://product-construction-prod.wittysky-0c79e3cc.westus2.azurecontainerapps.io/"; public const string PcsStagingUri = "https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"; public const string PcsLocalUri = "https://localhost:53180/"; @@ -30,12 +31,13 @@ public partial class ProductConstructionServiceApiOptions private static readonly Dictionary EntraAppIds = new Dictionary { [StagingMaestroUri.TrimEnd('/')] = MaestroStagingAppId, - [OldPcsStagingUri.TrimEnd('/')] = MaestroStagingAppId, + [OldStagingMaestroUri.TrimEnd('/')] = MaestroStagingAppId, [PcsStagingUri.TrimEnd('/')] = MaestroStagingAppId, [PcsLocalUri.TrimEnd('/')] = MaestroStagingAppId, [ProductionMaestroUri.TrimEnd('/')] = MaestroProductionAppId, [OldProductionMaestroUri.TrimEnd('/')] = MaestroProductionAppId, + [PcsProdUri.TrimEnd('/')] = MaestroProductionAppId }; /// diff --git a/src/ProductConstructionService/ProductConstructionService.Deployment/Deployer.cs b/src/ProductConstructionService/ProductConstructionService.Deployment/Deployer.cs index de647ceef2..f72620ce82 100644 --- a/src/ProductConstructionService/ProductConstructionService.Deployment/Deployer.cs +++ b/src/ProductConstructionService/ProductConstructionService.Deployment/Deployer.cs @@ -41,8 +41,8 @@ public Deployer( ArmClient client = armClient; SubscriptionResource subscription = client.GetSubscriptionResource(new ResourceIdentifier($"/subscriptions/{_options.SubscriptionId}")); - _resourceGroup = subscription.GetResourceGroups().Get("product-construction-service"); - _containerApp = _resourceGroup.GetContainerApp("product-construction-int").Value; + _resourceGroup = subscription.GetResourceGroups().Get(_options.ResourceGroupName); + _containerApp = _resourceGroup.GetContainerApp(_options.ContainerAppName).Value; _redisCacheFactory = redisCacheFactory; _serviceProvider = serviceProvider; _logger = logger; @@ -60,24 +60,33 @@ public async Task DeployAsync() var activeRevisionTrafficWeight = trafficWeights.FirstOrDefault(weight => weight.Weight == 100) ?? throw new ArgumentException("Container app has no active revision, please investigate manually"); - var activeRevision = (await _containerApp.GetContainerAppRevisionAsync(activeRevisionTrafficWeight.RevisionName)).Value; - var replicas = activeRevision.GetContainerAppReplicas().ToList(); + string inactiveRevisionLabel; + // When we create the ACA, the first revision won't have a name + if (activeRevisionTrafficWeight.RevisionName == null) + { + inactiveRevisionLabel = "blue"; + } + else + { + var activeRevision = (await _containerApp.GetContainerAppRevisionAsync(activeRevisionTrafficWeight.RevisionName)).Value; + var replicas = activeRevision.GetContainerAppReplicas().ToList(); - _logger.LogInformation("Currently active revision {revisionName} with label {label}", - activeRevisionTrafficWeight.RevisionName, - activeRevisionTrafficWeight.Label); + _logger.LogInformation("Currently active revision {revisionName} with label {label}", + activeRevisionTrafficWeight.RevisionName, + activeRevisionTrafficWeight.Label); - // Determine the label of the inactive revision - string inactiveRevisionLabel = activeRevisionTrafficWeight.Label == "blue" ? "green" : "blue"; + // Determine the label of the inactive revision + inactiveRevisionLabel = activeRevisionTrafficWeight.Label == "blue" ? "green" : "blue"; - _logger.LogInformation("Next revision will be deployed with label {inactiveLabel}", inactiveRevisionLabel); - _logger.LogInformation("Removing label {inactiveLabel} from inactive revision", inactiveRevisionLabel); + _logger.LogInformation("Next revision will be deployed with label {inactiveLabel}", inactiveRevisionLabel); + _logger.LogInformation("Removing label {inactiveLabel} from inactive revision", inactiveRevisionLabel); - // Cleanup all revisions except the currently active one - await CleanupRevisionsAsync(trafficWeights.Where(weight => weight != activeRevisionTrafficWeight)); + // Cleanup all revisions except the currently active one + await CleanupRevisionsAsync(trafficWeights.Where(weight => weight != activeRevisionTrafficWeight)); - // Tell the active revision to finish current work items and stop processing new ones - await StopProcessingNewJobs(activeRevisionTrafficWeight.RevisionName); + // Tell the active revision to finish current work items and stop processing new ones + await StopProcessingNewJobs(activeRevisionTrafficWeight.RevisionName); + } var newRevisionName = $"{_options.ContainerAppName}--{_options.NewImageTag}"; var newImageFullUrl = $"{_options.ContainerRegistryName}.azurecr.io/{_options.ImageName}:{_options.NewImageTag}"; diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Production.json b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Production.json new file mode 100644 index 0000000000..9b74b68cbf --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Production.json @@ -0,0 +1,9 @@ +{ + "BuildAssetRegistrySqlConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False; User Id=USER_ID_PLACEHOLDER", + "AzureDevOps": { + "default": { + "ManagedIdentityId": "520f92da-8df7-4bdf-afcd-400caf2c23b6" + } + }, + "ManagedIdentityClientId": "520f92da-8df7-4bdf-afcd-400caf2c23b6" +} diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/appsettings.Production.json b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/appsettings.Production.json new file mode 100644 index 0000000000..c12beb83c9 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/appsettings.Production.json @@ -0,0 +1,8 @@ +{ + "BuildAssetRegistrySqlConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False; User Id=USER_ID_PLACEHOLDER", + "Kusto": { + "Database": "engineeringdata", + "KustoClusterUri": "https://engsrvprod.westus.kusto.windows.net" + }, + "ManagedIdentityClientId": "f15e5ffe-fe05-4430-8226-a8ab343d0487" +} diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Production.json b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Production.json new file mode 100644 index 0000000000..46da7041cb --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Production.json @@ -0,0 +1,11 @@ +{ + "ConnectionStrings": { + "queues": "https://productconstructionprod.queue.core.windows.net" + }, + "ManagedIdentityClientId": "3a1499d3-24dc-4e87-960b-3cff340f0fd4", + "BuildAssetRegistrySqlConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False; User Id=USER_ID_PLACEHOLDER", + "Kusto": { + "Database": "engineeringdata", + "KustoClusterUri": "https://engsrvprod.westus.kusto.windows.net" + } +} diff --git a/src/ProductConstructionService/Readme.md b/src/ProductConstructionService/Readme.md index 6f11e52f70..017d3e35df 100644 --- a/src/ProductConstructionService/Readme.md +++ b/src/ProductConstructionService/Readme.md @@ -138,22 +138,19 @@ If the service is being recreated and the same Managed Identity name is reused, Once the resources are created and configured: - Go to the newly created User Assigned Managed Identity (the one that's assigned to the container app, not the deployment one) - - Copy the Client ID, and paste it in the correct appconfig.json, under `ManagedIdentityClientId` - - Add this identity as a user to AzDO so it can get AzDO tokens (you'll need a saw for this). You might have to remove the old user identity before doing this - - It needs to be able to manage code / pull requests and manage feeds (this is done in the artifact section). + - Copy the Client ID, and paste it in the correct appconfig.json, under `ManagedIdentityClientId`. Do this for all services (PCS, Subscription Triggerer, LongestBuildPathUpdater and FeedCleaner) + - Add the PCS and FeedCleaner identity as a user to AzDO so it can get AzDO tokens (you'll need a saw for this). You might have to remove the old user identity before doing this + - The PCS identity needs to be able to manage code / pull requests and manage feeds (this is done in the artifact section). + - The FeedCleaner identity needs to be in the `Feed Managers` permissions group in `dnceng/internal` - Update the `ProductConstructionServiceDeploymentProd` (or `ProductConstructionServiceDeploymentInt`) Service Connection with the new MI information (you'll also have to create a Federated Credential in the MI) - - Update the default PCS URI in `ProductConstructionServiceApiOptions`. + - Update the appropriate PCS URI in `ProductConstructionServiceApiOptions` and `MaestroApiOptions`. We're not able to configure a few Kusto things in bicep: - Give the PCS Managed Identity the permissions it needs: - Go to the Kusto Cluster, and select the database you want the MI to have access to - Go to permissions -> Add -> Viewer and select the newly created PCS Managed Identity - - Create a private endpoint between the Kusto cluster and PCS - - Go to the Kusto cluster -> Networking -> Private endpoint connections -> + Private endpoint - - Select the appropriate subscription and resource group. Name the private endpoint something meaningful, like `pcs-kusto-private-connection` - - On the Resource page, set the `Target sub-resource` to `cluster` - - On the Virtual Network page, select the product-construction-service-vntet-int/prod, and the private-endpoints-subnet, leave the rest as default - - leave the rest of the settings as default + +Give the Deployment Identity Reader role for the whole subscription. The last part is setting up the pipeline: - Make sure all of the resources referenced in the yaml have the correct names