From 5bab5a0267c3aead79c3bfa80ae3470797fbf5d0 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:05:02 +0200 Subject: [PATCH 01/13] Run scenario tests in PCS pipeline (#3917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Přemek Vysoký --- .vault-config/product-construction-dev.yaml | 7 ++ ...pipelines-product-construction-service.yml | 63 +++++++++- eng/templates/jobs/e2e-pcs-tests.yml | 111 ++++++++++++++++++ eng/templates/steps/docker-build.yml | 2 +- 4 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 eng/templates/jobs/e2e-pcs-tests.yml diff --git a/.vault-config/product-construction-dev.yaml b/.vault-config/product-construction-dev.yaml index d73c9d807e..56c9d14cf4 100644 --- a/.vault-config/product-construction-dev.yaml +++ b/.vault-config/product-construction-dev.yaml @@ -25,3 +25,10 @@ secrets: location: engkeyvault name: BotAccount-dotnet-bot gitHubBotAccountName: dotnet-bot + + github: + type: github-app-secret + parameters: + hasPrivateKey: true + hasWebhookSecret: false + hasOAuthSecret: true \ No newline at end of file diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index 16bad7e91a..52cc302cd8 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -1,6 +1,3 @@ -# Changes the format of the Build.BuildNumber variable -name: $(Date:yyyyMMdd)$(Rev:r) - trigger: batch: true branches: @@ -21,6 +18,14 @@ variables: value: product-construction-service.api - name: diffFolder value: $(Build.ArtifactStagingDirectory)/diff +- name: _TeamName + value: DotNetCore +- name: _PublishUsingPipelines + value: true +- name: _DotNetArtifactsCategory + value: .NETCore +- name: _SignType + value: test - ${{ if ne(variables['Build.SourceBranch'], 'refs/heads/production') }}: - name: subscriptionId value: e6b5f9f5-0ca4-4351-879b-014d78400ec2 @@ -133,3 +138,55 @@ stages: - publish: $(diffFolder) displayName: Upload snapshot diff artifact: DeploymentDiff + + - job: BuildAndPublish + displayName: Build and Publish Repo + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals 1es-windows-2019 + + variables: + - name: _BuildConfig + value: Release + - name: _BuildArgs + value: > + /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER)" + + steps: + - checkout: self + + - template: /eng/templates/steps/build.yml + parameters: + configuration: $(_BuildConfig) + buildArgs: $(_BuildArgs) + + - publish: $(Build.SourcesDirectory)\artifacts\bin\ProductConstructionService.ScenarioTests\$(_BuildConfig)\net8.0\publish + artifact: ProductConstructionService.ScenarioTests + +- stage: TestPCS + displayName: Run E2E Product Construction Service Tests + dependsOn: + - DeployPCS + + jobs: + - template: /eng/templates/jobs/e2e-pcs-tests.yml + parameters: + name: scenarioTests_GitHub + displayName: GitHub tests + testFilter: 'TestCategory=GitHub' + + - template: /eng/templates/jobs/e2e-pcs-tests.yml + parameters: + name: scenarioTests_AzDO + displayName: AzDO tests + testFilter: 'TestCategory=AzDO' + + - template: /eng/templates/jobs/e2e-pcs-tests.yml + parameters: + name: scenarioTests_Other + displayName: Other tests + testFilter: 'TestCategory!=GitHub&TestCategory!=AzDO' diff --git a/eng/templates/jobs/e2e-pcs-tests.yml b/eng/templates/jobs/e2e-pcs-tests.yml new file mode 100644 index 0000000000..1750a1750b --- /dev/null +++ b/eng/templates/jobs/e2e-pcs-tests.yml @@ -0,0 +1,111 @@ +parameters: +- name: name + type: string +- name: displayName + type: string +- name: testFilter + type: string + +jobs: +- job: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 60 + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals 1es-windows-2019 + variables: + # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=20&path=Publish-Build-Assets + # Required for MaestroAppClientId, MaestroStagingAppClientId + - group: Publish-Build-Assets + - group: MaestroInt KeyVault + - name: PcsTestEndpoint + value: https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io + - name: ScenarioTestSubscription + value: "Darc: Maestro Staging" + - name: MaestroAppId + value: $(MaestroStagingAppClientId) + steps: + - download: current + displayName: Download Darc + artifact: PackageArtifacts + patterns: Microsoft.DotNet.Darc.* + + - download: current + displayName: Download ScenarioTets + artifact: ProductConstructionService.ScenarioTests + + - task: NuGetToolInstaller@1 + displayName: Use NuGet + inputs: + versionSpec: 5.3.x + + - powershell: | + . .\eng\common\tools.ps1 + InitializeDotNetCli -install:$true + .\.dotnet\dotnet workload install aspire + displayName: Install .NET and Aspire Workload + + - powershell: .\eng\common\build.ps1 -restore + displayName: Install .NET + + - powershell: | + mkdir darc + .\.dotnet\dotnet tool install Microsoft.DotNet.Darc --prerelease --tool-path .\darc --add-source $(Pipeline.Workspace)\PackageArtifacts + displayName: Install Darc + + - task: AzureCLI@2 + name: GetAuthInfo + displayName: Get auth information + inputs: + azureSubscription: ${{ variables.ScenarioTestSubscription }} + addSpnToEnvironment: true + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + # Fetch token used for scenario tests + $token = (az account get-access-token --resource "$(MaestroAppId)" | ConvertFrom-Json).accessToken + echo "##vso[task.setvariable variable=Token;isOutput=true;isSecret=true]$token" + + # Set variables with auth info for tests below + echo "##vso[task.setvariable variable=ServicePrincipalId;isOutput=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=FederatedToken;isOutput=true;isSecret=true]$env:idToken" + echo "##vso[task.setvariable variable=TenantId;isOutput=true]$env:tenantId" + + - template: /eng/common/templates-official/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: "ArcadeServicesInternal" + outputVariableName: "AzdoToken" + + - task: DotNetCoreCLI@2 + displayName: Run E2E tests + inputs: + command: custom + projects: | + $(Pipeline.Workspace)/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.dll + custom: test + arguments: > + --filter "TestCategory=PostDeployment&${{ parameters.testFilter }}" + --no-build + --logger "trx;LogFilePrefix=TestResults-" + --parallel + -- + "RunConfiguration.ResultsDirectory=$(Build.ArtifactStagingDirectory)\TestResults" + RunConfiguration.MapCpuCount=4 + env: + PCS_BASEURI: ${{ variables.PcsTestEndpoint }} + PCS_TOKEN: $(GetAuthInfo.Token) + GITHUB_TOKEN: $(maestro-scenario-test-github-token) + AZDO_TOKEN: $(AzdoToken) + DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts + DARC_DIR: $(Build.SourcesDirectory)\darc + DARC_IS_CI: true + + - task: PublishTestResults@2 + displayName: Publish Core Test Results + condition: succeededOrFailed() + inputs: + testRunner: VSTest + testResultsFiles: '**/TestResults-*' + searchFolder: $(Build.ArtifactStagingDirectory)\TestResults + testRunTitle: ${{ parameters.displayName }} + mergeTestResults: true diff --git a/eng/templates/steps/docker-build.yml b/eng/templates/steps/docker-build.yml index 1add68f816..ffc5fdfbb7 100644 --- a/eng/templates/steps/docker-build.yml +++ b/eng/templates/steps/docker-build.yml @@ -9,7 +9,7 @@ steps: - powershell: | Write-Host "Dev branch suffix is ${{ parameters.devBranchSuffix }}" $shortSha = "$(Build.SourceVersion)".Substring(0,10) - $newDockerTag = "$(Build.BuildNumber)-$(System.JobAttempt)-$shortSha${{ parameters.devBranchSuffix }}" + $newDockerTag = "$(Build.BuildNumber)-$(System.JobAttempt)-$shortSha${{ parameters.devBranchSuffix }}".Replace(".", "") Write-Host "##vso[task.setvariable variable=newDockerImageTag;isOutput=true]$newDockerTag" Write-Host "set newDockerImageTag to $newDockerTag" displayName: Generate docker image tag From 580a2287010ce54dc05ddff08470fa456a37b6fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 4 Sep 2024 16:37:25 +0200 Subject: [PATCH 02/13] Fix the PCS pipeline --- azure-pipelines-product-construction-service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index 52cc302cd8..48341021b8 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -154,7 +154,7 @@ stages: /p:TeamName=$(_TeamName) /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER)" + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) steps: - checkout: self From 05bd01b1e0aa2bb7db80c5088b7b1173d44261fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 4 Sep 2024 17:06:08 +0200 Subject: [PATCH 03/13] Do not use GitHub PAT in VMR initialization (#3926) --- .../ProductConstructionService.Api/PcsStartup.cs | 4 +--- .../VirtualMonoRepo/VmrConfiguration.cs | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 0542b150c8..cc8a5d395a 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -53,7 +53,6 @@ private static class ConfigurationKeys // Secrets coming from the KeyVault public const string GitHubClientId = $"{KeyVaultSecretPrefix}github-app-id"; public const string GitHubClientSecret = $"{KeyVaultSecretPrefix}github-app-private-key"; - public const string GitHubToken = $"{KeyVaultSecretPrefix}BotAccount-dotnet-bot-repo-PAT"; // Configuration from appsettings.json public const string AzureDevOpsConfiguration = "AzureDevOps"; @@ -148,7 +147,6 @@ internal static async Task ConfigurePcs( string? managedIdentityId = builder.Configuration[ConfigurationKeys.ManagedIdentityId]; string databaseConnectionString = builder.Configuration.GetRequiredValue(ConfigurationKeys.DatabaseConnectionString) .Replace(SqlConnectionStringUserIdPlaceholder, managedIdentityId); - string? gitHubToken = builder.Configuration[ConfigurationKeys.GitHubToken]; builder.Services.Configure(ConfigurationKeys.AzureDevOpsConfiguration, (o, s) => s.Bind(o)); DefaultAzureCredential azureCredential = new(new DefaultAzureCredentialOptions @@ -174,7 +172,7 @@ internal static async Task ConfigurePcs( builder.AddBuildAssetRegistry(); builder.AddWorkItemQueues(azureCredential, waitForInitialization: initializeService); builder.AddDependencyFlowProcessors(); - builder.AddVmrRegistrations(gitHubToken); + builder.AddVmrRegistrations(); builder.AddMaestroApiClient(managedIdentityId); builder.AddGitHubClientFactory( builder.Configuration[ConfigurationKeys.GitHubClientId], diff --git a/src/ProductConstructionService/ProductConstructionService.Api/VirtualMonoRepo/VmrConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.Api/VirtualMonoRepo/VmrConfiguration.cs index dee4485c27..33e0cdf3e0 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/VirtualMonoRepo/VmrConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/VirtualMonoRepo/VmrConfiguration.cs @@ -14,12 +14,12 @@ internal static class VmrConfiguration private const string VmrReadyHealthCheckName = "VmrReady"; private const string VmrReadyHealthCheckTag = "vmrReady"; - public static void AddVmrRegistrations(this WebApplicationBuilder builder, string? gitHubToken) + public static void AddVmrRegistrations(this WebApplicationBuilder builder) { string vmrPath = builder.Configuration.GetRequiredValue(VmrPathKey); string tmpPath = builder.Configuration.GetRequiredValue(TmpPathKey); - builder.Services.AddVmrManagers("git", vmrPath, tmpPath, gitHubToken, azureDevOpsToken: null); + builder.Services.AddVmrManagers("git", vmrPath, tmpPath, gitHubToken: null, azureDevOpsToken: null); } public static void AddVmrInitialization(this WebApplicationBuilder builder) From 26f3dc05769700710973f86d5056574b9272f29f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 5 Sep 2024 12:04:25 +0200 Subject: [PATCH 04/13] Make E2E Arcade tests use maestro-auth-test arcade (#3928) --- docs/DevGuide.md | 1 + .../DarcLib/Helpers/DependencyFileManager.cs | 2 +- src/ProductConstructionService/Readme.md | 1 + .../MaestroScenarioTestBase.cs | 24 +++++----- .../ScenarioTests_SdkUpdate.cs | 44 +++++++++++-------- .../ScenarioTestBase.cs | 32 +++++++------- .../ScenarioTests_SdkUpdate.cs | 40 ++++++++++------- 7 files changed, 81 insertions(+), 63 deletions(-) diff --git a/docs/DevGuide.md b/docs/DevGuide.md index 9aa4912197..5f638014d0 100644 --- a/docs/DevGuide.md +++ b/docs/DevGuide.md @@ -17,6 +17,7 @@ ('https://github.com/maestro-auth-test/maestro-test', 289474), ('https://github.com/maestro-auth-test/maestro-test2', 289474), ('https://github.com/maestro-auth-test/maestro-test3', 289474), + ('https://github.com/maestro-auth-test/arcade', 289474), ('https://github.com/maestro-auth-test/dnceng-vmr', 289474); ``` diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs index 168aced128..ad2d0a2b5f 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs @@ -362,7 +362,7 @@ private Dictionary UpdateDotnetVersionGlobalJson(Se { try { - if (SemanticVersion.TryParse(globalJson.SelectToken("tools.dotnet").ToString(), out SemanticVersion repoDotnetVersion)) + if (SemanticVersion.TryParse(globalJson.SelectToken("tools.dotnet")?.ToString(), out SemanticVersion repoDotnetVersion)) { if (repoDotnetVersion.CompareTo(incomingDotnetVersion) < 0) { diff --git a/src/ProductConstructionService/Readme.md b/src/ProductConstructionService/Readme.md index b85e65a5cc..c7431e9be7 100644 --- a/src/ProductConstructionService/Readme.md +++ b/src/ProductConstructionService/Readme.md @@ -17,6 +17,7 @@ ('https://github.com/maestro-auth-test/maestro-test', 289474), ('https://github.com/maestro-auth-test/maestro-test2', 289474), ('https://github.com/maestro-auth-test/maestro-test3', 289474), + ('https://github.com/maestro-auth-test/arcade', 289474), ('https://github.com/maestro-auth-test/dnceng-vmr', 289474); ``` 1. Install Docker Desktop: https://www.docker.com/products/docker-desktop diff --git a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs index 628cd785f9..d65bb5139c 100644 --- a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs +++ b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs @@ -61,7 +61,7 @@ public void SetTestParameters(TestParameters parameters) { Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); - var attempts = 10; + var attempts = 20; while (attempts-- > 0) { IReadOnlyList prs = await GitHubApi.PullRequest.GetAllForRepository(repo.Id, new Octokit.PullRequestRequest @@ -85,13 +85,13 @@ public void SetTestParameters(TestParameters parameters) throw new MaestroTestException($"More than one pull request found in {targetRepo} targeting {targetBranch}"); } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new MaestroTestException($"No pull request was created in {targetRepo} targeting {targetBranch}"); } - private async Task WaitForUpdatedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) + private async Task WaitForUpdatedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 20) { Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); @@ -106,13 +106,13 @@ public void SetTestParameters(TestParameters parameters) return pr; } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new MaestroTestException($"The created pull request for {targetRepo} targeting {targetBranch} was not updated with subsequent subscriptions after creation"); } - private async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) + private async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 20) { Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); @@ -127,7 +127,7 @@ private async Task WaitForMergedPullRequestAsync(string targetRepo, string return true; } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new MaestroTestException($"The created pull request for {targetRepo} targeting {targetBranch} was not merged within {attempts} minutes"); @@ -138,7 +138,7 @@ private async Task GetAzDoPullRequestIdAsync(string targetRepoName, string var searchBaseUrl = GetAzDoRepoUrl(targetRepoName); IEnumerable prs = new List(); - var attempts = 10; + var attempts = 20; while (attempts-- > 0) { try @@ -162,7 +162,7 @@ private async Task GetAzDoPullRequestIdAsync(string targetRepoName, string throw new MaestroTestException($"More than one pull request found in {targetRepoName} targeting {targetBranch}"); } - await Task.Delay(60 * 1000); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new MaestroTestException($"No pull request was created in {searchBaseUrl} targeting {targetBranch}"); @@ -201,7 +201,7 @@ private async Task> GetAzDoPullRequestAsync(in throw new Exception($"{nameof(expectedPRTitle)} must be defined for AzDo PRs that require an update"); } - for (var tries = 10; tries > 0; tries--) + for (var tries = 20; tries > 0; tries--) { PullRequest pr = await AzDoClient.GetPullRequestAsync($"{apiBaseUrl}/pullRequests/{pullRequestId}"); var trimmedTitle = Regex.Replace(pr.Title, @"\s+", " "); @@ -230,7 +230,7 @@ private async Task> GetAzDoPullRequestAsync(in }); } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new MaestroTestException($"The created pull request for {targetRepoName} targeting {targetBranch} was not updated with subsequent subscriptions after creation"); @@ -309,7 +309,7 @@ protected async Task CheckNonBatchedAzDoPullRequest( await CheckAzDoPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedDependencies, repoDirectory, false, isUpdated, expectedFeeds, notExpectedFeeds); } - protected async Task CheckAzDoPullRequest( + protected async Task CheckAzDoPullRequest( string expectedPRTitle, string targetRepoName, string targetBranch, @@ -348,6 +348,8 @@ protected async Task CheckAzDoPullRequest( sources.Should().NotContain(notExpectedFeeds); } } + + return pullRequest.Value.HeadBranch; } private async Task ValidatePullRequestDependencies(string pullRequestBaseBranch, List expectedDependencies, int tries = 1) diff --git a/test/Maestro.ScenarioTests/ScenarioTests_SdkUpdate.cs b/test/Maestro.ScenarioTests/ScenarioTests_SdkUpdate.cs index 3c6b90ce8d..66dca15b9b 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_SdkUpdate.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_SdkUpdate.cs @@ -35,29 +35,32 @@ public async Task ArcadeSdkUpdate_E2E(bool targetAzDO) _parameters = await TestParameters.GetAsync(); SetTestParameters(_parameters); - string testChannelName = "Test Channel " + _random.Next(int.MaxValue); + var testChannelName = "Test Channel " + _random.Next(int.MaxValue); + const string sourceOrg = "maestro-auth-test"; const string sourceRepo = "arcade"; - const string sourceRepoUri = "https://github.com/dotnet/arcade"; + const string sourceRepoUri = $"https://github.com/{sourceOrg}/{sourceRepo}"; const string sourceBranch = "dependencyflow-tests"; - const string sourceCommit = "0b36b99e29b1751403e23cfad0a7dff585818051"; + const string sourceCommit = "f3d51d2c9af2a3eb046fa54c5acdef9fb37db172"; const string newArcadeSdkVersion = "2.1.0"; var sourceBuildNumber = _random.Next(int.MaxValue).ToString(); - ImmutableList sourceAssets = ImmutableList.Create() - .Add(new AssetData(true) + ImmutableList sourceAssets = + [ + new AssetData(true) { Name = DependencyFileManager.ArcadeSdkPackageName, - Version = newArcadeSdkVersion, - }); + Version = newArcadeSdkVersion + } + ]; var targetRepo = "maestro-test2"; var targetBranch = "test/" + _random.Next(int.MaxValue).ToString(); await using AsyncDisposableValue channel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); await using AsyncDisposableValue sub = - await CreateSubscriptionAsync(testChannelName, sourceRepo, targetRepo, targetBranch, "none", targetIsAzDo: targetAzDO); + await CreateSubscriptionAsync(testChannelName, sourceRepo, targetRepo, targetBranch, "none", sourceOrg: sourceOrg, targetIsAzDo: targetAzDO); Build build = - await CreateBuildAsync(GetRepoUrl("dotnet", sourceRepo), sourceBranch, sourceCommit, sourceBuildNumber, sourceAssets); + await CreateBuildAsync(GetRepoUrl(sourceOrg, sourceRepo), sourceBranch, sourceCommit, sourceBuildNumber, sourceAssets); await using IAsyncDisposable _ = await AddBuildToChannelAsync(build.Id, testChannelName); @@ -76,7 +79,7 @@ await RunDarcAsync("add-dependency", await using IAsyncDisposable ___ = await PushGitBranchAsync("origin", targetBranch); await TriggerSubscriptionAsync(sub.Value); - string expectedTitle = $"[{targetBranch}] Update dependencies from dotnet/arcade"; + var expectedTitle = $"[{targetBranch}] Update dependencies from {sourceOrg}/{sourceRepo}"; DependencyDetail expectedDependency = new() { Name = DependencyFileManager.ArcadeSdkPackageName, @@ -87,9 +90,10 @@ await RunDarcAsync("add-dependency", Pinned = false, }; + string prHead; if (targetAzDO) { - await CheckAzDoPullRequest( + prHead = await CheckAzDoPullRequest( expectedTitle, targetRepo, targetBranch, @@ -99,16 +103,18 @@ await CheckAzDoPullRequest( isUpdated: false, expectedFeeds: null, notExpectedFeeds: null); - return; + } + else + { + Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); + pr.Title.Should().BeEquivalentTo(expectedTitle); + prHead = pr.Head.Ref; } - Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); - pr.Title.Should().BeEquivalentTo(expectedTitle); - - await CheckoutRemoteRefAsync(pr.MergeCommitSha); + await CheckoutRemoteRefAsync(prHead); - string dependencies = await RunDarcAsync("get-dependencies"); - string[] dependencyLines = dependencies.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries); + var dependencies = await RunDarcAsync("get-dependencies"); + var dependencyLines = dependencies.Split(['\n', '\r'], StringSplitOptions.RemoveEmptyEntries); dependencyLines.Should().BeEquivalentTo( [ $"Name: {DependencyFileManager.ArcadeSdkPackageName}", @@ -119,7 +125,7 @@ await CheckAzDoPullRequest( "Pinned: False", ]); - using TemporaryDirectory arcadeRepo = await CloneRepositoryAsync("dotnet", sourceRepo); + using TemporaryDirectory arcadeRepo = await CloneRepositoryAsync(sourceOrg, sourceRepo); using (ChangeDirectory(arcadeRepo.Directory)) { await CheckoutRemoteRefAsync(sourceCommit); diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs index b911b2e19f..0cbd2f579c 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs @@ -55,7 +55,7 @@ public void SetTestParameters(TestParameters parameters) { Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); - var attempts = 10; + var attempts = 20; while (attempts-- > 0) { IReadOnlyList prs = await GitHubApi.PullRequest.GetAllForRepository(repo.Id, new Octokit.PullRequestRequest @@ -79,13 +79,13 @@ public void SetTestParameters(TestParameters parameters) throw new ScenarioTestException($"More than one pull request found in {targetRepo} targeting {targetBranch}"); } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new ScenarioTestException($"No pull request was created in {targetRepo} targeting {targetBranch}"); } - private async Task WaitForUpdatedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) + private async Task WaitForUpdatedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 20) { Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); @@ -100,13 +100,13 @@ public void SetTestParameters(TestParameters parameters) return pr; } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new ScenarioTestException($"The created pull request for {targetRepo} targeting {targetBranch} was not updated with subsequent subscriptions after creation"); } - private async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) + private async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 20) { Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); @@ -121,7 +121,7 @@ private async Task WaitForMergedPullRequestAsync(string targetRepo, string return true; } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new ScenarioTestException($"The created pull request for {targetRepo} targeting {targetBranch} was not merged within {attempts} minutes"); @@ -132,7 +132,7 @@ private async Task GetAzDoPullRequestIdAsync(string targetRepoName, string var searchBaseUrl = GetAzDoRepoUrl(targetRepoName); IEnumerable prs = new List(); - var attempts = 10; + var attempts = 20; while (attempts-- > 0) { try @@ -156,7 +156,7 @@ private async Task GetAzDoPullRequestIdAsync(string targetRepoName, string throw new ScenarioTestException($"More than one pull request found in {targetRepoName} targeting {targetBranch}"); } - await Task.Delay(60 * 1000); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new ScenarioTestException($"No pull request was created in {searchBaseUrl} targeting {targetBranch}"); @@ -195,7 +195,7 @@ private async Task> GetAzDoPullRequestAsync(in throw new Exception($"{nameof(expectedPRTitle)} must be defined for AzDo PRs that require an update"); } - for (var tries = 10; tries > 0; tries--) + for (var tries = 20; tries > 0; tries--) { PullRequest pr = await AzDoClient.GetPullRequestAsync($"{apiBaseUrl}/pullRequests/{pullRequestId}"); var trimmedTitle = Regex.Replace(pr.Title, @"\s+", " "); @@ -214,8 +214,8 @@ private async Task> GetAzDoPullRequestAsync(in projectName, $"_apis/git/repositories/{targetRepoName}/pullrequests/{pullRequestId}", new NUnitLogger(), - "{ \"status\" : \"abandoned\"}" - ); + "{ \"status\" : \"abandoned\"}", + logFailure: false); } catch { @@ -224,7 +224,7 @@ private async Task> GetAzDoPullRequestAsync(in }); } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new ScenarioTestException($"The created pull request for {targetRepoName} targeting {targetBranch} was not updated with subsequent subscriptions after creation"); @@ -303,7 +303,7 @@ protected async Task CheckNonBatchedAzDoPullRequest( await CheckAzDoPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedDependencies, repoDirectory, false, isUpdated, expectedFeeds, notExpectedFeeds); } - protected async Task CheckAzDoPullRequest( + protected async Task CheckAzDoPullRequest( string expectedPRTitle, string targetRepoName, string targetBranch, @@ -342,6 +342,8 @@ protected async Task CheckAzDoPullRequest( sources.Should().NotContain(notExpectedFeeds); } } + + return pullRequest.Value.HeadBranch; } private async Task ValidatePullRequestDependencies(string pullRequestBaseBranch, List expectedDependencies, int tries = 1) @@ -889,7 +891,7 @@ protected async Task GetRepositoryPolicies(string repoUri, string branch return await RunDarcAsync("get-repository-policies", "--all", "--repo", repoUri, "--branch", branchName); } - protected async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, Octokit.PullRequest pr, Octokit.Repository repo, int attempts = 7) + protected async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, Octokit.PullRequest pr, Octokit.Repository repo, int attempts = 20) { while (attempts-- > 0) { @@ -901,7 +903,7 @@ protected async Task WaitForMergedPullRequestAsync(string targetRepo, string tar return; } - await Task.Delay(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromSeconds(30)); } throw new ScenarioTestException($"The created pull request for {targetRepo} targeting {targetBranch} was not merged within {attempts} minutes"); diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_SdkUpdate.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_SdkUpdate.cs index 4bdea79004..6c52aa514c 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_SdkUpdate.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_SdkUpdate.cs @@ -32,28 +32,31 @@ public async Task ArcadeSdkUpdate_E2E(bool targetAzDO) SetTestParameters(_parameters); var testChannelName = "Test Channel " + _random.Next(int.MaxValue); + const string sourceOrg = "maestro-auth-test"; const string sourceRepo = "arcade"; - const string sourceRepoUri = "https://github.com/dotnet/arcade"; + const string sourceRepoUri = $"https://github.com/{sourceOrg}/{sourceRepo}"; const string sourceBranch = "dependencyflow-tests"; - const string sourceCommit = "0b36b99e29b1751403e23cfad0a7dff585818051"; + const string sourceCommit = "f3d51d2c9af2a3eb046fa54c5acdef9fb37db172"; const string newArcadeSdkVersion = "2.1.0"; var sourceBuildNumber = _random.Next(int.MaxValue).ToString(); - ImmutableList sourceAssets = ImmutableList.Create() - .Add(new AssetData(true) + ImmutableList sourceAssets = + [ + new AssetData(true) { Name = DependencyFileManager.ArcadeSdkPackageName, - Version = newArcadeSdkVersion, - }); + Version = newArcadeSdkVersion + } + ]; var targetRepo = "maestro-test2"; var targetBranch = "test/" + _random.Next(int.MaxValue).ToString(); await using AsyncDisposableValue channel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); await using AsyncDisposableValue sub = - await CreateSubscriptionAsync(testChannelName, sourceRepo, targetRepo, targetBranch, "none", targetIsAzDo: targetAzDO); + await CreateSubscriptionAsync(testChannelName, sourceRepo, targetRepo, targetBranch, "none", sourceOrg: sourceOrg, targetIsAzDo: targetAzDO); Build build = - await CreateBuildAsync(GetRepoUrl("dotnet", sourceRepo), sourceBranch, sourceCommit, sourceBuildNumber, sourceAssets); + await CreateBuildAsync(GetRepoUrl(sourceOrg, sourceRepo), sourceBranch, sourceCommit, sourceBuildNumber, sourceAssets); await using IAsyncDisposable _ = await AddBuildToChannelAsync(build.Id, testChannelName); @@ -72,7 +75,7 @@ await RunDarcAsync("add-dependency", await using IAsyncDisposable ___ = await PushGitBranchAsync("origin", targetBranch); await TriggerSubscriptionAsync(sub.Value); - var expectedTitle = $"[{targetBranch}] Update dependencies from dotnet/arcade"; + var expectedTitle = $"[{targetBranch}] Update dependencies from {sourceOrg}/{sourceRepo}"; DependencyDetail expectedDependency = new() { Name = DependencyFileManager.ArcadeSdkPackageName, @@ -83,9 +86,10 @@ await RunDarcAsync("add-dependency", Pinned = false, }; + string prHead; if (targetAzDO) { - await CheckAzDoPullRequest( + prHead = await CheckAzDoPullRequest( expectedTitle, targetRepo, targetBranch, @@ -95,16 +99,18 @@ await CheckAzDoPullRequest( isUpdated: false, expectedFeeds: null, notExpectedFeeds: null); - return; + } + else + { + Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); + pr.Title.Should().BeEquivalentTo(expectedTitle); + prHead = pr.Head.Ref; } - Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); - pr.Title.Should().BeEquivalentTo(expectedTitle); - - await CheckoutRemoteRefAsync(pr.MergeCommitSha); + await CheckoutRemoteRefAsync(prHead); var dependencies = await RunDarcAsync("get-dependencies"); - var dependencyLines = dependencies.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries); + var dependencyLines = dependencies.Split(['\n', '\r'], StringSplitOptions.RemoveEmptyEntries); dependencyLines.Should().BeEquivalentTo( [ $"Name: {DependencyFileManager.ArcadeSdkPackageName}", @@ -115,7 +121,7 @@ await CheckAzDoPullRequest( "Pinned: False", ]); - using TemporaryDirectory arcadeRepo = await CloneRepositoryAsync("dotnet", sourceRepo); + using TemporaryDirectory arcadeRepo = await CloneRepositoryAsync(sourceOrg, sourceRepo); using (ChangeDirectory(arcadeRepo.Directory)) { await CheckoutRemoteRefAsync(sourceCommit); From 56199b02e291b8cf7ebd40ef4ac61ee74b47008e Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:37:23 +0200 Subject: [PATCH 05/13] Update Kusto query in rollout issue (#3927) --- .github/ISSUE_TEMPLATE/rollout-issue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/rollout-issue.md b/.github/ISSUE_TEMPLATE/rollout-issue.md index e64f67418d..fa4ed8470c 100644 --- a/.github/ISSUE_TEMPLATE/rollout-issue.md +++ b/.github/ISSUE_TEMPLATE/rollout-issue.md @@ -71,7 +71,7 @@ In case the services don't work as expected after the rollout, it's necessary to ## Rollout times -Use the following [Kusto query](https://dataexplorer.azure.com/clusters/engsrvprod/databases/engineeringdata?query=H4sIAAAAAAAAA51Qy07DMBC89ytWuTSWwg+k6gFUCfWCqhZxQSha4m1j5Eew10B5/DubUETgiE+r8ezM7GhqLUaCx0zx2PQY0RFTTOU2WBsyr3UNNvgDLEEtZpYYLolXOSKb4AUsG3NiVNDsTUy8YzxQDYmjGUGLfzAFbzOQd20cWeNpS22IOo3YOzx3JHEusrF6rWG5BDH49XUlCcH4cuI2dVEncsrOYTSvJK4YWaI6WdoN82CsKiCvBxRfJuhpuY/hgVqGyaH7EB1yw8JKPfpy2D770q6g6LrauUIa+ljMejmSR6Hb+SbSeS9qT2ihHElqfidqHFKLUnw5afOnchEcCyhk2OR7a1IHOUl1sMLYFkpV3/Ih8f/0V9TbcBwMbtAajUygR8iRZ3FYfALfQdHDGQIAAA==) to gather data about rollout times: +Use the following [Kusto query](https://dataexplorer.azure.com/clusters/engsrvprod/databases/engineeringdata?query=H4sIAAAAAAAAA52QP0%2FDQAzF934KK0tzUlgYU2UAtUJdUNWyIRSZxG0O3eWCzwHKn%2B%2BOE4oIjNxkvbN%2Fz341VQ6Z4LEnPpYdMnoS4phug3Ohl3WdgwvtAQqzmDkSuCJZ9oxiQwsFpKU9NWRQ7i1H2QkeKIcobEfR4R%2FNwNsM9N1YT862tKUqcB1H7R2eG9JtLnvr6nUNRQFq8OvrWhcE26YTt6mLOTXH3ntk%2B0rqiiy6qteh3VAPxiYDautBxZeJehruODxQJTA5dB%2FYo5SiXbHDNh2mz77YGSRNk3ufaEIfi1mnR8oIup1vmC46pT2hg3RsMvM7pUmIFWru6STNn8QVOAaQaLHp752NDfRRo4MlcpUYk33jQ5T%2F8ZfUuXAcDFbnKxCKEpW7%2BATR1TCdDgIAAA%3D%3D) to gather data about rollout times: * Pre-Approval run time: `` * Post-Approval run time: `` From 3e2e7edb6768b71e97d8ee60d103246b3afae2ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 5 Sep 2024 13:13:04 +0200 Subject: [PATCH 06/13] Fix repo merge policies E2E test (#3929) --- .../Darc/Operations/SetRepositoryMergePoliciesOperation.cs | 6 ------ .../ScenarioTests_RepoPolicies.cs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/SetRepositoryMergePoliciesOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/SetRepositoryMergePoliciesOperation.cs index c2c24c234e..618fc801a6 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/SetRepositoryMergePoliciesOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/SetRepositoryMergePoliciesOperation.cs @@ -140,12 +140,6 @@ public override async Task ExecuteAsync() IRemote verifyRemote = RemoteFactory.GetRemote(_options, repository, _logger); IEnumerable targetRepository = await _barClient.GetRepositoriesAsync(repository, branch: null); - if (targetRepository == null || !targetRepository.Any()) - { - Console.WriteLine($"The target repository '{repository}' doesn't have a Maestro installation. Aborting merge policy creation."); - return Constants.ErrorCode; - } - if (!await UxHelpers.VerifyAndConfirmBranchExistsAsync(verifyRemote, repository, branch, !_options.Quiet)) { Console.WriteLine("Aborting merge policy creation."); diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs index 8e19f05f13..b500120094 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs @@ -23,7 +23,7 @@ public Task DisposeAsync() } [Test] - public async Task ArcadeRepoPolicies_EndToEnd() + public async Task RepoPolicies_EndToEnd() { TestContext.WriteLine("Repository merge policy handling"); TestContext.WriteLine("Running tests..."); From ecae5f322cacd97f17f2534c9a97d654a97ba6a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 5 Sep 2024 13:13:17 +0200 Subject: [PATCH 07/13] Add default expiration to all cache entries (#3921) --- .../ProductConstructionService.Common/RedisCache.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs b/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs index 2eada09dbe..f2b5198b9d 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs @@ -18,6 +18,8 @@ public interface IRedisCache public class RedisCache : IRedisCache { + internal static readonly TimeSpan DefaultExpiration = TimeSpan.FromDays(180); + private readonly string _stateKey; private readonly IConnectionMultiplexer _connection; @@ -31,7 +33,7 @@ public RedisCache(IConnectionMultiplexer connection, string stateKey) public async Task SetAsync(string value, TimeSpan? expiration = null) { - await Cache.StringSetAsync(_stateKey, value, expiration); + await Cache.StringSetAsync(_stateKey, value, expiration ?? DefaultExpiration); } public async Task TryGetAsync() @@ -94,7 +96,7 @@ public async Task SetAsync(T value, TimeSpan? expiration = null) return; } - await _stateManager.SetAsync(json, expiration); + await _stateManager.SetAsync(json, expiration ?? RedisCache.DefaultExpiration); } private async Task TryGetStateAsync(bool delete) From 680ecbe91e7e2e1cceeb4d8a75e7287e2d9c324b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 5 Sep 2024 13:39:45 +0200 Subject: [PATCH 08/13] Fix building repo in PCS pipeline & fix PCS token in E2E tests (#3930) --- ...pipelines-product-construction-service.yml | 48 ++++++++++++------- .../TestParameters.cs | 7 ++- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index 48341021b8..58876e8ab4 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -146,32 +146,46 @@ stages: demands: ImageOverride -equals 1es-windows-2019 variables: - - name: _BuildConfig - value: Release - - name: _BuildArgs - value: > - /p:DotNetSignType=$(_SignType) - /p:TeamName=$(_TeamName) - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) - /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + - name: BuildConfig + value: Release steps: - checkout: self - - template: /eng/templates/steps/build.yml - parameters: - configuration: $(_BuildConfig) - buildArgs: $(_BuildArgs) - - - publish: $(Build.SourcesDirectory)\artifacts\bin\ProductConstructionService.ScenarioTests\$(_BuildConfig)\net8.0\publish + - powershell: | + . .\eng\common\tools.ps1 + InitializeDotNetCli -install:$true + .\.dotnet\dotnet workload update --source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json + .\.dotnet\dotnet workload install aspire --source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json + displayName: Install .NET and Aspire Workload + + - powershell: > + .\eng\common\build.ps1 + -restore + -pack + -configuration $(BuildConfig) + -projects .\src\Microsoft.DotNet.Darc\Darc\Microsoft.DotNet.Darc.csproj + displayName: Build Darc + + - powershell: > + .\eng\common\build.ps1 + -restore + -build + -configuration $(BuildConfig) + -projects .\test\ProductConstructionService.ScenarioTests\ProductConstructionService.ScenarioTests.csproj + displayName: Build ScenarioTests + + - publish: $(Build.SourcesDirectory)\artifacts\bin\ProductConstructionService.ScenarioTests\$(BuildConfig)\net8.0\publish artifact: ProductConstructionService.ScenarioTests + - publish: $(Build.SourcesDirectory)\artifacts\packages\$(BuildConfig)\NonShipping + artifact: PackageArtifacts + - stage: TestPCS displayName: Run E2E Product Construction Service Tests - dependsOn: + dependsOn: - DeployPCS - + jobs: - template: /eng/templates/jobs/e2e-pcs-tests.yml parameters: diff --git a/test/ProductConstructionService.ScenarioTests/TestParameters.cs b/test/ProductConstructionService.ScenarioTests/TestParameters.cs index 1464773b7e..f6e5ff0b36 100644 --- a/test/ProductConstructionService.ScenarioTests/TestParameters.cs +++ b/test/ProductConstructionService.ScenarioTests/TestParameters.cs @@ -19,6 +19,7 @@ public class TestParameters : IDisposable { internal readonly TemporaryDirectory _dir; private static readonly string pcsBaseUri; + private static readonly string? pcsToken; private static readonly string githubToken; private static readonly string darcPackageSource; private static readonly string? azdoToken; @@ -37,6 +38,8 @@ static TestParameters() pcsBaseUri = Environment.GetEnvironmentVariable("PCS_BASEURI") ?? userSecrets["PCS_BASEURI"] ?? "https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"; + pcsToken = Environment.GetEnvironmentVariable("PCS_TOKEN") + ?? userSecrets["PCS_TOKEN"]; isCI = Environment.GetEnvironmentVariable("DARC_IS_CI")?.ToLower() == "true"; githubToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN") ?? userSecrets["GITHUB_TOKEN"] ?? throw new Exception("Please configure the GitHub token"); @@ -54,9 +57,9 @@ public static async Task GetAsync(bool useNonPrimaryEndpoint = f var testDir = TemporaryDirectory.Get(); var testDirSharedWrapper = Shareable.Create(testDir); - IProductConstructionServiceApi pcsApi = pcsBaseUri.Contains("localhost") + IProductConstructionServiceApi pcsApi = pcsBaseUri.Contains("localhost") || pcsBaseUri.Contains("127.0.0.1") ? PcsApiFactory.GetAnonymous(pcsBaseUri) - : PcsApiFactory.GetAuthenticated(pcsBaseUri, accessToken: null, managedIdentityId: null, disableInteractiveAuth: isCI); + : PcsApiFactory.GetAuthenticated(pcsBaseUri, accessToken: pcsToken, managedIdentityId: null, disableInteractiveAuth: isCI); var darcRootDir = darcDir; if (string.IsNullOrEmpty(darcRootDir)) From 7d49d3d238153cff9014f8fcfd480306493ab4f4 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Thu, 5 Sep 2024 18:29:55 +0200 Subject: [PATCH 09/13] Fix how were calling container jobs (#3931) --- .../ProductConstructionService/provision.bicep | 16 ++++++---------- .../scheduledContainerJob.bicep | 11 ++++------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index c552213a2d..287684bee0 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -52,7 +52,7 @@ param pcsIdentityName string = 'ProductConstructionServiceInt' param deploymentIdentityName string = 'ProductConstructionServiceDeploymentInt' @description('Bicep requires an image when creating a containerapp. Using a dummy image for that.') -param containerImageName = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' +param containerImageName string = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' @description('Virtual network name') param virtualNetworkName string = 'product-construction-service-vnet-int' @@ -600,8 +600,7 @@ module subscriptionTriggererTwiceDaily 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.dll' - argument: 'twicedaily' + command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId } @@ -622,8 +621,7 @@ module subscriptionTriggererDaily 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.dll' - argument: 'daily' + command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId } @@ -644,8 +642,7 @@ module subscriptionTriggererWeekly 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.dll' - argument: 'weekly' + command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId } @@ -666,8 +663,7 @@ module longestBuildPathUpdater 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.dll' - argument: '' + command: 'cd /app/LongestBuildPathUpdater && dotnet ProductConstructionService.LongestBuildPathUpdater.dll' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId } @@ -689,7 +685,7 @@ module feedCleaner 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/FeedCleaner/ProductConstructionService.FeedCleaner.dll' + command: 'cd /app/FeedCleaner && dotnet ProductConstructionService.FeedCleaner.dll' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId } diff --git a/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep b/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep index 0244ed32ec..8c02b214ca 100644 --- a/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep +++ b/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep @@ -7,8 +7,7 @@ param cronSchedule string param containerRegistryName string param containerAppsEnvironmentId string param containerImageName string -param dllFullPath string -param argument string = '' +param command string param contributorRoleId string param deploymentIdentityPrincipalId string @@ -75,11 +74,9 @@ resource containerJob 'Microsoft.App/jobs@2024-03-01' = { name: 'job' env: env command: [ - 'dotnet' - dllFullPath - ] - args: [ - argument + '/bin/sh' + '-c' + command ] resources: { cpu: json('1.0') From 8f15c7d246b4b7aeaa909c65c88965f7383d58dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 5 Sep 2024 18:30:49 +0200 Subject: [PATCH 10/13] Do not use Get/Delete in Redis (#3932) --- .../RedisCache.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs b/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs index f2b5198b9d..ef29fd3589 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs @@ -12,7 +12,7 @@ public interface IRedisCache { Task GetAsync(); Task SetAsync(string value, TimeSpan? expiration = null); - Task TryDeleteAsync(); + Task TryDeleteAsync(); Task TryGetAsync(); } @@ -42,9 +42,9 @@ public async Task SetAsync(string value, TimeSpan? expiration = null) return value.HasValue ? value.ToString() : null; } - public async Task TryDeleteAsync() + public async Task TryDeleteAsync() { - return await Cache.StringGetDeleteAsync(_stateKey); + await Cache.KeyDeleteAsync(_stateKey); } public async Task GetAsync() @@ -101,14 +101,17 @@ public async Task SetAsync(T value, TimeSpan? expiration = null) private async Task TryGetStateAsync(bool delete) { - var state = delete - ? await _stateManager.TryDeleteAsync() - : await _stateManager.TryGetAsync(); + var state = await _stateManager.TryGetAsync(); if (state == null) { return null; } + if (delete) + { + await _stateManager.TryDeleteAsync(); + } + try { var result = JsonSerializer.Deserialize(state, JsonSerializerOptions); From 3d25e8cbd3ce101db691fb048fdf3a3371a4592f Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:04:12 +0200 Subject: [PATCH 11/13] Refactor bicep, change redis name (#3935) --- .../container-app.bicep | 145 ++++ .../container-environment.bicep | 61 ++ .../container-registry.bicep | 89 ++ .../key-vaults.bicep | 98 +++ .../managed-identities.bicep | 42 + .../ProductConstructionService/nsg.bicep | 173 ++++ .../provision.bicep | 793 +++--------------- .../ProductConstructionService/redis.bicep | 32 + .../scheduledContainerJob.bicep | 2 +- .../storage-account.bicep | 77 ++ .../virtual-network.bicep | 41 + .../appsettings.Staging.json | 2 +- 12 files changed, 874 insertions(+), 681 deletions(-) create mode 100644 eng/service-templates/ProductConstructionService/container-app.bicep create mode 100644 eng/service-templates/ProductConstructionService/container-environment.bicep create mode 100644 eng/service-templates/ProductConstructionService/container-registry.bicep create mode 100644 eng/service-templates/ProductConstructionService/key-vaults.bicep create mode 100644 eng/service-templates/ProductConstructionService/managed-identities.bicep create mode 100644 eng/service-templates/ProductConstructionService/nsg.bicep create mode 100644 eng/service-templates/ProductConstructionService/redis.bicep create mode 100644 eng/service-templates/ProductConstructionService/storage-account.bicep create mode 100644 eng/service-templates/ProductConstructionService/virtual-network.bicep diff --git a/eng/service-templates/ProductConstructionService/container-app.bicep b/eng/service-templates/ProductConstructionService/container-app.bicep new file mode 100644 index 0000000000..e67047f2b2 --- /dev/null +++ b/eng/service-templates/ProductConstructionService/container-app.bicep @@ -0,0 +1,145 @@ +param location string +param containerRegistryName string +param containerImageName string +param containerCpuCoreCount string +param containerMemory string +param aspnetcoreEnvironment string +param productConstructionServiceName string +param applicationInsightsConnectionString string +param pcsIdentityId string +param containerEnvironmentId string +param contributorRoleId string +param deploymentIdentityPrincipalId string + +// common environment variables used by the app +var containerAppEnv = [ + { + name: 'ASPNETCORE_ENVIRONMENT' + value: aspnetcoreEnvironment + } + { + name: 'Logging__Console__FormatterName' + value: 'simple' + } + { + name: 'Logging__Console__FormatterOptions__SingleLine' + value: 'true' + } + { + name: 'Logging__Console__FormatterOptions__IncludeScopes' + value: 'true' + } + { + name: 'ASPNETCORE_LOGGING__CONSOLE__DISABLECOLORS' + value: 'true' + } + { + name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' + value: applicationInsightsConnectionString + } + { + name: 'VmrPath' + value: '/mnt/datadir/vmr' + } + { + name: 'TmpPath' + value: '/mnt/datadir/tmp' + } +] + +// container app hosting the Product Construction Service +resource containerApp 'Microsoft.App/containerApps@2023-04-01-preview' = { + name: productConstructionServiceName + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { '${pcsIdentityId}' : {}} + } + properties: { + managedEnvironmentId: containerEnvironmentId + configuration: { + activeRevisionsMode: 'Multiple' + maxInactiveRevisions: 5 + ingress: { + external: true + targetPort: 8080 + transport: 'http' + } + dapr: { enabled: false } + registries: [ + { + server: '${containerRegistryName}.azurecr.io' + identity: pcsIdentityId + } + ] + } + template: { + scale: { + minReplicas: 1 + maxReplicas: 1 + } + serviceBinds: [] + containers: [ + { + image: containerImageName + name: 'api' + env: containerAppEnv + resources: { + cpu: json(containerCpuCoreCount) + memory: containerMemory + ephemeralStorage: '50Gi' + } + volumeMounts: [ + { + volumeName: 'data' + mountPath: '/mnt/datadir' + } + ] + probes: [ + { + httpGet: { + path: '/alive' + port: 8080 + scheme: 'HTTP' + } + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + type: 'Startup' + } + { + httpGet: { + path: '/health' + port: 8080 + scheme: 'HTTP' + } + initialDelaySeconds: 60 + failureThreshold: 10 + successThreshold: 1 + periodSeconds: 30 + type: 'Readiness' + } + ] + } + ] + volumes: [ + { + name: 'data' + storageType: 'EmptyDir' + } + ] + } + } +} + +// Give the PCS Deployment MI the Contributor role in the containerapp to allow it to deploy +resource deploymentSubscriptionTriggererContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerApp + name: guid(subscription().id, resourceGroup().id, '${productConstructionServiceName}-contributor') + properties: { + roleDefinitionId: contributorRoleId + principalType: 'ServicePrincipal' + principalId: deploymentIdentityPrincipalId + } +} diff --git a/eng/service-templates/ProductConstructionService/container-environment.bicep b/eng/service-templates/ProductConstructionService/container-environment.bicep new file mode 100644 index 0000000000..9bd3fd48f7 --- /dev/null +++ b/eng/service-templates/ProductConstructionService/container-environment.bicep @@ -0,0 +1,61 @@ +param location string +param logAnalyticsName string +param containerEnvironmentName string +param productConstructionServiceSubnetId string +param infrastructureResourceGroupName string +param applicationInsightsName string + +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { + name: logAnalyticsName + location: location + properties: any({ + retentionInDays: 30 + features: { + searchVersion: 1 + } + sku: { + name: 'PerGB2018' + } + }) +} + +resource containerEnvironment 'Microsoft.App/managedEnvironments@2023-04-01-preview' = { + name: containerEnvironmentName + location: location + properties: { + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logAnalytics.properties.customerId + sharedKey: logAnalytics.listKeys().primarySharedKey + } + } + workloadProfiles: [ + { + name: 'Consumption' + workloadProfileType: 'Consumption' + } + ] + vnetConfiguration: { + infrastructureSubnetId: productConstructionServiceSubnetId + } + infrastructureResourceGroup: infrastructureResourceGroupName + } +} + +// application insights for service logging +resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { + name: applicationInsightsName + location: location + kind: 'web' + properties: { + Application_Type: 'web' + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + RetentionInDays: 120 + WorkspaceResourceId: logAnalytics.id + } +} + +output applicationInsightsConnectionString string = applicationInsights.properties.ConnectionString +output containerEnvironmentId string = containerEnvironment.id diff --git a/eng/service-templates/ProductConstructionService/container-registry.bicep b/eng/service-templates/ProductConstructionService/container-registry.bicep new file mode 100644 index 0000000000..69a6a6c74e --- /dev/null +++ b/eng/service-templates/ProductConstructionService/container-registry.bicep @@ -0,0 +1,89 @@ +param location string +param containerRegistryName string +param acrPullRole string +param pcsIdentityPrincipalId string +param subscriptionTriggererPricnipalId string +param longestBuildPathUpdaterIdentityPrincipalId string +param feedCleanerIdentityPrincipalId string +param acrPushRole string +param deploymentIdentityPrincipalId string + +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { + name: containerRegistryName + location: location + sku: { + name: 'Premium' + } + properties: { + adminUserEnabled: false + anonymousPullEnabled: false + dataEndpointEnabled: false + encryption: { + status: 'disabled' + } + networkRuleBypassOptions: 'AzureServices' + publicNetworkAccess: 'Enabled' + zoneRedundancy: 'Disabled' + policies: { + retentionPolicy: { + days: 60 + status: 'enabled' + } + } + } +} + +// allow acr pulls to the identity used for the pcs +resource aksAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerRegistry + name: guid(subscription().id, resourceGroup().id, 'pcsAcrPull') + properties: { + roleDefinitionId: acrPullRole + principalType: 'ServicePrincipal' + principalId: pcsIdentityPrincipalId + } +} + +// allow acr pulls to the identity used for the subscription triggerer +resource subscriptionTriggererIdentityAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerRegistry + name: guid(subscription().id, resourceGroup().id, 'subscriptionTriggererAcrPull') + properties: { + roleDefinitionId: acrPullRole + principalType: 'ServicePrincipal' + principalId: subscriptionTriggererPricnipalId + } +} + +// allow acr pulls to the identity used for the longest build path updater +resource longestBuildPathUpdaterIdentityAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerRegistry + name: guid(subscription().id, resourceGroup().id, 'longestBuildPathUpdaterAcrPull') + properties: { + roleDefinitionId: acrPullRole + principalType: 'ServicePrincipal' + principalId: longestBuildPathUpdaterIdentityPrincipalId + } +} + +// allow acr pulls to the identity used for the feed cleaner +resource feedCleanerIdentityAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerRegistry + name: guid(subscription().id, resourceGroup().id, 'feedCleanerAcrPull') + properties: { + roleDefinitionId: acrPullRole + principalType: 'ServicePrincipal' + principalId: feedCleanerIdentityPrincipalId + } +} + +// Give the PCS Deployment MI the ACR Push role to be able to push docker images +resource deploymentAcrPush 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerRegistry + name: guid(subscription().id, resourceGroup().id, 'deploymentAcrPush') + properties: { + roleDefinitionId: acrPushRole + principalType: 'ServicePrincipal' + principalId: deploymentIdentityPrincipalId + } +} diff --git a/eng/service-templates/ProductConstructionService/key-vaults.bicep b/eng/service-templates/ProductConstructionService/key-vaults.bicep new file mode 100644 index 0000000000..605244f42e --- /dev/null +++ b/eng/service-templates/ProductConstructionService/key-vaults.bicep @@ -0,0 +1,98 @@ +param aspnetcoreEnvironment string +param location string +param keyVaultName string +param devKeyVaultName string +param kvSecretUserRole string +param kvCryptoUserRole string +param pcsIdentityPrincipalId string + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { + name: keyVaultName + location: location + properties: { + sku: { + name: 'standard' + family: 'A' + } + tenantId: subscription().tenantId + enableSoftDelete: true + softDeleteRetentionInDays: 90 + accessPolicies: [] + enableRbacAuthorization: true + } +} + +// If we're creating the staging environment, also create a dev key vault +resource devKeyVault 'Microsoft.KeyVault/vaults@2022-07-01' = if (aspnetcoreEnvironment == 'Staging') { + name: devKeyVaultName + location: location + properties: { + sku: { + name: 'standard' + family: 'A' + } + tenantId: subscription().tenantId + enableSoftDelete: true + softDeleteRetentionInDays: 90 + accessPolicies: [] + enableRbacAuthorization: true + } +} + +// allow secret access to the identity used for the aca's +resource secretAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: keyVault + name: guid(subscription().id, resourceGroup().id, kvSecretUserRole) + properties: { + roleDefinitionId: kvSecretUserRole + principalType: 'ServicePrincipal' + principalId: pcsIdentityPrincipalId + } +} + +// allow crypto access to the identity used for the aca's +resource cryptoAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: keyVault + name: guid(subscription().id, resourceGroup().id, kvCryptoUserRole) + properties: { + roleDefinitionId: kvCryptoUserRole + principalType: 'ServicePrincipal' + principalId: pcsIdentityPrincipalId + } +} + +resource dataProtectionKey 'Microsoft.KeyVault/vaults/keys@2023-07-01' = { + name: 'data-protection-encryption-key' + parent: keyVault + properties: { + attributes: { + enabled: true + exportable: false + } + keyOps: [ + 'sign' + 'verify' + 'wrapKey' + 'unwrapKey' + 'encrypt' + 'decrypt' + ] + keySize: 2048 + kty: 'RSA' + rotationPolicy: { + attributes: { + expiryTime: 'P540D' + } + lifetimeActions: [ + { + action: { + type: 'rotate' + } + trigger: { + timeBeforeExpiry: 'P30D' + } + } + ] + } + } +} diff --git a/eng/service-templates/ProductConstructionService/managed-identities.bicep b/eng/service-templates/ProductConstructionService/managed-identities.bicep new file mode 100644 index 0000000000..e7fca31dca --- /dev/null +++ b/eng/service-templates/ProductConstructionService/managed-identities.bicep @@ -0,0 +1,42 @@ +param location string +param deploymentIdentityName string +param pcsIdentityName string +param subscriptionTriggererIdentityName string +param longestBuildPathUpdaterIdentityName string +param feedCleanerIdentityName string + +resource deploymentIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: deploymentIdentityName + location: location +} + +resource pcsIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: pcsIdentityName + location: location +} + +resource subscriptionTriggererIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: subscriptionTriggererIdentityName + location: location +} + +resource longestBuildPathUpdaterIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: longestBuildPathUpdaterIdentityName + location: location +} + +resource feedCleanerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: feedCleanerIdentityName + location: location +} + +output pcsIdentityPrincipalId string = pcsIdentity.properties.principalId +output pcsIdentityId string = pcsIdentity.id +output deploymentIdentityPrincipalId string = deploymentIdentity.properties.principalId +output deploymentIdentityId string = deploymentIdentity.id +output subscriptionTriggererIdentityPrincipalId string = subscriptionTriggererIdentity.properties.principalId +output subscriptionTriggererIdentityId string = subscriptionTriggererIdentity.id +output longestBuildPathUpdaterIdentityPrincipalId string = longestBuildPathUpdaterIdentity.properties.principalId +output longestBuildPathUpdaterIdentityId string = longestBuildPathUpdaterIdentity.id +output feedCleanerIdentityPrincipalId string = feedCleanerIdentity.properties.principalId +output feedCleanerIdentityId string = feedCleanerIdentity.id diff --git a/eng/service-templates/ProductConstructionService/nsg.bicep b/eng/service-templates/ProductConstructionService/nsg.bicep new file mode 100644 index 0000000000..68b458973e --- /dev/null +++ b/eng/service-templates/ProductConstructionService/nsg.bicep @@ -0,0 +1,173 @@ +param networkSecurityGroupName string +param location string + +resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-11-01' = { + name: networkSecurityGroupName + location: location + properties: { + securityRules: [ + // These are required by a corp policy + { + name: 'NRMS-Rule-101' + properties: { + priority: 101 + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: '*' + access: 'Allow' + direction: 'Inbound' + } + } + { + name: 'NRMS-Rule-103' + properties: { + priority: 103 + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'CorpNetPublic' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Allow' + direction: 'Inbound' + } + } + { + name: 'NRMS-Rule-104' + properties: { + priority: 104 + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'CorpNetSaw' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Allow' + direction: 'Inbound' + } + } + { + name: 'NRMS-Rule-105' + properties: { + priority: 105 + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRanges: [ + '1434' + '1433' + '3306' + '4333' + '5432' + '6379' + '7000' + '7001' + '7199' + '9042' + '9160' + '9300' + '16379' + '26379' + '27017' + ] + destinationAddressPrefix: '*' + access: 'Deny' + direction: 'Inbound' + } + } + { + name: 'NRMS-Rule-106' + properties: { + priority: 106 + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRanges: [ + '22' + '3389' + ] + destinationAddressPrefix: '*' + access: 'Deny' + direction: 'Inbound' + } + } + { + name: 'NRMS-Rule-107' + properties: { + priority: 107 + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRanges: [ + '23' + '135' + '445' + '5985' + '5986' + ] + destinationAddressPrefix: '*' + access: 'Deny' + direction: 'Inbound' + } + } + { + name: 'NRMS-Rule-108' + properties: { + priority: 108 + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRanges: [ + '13' + '17' + '19' + '53' + '69' + '111' + '123' + '512' + '514' + '593' + '873' + '1900' + '5353' + '11211' + ] + destinationAddressPrefix: '*' + access: 'Deny' + direction: 'Inbound' + } + } + { + name: 'NRMS-Rule-109' + properties: { + priority: 109 + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRanges: [ + '119' + '137' + '138' + '139' + '161' + '162' + '389' + '636' + '2049' + '2301' + '2381' + '3268' + '5800' + '5900' + ] + destinationAddressPrefix: '*' + access: 'Deny' + direction: 'Inbound' + } + } + ] + } +} + +output networkSecurityGroupId string = networkSecurityGroup.id diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 287684bee0..2a6f666fa4 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -31,13 +31,13 @@ param keyVaultName string = 'ProductConstructionInt' param devKeyVaultName string = 'ProductConstructionDev' @description('Azure Cache for Redis name') -param azureCacheRedisName string = 'prodconstaging' +param azureCacheRedisName string = 'product-construction-service-redis-int' @description('Log analytics workspace name') param logAnalyticsName string = 'product-construction-service-workspace-int' @description('Name of the container apps environment') -param containerAppsEnvironmentName string = 'product-construction-service-env-int' +param containerEnvironmentName string = 'product-construction-service-env-int' @description('Product construction service API name') param productConstructionServiceName string = 'product-construction-int' @@ -90,348 +90,6 @@ param networkSecurityGroupName string = 'product-construction-service-nsg-int' @description('Resource group where PCS IP resources will be created') param infrastructureResourceGroupName string = 'product-construction-service-ip-int' -resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-11-01' = { - name: networkSecurityGroupName - location: location - properties: { - securityRules: [ - // These are required by a corp policy - { - name: 'NRMS-Rule-101' - properties: { - priority: 101 - protocol: 'Tcp' - sourcePortRange: '*' - sourceAddressPrefix: 'VirtualNetwork' - destinationPortRange: '443' - destinationAddressPrefix: '*' - access: 'Allow' - direction: 'Inbound' - } - } - { - name: 'NRMS-Rule-103' - properties: { - priority: 103 - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: 'CorpNetPublic' - destinationPortRange: '*' - destinationAddressPrefix: '*' - access: 'Allow' - direction: 'Inbound' - } - } - { - name: 'NRMS-Rule-104' - properties: { - priority: 104 - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: 'CorpNetSaw' - destinationPortRange: '*' - destinationAddressPrefix: '*' - access: 'Allow' - direction: 'Inbound' - } - } - { - name: 'NRMS-Rule-105' - properties: { - priority: 105 - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: 'Internet' - destinationPortRanges: [ - '1434' - '1433' - '3306' - '4333' - '5432' - '6379' - '7000' - '7001' - '7199' - '9042' - '9160' - '9300' - '16379' - '26379' - '27017' - ] - destinationAddressPrefix: '*' - access: 'Deny' - direction: 'Inbound' - } - } - { - name: 'NRMS-Rule-106' - properties: { - priority: 106 - protocol: 'Tcp' - sourcePortRange: '*' - sourceAddressPrefix: 'Internet' - destinationPortRanges: [ - '22' - '3389' - ] - destinationAddressPrefix: '*' - access: 'Deny' - direction: 'Inbound' - } - } - { - name: 'NRMS-Rule-107' - properties: { - priority: 107 - protocol: 'Tcp' - sourcePortRange: '*' - sourceAddressPrefix: 'Internet' - destinationPortRanges: [ - '23' - '135' - '445' - '5985' - '5986' - ] - destinationAddressPrefix: '*' - access: 'Deny' - direction: 'Inbound' - } - } - { - name: 'NRMS-Rule-108' - properties: { - priority: 108 - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: 'Internet' - destinationPortRanges: [ - '13' - '17' - '19' - '53' - '69' - '111' - '123' - '512' - '514' - '593' - '873' - '1900' - '5353' - '11211' - ] - destinationAddressPrefix: '*' - access: 'Deny' - direction: 'Inbound' - } - } - { - name: 'NRMS-Rule-109' - properties: { - priority: 109 - protocol: '*' - sourcePortRange: '*' - sourceAddressPrefix: 'Internet' - destinationPortRanges: [ - '119' - '137' - '138' - '139' - '161' - '162' - '389' - '636' - '2049' - '2301' - '2381' - '3268' - '5800' - '5900' - ] - destinationAddressPrefix: '*' - access: 'Deny' - direction: 'Inbound' - } - } - ] - } -} - -// log analytics -resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { - name: logAnalyticsName - location: location - properties: any({ - retentionInDays: 30 - features: { - searchVersion: 1 - } - sku: { - name: 'PerGB2018' - } - }) -} - -// virtual network -resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { - name: virtualNetworkName - location: location - properties: { - addressSpace: { - addressPrefixes: [ - '10.0.0.0/16' - ] - } - } - tags: { - 'ms.inv.v0.networkUsage': 'mixedTraffic' - } -} - -// subnet for the product construction service -resource productConstructionServiceSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-04-01' = { - name: productConstructionServiceSubnetName - parent: virtualNetwork - properties: { - addressPrefix: '10.0.0.0/24' - delegations: [ - { - name: 'Microsoft.App/environments' - properties: { - serviceName: 'Microsoft.App/environments' - } - } - ] - networkSecurityGroup: { - id: networkSecurityGroup.id - } - } -} - -// the container apps environment -resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-04-01-preview' = { - name: containerAppsEnvironmentName - location: location - properties: { - appLogsConfiguration: { - destination: 'log-analytics' - logAnalyticsConfiguration: { - customerId: logAnalytics.properties.customerId - sharedKey: logAnalytics.listKeys().primarySharedKey - } - } - workloadProfiles: [ - { - name: 'Consumption' - workloadProfileType: 'Consumption' - } - ] - vnetConfiguration: { - infrastructureSubnetId: productConstructionServiceSubnet.id - } - infrastructureResourceGroup: infrastructureResourceGroupName - } -} - -// the container registry -resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { - name: containerRegistryName - location: location - sku: { - name: 'Premium' - } - properties: { - adminUserEnabled: false - anonymousPullEnabled: false - dataEndpointEnabled: false - encryption: { - status: 'disabled' - } - networkRuleBypassOptions: 'AzureServices' - publicNetworkAccess: 'Enabled' - zoneRedundancy: 'Disabled' - policies: { - retentionPolicy: { - days: 60 - status: 'enabled' - } - } - } -} - - -resource deploymentIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: deploymentIdentityName - location: location -} - -resource pcsIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: pcsIdentityName - location: location -} - -resource subscriptionTriggererIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: subscriptionTriggererIdentityName - location: location -} - -resource longestBuildPathUpdaterIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: longestBuildPathUpdaterIdentityName - location: location -} - -resource feedCleanerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: feedCleanerIdentityName - location: location -} - -// allow acr pulls to the identity used for the aca's -resource aksAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: containerRegistry // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, acrPullRole) - properties: { - roleDefinitionId: acrPullRole - principalType: 'ServicePrincipal' - principalId: pcsIdentity.properties.principalId - } -} - -// allow acr pulls to the identity used for the subscription triggerer -resource subscriptionTriggererIdentityAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: containerRegistry // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, acrPullRole) - properties: { - roleDefinitionId: acrPullRole - principalType: 'ServicePrincipal' - principalId: subscriptionTriggererIdentity.properties.principalId - } -} - -// allow acr pulls to the identity used for the longest build path updater -resource longestBuildPathUpdaterIdentityAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: containerRegistry // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, acrPullRole) - properties: { - roleDefinitionId: acrPullRole - principalType: 'ServicePrincipal' - principalId: longestBuildPathUpdaterIdentity.properties.principalId - } -} - -// allow acr pulls to the identity used for the feed cleaner -resource feedCleanerIdentityAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: containerRegistry // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, acrPullRole) - properties: { - roleDefinitionId: acrPullRole - principalType: 'ServicePrincipal' - principalId: feedCleanerIdentity.properties.principalId - } -} - - // azure system role for setting up acr pull access var acrPullRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') // azure system role for granting push access @@ -442,149 +100,86 @@ var kvSecretUserRole = subscriptionResourceId('Microsoft.Authorization/roleDefin var storageQueueContrubutorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '974c5e8b-45b9-4653-ba55-5f855dd0fb88') // azure system role for setting contributor access var contributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') -// azure system role Key Vault Reader -var keyVaultReaderRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '21090545-7ca7-4776-b22c-e363652d74d2') // storage account blob contributor var blobContributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') // Key Vault Crypto User role var kvCryptoUserRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') -// application insights for service logging -resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { - name: applicationInsightsName - location: location - kind: 'web' - properties: { - Application_Type: 'web' - publicNetworkAccessForIngestion: 'Enabled' - publicNetworkAccessForQuery: 'Enabled' - RetentionInDays: 120 - WorkspaceResourceId: logAnalytics.id +module networkSecurityGroupModule 'nsg.bicep' = { + name: 'networkSecurityGroupModule' + params: { + networkSecurityGroupName: networkSecurityGroupName + location: location } } -// common environment variables used by each of the apps -var containerAppEnv = [ - { - name: 'ASPNETCORE_ENVIRONMENT' - value: aspnetcoreEnvironment - } - { - name: 'Logging__Console__FormatterName' - value: 'simple' - } - { - name: 'Logging__Console__FormatterOptions__SingleLine' - value: 'true' - } - { - name: 'Logging__Console__FormatterOptions__IncludeScopes' - value: 'true' - } - { - name: 'ASPNETCORE_LOGGING__CONSOLE__DISABLECOLORS' - value: 'true' - } - { - name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' - value: applicationInsights.properties.ConnectionString +module virtualNetworkModule 'virtual-network.bicep' = { + name: 'virtualNetworkModule' + params: { + location: location + virtualNetworkName: virtualNetworkName + networkSecurityGroupId: networkSecurityGroupModule.outputs.networkSecurityGroupId + productConstructionServiceSubnetName: productConstructionServiceSubnetName } - { - name: 'VmrPath' - value: '/mnt/datadir/vmr' +} + +module containerEnvironmentModule 'container-environment.bicep' = { + name: 'containerEnvironmentModule' + params: { + location: location + logAnalyticsName: logAnalyticsName + containerEnvironmentName: containerEnvironmentName + productConstructionServiceSubnetId: virtualNetworkModule.outputs.productConstructionServiceSubnetId + infrastructureResourceGroupName: infrastructureResourceGroupName + applicationInsightsName: applicationInsightsName } - { - name: 'TmpPath' - value: '/mnt/datadir/tmp' +} + +module managedIdentitiesModule 'managed-identities.bicep' = { + name: 'managedIdentitiesModule' + params: { + location: location + deploymentIdentityName: deploymentIdentityName + pcsIdentityName: pcsIdentityName + subscriptionTriggererIdentityName: subscriptionTriggererIdentityName + longestBuildPathUpdaterIdentityName: longestBuildPathUpdaterIdentityName + feedCleanerIdentityName: feedCleanerIdentityName } -] - -// container app hosting the Product Construction Service -resource containerapp 'Microsoft.App/containerApps@2023-04-01-preview' = { - name: productConstructionServiceName - location: location - identity: { - type: 'UserAssigned' - userAssignedIdentities: { '${pcsIdentity.id}' : {}} +} + +module containerRegistryModule 'container-registry.bicep' = { + name: 'containerRegistryModule' + params: { + location: location + containerRegistryName: containerRegistryName + acrPullRole: acrPullRole + pcsIdentityPrincipalId: managedIdentitiesModule.outputs.pcsIdentityPrincipalId + subscriptionTriggererPricnipalId: managedIdentitiesModule.outputs.subscriptionTriggererIdentityPrincipalId + longestBuildPathUpdaterIdentityPrincipalId: managedIdentitiesModule.outputs.longestBuildPathUpdaterIdentityPrincipalId + feedCleanerIdentityPrincipalId: managedIdentitiesModule.outputs.feedCleanerIdentityPrincipalId + acrPushRole: acrPushRole + deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId } - properties: { - managedEnvironmentId: containerAppsEnvironment.id - configuration: { - activeRevisionsMode: 'Multiple' - maxInactiveRevisions: 5 - ingress: { - external: true - targetPort: 8080 - transport: 'http' - } - dapr: { enabled: false } - registries: [ - { - server: '${containerRegistryName}.azurecr.io' - identity: pcsIdentity.id - } - ] - } - template: { - scale: { - minReplicas: 1 - maxReplicas: 1 - } - serviceBinds: [] - containers: [ - { - image: containerImageName - name: 'api' - env: containerAppEnv - resources: { - cpu: json(containerCpuCoreCount) - memory: containerMemory - ephemeralStorage: '50Gi' - } - volumeMounts: [ - { - volumeName: 'data' - mountPath: '/mnt/datadir' - } - ] - probes: [ - { - httpGet: { - path: '/alive' - port: 8080 - scheme: 'HTTP' - } - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 3 - type: 'Startup' - } - { - httpGet: { - path: '/health' - port: 8080 - scheme: 'HTTP' - } - initialDelaySeconds: 60 - failureThreshold: 10 - successThreshold: 1 - periodSeconds: 30 - type: 'Readiness' - } - ] - } - ] - volumes: [ - { - name: 'data' - storageType: 'EmptyDir' - } - ] - } +} + +module containerAppModule 'container-app.bicep' = { + name: 'containerAppModule' + params: { + location: location + containerEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId + containerImageName: containerImageName + containerRegistryName: containerRegistryName + containerCpuCoreCount: containerCpuCoreCount + containerMemory: containerMemory + productConstructionServiceName: productConstructionServiceName + applicationInsightsConnectionString: containerEnvironmentModule.outputs.applicationInsightsConnectionString + aspnetcoreEnvironment: aspnetcoreEnvironment + contributorRoleId: contributorRole + deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId + pcsIdentityId: managedIdentitiesModule.outputs.pcsIdentityId } dependsOn: [ - aksAcrPull + containerRegistryModule ] } @@ -594,18 +189,18 @@ module subscriptionTriggererTwiceDaily 'scheduledContainerJob.bicep' = { jobName: subscriptionTriggererTwiceDailyJobName location: location aspnetcoreEnvironment: aspnetcoreEnvironment - applicationInsightsConnectionString: applicationInsights.properties.ConnectionString - userAssignedIdentityId: subscriptionTriggererIdentity.id + applicationInsightsConnectionString: containerEnvironmentModule.outputs.applicationInsightsConnectionString + userAssignedIdentityId: managedIdentitiesModule.outputs.subscriptionTriggererIdentityId cronSchedule: '0 5,19 * * *' containerRegistryName: containerRegistryName - containerAppsEnvironmentId: containerAppsEnvironment.id + containerAppsEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId containerImageName: containerImageName command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll' contributorRoleId: contributorRole - deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId + deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId } dependsOn: [ - subscriptionTriggererIdentityAcrPull + containerRegistryModule ] } @@ -615,15 +210,15 @@ module subscriptionTriggererDaily 'scheduledContainerJob.bicep' = { jobName: subscriptionTriggererDailyJobName location: location aspnetcoreEnvironment: aspnetcoreEnvironment - applicationInsightsConnectionString: applicationInsights.properties.ConnectionString - userAssignedIdentityId: subscriptionTriggererIdentity.id + applicationInsightsConnectionString: containerEnvironmentModule.outputs.applicationInsightsConnectionString + userAssignedIdentityId: managedIdentitiesModule.outputs.subscriptionTriggererIdentityId cronSchedule: '0 5 * * *' containerRegistryName: containerRegistryName - containerAppsEnvironmentId: containerAppsEnvironment.id + containerAppsEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId containerImageName: containerImageName command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll' contributorRoleId: contributorRole - deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId + deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId } dependsOn: [ subscriptionTriggererTwiceDaily @@ -636,15 +231,15 @@ module subscriptionTriggererWeekly 'scheduledContainerJob.bicep' = { jobName: subscriptionTriggererWeeklyJobName location: location aspnetcoreEnvironment: aspnetcoreEnvironment - applicationInsightsConnectionString: applicationInsights.properties.ConnectionString - userAssignedIdentityId: subscriptionTriggererIdentity.id + applicationInsightsConnectionString: containerEnvironmentModule.outputs.applicationInsightsConnectionString + userAssignedIdentityId: managedIdentitiesModule.outputs.subscriptionTriggererIdentityId cronSchedule: '0 5 * * MON' containerRegistryName: containerRegistryName - containerAppsEnvironmentId: containerAppsEnvironment.id + containerAppsEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId containerImageName: containerImageName command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll' contributorRoleId: contributorRole - deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId + deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId } dependsOn: [ subscriptionTriggererDaily @@ -657,19 +252,18 @@ module longestBuildPathUpdater 'scheduledContainerJob.bicep' = { jobName: longestBuildPathUpdaterJobName location: location aspnetcoreEnvironment: aspnetcoreEnvironment - applicationInsightsConnectionString: applicationInsights.properties.ConnectionString - userAssignedIdentityId: longestBuildPathUpdaterIdentity.id + applicationInsightsConnectionString: containerEnvironmentModule.outputs.applicationInsightsConnectionString + userAssignedIdentityId: managedIdentitiesModule.outputs.longestBuildPathUpdaterIdentityId cronSchedule: '0 5 * * MON' containerRegistryName: containerRegistryName - containerAppsEnvironmentId: containerAppsEnvironment.id + containerAppsEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId containerImageName: containerImageName command: 'cd /app/LongestBuildPathUpdater && dotnet ProductConstructionService.LongestBuildPathUpdater.dll' contributorRoleId: contributorRole - deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId + deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId } dependsOn: [ subscriptionTriggererWeekly - longestBuildPathUpdaterIdentityAcrPull ] } @@ -679,210 +273,51 @@ module feedCleaner 'scheduledContainerJob.bicep' = { jobName: feedCleanerJobName location: location aspnetcoreEnvironment: aspnetcoreEnvironment - applicationInsightsConnectionString: applicationInsights.properties.ConnectionString - userAssignedIdentityId: feedCleanerIdentity.id + applicationInsightsConnectionString: containerEnvironmentModule.outputs.applicationInsightsConnectionString + userAssignedIdentityId: managedIdentitiesModule.outputs.feedCleanerIdentityId cronSchedule: '0 2 * * *' containerRegistryName: containerRegistryName - containerAppsEnvironmentId: containerAppsEnvironment.id + containerAppsEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId containerImageName: containerImageName command: 'cd /app/FeedCleaner && dotnet ProductConstructionService.FeedCleaner.dll' contributorRoleId: contributorRole - deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId + deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId } dependsOn: [ - feedCleanerIdentityAcrPull longestBuildPathUpdater ] } -resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { - name: keyVaultName - location: location - properties: { - sku: { - name: 'standard' - family: 'A' - } - tenantId: subscription().tenantId - enableSoftDelete: true - softDeleteRetentionInDays: 90 - accessPolicies: [] - enableRbacAuthorization: true - } -} - -// If we're creating the staging environment, also create a dev key vault -resource devKeyVault 'Microsoft.KeyVault/vaults@2022-07-01' = if (aspnetcoreEnvironment == 'Staging') { - name: devKeyVaultName - location: location - properties: { - sku: { - name: 'standard' - family: 'A' - } - tenantId: subscription().tenantId - enableSoftDelete: true - softDeleteRetentionInDays: 90 - accessPolicies: [] - enableRbacAuthorization: true - } -} - -resource redisCache 'Microsoft.Cache/redis@2024-03-01' = { - name: azureCacheRedisName - location: location - properties: { - enableNonSslPort: false - minimumTlsVersion: '1.2' - sku: { - capacity: 0 - family: 'C' - name: 'Basic' - } - redisConfiguration: { - 'aad-enabled': 'true' - } - disableAccessKeyAuthentication: true - } -} - -// allow secret access to the identity used for the aca's -resource secretAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: keyVault // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, kvSecretUserRole) - properties: { - roleDefinitionId: kvSecretUserRole - principalType: 'ServicePrincipal' - principalId: pcsIdentity.properties.principalId - } -} - -// allow crypto access to the identity used for the aca's -resource cryptoAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: keyVault // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, kvCryptoUserRole) - properties: { - roleDefinitionId: kvCryptoUserRole - principalType: 'ServicePrincipal' - principalId: pcsIdentity.properties.principalId - } -} - - -resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { - name: storageAccountName - location: location - kind: 'StorageV2' - sku: { - name: 'Standard_LRS' - } - properties: { - allowBlobPublicAccess: false - publicNetworkAccess: 'Enabled' - allowSharedKeyAccess: false - networkAcls: { - defaultAction: 'Deny' - } - } -} - -// Create the dataprotection container in the storage account -resource storageAccountBlobService 'Microsoft.Storage/storageAccounts/blobServices@2022-09-01' = { - name: 'default' - parent: storageAccount -} - -resource dataProtectionContainer 'Microsoft.Storage/storageAccounts/blobServices/containers@2022-09-01' = { - name: 'dataprotection' - parent: storageAccountBlobService -} - -// allow identity access to the storage account -resource storageAccountContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: dataProtectionContainer // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, blobContributorRole) - properties: { - roleDefinitionId: blobContributorRole - principalType: 'ServicePrincipal' - principalId: pcsIdentity.properties.principalId - } -} - -resource storageAccountQueueService 'Microsoft.Storage/storageAccounts/queueServices@2022-09-01' = { - name: 'default' - parent: storageAccount -} - -resource storageAccountQueue 'Microsoft.Storage/storageAccounts/queueServices/queues@2022-09-01' = { - name: 'pcs-workitems' - parent: storageAccountQueueService -} - -// allow storage queue access to the identity used for the aca's -resource pcsStorageQueueAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: storageAccount // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, storageQueueContrubutorRole) - properties: { - roleDefinitionId: storageQueueContrubutorRole - principalType: 'ServicePrincipal' - principalId: pcsIdentity.properties.principalId - } -} - -// allow storage queue access to the identity used for the SubscriptionTriggerer -resource subscriptionTriggererStorageQueueAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: storageAccount // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, storageQueueContrubutorRole) - properties: { - roleDefinitionId: storageQueueContrubutorRole - principalType: 'ServicePrincipal' - principalId: subscriptionTriggererIdentity.properties.principalId - } - } - -// allow redis cache read / write access to the service's identity -resource redisCacheBuiltInAccessPolicyAssignment 'Microsoft.Cache/redis/accessPolicyAssignments@2024-03-01' = { - name: guid(subscription().id, resourceGroup().id, 'pcsDataContributor') - parent: redisCache - properties: { - accessPolicyName: 'Data Contributor' - objectId: pcsIdentity.properties.principalId - objectIdAlias: 'PCS Managed Identity' - } -} - -// Give the PCS Deployment MI the Contributor role in the containerapp to allow it to deploy -resource deploymentContainerAppContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: containerapp // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, contributorRole) - properties: { - roleDefinitionId: contributorRole - principalType: 'ServicePrincipal' - principalId: deploymentIdentity.properties.principalId +module keyVaultsModule 'key-vaults.bicep' = { + name: 'keyVaultsModule' + params: { + location: location + keyVaultName: keyVaultName + devKeyVaultName: devKeyVaultName + aspnetcoreEnvironment: aspnetcoreEnvironment + kvSecretUserRole: kvSecretUserRole + kvCryptoUserRole: kvCryptoUserRole + pcsIdentityPrincipalId: managedIdentitiesModule.outputs.pcsIdentityPrincipalId } } -// Give the PCS Deployment MI the Key Vault Reader role to be able to read secrets during the deployment -resource deploymentKeyVaultReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: keyVault // Use when specifying a scope that is different than the deployment scope - name: guid(subscription().id, resourceGroup().id, keyVaultReaderRole) - properties: { - roleDefinitionId: keyVaultReaderRole - principalType: 'ServicePrincipal' - principalId: deploymentIdentity.properties.principalId +module redisModule 'redis.bicep' = { + name: 'redisModule' + params: { + location: location + azureCacheRedisName: azureCacheRedisName + pcsIdentityPrincipalId: managedIdentitiesModule.outputs.pcsIdentityPrincipalId } - dependsOn: [ - deploymentContainerAppContributor - ] } -// Give the PCS Deployment MI the ACR Push role to be able to push docker images -resource deploymentAcrPush 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: containerRegistry - name: guid(subscription().id, resourceGroup().id, 'deploymentAcrPush') - properties: { - roleDefinitionId: acrPushRole - principalType: 'ServicePrincipal' - principalId: deploymentIdentity.properties.principalId +module storageAccountModule 'storage-account.bicep' = { + name: 'storageAccountModule' + params: { + location: location + storageAccountName: storageAccountName + pcsIdentityPrincipalId: managedIdentitiesModule.outputs.pcsIdentityPrincipalId + subscriptionTriggererIdentityPrincipalId: managedIdentitiesModule.outputs.subscriptionTriggererIdentityPrincipalId + blobContributorRole: blobContributorRole + storageQueueContrubutorRole: storageQueueContrubutorRole } } diff --git a/eng/service-templates/ProductConstructionService/redis.bicep b/eng/service-templates/ProductConstructionService/redis.bicep new file mode 100644 index 0000000000..09d753f15e --- /dev/null +++ b/eng/service-templates/ProductConstructionService/redis.bicep @@ -0,0 +1,32 @@ +param location string +param azureCacheRedisName string +param pcsIdentityPrincipalId string + +resource redisCache 'Microsoft.Cache/redis@2024-03-01' = { + name: azureCacheRedisName + location: location + properties: { + enableNonSslPort: false + minimumTlsVersion: '1.2' + sku: { + capacity: 0 + family: 'C' + name: 'Basic' + } + redisConfiguration: { + 'aad-enabled': 'true' + } + disableAccessKeyAuthentication: true + } +} + +// allow redis cache read / write access to the service's identity +resource redisCacheBuiltInAccessPolicyAssignment 'Microsoft.Cache/redis/accessPolicyAssignments@2024-03-01' = { + name: guid(subscription().id, resourceGroup().id, 'pcsDataContributor') + parent: redisCache + properties: { + accessPolicyName: 'Data Contributor' + objectId: pcsIdentityPrincipalId + objectIdAlias: 'PCS Managed Identity' + } +} diff --git a/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep b/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep index 8c02b214ca..382a5c73fb 100644 --- a/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep +++ b/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep @@ -89,7 +89,7 @@ resource containerJob 'Microsoft.App/jobs@2024-03-01' = { } resource deploymentSubscriptionTriggererContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: containerJob // Use when specifying a scope that is different than the deployment scope + scope: containerJob name: guid(subscription().id, resourceGroup().id, '${jobName}-contributor') properties: { roleDefinitionId: contributorRoleId diff --git a/eng/service-templates/ProductConstructionService/storage-account.bicep b/eng/service-templates/ProductConstructionService/storage-account.bicep new file mode 100644 index 0000000000..7749e55ccd --- /dev/null +++ b/eng/service-templates/ProductConstructionService/storage-account.bicep @@ -0,0 +1,77 @@ +param location string +param storageAccountName string +param pcsIdentityPrincipalId string +param subscriptionTriggererIdentityPrincipalId string +param storageQueueContrubutorRole string +param blobContributorRole string + +resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { + name: storageAccountName + location: location + kind: 'StorageV2' + sku: { + name: 'Standard_LRS' + } + properties: { + allowBlobPublicAccess: false + publicNetworkAccess: 'Enabled' + allowSharedKeyAccess: false + networkAcls: { + defaultAction: 'Deny' + } + } +} + +// Create the dataprotection container in the storage account +resource storageAccountBlobService 'Microsoft.Storage/storageAccounts/blobServices@2022-09-01' = { + name: 'default' + parent: storageAccount +} + +resource dataProtectionContainer 'Microsoft.Storage/storageAccounts/blobServices/containers@2022-09-01' = { + name: 'dataprotection' + parent: storageAccountBlobService +} + +resource storageAccountQueueService 'Microsoft.Storage/storageAccounts/queueServices@2022-09-01' = { + name: 'default' + parent: storageAccount +} + +resource storageAccountQueue 'Microsoft.Storage/storageAccounts/queueServices/queues@2022-09-01' = { + name: 'pcs-workitems' + parent: storageAccountQueueService +} + +// allow storage queue access to the identity used for the aca's +resource pcsStorageQueueAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: storageAccount + name: guid(subscription().id, resourceGroup().id, 'pcs-queue-access') + properties: { + roleDefinitionId: storageQueueContrubutorRole + principalType: 'ServicePrincipal' + principalId: pcsIdentityPrincipalId + } +} + +// allow storage queue access to the identity used for the SubscriptionTriggerer +resource subscriptionTriggererStorageQueueAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: storageAccount + name: guid(subscription().id, resourceGroup().id, 'sub-triggerer-queue-access') + properties: { + roleDefinitionId: storageQueueContrubutorRole + principalType: 'ServicePrincipal' + principalId: subscriptionTriggererIdentityPrincipalId + } +} + +// allow data protection container access to the identity used for the pcs +resource storageAccountContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: dataProtectionContainer + name: guid(subscription().id, resourceGroup().id, blobContributorRole) + properties: { + roleDefinitionId: blobContributorRole + principalType: 'ServicePrincipal' + principalId: pcsIdentityPrincipalId + } +} diff --git a/eng/service-templates/ProductConstructionService/virtual-network.bicep b/eng/service-templates/ProductConstructionService/virtual-network.bicep new file mode 100644 index 0000000000..e30189d51e --- /dev/null +++ b/eng/service-templates/ProductConstructionService/virtual-network.bicep @@ -0,0 +1,41 @@ +param virtualNetworkName string +param location string +param productConstructionServiceSubnetName string +param networkSecurityGroupId string + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + '10.0.0.0/16' + ] + } + } + tags: { + 'ms.inv.v0.networkUsage': 'mixedTraffic' + } +} + +// subnet for the product construction service +resource productConstructionServiceSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-04-01' = { + name: productConstructionServiceSubnetName + parent: virtualNetwork + properties: { + addressPrefix: '10.0.0.0/24' + delegations: [ + { + name: 'Microsoft.App/environments' + properties: { + serviceName: 'Microsoft.App/environments' + } + } + ] + networkSecurityGroup: { + id: networkSecurityGroupId + } + } +} + +output productConstructionServiceSubnetId string = productConstructionServiceSubnet.id diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json index 8dc3c18e2b..4c3b8bbf6d 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json @@ -2,7 +2,7 @@ "KeyVaultName": "ProductConstructionInt", "ConnectionStrings": { "queues": "https://productconstructionint.queue.core.windows.net", - "redis": "prodconstaging.redis.cache.windows.net:6380,ssl=true" + "redis": "product-construction-service-redis-int.redis.cache.windows.net:6380,ssl=true" }, "ManagedIdentityClientId": "1d43ba8a-c2a6-4fad-b064-6d8c16fc0745", "VmrUri": "https://github.com/maestro-auth-test/dnceng-vmr", From f0c6ffa92beea75290ef29fb72bf13fc0a1bdfe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Mon, 9 Sep 2024 11:33:24 +0200 Subject: [PATCH 12/13] Measure time messages wait in the work item queue (#3934) --- .../PcsStartup.cs | 5 +-- .../ProductConstructionService.Api/Program.cs | 1 + .../MetricRecorder.cs | 32 +++++++++++++++++++ .../ProductConstructionServiceExtension.cs | 5 +++ .../Extensions.cs | 8 +++-- ...ConstructionService.ServiceDefaults.csproj | 4 +++ .../WorkItem.cs | 12 ++++++- .../WorkItemConsumer.cs | 31 +++++++++++------- .../WorkItemProducer.cs | 6 ++++ .../UpdaterTests.cs | 6 ++-- 10 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.Common/MetricRecorder.cs diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index cc8a5d395a..bbb84f64b4 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -38,6 +38,7 @@ using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; using ProductConstructionService.DependencyFlow; +using ProductConstructionService.ServiceDefaults; namespace ProductConstructionService.Api; @@ -169,7 +170,9 @@ internal static async Task ConfigurePcs( // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator builder.Services.AddSingleton(sp => new(RunningService.PCS)); + await builder.AddRedisCache(authRedis); builder.AddBuildAssetRegistry(); + builder.AddMetricRecorder(); builder.AddWorkItemQueues(azureCredential, waitForInitialization: initializeService); builder.AddDependencyFlowProcessors(); builder.AddVmrRegistrations(); @@ -193,8 +196,6 @@ internal static async Task ConfigurePcs( builder.Services.AddMergePolicies(); builder.Services.Configure(builder.Configuration.GetSection(ConfigurationKeys.DependencyFlowSLAs)); - await builder.AddRedisCache(authRedis); - if (initializeService) { builder.AddVmrInitialization(); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs index 19f9bf70c6..09e89ab34a 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs @@ -6,6 +6,7 @@ using ProductConstructionService.Api; using ProductConstructionService.Api.Configuration; using ProductConstructionService.Common; +using ProductConstructionService.ServiceDefaults; using ProductConstructionService.WorkItems; var builder = WebApplication.CreateBuilder(args); diff --git a/src/ProductConstructionService/ProductConstructionService.Common/MetricRecorder.cs b/src/ProductConstructionService/ProductConstructionService.Common/MetricRecorder.cs new file mode 100644 index 0000000000..179c2211ab --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/MetricRecorder.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.Metrics; +using Azure.Storage.Queues.Models; + +namespace ProductConstructionService.Common; + +public interface IMetricRecorder +{ + void QueueMessageReceived(QueueMessage message, TimeSpan delay); +} + +public class MetricRecorder : IMetricRecorder +{ + public const string PcsMetricsNamespace = "ProductConstructionService.Metrics"; + private const string WaitTimeMetricName = "pcs.queue.wait_time"; + + private readonly Counter _queueWaitTimeCounter; + + public MetricRecorder(IMeterFactory meterFactory) + { + var meter = meterFactory.Create(PcsMetricsNamespace); + _queueWaitTimeCounter = meter.CreateCounter(WaitTimeMetricName); + } + + public void QueueMessageReceived(QueueMessage message, TimeSpan delay) + { + TimeSpan timeInQueue = DateTimeOffset.UtcNow - message.InsertedOn!.Value - delay; + _queueWaitTimeCounter.Add((int)timeInQueue.TotalSeconds); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs b/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs index 99545e3fcd..fc97d46c00 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs @@ -74,4 +74,9 @@ public static async Task AddRedisCache( builder.Services.AddSingleton(redisConfig); builder.Services.AddScoped(); } + + public static void AddMetricRecorder(this IHostApplicationBuilder builder) + { + builder.Services.AddSingleton(); + } } diff --git a/src/ProductConstructionService/ProductConstructionService.ServiceDefaults/Extensions.cs b/src/ProductConstructionService/ProductConstructionService.ServiceDefaults/Extensions.cs index 8730cbba42..3659f9b463 100644 --- a/src/ProductConstructionService/ProductConstructionService.ServiceDefaults/Extensions.cs +++ b/src/ProductConstructionService/ProductConstructionService.ServiceDefaults/Extensions.cs @@ -6,12 +6,14 @@ using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using OpenTelemetry.Logs; using OpenTelemetry.Metrics; using OpenTelemetry.Trace; +using ProductConstructionService.Common; -namespace Microsoft.Extensions.Hosting; +namespace ProductConstructionService.ServiceDefaults; public static class Extensions { @@ -47,7 +49,8 @@ public static IHostApplicationBuilder ConfigureOpenTelemetry(this IHostApplicati .WithMetrics(metrics => { metrics.AddRuntimeInstrumentation() - .AddBuiltInMeters(); + .AddBuiltInMeters() + .AddMeter(MetricRecorder.PcsMetricsNamespace); }) .WithTracing(tracing => { @@ -75,6 +78,7 @@ private static IHostApplicationBuilder AddOpenTelemetryExporters(this IHostAppli { builder.Services.Configure(logging => logging.AddOtlpExporter()); builder.Services.ConfigureOpenTelemetryMeterProvider(metrics => metrics.AddOtlpExporter()); + builder.Services.ConfigureOpenTelemetryMeterProvider(metrics => metrics.AddMeter(MetricRecorder.PcsMetricsNamespace)); builder.Services.ConfigureOpenTelemetryTracerProvider(tracing => tracing.AddOtlpExporter()); } diff --git a/src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj b/src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj index 26b9a08102..f3f3c7dbe2 100644 --- a/src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj +++ b/src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj @@ -23,4 +23,8 @@ + + + + diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs index 60d1bc1846..7c5b7cd9fd 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs @@ -1,10 +1,20 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json.Serialization; + namespace ProductConstructionService.WorkItems; public abstract class WorkItem { - public Guid Id { get; init; } = Guid.NewGuid(); + /// + /// Type of the message for easier deserialization. + /// public string Type => GetType().Name; + + /// + /// Period of time before the WorkItem becomes visible in the queue. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault | JsonIgnoreCondition.WhenWritingNull)] + public TimeSpan? Delay { get; internal set; } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs index 44697d9889..76e80caf0e 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs @@ -7,19 +7,22 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ProductConstructionService.Common; namespace ProductConstructionService.WorkItems; internal class WorkItemConsumer( - ILogger logger, - IOptions options, - WorkItemScopeManager scopeManager, - QueueServiceClient queueServiceClient) + ILogger logger, + IOptions options, + WorkItemScopeManager scopeManager, + QueueServiceClient queueServiceClient, + IMetricRecorder metricRecorder) : BackgroundService { private readonly ILogger _logger = logger; private readonly IOptions _options = options; private readonly WorkItemScopeManager _scopeManager = scopeManager; + private readonly IMetricRecorder _metricRecorder = metricRecorder; protected override async Task ExecuteAsync(CancellationToken cancellationToken) { @@ -66,14 +69,18 @@ private async Task ReadAndProcessWorkItemAsync(QueueClient queueClient, WorkItem return; } - string workItemId; string workItemType; + TimeSpan? delay; JsonNode node; try { node = JsonNode.Parse(message.Body)!; - workItemId = node["id"]!.ToString(); workItemType = node["type"]!.ToString(); + + var d = node["delay"]?.GetValue(); + delay = d.HasValue + ? TimeSpan.FromSeconds(d.Value) + : null; } catch (Exception ex) { @@ -82,9 +89,11 @@ private async Task ReadAndProcessWorkItemAsync(QueueClient queueClient, WorkItem return; } + _metricRecorder.QueueMessageReceived(message, delay ?? default); + try { - _logger.LogInformation("Starting attempt {attemptNumber} for work item {workItemId}, type {workItemType}", message.DequeueCount, workItemId, workItemType); + _logger.LogInformation("Starting attempt {attemptNumber} for {workItemType}", message.DequeueCount, workItemType); await workItemScope.RunWorkItemAsync(node, cancellationToken); await queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, cancellationToken); } @@ -95,13 +104,13 @@ private async Task ReadAndProcessWorkItemAsync(QueueClient queueClient, WorkItem } catch (Exception ex) { - _logger.LogError(ex, "Processing work item {workItemId} attempt {attempt}/{maxAttempts} failed", - workItemId, message.DequeueCount, _options.Value.MaxWorkItemRetries); + _logger.LogError(ex, "Processing work item {workItemType} attempt {attempt}/{maxAttempts} failed", + workItemType, message.DequeueCount, _options.Value.MaxWorkItemRetries); // Let the workItem retry a few times. If it fails a few times, delete it from the queue, it's a bad work item if (message.DequeueCount == _options.Value.MaxWorkItemRetries || ex is NonRetriableException) { - _logger.LogError("Work item {workItemId} has failed {maxAttempts} times. Discarding the message {message} from the queue", - workItemId, _options.Value.MaxWorkItemRetries, message.Body.ToString()); + _logger.LogError("Work item {type} has failed {maxAttempts} times. Discarding the message {message} from the queue", + workItemType, _options.Value.MaxWorkItemRetries, message.Body.ToString()); await queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, cancellationToken); } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs index 001db4bcf7..620db1d715 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs @@ -28,6 +28,12 @@ public class WorkItemProducer(QueueServiceClient queueServiceClient, string q public async Task ProduceWorkItemAsync(T payload, TimeSpan delay = default) { var client = _queueServiceClient.GetQueueClient(_queueName); + + if (delay != default) + { + payload.Delay = delay; + } + var json = JsonSerializer.Serialize(payload, WorkItemConfiguration.JsonSerializerOptions); return await client.SendMessageAsync(json, delay); } diff --git a/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs index d9ccfa64eb..ec5404f5cd 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs @@ -96,10 +96,8 @@ public void UpdaterTests_SetUp() [TearDown] public void UpdaterTests_TearDown() { - var ExcludeWorkItemId = static (FluentAssertions.Equivalency.EquivalencyAssertionOptions> opt) - => opt.Excluding(member => member.DeclaringType == typeof(WorkItem) && member.Name.Equals(nameof(WorkItem.Id))); - Cache.Data.Should().BeEquivalentTo(ExpectedCacheState, ExcludeWorkItemId); - Reminders.Reminders.Should().BeEquivalentTo(ExpectedReminders, ExcludeWorkItemId); + Cache.Data.Should().BeEquivalentTo(ExpectedCacheState); + Reminders.Reminders.Should().BeEquivalentTo(ExpectedReminders); } protected void SetState(Subscription subscription, T state) where T : class From b58b1d2074aab08d54eb799e909a1824a0b21e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Mon, 9 Sep 2024 11:54:49 +0200 Subject: [PATCH 13/13] Split PCS and repo build into separate stages (#3936) --- azure-pipelines-product-construction-service.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index 58876e8ab4..812a5e9780 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -139,6 +139,10 @@ stages: displayName: Upload snapshot diff artifact: DeploymentDiff +- stage: BuildRepo + displayName: Build and Publish Repo + dependsOn: [] + jobs: - job: BuildAndPublish displayName: Build and Publish Repo pool: @@ -185,6 +189,7 @@ stages: displayName: Run E2E Product Construction Service Tests dependsOn: - DeployPCS + - BuildRepo jobs: - template: /eng/templates/jobs/e2e-pcs-tests.yml