From b7f2f361674fb05e95d9806671952b464e014928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 12 Jun 2024 15:57:13 +0200 Subject: [PATCH 01/11] Allow linking AzDO issues in PR descriptions (#3626) --- eng/enforce-issue.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eng/enforce-issue.ps1 b/eng/enforce-issue.ps1 index 635cab8b4c..3fb7edd7b1 100644 --- a/eng/enforce-issue.ps1 +++ b/eng/enforce-issue.ps1 @@ -28,10 +28,15 @@ elseif ($prDetail.title -match "\[automated\]") { exit 0 } +# Look for https://github.com/dotnet/arcade-services/issues/3625 $hasIssue = $prDetail.body -Match "github\.com/dotnet/(.+)/issues/(\d+)" if (-not $hasIssue) { - Write-Host "##vso[task.LogIssue type=error;]Link to the corresponding GitHub issue is missing in the PR description. Check failed." - exit 1 + # Or for https://dev.azure.com/dnceng/internal/_workitems/edit/45126 + $hasIssue = $prDetail.body -Match "dev\.azure\.com/dnceng/internal/_workitems" + if (-not $hasIssue) { + Write-Host "##vso[task.LogIssue type=error;]Link to the corresponding GitHub/AzDO issue is missing in the PR description. Check failed." + exit 1 + } } if (HasReleaseNotes $prDetail.body) { From fc5dc0cbd9f438858279fcb6b5f43e1194a571fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 12 Jun 2024 16:49:44 +0200 Subject: [PATCH 02/11] Use system MI to call PCS from Maestro (#3625) --- .../.config/settings.Development.json | 2 +- .../.config/settings.Production.json | 2 +- .../.config/settings.Staging.json | 3 +-- src/Maestro/SubscriptionActorService/Program.cs | 15 ++++++++------- .../PcsApiCredential.cs | 4 +++- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/Maestro/SubscriptionActorService/.config/settings.Development.json b/src/Maestro/SubscriptionActorService/.config/settings.Development.json index 5d4327729e..35afe1e7e5 100644 --- a/src/Maestro/SubscriptionActorService/.config/settings.Development.json +++ b/src/Maestro/SubscriptionActorService/.config/settings.Development.json @@ -9,7 +9,7 @@ }, "ProductConstructionService": { "Uri": "http://localhost:53181/", - "Token": "" + "NoAuth": true }, "Kusto": { "Database": "engineeringdata", diff --git a/src/Maestro/SubscriptionActorService/.config/settings.Production.json b/src/Maestro/SubscriptionActorService/.config/settings.Production.json index 804c617f1c..becdb23829 100644 --- a/src/Maestro/SubscriptionActorService/.config/settings.Production.json +++ b/src/Maestro/SubscriptionActorService/.config/settings.Production.json @@ -7,7 +7,7 @@ "DarcTemporaryRepoRoot": "D:\\", "ProductConstructionService": { "Uri": "https://github.com/dotnet/arcade-services/issues/3183", - "Token": "TODO https://github.com/dotnet/arcade-services/issues/3183" + "NoAuth": true // TODO https://github.com/dotnet/arcade-services/issues/3183 }, "BuildAssetRegistry": { "ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" diff --git a/src/Maestro/SubscriptionActorService/.config/settings.Staging.json b/src/Maestro/SubscriptionActorService/.config/settings.Staging.json index c20b744196..a98d5209ca 100644 --- a/src/Maestro/SubscriptionActorService/.config/settings.Staging.json +++ b/src/Maestro/SubscriptionActorService/.config/settings.Staging.json @@ -6,8 +6,7 @@ "KeyVaultUri": "https://maestroint.vault.azure.net/", "DarcTemporaryRepoRoot": "D:\\", "ProductConstructionService": { - "Uri": "https://product-construction-int.wittytree-28a89311.westus2.azurecontainerapps.io/", - "Token": "[vault(product-construction-service-int-token)]" + "Uri": "https://product-construction-int.wittytree-28a89311.westus2.azurecontainerapps.io/" }, "BuildAssetRegistry": { "ConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" diff --git a/src/Maestro/SubscriptionActorService/Program.cs b/src/Maestro/SubscriptionActorService/Program.cs index 27d0d1a3ae..cc1a2c06b5 100644 --- a/src/Maestro/SubscriptionActorService/Program.cs +++ b/src/Maestro/SubscriptionActorService/Program.cs @@ -75,20 +75,21 @@ public static void Configure(IServiceCollection services) var tokenMap = s.GetChildren(); foreach (IConfigurationSection token in tokenMap) { - o.Tokens.Add(token.GetValue("Account"), token.GetValue("Token")); + o.Tokens.Add(token["Account"], token["Token"]); } }); services.AddSingleton(s => { var config = s.GetRequiredService(); - var uri = config.GetValue("ProductConstructionService:Uri"); + var uri = config["ProductConstructionService:Uri"]; - // TODO https://dev.azure.com/dnceng/internal/_workitems/edit/6451: Implement Maestro - PCS communication - var token = config.GetValue("ProductConstructionService:Token"); + var noAuth = config.GetValue("ProductConstructionService:NoAuth"); + if (noAuth) + { + return PcsApiFactory.GetAnonymous(uri); + } - return string.IsNullOrEmpty(token) - ? PcsApiFactory.GetAnonymous(uri) - : PcsApiFactory.GetAuthenticated(uri, token); + return PcsApiFactory.GetAuthenticated(uri, managedIdentityId: "system"); }); services.AddMergePolicies(); diff --git a/src/ProductConstructionService/ProductConstructionService.Client/PcsApiCredential.cs b/src/ProductConstructionService/ProductConstructionService.Client/PcsApiCredential.cs index dc13289eed..d023169310 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/PcsApiCredential.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/PcsApiCredential.cs @@ -49,7 +49,9 @@ internal static PcsApiCredential CreateManagedIdentityCredential(string barApiBa { string appId = EntraAppIds[barApiBaseUri.TrimEnd('/')]; - var miCredential = new ManagedIdentityCredential(managedIdentityId); + var miCredential = managedIdentityId == "system" + ? new ManagedIdentityCredential() + : new ManagedIdentityCredential(managedIdentityId); var appCredential = new ClientAssertionCredential( TENANT_ID, From c22442f69174abd59c3d8fe4bf5d3d5c744faf46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 12 Jun 2024 19:10:04 +0200 Subject: [PATCH 03/11] Fix an NRE in LocalSettings (#3632) --- src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs b/src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs index b865415bf4..0a735dc0ce 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs @@ -90,14 +90,15 @@ static string PreferOptionToSetting(string option, string localSetting) return !string.IsNullOrEmpty(option) ? option : localSetting; } + // Prefer the command line options over the settings file + localSettings ??= new LocalSettings(); + if (string.IsNullOrEmpty(localSettings.BuildAssetRegistryToken)) { // Old way of storing the settings had the password and not the token localSettings.BuildAssetRegistryToken = localSettings.BuildAssetRegistryPassword; } - // Prefer the command line options over the settings file - localSettings ??= new LocalSettings(); localSettings.AzureDevOpsToken = PreferOptionToSetting(options.AzureDevOpsPat, localSettings.AzureDevOpsToken); localSettings.GitHubToken = PreferOptionToSetting(options.GitHubPat, localSettings.GitHubToken); localSettings.BuildAssetRegistryToken = PreferOptionToSetting(options.BuildAssetRegistryToken, localSettings.BuildAssetRegistryToken); From 57f9755405b9a4fe00cf6e1dcbba11c2e5ef032f Mon Sep 17 00:00:00 2001 From: Oleksandr Didyk <106967057+oleksandr-didyk@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:27:51 +0200 Subject: [PATCH 04/11] Switch auth for scenario tests (#3630) --- azure-pipelines.yml | 2 + eng/templates/stages/deploy.yaml | 45 +++++++++++++++- .../MaestroScenarioTestBase.cs | 22 +++++--- .../ScenarioTests_Channels.cs | 1 + test/Maestro.ScenarioTests/TestParameters.cs | 52 +++++++++++++------ 5 files changed, 96 insertions(+), 26 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0d23900438..d27e407cff 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -189,6 +189,7 @@ extends: MaestroTestEndpoints: https://maestro-int.westus2.cloudapp.azure.com,https://maestro.int-dot.net PublishProfile: Int Subscription: NetHelixStaging + ScenarioTestSubscription: "Darc: Maestro Staging" VariableGroup: MaestroInt KeyVault BarConnectionString: "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Default; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=120; Encrypt=True; TrustServerCertificate=False; User Id=736067df-a2c8-4c63-ad30-12db55c186cc" BarMigrationSubscription: BarMigrationInt @@ -197,6 +198,7 @@ extends: MaestroTestEndpoints: https://maestro-prod.westus2.cloudapp.azure.com,https://maestro.dot.net PublishProfile: Prod Subscription: NetHelix + ScenarioTestSubscription: "Darc: Maestro Production" VariableGroup: MaestroProd KeyVault BarConnectionString: "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Default; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=120; Encrypt=True; TrustServerCertificate=False; User Id=1093df3b-c754-4788-a4ae-ea33b86b82aa" BarMigrationSubscription: BarMigrationProd \ No newline at end of file diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index 18f6ff096a..8fa9bb6ca1 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -1,6 +1,8 @@ parameters: - name: Subscription type: string +- name: ScenarioTestSubscription + type: string - name: PublishProfile type: string values: ['Int', 'Prod'] @@ -141,6 +143,45 @@ stages: - powershell: .\eng\common\build.ps1 -restore displayName: Install .NET + - powershell: | + mkdir darc + + $dotnetDir = cmd /c "where dotnet" + Invoke-Expression "& '$dotnetDir' tool install Microsoft.DotNet.Darc --prerelease --tool-path .\darc --add-source $(Pipeline.Workspace)\PackageArtifacts" + displayName: Install Darc + + - task: AzureCLI@2 + name: GetAuthInfo + displayName: Get auth information + inputs: + azureSubscription: ${{ parameters.ScenarioTestSubscription }} + addSpnToEnvironment: true + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + # Fetch token used for scenario tests + $token = (az account get-access-token --resource "$env:servicePrincipalId" | ConvertFrom-Json).accessToken + echo "##vso[task.setvariable variable=Token;isOutput=true;isSecret=true]$token" + + # Set variables with auth info for tests below + echo "##vso[task.setvariable variable=ServicePrincipalId;isOutput=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=FederatedToken;isOutput=true;isSecret=true]$env:idToken" + echo "##vso[task.setvariable variable=TenantId;isOutput=true]$env:tenantId" + + # Either of the URIs will do + $barUri = "${{ split(parameters.MaestroTestEndpoints, ',')[0] }}" + echo "##vso[task.setvariable variable=BarUri;isOutput=true]$barUri" + + - powershell: + az login --service-principal -u "$(GetAuthInfo.ServicePrincipalId)" --federated-token "$(GetAuthInfo.FederatedToken)" --tenant "$(GetAuthInfo.TenantId)" --allow-no-subscriptions + + .\darc\darc.exe get-subscriptions --ci --source-enabled true --bar-uri "$(GetAuthInfo.BarUri)" --debug + displayName: Test Azure CLI authentication + + - powershell: + .\darc\darc.exe get-subscriptions -t "$(GetAuthInfo.FederatedToken)" --source-enabled true --bar-uri "$(GetAuthInfo.BarUri)" --debug + displayName: Test Federated token authentication + - task: VSTest@2 displayName: Maestro Scenario Tests inputs: @@ -151,10 +192,12 @@ stages: runInParallel: true env: MAESTRO_BASEURIS: ${{ parameters.MaestroTestEndpoints }} - MAESTRO_TOKEN: $(scenario-test-maestro-token) + MAESTRO_TOKEN: $(GetAuthInfo.Token) GITHUB_TOKEN: $(maestro-scenario-test-github-token) AZDO_TOKEN: $(dn-bot-dnceng-build-rw-code-rw-release-rw) DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts + DARC_DIR: $(Build.SourcesDirectory)\darc + DARC_IS_CI: true - powershell: | nuget sources add -Name "arcade" -Source "https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json" diff --git a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs index 3388e0f438..da6c7496e3 100644 --- a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs +++ b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs @@ -30,6 +30,7 @@ namespace Maestro.ScenarioTests; internal abstract class MaestroScenarioTestBase { private TestParameters _parameters = null!; + private List _baseDarcRunArgs = new List(); protected IMaestroApi MaestroApi => _parameters.MaestroApi; @@ -40,6 +41,17 @@ internal abstract class MaestroScenarioTestBase public void SetTestParameters(TestParameters parameters) { _parameters = parameters; + _baseDarcRunArgs = [ + "--bar-uri", _parameters.MaestroBaseUri, + "--github-pat", _parameters.GitHubToken, + "--azdev-pat", _parameters.AzDoToken, + _parameters.IsCI ? "--ci" : "" + ]; + + if (!string.IsNullOrEmpty(_parameters.MaestroToken)) + { + _baseDarcRunArgs.AddRange(["--p", _parameters.MaestroToken]); + } } protected async Task WaitForPullRequestAsync(string targetRepo, string targetBranch) @@ -401,10 +413,7 @@ protected Task RunDarcAsyncWithInput(string input, params string[] args) return TestHelpers.RunExecutableAsyncWithInput(_parameters.DarcExePath, input, [ .. args, - "-p", _parameters.MaestroToken ?? string.Empty, - "--bar-uri", _parameters.MaestroBaseUri, - "--github-pat", _parameters.GitHubToken, - "--azdev-pat", _parameters.AzDoToken, + .. _baseDarcRunArgs, ]); } @@ -413,10 +422,7 @@ protected Task RunDarcAsync(params string[] args) return TestHelpers.RunExecutableAsync(_parameters.DarcExePath, [ .. args, - "-p", _parameters.MaestroToken ?? string.Empty, - "--bar-uri", _parameters.MaestroBaseUri, - "--github-pat", _parameters.GitHubToken, - "--azdev-pat", _parameters.AzDoToken, + .. _baseDarcRunArgs, ]); } diff --git a/test/Maestro.ScenarioTests/ScenarioTests_Channels.cs b/test/Maestro.ScenarioTests/ScenarioTests_Channels.cs index 6e60dafa7a..4130d5399e 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_Channels.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_Channels.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.Threading.Tasks; using FluentAssertions; using NUnit.Framework; diff --git a/test/Maestro.ScenarioTests/TestParameters.cs b/test/Maestro.ScenarioTests/TestParameters.cs index f73acb17ca..cbae51745e 100644 --- a/test/Maestro.ScenarioTests/TestParameters.cs +++ b/test/Maestro.ScenarioTests/TestParameters.cs @@ -26,6 +26,8 @@ public class TestParameters : IDisposable private static readonly string githubToken; private static readonly string darcPackageSource; private static readonly string azdoToken; + private static readonly bool isCI; + private static readonly string? darcDir; static TestParameters() { @@ -38,12 +40,14 @@ static TestParameters() ?? "https://maestro.int-dot.net") .Split(','); maestroToken = Environment.GetEnvironmentVariable("MAESTRO_TOKEN") ?? userSecrets["MAESTRO_TOKEN"]; + isCI = Environment.GetEnvironmentVariable("DARC_IS_CI")?.ToLower() == "true"; githubToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN") ?? userSecrets["GITHUB_TOKEN"] ?? throw new Exception("Please configure the GitHub token"); darcPackageSource = Environment.GetEnvironmentVariable("DARC_PACKAGE_SOURCE") ?? throw new Exception("Please configure the Darc package source"); azdoToken = Environment.GetEnvironmentVariable("AZDO_TOKEN") ?? userSecrets["AZDO_TOKEN"] ?? throw new Exception("Please configure the Azure DevOps token"); + darcDir = Environment.GetEnvironmentVariable("DARC_DIR"); } /// If set to true, the test will attempt to use the non primary endpoint, if provided @@ -61,16 +65,39 @@ public static async Task GetAsync(bool useNonPrimaryEndpoint = f maestroToken, managedIdentityId: null, federatedToken: null, - disableInteractiveAuth: true); + disableInteractiveAuth: isCI); + string darcRootDir = darcDir ?? ""; + if (string.IsNullOrEmpty(darcRootDir)) + { + await InstallDarc(maestroApi, testDirSharedWrapper); + darcRootDir = testDirSharedWrapper.Peek()!.Directory; + } + + string darcExe = Path.Join(darcRootDir, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "darc.exe" : "darc"); + + Assembly assembly = typeof(TestParameters).Assembly; + var githubApi = + new GitHubClient( + new ProductHeaderValue(assembly.GetName().Name, assembly.GetCustomAttribute()?.InformationalVersion), + new InMemoryCredentialStore(new Credentials(githubToken))); + var azDoClient = + new Microsoft.DotNet.DarcLib.AzureDevOpsClient(await TestHelpers.Which("git"), azdoToken, new NUnitLogger(), testDirSharedWrapper.TryTake()!.Directory); + + return new TestParameters( + darcExe, await TestHelpers.Which("git"), maestroBaseUri, maestroToken, githubToken, maestroApi, githubApi, azDoClient, testDir, azdoToken, isCI); + } + + private static async Task InstallDarc(IMaestroApi maestroApi, Shareable toolPath) + { string darcVersion = await maestroApi.Assets.GetDarcVersionAsync(); string dotnetExe = await TestHelpers.Which("dotnet"); var toolInstallArgs = new List { "tool", "install", - "--tool-path", testDirSharedWrapper.Peek()!.Directory, "--version", darcVersion, + "--tool-path", toolPath.Peek()!.Directory, "Microsoft.DotNet.Darc", }; @@ -81,22 +108,10 @@ public static async Task GetAsync(bool useNonPrimaryEndpoint = f } await TestHelpers.RunExecutableAsync(dotnetExe, [.. toolInstallArgs]); - - string darcExe = Path.Join(testDirSharedWrapper.Peek()!.Directory, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "darc.exe" : "darc"); - - Assembly assembly = typeof(TestParameters).Assembly; - var githubApi = - new GitHubClient( - new ProductHeaderValue(assembly.GetName().Name, assembly.GetCustomAttribute()?.InformationalVersion), - new InMemoryCredentialStore(new Credentials(githubToken))); - var azDoClient = - new Microsoft.DotNet.DarcLib.AzureDevOpsClient(await TestHelpers.Which("git"), azdoToken, new NUnitLogger(), testDirSharedWrapper.TryTake()!.Directory); - - return new TestParameters(darcExe, await TestHelpers.Which("git"), maestroBaseUri, maestroToken!, githubToken, maestroApi, githubApi, azDoClient, testDir, azdoToken); } - private TestParameters(string darcExePath, string gitExePath, string maestroBaseUri, string maestroToken, string gitHubToken, - IMaestroApi maestroApi, GitHubClient gitHubApi, Microsoft.DotNet.DarcLib.AzureDevOpsClient azdoClient, TemporaryDirectory dir, string azdoToken) + private TestParameters(string darcExePath, string gitExePath, string maestroBaseUri, string? maestroToken, string gitHubToken, + IMaestroApi maestroApi, GitHubClient gitHubApi, Microsoft.DotNet.DarcLib.AzureDevOpsClient azdoClient, TemporaryDirectory dir, string azdoToken, bool isCI) { _dir = dir; DarcExePath = darcExePath; @@ -108,6 +123,7 @@ private TestParameters(string darcExePath, string gitExePath, string maestroBase GitHubApi = gitHubApi; AzDoClient = azdoClient; AzDoToken = azdoToken; + IsCI = isCI; } public string DarcExePath { get; } @@ -120,7 +136,7 @@ private TestParameters(string darcExePath, string gitExePath, string maestroBase public string MaestroBaseUri { get; } - public string MaestroToken { get; } + public string? MaestroToken { get; } public string GitHubToken { get; } @@ -140,6 +156,8 @@ private TestParameters(string darcExePath, string gitExePath, string maestroBase public string AzDoToken { get; } + public bool IsCI { get; } + public void Dispose() { _dir?.Dispose(); From 2b36f239f2712af995b084ddacc7c4c3d0690d05 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:31:51 +0200 Subject: [PATCH 05/11] Create GitHub releases during production rollouts (#3634) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Přemek Vysoký --- .vault-config/maestroprod.yaml | 17 +++++++++++++---- azure-pipelines.yml | 2 +- eng/templates/stages/deploy.yaml | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.vault-config/maestroprod.yaml b/.vault-config/maestroprod.yaml index cada11a522..ffe9533729 100644 --- a/.vault-config/maestroprod.yaml +++ b/.vault-config/maestroprod.yaml @@ -10,6 +10,12 @@ references: parameters: subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 name: helixkv + + engkeyvault: + type: azure-key-vault + parameters: + subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 + name: engkeyvault keys: data-protection-encryption-key: @@ -19,8 +25,11 @@ keys: importSecretsFrom: shared/maestro-secrets.yaml secrets: - maestro-storage-account: - type: azure-storage-connection-string + # Needed during Maestro rollouts to create GitHub releases in arcade-services + BotAccount-dotnet-bot-repo-PAT: + type: github-access-token parameters: - subscription: 68672ab8-de0c-40f1-8d1b-ffb20bd62c0f - account: maestroprod1337 + gitHubBotAccountSecret: + location: engkeyvault + name: BotAccount-dotnet-bot + gitHubBotAccountName: dotnet-bot diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d27e407cff..e501a345d8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -201,4 +201,4 @@ extends: ScenarioTestSubscription: "Darc: Maestro Production" VariableGroup: MaestroProd KeyVault BarConnectionString: "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Default; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=120; Encrypt=True; TrustServerCertificate=False; User Id=1093df3b-c754-4788-a4ae-ea33b86b82aa" - BarMigrationSubscription: BarMigrationProd \ No newline at end of file + BarMigrationSubscription: BarMigrationProd diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index 8fa9bb6ca1..d94d69f46a 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -48,6 +48,8 @@ stages: - approval variables: + - name: currentDate + value: $[format('{0:yyyy}-{0:MM}-{0:dd}', pipeline.startTime)] - group: ${{ parameters.VariableGroup }} jobs: @@ -106,6 +108,26 @@ stages: PublishProfilePath: $(Pipeline.Workspace)/MaestroApplication/projectartifacts/PublishProfiles/${{ parameters.PublishProfile }}.xml ApplicationPackage: $(Pipeline.Workspace)/MaestroApplication/applicationpackage + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/production')}}: + - download: current + artifact: AssetManifests + + - powershell: | + $xmlPaths = Get-ChildItem $(Pipeline.Workspace)/AssetManifests/ -Filter *.xml + $xml = [xml](Get-Content $xmlPaths[0].FullName) + $releaseVersion = $xml.Build.Package[0].Version + gh release create "v$(Build.BuildNumber)-$releaseVersion" ` + --generate-notes ` + --latest ` + --title "Rollout $(currentDate) / $(Build.BuildNumber)" ` + --target $(Build.SourceVersion) ` + --notes "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" ` + --repo dotnet/arcade-services + displayName: Create GitHub release + env: + GH_TOKEN: $(BotAccount-dotnet-bot-repo-PAT) + continueOnError: true + - stage: validateDeployment displayName: Validate deployment dependsOn: From db08c90c12f7d2ebeea6cbd68f1d18b407af0fdf Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 17 Jun 2024 10:22:52 +0200 Subject: [PATCH 06/11] Throw an exception if we exceed GitHub rate limit (#3428) --- .../src/PushMetadataToBuildAssetRegistry.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Maestro/Microsoft.DotNet.Maestro.Tasks/src/PushMetadataToBuildAssetRegistry.cs b/src/Maestro/Microsoft.DotNet.Maestro.Tasks/src/PushMetadataToBuildAssetRegistry.cs index 0731781fc1..98951a6ddd 100644 --- a/src/Maestro/Microsoft.DotNet.Maestro.Tasks/src/PushMetadataToBuildAssetRegistry.cs +++ b/src/Maestro/Microsoft.DotNet.Maestro.Tasks/src/PushMetadataToBuildAssetRegistry.cs @@ -683,6 +683,12 @@ private void LookupForMatchingGitHubRepository(Manifest manifest) } else { + if (response.StatusCode == System.Net.HttpStatusCode.Forbidden + || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + { + string responseBody = response.Content.ReadAsStringAsync().Result; + throw new HttpRequestException($"API rate limit exceeded, HttpResponse: {response.StatusCode} {responseBody}. Please retry"); + } Log.LogMessage(MessageImportance.High, $" Unable to translate AzDO to GitHub URL. HttpResponse: {response.StatusCode} {response.ReasonPhrase} for repoIdentity: {repoIdentity} and commit: {manifest.Commit}."); _gitHubRepository = null; From ad5f796a93e5db68ab99c42a66a058d980c8446b Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 17 Jun 2024 11:48:52 +0200 Subject: [PATCH 07/11] Deploy PCS using the new authentication method (#3635) --- .vault-config/product-construction-int.yaml | 5 ----- ...pipelines-product-construction-service.yml | 20 ++++++++++++++++--- .../product-construction-service-deploy.ps1 | 5 +++-- .../provision.bicep | 9 --------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.vault-config/product-construction-int.yaml b/.vault-config/product-construction-int.yaml index 106242e6c4..e7c3cb9eb3 100644 --- a/.vault-config/product-construction-int.yaml +++ b/.vault-config/product-construction-int.yaml @@ -43,8 +43,3 @@ secrets: hasPrivateKey: true hasWebhookSecret: false hasOAuthSecret: true - - maestro-token: - type: maestro-access-token - parameters: - environment: maestro.int-dot.net/ diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index 96d217a9b4..c6fe4fb83e 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -31,6 +31,8 @@ variables: value: productconstructionint.azurecr.io - name: serviceConnectionName value: ProductConstructionServiceDeploymentInt + - name: authServiceConnection + value: "Darc: Maestro Staging" - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - name: devBranchSuffix value: @@ -55,7 +57,7 @@ stages: - powershell: | Write-Host "Dev branch suffix is $(devBranchSuffix)" $shortSha = "$(Build.SourceVersion)".Substring(0,10) - $newDockerTag = "$(Build.BuildNumber)-$shortSha$(devBranchSuffix)" + $newDockerTag = "$(Build.BuildNumber)-$(System.JobAttempt)-$shortSha$(devBranchSuffix)" Write-Host "##vso[task.setvariable variable=newDockerImageTag]$newDockerTag" Write-Host "set newDockerImageTag to $newDockerTag" displayName: Generate docker image tag @@ -85,6 +87,19 @@ stages: Set-Content -Path $(diffFolder)/before.json -Value $before displayName: Snapshot configuration (before) + - task: AzureCLI@2 + name: GetAuthInfo + displayName: Get PCS Token + inputs: + azureSubscription: $(authServiceConnection) + addSpnToEnvironment: true + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + # Fetch pcs token + $token = (az account get-access-token --resource "$env:servicePrincipalId" | ConvertFrom-Json).accessToken + "##vso[task.setvariable variable=Token;isOutput=true;isSecret=true]$token" + - task: AzureCLI@2 inputs: azureSubscription: $(serviceConnectionName) @@ -97,8 +112,7 @@ stages: -newImageTag $(newDockerImageTag) -containerRegistryName $(containerRegistryName) -imageName $(containerName) - -pcsUrl $(pcsUrl) - -token $(maestro-token) + -token $(GetAuthInfo.Token) displayName: Deploy container app - task: AzureCLI@2 diff --git a/eng/deployment/product-construction-service-deploy.ps1 b/eng/deployment/product-construction-service-deploy.ps1 index 40b178a595..52a9b4b566 100644 --- a/eng/deployment/product-construction-service-deploy.ps1 +++ b/eng/deployment/product-construction-service-deploy.ps1 @@ -7,10 +7,11 @@ param( [Parameter(Mandatory=$true)][string]$newImageTag, [Parameter(Mandatory=$true)][string]$containerRegistryName, [Parameter(Mandatory=$true)][string]$imageName, - [Parameter(Mandatory=$true)][string]$pcsUrl, [Parameter(Mandatory=$true)][string]$token ) +$containerapp = az containerapp show -g $resourceGroupName -n $containerappName | ConvertFrom-Json +$pcsUrl = "https://$($containerapp.properties.configuration.ingress.fqdn)" $pcsStatusUrl = $pcsUrl + "/status" $pcsStopUrl = $pcsStatusUrl + "/stop" $pcsStartUrl = $pcsStatusUrl + "/start" @@ -88,7 +89,7 @@ else # Tell the service to stop processing jobs after it finishes the current one Write-Host "Stopping the service from processing new jobs" -StopAndWait -pcsStatusUrl $pcsStatusUrl -pcsStopUrl $pcsStopUrl +StopAndWait -pcsStatusUrl $pcsStatusUrl -pcsStopUrl $pcsStopUrl -authenticationHeader $authenticationHeader # deploy the new image $newImage = "$containerRegistryName.azurecr.io/$imageName`:$newImageTag" diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 92375e3633..58ca8f9b8f 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -589,15 +589,6 @@ resource containerRegistryPasswordSecret 'Microsoft.KeyVault/vaults/secrets@2022 } } -// Automatically add the containerapp url to the keyVault. This isn't really a secret, but we do need it for the deployments and this makes it so we don't have to manually update it -resource contianerAppFqdn 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { - parent: keyVault - name: 'pcsUrl' - properties: { - value: containerapp.properties.configuration.ingress.fqdn - } -} - // If we're creating the staging environment, also create a dev key vault resource devKeyVault 'Microsoft.KeyVault/vaults@2022-07-01' = if (aspnetcoreEnvironment == 'Staging') { name: devKeyVaultName From d1d3a08ffb2721ccb93b0934dd9e7ce2f6330694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Mon, 17 Jun 2024 11:52:58 +0200 Subject: [PATCH 08/11] Support system assigned managed identities in Maestro credential (#3638) --- src/Maestro/Client/src/MaestroApiCredential.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Maestro/Client/src/MaestroApiCredential.cs b/src/Maestro/Client/src/MaestroApiCredential.cs index e0c11b9509..3c60dc5a80 100644 --- a/src/Maestro/Client/src/MaestroApiCredential.cs +++ b/src/Maestro/Client/src/MaestroApiCredential.cs @@ -142,13 +142,16 @@ internal static MaestroApiCredential CreateFederatedCredential(string barApiBase } /// - /// Use this for darc invocations from services using an MI + /// Use this for darc invocations from services using an MI. + /// ID can be "system" for system-assigned identity or GUID for a user assigned one. /// internal static MaestroApiCredential CreateManagedIdentityCredential(string barApiBaseUri, string managedIdentityId) { string appId = EntraAppIds[barApiBaseUri.TrimEnd('/')]; - var miCredential = new ManagedIdentityCredential(managedIdentityId); + var miCredential = managedIdentityId == "system" + ? new ManagedIdentityCredential() + : new ManagedIdentityCredential(managedIdentityId); var appCredential = new ClientAssertionCredential( TENANT_ID, From 426e3be30cfb90fad9f93c26b2b3c5619743f034 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:09:06 +0200 Subject: [PATCH 09/11] [main] Update dependencies from dotnet/arcade (#3640) Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 24 ++++++++++++------------ eng/Versions.props | 10 +++++----- global.json | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 065b1c6d73..8cf11b3165 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -91,29 +91,29 @@ - + https://github.com/dotnet/arcade - 9f6799fdc16ae19b3e9478c55b997a6aab839d09 + c214b6ad17aedca4fa48294d80f6c52ef2463081 - + https://github.com/dotnet/arcade - 9f6799fdc16ae19b3e9478c55b997a6aab839d09 + c214b6ad17aedca4fa48294d80f6c52ef2463081 - + https://github.com/dotnet/arcade - 9f6799fdc16ae19b3e9478c55b997a6aab839d09 + c214b6ad17aedca4fa48294d80f6c52ef2463081 - + https://github.com/dotnet/arcade - 9f6799fdc16ae19b3e9478c55b997a6aab839d09 + c214b6ad17aedca4fa48294d80f6c52ef2463081 - + https://github.com/dotnet/arcade - 9f6799fdc16ae19b3e9478c55b997a6aab839d09 + c214b6ad17aedca4fa48294d80f6c52ef2463081 - + https://github.com/dotnet/arcade - 9f6799fdc16ae19b3e9478c55b997a6aab839d09 + c214b6ad17aedca4fa48294d80f6c52ef2463081 https://github.com/dotnet/dnceng diff --git a/eng/Versions.props b/eng/Versions.props index 6c7e43b5f7..e84dd12bcf 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,11 +9,11 @@ true 1.0.0-preview.1 - 8.0.0-beta.24310.5 - 8.0.0-beta.24310.5 - 8.0.0-beta.24310.5 - 8.0.0-beta.24310.5 - 8.0.0-beta.24310.5 + 8.0.0-beta.24311.3 + 8.0.0-beta.24311.3 + 8.0.0-beta.24311.3 + 8.0.0-beta.24311.3 + 8.0.0-beta.24311.3 17.4.1 1.1.0-beta.24303.1 1.1.0-beta.24303.1 diff --git a/global.json b/global.json index b538d6396c..6200593810 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24310.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24311.3" } } From 52d0c7eca90c236dacb9165f551f791383b1aba9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:09:16 +0200 Subject: [PATCH 10/11] [main] Update dependencies from dotnet/dnceng (#3639) Co-authored-by: dotnet-maestro[bot] --- .config/dotnet-tools.json | 4 ++-- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 337c1a7167..e33fdbe6f6 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dnceng.secretmanager": { - "version": "1.1.0-beta.24280.1", + "version": "1.1.0-beta.24313.1", "commands": [ "secret-manager" ] @@ -15,7 +15,7 @@ ] }, "microsoft.dnceng.configuration.bootstrap": { - "version": "1.1.0-beta.24280.1", + "version": "1.1.0-beta.24313.1", "commands": [ "bootstrap-dnceng-configuration" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8cf11b3165..9669fbad76 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -115,13 +115,13 @@ https://github.com/dotnet/arcade c214b6ad17aedca4fa48294d80f6c52ef2463081 - + https://github.com/dotnet/dnceng - ca0a1eb1e36df8a36ec9393d5ef53f3f03dfbbfa + 52ec432649b0001426a6c13aea988bb8d74ebf91 - + https://github.com/dotnet/dnceng - ca0a1eb1e36df8a36ec9393d5ef53f3f03dfbbfa + 52ec432649b0001426a6c13aea988bb8d74ebf91 diff --git a/eng/Versions.props b/eng/Versions.props index e84dd12bcf..0716f99b1d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -37,8 +37,8 @@ 1.1.0-beta.24303.1 1.1.0-beta.24303.1 1.1.0-beta.24303.1 - 1.1.0-beta.24280.1 - 1.1.0-beta.24280.1 + 1.1.0-beta.24313.1 + 1.1.0-beta.24313.1 From 937147daa2585553fd1760d9c0e70c6fb3747352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Mon, 17 Jun 2024 16:11:18 +0200 Subject: [PATCH 11/11] Stop requiring rollout notes in PRs (#3641) --- .github/pull_request_template.md | 12 +---------- eng/enforce-issue.ps1 | 37 +------------------------------- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 76f828d737..13787fa314 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,11 +1 @@ - - - - - -### Release Note Category -- [ ] Feature changes/additions -- [ ] Bug fixes -- [ ] Internal Infrastructure Improvements - -### Release Note Description + diff --git a/eng/enforce-issue.ps1 b/eng/enforce-issue.ps1 index 3fb7edd7b1..c27e3d2481 100644 --- a/eng/enforce-issue.ps1 +++ b/eng/enforce-issue.ps1 @@ -6,10 +6,6 @@ param ( $RepositoryName ) -function HasReleaseNotes($description) { - return $description -Match "### Release Note Description(?:\r?\n)+([^\s]+)" -} - $prDetail = Invoke-WebRequest ` -UseBasicParsing ` -Uri "https://api.github.com/repos/$RepositoryName/pulls/$PullRequestNumber" ` @@ -39,35 +35,4 @@ if (-not $hasIssue) { } } -if (HasReleaseNotes $prDetail.body) { - Write-Host "PR has release notes in the description. Check passed." - exit 0 -} - - -try { - $issueDetail = Invoke-WebRequest ` - -UseBasicParsing ` - -Uri "https://api.github.com/repos/dotnet/$($matches[1])/issues/$($matches[2])" ` - | ConvertFrom-Json -} -catch { - Write-Host "##vso[task.LogIssue type=error;]Error fetching issue dotnet/$($matches[1])#$($matches[2]) from arcade. Does it exist?" - exit 1 -} - -if (HasReleaseNotes $issueDetail.body) { - Write-Host "PR links a GitHub issue with release notes. Check passed." - exit 0 -} - -$issueIsAzdoMirror = $issueDetail.title -like "AzDO Issue*" -if ($issueIsAzdoMirror) { - Write-Host "##vso[task.LogIssue type=warning;]Linked GitHub issue is a mirrored Azure DevOps workitem. Please ensure the workitem has release notes in it." - exit 0 -} - -Write-Host "##vso[task.LogIssue type=error;]Linked GitHub issue does not have release notes. Check failed." -Write-Host "Ensure your issue has release notes. They should be in the following format:`n`n### Release Note Description`n`n" -Write-Host "For more information, see https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/983/ReleaseNotesGuidance?anchor=mechanics" -exit 1 +exit 0