diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index e33fdbe6f6..5aa9bf38b7 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dnceng.secretmanager": { - "version": "1.1.0-beta.24313.1", + "version": "1.1.0-beta.24321.4", "commands": [ "secret-manager" ] @@ -15,7 +15,7 @@ ] }, "microsoft.dnceng.configuration.bootstrap": { - "version": "1.1.0-beta.24313.1", + "version": "1.1.0-beta.24321.4", "commands": [ "bootstrap-dnceng-configuration" ] diff --git a/.github/ISSUE_TEMPLATE/issue-with-release-notes.md b/.github/ISSUE_TEMPLATE/issue-with-release-notes.md deleted file mode 100644 index 19cf763c99..0000000000 --- a/.github/ISSUE_TEMPLATE/issue-with-release-notes.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: Issue with release notes -about: Create issue containing the release notes section ---- - - - -### Release Note Category -- [ ] Feature changes/additions -- [ ] Bug fixes -- [ ] Internal Infrastructure Improvements - -### Release Note Description diff --git a/.github/ISSUE_TEMPLATE/rollout-issue.md b/.github/ISSUE_TEMPLATE/rollout-issue.md index e97d8f7c39..3f8de9edc2 100644 --- a/.github/ISSUE_TEMPLATE/rollout-issue.md +++ b/.github/ISSUE_TEMPLATE/rollout-issue.md @@ -13,33 +13,32 @@ This issue tracks the `arcade-services` repository rollout. On top of the [Rollo # Process -## Build status check (Monday) +## Build status check - [ ] Check the status of the [dotnet-arcade-services-weekly](https://dev.azure.com/dnceng/internal/_build?definitionId=993) pipeline - [ ] Rotate any secrets that need manual rotation - [ ] Check the status of the [arcade-services-internal-ci](https://dev.azure.com/dnceng/internal/_build?definitionId=252) pipeline. Try to fix issues, if any, so that we have a green build before the rollout day. - [ ] Check the `Rollout` column in the [Product Construction](https://github.com/orgs/dotnet/projects/276) board - move any issues rolled-out last week into `Done` -## Rollout preparation (Tuesday) -- [ ] Check that the vendor prepared the rollout: - - Thread on the [Rollout channel](https://teams.microsoft.com/l/channel/19%3a72e283b51f9e4567ba24a35328562df4%40thread.skype/Rollout?groupId=147df318-61de-4f04-8f7b-ecd328c256bb&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47) - - Rollout issue in [AzDO](https://dev.azure.com/dnceng/internal/_workitems/) - - Rollout PR in `arcade-services` -- [ ] In case there is a problem with the CI build, notify the [Rollout channel](https://teams.microsoft.com/l/channel/19%3a72e283b51f9e4567ba24a35328562df4%40thread.skype/Rollout?groupId=147df318-61de-4f04-8f7b-ecd328c256bb&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47) +## Rollout preparation +- [ ] Create the rollout PR: + - Find a commit on `main` that you want to rollout + - Create a branch named `rollout/YYYY-MM-DD` from that commit + - Create a PR on GitHub from the `rollout/YYYY-MM-DD` branch to `production` + - Name the PR `[Rollout] Production rollout YYYY-MM-DD` + - Link this issue in the PR description - [ ] Link the rollout PR to the [Rollout PRs](#rollout-prs) section of this issue -- [ ] Double-check that the release notes contain all information -- [ ] Merge the already prepared rollout PR (⚠️ **DO NOT SQUASH**) +- [ ] Merge the prepared rollout PR (⚠️ **DO NOT SQUASH**) - [ ] Link the rollout build to the [Rollout build](#rollout-build) section of this issue -- [ ] Verify that Maestro opened a production => main PR in `arcade-services` with the rollout merge commit ([example](https://github.com/dotnet/arcade-services/pull/2741)). There should be no changes in the PR to any files. **Do not merge the PR yet**. +- [ ] Verify that Maestro opened a `production => main` PR in `arcade-services` with the rollout merge commit ([example](https://github.com/dotnet/arcade-services/pull/2741)). There should be no changes in the PR to any files. **Do not merge the PR yet**. - [ ] Ensure the build is green and stops at the `Approval` phase -## Rollout day (Wednesday) -- [ ] Approve the `Approval` stage of the rollout build (that has been already started the day before) +## Rollout +- [ ] Approve the `Approval` stage of the rollout build. - [ ] Monitor the rollout build for failures. - Note: this [Maestro exceptions query](https://ms.portal.azure.com/#view/Microsoft_OperationsManagementSuite_Workspace/Logs.ReactView/resourceId/%2Fsubscriptions%2F68672ab8-de0c-40f1-8d1b-ffb20bd62c0f%2FresourceGroups%2Fmaestro-prod-cluster%2Fproviders%2Fmicrosoft.insights%2Fcomponents%2Fmaestro-prod/source/LogsBlade.AnalyticsShareLinkToQuery/q/H4sIAAAAAAAAAz2MOw6DMBBE%252B5xiSlsiRZDS5i7GjGQXu0brRSSIwyekoH4fvjMXr0377cBWaIRXYfckC17QtoV4H%252Bcf7KtIsroTua3qIWL6YKoaLn%252FA4ylxgNBLOxOjzrT%252FMJdk%252FgV08ryabQAAAA%253D%253D) might help in diagnosing issues. - [ ] Keep track of any issues encountered during the rollout either directly in this issue, or in a dedicated issue linked to this issue -- [ ] Update the rollout stats in the [Stats](#stats) section below. The statistics will be available in Kusto a few minutes after the build was finished -- [ ] Notify the [Rollout channel](https://teams.microsoft.com/l/channel/19%3a72e283b51f9e4567ba24a35328562df4%40thread.skype/Rollout?groupId=147df318-61de-4f04-8f7b-ecd328c256bb&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47) -- [ ] Merge the production => main Maestro PR in `arcade-services` (⚠️ **DO NOT SQUASH**) +- [ ] When finished, update the rollout stats in the [Stats](#stats) section below. The statistics will be available in Kusto a few minutes after the build was finished +- [ ] Merge the `production => main` PR in `arcade-services` (⚠️ **DO NOT SQUASH**) - [ ] Move rolled-out issues in the `Rollout` column of the [Product Construction](https://github.com/orgs/dotnet/projects/276) board into `Done`. Add a link in to this rollout issue in the comments before closing them ([example](https://github.com/dotnet/arcade-services/issues/2681#issuecomment-1632288755)) - [ ] Close this issue with closing comment describing a high-level summary of issues encountered during the rollout - In case of rollback, uncomment the *Rollback* section below and follow the steps there @@ -64,11 +63,11 @@ In case the services don't work as expected after the rollout, it's necessary to ## Rollout PRs -* The main PR: `` +* The main PR: ## Rollout build -* Rollout AzDO build: `` +* Rollout AzDO build: ## Rollout times diff --git a/arcade-services.sln b/arcade-services.sln index 00ad56aa59..916acf4af6 100644 --- a/arcade-services.sln +++ b/arcade-services.sln @@ -21,8 +21,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Maestro.MergePolicies", "sr EndProject Project("{A07B5EB6-E848-4116-A8D0-A826331D98C6}") = "MaestroApplication", "src\Maestro\MaestroApplication\MaestroApplication.sfproj", "{93F066A5-A2D8-4926-A255-81077AEE5972}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Maestro.AzureDevOps", "src\Maestro\Maestro.AzureDevOps\Maestro.AzureDevOps.csproj", "{423EDB52-F832-4AE9-B85F-7F427056940A}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{C9012867-A5A4-464F-A104-99A8D6C97DAF}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig @@ -118,7 +116,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Maestro.Authentication", "s EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProductConstructionService.Client", "src\ProductConstructionService\ProductConstructionService.Client\ProductConstructionService.Client.csproj", "{964FA796-358E-48AE-B75C-E42132600BCC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Maestro.Common", "src\Maestro\Maestro.Common\Maestro.Common.csproj", "{16F086DD-8387-44BB-87D5-CD804355A110}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Maestro.Common", "src\Maestro\Maestro.Common\Maestro.Common.csproj", "{16F086DD-8387-44BB-87D5-CD804355A110}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -226,18 +224,6 @@ Global {93F066A5-A2D8-4926-A255-81077AEE5972}.Release|x64.Build.0 = Release|x64 {93F066A5-A2D8-4926-A255-81077AEE5972}.Release|x64.Deploy.0 = Release|x64 {93F066A5-A2D8-4926-A255-81077AEE5972}.Release|x86.ActiveCfg = Release|x64 - {423EDB52-F832-4AE9-B85F-7F427056940A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Debug|x64.ActiveCfg = Debug|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Debug|x64.Build.0 = Debug|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Debug|x86.ActiveCfg = Debug|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Debug|x86.Build.0 = Debug|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Release|Any CPU.Build.0 = Release|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Release|x64.ActiveCfg = Release|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Release|x64.Build.0 = Release|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Release|x86.ActiveCfg = Release|Any CPU - {423EDB52-F832-4AE9-B85F-7F427056940A}.Release|x86.Build.0 = Release|Any CPU {7D7E290E-7A94-4F7B-AA05-27156C4A3CAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7D7E290E-7A94-4F7B-AA05-27156C4A3CAC}.Debug|Any CPU.Build.0 = Debug|Any CPU {7D7E290E-7A94-4F7B-AA05-27156C4A3CAC}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -552,7 +538,6 @@ Global {EDF632B3-48E2-43FD-B014-79CDED8F4240} = {AE791E26-78E1-4936-BCF4-1BF5152CBBD6} {37D70EEA-9621-44EB-921A-5D303917F851} = {AE791E26-78E1-4936-BCF4-1BF5152CBBD6} {93F066A5-A2D8-4926-A255-81077AEE5972} = {AE791E26-78E1-4936-BCF4-1BF5152CBBD6} - {423EDB52-F832-4AE9-B85F-7F427056940A} = {AE791E26-78E1-4936-BCF4-1BF5152CBBD6} {939DD755-B5FC-46EF-AE8B-5073DA6D4490} = {F92E1B5C-26D4-4244-A4F9-41E22BA8E1CE} {7D7E290E-7A94-4F7B-AA05-27156C4A3CAC} = {AE791E26-78E1-4936-BCF4-1BF5152CBBD6} {C7E8C999-F8AE-427D-B748-6BCF7202B476} = {AE791E26-78E1-4936-BCF4-1BF5152CBBD6} diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e913c4943c..f17b8329a5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -91,37 +91,37 @@ - + https://github.com/dotnet/arcade - c214b6ad17aedca4fa48294d80f6c52ef2463081 + 748cd976bf8b0f69b809e569943635ab8be36dc8 - + https://github.com/dotnet/arcade - c214b6ad17aedca4fa48294d80f6c52ef2463081 + 748cd976bf8b0f69b809e569943635ab8be36dc8 - + https://github.com/dotnet/arcade - c214b6ad17aedca4fa48294d80f6c52ef2463081 + 748cd976bf8b0f69b809e569943635ab8be36dc8 - + https://github.com/dotnet/arcade - c214b6ad17aedca4fa48294d80f6c52ef2463081 + 748cd976bf8b0f69b809e569943635ab8be36dc8 - + https://github.com/dotnet/arcade - c214b6ad17aedca4fa48294d80f6c52ef2463081 + 748cd976bf8b0f69b809e569943635ab8be36dc8 - + https://github.com/dotnet/arcade - c214b6ad17aedca4fa48294d80f6c52ef2463081 + 748cd976bf8b0f69b809e569943635ab8be36dc8 - + https://github.com/dotnet/dnceng - 52ec432649b0001426a6c13aea988bb8d74ebf91 + 44c25f86ba374d93ba9c22f12552deeed2221588 - + https://github.com/dotnet/dnceng - 52ec432649b0001426a6c13aea988bb8d74ebf91 + 44c25f86ba374d93ba9c22f12552deeed2221588 diff --git a/eng/Versions.props b/eng/Versions.props index 019ce58d75..1e39a902b6 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,11 +9,11 @@ true 1.0.0-preview.1 - 8.0.0-beta.24311.3 - 8.0.0-beta.24311.3 - 8.0.0-beta.24311.3 - 8.0.0-beta.24311.3 - 8.0.0-beta.24311.3 + 8.0.0-beta.24324.1 + 8.0.0-beta.24324.1 + 8.0.0-beta.24324.1 + 8.0.0-beta.24324.1 + 8.0.0-beta.24324.1 17.4.1 1.1.0-beta.24319.1 1.1.0-beta.24319.1 @@ -37,8 +37,8 @@ 1.1.0-beta.24319.1 1.1.0-beta.24319.1 1.1.0-beta.24319.1 - 1.1.0-beta.24313.1 - 1.1.0-beta.24313.1 + 1.1.0-beta.24321.4 + 1.1.0-beta.24321.4 diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f193dfbe23..f983033bb0 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -62,6 +68,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - template: /eng/common/templates-official/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 08e5db9bb1..5cf6a269c0 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates-official/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 0000000000..93a8394a66 --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 0000000000..c0e8f91317 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 0000000000..e3786cef6d --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 8a3deef2b7..c0ff472b69 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -61,6 +67,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - template: /eng/common/templates/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index a15b07eb51..5f46bfa895 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 0000000000..54dc9416c5 --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 0000000000..c0e8f91317 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 0000000000..c8c49cc0e8 --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index 31500bbc24..cc253f8c3b 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -135,9 +135,7 @@ stages: variables: - group: ${{ parameters.VariableGroup }} - # Secret-Manager-Scenario-Tests provides: secret-manager-scenario-tests-client-secret - - group: Secret-Manager-Scenario-Tests - + jobs: - job: scenario displayName: Scenario tests diff --git a/github-merge-flow.jsonc b/github-merge-flow.jsonc new file mode 100644 index 0000000000..ce590880b8 --- /dev/null +++ b/github-merge-flow.jsonc @@ -0,0 +1,8 @@ +// IMPORTANT: This file is read by the merge flow from main branch only. +{ + "merge-flow-configurations": { + "production":{ + "MergeToBranch": "main" + } + } +} \ No newline at end of file diff --git a/global.json b/global.json index 6200593810..8136e023fa 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24311.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24324.1" } } diff --git a/src/Maestro/Client/src/MaestroApiOptions.cs b/src/Maestro/Client/src/MaestroApiOptions.cs index ed7ef2bc88..9b98163bd2 100644 --- a/src/Maestro/Client/src/MaestroApiOptions.cs +++ b/src/Maestro/Client/src/MaestroApiOptions.cs @@ -5,8 +5,7 @@ using System.Collections.Generic; using Azure.Core; using Azure.Core.Pipeline; -using Microsoft.DotNet.Maestro.Common; - +using Maestro.Common.AppCredentials; namespace Microsoft.DotNet.Maestro.Client { @@ -46,12 +45,14 @@ public MaestroApiOptions(string baseUri, string accessToken, string managedIdent : this( new Uri(baseUri), AppCredentialResolver.CreateCredential( - EntraAppIds[(baseUri ?? ProductionBuildAssetRegistryBaseUri).TrimEnd('/')], - disableInteractiveAuth, - accessToken, - federatedToken, - managedIdentityId, - APP_USER_SCOPE)) + new AppCredentialResolverOptions(EntraAppIds[(baseUri ?? ProductionBuildAssetRegistryBaseUri).TrimEnd('/')]) + { + DisableInteractiveAuth = disableInteractiveAuth, + Token = accessToken, + FederatedToken = federatedToken, + ManagedIdentityId = managedIdentityId, + UserScope = APP_USER_SCOPE, + })) { } diff --git a/src/Maestro/DependencyUpdateErrorProcessor/DependencyUpdateErrorProcessor.csproj b/src/Maestro/DependencyUpdateErrorProcessor/DependencyUpdateErrorProcessor.csproj index 80ad77193b..52829b08b5 100644 --- a/src/Maestro/DependencyUpdateErrorProcessor/DependencyUpdateErrorProcessor.csproj +++ b/src/Maestro/DependencyUpdateErrorProcessor/DependencyUpdateErrorProcessor.csproj @@ -29,7 +29,7 @@ - + diff --git a/src/Maestro/DependencyUpdater/.config/settings.Development.json b/src/Maestro/DependencyUpdater/.config/settings.Development.json index 3a5dba9dc2..887d9725c1 100644 --- a/src/Maestro/DependencyUpdater/.config/settings.Development.json +++ b/src/Maestro/DependencyUpdater/.config/settings.Development.json @@ -1,15 +1,22 @@ { - "HealthReportSettings": { - "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", - "TableName": "healthreport" - }, - "KeyVaultUri": "https://maestrolocal.vault.azure.net/", - "BuildAssetRegistry": { - "ConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true" - }, - "Kusto": { - "Database": "engineeringdata", - "KustoClusterUri": "https://engdata.westus2.kusto.windows.net", - "UseAzCliAuthentication": true - } -} + "HealthReportSettings": { + "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", + "TableName": "healthreport" + }, + "KeyVaultUri": "https://maestrolocal.vault.azure.net/", + "BuildAssetRegistry": { + "ConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true" + }, + "Kusto": { + "Database": "engineeringdata", + "KustoClusterUri": "https://engdata.westus2.kusto.windows.net", + "UseAzCliAuthentication": true + }, + "AzureDevOps": { + "Tokens": { + "dnceng": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]", + "devdiv": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]", + "domoreexp": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" + } + } +} \ No newline at end of file diff --git a/src/Maestro/DependencyUpdater/.config/settings.Production.json b/src/Maestro/DependencyUpdater/.config/settings.Production.json index dcaecad4c8..7b67bbc51c 100644 --- a/src/Maestro/DependencyUpdater/.config/settings.Production.json +++ b/src/Maestro/DependencyUpdater/.config/settings.Production.json @@ -1,14 +1,14 @@ { - "HealthReportSettings": { - "StorageAccountTablesUri": "https://maestroprod1337.table.core.windows.net", - "TableName": "healthreport" - }, - "KeyVaultUri": "https://maestroprod.vault.azure.net/", - "BuildAssetRegistry": { - "ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" - }, - "Kusto": { - "Database": "engineeringdata", - "KustoClusterUri": "https://engsrvprod.westus.kusto.windows.net" - } + "HealthReportSettings": { + "StorageAccountTablesUri": "https://maestroprod1337.table.core.windows.net", + "TableName": "healthreport" + }, + "KeyVaultUri": "https://maestroprod.vault.azure.net/", + "BuildAssetRegistry": { + "ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" + }, + "Kusto": { + "Database": "engineeringdata", + "KustoClusterUri": "https://engsrvprod.westus.kusto.windows.net" + } } \ No newline at end of file diff --git a/src/Maestro/DependencyUpdater/.config/settings.Staging.json b/src/Maestro/DependencyUpdater/.config/settings.Staging.json index 4fccff13d6..270efe35a7 100644 --- a/src/Maestro/DependencyUpdater/.config/settings.Staging.json +++ b/src/Maestro/DependencyUpdater/.config/settings.Staging.json @@ -1,14 +1,14 @@ { - "HealthReportSettings": { - "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", - "TableName": "healthreport" - }, - "KeyVaultUri": "https://maestroint.vault.azure.net/", - "BuildAssetRegistry": { - "ConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" - }, - "Kusto": { - "Database": "engineeringdata", - "KustoClusterUri": "https://engdata.westus2.kusto.windows.net" - } -} \ No newline at end of file + "HealthReportSettings": { + "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", + "TableName": "healthreport" + }, + "KeyVaultUri": "https://maestroint.vault.azure.net/", + "BuildAssetRegistry": { + "ConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" + }, + "Kusto": { + "Database": "engineeringdata", + "KustoClusterUri": "https://engdata.westus2.kusto.windows.net" + } +} diff --git a/src/Maestro/DependencyUpdater/.config/settings.json b/src/Maestro/DependencyUpdater/.config/settings.json index 8c30caa817..4b7dd5d419 100644 --- a/src/Maestro/DependencyUpdater/.config/settings.json +++ b/src/Maestro/DependencyUpdater/.config/settings.json @@ -1,22 +1,11 @@ { - "GitHub": { - "GitHubAppId": "[vault(github-app-id)]", - "PrivateKey": "[vault(github-app-private-key)]" - }, - "AzureDevOps": { - "Tokens": [ - { - "Account": "dnceng", - "Token": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]" - }, - { - "Account": "devdiv", - "Token": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]" - }, - { - "Account": "domoreexp", - "Token": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" - } - ] - } + "GitHub": { + "GitHubAppId": "[vault(github-app-id)]", + "PrivateKey": "[vault(github-app-private-key)]" + }, + "AzureDevOps": { + "ManagedIdentities": { + "default": "system" + } + } } diff --git a/src/Maestro/DependencyUpdater/DependencyUpdater.csproj b/src/Maestro/DependencyUpdater/DependencyUpdater.csproj index 72d040bd99..bb7d1e2652 100644 --- a/src/Maestro/DependencyUpdater/DependencyUpdater.csproj +++ b/src/Maestro/DependencyUpdater/DependencyUpdater.csproj @@ -29,6 +29,7 @@ + diff --git a/src/Maestro/DependencyUpdater/Program.cs b/src/Maestro/DependencyUpdater/Program.cs index 54725f0223..e26e7956d4 100644 --- a/src/Maestro/DependencyUpdater/Program.cs +++ b/src/Maestro/DependencyUpdater/Program.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Reflection; -using Maestro.AzureDevOps; +using Maestro.Common.AzureDevOpsTokens; using Maestro.Data; using Maestro.DataProviders; using Microsoft.DncEng.Configuration.Extensions; @@ -46,21 +46,11 @@ public static void Configure(IServiceCollection services) .GetCustomAttribute() ?.InformationalVersion); }); - services.Configure("GitHub", (o, s) => - { - s.Bind(o); - }); + services.Configure("GitHub", (o, s) => s.Bind(o)); services.AddGitHubTokenProvider(); + services.Configure("AzureDevOps", (o, s) => s.Bind(o)); services.AddAzureDevOpsTokenProvider(); - services.Configure("AzureDevOps:Tokens", (o, s) => - { - var tokenMap = s.GetChildren(); - foreach (IConfigurationSection token in tokenMap) - { - o.Tokens.Add(token.GetValue("Account"), token.GetValue("Token")); - } - }); // We do not use AddMemoryCache here. We use our own cache because we wish to // use a sized cache and some components, such as EFCore, do not implement their caching diff --git a/src/Maestro/FeedCleanerService/.config/settings.Development.json b/src/Maestro/FeedCleanerService/.config/settings.Development.json index 2db879a62e..4aa6c7834f 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.Development.json +++ b/src/Maestro/FeedCleanerService/.config/settings.Development.json @@ -1,10 +1,15 @@ { - "HealthReportSettings": { - "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", - "TableName": "healthreport" - }, - "KeyVaultUri": "https://maestrolocal.vault.azure.net/", - "BuildAssetRegistry": { - "ConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true" - } -} + "HealthReportSettings": { + "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", + "TableName": "healthreport" + }, + "KeyVaultUri": "https://maestrolocal.vault.azure.net/", + "BuildAssetRegistry": { + "ConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true" + }, + "AzureDevOps": { + "Tokens": { + "dnceng": "[vault(dn-bot-dnceng-packaging-rwm)]" + } + } +} \ No newline at end of file diff --git a/src/Maestro/FeedCleanerService/.config/settings.Production.json b/src/Maestro/FeedCleanerService/.config/settings.Production.json index 94c404d56c..68ae60b3d1 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.Production.json +++ b/src/Maestro/FeedCleanerService/.config/settings.Production.json @@ -1,13 +1,13 @@ { - "HealthReportSettings": { - "StorageAccountTablesUri": "https://maestroprod1337.table.core.windows.net", - "TableName": "healthreport" - }, - "KeyVaultUri": "https://maestroprod.vault.azure.net/", - "FeedCleaner": { - "Enabled": true - }, - "BuildAssetRegistry": { - "ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" - } -} + "HealthReportSettings": { + "StorageAccountTablesUri": "https://maestroprod1337.table.core.windows.net", + "TableName": "healthreport" + }, + "KeyVaultUri": "https://maestroprod.vault.azure.net/", + "FeedCleaner": { + "Enabled": true + }, + "BuildAssetRegistry": { + "ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" + } +} \ No newline at end of file diff --git a/src/Maestro/FeedCleanerService/.config/settings.Staging.json b/src/Maestro/FeedCleanerService/.config/settings.Staging.json index 73be35ea1f..f322dd3dff 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.Staging.json +++ b/src/Maestro/FeedCleanerService/.config/settings.Staging.json @@ -1,10 +1,10 @@ { - "HealthReportSettings": { - "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", - "TableName": "healthreport" - }, - "KeyVaultUri": "https://maestroint.vault.azure.net/", - "BuildAssetRegistry": { - "ConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" - } + "HealthReportSettings": { + "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", + "TableName": "healthreport" + }, + "KeyVaultUri": "https://maestroint.vault.azure.net/", + "BuildAssetRegistry": { + "ConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" + } } diff --git a/src/Maestro/FeedCleanerService/.config/settings.json b/src/Maestro/FeedCleanerService/.config/settings.json index 4cc756961c..b87d8cc284 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.json +++ b/src/Maestro/FeedCleanerService/.config/settings.json @@ -1,35 +1,32 @@ { - "FeedCleaner": { - "Enabled": false, - "ReleasePackageFeeds": [ - { - "Account": "dnceng", - "Project": "public", - "Name": "dotnet3" - }, - { - "Account": "dnceng", - "Project": "public", - "Name": "dotnet3.1" - }, - { - "Account": "dnceng", - "Project": "public", - "Name": "dotnet5" - }, - { - "Account": "dnceng", - "Project": "public", - "Name": "dotnet-tools" - } - ] - }, - "AzureDevOps": { - "Tokens": [ - { - "Account": "dnceng", - "Token": "[vault(dn-bot-dnceng-packaging-rwm)]" - } - ] - } -} + "FeedCleaner": { + "Enabled": false, + "ReleasePackageFeeds": [ + { + "Account": "dnceng", + "Project": "public", + "Name": "dotnet3" + }, + { + "Account": "dnceng", + "Project": "public", + "Name": "dotnet3.1" + }, + { + "Account": "dnceng", + "Project": "public", + "Name": "dotnet5" + }, + { + "Account": "dnceng", + "Project": "public", + "Name": "dotnet-tools" + } + ] + }, + "AzureDevOps": { + "ManagedIdentities": { + "default": "system" + } + } +} \ No newline at end of file diff --git a/src/Maestro/FeedCleanerService/FeedCleanerService.cs b/src/Maestro/FeedCleanerService/FeedCleanerService.cs index ae8ebe4d84..5496a0c768 100644 --- a/src/Maestro/FeedCleanerService/FeedCleanerService.cs +++ b/src/Maestro/FeedCleanerService/FeedCleanerService.cs @@ -9,7 +9,7 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Maestro.AzureDevOps; +using Maestro.Common.AzureDevOpsTokens; using Maestro.Contracts; using Maestro.Data; using Maestro.Data.Models; diff --git a/src/Maestro/FeedCleanerService/FeedCleanerService.csproj b/src/Maestro/FeedCleanerService/FeedCleanerService.csproj index 645f040763..f8571474f6 100644 --- a/src/Maestro/FeedCleanerService/FeedCleanerService.csproj +++ b/src/Maestro/FeedCleanerService/FeedCleanerService.csproj @@ -27,7 +27,7 @@ - + diff --git a/src/Maestro/FeedCleanerService/Program.cs b/src/Maestro/FeedCleanerService/Program.cs index 9527a197a4..22032ade7a 100644 --- a/src/Maestro/FeedCleanerService/Program.cs +++ b/src/Maestro/FeedCleanerService/Program.cs @@ -1,7 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Maestro.AzureDevOps; +using System.Collections.Generic; +using System.Linq; +using Maestro.Common.AzureDevOpsTokens; using Maestro.Data; using Microsoft.DncEng.Configuration.Extensions; using Microsoft.DotNet.ServiceFabric.ServiceHost; @@ -29,19 +31,20 @@ public static void Configure(IServiceCollection services) { services.Configure((options, provider) => { - var config1 = provider.GetRequiredService(); - options.Enabled = config1.GetSection("FeedCleaner").GetValue("Enabled"); - var releaseFeedsTokenMap = config1.GetSection("FeedCleaner:ReleasePackageFeeds").GetChildren(); + var config = provider.GetRequiredService(); + options.Enabled = config.GetSection("FeedCleaner").GetValue("Enabled"); + var releaseFeedsTokenMap = config.GetSection("FeedCleaner:ReleasePackageFeeds").GetChildren(); foreach (IConfigurationSection token1 in releaseFeedsTokenMap) { options.ReleasePackageFeeds.Add((token1.GetValue("Account"), token1.GetValue("Project"), token1.GetValue("Name"))); } - var azdoAccountTokenMap = config1.GetSection("AzureDevOps:Tokens").GetChildren(); - foreach (IConfigurationSection token2 in azdoAccountTokenMap) - { - options.AzdoAccounts.Add(token2.GetValue("Account")); - } + AzureDevOpsTokenProviderOptions azdoConfig = new(); + config.GetSection("AzureDevOps").Bind(azdoConfig); + IEnumerable allOrgs = azdoConfig.Tokens.Keys + .Concat(azdoConfig.ManagedIdentities.Keys) + .Distinct(); + options.AzdoAccounts.AddRange(allOrgs); }); services.AddDefaultJsonConfiguration(); services.AddBuildAssetRegistry((provider, options) => @@ -50,14 +53,6 @@ public static void Configure(IServiceCollection services) options.UseSqlServerWithRetry(config.GetSection("BuildAssetRegistry")["ConnectionString"]); }); services.AddAzureDevOpsTokenProvider(); - services.Configure((options, provider) => - { - var config = provider.GetRequiredService(); - var tokenMap = config.GetSection("AzureDevOps:Tokens").GetChildren(); - foreach (IConfigurationSection token in tokenMap) - { - options.Tokens.Add(token.GetValue("Account"), token.GetValue("Token")); - } - }); + services.Configure("AzureDevOps", (o, s) => s.Bind(o)); } } diff --git a/src/Maestro/Maestro.AzureDevOps/AzureDevOpsTokenProvider.cs b/src/Maestro/Maestro.AzureDevOps/AzureDevOpsTokenProvider.cs deleted file mode 100644 index 38728647e9..0000000000 --- a/src/Maestro/Maestro.AzureDevOps/AzureDevOpsTokenProvider.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Extensions.Options; -using System; -using System.Threading.Tasks; - -namespace Maestro.AzureDevOps; - -public class AzureDevOpsTokenProvider : IAzureDevOpsTokenProvider -{ - private readonly IOptionsMonitor _options; - - public AzureDevOpsTokenProvider(IOptionsMonitor options) - { - _options = options; - } - - public Task GetTokenForAccount(string account) - { - var options = _options.CurrentValue; - if (!options.Tokens.TryGetValue(account, out var pat) || string.IsNullOrEmpty(pat)) - { - throw new ArgumentOutOfRangeException($"Azure DevOps account {account} does not have a configured PAT. " + - $"Please ensure the 'Tokens' array in the 'AzureDevOps' section of settings.json contains a PAT for {account}"); - } - - return Task.FromResult(pat); - } -} diff --git a/src/Maestro/Maestro.AzureDevOps/Maestro.AzureDevOps.csproj b/src/Maestro/Maestro.AzureDevOps/Maestro.AzureDevOps.csproj deleted file mode 100644 index e667e56052..0000000000 --- a/src/Maestro/Maestro.AzureDevOps/Maestro.AzureDevOps.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - - false - - - - - - - - diff --git a/src/Maestro/Maestro.Common/AppCredentialResolver.cs b/src/Maestro/Maestro.Common/AppCredentialResolver.cs deleted file mode 100644 index 62b92a1202..0000000000 --- a/src/Maestro/Maestro.Common/AppCredentialResolver.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Azure.Core; - -namespace Microsoft.DotNet.Maestro.Common; - -public static class AppCredentialResolver -{ - /// - /// Creates a credential based on parameters provided. - /// - /// Client ID of the Azure application to request the token for - /// Whether to include interactive login flows - /// Token to use directly instead of authenticating. - /// Federated token to use for fetching the token. If none supplied, will try other flows. - /// Managed Identity to use for the auth - /// Credential that can be used to call the Maestro API - public static TokenCredential CreateCredential( - string appId, - bool disableInteractiveAuth, - string? token = null, - string? federatedToken = null, - string? managedIdentityId = null, - string userScope = ".default") - { - // 1. BAR or Entra token that can directly be used to authenticate against a service - if (!string.IsNullOrEmpty(token)) - { - return new ResolvedCredential(token!); - } - - // 2. Federated token that can be used to fetch an app token (for CI scenarios) - if (!string.IsNullOrEmpty(federatedToken)) - { - return AppCredential.CreateFederatedCredential(appId, federatedToken!); - } - - // 3. Managed identity (for server-to-server scenarios - e.g. PCS->Maestro) - if (!string.IsNullOrEmpty(managedIdentityId)) - { - return AppCredential.CreateManagedIdentityCredential(appId, managedIdentityId!); - } - - // 4. Azure CLI authentication setup by the caller (for CI scenarios) - if (disableInteractiveAuth) - { - return AppCredential.CreateNonUserCredential(appId); - } - - // 5. Interactive login (user-based scenario) - return AppCredential.CreateUserCredential(appId, userScope); - } - -} diff --git a/src/Maestro/Maestro.Common/AppCredential.cs b/src/Maestro/Maestro.Common/AppCredentials/AppCredential.cs similarity index 96% rename from src/Maestro/Maestro.Common/AppCredential.cs rename to src/Maestro/Maestro.Common/AppCredentials/AppCredential.cs index eb2146533e..ad9732d8b5 100644 --- a/src/Maestro/Maestro.Common/AppCredential.cs +++ b/src/Maestro/Maestro.Common/AppCredentials/AppCredential.cs @@ -4,7 +4,7 @@ using Azure.Core; using Azure.Identity; -namespace Microsoft.DotNet.Maestro.Common; +namespace Maestro.Common.AppCredentials; /// /// A credential for authenticating against Azure applications. @@ -45,7 +45,7 @@ public static AppCredential CreateUserCredential(string appId, string userScope { var requestContext = new TokenRequestContext(new string[] { $"api://{appId}/{userScope}" }); - string authRecordPath = Path.Combine(AUTH_CACHE, $"{AUTH_RECORD_PREFIX}-{appId}"); + var authRecordPath = Path.Combine(AUTH_CACHE, $"{AUTH_RECORD_PREFIX}-{appId}"); var credential = GetInteractiveCredential(appId, requestContext, authRecordPath); return new AppCredential(credential, requestContext); @@ -67,14 +67,14 @@ private static InteractiveBrowserCredential GetInteractiveCredential( ClientId = appId, RedirectUri = new Uri("http://localhost"), // These options describe credential caching only during runtime - TokenCachePersistenceOptions = new TokenCachePersistenceOptions() + TokenCachePersistenceOptions = new TokenCachePersistenceOptions() { Name = "maestro" }, }; - string authRecordDir = Path.GetDirectoryName(authRecordPath) ?? + var authRecordDir = Path.GetDirectoryName(authRecordPath) ?? throw new ArgumentException($"Cannot resolve cache dir from auth record: {authRecordPath}"); if (!Directory.Exists(authRecordDir)) diff --git a/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolver.cs b/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolver.cs new file mode 100644 index 0000000000..6feadf090b --- /dev/null +++ b/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolver.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Azure.Core; + +namespace Maestro.Common.AppCredentials; + +public static class AppCredentialResolver +{ + /// + /// Creates a credential based on parameters provided. + /// + public static TokenCredential CreateCredential(AppCredentialResolverOptions options) + { + // 1. BAR or Entra token that can directly be used to authenticate against a service + if (!string.IsNullOrEmpty(options.Token)) + { + return new ResolvedCredential(options.Token!); + } + + // 2. Federated token that can be used to fetch an app token (for CI scenarios) + if (!string.IsNullOrEmpty(options.FederatedToken)) + { + return AppCredential.CreateFederatedCredential(options.AppId, options.FederatedToken!); + } + + // 3. Managed identity (for server-to-server scenarios - e.g. PCS->Maestro) + if (!string.IsNullOrEmpty(options.ManagedIdentityId)) + { + return AppCredential.CreateManagedIdentityCredential(options.AppId, options.ManagedIdentityId!); + } + + // 4. Azure CLI authentication setup by the caller (for CI scenarios) + if (options.DisableInteractiveAuth) + { + return AppCredential.CreateNonUserCredential(options.AppId); + } + + // 5. Interactive login (user-based scenario) + return AppCredential.CreateUserCredential(options.AppId, options.UserScope); + } +} diff --git a/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolverOptions.cs b/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolverOptions.cs new file mode 100644 index 0000000000..dac9841887 --- /dev/null +++ b/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolverOptions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Maestro.Common.AppCredentials; + +public class AppCredentialResolverOptions : CredentialResolverOptions +{ + /// + /// Client ID of the Azure application to request the token for + /// + public string AppId { get; set; } + + /// + /// User scope to request the token for (in case of user flows). + /// + public string UserScope { get; set; } = ".default"; + + public AppCredentialResolverOptions(string appId) + { + AppId = appId; + } +} diff --git a/src/Maestro/Maestro.Common/AppCredentials/CredentialResolverOptions.cs b/src/Maestro/Maestro.Common/AppCredentials/CredentialResolverOptions.cs new file mode 100644 index 0000000000..2034bece49 --- /dev/null +++ b/src/Maestro/Maestro.Common/AppCredentials/CredentialResolverOptions.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Maestro.Common.AppCredentials; + +public class CredentialResolverOptions +{ + /// + /// Whether to include interactive login flows + /// + public bool DisableInteractiveAuth { get; set; } + + /// + /// Token to use directly instead of authenticating. + /// + public string? Token { get; set; } + + /// + /// Federated token to use for fetching the token. If none supplied, will try other flows. + /// + public string? FederatedToken { get; set; } + + /// + /// Managed Identity to use for the auth + /// + public string? ManagedIdentityId { get; set; } +} diff --git a/src/Maestro/Maestro.Common/ResolvedCredential.cs b/src/Maestro/Maestro.Common/AppCredentials/ResolvedCredential.cs similarity index 94% rename from src/Maestro/Maestro.Common/ResolvedCredential.cs rename to src/Maestro/Maestro.Common/AppCredentials/ResolvedCredential.cs index cc8ac16196..17bd08e7df 100644 --- a/src/Maestro/Maestro.Common/ResolvedCredential.cs +++ b/src/Maestro/Maestro.Common/AppCredentials/ResolvedCredential.cs @@ -3,7 +3,7 @@ using Azure.Core; -namespace Microsoft.DotNet.Maestro.Common; +namespace Maestro.Common.AppCredentials; /// /// Credential with a set token. diff --git a/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProvider.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProvider.cs new file mode 100644 index 0000000000..37c2de674a --- /dev/null +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProvider.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Azure.Core; +using Azure.Identity; +using Microsoft.Extensions.Options; + +namespace Maestro.Common.AzureDevOpsTokens; + +/// +/// This token provider expects to have a token or an MI defined per each Azure DevOps account (dnceng, devdiv..). +/// Token has precedence over MI. +/// Example configuration: +/// +/// { +/// "Tokens": { +/// "dnceng": "[some PAT]", +/// }, +/// "ManagedIdentities": { +/// "devdiv": "123b84ac-1321-425f-a117-222ca6974498", +/// "default": "system", // Use the system-assigned identity for every other org not defined +/// } +/// +/// The config above will use PAT for dnceng, a specific MI for devdiv and the system-assigned MI for any other org. +/// +public class AzureDevOpsTokenProvider : IAzureDevOpsTokenProvider +{ + private const string AzureDevOpsScope = "499b84ac-1321-427f-aa17-267ca6975798/.default"; + + private readonly Dictionary _tokenCredentials = []; + private readonly IOptionsMonitor _options; + + public AzureDevOpsTokenProvider(IOptionsMonitor options) + { + _options = options; + + foreach (var credential in options.CurrentValue.ManagedIdentities) + { + _tokenCredentials[credential.Key] = credential.Value == "system" + ? new ManagedIdentityCredential() + : new ManagedIdentityCredential(credential.Value); + } + } + + public async Task GetTokenForAccount(string account) + { + if (_options.CurrentValue.Tokens.TryGetValue(account, out var pat) && !string.IsNullOrEmpty(pat)) + { + return pat; + } + + if (_tokenCredentials.TryGetValue(account, out var credential)) + { + return (await credential.GetTokenAsync(new TokenRequestContext([AzureDevOpsScope]))).Token; + } + + // We can also define just one MI for all accounts + if (_tokenCredentials.TryGetValue("default", out var defaultCredential)) + { + return (await defaultCredential.GetTokenAsync(new TokenRequestContext([AzureDevOpsScope]))).Token; + } + + throw new ArgumentOutOfRangeException( + $"Azure DevOps account {account} does not have a configured PAT or credential. " + + $"Please add the account to the 'AzureDevOps.Tokens' or 'AzureDevOps.ManagedIdentities' configuration section"); + } +} diff --git a/src/Maestro/Maestro.AzureDevOps/AzureDevOpsTokenProviderOptions.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProviderOptions.cs similarity index 68% rename from src/Maestro/Maestro.AzureDevOps/AzureDevOpsTokenProviderOptions.cs rename to src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProviderOptions.cs index 1f211e1227..cd8905ab69 100644 --- a/src/Maestro/Maestro.AzureDevOps/AzureDevOpsTokenProviderOptions.cs +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProviderOptions.cs @@ -1,11 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; - -namespace Maestro.AzureDevOps; +namespace Maestro.Common.AzureDevOpsTokens; public class AzureDevOpsTokenProviderOptions { public Dictionary Tokens { get; } = []; + + public Dictionary ManagedIdentities { get; } = []; } diff --git a/src/Maestro/Maestro.AzureDevOps/IAzureDevOpsTokenProvider.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/IAzureDevOpsTokenProvider.cs similarity index 86% rename from src/Maestro/Maestro.AzureDevOps/IAzureDevOpsTokenProvider.cs rename to src/Maestro/Maestro.Common/AzureDevOpsTokens/IAzureDevOpsTokenProvider.cs index 4cdc0e352b..a43042e143 100644 --- a/src/Maestro/Maestro.AzureDevOps/IAzureDevOpsTokenProvider.cs +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/IAzureDevOpsTokenProvider.cs @@ -1,11 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Text.RegularExpressions; -using System.Threading.Tasks; -namespace Maestro.AzureDevOps; +namespace Maestro.Common.AzureDevOpsTokens; public interface IAzureDevOpsTokenProvider { @@ -23,7 +21,7 @@ public static Task GetTokenForRepository(this IAzureDevOpsTokenProvider { throw new ArgumentException($"{repositoryUrl} is not a valid Azure DevOps repository URL"); } - string account = m.Groups["account"].Value; + var account = m.Groups["account"].Value; return that.GetTokenForAccount(account); } } diff --git a/src/Maestro/Maestro.AzureDevOps/MaestroAzureDevOpsServiceCollectionExtensions.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/MaestroAzureDevOpsServiceCollectionExtensions.cs similarity index 91% rename from src/Maestro/Maestro.AzureDevOps/MaestroAzureDevOpsServiceCollectionExtensions.cs rename to src/Maestro/Maestro.Common/AzureDevOpsTokens/MaestroAzureDevOpsServiceCollectionExtensions.cs index 348c62f57c..0eee5b95f7 100644 --- a/src/Maestro/Maestro.AzureDevOps/MaestroAzureDevOpsServiceCollectionExtensions.cs +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/MaestroAzureDevOpsServiceCollectionExtensions.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.DependencyInjection; -namespace Maestro.AzureDevOps; +namespace Maestro.Common.AzureDevOpsTokens; public static class MaestroAzureDevOpsServiceCollectionExtensions { diff --git a/src/Maestro/Maestro.Common/Maestro.Common.csproj b/src/Maestro/Maestro.Common/Maestro.Common.csproj index ca5fb70831..e08e86ecc6 100644 --- a/src/Maestro/Maestro.Common/Maestro.Common.csproj +++ b/src/Maestro/Maestro.Common/Maestro.Common.csproj @@ -1,15 +1,18 @@ - enable - enable netstandard2.0;net6.0 + true + enable + enable + + diff --git a/src/Maestro/Maestro.Data/GlobalSuppressions.cs b/src/Maestro/Maestro.Data/GlobalSuppressions.cs index 8329ea847c..6a1fdaff9c 100644 --- a/src/Maestro/Maestro.Data/GlobalSuppressions.cs +++ b/src/Maestro/Maestro.Data/GlobalSuppressions.cs @@ -1,8 +1,10 @@ -// This file is used by Code Analysis to maintain SuppressMessage -// attributes that are applied to this project. -// Project-level suppressions either have no target or are given -// a specific target and scoped to a namespace, type, member, etc. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. [assembly: SuppressMessage("Style", "IDE0161:Convert to file-scoped namespace", Justification = "", Scope = "namespace", Target = "~N:Maestro.Data.Migrations")] diff --git a/src/Maestro/Maestro.Data/Migrations/20220427230202_initial-squashed.Designer.cs b/src/Maestro/Maestro.Data/Migrations/20220427230202_initial-squashed.Designer.cs index 7ec013318c..a37b190f9a 100644 --- a/src/Maestro/Maestro.Data/Migrations/20220427230202_initial-squashed.Designer.cs +++ b/src/Maestro/Maestro.Data/Migrations/20220427230202_initial-squashed.Designer.cs @@ -1,11 +1,11 @@ -// +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// using System; -using Maestro.Data; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; #nullable disable diff --git a/src/Maestro/Maestro.Data/Migrations/20240202111944_CodeEnabledSubscriptions.cs b/src/Maestro/Maestro.Data/Migrations/20240202111944_CodeEnabledSubscriptions.cs index 5252ea76ad..7853d70058 100644 --- a/src/Maestro/Maestro.Data/Migrations/20240202111944_CodeEnabledSubscriptions.cs +++ b/src/Maestro/Maestro.Data/Migrations/20240202111944_CodeEnabledSubscriptions.cs @@ -1,4 +1,7 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; using Microsoft.EntityFrameworkCore.Migrations; #nullable disable diff --git a/src/Maestro/Maestro.Data/Migrations/20240216101522_CodeEnabledSubscriptions2.cs b/src/Maestro/Maestro.Data/Migrations/20240216101522_CodeEnabledSubscriptions2.cs index fddfc25beb..ab97fca364 100644 --- a/src/Maestro/Maestro.Data/Migrations/20240216101522_CodeEnabledSubscriptions2.cs +++ b/src/Maestro/Maestro.Data/Migrations/20240216101522_CodeEnabledSubscriptions2.cs @@ -1,4 +1,7 @@ -using Microsoft.EntityFrameworkCore.Migrations; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.EntityFrameworkCore.Migrations; #nullable disable diff --git a/src/Maestro/Maestro.Data/Migrations/20240403133733_CodeEnabledSubscriptions3.cs b/src/Maestro/Maestro.Data/Migrations/20240403133733_CodeEnabledSubscriptions3.cs index 3e0c27796e..56e86327be 100644 --- a/src/Maestro/Maestro.Data/Migrations/20240403133733_CodeEnabledSubscriptions3.cs +++ b/src/Maestro/Maestro.Data/Migrations/20240403133733_CodeEnabledSubscriptions3.cs @@ -1,4 +1,7 @@ -using Microsoft.EntityFrameworkCore.Migrations; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.EntityFrameworkCore.Migrations; #nullable disable diff --git a/src/Maestro/Maestro.DataProviders/DarcRemoteFactory.cs b/src/Maestro/Maestro.DataProviders/DarcRemoteFactory.cs index b0bfdae74c..fd5967a587 100644 --- a/src/Maestro/Maestro.DataProviders/DarcRemoteFactory.cs +++ b/src/Maestro/Maestro.DataProviders/DarcRemoteFactory.cs @@ -3,7 +3,7 @@ using System; using System.Threading.Tasks; -using Maestro.AzureDevOps; +using Maestro.Common.AzureDevOpsTokens; using Maestro.Data; using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.DarcLib.Helpers; @@ -74,9 +74,9 @@ private async Task GetRemoteGitClient(string repoUrl, ILogger lo var remoteConfiguration = repoType switch { - GitRepoType.GitHub => new RemoteConfiguration( + GitRepoType.GitHub => new RemoteTokenProvider( gitHubToken: await _gitHubTokenProvider.GetTokenForInstallationAsync(installationId)), - GitRepoType.AzureDevOps => new RemoteConfiguration( + GitRepoType.AzureDevOps => new RemoteTokenProvider( azureDevOpsToken: await _azureDevOpsTokenProvider.GetTokenForRepository(normalizedUrl)), _ => throw new NotImplementedException($"Unknown repo url type {normalizedUrl}"), diff --git a/src/Maestro/Maestro.DataProviders/Maestro.DataProviders.csproj b/src/Maestro/Maestro.DataProviders/Maestro.DataProviders.csproj index f87f90930f..519740f5dc 100644 --- a/src/Maestro/Maestro.DataProviders/Maestro.DataProviders.csproj +++ b/src/Maestro/Maestro.DataProviders/Maestro.DataProviders.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/Maestro/Maestro.Web/.config/settings.Development.json b/src/Maestro/Maestro.Web/.config/settings.Development.json index c6902294e1..70b6f27c98 100644 --- a/src/Maestro/Maestro.Web/.config/settings.Development.json +++ b/src/Maestro/Maestro.Web/.config/settings.Development.json @@ -1,21 +1,28 @@ { - "HealthReportSettings": { - "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", - "TableName": "healthreport" - }, - "KeyVaultUri": "https://maestrolocal.vault.azure.net/", - "AppConfigurationUri": "https://maestrolocal.azconfig.io/", - "BuildAssetRegistry": { - "ConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true" - }, - "ForceLocalApi": false, - "DataProtection": { - "KeyFileUri": "", - "KeyIdentifier": "" - }, - "Kusto": { - "Database": "engineeringdata", - "KustoClusterUri": "https://engdata.westus2.kusto.windows.net", - "UseAzCliAuthentication": true - } -} + "HealthReportSettings": { + "StorageAccountTablesUri": "https://maestroint.table.core.windows.net", + "TableName": "healthreport" + }, + "KeyVaultUri": "https://maestrolocal.vault.azure.net/", + "AppConfigurationUri": "https://maestrolocal.azconfig.io/", + "BuildAssetRegistry": { + "ConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true" + }, + "ForceLocalApi": false, + "DataProtection": { + "KeyFileUri": "", + "KeyIdentifier": "" + }, + "Kusto": { + "Database": "engineeringdata", + "KustoClusterUri": "https://engdata.westus2.kusto.windows.net", + "UseAzCliAuthentication": true + }, + "AzureDevOps": { + "Tokens": { + "dnceng": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]", + "devdiv": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]", + "domoreexp": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" + } + } +} \ No newline at end of file diff --git a/src/Maestro/Maestro.Web/.config/settings.json b/src/Maestro/Maestro.Web/.config/settings.json index 81fcaf65a5..81098fab20 100644 --- a/src/Maestro/Maestro.Web/.config/settings.json +++ b/src/Maestro/Maestro.Web/.config/settings.json @@ -1,43 +1,32 @@ { - "GitHubAuthentication": { - "ClientId": "[vault(github-oauth-id)]", - "ClientSecret": "[vault(github-oauth-secret)]", - "SaveTokens": true, - "CallbackPath": "/signin/github", - "UserAgentProduct": "", - "ClaimsIssuer": "github" - }, - "GitHub": { - "GitHubAppId": "[vault(github-app-id)]", - "PrivateKey": "[vault(github-app-private-key)]" - }, - "AzureDevOps": { - "Tokens": [ - { - "Account": "dnceng", - "Token": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]" - }, - { - "Account": "devdiv", - "Token": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]" - }, - { - "Account": "domoreexp", - "Token": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" - } - ] - }, - "WebHooks": { - "github": { - "SecretKey": { - "default": "[vault(github-app-webhook-secret)]" - } - } - }, - "ApiRedirect": { - "uri": "https://maestro.dot.net/", - "token": "[vault(prod-maestro-token)]" - }, - "GitDownloadLocation": "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/git/Git-2.32.0-64-bit.zip", - "EnableAutoBuildPromotion": "[config(FeatureManagement:AutoBuildPromotion)]" + "GitHubAuthentication": { + "ClientId": "[vault(github-oauth-id)]", + "ClientSecret": "[vault(github-oauth-secret)]", + "SaveTokens": true, + "CallbackPath": "/signin/github", + "UserAgentProduct": "", + "ClaimsIssuer": "github" + }, + "GitHub": { + "GitHubAppId": "[vault(github-app-id)]", + "PrivateKey": "[vault(github-app-private-key)]" + }, + "AzureDevOps": { + "ManagedIdentities": { + "default": "system" + } + }, + "WebHooks": { + "github": { + "SecretKey": { + "default": "[vault(github-app-webhook-secret)]" + } + } + }, + "ApiRedirect": { + "uri": "https://maestro.dot.net/", + "token": "[vault(prod-maestro-token)]" + }, + "GitDownloadLocation": "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/git/Git-2.32.0-64-bit.zip", + "EnableAutoBuildPromotion": "[config(FeatureManagement:AutoBuildPromotion)]" } diff --git a/src/Maestro/Maestro.Web/Controllers/AzDevController.cs b/src/Maestro/Maestro.Web/Controllers/AzDevController.cs index 192f295233..3b1b7c9497 100644 --- a/src/Maestro/Maestro.Web/Controllers/AzDevController.cs +++ b/src/Maestro/Maestro.Web/Controllers/AzDevController.cs @@ -6,7 +6,7 @@ using System.Net.Http.Headers; using System.Text; using System.Threading.Tasks; -using Maestro.AzureDevOps; +using Maestro.Common.AzureDevOpsTokens; using Microsoft.AspNetCore.Mvc; namespace Maestro.Web.Controllers; diff --git a/src/Maestro/Maestro.Web/Maestro.Web.csproj b/src/Maestro/Maestro.Web/Maestro.Web.csproj index 38be8f998c..1b27380b23 100644 --- a/src/Maestro/Maestro.Web/Maestro.Web.csproj +++ b/src/Maestro/Maestro.Web/Maestro.Web.csproj @@ -63,7 +63,7 @@ - + diff --git a/src/Maestro/Maestro.Web/Pages/_Layout.cshtml b/src/Maestro/Maestro.Web/Pages/_Layout.cshtml index ed645ef52a..ebb81f5a6b 100644 --- a/src/Maestro/Maestro.Web/Pages/_Layout.cshtml +++ b/src/Maestro/Maestro.Web/Pages/_Layout.cshtml @@ -82,14 +82,9 @@ public class LocalGitClient : ILocalGitClient { - private readonly RemoteConfiguration _remoteConfiguration; + private readonly RemoteTokenProvider _remoteConfiguration; private readonly ITelemetryRecorder _telemetryRecorder; private readonly IProcessManager _processManager; private readonly IFileSystem _fileSystem; private readonly ILogger _logger; public LocalGitClient( - RemoteConfiguration remoteConfiguration, + RemoteTokenProvider remoteConfiguration, ITelemetryRecorder telemetryRecorder, IProcessManager processManager, IFileSystem fileSystem, diff --git a/src/Microsoft.DotNet.Darc/DarcLib/LocalLibGit2Client.cs b/src/Microsoft.DotNet.Darc/DarcLib/LocalLibGit2Client.cs index b37504ad7f..22d7cb7081 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/LocalLibGit2Client.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/LocalLibGit2Client.cs @@ -20,11 +20,11 @@ namespace Microsoft.DotNet.DarcLib; /// public class LocalLibGit2Client : LocalGitClient, ILocalLibGit2Client { - private readonly RemoteConfiguration _remoteConfiguration; + private readonly RemoteTokenProvider _remoteConfiguration; private readonly IProcessManager _processManager; private readonly ILogger _logger; - public LocalLibGit2Client(RemoteConfiguration remoteConfiguration, ITelemetryRecorder telemetryRecorder, IProcessManager processManager, IFileSystem fileSystem, ILogger logger) + public LocalLibGit2Client(RemoteTokenProvider remoteConfiguration, ITelemetryRecorder telemetryRecorder, IProcessManager processManager, IFileSystem fileSystem, ILogger logger) : base(remoteConfiguration, telemetryRecorder, processManager, fileSystem, logger) { _remoteConfiguration = remoteConfiguration; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Models/Darc/DependencyGraph.cs b/src/Microsoft.DotNet.Darc/DarcLib/Models/Darc/DependencyGraph.cs index d5e4e6115f..9b0debd8e6 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Models/Darc/DependencyGraph.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Models/Darc/DependencyGraph.cs @@ -782,7 +782,7 @@ private static async Task GetRepoPathAsync( // If a repo folder or a mapping was not set we use the current parent's // parent folder. var gitClient = new LocalLibGit2Client( - new RemoteConfiguration(null, null), + new RemoteTokenProvider(null, null), new NoTelemetryRecorder(), new ProcessManager(logger, gitExecutable), new FileSystem(), @@ -830,7 +830,7 @@ private static async Task> GetDependenciesAsync( if (Directory.Exists(testPath)) { - var local = new Local(new RemoteConfiguration(), logger, testPath); + var local = new Local(new RemoteTokenProvider(), logger, testPath); dependencies = await local.GetDependenciesAsync(); } } @@ -847,7 +847,7 @@ private static async Task> GetDependenciesAsync( if (!string.IsNullOrEmpty(repoPath)) { - var local = new Local(new RemoteConfiguration(), logger); + var local = new Local(new RemoteTokenProvider(), logger); string fileContents = await GitShowAsync( gitExecutable, repoPath, diff --git a/src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs b/src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs index 36decc0d41..72d917f4d7 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs @@ -24,7 +24,7 @@ protected RemoteRepoBase( string temporaryRepositoryPath, IMemoryCache cache, ILogger logger, - RemoteConfiguration remoteConfiguration) + RemoteTokenProvider remoteConfiguration) : this(gitExecutable, temporaryRepositoryPath, cache, logger, new ProcessManager(logger, gitExecutable), remoteConfiguration) { } @@ -35,7 +35,7 @@ private RemoteRepoBase( IMemoryCache cache, ILogger logger, ProcessManager processManager, - RemoteConfiguration remoteConfiguration) + RemoteTokenProvider remoteConfiguration) : base(remoteConfiguration, new LocalLibGit2Client(remoteConfiguration, new NoTelemetryRecorder(), processManager, new FileSystem(), logger), logger) { TemporaryRepositoryPath = temporaryRepositoryPath; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/RemoteConfiguration.cs b/src/Microsoft.DotNet.Darc/DarcLib/RemoteTokenProvider.cs similarity index 90% rename from src/Microsoft.DotNet.Darc/DarcLib/RemoteConfiguration.cs rename to src/Microsoft.DotNet.Darc/DarcLib/RemoteTokenProvider.cs index b7a7aa17a4..f7270300e9 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/RemoteConfiguration.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/RemoteTokenProvider.cs @@ -7,9 +7,9 @@ namespace Microsoft.DotNet.DarcLib; -public class RemoteConfiguration +public class RemoteTokenProvider { - public RemoteConfiguration(string? gitHubToken = null, string? azureDevOpsToken = null) + public RemoteTokenProvider(string? gitHubToken = null, string? azureDevOpsToken = null) { GitHubToken = gitHubToken; AzureDevOpsToken = azureDevOpsToken; @@ -19,6 +19,7 @@ public RemoteConfiguration(string? gitHubToken = null, string? azureDevOpsToken public string? GitHubToken { get; } + public string? AzureDevOpsToken { get; } public string? GetTokenForUri(string repoUri) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs index c98f5c9c2b..4e785e2ddb 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs @@ -27,7 +27,7 @@ public static IServiceCollection AddVmrManagers( { // Configuration based registrations services.TryAddSingleton(new VmrInfo(Path.GetFullPath(vmrPath), Path.GetFullPath(tmpPath))); - services.TryAddSingleton(new RemoteConfiguration(gitHubToken, azureDevOpsToken)); + services.TryAddSingleton(new RemoteTokenProvider(gitHubToken, azureDevOpsToken)); services.TryAddTransient(sp => ActivatorUtilities.CreateInstance(sp, gitLocation)); services.TryAddTransient(sp => sp.GetRequiredService>()); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile b/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile index 7391e651b3..e5f5cfeeeb 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile +++ b/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile @@ -16,7 +16,6 @@ COPY ["eng/Versions.props", "./eng/"] COPY ["src/Maestro/Client/src/Microsoft.DotNet.Maestro.Client.csproj", "./Maestro/Client/src/"] COPY ["src/Maestro/Maestro.Authentication/Maestro.Authentication.csproj", "./Maestro/Maestro.Authentication/"] -COPY ["src/Maestro/Maestro.AzureDevOps/Maestro.AzureDevOps.csproj", "./Maestro/Maestro.AzureDevOps/"] COPY ["src/Maestro/Maestro.Data/Maestro.Data.csproj", "./Maestro/Maestro.Data/"] COPY ["src/Maestro/Maestro.Common/Maestro.Common.csproj", "./Maestro/Maestro.Common/"] COPY ["src/Maestro/Maestro.DataProviders/Maestro.DataProviders.csproj", "./Maestro/Maestro.DataProviders/"] @@ -31,7 +30,6 @@ RUN dotnet restore "./ProductConstructionService/ProductConstructionService.Api/ COPY ["src/Maestro/Client/src", "./Maestro/Client/src"] COPY ["src/Maestro/Maestro.Authentication", "./Maestro/Maestro.Authentication"] -COPY ["src/Maestro/Maestro.AzureDevOps", "./Maestro/Maestro.AzureDevOps"] COPY ["src/Maestro/Maestro.Data", "./Maestro/Maestro.Data"] COPY ["src/Maestro/Maestro.Common", "./Maestro/Maestro.Common"] COPY ["src/Maestro/Maestro.DataProviders", "./Maestro/Maestro.DataProviders"] @@ -52,4 +50,4 @@ RUN git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com && git config --global user.name "dotnet-maestro[bot]" WORKDIR /app COPY --from=publish /app/publish . -ENTRYPOINT ["dotnet", "ProductConstructionService.Api.dll"] \ No newline at end of file +ENTRYPOINT ["dotnet", "ProductConstructionService.Api.dll"] diff --git a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs index eabc3a0783..1dbfe0212f 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs @@ -3,7 +3,7 @@ using System; using System.Collections.Generic; -using Microsoft.DotNet.Maestro.Common; +using Maestro.Common.AppCredentials; namespace ProductConstructionService.Client { @@ -26,11 +26,13 @@ public ProductConstructionServiceApiOptions(string baseUri, string accessToken, : this( new Uri(baseUri), AppCredentialResolver.CreateCredential( - EntraAppIds[baseUri.TrimEnd('/')], - disableInteractiveAuth: true, // the client is only used in Maestro for now - token: accessToken, - federatedToken: null, - managedIdentityId: managedIdentityId)) + new AppCredentialResolverOptions(EntraAppIds[baseUri.TrimEnd('/')]) + { + DisableInteractiveAuth = true, // the client is only used in Maestro for now + Token = accessToken, + FederatedToken = null, + ManagedIdentityId = managedIdentityId, + })) { } @@ -41,14 +43,7 @@ public ProductConstructionServiceApiOptions(string baseUri, string accessToken, /// Optional BAR token. When provided, will be used as the primary auth method. /// Managed Identity to use for the auth public ProductConstructionServiceApiOptions(string accessToken, string managedIdentityId) - : this( - new Uri(StagingPcsBaseUri), - AppCredentialResolver.CreateCredential( - EntraAppIds[StagingPcsBaseUri.TrimEnd('/')], - disableInteractiveAuth: true, // the client is only used in Maestro for now - token: accessToken, - federatedToken: null, - managedIdentityId: managedIdentityId)) + : this(StagingPcsBaseUri, accessToken, managedIdentityId) { } } diff --git a/src/ProductConstructionService/Readme.md b/src/ProductConstructionService/Readme.md index 04d405a3ea..e629458db2 100644 --- a/src/ProductConstructionService/Readme.md +++ b/src/ProductConstructionService/Readme.md @@ -54,7 +54,8 @@ If the service is being recreated and the same Managed Identity name is reused, Once the resources are created and configured: - Go to the newly created User Assigned Managed Identity (the one that's assigned to the container app, not the deployment one) - Copy the Client ID, and paste it in the correct appconfig.json, under `ManagedIdentityClientId` - - Add this identity as a user to AzDo so it can get AzDo tokens (you'll need a saw for this). You might have to remove the old user identity before doing this + - Add this identity as a user to AzDO so it can get AzDO tokens (you'll need a saw for this). You might have to remove the old user identity before doing this + - It needs to be able to manage code / pull requests and manage feeds (this is done in the artifact section). - Update the `ProductConstructionServiceDeploymentProd` (or `ProductConstructionServiceDeploymentInt`) Service Connection with the new MI information (you'll also have to create a Federated Credential in the MI) - Update the default PCS URI in `ProductConstructionServiceApiOptions`. diff --git a/test/FeedCleaner.Tests/FeedCleanerServiceTests.cs b/test/FeedCleaner.Tests/FeedCleanerServiceTests.cs index 5c6c386702..02bfcdd866 100644 --- a/test/FeedCleaner.Tests/FeedCleanerServiceTests.cs +++ b/test/FeedCleaner.Tests/FeedCleanerServiceTests.cs @@ -6,7 +6,7 @@ using System.Linq; using System.Threading.Tasks; using FluentAssertions; -using Maestro.AzureDevOps; +using Maestro.Common.AzureDevOpsTokens; using Maestro.Data; using Maestro.Data.Models; using Microsoft.DotNet.DarcLib; diff --git a/test/Microsoft.DotNet.Darc.Tests/DependencyTestDriver.cs b/test/Microsoft.DotNet.Darc.Tests/DependencyTestDriver.cs index cb8ed8e6b3..334413c30b 100644 --- a/test/Microsoft.DotNet.Darc.Tests/DependencyTestDriver.cs +++ b/test/Microsoft.DotNet.Darc.Tests/DependencyTestDriver.cs @@ -65,7 +65,7 @@ public async Task Setup() // Set up a git file manager var processManager = new ProcessManager(NullLogger.Instance, "git"); - GitClient = new LocalLibGit2Client(new RemoteConfiguration(), new NoTelemetryRecorder(), processManager, new FileSystem(), NullLogger.Instance); + GitClient = new LocalLibGit2Client(new RemoteTokenProvider(), new NoTelemetryRecorder(), processManager, new FileSystem(), NullLogger.Instance); _versionDetailsParser = new VersionDetailsParser(); DependencyFileManager = new DependencyFileManager(GitClient, _versionDetailsParser, NullLogger.Instance);