[Rollout] Production rollout 2024-11-13 (#4147)

.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "microsoft.dnceng.secretmanager": {
-      "version": "1.1.0-beta.24474.1",
+      "version": "1.1.0-beta.24555.1",
       "commands": [
         "secret-manager"
       ]
@@ -15,7 +15,7 @@
       ]
     },
     "microsoft.dnceng.configuration.bootstrap": {
-      "version": "1.1.0-beta.24474.1",
+      "version": "1.1.0-beta.24555.1",
       "commands": [
         "bootstrap-dnceng-configuration"
       ]

Directory.Build.props

@@ -31,6 +31,14 @@
     <LangVersion>latest</LangVersion>
     <IsShipping>false</IsShipping>
     <WarnOnPackingNonPackableProject>false</WarnOnPackingNonPackableProject>
+    <!-- 
+      This compiler property is required for debugging the BarViz project locally.
+      
+      When set to `full`, all debugging information will be made available together with the built
+      artifacts. If not present, Visual Studio debugger will fail to connect to the browser during
+      the debugging session
+    -->
+    <DebugType>Full</DebugType>
 
     <EnableCentralPackageVersions>true</EnableCentralPackageVersions>
     <CentralPackagesFile>$(MSBuildThisFileDirectory)eng/Packages.props</CentralPackagesFile>

NuGet.config

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
     <clear />

eng/Version.Details.xml

@@ -91,37 +91,37 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24516.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24525.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
+      <Sha>24e02f80c5458d1f75240ae57fc2a98fb8a9022a</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="8.0.0-beta.24516.1">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="8.0.0-beta.24525.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
+      <Sha>24e02f80c5458d1f75240ae57fc2a98fb8a9022a</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.24516.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.24525.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
+      <Sha>24e02f80c5458d1f75240ae57fc2a98fb8a9022a</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="8.0.0-beta.24516.1">
+    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="8.0.0-beta.24525.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
+      <Sha>24e02f80c5458d1f75240ae57fc2a98fb8a9022a</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Git.IssueManager" Version="8.0.0-beta.24516.1">
+    <Dependency Name="Microsoft.DotNet.Git.IssueManager" Version="8.0.0-beta.24525.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
+      <Sha>24e02f80c5458d1f75240ae57fc2a98fb8a9022a</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.VersionTools" Version="8.0.0-beta.24516.1">
+    <Dependency Name="Microsoft.DotNet.VersionTools" Version="8.0.0-beta.24525.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f7fb1fec01b91be69e4dcc5290a0bff3f28e214f</Sha>
+      <Sha>24e02f80c5458d1f75240ae57fc2a98fb8a9022a</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DncEng.SecretManager" Version="1.1.0-beta.24474.1">
+    <Dependency Name="Microsoft.DncEng.SecretManager" Version="1.1.0-beta.24555.1">
       <Uri>https://github.com/dotnet/dnceng</Uri>
-      <Sha>fa5f53c54c7d30fac266d7e70e7adfab720a55fa</Sha>
+      <Sha>64878fb972e889fc51b1f2e80a1ad2285fbd896c</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DncEng.Configuration.Bootstrap" Version="1.1.0-beta.24474.1">
+    <Dependency Name="Microsoft.DncEng.Configuration.Bootstrap" Version="1.1.0-beta.24555.1">
       <Uri>https://github.com/dotnet/dnceng</Uri>
-      <Sha>fa5f53c54c7d30fac266d7e70e7adfab720a55fa</Sha>
+      <Sha>64878fb972e889fc51b1f2e80a1ad2285fbd896c</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -9,11 +9,11 @@
     <UsingToolNetFrameworkReferenceAssemblies>true</UsingToolNetFrameworkReferenceAssemblies>
     <MicrosoftNetFrameworkReferenceAssembliesVersion>1.0.0-preview.1</MicrosoftNetFrameworkReferenceAssembliesVersion>
     <!-- Libs -->
-    <MicrosoftDotNetSignToolVersion>8.0.0-beta.24516.1</MicrosoftDotNetSignToolVersion>
-    <MicrosoftDotNetBuildTasksFeedVersion>8.0.0-beta.24516.1</MicrosoftDotNetBuildTasksFeedVersion>
-    <MicrosoftDotNetSwaggerGeneratorMSBuildVersion>8.0.0-beta.24516.1</MicrosoftDotNetSwaggerGeneratorMSBuildVersion>
-    <MicrosoftDotNetGitIssueManagerVersion>8.0.0-beta.24516.1</MicrosoftDotNetGitIssueManagerVersion>
-    <MicrosoftDotNetVersionToolsVersion>8.0.0-beta.24516.1</MicrosoftDotNetVersionToolsVersion>
+    <MicrosoftDotNetSignToolVersion>8.0.0-beta.24525.2</MicrosoftDotNetSignToolVersion>
+    <MicrosoftDotNetBuildTasksFeedVersion>8.0.0-beta.24525.2</MicrosoftDotNetBuildTasksFeedVersion>
+    <MicrosoftDotNetSwaggerGeneratorMSBuildVersion>8.0.0-beta.24525.2</MicrosoftDotNetSwaggerGeneratorMSBuildVersion>
+    <MicrosoftDotNetGitIssueManagerVersion>8.0.0-beta.24525.2</MicrosoftDotNetGitIssueManagerVersion>
+    <MicrosoftDotNetVersionToolsVersion>8.0.0-beta.24525.2</MicrosoftDotNetVersionToolsVersion>
     <MicrosoftNetTestSdkVersion>17.4.1</MicrosoftNetTestSdkVersion>
     <MicrosoftDotNetInternalLoggingVersion>1.1.0-beta.24376.1</MicrosoftDotNetInternalLoggingVersion>
     <MicrosoftAspNetCoreApiPaginationVersion>1.1.0-beta.24376.1</MicrosoftAspNetCoreApiPaginationVersion>
@@ -37,8 +37,8 @@
     <MicrosoftDotNetWebAuthenticationVersion>1.1.0-beta.24376.1</MicrosoftDotNetWebAuthenticationVersion>
     <ServiceFabricMocksVersion>1.1.0-beta.24376.1</ServiceFabricMocksVersion>
     <CoreHealthMonitorVersion>1.1.0-beta.24376.1</CoreHealthMonitorVersion>
-    <MicrosoftDncEngSecretManagerVersion>1.1.0-beta.24474.1</MicrosoftDncEngSecretManagerVersion>
-    <MicrosoftDncEngConfigurationBootstrapVersion>1.1.0-beta.24474.1</MicrosoftDncEngConfigurationBootstrapVersion>
+    <MicrosoftDncEngSecretManagerVersion>1.1.0-beta.24555.1</MicrosoftDncEngSecretManagerVersion>
+    <MicrosoftDncEngConfigurationBootstrapVersion>1.1.0-beta.24555.1</MicrosoftDncEngConfigurationBootstrapVersion>
   </PropertyGroup>
   <!--Package names-->
   <PropertyGroup>

eng/service-templates/ProductConstructionService/provision.bicep

@@ -202,7 +202,7 @@ module subscriptionTriggererTwiceDaily 'scheduledContainerJob.bicep' = {
         containerRegistryName: containerRegistryName
         containerAppsEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId
         containerImageName: containerImageName
-        command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll'
+        command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll twicedaily'
         contributorRoleId: contributorRole
         deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId
     }
@@ -223,7 +223,7 @@ module subscriptionTriggererDaily 'scheduledContainerJob.bicep' = {
         containerRegistryName: containerRegistryName
         containerAppsEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId
         containerImageName: containerImageName
-        command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll'
+        command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll daily'
         contributorRoleId: contributorRole
         deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId
     }
@@ -244,7 +244,7 @@ module subscriptionTriggererWeekly 'scheduledContainerJob.bicep' = {
         containerRegistryName: containerRegistryName
         containerAppsEnvironmentId: containerEnvironmentModule.outputs.containerEnvironmentId
         containerImageName: containerImageName
-        command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll'
+        command: 'cd /app/SubscriptionTriggerer && dotnet ProductConstructionService.SubscriptionTriggerer.dll weekly'
         contributorRoleId: contributorRole
         deploymentIdentityPrincipalId: managedIdentitiesModule.outputs.deploymentIdentityPrincipalId
     }

global.json

@@ -15,6 +15,6 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24516.1"
+    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24525.2"
   }
 }

src/Maestro/Maestro.Data/Models/Build.cs

@@ -5,7 +5,6 @@
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
-using System.Reflection.Metadata.Ecma335;
 using Microsoft.DotNet.DarcLib;
 using Microsoft.DotNet.Services.Utility;
 

src/Maestro/Maestro.Web/Api/v2018_07_16/Controllers/SubscriptionsController.cs

@@ -125,7 +125,7 @@ protected async Task<IActionResult> TriggerSubscriptionCore(Guid id, int buildId
         // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator
         if (!_subscriptionIdGenerator.ShouldTriggerSubscription(id))
         {
-            return BadRequest("Maestro shouldn't trigger PCS subscriptions");
+            return BadRequest(new ApiError("Maestro shouldn't trigger PCS subscriptions"));
         }
         Data.Models.Subscription subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild)
             .Include(sub => sub.Channel)
@@ -137,13 +137,13 @@ protected async Task<IActionResult> TriggerSubscriptionCore(Guid id, int buildId
             // Non-existent build
             if (build == null)
             {
-                return BadRequest($"Build {buildId} was not found");
+                return BadRequest(new ApiError($"Build {buildId} was not found"));
             }
             // Build doesn't match source repo
             if (!(build.GitHubRepository?.Equals(subscription.SourceRepository, StringComparison.InvariantCultureIgnoreCase) == true ||
                   build.AzureDevOpsRepository?.Equals(subscription.SourceRepository, StringComparison.InvariantCultureIgnoreCase) == true))
             {
-                return BadRequest($"Build {buildId} does not match source repo");
+                return BadRequest(new ApiError($"Build {buildId} does not match source repo"));
             }
         }
 

src/Microsoft.DotNet.Darc/Darc/Options/AddBuildToChannelCommandLineOptions.cs

@@ -58,9 +58,6 @@ internal class AddBuildToChannelCommandLineOptions : CommandLineOptions<AddBuild
     [RedactFromLogging]
     public string ArtifactPublishingAdditionalParameters { get; set; }
 
-    [Option("publish-installers-and-checksums", HelpText = "Whether installers and checksums should be published. This option is ignored")]
-    public bool PublishInstallersAndChecksums { get; set; }
-
     [Option("skip-assets-publishing", HelpText = "Add the build to the channel without publishing assets to the channel's feeds.")]
     public bool SkipAssetsPublishing { get; set; }
 

src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs

@@ -1328,14 +1328,15 @@ public async Task<AzureDevOpsFeed> GetFeedAndPackagesAsync(string accountName, s
     /// <param name="accountName">Azure DevOps account name</param>
     /// <param name="project">Project that the feed was created in</param>
     /// <param name="feedIdentifier">Name or id of the feed</param>
+    /// <param name="includeDeleted">Include deleted packages</param>
     /// <returns>List of packages in the feed</returns>
-    public async Task<List<AzureDevOpsPackage>> GetPackagesForFeedAsync(string accountName, string project, string feedIdentifier)
+    public async Task<List<AzureDevOpsPackage>> GetPackagesForFeedAsync(string accountName, string project, string feedIdentifier, bool includeDeleted = true)
     {
         JObject content = await ExecuteAzureDevOpsAPIRequestAsync(
             HttpMethod.Get,
             accountName,
             project,
-            $"_apis/packaging/feeds/{feedIdentifier}/packages?includeAllVersions=true&includeDeleted=true",
+            $"_apis/packaging/feeds/{feedIdentifier}/packages?includeAllVersions=true" + (includeDeleted ? "&includeDeleted=true" : string.Empty),
             _logger,
             versionOverride: "5.1-preview.1",
             baseAddressSubpath: "feeds.");

src/Microsoft.DotNet.Darc/DarcLib/IAzureDevOpsClient.cs

@@ -111,8 +111,11 @@ public interface IAzureDevOpsClient
     ///   Gets all Artifact feeds along with their packages in an Azure DevOps account.
     /// </summary>
     /// <param name="accountName">Azure DevOps account name.</param>
+    /// <param name="projectName">Azure DevOps project to get the ID for</param>
+    /// <param name="feedIdentifier">ID or name of the feed</param>
+    /// <param name="includeDeleted">Include deleted packages</param>
     /// <returns>List of Azure DevOps feeds in the account.</returns>
-    Task<List<AzureDevOpsPackage>> GetPackagesForFeedAsync(string accountName, string project, string feedIdentifier);
+    Task<List<AzureDevOpsPackage>> GetPackagesForFeedAsync(string accountName, string projectName, string feedIdentifier, bool includeDeleted = true);
 
     /// <summary>
     ///   Returns the project ID for a combination of Azure DevOps account and project name

src/Microsoft.DotNet.Darc/DarcLib/ILocalGitClient.cs

@@ -36,6 +36,12 @@ Task<string> BlameLineAsync(
         int line,
         string? blameFromCommit = null);
 
+    /// <summary>
+    /// Checks if the repository has any working tree changes.
+    /// </summary>
+    /// <param name="repoPath">Path to the repository</param>
+    Task<bool> HasWorkingTreeChangesAsync(string repoPath);
+
     /// <summary>
     ///     Checkout the repo to the specified state.
     /// </summary>

src/Microsoft.DotNet.Darc/DarcLib/ILocalGitRepo.cs

@@ -47,6 +47,11 @@ Task<string> BlameLineAsync(
         int line,
         string? blameFromCommit = null);
 
+    /// <summary>
+    /// Checks if the repository has any working tree changes.
+    /// </summary>
+    Task<bool> HasWorkingTreeChangesAsync();
+
     /// <summary>
     ///     Checkout the repo to the specified state.
     /// </summary>

src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs

@@ -436,6 +436,12 @@ public async Task<string> BlameLineAsync(string repoPath, string relativeFilePat
         return result.StandardOutput.Trim().Split(' ').First();
     }
 
+    public async Task<bool> HasWorkingTreeChangesAsync(string repoPath)
+    {
+        var result = await _processManager.ExecuteGit(repoPath, ["diff", "--exit-code"]);
+        return !result.Succeeded;
+    }
+
     public async Task AddGitAuthHeader(IList<string> args, IDictionary<string, string> envVars, string repoUri)
     {
         var token = await _remoteConfiguration.GetTokenForRepositoryAsync(repoUri);

src/Microsoft.DotNet.Darc/DarcLib/LocalGitRepo.cs

@@ -35,6 +35,9 @@ public async Task<string> AddRemoteIfMissingAsync(string repoUrl, CancellationTo
     public async Task<string> BlameLineAsync(string relativeFilePath, int line, string? blameFromCommit = null)
         => await _localGitClient.BlameLineAsync(Path, relativeFilePath, line, blameFromCommit);
 
+    public async Task<bool> HasWorkingTreeChangesAsync()
+        => await _localGitClient.HasWorkingTreeChangesAsync(Path);
+
     public async Task CheckoutAsync(string refToCheckout)
         => await _localGitClient.CheckoutAsync(Path, refToCheckout);
 

src/Microsoft.DotNet.Darc/DarcLib/Models/MsBuildPropsFile.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Xml;
 using System.Xml.Linq;
 using System.Xml.Serialization;
@@ -14,7 +15,16 @@ namespace Microsoft.DotNet.DarcLib.Models;
 
 public interface IMsBuildPropsFile
 {
+    /// <summary>
+    /// Serializes the properties to an XML file.
+    /// </summary>
+    /// <param name="path">Path to the file</param>
     void SerializeToXml(string path);
+
+    /// <summary>
+    /// Serializes the properties to an XML string.
+    /// </summary>
+    string SerializeToXml();
 }
 
 /// <summary>
@@ -53,8 +63,38 @@ public void SerializeToXml(string path)
         {
             Directory.CreateDirectory(directory);
         }
-
-        XmlSerializer serializer = new XmlSerializer(typeof(XmlElement));
+
+        var settings = new XmlWriterSettings
+        {
+            Indent = true,
+            Encoding = Encoding.UTF8,
+        };
+
+        using var writer = XmlWriter.Create(path, settings);
+        SerializeToXml(writer);
+    }
+
+    public string SerializeToXml()
+    {
+        var settings = new XmlWriterSettings
+        {
+            Indent = true,
+            Encoding = Encoding.UTF8,
+        };
+
+        using var stringWriter = new StringWriter();
+        using (var writer = XmlWriter.Create(stringWriter, settings))
+        {
+            SerializeToXml(writer);
+            writer.Flush();
+        }
+
+        return stringWriter.ToString();
+    }
+
+    private void SerializeToXml(XmlWriter xmlWriter)
+    {
+        var serializer = new XmlSerializer(typeof(XmlElement));
         var xmlDocument = new XmlDocument();
         XmlElement root = xmlDocument.CreateElement(ProjectPropertyName);
         var propertyGroup = xmlDocument.CreateElement(PropertyGroupName);
@@ -65,11 +105,10 @@ public void SerializeToXml(string path)
         var settings = new XmlWriterSettings
         {
             Indent = true,
-            Encoding = System.Text.Encoding.UTF8,
+            Encoding = Encoding.UTF8,
         };
 
-        using var writer = XmlWriter.Create(path, settings);
-        serializer.Serialize(writer, root);
+        serializer.Serialize(xmlWriter, root);
     }
 
     protected abstract void SerializeProperties(XmlElement propertyGroup, Func<string, XmlElement> createElement);

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrPatchHandler.cs

@@ -39,9 +39,7 @@ Task<List<VmrIngestionPatch>> CreatePatches(
         UnixPath? applicationPath,
         CancellationToken cancellationToken);
 
-    IReadOnlyCollection<string> GetVmrPatches(SourceMapping mapping) => GetVmrPatches(mapping.Name);
-
-    IReadOnlyCollection<string> GetVmrPatches(string mappingName);
+    IReadOnlyCollection<VmrIngestionPatch> GetVmrPatches();
 
     Task<IReadOnlyCollection<UnixPath>> GetPatchedFiles(string patchPath, CancellationToken cancellationToken);
 }

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PatchApplicationFailedException.cs

@@ -10,8 +10,8 @@ namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo;
 
 internal class PatchApplicationFailedException : Exception
 {
-    public PatchApplicationFailedException(VmrIngestionPatch patch, ProcessExecutionResult result)
-        : base($"Failed to apply the patch {Path.GetFileName(patch.Path)} to {patch.ApplicationPath ?? "/"}."
+    public PatchApplicationFailedException(VmrIngestionPatch patch, ProcessExecutionResult result, bool reverseApply)
+        : base($"Failed to {(reverseApply ? "reverse-apply" : "apply")} the patch {Path.GetFileName(patch.Path)} to {patch.ApplicationPath ?? "/"}."
             + Environment.NewLine
             + Environment.NewLine
             + result)

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrForwardFlower.cs

@@ -60,7 +60,7 @@ public PcsVmrForwardFlower(
         IAssetLocationResolver assetLocationResolver,
         IFileSystem fileSystem,
         ILogger<VmrCodeFlower> logger)
-        : base(vmrInfo, sourceManifest, vmrUpdater, dependencyTracker, vmrCloneManager, dependencyFileManager, repositoryCloneManager, localGitClient, libGit2Client, basicBarClient, localGitRepoFactory, versionDetailsParser, processManager, workBranchFactory, coherencyUpdateResolver, assetLocationResolver, fileSystem, logger)
+        : base(vmrInfo, sourceManifest, vmrUpdater, dependencyTracker, vmrCloneManager, dependencyFileManager, localGitClient, libGit2Client, basicBarClient, localGitRepoFactory, versionDetailsParser, processManager, coherencyUpdateResolver, assetLocationResolver, fileSystem, logger)
     {
         _sourceManifest = sourceManifest;
         _dependencyTracker = dependencyTracker;