diff --git a/.vault-config/maestroprod.yaml b/.vault-config/maestroprod.yaml index ffe9533729..a71c1895b2 100644 --- a/.vault-config/maestroprod.yaml +++ b/.vault-config/maestroprod.yaml @@ -24,12 +24,4 @@ keys: importSecretsFrom: shared/maestro-secrets.yaml -secrets: - # Needed during Maestro rollouts to create GitHub releases in arcade-services - BotAccount-dotnet-bot-repo-PAT: - type: github-access-token - parameters: - gitHubBotAccountSecret: - location: engkeyvault - name: BotAccount-dotnet-bot - gitHubBotAccountName: dotnet-bot +secrets: {} diff --git a/.vault-config/product-construction-dev.yaml b/.vault-config/product-construction-dev.yaml index 56c9d14cf4..6521a799a5 100644 --- a/.vault-config/product-construction-dev.yaml +++ b/.vault-config/product-construction-dev.yaml @@ -18,13 +18,6 @@ references: name: engkeyvault secrets: - BotAccount-dotnet-bot-repo-PAT: - type: github-access-token - parameters: - gitHubBotAccountSecret: - location: engkeyvault - name: BotAccount-dotnet-bot - gitHubBotAccountName: dotnet-bot github: type: github-app-secret diff --git a/.vault-config/product-construction-prod.yaml b/.vault-config/product-construction-prod.yaml index 52c2d7ac65..b61c5f2a86 100644 --- a/.vault-config/product-construction-prod.yaml +++ b/.vault-config/product-construction-prod.yaml @@ -12,14 +12,7 @@ references: name: engkeyvault secrets: - BotAccount-dotnet-bot-repo-PAT: - type: github-access-token - parameters: - gitHubBotAccountSecret: - location: engkeyvault - name: BotAccount-dotnet-bot - gitHubBotAccountName: dotnet-bot - + github: type: github-app-secret parameters: diff --git a/.vault-config/vmr-synchronization.1.yaml b/.vault-config/vmr-synchronization.1.yaml index ff88b7b266..c65c3a67c1 100644 --- a/.vault-config/vmr-synchronization.1.yaml +++ b/.vault-config/vmr-synchronization.1.yaml @@ -42,9 +42,3 @@ secrets: name: dn-bot-dnceng-build organizations: dnceng scopes: build_execute code_write - - BotAccount-dotnet-bot-repo-PAT: - type: github-access-token - parameters: - gitHubBotAccountSecret: BotAccount-dotnet-bot - gitHubBotAccountName: dotnet-bot diff --git a/eng/service-templates/ProductConstructionService/storage-account.bicep b/eng/service-templates/ProductConstructionService/storage-account.bicep index 7749e55ccd..096a697555 100644 --- a/eng/service-templates/ProductConstructionService/storage-account.bicep +++ b/eng/service-templates/ProductConstructionService/storage-account.bicep @@ -43,6 +43,11 @@ resource storageAccountQueue 'Microsoft.Storage/storageAccounts/queueServices/qu parent: storageAccountQueueService } +resource codeFlowQueue 'Microsoft.Storage/storageAccounts/queueServices/queues@2022-09-01' = { + name: 'pcs-codeflow-workitems' + parent: storageAccountQueueService +} + // allow storage queue access to the identity used for the aca's resource pcsStorageQueueAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: storageAccount diff --git a/src/Maestro/FeedCleanerService/.config/settings.Production.json b/src/Maestro/FeedCleanerService/.config/settings.Production.json index 3d95f6c7e5..d5460f2425 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.Production.json +++ b/src/Maestro/FeedCleanerService/.config/settings.Production.json @@ -4,9 +4,6 @@ "TableName": "healthreport" }, "KeyVaultUri": "https://maestroprod.vault.azure.net/", - "FeedCleaner": { - "Enabled": true - }, "BuildAssetRegistry": { "ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" }, diff --git a/src/Maestro/SubscriptionActorService/PullRequestActor.cs b/src/Maestro/SubscriptionActorService/PullRequestActor.cs index 9d87a786bf..aa8cbe4a4b 100644 --- a/src/Maestro/SubscriptionActorService/PullRequestActor.cs +++ b/src/Maestro/SubscriptionActorService/PullRequestActor.cs @@ -18,6 +18,7 @@ using Microsoft.Extensions.Logging; using Microsoft.ServiceFabric.Actors; using Microsoft.ServiceFabric.Actors.Runtime; +using Microsoft.VisualStudio.Services.Common; using ProductConstructionService.Client; using ProductConstructionService.Client.Models; using SubscriptionActorService.StateModel; @@ -872,8 +873,11 @@ await AddDependencyFlowEventsAsync( MergePolicyCheckResult.PendingPolicies, pr.Url); + var requiredDescriptionUpdates = + await CalculateOriginalDependencies(darcRemote, targetRepository, targetBranch, targetRepositoryUpdates); + pullRequest.Description = await _pullRequestBuilder.CalculatePRDescriptionAndCommitUpdatesAsync( - targetRepositoryUpdates.RequiredUpdates, + requiredDescriptionUpdates, pullRequest.Description, targetRepository, pullRequest.HeadBranch); @@ -1075,6 +1079,48 @@ private async Task GetRepositoryBranchUpdate() private static string GetNewBranchName(string targetBranch) => $"darc-{targetBranch}-{Guid.NewGuid()}"; + /// + /// Given a set of updates, replace the `from` version of every dependency update with the corresponding version + /// from the target branch + /// + /// Darc client used to fetch target branch dependencies. + /// Target repository to fetch the dependencies from. + /// Target branch to fetch the dependencies from. + /// Incoming updates to the repository + /// + /// Asset update and the corresponding list of altered dependencies + /// + /// + /// This method is intended for use in situations where we want to keep the information about the original dependency + /// version, such as when updating PR descriptions. + /// + private static async Task deps)>> CalculateOriginalDependencies( + IRemote darcRemote, + string targetRepository, + string targetBranch, + TargetRepoDependencyUpdate targetRepositoryUpdates) + { + List targetBranchDeps = [..await darcRemote.GetDependenciesAsync(targetRepository, targetBranch)]; + + List<(UpdateAssetsParameters update, List deps)> alteredUpdates = []; + foreach (var requiredUpdate in targetRepositoryUpdates.RequiredUpdates) + { + var updatedDependencies = requiredUpdate.deps + .Select(dependency => new DependencyUpdate() + { + From = targetBranchDeps + .Where(replace => dependency.From.Name == replace.Name) + .FirstOrDefault(dependency.From), + To = dependency.To, + }) + .ToList(); + + alteredUpdates.Add((requiredUpdate.update, updatedDependencies)); + } + + return alteredUpdates; + } + #region Code flow subscriptions /// diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CloneManager.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CloneManager.cs index d7c937c983..5e38b7ada0 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CloneManager.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CloneManager.cs @@ -151,7 +151,19 @@ protected async Task PrepareCloneInternal(string remoteUri, string d else { _logger.LogDebug("Clone of {repo} found in {clonePath}", remoteUri, clonePath); - var remote = await _localGitRepo.AddRemoteIfMissingAsync(clonePath, remoteUri, cancellationToken); + + string remote; + + try + { + remote = await _localGitRepo.AddRemoteIfMissingAsync(clonePath, remoteUri, cancellationToken); + } + catch (Exception e) when (e.Message.Contains("fatal: not a git repository")) + { + _logger.LogWarning("Clone at {clonePath} is not a git repository, re-cloning", clonePath); + _fileSystem.DeleteDirectory(clonePath, recursive: true); + return await PrepareCloneInternal(remoteUri, dirName, cancellationToken); + } // We cannot do `fetch --all` as tokens might be needed but fetch +refs/heads/*:+refs/remotes/origin/* doesn't fetch new refs // So we need to call `remote update origin` to fetch everything diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index e89f6e89fa..e5b1310f52 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -187,7 +187,7 @@ where sub.Enabled if (subscriptionToUpdate != null) { - await _workItemProducerFactory.CreateProducer().ProduceWorkItemAsync(new() + await _workItemProducerFactory.CreateProducer(subscriptionToUpdate.SourceEnabled).ProduceWorkItemAsync(new() { SubscriptionId = subscriptionToUpdate.Id, BuildId = buildId diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/StatusController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/StatusController.cs index 2b6e55168b..829fc0dbec 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/StatusController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/StatusController.cs @@ -45,8 +45,20 @@ public async Task StopPcsWorkItemProcessors() { return Ok(await PerformActionOnAllProcessors(async stateCache => { - await stateCache.SetStateAsync(WorkItemProcessorState.Stopping); - return (stateCache.ReplicaName, WorkItemProcessorState.Stopping); + var state = await stateCache.GetStateAsync(); + switch (state) + { + case WorkItemProcessorState.Stopping: + case WorkItemProcessorState.Working: + await stateCache.SetStateAsync(WorkItemProcessorState.Stopping); + return (stateCache.ReplicaName, WorkItemProcessorState.Stopping); + case WorkItemProcessorState.Initializing: + throw new BadHttpRequestException("Can't stop the service while initializing, try again later"); + case WorkItemProcessorState.Stopped: + return (stateCache.ReplicaName, WorkItemProcessorState.Stopped); + default: + throw new Exception("PCS is in an unsupported state"); + } })); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 49fdfa9a02..59bd42ce31 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -81,7 +81,7 @@ static PcsStartup() { var context = (BuildAssetRegistryContext)entry.Context; ILogger logger = context.GetService>(); - var workItemProducer = context.GetService().CreateProducer(); + var workItemProducerFactory = context.GetService(); var subscriptionIdGenerator = context.GetService(); BuildChannel entity = entry.Entity; @@ -118,6 +118,7 @@ static PcsStartup() foreach (Subscription subscription in subscriptionsToUpdate) { + var workItemProducer = workItemProducerFactory.CreateProducer(subscription.SourceEnabled); workItemProducer.ProduceWorkItemAsync(new() { BuildId = entity.BuildId, diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs index d4f8e43b13..741b0da1bc 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs @@ -48,8 +48,9 @@ await builder.ConfigurePcs( if (isDevelopment) { app.UseDeveloperExceptionPage(); - await app.Services.UseLocalWorkItemQueues( - app.Configuration.GetRequiredValue(WorkItemConfiguration.WorkItemQueueNameConfigurationKey)); + await app.Services.UseLocalWorkItemQueues([ + app.Configuration.GetRequiredValue(WorkItemConfiguration.DefaultWorkItemQueueNameConfigurationKey), + app.Configuration.GetRequiredValue(WorkItemConfiguration.CodeFlowWorkItemQueueNameConfigurationKey)]); if (useSwagger) { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json index 2d14aa38fd..9879c73c7d 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json @@ -12,8 +12,9 @@ } }, "AllowedHosts": "*", - "WorkItemQueueName": "pcs-workitems", - "WorkItemConsumerCount": 5, + "DefaultWorkItemQueueName": "pcs-workitems", + "DefaultWorkItemConsumerCount": 4, + "CodeFlowWorkItemQueueName": "pcs-codeflow-workitems", "WorkItemConsumerOptions": { "QueuePollTimeout": "00:01:00", "MaxWorkItemRetries": 3, diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestUpdater.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestUpdater.cs index e7896ddff3..6f6cb042e3 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestUpdater.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestUpdater.cs @@ -66,8 +66,12 @@ public NonBatchedPullRequestUpdater( Subscription? subscription = await _context.Subscriptions.FindAsync(SubscriptionId); if (subscription == null) { - await _pullRequestCheckReminders.UnsetReminderAsync(); - await _pullRequestUpdateReminders.UnsetReminderAsync(); + // We don't know if the subscription was a code flow one, so just unset both + await _pullRequestCheckReminders.UnsetReminderAsync(isCodeFlow: true); + await _pullRequestCheckReminders.UnsetReminderAsync(isCodeFlow: false); + await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow: true); + await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow: false); + return null; } @@ -98,7 +102,8 @@ protected override async Task> GetMergePoli } protected override async Task CheckInProgressPullRequestAsync( - InProgressPullRequest pullRequestCheck) + InProgressPullRequest pullRequestCheck, + bool isCodeFlow) { Subscription? subscription = await GetSubscription(); if (subscription == null) @@ -106,6 +111,6 @@ protected override async Task CheckInProgressPullRequestAsync( return false; } - return await base.CheckInProgressPullRequestAsync(pullRequestCheck); + return await base.CheckInProgressPullRequestAsync(pullRequestCheck, isCodeFlow); } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs index 61b07c9f1c..1b388cfac0 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Net; using Maestro.Contracts; using Maestro.Data.Models; @@ -101,6 +102,7 @@ public async Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem up // Check if we track an on-going PR already InProgressPullRequest? pr = await _pullRequestState.TryGetStateAsync(); + bool isCodeFlow = update.SubscriptionType == SubscriptionType.DependenciesAndSources; if (pr == null) { @@ -108,7 +110,7 @@ public async Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem up } else { - switch (await GetPullRequestStatusAsync(pr)) + switch (await GetPullRequestStatusAsync(pr, isCodeFlow)) { case PullRequestStatus.Completed: case PullRequestStatus.Invalid: @@ -120,14 +122,14 @@ public async Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem up break; default: _logger.LogInformation("PR {url} for subscription {subscriptionId} cannot be updated at this time", pr.Url, update.SubscriptionId); - await _pullRequestUpdateReminders.SetReminderAsync(update, DefaultReminderDelay); - await _pullRequestCheckReminders.UnsetReminderAsync(); + await _pullRequestUpdateReminders.SetReminderAsync(update, DefaultReminderDelay, isCodeFlow); + await _pullRequestCheckReminders.UnsetReminderAsync(isCodeFlow); return false; } } // Code flow updates are handled separetely - if (update.SubscriptionType == SubscriptionType.DependenciesAndSources) + if (isCodeFlow) { return await ProcessCodeFlowUpdateAsync(update, pr); } @@ -137,7 +139,7 @@ public async Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem up { await UpdatePullRequestAsync(pr, update); _logger.LogInformation("Pull request {url} for subscription {subscriptionId} was updated", pr.Url, update.SubscriptionId); - await _pullRequestUpdateReminders.UnsetReminderAsync(); + await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow); return true; } @@ -152,7 +154,7 @@ public async Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem up _logger.LogInformation("Pull request '{url}' for subscription {subscriptionId} created", prUrl, update.SubscriptionId); } - await _pullRequestUpdateReminders.UnsetReminderAsync(); + await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow); return true; } @@ -167,13 +169,13 @@ public async Task CheckPullRequestAsync(PullRequestCheck pullRequestCheck) return false; } - return await CheckInProgressPullRequestAsync(inProgressPr); + return await CheckInProgressPullRequestAsync(inProgressPr, pullRequestCheck.IsCodeFlow); } - protected virtual async Task CheckInProgressPullRequestAsync(InProgressPullRequest pullRequestCheck) + protected virtual async Task CheckInProgressPullRequestAsync(InProgressPullRequest pullRequestCheck, bool isCodeFlow) { _logger.LogInformation("Checking in-progress pull request {url}", pullRequestCheck.Url); - var status = await GetPullRequestStatusAsync(pullRequestCheck); + var status = await GetPullRequestStatusAsync(pullRequestCheck, isCodeFlow); return status != PullRequestStatus.Invalid; } @@ -183,7 +185,7 @@ protected virtual Task TagSourceRepositoryGitHubContactsIfPossibleAsync(InProgre return Task.CompletedTask; } - protected async Task GetPullRequestStatusAsync(InProgressPullRequest pr) + protected async Task GetPullRequestStatusAsync(InProgressPullRequest pr, bool isCodeFlow) { _logger.LogInformation("Querying status for pull request {prUrl}", pr.Url); @@ -223,12 +225,12 @@ await AddDependencyFlowEventsAsync( case MergePolicyCheckResult.NoPolicies: case MergePolicyCheckResult.FailedToMerge: _logger.LogInformation("Pull request {url} still active (updatable) - keeping tracking it", pr.Url); - await SetPullRequestCheckReminder(pr); + await SetPullRequestCheckReminder(pr, isCodeFlow); return PullRequestStatus.InProgressCanUpdate; case MergePolicyCheckResult.PendingPolicies: _logger.LogInformation("Pull request {url} still active (not updatable at the moment) - keeping tracking it", pr.Url); - await SetPullRequestCheckReminder(pr); + await SetPullRequestCheckReminder(pr, isCodeFlow); return PullRequestStatus.InProgressCannotUpdate; default: @@ -398,6 +400,8 @@ public async Task UpdateAssetsAsync( { // Check if we track an on-going PR already InProgressPullRequest? pr = await _pullRequestState.TryGetStateAsync(); + bool isCodeFlow = type == SubscriptionType.DependenciesAndSources; + bool canUpdate; if (pr == null) { @@ -406,7 +410,7 @@ public async Task UpdateAssetsAsync( } else { - var status = await GetPullRequestStatusAsync(pr); + var status = await GetPullRequestStatusAsync(pr, isCodeFlow); canUpdate = status == PullRequestStatus.InProgressCanUpdate; if (status == PullRequestStatus.Completed || status == PullRequestStatus.Invalid) @@ -431,8 +435,8 @@ public async Task UpdateAssetsAsync( // Regardless of code flow or regular PR, if the PR are not complete, postpone the update if (pr != null && !canUpdate) { - await _pullRequestUpdateReminders.SetReminderAsync(update, DefaultReminderDelay); - await _pullRequestCheckReminders.UnsetReminderAsync(); + await _pullRequestUpdateReminders.SetReminderAsync(update, DefaultReminderDelay, isCodeFlow); + await _pullRequestCheckReminders.UnsetReminderAsync(isCodeFlow); _logger.LogInformation("Pull request '{prUrl}' cannot be updated, update queued", pr!.Url); return true; } @@ -480,6 +484,7 @@ public async Task UpdateAssetsAsync( private async Task CreatePullRequestAsync(SubscriptionUpdateWorkItem update) { (var targetRepository, var targetBranch) = await GetTargetAsync(); + bool isCodeFlow = update.SubscriptionType == SubscriptionType.DependenciesAndSources; IRemote darcRemote = await _remoteFactory.GetRemoteAsync(targetRepository, _logger); @@ -557,7 +562,7 @@ await AddDependencyFlowEventsAsync( MergePolicyCheckResult.PendingPolicies, prUrl); - await SetPullRequestCheckReminder(inProgressPr); + await SetPullRequestCheckReminder(inProgressPr, isCodeFlow); return prUrl; } @@ -584,6 +589,7 @@ await AddDependencyFlowEventsAsync( private async Task UpdatePullRequestAsync(InProgressPullRequest pr, SubscriptionUpdateWorkItem update) { (var targetRepository, var targetBranch) = await GetTargetAsync(); + bool isCodeFlow = update.SubscriptionType == SubscriptionType.DependenciesAndSources; _logger.LogInformation("Updating pull request {url} branch {targetBranch} in {targetRepository}", pr.Url, targetBranch, targetRepository); @@ -649,8 +655,11 @@ await AddDependencyFlowEventsAsync( MergePolicyCheckResult.PendingPolicies, pr.Url); + var requiredDescriptionUpdates = + await CalculateOriginalDependencies(darcRemote, targetRepository, targetBranch, targetRepositoryUpdates); + pullRequest.Description = await _pullRequestBuilder.CalculatePRDescriptionAndCommitUpdatesAsync( - targetRepositoryUpdates.RequiredUpdates, + requiredDescriptionUpdates, pullRequest.Description, targetRepository, pullRequest.HeadBranch); @@ -658,7 +667,7 @@ await AddDependencyFlowEventsAsync( pullRequest.Title = await _pullRequestBuilder.GeneratePRTitleAsync(pr.ContainedSubscriptions, targetBranch); await darcRemote.UpdatePullRequestAsync(pr.Url, pullRequest); - await SetPullRequestCheckReminder(pr); + await SetPullRequestCheckReminder(pr, isCodeFlow); _logger.LogInformation("Pull request '{prUrl}' updated", pr.Url); } @@ -818,22 +827,67 @@ await UpdateSubscriptionsForMergedPRAsync( private static string GetNewBranchName(string targetBranch) => $"darc-{targetBranch}-{Guid.NewGuid()}"; - private async Task SetPullRequestCheckReminder(InProgressPullRequest prState) + private async Task SetPullRequestCheckReminder(InProgressPullRequest prState, bool isCodeFlow) { var reminder = new PullRequestCheck() { UpdaterId = Id.ToString(), Url = prState.Url, + IsCodeFlow = isCodeFlow }; - await _pullRequestCheckReminders.SetReminderAsync(reminder, DefaultReminderDelay); + await _pullRequestCheckReminders.SetReminderAsync(reminder, DefaultReminderDelay, isCodeFlow); await _pullRequestState.SetAsync(prState); } private async Task ClearAllStateAsync() { await _pullRequestState.TryDeleteAsync(); - await _pullRequestCheckReminders.UnsetReminderAsync(); - await _pullRequestUpdateReminders.UnsetReminderAsync(); + await _pullRequestCheckReminders.UnsetReminderAsync(isCodeFlow: true); + await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow: true); + await _pullRequestCheckReminders.UnsetReminderAsync(isCodeFlow: false); + await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow: false); + } + + /// + /// Given a set of updates, replace the `from` version of every dependency update with the corresponding version + /// from the target branch + /// + /// Darc client used to fetch target branch dependencies. + /// Target repository to fetch the dependencies from. + /// Target branch to fetch the dependencies from. + /// Incoming updates to the repository + /// + /// Subscription update and the corresponding list of altered dependencies + /// + /// + /// This method is intended for use in situations where we want to keep the information about the original dependency + /// version, such as when updating PR descriptions. + /// + private static async Task deps)>> CalculateOriginalDependencies( + IRemote darcRemote, + string targetRepository, + string targetBranch, + TargetRepoDependencyUpdate targetRepositoryUpdates) + { + List targetBranchDeps = [.. await darcRemote.GetDependenciesAsync(targetRepository, targetBranch)]; + + List<(SubscriptionUpdateWorkItem update, List deps)> alteredUpdates = []; + foreach (var requiredUpdate in targetRepositoryUpdates.RequiredUpdates) + { + var updatedDependencies = requiredUpdate.deps + .Select(dependency => new DependencyUpdate() + { + From = targetBranchDeps + .Where(replace => dependency.From.Name == replace.Name) + .FirstOrDefault(dependency.From), + To = dependency.To, + }) + .ToList(); + + alteredUpdates.Add((requiredUpdate.update, updatedDependencies)); + } + + return alteredUpdates; } #region Code flow subscriptions @@ -845,6 +899,7 @@ private async Task ProcessCodeFlowUpdateAsync( SubscriptionUpdateWorkItem update, InProgressPullRequest? pr) { + bool isCodeFlow = update.SubscriptionType == SubscriptionType.DependenciesAndSources; // Compare last SHA with the build SHA to see if we already have this SHA in the PR if (update.SourceSha == pr?.SourceSha) { @@ -853,8 +908,8 @@ private async Task ProcessCodeFlowUpdateAsync( update.SubscriptionId, update.SourceSha); - await SetPullRequestCheckReminder(pr); - await _pullRequestUpdateReminders.UnsetReminderAsync(); + await SetPullRequestCheckReminder(pr, isCodeFlow); + await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow: true); return true; } @@ -1003,8 +1058,8 @@ private async Task UpdateAssetsAndSources(SubscriptionUpdateWorkItem updat Description = description }); - await SetPullRequestCheckReminder(pullRequest); - await _pullRequestUpdateReminders.UnsetReminderAsync(); + await SetPullRequestCheckReminder(pullRequest, true); + await _pullRequestUpdateReminders.UnsetReminderAsync(true); return true; } @@ -1103,6 +1158,7 @@ private async Task CreateCodeFlowPullRequestAsync( string prBranch, string targetBranch) { + bool isCodeFlow = update.SubscriptionType == SubscriptionType.DependenciesAndSources; IRemote darcRemote = await _remoteFactory.GetRemoteAsync(targetRepository, _logger); try @@ -1144,8 +1200,8 @@ await AddDependencyFlowEventsAsync( MergePolicyCheckResult.PendingPolicies, prUrl); - await SetPullRequestCheckReminder(inProgressPr); - await _pullRequestUpdateReminders.UnsetReminderAsync(); + await SetPullRequestCheckReminder(inProgressPr, isCodeFlow); + await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow); _logger.LogInformation("Code flow pull request created: {prUrl}", prUrl); } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/PullRequestCheck.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/PullRequestCheck.cs index 83c924381a..48ad2ecb30 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/PullRequestCheck.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/PullRequestCheck.cs @@ -11,4 +11,6 @@ public class PullRequestCheck : DependencyFlowWorkItem { [DataMember] public required string Url { get; set; } + [DataMember] + public required bool IsCodeFlow { get; set; } } diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs index 944ee035ad..a0feea809c 100644 --- a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs @@ -98,14 +98,15 @@ private async Task CleanFeedAsync(AzureDevOpsFeed feed, PackagesInReleaseFeeds p updatedCount += updatedVersions.Count; } - _logger.LogInformation("Feed {feed} cleaning finished with {count} updated packages", feed.Name, updatedCount); - - packages = await _azureDevOpsClient.GetPackagesForFeedAsync(feed.Account, feed.Project?.Name, feed.Name); - if (!packages.Any(packages => packages.Versions.Any(v => !v.IsDeleted))) - { - _logger.LogInformation("Feed {feed} has no packages left, deleting the feed", feed.Name); - await _azureDevOpsClient.DeleteFeedAsync(feed.Account, feed.Project?.Name, feed.Name); - } + _logger.LogInformation("Feed {feed} cleaning finished with {count}/{totalCount} updated packages", feed.Name, updatedCount, packages.Count); + + // TODO https://github.com/dotnet/core-eng/issues/9366: Do not remove feeds because it can break branches that still depend on those + // packages = await _azureDevOpsClient.GetPackagesForFeedAsync(feed.Account, feed.Project?.Name, feed.Name); + // if (!packages.Any(packages => packages.Versions.Any(v => !v.IsDeleted))) + // { + // _logger.LogInformation("Feed {feed} has no packages left, deleting the feed", feed.Name); + // await _azureDevOpsClient.DeleteFeedAsync(feed.Account, feed.Project?.Name, feed.Name); + // } } catch (Exception ex) { diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs index 9bac5f153e..8701288878 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs @@ -36,8 +36,10 @@ if (builder.Environment.IsDevelopment()) { var config = applicationScope.ServiceProvider.GetRequiredService(); - await applicationScope.ServiceProvider.UseLocalWorkItemQueues( - config.GetRequiredValue(WorkItemConfiguration.WorkItemQueueNameConfigurationKey)); + await applicationScope.ServiceProvider.UseLocalWorkItemQueues([ + config.GetRequiredValue(WorkItemConfiguration.DefaultWorkItemQueueNameConfigurationKey), + config.GetRequiredValue(WorkItemConfiguration.CodeFlowWorkItemQueueNameConfigurationKey) + ]); } var triggerer = applicationScope.ServiceProvider.GetRequiredService(); diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs index 224254e7e2..ce81a891d6 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs @@ -4,6 +4,7 @@ using Maestro.Data; using Maestro.Data.Models; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; @@ -31,19 +32,19 @@ public SubscriptionTriggerer( public async Task TriggerSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) { - var workItemProducer = _workItemProducerFactory.CreateProducer(); foreach (var updateSubscriptionWorkItem in await GetSubscriptionsToTrigger(targetUpdateFrequency)) { - await workItemProducer.ProduceWorkItemAsync(updateSubscriptionWorkItem); + await _workItemProducerFactory.CreateProducer(updateSubscriptionWorkItem.sourceEnabled) + .ProduceWorkItemAsync(updateSubscriptionWorkItem.item); _logger.LogInformation("Queued update for subscription '{subscriptionId}' with build '{buildId}'", - updateSubscriptionWorkItem.SubscriptionId, - updateSubscriptionWorkItem.BuildId); + updateSubscriptionWorkItem.item.SubscriptionId, + updateSubscriptionWorkItem.item.BuildId); } } - private async Task> GetSubscriptionsToTrigger(UpdateFrequency targetUpdateFrequency) + private async Task> GetSubscriptionsToTrigger(UpdateFrequency targetUpdateFrequency) { - List subscriptionsToTrigger = new(); + List<(bool, SubscriptionTriggerWorkItem)> subscriptionsToTrigger = new(); var enabledSubscriptionsWithTargetFrequency = (await _context.Subscriptions .Where(s => s.Enabled) @@ -80,11 +81,13 @@ private async Task> GetSubscriptionsToTrigger( if (isThereAnUnappliedBuildInTargetChannel && latestBuildInTargetChannel != null) { - subscriptionsToTrigger.Add(new SubscriptionTriggerWorkItem - { - BuildId = latestBuildInTargetChannel.Id, - SubscriptionId = subscription.Id, - }); + subscriptionsToTrigger.Add(( + subscription.SourceEnabled, + new SubscriptionTriggerWorkItem + { + BuildId = latestBuildInTargetChannel.Id, + SubscriptionId = subscription.Id, + })); } } diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.json b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.json index e6cce8db67..fd21f047dc 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.json +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.json @@ -1,3 +1,4 @@ { - "WorkItemQueueName": "pcs-workitems" + "DefaultWorkItemQueueName": "pcs-workitems", + "CodeflowWorkItemQueueName": "pcs-codeflow-workitems" } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs index b86b7900e5..1a925a8594 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs @@ -2,15 +2,16 @@ // The .NET Foundation licenses this file to you under the MIT license. using Azure; +using Microsoft.Extensions.DependencyInjection; using ProductConstructionService.Common; namespace ProductConstructionService.WorkItems; public interface IReminderManager where T : WorkItem { - Task SetReminderAsync(T reminder, TimeSpan dueTime); + Task SetReminderAsync(T reminder, TimeSpan dueTime, bool isCodeFlow); - Task UnsetReminderAsync(); + Task UnsetReminderAsync(bool isCodeFlow); Task ReminderReceivedAsync(); } @@ -29,14 +30,14 @@ public ReminderManager( _receiptCache = cacheFactory.Create($"Reminder_{key}", includeTypeInKey: false); } - public async Task SetReminderAsync(T payload, TimeSpan visibilityTimeout) + public async Task SetReminderAsync(T payload, TimeSpan visibilityTimeout, bool isCodeFlow) { - var client = _workItemProducerFactory.CreateProducer(); + var client = _workItemProducerFactory.CreateProducer(isCodeFlow); var sendReceipt = await client.ProduceWorkItemAsync(payload, visibilityTimeout); await _receiptCache.SetAsync(new ReminderArguments(sendReceipt.PopReceipt, sendReceipt.MessageId), visibilityTimeout + TimeSpan.FromHours(4)); } - public async Task UnsetReminderAsync() + public async Task UnsetReminderAsync(bool isCodeFlow) { var receipt = await _receiptCache.TryDeleteAsync(); if (receipt == null) @@ -44,7 +45,7 @@ public async Task UnsetReminderAsync() return; } - var client = _workItemProducerFactory.CreateProducer(); + var client = _workItemProducerFactory.CreateProducer(isCodeFlow); try { diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs index 90bd696a0c..f80b325de3 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.Extensions.DependencyInjection; using ProductConstructionService.Common; namespace ProductConstructionService.WorkItems; @@ -15,7 +16,9 @@ public class ReminderManagerFactory : IReminderManagerFactory private readonly IWorkItemProducerFactory _workItemProducerFactory; private readonly IRedisCacheFactory _cacheFactory; - public ReminderManagerFactory(IWorkItemProducerFactory workItemProducerFactory, IRedisCacheFactory cacheFactory) + public ReminderManagerFactory( + IWorkItemProducerFactory workItemProducerFactory, + IRedisCacheFactory cacheFactory) { _workItemProducerFactory = workItemProducerFactory; _cacheFactory = cacheFactory; diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReplicaWorkItemProcessorStateCache.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReplicaWorkItemProcessorStateCache.cs index 7a71f7a504..591fdb67db 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReplicaWorkItemProcessorStateCache.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReplicaWorkItemProcessorStateCache.cs @@ -17,7 +17,7 @@ public interface IReplicaWorkItemProcessorStateCacheFactory public class ReplicaWorkItemProcessorStateCache : IReplicaWorkItemProcessorStateCacheFactory { - private readonly ContainerAppResource _containerApp; + private ContainerAppResource _containerApp; private readonly IRedisCacheFactory _redisCacheFactory; private readonly ILogger _logger; @@ -33,6 +33,9 @@ public ReplicaWorkItemProcessorStateCache( public async Task> GetAllWorkItemProcessorStateCachesAsync() { + // Always fetch the latest container app information, in case there was a deployment or something like that + // in between calls + _containerApp = await _containerApp.GetAsync(); ContainerAppRevisionTrafficWeight activeRevisionTrafficWeight = _containerApp.Data.Configuration.Ingress.Traffic .Single(traffic => traffic.Weight == 100); diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs index 686ec2d257..b8d51ee318 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs @@ -17,7 +17,9 @@ namespace ProductConstructionService.WorkItems; public static class WorkItemConfiguration { public const string WorkItemQueueNameConfigurationKey = "WorkItemQueueName"; - public const string WorkItemConsumerCountConfigurationKey = "WorkItemConsumerCount"; + public const string DefaultWorkItemQueueNameConfigurationKey = "DefaultWorkItemQueueName"; + public const string CodeFlowWorkItemQueueNameConfigurationKey = "CodeFlowWorkItemQueueName"; + public const string DefaultWorkItemConsumerCountConfigurationKey = "DefaultWorkItemConsumerCount"; public const string ReplicaNameKey = "CONTAINER_APP_REPLICA_NAME"; public const string SubscriptionIdKey = "SubscriptionId"; public const string ResourceGroupNameKey = "ResourceGroupName"; @@ -25,6 +27,9 @@ public static class WorkItemConfiguration public const int PollingRateSeconds = 10; public const string LocalReplicaName = "localReplica"; + public const string DefaultWorkItemType = "Default"; + public const string CodeFlowWorkItemType = "CodeFlow"; + internal static readonly JsonSerializerOptions JsonSerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -45,22 +50,17 @@ public static void AddWorkItemQueues(this IHostApplicationBuilder builder, Defau sp, PollingRateSeconds)); - builder.Configuration[$"{WorkItemConsumerOptions.ConfigurationKey}:{WorkItemQueueNameConfigurationKey}"] = - builder.Configuration.GetRequiredValue(WorkItemQueueNameConfigurationKey); builder.Services.Configure( builder.Configuration.GetSection(WorkItemConsumerOptions.ConfigurationKey)); - - var consumerCount = int.Parse( - builder.Configuration.GetRequiredValue(WorkItemConsumerCountConfigurationKey)); - - for (int i = 0; i < consumerCount; i++) - { - var consumerId = $"WorkItemConsumer_{i}"; - - // https://github.com/dotnet/runtime/issues/38751 - builder.Services.AddSingleton( - p => ActivatorUtilities.CreateInstance(p, consumerId)); - } + builder.RegisterWorkItemConsumers( + int.Parse(builder.Configuration.GetRequiredValue(DefaultWorkItemConsumerCountConfigurationKey)), + DefaultWorkItemType, + builder.Configuration.GetRequiredValue(DefaultWorkItemQueueNameConfigurationKey)); + // We should only ever have one consumer listening to the CodeFlow Work Item Queue + builder.RegisterWorkItemConsumers( + 1, + CodeFlowWorkItemType, + builder.Configuration.GetRequiredValue(CodeFlowWorkItemQueueNameConfigurationKey)); builder.Services.AddTransient(); if (builder.Environment.IsDevelopment()) @@ -83,18 +83,23 @@ public static void AddWorkItemProducerFactory(this IHostApplicationBuilder build { builder.AddAzureQueueClient("queues", settings => settings.Credential = credential); - var queueName = builder.Configuration.GetRequiredValue(WorkItemQueueNameConfigurationKey); + var defaultWorkItemQueueName = builder.Configuration.GetRequiredValue(DefaultWorkItemQueueNameConfigurationKey); + var codeFlowWorkItemQueueName = builder.Configuration.GetRequiredValue(CodeFlowWorkItemQueueNameConfigurationKey); builder.Services.AddTransient(sp => - ActivatorUtilities.CreateInstance(sp, queueName)); + ActivatorUtilities.CreateInstance(sp, defaultWorkItemQueueName, codeFlowWorkItemQueueName)); } // When running locally, create the workitem queue, if it doesn't already exist - public static async Task UseLocalWorkItemQueues(this IServiceProvider serviceProvider, string queueName) + public static async Task UseLocalWorkItemQueues(this IServiceProvider serviceProvider, string[] queueNames) { var queueServiceClient = serviceProvider.GetRequiredService(); - var queueClient = queueServiceClient.GetQueueClient(queueName); - await queueClient.CreateIfNotExistsAsync(); + + foreach (var queueName in queueNames) + { + var queueClient = queueServiceClient.GetQueueClient(queueName); + await queueClient.CreateIfNotExistsAsync(); + } } public static void AddWorkItemProcessor( @@ -122,4 +127,20 @@ public static void AddWorkItemProcessor( registrations.RegisterProcessor(); }); } + + private static void RegisterWorkItemConsumers( + this IHostApplicationBuilder builder, + int count, + string type, + string queueName) + { + for (int i = 0; i < count; i++) + { + var consumerId = $"{type}WorkItemConsumer_{i}"; + + // https://github.com/dotnet/runtime/issues/38751 + builder.Services.AddSingleton( + p => ActivatorUtilities.CreateInstance(p, consumerId, queueName)); + } + } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs index 655afc25f2..c7f5b4c770 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs @@ -13,6 +13,7 @@ namespace ProductConstructionService.WorkItems; internal class WorkItemConsumer( string consumerId, + string queueName, ILogger logger, IOptions options, WorkItemScopeManager scopeManager, @@ -21,6 +22,7 @@ internal class WorkItemConsumer( : BackgroundService { private readonly string _consumerId = consumerId; + private readonly string _queueName = queueName; private readonly ILogger _logger = logger; private readonly IOptions _options = options; private readonly WorkItemScopeManager _scopeManager = scopeManager; @@ -32,8 +34,8 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) // Otherwise, the service will be stuck here await Task.Yield(); - QueueClient queueClient = queueServiceClient.GetQueueClient(_options.Value.WorkItemQueueName); - _logger.LogInformation("Consumer {consumerId} starting to process PCS queue {queueName}", _consumerId, _options.Value.WorkItemQueueName); + QueueClient queueClient = queueServiceClient.GetQueueClient(_queueName); + _logger.LogInformation("Consumer {consumerId} starting to process PCS queue {queueName}", _consumerId, _queueName); while (!cancellationToken.IsCancellationRequested) { @@ -67,7 +69,7 @@ private async Task ReadAndProcessWorkItemAsync(QueueClient queueClient, WorkItem if (message?.Body == null) { // Queue is empty, wait a bit - _logger.LogDebug("Queue {queueName} is empty. Sleeping for {sleepingTime} seconds", _options.Value.WorkItemQueueName, (int)_options.Value.QueuePollTimeout.TotalSeconds); + _logger.LogDebug("Queue {queueName} is empty. Sleeping for {sleepingTime} seconds", _queueName, (int)_options.Value.QueuePollTimeout.TotalSeconds); await Task.Delay(_options.Value.QueuePollTimeout, cancellationToken); return; } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumerOptions.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumerOptions.cs index b4f7fb6a1c..74b63f5b9b 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumerOptions.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumerOptions.cs @@ -8,7 +8,6 @@ public class WorkItemConsumerOptions public const string ConfigurationKey = "WorkItemConsumerOptions"; public required TimeSpan QueuePollTimeout { get; init; } - public required string WorkItemQueueName { get; init; } public required int MaxWorkItemRetries { get; init; } public required TimeSpan QueueMessageInvisibilityTime { get; init; } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs index 3f369b364e..541e8fbb14 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs @@ -7,14 +7,17 @@ namespace ProductConstructionService.WorkItems; public interface IWorkItemProducerFactory { - public IWorkItemProducer CreateProducer() where T : WorkItem; + public IWorkItemProducer CreateProducer(bool IsCodeFlowSubscription = false) where T : WorkItem; } -public class WorkItemProducerFactory(QueueServiceClient queueServiceClient, string queueName) : IWorkItemProducerFactory +public class WorkItemProducerFactory(QueueServiceClient queueServiceClient, string defaultQueueName, string codeFlowQueueName) : IWorkItemProducerFactory { private readonly QueueServiceClient _queueServiceClient = queueServiceClient; - private readonly string _queueName = queueName; + private readonly string _defaultQueueName = defaultQueueName; + private readonly string _codeFlowQueueName = codeFlowQueueName; - public IWorkItemProducer CreateProducer() where T : WorkItem - => new WorkItemProducer(_queueServiceClient, _queueName); + public IWorkItemProducer CreateProducer(bool isCodeFlowSubscription = false) where T : WorkItem + => isCodeFlowSubscription + ? new WorkItemProducer(_queueServiceClient, _codeFlowQueueName) + : new WorkItemProducer(_queueServiceClient, _defaultQueueName); } diff --git a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs index 94a3466004..ff19de102f 100644 --- a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs +++ b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs @@ -54,7 +54,7 @@ public void SetTestParameters(TestParameters parameters) if (!string.IsNullOrEmpty(_parameters.MaestroToken)) { _baseDarcRunArgs.AddRange(["-p", _parameters.MaestroToken]); - } + } } protected async Task WaitForPullRequestAsync(string targetRepo, string targetBranch) @@ -487,7 +487,10 @@ protected async Task AddDependenciesToLocalRepo(string repoPath, string name, st { using (ChangeDirectory(repoPath)) { - await RunDarcAsync(["add-dependency", "--name", name, "--type", isToolset ? "toolset" : "product", "--repo", repoUri, "--version", "0.0.1"]); + await RunDarcAsync( + [ + "add-dependency", "--name", name, "--version", "0.0.1", "--type", isToolset ? "toolset" : "product", "--repo", repoUri + ]); } } protected async Task GetTestChannelsAsync() @@ -682,7 +685,7 @@ protected async Task AddDependenciesToLocalRepo(string repoPath, List protected async Task GatherDrop(int buildId, string outputDir, bool includeReleased, string extraAssetsRegex) { - string[] args = [ "gather-drop", "--id", buildId.ToString(), "--dry-run", "--output-dir", outputDir ]; + string[] args = ["gather-drop", "--id", buildId.ToString(), "--dry-run", "--output-dir", outputDir]; if (includeReleased) { diff --git a/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs b/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs index 34ff5da254..5e5e279adf 100644 --- a/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs +++ b/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs @@ -94,7 +94,7 @@ public static async Task Dependencies(IServiceCollection collection) var mockIRemote = new Mock(); var mockWorkItemProducerFactory = new Mock(); var mockWorkItemProducer = new Mock>(); - mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockWorkItemProducer.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer(false)).Returns(mockWorkItemProducer.Object); mockIRemoteFactory.Setup(f => f.GetRemoteAsync(Repository, It.IsAny())).ReturnsAsync(mockIRemote.Object); mockIRemote.Setup(f => f.GetCommitAsync(Repository, CommitHash)).ReturnsAsync(new Microsoft.DotNet.DarcLib.Commit(Account, CommitHash, CommitMessage)); diff --git a/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs b/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs index ca4485c4ba..269d99425e 100644 --- a/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs +++ b/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs @@ -170,7 +170,7 @@ public static async Task Dependencies(IServiceCollection collection) var mockWorkItemProducerFactory = new Mock(); var mockWorkItemProducer = new Mock>(); mockWorkItemProducerFactory - .Setup(f => f.CreateProducer()) + .Setup(f => f.CreateProducer(false)) .Returns(mockWorkItemProducer.Object); collection.AddSingleton(mockWorkItemProducerFactory.Object); diff --git a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs index 2d412c6c06..7fa62bf22e 100644 --- a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs @@ -175,7 +175,7 @@ public static async Task Dependencies(IServiceCollection collection) var mockWorkItemProducerFactory = new Mock(); var mockWorkItemProducer = new Mock>(); mockWorkItemProducerFactory - .Setup(f => f.CreateProducer()) + .Setup(f => f.CreateProducer(false)) .Returns(mockWorkItemProducer.Object); collection.AddSingleton(mockWorkItemProducerFactory.Object); diff --git a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs index f99ba15426..96bd971db4 100644 --- a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs @@ -454,10 +454,12 @@ private static class TestDataConfiguration public static void Dependencies(IServiceCollection collection) { var mockWorkItemProducerFactory = new Mock(); - var mockUpdateSubscriptionWorkItemProducer = new Mock>(); + var mockSubscriptionTriggerWorkItemProducer = new Mock>(); var mockBuildCoherencyInfoWorkItem = new Mock>(); - mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockUpdateSubscriptionWorkItemProducer.Object); - mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockBuildCoherencyInfoWorkItem.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer(false)).Returns(mockSubscriptionTriggerWorkItemProducer.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer(false)).Returns(mockBuildCoherencyInfoWorkItem.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer(true)).Returns(mockSubscriptionTriggerWorkItemProducer.Object); + collection.AddLogging(l => l.AddProvider(new NUnitLogger())); collection.AddSingleton(new HostingEnvironment { diff --git a/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManager.cs b/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManager.cs index b168a77a7d..b19c84416b 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManager.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManager.cs @@ -21,13 +21,13 @@ public MockReminderManager(string key, Dictionary data) Data = data; } - public Task SetReminderAsync(T reminder, TimeSpan dueTime) + public Task SetReminderAsync(T reminder, TimeSpan dueTime, bool isCodeFlow) { Data[_key] = reminder; return Task.CompletedTask; } - public Task UnsetReminderAsync() + public Task UnsetReminderAsync(bool isCodeFlow) { Data.Remove(_key); return Task.CompletedTask; diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestUpdaterTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestUpdaterTests.cs index e4c3a06b03..148c91b68b 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestUpdaterTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestUpdaterTests.cs @@ -455,6 +455,7 @@ protected void AndShouldHavePullRequestCheckReminder() { UpdaterId = GetPullRequestUpdaterId().ToString(), Url = prUrl, + IsCodeFlow = Subscription.SourceEnabled }); } diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs index 1ab0ae4b35..d781a8d2c9 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs @@ -660,7 +660,10 @@ protected async Task AddDependenciesToLocalRepo(string repoPath, List { foreach (AssetData asset in dependencies) { - List parameters = ["add-dependency", "--name", asset.Name, "--type", "product", "--repo", repoUri]; + List parameters = + [ + "add-dependency", "--name", asset.Name,"--type", "product", "--repo", repoUri, + ]; if (!string.IsNullOrEmpty(coherentParent)) { diff --git a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs index c0478180eb..ddf9176876 100644 --- a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs +++ b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs @@ -35,7 +35,7 @@ public void Setup() workItemProducerMock.Setup(w => w.ProduceWorkItemAsync(It.IsAny(), TimeSpan.Zero)) .ReturnsAsync(QueuesModelFactory.SendReceipt("message", DateTimeOffset.Now, DateTimeOffset.Now, "popReceipt", DateTimeOffset.Now)) .Callback((item, _) => _updateSubscriptionWorkItems.Add(item)); - workItemProducerFactoryMock.Setup(w => w.CreateProducer()) + workItemProducerFactoryMock.Setup(w => w.CreateProducer(false)) .Returns(workItemProducerMock.Object); services.AddLogging();