Rename and separate queues/jobs into PCS.WorkItems (#3845)

Co-authored-by: Oleksandr Didyk <[email protected]>
Co-authored-by: Djuradj Kurepa <[email protected]>

arcade-services.sln

@@ -126,9 +126,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProductConstructionService.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProductConstructionService.Common", "src\ProductConstructionService\ProductConstructionService.Common\ProductConstructionService.Common.csproj", "{9C6F75EE-618D-4268-88B6-9E4C99B062FF}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.LongestBuildPathUpdater", "src\ProductConstructionService\ProductConstructionService.LongestBuildPathUpdater\ProductConstructionService.LongestBuildPathUpdater.csproj", "{BE0088E3-A8FF-4F05-9456-E8BAD2E50A19}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProductConstructionService.LongestBuildPathUpdater", "src\ProductConstructionService\ProductConstructionService.LongestBuildPathUpdater\ProductConstructionService.LongestBuildPathUpdater.csproj", "{BE0088E3-A8FF-4F05-9456-E8BAD2E50A19}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.LongestBuildPathUpdater.Tests", "test\ProductConstructionService.LongestBuildPathUpdater.Tests\ProductConstructionService.LongestBuildPathUpdater.Tests.csproj", "{D40EADB7-5D48-421B-806D-6E2F79C077F8}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProductConstructionService.LongestBuildPathUpdater.Tests", "test\ProductConstructionService.LongestBuildPathUpdater.Tests\ProductConstructionService.LongestBuildPathUpdater.Tests.csproj", "{D40EADB7-5D48-421B-806D-6E2F79C077F8}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProductConstructionService.WorkItems", "src\ProductConstructionService\ProductConstructionService.WorkItems\ProductConstructionService.WorkItems.csproj", "{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -609,6 +611,18 @@ Global
 		{D40EADB7-5D48-421B-806D-6E2F79C077F8}.Release|x64.Build.0 = Release|Any CPU
 		{D40EADB7-5D48-421B-806D-6E2F79C077F8}.Release|x86.ActiveCfg = Release|Any CPU
 		{D40EADB7-5D48-421B-806D-6E2F79C077F8}.Release|x86.Build.0 = Release|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Debug|x64.Build.0 = Debug|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Debug|x86.Build.0 = Debug|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|x64.ActiveCfg = Release|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|x64.Build.0 = Release|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|x86.ActiveCfg = Release|Any CPU
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -657,6 +671,7 @@ Global
 		{9C6F75EE-618D-4268-88B6-9E4C99B062FF} = {243A4561-BF35-405A-AF12-AC57BB27796D}
 		{BE0088E3-A8FF-4F05-9456-E8BAD2E50A19} = {243A4561-BF35-405A-AF12-AC57BB27796D}
 		{D40EADB7-5D48-421B-806D-6E2F79C077F8} = {1A456CF0-C09A-4DE6-89CE-1110EED31180}
+		{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA} = {243A4561-BF35-405A-AF12-AC57BB27796D}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {32B9C883-432E-4FC8-A1BF-090EB033DD5B}

eng/service-templates/ProductConstructionService/provision.bicep

@@ -715,7 +715,7 @@ resource storageAccountQueueService 'Microsoft.Storage/storageAccounts/queueServ
 }
 
 resource storageAccountQueue 'Microsoft.Storage/storageAccounts/queueServices/queues@2022-09-01' = {
-    name: 'pcs-jobs'
+    name: 'pcs-workitems'
     parent: storageAccountQueueService
 }
 

src/Microsoft.DotNet.Darc/DarcLib/TelemetryRecorder.cs

@@ -15,9 +15,9 @@ public enum TrackedGitOperation
 public interface ITelemetryRecorder
 {
     /// <summary>
-    /// Records job duration and result in the customEvents table, with the `job.{jobType}` name
+    /// Records work item duration and result in the customEvents table, with the `workItem.{type}` name
     /// </summary>
-    ITelemetryScope RecordJobCompletion(string jobName);
+    ITelemetryScope RecordWorkItemCompletion(string workItemName);
 
     /// <summary>
     /// Records git operation duration and result.
@@ -40,7 +40,7 @@ public class NoTelemetryRecorder : ITelemetryRecorder
 {
     private static readonly NoTelemetryScope _instance = new();
 
-    public ITelemetryScope RecordJobCompletion(string jobName) => _instance;
+    public ITelemetryScope RecordWorkItemCompletion(string workItemName) => _instance;
     public ITelemetryScope RecordGitOperation(TrackedGitOperation operation, string repoUri) => _instance;
 
     public class NoTelemetryScope : ITelemetryScope

src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs

@@ -6,20 +6,20 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.DotNet.DarcLib;
 using Microsoft.DotNet.Maestro.Client.Models;
-using ProductConstructionService.Api.Queue;
-using ProductConstructionService.Api.Queue.Jobs;
+using ProductConstructionService.WorkItems;
+using ProductConstructionService.WorkItems.WorkItemDefinitions;
 
 namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers;
 
 [Route("codeflow")]
 [ApiVersion("2020-02-20")]
 public class CodeFlowController(
         IBasicBarClient barClient,
-        JobProducerFactory jobProducerFactory)
+        WorkItemProducerFactory workItemProducerFactory)
     : ControllerBase
 {
     private readonly IBasicBarClient _barClient = barClient;
-    private readonly JobProducerFactory _jobProducerFactory = jobProducerFactory;
+    private readonly WorkItemProducerFactory _workItemProducerFactory = workItemProducerFactory;
 
     [HttpPost(Name = "Flow")]
     public async Task<IActionResult> FlowBuild([Required, FromBody] Maestro.Api.Model.v2020_02_20.CodeFlowRequest request)
@@ -41,7 +41,7 @@ public async Task<IActionResult> FlowBuild([Required, FromBody] Maestro.Api.Mode
             return NotFound($"Build {request.BuildId} not found");
         }
 
-        await _jobProducerFactory.Create<CodeFlowJob>().ProduceJobAsync(new()
+        await _workItemProducerFactory.Create<CodeFlowWorkItem>().ProduceWorkItemAsync(new()
         {
             BuildId = request.BuildId,
             SubscriptionId = request.SubscriptionId,

src/ProductConstructionService/ProductConstructionService.Api/Controllers/StatusController.cs

@@ -5,38 +5,38 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.OpenApi.Extensions;
 using ProductConstructionService.Api.Controllers.ActionResults;
-using ProductConstructionService.Api.Queue;
+using ProductConstructionService.WorkItems;
 
 namespace ProductConstructionService.Api.Controllers;
 
 [Route("status")]
-public class StatusController(JobScopeManager jobProcessorScopeManager)
+public class StatusController(WorkItemScopeManager workItemScopeManager)
     : ControllerBase
 {
     [HttpPut("stop", Name = "Stop")]
-    public IActionResult StopPcsJobProcessor()
+    public IActionResult StopPcsWorkItemProcessor()
     {
-        jobProcessorScopeManager.FinishJobAndStop();
-        return GetPcsJobProcessorStatus();
+        workItemScopeManager.FinishWorkItemAndStop();
+        return GetPcsWorkItemProcessorStatus();
     }
 
     [HttpPut("start", Name = "Start")]
-    public IActionResult StartPcsJobProcessor()
+    public IActionResult StartPcsWorkItemProcessor()
     {
-        if (jobProcessorScopeManager.State == JobsProcessorState.Initializing)
+        if (workItemScopeManager.State == WorkItemProcessorState.Initializing)
         {
             return new PreconditionFailedActionResult("The background worker can't be started until the VMR is cloned");
         }
 
-        jobProcessorScopeManager.Start();
+        workItemScopeManager.Start();
 
-        return GetPcsJobProcessorStatus();
+        return GetPcsWorkItemProcessorStatus();
     }
 
     [AllowAnonymous]
     [HttpGet(Name = "Status")]
-    public IActionResult GetPcsJobProcessorStatus()
+    public IActionResult GetPcsWorkItemProcessorStatus()
     {
-        return Ok(jobProcessorScopeManager.State.GetDisplayName());
+        return Ok(workItemScopeManager.State.GetDisplayName());
     }
 }

src/ProductConstructionService/ProductConstructionService.Api/InitializationBackgroundService.cs

@@ -3,7 +3,7 @@
 
 using Microsoft.DotNet.DarcLib;
 using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
-using ProductConstructionService.Api.Queue;
+using ProductConstructionService.WorkItems;
 
 namespace ProductConstructionService.Api;
 
@@ -16,7 +16,7 @@ internal class InitializationBackgroundService(
         IServiceScopeFactory serviceScopeFactory,
         ITelemetryRecorder telemetryRecorder,
         InitializationBackgroundServiceOptions options,
-        JobScopeManager jobScopeManager)
+        WorkItemScopeManager workItemScopeManager)
     : BackgroundService
 {
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -32,7 +32,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             linkedTokenSource.Token.ThrowIfCancellationRequested();
 
             telemetryScope.SetSuccess();
-            jobScopeManager.InitializingDone();
+            workItemScopeManager.InitializingDone();
         }
     }
 }

src/ProductConstructionService/ProductConstructionService.Api/InitializationHealthCheck.cs

@@ -2,15 +2,15 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using Microsoft.Extensions.Diagnostics.HealthChecks;
-using ProductConstructionService.Api.Queue;
+using ProductConstructionService.WorkItems;
 
 namespace ProductConstructionService.Api;
 
-internal class InitializationHealthCheck(JobScopeManager jobProcessorScopeManager) : IHealthCheck
+internal class InitializationHealthCheck(WorkItemScopeManager workItemProcessorScopeManager) : IHealthCheck
 {
     public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
     {
-        if (jobProcessorScopeManager.State == JobsProcessorState.Initializing)
+        if (workItemProcessorScopeManager.State == WorkItemProcessorState.Initializing)
         {
             return Task.FromResult(HealthCheckResult.Unhealthy("Background worker is waiting for initialization to finish"));
         }

src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs

@@ -32,10 +32,10 @@
 using ProductConstructionService.Api.Configuration;
 using ProductConstructionService.Api.Controllers;
 using ProductConstructionService.Api.Pages.DependencyFlow;
-using ProductConstructionService.Api.Queue;
 using ProductConstructionService.Api.Telemetry;
 using ProductConstructionService.Api.VirtualMonoRepo;
 using ProductConstructionService.Common;
+using ProductConstructionService.WorkItems;
 
 namespace ProductConstructionService.Api;
 
@@ -138,7 +138,8 @@ internal static void ConfigurePcs(
         }
 
         builder.Services.RegisterBuildAssetRegistry(builder.Configuration);
-        builder.AddWorkitemQueues(azureCredential, waitForInitialization: initializeService);
+        builder.AddWorkItemQueues(azureCredential, waitForInitialization: initializeService);
+        builder.Services.AddWorkItemProcessors();
         builder.AddVmrRegistrations(gitHubToken);
         builder.AddMaestroApiClient(managedIdentityId);
         builder.AddGitHubClientFactory();

src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj

@@ -47,6 +47,7 @@
     <ProjectReference Include="..\..\Maestro\Maestro.MergePolicies\Maestro.MergePolicies.csproj" />
     <ProjectReference Include="..\..\Microsoft.DotNet.Darc\DarcLib\Microsoft.DotNet.DarcLib.csproj" />
     <ProjectReference Include="..\ProductConstructionService.Common\ProductConstructionService.Common.csproj" />
+    <ProjectReference Include="..\ProductConstructionService.WorkItems\ProductConstructionService.WorkItems.csproj" />
     <ProjectReference Include="..\ProductConstructionService.ServiceDefaults\ProductConstructionService.ServiceDefaults.csproj" />
   </ItemGroup>
 </Project>

src/ProductConstructionService/ProductConstructionService.Api/Program.cs

@@ -1,13 +1,11 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using Azure.Storage.Queues;
 using Microsoft.AspNetCore.Rewrite;
 using Microsoft.Extensions.FileProviders;
 using ProductConstructionService.Api;
 using ProductConstructionService.Api.Configuration;
-using ProductConstructionService.Api.Queue;
-using ProductConstructionService.Common;
+using ProductConstructionService.WorkItems;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -55,10 +53,7 @@
             new PhysicalFileProvider(Path.Combine(Environment.CurrentDirectory, "wwwroot"))),
     });
 
-    // When running locally, create the workitem queue, if it doesn't already exist
-    var queueServiceClient = app.Services.GetRequiredService<QueueServiceClient>();
-    var queueClient = queueServiceClient.GetQueueClient(app.Configuration.GetRequiredValue(QueueConfiguration.JobQueueNameConfigurationKey));
-    await queueClient.CreateIfNotExistsAsync();
+    await app.UseLocalWorkItemQueues();
 
     if (useSwagger)
     {