Skip to content

Commit

Permalink
[Rollout] Production Rollout 2024-11-06 (#4126)
Browse files Browse the repository at this point in the history
  • Loading branch information
dkurepa authored Nov 5, 2024
2 parents 09ffb59 + fc65aa7 commit 84bb10b
Show file tree
Hide file tree
Showing 16 changed files with 299 additions and 325 deletions.
1 change: 1 addition & 0 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
<PackageVersion Include="NUnit" Version="4.0.1" />
<PackageVersion Include="NUnit3TestAdapter" Version="4.5.0" />
<PackageVersion Include="Octokit" Version="13.0.1" />
<PackageVersion Include="Octokit.Webhooks.AspNetCore" Version="2.4.1" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
Expand Down
2 changes: 1 addition & 1 deletion NuGet.config
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@
<package pattern="NuGet.*" />
<package pattern="NUnit" />
<package pattern="NUnit3TestAdapter" />
<package pattern="Octokit" />
<package pattern="Octokit*" />
<package pattern="OpenTelemetry" />
<package pattern="OpenTelemetry.*" />
<package pattern="Perfolizer" />
Expand Down
2 changes: 0 additions & 2 deletions eng/templates/jobs/e2e-pcs-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,13 @@ jobs:
# Required for dotnet-bot-maestro-auth-test-content-rw
- group: Arcade-Services-Scenario-Tests
- ${{ if not(or(startsWith(variables['Build.SourceBranch'], 'refs/heads/production'), startsWith(variables['Build.SourceBranch'], 'refs/heads/production-'), eq(variables['Build.SourceBranch'], 'refs/heads/production'))) }}:
- group: MaestroInt KeyVault
- name: PcsTestEndpoint
value: https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io
- name: ScenarioTestSubscription
value: "Darc: Maestro Staging"
- name: MaestroAppId
value: $(MaestroStagingAppClientId)
- ${{ else }}:
- group: MaestroProd KeyVault
- name: PcsTestEndpoint
value: https://product-construction-prod.wittysky-0c79e3cc.westus2.azurecontainerapps.io
- name: ScenarioTestSubscription
Expand Down
2 changes: 1 addition & 1 deletion eng/templates/stages/deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ stages:
--repo dotnet/arcade-services
displayName: Create GitHub release
env:
GH_TOKEN: $(dotnet-bot-arcade-services-content-write)
GH_TOKEN: $(dotnet-bot-arcade-services-content-rw)
continueOnError: true
- stage: validateDeployment
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public static class AuthenticationConfiguration
{
public const string EntraAuthorizationPolicyName = "Entra";
public const string MsftAuthorizationPolicyName = "msft";
public const string AdminAuthorizationPolicyName = "RequireAdminAccess";

public const string AccountSignInRoute = "/Account/SignIn";

Expand Down Expand Up @@ -111,6 +112,12 @@ public static void ConfigureAuthServices(this IServiceCollection services, IConf
|| context.User.IsInRole(prodconSvcsRole);
});
});
options.AddPolicy(AdminAuthorizationPolicyName, policy =>
{
policy.AddAuthenticationSchemes(AuthenticationSchemes);
policy.RequireAuthenticatedUser();
policy.RequireRole("Admin");
});
});

services.Configure<MvcOptions>(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Collections.Generic;
using CommandLine;
using Microsoft.DotNet.Darc.Operations.VirtualMonoRepo;
using Microsoft.Extensions.DependencyInjection;

#nullable enable
namespace Microsoft.DotNet.Darc.Options.VirtualMonoRepo;
Expand All @@ -14,4 +15,10 @@ internal class GetRepoVersionCommandLineOptions : VmrCommandLineOptionsBase<GetR
{
[Value(0, HelpText = "Repository names (e.g. runtime) to get the versions for.")]
public IEnumerable<string> Repositories { get; set; } = Array.Empty<string>();

public override IServiceCollection RegisterServices(IServiceCollection services)
{
RegisterVmrServices(services, tmpPath: null);
return base.RegisterServices(services);
}
}
Original file line number Diff line number Diff line change
@@ -1,15 +1,9 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.IO;
using CommandLine;
using Microsoft.DotNet.Darc.Helpers;
using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
using Microsoft.DotNet.DarcLib;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.DotNet.Darc.Operations;
using Microsoft.Extensions.DependencyInjection;

namespace Microsoft.DotNet.Darc.Options.VirtualMonoRepo;

Expand All @@ -20,31 +14,7 @@ internal abstract class VmrCommandLineOptions<T> : VmrCommandLineOptionsBase<T>

public override IServiceCollection RegisterServices(IServiceCollection services)
{
string tmpPath = Path.GetFullPath(TmpPath ?? Path.GetTempPath());
LocalSettings localDarcSettings = null;

var gitHubToken = GitHubPat;
var azureDevOpsToken = AzureDevOpsPat;

// Read tokens from local settings if not provided
// We silence errors because the VMR synchronization often works with public repositories where tokens are not required
if (gitHubToken == null || azureDevOpsToken == null)
{
try
{
localDarcSettings = LocalSettings.GetSettings(this, NullLogger.Instance);
}
catch (DarcException)
{
// The VMR synchronization often works with public repositories where tokens are not required
}

gitHubToken ??= localDarcSettings?.GitHubToken;
azureDevOpsToken ??= localDarcSettings?.AzureDevOpsToken;
}

services.AddVmrManagers(GitLocation, VmrPath, tmpPath, gitHubToken, azureDevOpsToken);
services.TryAddTransient<IVmrScanner, VmrCloakedFileScanner>();
RegisterVmrServices(services, TmpPath);
return base.RegisterServices(services);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,14 @@

using System;
using CommandLine;
using Microsoft.DotNet.Darc.Helpers;
using Microsoft.DotNet.Darc.Operations;
using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
using Microsoft.DotNet.DarcLib;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging.Abstractions;
using System.IO;

#nullable enable
namespace Microsoft.DotNet.Darc.Options.VirtualMonoRepo;
Expand All @@ -12,4 +19,34 @@ internal abstract class VmrCommandLineOptionsBase<T> : CommandLineOptions<T> whe
{
[Option("vmr", HelpText = "Path to the VMR; defaults to nearest git root above the current working directory.")]
public string VmrPath { get; set; } = Environment.CurrentDirectory;

protected void RegisterVmrServices(IServiceCollection services, string? tmpPath)
{
LocalSettings? localDarcSettings = null;

var gitHubToken = GitHubPat;
var azureDevOpsToken = AzureDevOpsPat;

// Read tokens from local settings if not provided
// We silence errors because the VMR synchronization often works with public repositories where tokens are not required
if (gitHubToken == null || azureDevOpsToken == null)
{
try
{
localDarcSettings = LocalSettings.GetSettings(this, NullLogger.Instance);
}
catch (DarcException)
{
// The VMR synchronization often works with public repositories where tokens are not required
}

gitHubToken ??= localDarcSettings?.GitHubToken;
azureDevOpsToken ??= localDarcSettings?.AzureDevOpsToken;
}

tmpPath = Path.GetFullPath(tmpPath ?? Path.GetTempPath());

services.AddVmrManagers(GitLocation, VmrPath, tmpPath, gitHubToken, azureDevOpsToken);
services.TryAddTransient<IVmrScanner, VmrCloakedFileScanner>();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Reflection;
using Maestro.Data;
using Microsoft.DotNet.GitHub.Authentication;
using Octokit.Internal;
using Octokit;
using Octokit.Webhooks;
using Octokit.Webhooks.Events;
using Octokit.Webhooks.Events.Installation;
using Octokit.Webhooks.Events.InstallationRepositories;
using Microsoft.EntityFrameworkCore;
using System.Diagnostics.CodeAnalysis;

namespace ProductConstructionService.Api.Controllers;

public class GitHubWebhookEventProcessor : WebhookEventProcessor
{
private readonly ILogger<GitHubWebhookEventProcessor> _logger;
private readonly IGitHubTokenProvider _tokenProvider;
private readonly BuildAssetRegistryContext _context;

public GitHubWebhookEventProcessor(
ILogger<GitHubWebhookEventProcessor> logger,
IGitHubTokenProvider tokenProvider,
BuildAssetRegistryContext context)
{
_logger = logger;
_tokenProvider = tokenProvider;
_context = context;
}

protected override async Task ProcessInstallationWebhookAsync(
WebhookHeaders headers,
InstallationEvent installationEvent,
InstallationAction action)
{
switch(installationEvent.Action)
{
case InstallationActionValue.Deleted:
await RemoveInstallationRepositoriesAsync(installationEvent.Installation.Id);
break;
case InstallationActionValue.Created:
await SynchronizeInstallationRepositoriesAsync(installationEvent.Installation.Id);
break;
default:
_logger.LogError("Received unknown installation action `{action}`", installationEvent.Action);
break;
}
}

protected override async Task ProcessInstallationRepositoriesWebhookAsync(WebhookHeaders headers, InstallationRepositoriesEvent installationRepositoriesEvent, InstallationRepositoriesAction action)
{
await SynchronizeInstallationRepositoriesAsync(installationRepositoriesEvent.Installation.Id);
}

private async Task RemoveInstallationRepositoriesAsync(long installationId)
{
foreach (Maestro.Data.Models.Repository repo in await _context.Repositories.Where(r => r.InstallationId == installationId).ToListAsync())
{
repo.InstallationId = 0;
_context.Update(repo);
}

await _context.SaveChangesAsync();
}

private async Task SynchronizeInstallationRepositoriesAsync(long installationId)
{
var token = await _tokenProvider.GetTokenForInstallationAsync(installationId);
IReadOnlyList<Repository> gitHubRepos = await GetAllInstallationRepositories(token);

List<Maestro.Data.Models.Repository> barRepos =
await _context.Repositories.Where(r => r.InstallationId == installationId).ToListAsync();

List<Maestro.Data.Models.Repository> updatedRepos = [];
// Go through all Repos that currently have a given installation id. If the repo is already present in BAR, make sure it has the correct installationId,
// and add it to the list of update repos.
// If not, add it to BAR
foreach (Repository repo in gitHubRepos)
{
Maestro.Data.Models.Repository? existing = await _context.Repositories.FindAsync(repo.HtmlUrl);

if (existing == null)
{
_context.Repositories.Add(
new Maestro.Data.Models.Repository
{
RepositoryName = repo.HtmlUrl,
InstallationId = installationId,
});
}
else
{
existing.InstallationId = installationId;
_context.Update(existing);
updatedRepos.Add(existing);
}
}

// If a repo is present in BAR, but not in the updateRepos list, that means the GH app was uninstalled from it. Set its installationID to 0
foreach (Maestro.Data.Models.Repository repo in barRepos.Except(updatedRepos, new RepositoryNameComparer()))
{
repo.InstallationId = 0;
_context.Update(repo);
}

await _context.SaveChangesAsync();
}

private static Task<IReadOnlyList<Repository>> GetAllInstallationRepositories(string token)
{
var product = new ProductHeaderValue(
"Maestro",
Assembly.GetEntryAssembly()?
.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?
.InformationalVersion);
var client = new GitHubClient(product) { Credentials = new Credentials(token, AuthenticationType.Bearer) };
var pagination = new ApiPagination();
var uri = new Uri("installation/repositories", UriKind.Relative);

async Task<IApiResponse<List<Repository>>> GetInstallationRepositories(Uri requestUri)
{
IApiResponse<InstallationRepositoriesResponse> response =
await client.Connection.Get<InstallationRepositoriesResponse>(requestUri, parameters: null);
return new ApiResponse<List<Repository>>(response.HttpResponse, response.Body.Repositories);
}

return pagination.GetAllPages<Repository>(
async () => new ReadOnlyPagedCollection<Repository>(
await GetInstallationRepositories(uri),
GetInstallationRepositories),
uri);
}

public class InstallationRepositoriesResponse
{
public int TotalCount { get; set; }
public StringEnum<InstallationRepositorySelection> RepositorySelection { get; set; }
public required List<Repository> Repositories { get; set; }
}

private class RepositoryNameComparer : IEqualityComparer<Maestro.Data.Models.Repository>
{
public bool Equals(Maestro.Data.Models.Repository? x, Maestro.Data.Models.Repository? y) =>
string.Equals(x?.RepositoryName, y?.RepositoryName);

public int GetHashCode([DisallowNull] Maestro.Data.Models.Repository obj) => throw new NotImplementedException();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.

using System.Net;
using Maestro.Authentication;
using Microsoft.AspNetCore.ApiVersioning;
using Microsoft.AspNetCore.ApiVersioning.Swashbuckle;
using Microsoft.AspNetCore.Authorization;
Expand All @@ -12,6 +13,7 @@ namespace ProductConstructionService.Api.Controllers;

[Route("status")]
[ApiVersion("2020-02-20")]
[Authorize(Policy = AuthenticationConfiguration.AdminAuthorizationPolicyName)]
public class StatusController(IReplicaWorkItemProcessorStateCacheFactory replicaWorkItemProcessorStateCacheFactory) : ControllerBase
{
private readonly IReplicaWorkItemProcessorStateCacheFactory _replicaWorkItemProcessorStateCacheFactory = replicaWorkItemProcessorStateCacheFactory;
Expand Down
Loading

0 comments on commit 84bb10b

Please sign in to comment.