-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
24 changed files
with
1,049 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
60 changes: 60 additions & 0 deletions
60
src/StreetNameRegistry.Consumer.Read.Postal/ConsumerPostal.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| namespace StreetNameRegistry.Consumer.Read.Postal | ||
| { | ||
| using System; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Be.Vlaanderen.Basisregisters.MessageHandling.Kafka.Consumer; | ||
| using Be.Vlaanderen.Basisregisters.ProjectionHandling.Connector; | ||
| using Microsoft.EntityFrameworkCore; | ||
| using Microsoft.Extensions.Hosting; | ||
| using Microsoft.Extensions.Logging; | ||
| using Projections; | ||
|
|
||
| public class ConsumerPostal : BackgroundService | ||
| { | ||
| private readonly IHostApplicationLifetime _hostApplicationLifetime; | ||
| private readonly IDbContextFactory<ConsumerPostalContext> _consumerPostalDbContextFactory; | ||
| private readonly IConsumer _consumer; | ||
| private readonly ILogger<ConsumerPostal> _logger; | ||
|
|
||
| public ConsumerPostal( | ||
| IHostApplicationLifetime hostApplicationLifetime, | ||
| IDbContextFactory<ConsumerPostalContext> consumerPostalDbContextFactory, | ||
| IConsumer consumer, | ||
| ILoggerFactory loggerFactory) | ||
| { | ||
| _hostApplicationLifetime = hostApplicationLifetime; | ||
| _consumerPostalDbContextFactory = consumerPostalDbContextFactory; | ||
| _consumer = consumer; | ||
| _logger = loggerFactory.CreateLogger<ConsumerPostal>(); | ||
| } | ||
|
|
||
| protected override async Task ExecuteAsync(CancellationToken stoppingToken) | ||
| { | ||
| var projector = | ||
| new ConnectedProjector<ConsumerPostalContext>( | ||
| Resolve.WhenEqualToHandlerMessageType(new PostalKafkaProjection().Handlers)); | ||
|
|
||
| try | ||
| { | ||
| await _consumer.ConsumeContinuously(async message => | ||
| { | ||
| _logger.LogInformation("Handling next message"); | ||
|
|
||
| await using var context = await _consumerPostalDbContextFactory.CreateDbContextAsync(stoppingToken); | ||
| await projector.ProjectAsync(context, message, stoppingToken).ConfigureAwait(false); | ||
|
|
||
| //CancellationToken.None to prevent halfway consumption | ||
| await context.SaveChangesAsync(CancellationToken.None); | ||
|
|
||
| }, stoppingToken); | ||
| } | ||
| catch (Exception exception) | ||
| { | ||
| _logger.LogCritical(exception, $"Critical error occured in {nameof(ConsumerPostal)}."); | ||
| _hostApplicationLifetime.StopApplication(); | ||
| throw; | ||
| } | ||
| } | ||
| } | ||
| } |
61 changes: 61 additions & 0 deletions
61
src/StreetNameRegistry.Consumer.Read.Postal/ConsumerPostalContext.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| namespace StreetNameRegistry.Consumer.Read.Postal | ||
| { | ||
| using System; | ||
| using System.IO; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Be.Vlaanderen.Basisregisters.ProjectionHandling.Runner; | ||
| using Be.Vlaanderen.Basisregisters.ProjectionHandling.Runner.MigrationExtensions; | ||
| using StreetNameRegistry.Infrastructure; | ||
| using Microsoft.EntityFrameworkCore; | ||
| using Microsoft.EntityFrameworkCore.Design; | ||
| using Microsoft.Extensions.Configuration; | ||
|
|
||
| public class ConsumerPostalContext : RunnerDbContext<ConsumerPostalContext> | ||
| { | ||
| public DbSet<PostalConsumerItem> PostalConsumerItems { get; set; } | ||
|
|
||
| // This needs to be here to please EF | ||
| public ConsumerPostalContext() | ||
| { } | ||
|
|
||
| // This needs to be DbContextOptions<T> for Autofac! | ||
| public ConsumerPostalContext(DbContextOptions<ConsumerPostalContext> options) | ||
| : base(options) | ||
| { } | ||
|
|
||
| public override string ProjectionStateSchema => Schema.ConsumerReadPostal; | ||
| } | ||
|
|
||
| public class ConsumerContextFactory : IDesignTimeDbContextFactory<ConsumerPostalContext> | ||
| { | ||
| public ConsumerPostalContext CreateDbContext(string[] args) | ||
| { | ||
| const string migrationConnectionStringName = "ConsumerPostalAdmin"; | ||
|
|
||
| var configuration = new ConfigurationBuilder() | ||
| .SetBasePath(Directory.GetCurrentDirectory()) | ||
| .AddJsonFile("appsettings.json", optional: false, reloadOnChange: false) | ||
| .AddJsonFile($"appsettings.{Environment.MachineName.ToLowerInvariant()}.json", optional: true, reloadOnChange: false) | ||
| .AddEnvironmentVariables() | ||
| .Build(); | ||
|
|
||
| var builder = new DbContextOptionsBuilder<ConsumerPostalContext>(); | ||
|
|
||
| var connectionString = configuration.GetConnectionString(migrationConnectionStringName); | ||
| if (string.IsNullOrEmpty(connectionString)) | ||
| throw new InvalidOperationException($"Could not find a connection string with name '{migrationConnectionStringName}'"); | ||
|
|
||
| builder | ||
| .UseSqlServer(connectionString, sqlServerOptions => | ||
| { | ||
| sqlServerOptions.EnableRetryOnFailure(); | ||
| sqlServerOptions.MigrationsHistoryTable(MigrationTables.ConsumerReadPostal, Schema.ConsumerReadPostal); | ||
| sqlServerOptions.UseNetTopologySuite(); | ||
| }) | ||
| .UseExtendedSqlServerMigrations(); | ||
|
|
||
| return new ConsumerPostalContext(builder.Options); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| FROM mcr.microsoft.com/dotnet/runtime-deps:6.0.3 | ||
|
|
||
| # create group & user | ||
| RUN addgroup --gid 1000 --system app && adduser --uid 1000 -system app --gid 1000 | ||
|
|
||
| # create work dir and set permissions as WORKDIR sets permissions as root | ||
| RUN mkdir /app && chown -R app:app /app | ||
| WORKDIR /app | ||
|
|
||
| LABEL maintainer "Digitaal Vlaanderen <[email protected]>" | ||
| LABEL registry="building-registry" | ||
|
|
||
| COPY / /app | ||
| WORKDIR /app | ||
|
|
||
| RUN apt-get update && \ | ||
| apt-get install curl jq -y && \ | ||
| chmod +x ./init.sh | ||
|
|
||
| # switch to created user | ||
| USER app | ||
|
|
||
| ENTRYPOINT ["./init.sh"] |
60 changes: 60 additions & 0 deletions
60
src/StreetNameRegistry.Consumer.Read.Postal/Infrastructure/MigrationsHelper.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| namespace StreetNameRegistry.Consumer.Read.Postal.Infrastructure | ||
| { | ||
| using System; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using StreetNameRegistry.Infrastructure; | ||
| using Microsoft.Data.SqlClient; | ||
| using Microsoft.EntityFrameworkCore; | ||
| using Microsoft.Extensions.Logging; | ||
| using Polly; | ||
|
|
||
| public class MigrationsLogger { } | ||
|
|
||
| public static class MigrationsHelper | ||
| { | ||
| public static Task RunAsync( | ||
| string connectionString, | ||
| ILoggerFactory loggerFactory, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| var logger = loggerFactory.CreateLogger<MigrationsLogger>(); | ||
|
|
||
| return Policy | ||
| .Handle<SqlException>() | ||
| .WaitAndRetryAsync( | ||
| 5, | ||
| retryAttempt => | ||
| { | ||
| var value = Math.Pow(2, retryAttempt) / 4; | ||
| var randomValue = new Random().Next((int)value * 3, (int)value * 5); | ||
| logger?.LogInformation("Retrying after {Seconds} seconds...", randomValue); | ||
| return TimeSpan.FromSeconds(randomValue); | ||
| }) | ||
| .ExecuteAsync(async ct => | ||
| { | ||
| logger?.LogInformation("Running EF Migrations."); | ||
| await RunInternal(connectionString, loggerFactory, ct); | ||
| }, | ||
| cancellationToken); | ||
| } | ||
|
|
||
| private static async Task RunInternal(string connectionString, ILoggerFactory loggerFactory, CancellationToken cancellationToken) | ||
| { | ||
| var migratorOptions = new DbContextOptionsBuilder<ConsumerPostalContext>() | ||
| .UseSqlServer( | ||
| connectionString, | ||
| sqlServerOptions => | ||
| { | ||
| sqlServerOptions.EnableRetryOnFailure(); | ||
| sqlServerOptions.MigrationsHistoryTable(MigrationTables.ConsumerReadPostal, Schema.ConsumerReadPostal); | ||
| sqlServerOptions.UseNetTopologySuite(); | ||
| }); | ||
|
|
||
| migratorOptions = migratorOptions.UseLoggerFactory(loggerFactory); | ||
|
|
||
| await using var migrator = new ConsumerPostalContext(migratorOptions.Options); | ||
| await migrator.Database.MigrateAsync(cancellationToken); | ||
| } | ||
| } | ||
| } |
69 changes: 69 additions & 0 deletions
69
src/StreetNameRegistry.Consumer.Read.Postal/Infrastructure/Modules/ConsumerPostalModule.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| namespace StreetNameRegistry.Consumer.Read.Postal.Infrastructure.Modules | ||
| { | ||
| using System; | ||
| using Autofac; | ||
| using Be.Vlaanderen.Basisregisters.DataDog.Tracing.Sql.EntityFrameworkCore; | ||
| using StreetNameRegistry.Infrastructure; | ||
| using Microsoft.Data.SqlClient; | ||
| using Microsoft.EntityFrameworkCore; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| public class ConsumerPostalModule : Module | ||
| { | ||
| public ConsumerPostalModule( | ||
| IConfiguration configuration, | ||
| IServiceCollection services, | ||
| ILoggerFactory loggerFactory, | ||
| ServiceLifetime serviceLifetime = ServiceLifetime.Scoped) | ||
| { | ||
| var logger = loggerFactory.CreateLogger<ConsumerPostalModule>(); | ||
| var connectionString = configuration.GetConnectionString("ConsumerPostal"); | ||
|
|
||
| var hasConnectionString = !string.IsNullOrWhiteSpace(connectionString); | ||
| if (hasConnectionString) | ||
| { | ||
| RunOnSqlServer(configuration, services, serviceLifetime, loggerFactory, connectionString); | ||
| } | ||
| else | ||
| { | ||
| RunInMemoryDb(services, loggerFactory, logger); | ||
| } | ||
| } | ||
|
|
||
| private static void RunOnSqlServer( | ||
| IConfiguration configuration, | ||
| IServiceCollection services, | ||
| ServiceLifetime serviceLifetime, | ||
| ILoggerFactory loggerFactory, | ||
| string consumerProjectionsConnectionString) | ||
| { | ||
| services | ||
| .AddScoped(s => new TraceDbConnection<ConsumerPostalContext>( | ||
| new SqlConnection(consumerProjectionsConnectionString), | ||
| configuration["DataDog:ServiceName"])) | ||
| .AddDbContext<ConsumerPostalContext>((provider, options) => options | ||
| .UseLoggerFactory(loggerFactory) | ||
| .UseSqlServer(provider.GetRequiredService<TraceDbConnection<ConsumerPostalContext>>(), sqlServerOptions => | ||
| { | ||
| sqlServerOptions.EnableRetryOnFailure(); | ||
| sqlServerOptions.MigrationsHistoryTable(MigrationTables.ConsumerReadPostal, Schema.ConsumerReadPostal); | ||
| sqlServerOptions.UseNetTopologySuite(); | ||
| }), serviceLifetime); | ||
| } | ||
|
|
||
| private static void RunInMemoryDb( | ||
| IServiceCollection services, | ||
| ILoggerFactory loggerFactory, | ||
| ILogger logger) | ||
| { | ||
| services | ||
| .AddDbContext<ConsumerPostalContext>(options => options | ||
| .UseLoggerFactory(loggerFactory) | ||
| .UseInMemoryDatabase(Guid.NewGuid().ToString(), sqlServerOptions => { })); | ||
|
|
||
| logger.LogWarning("Running InMemory for {Context}!", nameof(ConsumerPostalContext)); | ||
| } | ||
| } | ||
| } |
36 changes: 36 additions & 0 deletions
36
src/StreetNameRegistry.Consumer.Read.Postal/Infrastructure/Modules/LoggingModule.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| namespace StreetNameRegistry.Consumer.Read.Postal.Infrastructure.Modules | ||
| { | ||
| using System; | ||
| using Autofac; | ||
| using Destructurama; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Logging; | ||
| using Serilog; | ||
| using Serilog.Debugging; | ||
|
|
||
| public class LoggingModule : Module | ||
| { | ||
| public LoggingModule( | ||
| IConfiguration configuration, | ||
| IServiceCollection services) | ||
| { | ||
| SelfLog.Enable(Console.WriteLine); | ||
|
|
||
| Log.Logger = new LoggerConfiguration() | ||
| .ReadFrom.Configuration(configuration) | ||
| .Enrich.FromLogContext() | ||
| .Enrich.WithMachineName() | ||
| .Enrich.WithThreadId() | ||
| .Enrich.WithEnvironmentUserName() | ||
| .Destructure.JsonNetTypes() | ||
| .CreateLogger(); | ||
|
|
||
| services.AddLogging(l => | ||
| { | ||
| l.ClearProviders(); | ||
| l.AddSerilog(Log.Logger); | ||
| }); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.