diff --git a/DragonFruit.OnionFruit.Web.Worker/Worker.cs b/DragonFruit.OnionFruit.Web.Worker/Worker.cs index 8c1263f..51245d1 100644 --- a/DragonFruit.OnionFruit.Web.Worker/Worker.cs +++ b/DragonFruit.OnionFruit.Web.Worker/Worker.cs @@ -47,8 +47,8 @@ private async Task PerformUpdate() { using var scope = _ssf.CreateScope(); + var dataSourceUpdated = false; var sourceInstances = new Dictionary(); - var sourcesTypesToUse = new HashSet(); var redis = scope.ServiceProvider.GetRequiredService().GetDatabase(); var nextVersion = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); @@ -63,59 +63,38 @@ private async Task PerformUpdate() _stopwatch.Restart(); - // populate list with the sources that have been updated since last check + // create instances of all source types and set switch if data source has been updated foreach (var sourceType in _descriptors.SelectMany(x => x.SourceTypes).Distinct()) { var source = (IDataSource)ActivatorUtilities.CreateInstance(scope.ServiceProvider, sourceType); - if (await source.HasDataChanged(lastVersion).ConfigureAwait(false)) - { - sourcesTypesToUse.Add(sourceType); - } - sourceInstances[sourceType] = source; + dataSourceUpdated |= await source.HasDataChanged(lastVersion).ConfigureAwait(false); } - if (!sourcesTypesToUse.Any()) + if (!dataSourceUpdated) { _stopwatch.Stop(); _logger.LogInformation("No sources have been updated. Generator execution skipped (after {ts}).", _stopwatch.Elapsed); return; } - // add any source from full list if a generator that needs an updated source also needs one of the old sources - var generatorsToUse = _descriptors.Where(x => sourcesTypesToUse.Overlaps(x.SourceTypes)).ToList(); - - foreach (var sourceType in generatorsToUse.SelectMany(x => x.SourceTypes)) - { - // hashset doesn't have addrange... - sourcesTypesToUse.Add(sourceType); - } - - try - { - _logger.LogInformation("Collecting data for {sources}", string.Join(", ", sourcesTypesToUse.Select(x => x.Name))); - await Task.WhenAll(sourcesTypesToUse.Select(x => sourceInstances[x].CollectData())).ConfigureAwait(false); - } - catch (Exception e) - { - _logger.LogError(e, "Failed to collect data for one or more sources: {message}", e.Message); - return; - } + // fetch all data sources + await Task.WhenAll(sourceInstances.Select(x => x.Value.CollectData())).ConfigureAwait(false); // file sink used to store static-generated assets for uploading to s3 or saving to a local path using var fileSink = new FileSink(); - foreach (var generator in generatorsToUse) + foreach (var generatorDescriptor in _descriptors) { IDisposable disposableGeneratorInstance = null; try { - _logger.LogInformation("Running generator for {name}...", generator.OutputFormat.Name); + _logger.LogInformation("Running generator for {name}...", generatorDescriptor.OutputFormat.Name); - var instanceSources = generator.SourceTypes.Select(x => (object)sourceInstances[x]).ToArray(); - var generatorInstance = (IDatabaseGenerator)ActivatorUtilities.CreateInstance(scope.ServiceProvider, generator.OutputFormat, instanceSources); + var instanceSources = generatorDescriptor.SourceTypes.Select(x => (object)sourceInstances[x]).ToArray(); + var generatorInstance = (IDatabaseGenerator)ActivatorUtilities.CreateInstance(scope.ServiceProvider, generatorDescriptor.OutputFormat, instanceSources); disposableGeneratorInstance = generatorInstance as IDisposable; @@ -124,7 +103,7 @@ private async Task PerformUpdate() } catch (Exception e) { - _logger.LogError(e, "Database Generator {x} has failed: {err}", generator.OutputFormat.Name, e.Message); + _logger.LogError(e, "Database Generator {x} has failed: {err}", generatorDescriptor.OutputFormat.Name, e.Message); } finally { @@ -187,7 +166,7 @@ private IReadOnlyCollection GetDescriptors(IConfiguration c return listing; } - private IReadOnlyCollection GetExporters(IConfiguration config) + private static IReadOnlyCollection GetExporters(IConfiguration config) { var exporters = new List(); @@ -216,7 +195,7 @@ private IReadOnlyCollection GetExporters(IConfiguration config) Task IHostedService.StartAsync(CancellationToken cancellationToken) { _workerTimer?.Dispose(); - _workerTimer = new Timer(_ => PerformUpdate(), null, TimeSpan.Zero, TimeSpan.FromHours(6)); + _workerTimer = new Timer(_ => PerformUpdate(), null, TimeSpan.Zero, TimeSpan.FromHours(12)); return Task.CompletedTask; }