From 14b107e4598c2553333f4ac59ad439507034cfe9 Mon Sep 17 00:00:00 2001 From: halgari Date: Thu, 24 Oct 2024 14:49:26 +0100 Subject: [PATCH] Add a bunch of fixes around empty transactions (that broke the UI). Now we no longer advance the TxId if no datoms were inserted as part of the tx. Also includes some read benchmarks I was working on --- .../Benchmarks/ColdStartBenchmarks.cs | 98 +++++++++++++++++++ .../Program.cs | 2 +- src/NexusMods.MnemonicDB.Abstractions/IDb.cs | 5 + .../Caching/IndexSegmentCache.cs | 9 +- src/NexusMods.MnemonicDB/Db.cs | 5 + .../Storage/DatomStore.cs | 5 +- .../ComplexModelTests.cs | 8 ++ 7 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ColdStartBenchmarks.cs diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ColdStartBenchmarks.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ColdStartBenchmarks.cs new file mode 100644 index 0000000..d205052 --- /dev/null +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ColdStartBenchmarks.cs @@ -0,0 +1,98 @@ +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; +using NexusMods.MnemonicDB.Abstractions.IndexSegments; +using NexusMods.MnemonicDB.TestModel; +using NexusMods.Paths; + +namespace NexusMods.MnemonicDB.Benchmarks.Benchmarks; + +[MemoryDiagnoser] +public class ColdStartBenchmarks : ABenchmark +{ + [GlobalSetup] + public async ValueTask GlobalSetup() + { + await InitializeAsync(); + await InsertData(); + } + + private async Task InsertData() + { + foreach (var loadout in Enumerable.Range(0, 10)) + { + using var tx = Connection.BeginTransaction(); + var loadoutEntity = new Loadout.New(tx) + { + Name = $"Loadout {loadout}" + }; + foreach (var mod in Enumerable.Range(0, 1000)) + { + var modEntity = new Mod.New(tx) + { + Name = $"Mod {mod}", + Source = new System.Uri($"http://mod{mod}.com"), + LoadoutId = loadoutEntity, + OptionalHash = Hashing.xxHash3.Hash.FromLong(0) + }; + foreach (var file in Enumerable.Range(0, 100)) + { + _ = new File.New(tx) + { + Path = $"File {file}", + ModId = modEntity, + Size = Size.FromLong(file), + Hash = Hashing.xxHash3.Hash.FromLong(file) + }; + } + } + await tx.Commit(); + } + } + + [IterationSetup] + public void IterationSetup() + { + Connection.Db.ClearIndexCache(); + } + + [Benchmark] + public Size TotalSizeEAVT() + { + var loadout = Loadout.FindByName(Connection.Db, "Loadout 5").First(); + var totalSize = Size.FromLong(0); + + foreach (var mod in loadout.Mods) + { + foreach (var file in mod.Files) + { + totalSize += file.Size; + } + } + + return totalSize; + } + + + [Benchmark] + public Size TotalSizeAEVT() + { + var loadoutId = Connection.Db.Datoms(Loadout.Name, "Loadout 5").First(); + var modIds = Connection.Db.Datoms(Mod.LoadoutId, loadoutId.E).Select(e => e.E).ToHashSet(); + var fileIds = Connection.Db.Datoms(File.ModId).ToLookup(d => ValueTag.Reference.Read(d.ValueSpan), d => d.E); + var fileSizes = Connection.Db.Datoms(File.Size) + .ToDictionary(d => d.E, d => Size.From(ValueTag.UInt64.Read(d.ValueSpan))); + + var totalSize = Size.FromLong(0); + foreach (var modId in modIds) + { + foreach (var fileId in fileIds[modId]) + { + totalSize += fileSizes[fileId]; + } + } + return totalSize; + } +} diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs index bd31319..21b7931 100644 --- a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs @@ -29,7 +29,7 @@ #else -BenchmarkRunner.Run(config: DefaultConfig.Instance.WithOption(ConfigOptions.DisableOptimizationsValidator, true)); +BenchmarkRunner.Run(); #endif diff --git a/src/NexusMods.MnemonicDB.Abstractions/IDb.cs b/src/NexusMods.MnemonicDB.Abstractions/IDb.cs index c34f918..3b77c1f 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IDb.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IDb.cs @@ -85,4 +85,9 @@ public interface IDb : IEquatable /// TReturn AnalyzerData() where TAnalyzer : IAnalyzer; + + /// + /// Clears the internal cache of the database. + /// + void ClearIndexCache(); } diff --git a/src/NexusMods.MnemonicDB/Caching/IndexSegmentCache.cs b/src/NexusMods.MnemonicDB/Caching/IndexSegmentCache.cs index 9c128ad..9e6e187 100644 --- a/src/NexusMods.MnemonicDB/Caching/IndexSegmentCache.cs +++ b/src/NexusMods.MnemonicDB/Caching/IndexSegmentCache.cs @@ -283,5 +283,12 @@ private void UpdateEntry(CacheKey key, IndexSegment segment) return; } } - + + /// + /// Clears the cache. + /// + public void Clear() + { + _root = new CacheRoot(ImmutableDictionary.Empty); + } } diff --git a/src/NexusMods.MnemonicDB/Db.cs b/src/NexusMods.MnemonicDB/Db.cs index d91fcd4..0a1fae3 100644 --- a/src/NexusMods.MnemonicDB/Db.cs +++ b/src/NexusMods.MnemonicDB/Db.cs @@ -114,6 +114,11 @@ TReturn IDb.AnalyzerData() throw new KeyNotFoundException($"Analyzer {typeof(TAnalyzer).Name} not found"); } + public void ClearIndexCache() + { + _cache.Clear(); + } + public IndexSegment Datoms(IWritableAttribute attribute, TValue value) { return Datoms(SliceDescriptor.Create(attribute, value, AttributeCache)); diff --git a/src/NexusMods.MnemonicDB/Storage/DatomStore.cs b/src/NexusMods.MnemonicDB/Storage/DatomStore.cs index 7a7cda2..ff44a42 100644 --- a/src/NexusMods.MnemonicDB/Storage/DatomStore.cs +++ b/src/NexusMods.MnemonicDB/Storage/DatomStore.cs @@ -103,7 +103,8 @@ public DatomStore( Backend.Init(settings.Path); - if (bootstrap) Bootstrap(); + if (bootstrap) + Bootstrap(); } /// @@ -308,7 +309,7 @@ private StoreResult Log(IInternalTxFunctionImpl pendingTransaction) return new StoreResult { - AssignedTxId = _thisTx, + AssignedTxId = _asOfTx, Remaps = _remaps.ToFrozenDictionary(), Snapshot = CurrentSnapshot }; diff --git a/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs b/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs index 55a4521..7eafcf3 100644 --- a/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs +++ b/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs @@ -173,7 +173,15 @@ public async Task CanRestartStorage(int modCount, int filesPerMod, int extraFile await extraTx.Commit(); Logger.LogInformation("Restarting storage"); + Connection.Db.RecentlyAdded.Should().NotBeEmpty("the last transaction added data"); + + var lastTxId = Connection.TxId; await RestartDatomStore(); + + Connection.TxId.Should().Be(lastTxId, "the transaction id should be the same after a restart"); + Connection.Db.BasisTxId.Should().Be(lastTxId, "the basis transaction id should be the same after a restart"); + + Connection.Db.RecentlyAdded.Should().NotBeEmpty("the restarted database should populate the recently added"); Logger.LogInformation("Storage restarted");