From 452120a1fa0482bf3ce04f58fa3e23c0505c87e3 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 19 Nov 2024 14:14:58 +0700 Subject: [PATCH 01/30] Add test that shows bug and also add bugfix The second sync, that should do nothing, was importing the seed data, including the parts of speech and semantic domains that weren't in fwdata after the first sync. --- .../FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs | 2 +- backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index 0b0443e0d..822da9c54 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -58,7 +58,7 @@ public async Task InitializeAsync() if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); Directory.CreateDirectory(crdtProjectsFolder); var crdtProject = await _services.ServiceProvider.GetRequiredService() - .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: true)); + .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: false)); CrdtApi = (CrdtMiniLcmApi) await _services.ServiceProvider.OpenCrdtProject(crdtProject); } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index 752d77fa2..10ac23c21 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -87,6 +87,17 @@ public async Task FirstSyncJustDoesAnImport() .For(e => e.ComplexForms).Exclude(c => c.Id)); } + [Fact] + public async Task SecondSyncDoesNothing() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); + } + [Fact] public static async Task SyncFailsWithMismatchedProjectIds() { From 622856de5b3cac5b5b635791cd76dc2261f275d5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 19 Nov 2024 17:10:17 +0700 Subject: [PATCH 02/30] Attempt to create unit test to repro sena-3 issue This was not successful at reproducing the issue we encounter syncing sena-3, because this test is passing. I'll keep working at it. --- .../FwLiteProjectSync.Tests/SyncTests.cs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index 10ac23c21..bbb89c7a0 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -149,6 +149,55 @@ await crdtApi.CreateEntry(new Entry() .For(e => e.ComplexForms).Exclude(c => c.Id)); } + [Fact] + public async Task CreatingAComplexEntryInFwDataSyncsWithoutIssue() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + + var hat = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Hat" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Hat" } }, } + ] + }); + var stand = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Stand" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Stand" } }, } + ] + }); + var hatstand = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Hatstand" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Hatstand" } }, } + ], + }); + var component1 = ComplexFormComponent.FromEntries(hatstand, hat); + var component2 = ComplexFormComponent.FromEntries(hatstand, stand); + hatstand.Components = [component1, component2]; + await _syncService.Sync(crdtApi, fwdataApi); + + var crdtEntries = await crdtApi.GetEntries().ToArrayAsync(); + var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); + crdtEntries.Should().BeEquivalentTo(fwdataEntries, + options => options.For(e => e.Components).Exclude(c => c.Id) + .For(e => e.ComplexForms).Exclude(c => c.Id)); + + // Sync again, ensure no problems or changes + var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); + } + + [Fact] public async Task PartsOfSpeechSyncBothWays() { From cfcabd4fe402f6f1de292342c99f0adb176af81b Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 22 Nov 2024 13:41:13 +0700 Subject: [PATCH 03/30] write failing test --- .../FwLiteProjectSync.Tests/EntrySyncTests.cs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs index d1b264839..d7a631759 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs @@ -105,4 +105,27 @@ public async Task CanChangeComplexFormTypeViaSync() actual.Should().NotBeNull(); actual.Should().BeEquivalentTo(after, options => options); } + + [Fact] + public async Task CanCreateAComplexFormAndItsComponentInOneSync() + { + //ensure they are synced so a real sync will happen when we want it to + await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + + var complexFormEntry = await _fixture.CrdtApi.CreateEntry(new() { LexemeForm = { { "en", "complexForm" } } }); + var componentEntry = await _fixture.CrdtApi.CreateEntry(new() + { + LexemeForm = { { "en", "component" } }, + ComplexForms = + [ + new ComplexFormComponent() + { + ComplexFormEntryId = complexFormEntry.Id, ComponentEntryId = Guid.Empty + } + ] + }); + + //one of the entries will be created first, it will try to create the reference to the other but it won't exist yet + await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + } } From 429b74ba219f1d98fa9af45f31bb0d8d461df0c8 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 22 Nov 2024 14:05:55 +0700 Subject: [PATCH 04/30] change Entry.Sync to create entries first without complex forms, then in a second pass update those entries so they contain those complex forms --- .../Api/UpdateProxy/UpdateEntryProxy.cs | 2 +- backend/FwLite/MiniLcm/Models/Entry.cs | 7 +- .../MiniLcm/SyncHelpers/DiffCollection.cs | 68 +++++++++++++++++++ .../FwLite/MiniLcm/SyncHelpers/EntrySync.cs | 11 +-- 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs index 503222fca..d0defe8ad 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs @@ -4,7 +4,7 @@ namespace FwDataMiniLcmBridge.Api.UpdateProxy; -public class UpdateEntryProxy : Entry +public record UpdateEntryProxy : Entry { private readonly ILexEntry _lcmEntry; private readonly FwDataMiniLcmApi _lexboxLcmApi; diff --git a/backend/FwLite/MiniLcm/Models/Entry.cs b/backend/FwLite/MiniLcm/Models/Entry.cs index 191db2753..7e7239ca5 100644 --- a/backend/FwLite/MiniLcm/Models/Entry.cs +++ b/backend/FwLite/MiniLcm/Models/Entry.cs @@ -1,6 +1,6 @@ namespace MiniLcm.Models; -public class Entry : IObjectWithId +public record Entry : IObjectWithId { public Guid Id { get; set; } public DateTimeOffset? DeletedAt { get; set; } @@ -68,6 +68,11 @@ public Guid[] GetReferences() public void RemoveReference(Guid id, DateTimeOffset time) { } + + public Entry WithoutEntryRefs() + { + return this with { Components = [], ComplexForms = [] }; + } } public class Variants diff --git a/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs b/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs index 2cc251f2b..33bda1e46 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs @@ -4,6 +4,73 @@ namespace MiniLcm.SyncHelpers; public static class DiffCollection { + /// + /// Diffs a list, for new items calls add, it will then call update for the item returned from the add, using that as the before item for the replace call + /// + /// + /// + /// + /// + /// api, value, return value to be used as the before item for the replace call + /// + /// api, before, after is the parameter order + /// + /// + /// + public static async Task DiffAddThenUpdate( + IMiniLcmApi api, + IList before, + IList after, + Func identity, + Func> add, + Func> remove, + Func> replace) where TId : notnull + { + var changes = 0; + var afterEntriesDict = after.ToDictionary(identity); + + foreach (var beforeEntry in before) + { + if (afterEntriesDict.TryGetValue(identity(beforeEntry), out var afterEntry)) + { + changes += await replace(api, beforeEntry, afterEntry); + } + else + { + changes += await remove(api, beforeEntry); + } + + afterEntriesDict.Remove(identity(beforeEntry)); + } + + var postAddUpdates = new List<(T created, T after)>(afterEntriesDict.Values.Count); + foreach (var value in afterEntriesDict.Values) + { + changes++; + postAddUpdates.Add((await add(api, value), value)); + } + foreach ((T createdItem, T afterItem) in postAddUpdates) + { + //todo this may do a lot more work than it needs to, eg sense will be created during add, but they will be checked again here when we know they didn't change + await replace(api, createdItem, afterItem); + } + + return changes; + } + + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// api, before, after is the parameter order + /// + /// + /// public static async Task Diff( IMiniLcmApi api, IList before, @@ -36,6 +103,7 @@ public static async Task Diff( return changes; } + public static async Task Diff( IMiniLcmApi api, IList before, diff --git a/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs b/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs index 50fe78892..a61a7fc58 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs @@ -10,10 +10,13 @@ public static async Task Sync(Entry[] afterEntries, Entry[] beforeEntries, IMiniLcmApi api) { - Func> add = static async (api, afterEntry) => + Func> add = static async (api, afterEntry) => { - await api.CreateEntry(afterEntry); - return 1; + //create each entry without components. + //After each entry is created, then replace will be called to create those components + var entryWithoutEntryRefs = afterEntry.WithoutEntryRefs(); + await api.CreateEntry(entryWithoutEntryRefs); + return entryWithoutEntryRefs; }; Func> remove = static async (api, beforeEntry) => { @@ -21,7 +24,7 @@ public static async Task Sync(Entry[] afterEntries, return 1; }; Func> replace = static async (api, beforeEntry, afterEntry) => await Sync(afterEntry, beforeEntry, api); - return await DiffCollection.Diff(api, beforeEntries, afterEntries, add, remove, replace); + return await DiffCollection.DiffAddThenUpdate(api, beforeEntries, afterEntries, entry => entry.Id, add, remove, replace); } public static async Task Sync(Entry afterEntry, Entry beforeEntry, IMiniLcmApi api) From ceb7ade57da7d5ba85d44420ae9409c7c99ebb58 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 22 Nov 2024 14:20:50 +0700 Subject: [PATCH 05/30] Add integration test for syncing sena-3 --- .../Fixtures/LexboxConfig.cs | 14 +++ .../Fixtures/MercurialTestHelper.cs | 31 +++++ .../Fixtures/Sena3SyncFixture.cs | 114 ++++++++++++++++++ .../Fixtures/SyncFixture.cs | 11 +- .../FwLiteProjectSync.Tests.csproj | 11 ++ .../FwLiteProjectSync.Tests/Sena3SyncTests.cs | 100 +++++++++++++++ 6 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Fixtures/LexboxConfig.cs create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/LexboxConfig.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/LexboxConfig.cs new file mode 100644 index 000000000..7385cbbc8 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/LexboxConfig.cs @@ -0,0 +1,14 @@ +using System.ComponentModel.DataAnnotations; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public class LexboxConfig +{ + [Required, Url, RegularExpression(@"^.+/$", ErrorMessage = "Must end with '/'")] + public required string LexboxUrl { get; set; } + public string HgWebUrl => $"{LexboxUrl}hg/"; + [Required] + public required string LexboxUsername { get; set; } + [Required] + public required string LexboxPassword { get; set; } +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs new file mode 100644 index 000000000..afccd122a --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs @@ -0,0 +1,31 @@ +using SIL.CommandLineProcessing; +using SIL.PlatformUtilities; +using SIL.Progress; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public static class MercurialTestHelper +{ + public static string HgCommand => + Path.Combine("Mercurial", Platform.IsWindows ? "hg.exe" : "hg"); + + private static string RunHgCommand(string repoPath, string args) + { + var result = CommandLineRunner.Run(HgCommand, args, repoPath, 120, new NullProgress()); + if (result.ExitCode == 0) return result.StandardOutput; + throw new Exception( + $"hg {args} failed.\nStdOut: {result.StandardOutput}\nStdErr: {result.StandardError}"); + + } + + public static void HgClean(string repoPath, string exclude) + { + RunHgCommand(repoPath, $"purge --no-confirm --exclude {exclude}"); + } + + public static void HgUpdate(string repoPath, string rev) + { + RunHgCommand(repoPath, $"update \"{rev}\""); + } +} + diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs new file mode 100644 index 000000000..f504a8dd2 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs @@ -0,0 +1,114 @@ +using System.IO.Compression; +using System.Net.Http.Json; +using System.Runtime.CompilerServices; +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.Api; +using FwDataMiniLcmBridge.LcmUtils; +using FwDataMiniLcmBridge.Tests.Fixtures; +using LcmCrdt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MiniLcm; +using SIL.IO; +using SIL.Progress; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public class Sena3Fixture : IAsyncLifetime +{ + private readonly SyncFixture syncFixture; + public CrdtFwdataProjectSyncService SyncService => + Services.GetRequiredService(); + public IServiceProvider Services => syncFixture.Services; + private readonly LexboxConfig lexboxConfig; + private readonly HttpClient http; + public CrdtMiniLcmApi CrdtApi { get; set; } = null!; + public FwDataMiniLcmApi FwDataApi { get; set; } = null!; + private bool AlreadyLoggedIn { get; set; } = false; + + public Sena3Fixture() + { + syncFixture = SyncFixture.Create(services => + { + services.AddOptions() + .BindConfiguration("LexboxConfig") + .ValidateDataAnnotations() + .ValidateOnStart(); + // TODO: How do I set default values if and only if they're not already set (e.g., via environment variables)? + services.Configure(c => + { + c.LexboxUrl = "http://localhost/"; + c.LexboxUsername = "admin"; + c.LexboxPassword = "pass"; + }); + }, nameof(Sena3Fixture)); // TODO: Or create the project name in the constructor rather than in InitializeAsync, then pass it in here + lexboxConfig = Services.GetRequiredService>().Value; + var factory = Services.GetRequiredService(); + http = factory.CreateClient(nameof(Sena3Fixture)); + } + + public async Task InitializeAsync() + { + var sena3MasterCopy = await DownloadSena3(); + var projectName = "sena-3_" + Guid.NewGuid().ToString("N"); + + var projectsFolder = Services.GetRequiredService>().Value + .ProjectsFolder; + if (Path.Exists(projectsFolder)) Directory.Delete(projectsFolder, true); + Directory.CreateDirectory(projectsFolder); + var fwDataProject = new FwDataProject(projectName, projectsFolder); + var fwDataProjectPath = Path.Combine(fwDataProject.ProjectsPath, fwDataProject.Name); + DirectoryHelper.Copy(sena3MasterCopy, fwDataProjectPath); + File.Move(Path.Combine(fwDataProjectPath, "sena-3.fwdata"), fwDataProject.FilePath); + + Services.GetRequiredService().LoadCache(fwDataProject); + FwDataApi = Services.GetRequiredService().GetFwDataMiniLcmApi(projectName, false); + + var crdtProjectsFolder = + Services.GetRequiredService>().Value.ProjectPath; + if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); + Directory.CreateDirectory(crdtProjectsFolder); + var crdtProject = await Services.GetRequiredService() + .CreateProject(new(projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: false)); + CrdtApi = (CrdtMiniLcmApi) await Services.OpenCrdtProject(crdtProject); + } + + public async Task DisposeAsync() + { + await syncFixture.DisposeAsync(); + } + + public async Task DownloadProjectBackupStream(string code) + { + var backupUrl = new Uri($"{lexboxConfig.LexboxUrl}api/project/backupProject/{code}"); + var result = await http.GetAsync(backupUrl); + return await result.Content.ReadAsStreamAsync(); + } + + public async Task LoginAs(string lexboxUsername, string lexboxPassword) + { + if (AlreadyLoggedIn) return; + await http.PostAsync($"{lexboxConfig.LexboxUrl}api/login", JsonContent.Create(new { EmailOrUsername=lexboxUsername, Password=lexboxPassword })); + AlreadyLoggedIn = true; + } + + public async Task DownloadSena3() + { + + var tempFolder = Path.Combine(Path.GetTempPath(), nameof(Sena3Fixture)); + var sena3MasterCopy = Path.Combine(tempFolder, "sena-3"); + if (!Directory.Exists(sena3MasterCopy) || !File.Exists(Path.Combine(sena3MasterCopy, "sena-3.fwdata"))) + { + await LoginAs(lexboxConfig.LexboxUsername, lexboxConfig.LexboxPassword); + Directory.CreateDirectory(sena3MasterCopy); + var zipStream = await DownloadProjectBackupStream("sena-3"); + ZipFile.ExtractToDirectory(zipStream, sena3MasterCopy); + MercurialTestHelper.HgUpdate(sena3MasterCopy, "tip"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(new NullProgress(), false, Path.Combine(sena3MasterCopy, "sena-3.fwdata")); + MercurialTestHelper.HgClean(sena3MasterCopy, "sena-3.fwdata"); + } + return sena3MasterCopy; + } +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index 822da9c54..788605001 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -24,7 +24,9 @@ public class SyncFixture : IAsyncLifetime public static SyncFixture Create([CallerMemberName] string projectName = "") => new(projectName); - private SyncFixture(string projectName) + public static SyncFixture Create(Action extraServiceConfiguration, [CallerMemberName] string projectName = "") => new(projectName, extraServiceConfiguration); + + private SyncFixture(string projectName, Action? extraServiceConfiguration = null) { _projectName = projectName; var crdtServices = new ServiceCollection() @@ -34,9 +36,9 @@ private SyncFixture(string projectName) .AddFwLiteProjectSync() .Configure(c => c.ProjectsFolder = Path.Combine(".", _projectName, "FwData")) .Configure(c => c.ProjectPath = Path.Combine(".", _projectName, "LcmCrdt")) - .AddLogging(builder => builder.AddDebug()) - .BuildServiceProvider(); - _services = crdtServices.CreateAsyncScope(); + .AddLogging(builder => builder.AddDebug()); + if (extraServiceConfiguration is not null) extraServiceConfiguration(crdtServices); + _services = crdtServices.BuildServiceProvider().CreateAsyncScope(); } public SyncFixture(): this("sena-3_" + Guid.NewGuid().ToString("N")) @@ -64,6 +66,7 @@ public async Task InitializeAsync() public async Task DisposeAsync() { + // CAUTION: Do not assume that InitializeAsync() has been called await _services.DisposeAsync(); } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj b/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj index ea6572e46..fa5e2b83c 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj +++ b/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj @@ -7,6 +7,7 @@ false true + $(MSBuildProjectDirectory) @@ -28,6 +29,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + @@ -41,4 +44,12 @@ + + + + + + + + diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs new file mode 100644 index 000000000..929bdb471 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -0,0 +1,100 @@ +using FluentAssertions.Equivalency; +using FwLiteProjectSync.Tests.Fixtures; +using LcmCrdt; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using MiniLcm; +using MiniLcm.Models; +using SystemTextJsonPatch; + +namespace FwLiteProjectSync.Tests; + +public class Sena3SyncTests : IClassFixture, IAsyncLifetime +{ + private readonly Sena3Fixture _fixture; + private readonly CrdtFwdataProjectSyncService _syncService; + + private readonly Guid _complexEntryId = Guid.NewGuid(); + private Entry _testEntry = new Entry + { + Id = Guid.NewGuid(), + LexemeForm = { Values = { { "en", "Apple" } } }, + Note = { Values = { { "en", "this is a test note" } } }, + Senses = + [ + new Sense + { + Gloss = { Values = { { "en", "Apple" } } }, + Definition = { Values = { { "en", "a round fruit with a hard, crisp skin" } } }, + ExampleSentences = + [ + new ExampleSentence { Sentence = { Values = { { "en", "I went to the store to buy an apple." } } } } + ] + } + ] + }; + + public async Task InitializeAsync() + { + await _fixture.FwDataApi.CreateEntry(_testEntry); + await _fixture.FwDataApi.CreateEntry(new Entry() + { + Id = _complexEntryId, + LexemeForm = { { "en", "Pineapple" } }, + Components = + [ + new ComplexFormComponent() + { + Id = Guid.NewGuid(), + ComplexFormEntryId = _complexEntryId, + ComplexFormHeadword = "Pineapple", + ComponentEntryId = _testEntry.Id, + ComponentHeadword = "Apple" + } + ] + }); + } + + public async Task DisposeAsync() + { + await foreach (var entry in _fixture.FwDataApi.GetEntries()) + { + await _fixture.FwDataApi.DeleteEntry(entry.Id); + } + foreach (var entry in await _fixture.CrdtApi.GetEntries().ToArrayAsync()) + { + await _fixture.CrdtApi.DeleteEntry(entry.Id); + } + } + + public Sena3SyncTests(Sena3Fixture fixture) + { + _fixture = fixture; + _syncService = _fixture.SyncService; + } + + [Fact] + public async Task FirstSena3SyncJustDoesAnImport() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + + var crdtEntries = await crdtApi.GetEntries().ToArrayAsync(); + var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); + crdtEntries.Should().BeEquivalentTo(fwdataEntries, + options => options.For(e => e.Components).Exclude(c => c.Id) + .For(e => e.ComplexForms).Exclude(c => c.Id)); + } + + [Fact] + public async Task SecondSena3SyncDoesNothing() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); + } +} From 01e27b301934a054a0b464034d3689d4a3a5329b Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 22 Nov 2024 16:00:26 +0700 Subject: [PATCH 06/30] get sena 3 sync tests working --- .../Fixtures/FwDataTestsKernel.cs | 11 +++-- .../Fixtures/Sena3SyncFixture.cs | 36 +++++++-------- .../Fixtures/SyncFixture.cs | 19 +------- .../Fixtures/TestingKernel.cs | 27 +++++++++++ .../FwLiteProjectSync.Tests/Sena3SyncTests.cs | 45 ------------------- 5 files changed, 54 insertions(+), 84 deletions(-) create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs index cc951c27d..84a55beee 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs @@ -6,13 +6,16 @@ namespace FwDataMiniLcmBridge.Tests.Fixtures; public static class FwDataTestsKernel { - public static IServiceCollection AddTestFwDataBridge(this IServiceCollection services) + public static IServiceCollection AddTestFwDataBridge(this IServiceCollection services, bool mockProjectLoader = true) { services.AddFwDataBridge(); services.AddSingleton(_ => new ConfigurationRoot([])); - services.AddSingleton(); - services.AddSingleton(sp => sp.GetRequiredService()); - services.AddSingleton(); + if (mockProjectLoader) + { + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + } return services; } } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs index f504a8dd2..5a78db2cf 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs @@ -18,10 +18,12 @@ namespace FwLiteProjectSync.Tests.Fixtures; public class Sena3Fixture : IAsyncLifetime { - private readonly SyncFixture syncFixture; + private readonly AsyncServiceScope _services; + public CrdtFwdataProjectSyncService SyncService => - Services.GetRequiredService(); - public IServiceProvider Services => syncFixture.Services; + _services.ServiceProvider.GetRequiredService(); + + public IServiceProvider Services => _services.ServiceProvider; private readonly LexboxConfig lexboxConfig; private readonly HttpClient http; public CrdtMiniLcmApi CrdtApi { get; set; } = null!; @@ -30,20 +32,20 @@ public class Sena3Fixture : IAsyncLifetime public Sena3Fixture() { - syncFixture = SyncFixture.Create(services => - { - services.AddOptions() - .BindConfiguration("LexboxConfig") - .ValidateDataAnnotations() - .ValidateOnStart(); - // TODO: How do I set default values if and only if they're not already set (e.g., via environment variables)? - services.Configure(c => + var services = new ServiceCollection() + .AddSyncServices(nameof(Sena3Fixture), false); + services.AddOptions() + .BindConfiguration("LexboxConfig") + .Configure(c => { + // TODO: How do I set default values if and only if they're not already set (e.g., via environment variables)? c.LexboxUrl = "http://localhost/"; c.LexboxUsername = "admin"; c.LexboxPassword = "pass"; - }); - }, nameof(Sena3Fixture)); // TODO: Or create the project name in the constructor rather than in InitializeAsync, then pass it in here + }) + .ValidateDataAnnotations() + .ValidateOnStart(); + _services = services.BuildServiceProvider().CreateAsyncScope(); lexboxConfig = Services.GetRequiredService>().Value; var factory = Services.GetRequiredService(); http = factory.CreateClient(nameof(Sena3Fixture)); @@ -63,8 +65,7 @@ public async Task InitializeAsync() DirectoryHelper.Copy(sena3MasterCopy, fwDataProjectPath); File.Move(Path.Combine(fwDataProjectPath, "sena-3.fwdata"), fwDataProject.FilePath); - Services.GetRequiredService().LoadCache(fwDataProject); - FwDataApi = Services.GetRequiredService().GetFwDataMiniLcmApi(projectName, false); + FwDataApi = Services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); var crdtProjectsFolder = Services.GetRequiredService>().Value.ProjectPath; @@ -77,7 +78,7 @@ public async Task InitializeAsync() public async Task DisposeAsync() { - await syncFixture.DisposeAsync(); + await _services.DisposeAsync(); } public async Task DownloadProjectBackupStream(string code) @@ -96,14 +97,13 @@ public async Task LoginAs(string lexboxUsername, string lexboxPassword) public async Task DownloadSena3() { - var tempFolder = Path.Combine(Path.GetTempPath(), nameof(Sena3Fixture)); var sena3MasterCopy = Path.Combine(tempFolder, "sena-3"); if (!Directory.Exists(sena3MasterCopy) || !File.Exists(Path.Combine(sena3MasterCopy, "sena-3.fwdata"))) { await LoginAs(lexboxConfig.LexboxUsername, lexboxConfig.LexboxPassword); Directory.CreateDirectory(sena3MasterCopy); - var zipStream = await DownloadProjectBackupStream("sena-3"); + await using var zipStream = await DownloadProjectBackupStream("sena-3"); ZipFile.ExtractToDirectory(zipStream, sena3MasterCopy); MercurialTestHelper.HgUpdate(sena3MasterCopy, "tip"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(new NullProgress(), false, Path.Combine(sena3MasterCopy, "sena-3.fwdata")); diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index 788605001..a9e324753 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -20,24 +20,14 @@ public class SyncFixture : IAsyncLifetime _services.ServiceProvider.GetRequiredService(); public IServiceProvider Services => _services.ServiceProvider; private readonly string _projectName; - private readonly MockProjectContext _projectContext = new(null); public static SyncFixture Create([CallerMemberName] string projectName = "") => new(projectName); - public static SyncFixture Create(Action extraServiceConfiguration, [CallerMemberName] string projectName = "") => new(projectName, extraServiceConfiguration); - - private SyncFixture(string projectName, Action? extraServiceConfiguration = null) + private SyncFixture(string projectName) { _projectName = projectName; var crdtServices = new ServiceCollection() - .AddLcmCrdtClient() - .AddSingleton(_projectContext) - .AddTestFwDataBridge() - .AddFwLiteProjectSync() - .Configure(c => c.ProjectsFolder = Path.Combine(".", _projectName, "FwData")) - .Configure(c => c.ProjectPath = Path.Combine(".", _projectName, "LcmCrdt")) - .AddLogging(builder => builder.AddDebug()); - if (extraServiceConfiguration is not null) extraServiceConfiguration(crdtServices); + .AddSyncServices(_projectName); _services = crdtServices.BuildServiceProvider().CreateAsyncScope(); } @@ -73,8 +63,3 @@ public async Task DisposeAsync() public CrdtMiniLcmApi CrdtApi { get; set; } = null!; public FwDataMiniLcmApi FwDataApi { get; set; } = null!; } - -public class MockProjectContext(CrdtProject? project) : ProjectContext -{ - public override CrdtProject? Project { get; set; } = project; -} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs new file mode 100644 index 000000000..6eb9f8b40 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs @@ -0,0 +1,27 @@ +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.Tests.Fixtures; +using LcmCrdt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public static class TestingKernel +{ + public static IServiceCollection AddSyncServices(this IServiceCollection services, string projectName, bool mockFwProjectLoader = true) + { + return services.AddLcmCrdtClient() + .AddTestFwDataBridge(mockFwProjectLoader) + .AddFwLiteProjectSync() + .AddSingleton(new MockProjectContext(null)) + .Configure(c => c.ProjectsFolder = Path.Combine(".", projectName, "FwData")) + .Configure(c => c.ProjectPath = Path.Combine(".", projectName, "LcmCrdt")) + .AddLogging(builder => builder.AddDebug()); + } + + public class MockProjectContext(CrdtProject? project) : ProjectContext + { + public override CrdtProject? Project { get; set; } = project; + } + +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs index 929bdb471..17b9216b6 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -14,57 +14,12 @@ public class Sena3SyncTests : IClassFixture, IAsyncLifetime private readonly Sena3Fixture _fixture; private readonly CrdtFwdataProjectSyncService _syncService; - private readonly Guid _complexEntryId = Guid.NewGuid(); - private Entry _testEntry = new Entry - { - Id = Guid.NewGuid(), - LexemeForm = { Values = { { "en", "Apple" } } }, - Note = { Values = { { "en", "this is a test note" } } }, - Senses = - [ - new Sense - { - Gloss = { Values = { { "en", "Apple" } } }, - Definition = { Values = { { "en", "a round fruit with a hard, crisp skin" } } }, - ExampleSentences = - [ - new ExampleSentence { Sentence = { Values = { { "en", "I went to the store to buy an apple." } } } } - ] - } - ] - }; - public async Task InitializeAsync() { - await _fixture.FwDataApi.CreateEntry(_testEntry); - await _fixture.FwDataApi.CreateEntry(new Entry() - { - Id = _complexEntryId, - LexemeForm = { { "en", "Pineapple" } }, - Components = - [ - new ComplexFormComponent() - { - Id = Guid.NewGuid(), - ComplexFormEntryId = _complexEntryId, - ComplexFormHeadword = "Pineapple", - ComponentEntryId = _testEntry.Id, - ComponentHeadword = "Apple" - } - ] - }); } public async Task DisposeAsync() { - await foreach (var entry in _fixture.FwDataApi.GetEntries()) - { - await _fixture.FwDataApi.DeleteEntry(entry.Id); - } - foreach (var entry in await _fixture.CrdtApi.GetEntries().ToArrayAsync()) - { - await _fixture.CrdtApi.DeleteEntry(entry.Id); - } } public Sena3SyncTests(Sena3Fixture fixture) From ff499196c2e99de0ab0497b4e6ab18df7dbf22e5 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 12:12:22 +0700 Subject: [PATCH 07/30] prevent duplicating complex forms which are the same/already exist --- ...ModelSnapshotTests.VerifyDbModel.verified.txt | 9 ++++++++- backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs | 5 +++++ backend/FwLite/LcmCrdt/LcmCrdtKernel.cs | 15 +++++++++++++++ .../ComplexFormComponentTestsBase.cs | 16 ++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt index a7b2cc8c5..129ee865c 100644 --- a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt @@ -24,6 +24,8 @@ ComponentEntryId (Guid) Required FK Index ComponentHeadword (string) ComponentSenseId (Guid?) FK Index + Annotations: + Relational:ColumnName: ComponentSenseId DeletedAt (DateTimeOffset?) SnapshotId (no field, Guid?) Shadow FK Index Keys: @@ -34,10 +36,15 @@ ComplexFormComponent {'ComponentSenseId'} -> Sense {'Id'} Cascade ComplexFormComponent {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull Indexes: - ComplexFormEntryId ComponentEntryId ComponentSenseId SnapshotId Unique + ComplexFormEntryId, ComponentEntryId Unique + Annotations: + Relational:Filter: ComponentSenseId IS NULL + ComplexFormEntryId, ComponentEntryId, ComponentSenseId Unique + Annotations: + Relational:Filter: ComponentSenseId IS NOT NULL Annotations: DiscriminatorProperty: Relational:FunctionName: diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index 241c8daad..c39cb33d1 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -151,6 +151,11 @@ public async Task CreateComplexFormType(ComplexFormType complex public async Task CreateComplexFormComponent(ComplexFormComponent complexFormComponent) { + var existing = await ComplexFormComponents.SingleOrDefaultAsync(c => + c.ComplexFormEntryId == complexFormComponent.ComplexFormEntryId + && c.ComponentEntryId == complexFormComponent.ComponentEntryId + && c.ComponentSenseId == complexFormComponent.ComponentSenseId); + if (existing is not null) return existing; var addEntryComponentChange = new AddEntryComponentChange(complexFormComponent); await dataModel.AddChange(ClientId, addEntryComponentChange); return (await ComplexFormComponents.SingleOrDefaultAsync(c => c.Id == addEntryComponentChange.EntityId)) ?? throw NotFoundException.ForType(); diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index 9fd13ce3e..74562c440 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -134,7 +134,22 @@ public static void ConfigureCrdt(CrdtConfig config) .Add() .Add(builder => { + const string componentSenseId = "ComponentSenseId"; builder.ToTable("ComplexFormComponents"); + builder.Property(c => c.ComponentSenseId).HasColumnName(componentSenseId); + //these indexes are used to ensure that we don't create duplicate complex form components + //we need the filter otherwise 2 components which are the same and have a null sense id can be created because 2 rows with the same null are not considered duplicates + builder.HasIndex(component => new + { + component.ComplexFormEntryId, + component.ComponentEntryId, + component.ComponentSenseId + }).IsUnique().HasFilter($"{componentSenseId} IS NOT NULL"); + builder.HasIndex(component => new + { + component.ComplexFormEntryId, + component.ComponentEntryId + }).IsUnique().HasFilter($"{componentSenseId} IS NULL"); }); config.ChangeTypeListBuilder.Add>() diff --git a/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs b/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs index 4f51ca275..820b660b1 100644 --- a/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs @@ -50,6 +50,22 @@ public async Task CreateComplexFormComponent_Works() component.ComponentHeadword.Should().Be("component"); } + [Fact] + public async Task CreateComplexFormComponent_UsingTheSameComponentWithNullSenseDoesNothing() + { + var component1 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + var component2 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + component2.Should().BeEquivalentTo(component1); + } + + [Fact] + public async Task CreateComplexFormComponent_UsingTheSameComponentWithSenseDoesNothing() + { + var component1 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry, _componentSenseId1)); + var component2 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry, _componentSenseId1)); + component2.Should().BeEquivalentTo(component1); + } + [Fact] public async Task CreateComplexFormComponent_WorksWithSense() { From eba81b9e49e3279e36cf1c015187771b1a20b44e Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 12:12:50 +0700 Subject: [PATCH 08/30] catch and throw some exceptions with additional context to help debugging --- .../Api/FwDataMiniLcmApi.cs | 75 +++++++++++-------- .../Exceptions/CreateObjectException.cs | 12 +++ .../MiniLcm/Exceptions/SyncObjectException.cs | 12 +++ .../FwLite/MiniLcm/SyncHelpers/EntrySync.cs | 21 ++++-- 4 files changed, 80 insertions(+), 40 deletions(-) create mode 100644 backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs create mode 100644 backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index 8e132b2e6..a258593ed 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -6,6 +6,7 @@ using FwDataMiniLcmBridge.LcmUtils; using Microsoft.Extensions.Logging; using MiniLcm; +using MiniLcm.Exceptions; using MiniLcm.Models; using MiniLcm.SyncHelpers; using SIL.LCModel; @@ -559,40 +560,48 @@ public IAsyncEnumerable SearchEntries(string query, QueryOptions? options public async Task CreateEntry(Entry entry) { entry.Id = entry.Id == default ? Guid.NewGuid() : entry.Id; - UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create Entry", - "Remove entry", - Cache.ServiceLocator.ActionHandler, - () => - { - var lexEntry = LexEntryFactory.Create(entry.Id, Cache.ServiceLocator.GetInstance().Singleton.LexDbOA); - lexEntry.LexemeFormOA = Cache.ServiceLocator.GetInstance().Create(); - UpdateLcmMultiString(lexEntry.LexemeFormOA.Form, entry.LexemeForm); - UpdateLcmMultiString(lexEntry.CitationForm, entry.CitationForm); - UpdateLcmMultiString(lexEntry.LiteralMeaning, entry.LiteralMeaning); - UpdateLcmMultiString(lexEntry.Comment, entry.Note); - - foreach (var sense in entry.Senses) - { - CreateSense(lexEntry, sense); - } - - //form types should be created before components, otherwise the form type "unspecified" will be added - foreach (var complexFormType in entry.ComplexFormTypes) - { - AddComplexFormType(lexEntry, complexFormType.Id); - } - - foreach (var component in entry.Components) - { - AddComplexFormComponent(lexEntry, component); - } - - foreach (var complexForm in entry.ComplexForms) + try + { + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create Entry", + "Remove entry", + Cache.ServiceLocator.ActionHandler, + () => { - var complexLexEntry = EntriesRepository.GetObject(complexForm.ComplexFormEntryId); - AddComplexFormComponent(complexLexEntry, complexForm); - } - }); + var lexEntry = LexEntryFactory.Create(entry.Id, + Cache.ServiceLocator.GetInstance().Singleton.LexDbOA); + lexEntry.LexemeFormOA = Cache.ServiceLocator.GetInstance().Create(); + UpdateLcmMultiString(lexEntry.LexemeFormOA.Form, entry.LexemeForm); + UpdateLcmMultiString(lexEntry.CitationForm, entry.CitationForm); + UpdateLcmMultiString(lexEntry.LiteralMeaning, entry.LiteralMeaning); + UpdateLcmMultiString(lexEntry.Comment, entry.Note); + + foreach (var sense in entry.Senses) + { + CreateSense(lexEntry, sense); + } + + //form types should be created before components, otherwise the form type "unspecified" will be added + foreach (var complexFormType in entry.ComplexFormTypes) + { + AddComplexFormType(lexEntry, complexFormType.Id); + } + + foreach (var component in entry.Components) + { + AddComplexFormComponent(lexEntry, component); + } + + foreach (var complexForm in entry.ComplexForms) + { + var complexLexEntry = EntriesRepository.GetObject(complexForm.ComplexFormEntryId); + AddComplexFormComponent(complexLexEntry, complexForm); + } + }); + } + catch (Exception e) + { + throw new CreateObjectException($"Failed to create entry {entry}", e); + } return await GetEntry(entry.Id) ?? throw new InvalidOperationException("Entry was not created"); } diff --git a/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs b/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs new file mode 100644 index 000000000..128cb0515 --- /dev/null +++ b/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs @@ -0,0 +1,12 @@ +namespace MiniLcm.Exceptions; + +public class CreateObjectException: Exception +{ + public CreateObjectException(string? message) : base(message) + { + } + + public CreateObjectException(string? message, Exception? innerException) : base(message, innerException) + { + } +} diff --git a/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs b/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs new file mode 100644 index 000000000..e1f284cde --- /dev/null +++ b/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs @@ -0,0 +1,12 @@ +namespace MiniLcm.Exceptions; + +public class SyncObjectException: Exception +{ + public SyncObjectException(string? message) : base(message) + { + } + + public SyncObjectException(string? message, Exception? innerException) : base(message, innerException) + { + } +} diff --git a/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs b/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs index a61a7fc58..c8f2db5a3 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs @@ -29,14 +29,21 @@ public static async Task Sync(Entry[] afterEntries, public static async Task Sync(Entry afterEntry, Entry beforeEntry, IMiniLcmApi api) { - var updateObjectInput = EntryDiffToUpdate(beforeEntry, afterEntry); - if (updateObjectInput is not null) await api.UpdateEntry(afterEntry.Id, updateObjectInput); - var changes = await SensesSync(afterEntry.Id, afterEntry.Senses, beforeEntry.Senses, api); + try + { + var updateObjectInput = EntryDiffToUpdate(beforeEntry, afterEntry); + if (updateObjectInput is not null) await api.UpdateEntry(afterEntry.Id, updateObjectInput); + var changes = await SensesSync(afterEntry.Id, afterEntry.Senses, beforeEntry.Senses, api); - changes += await Sync(afterEntry.Components, beforeEntry.Components, api); - changes += await Sync(afterEntry.ComplexForms, beforeEntry.ComplexForms, api); - changes += await Sync(afterEntry.Id, afterEntry.ComplexFormTypes, beforeEntry.ComplexFormTypes, api); - return changes + (updateObjectInput is null ? 0 : 1); + changes += await Sync(afterEntry.Components, beforeEntry.Components, api); + changes += await Sync(afterEntry.ComplexForms, beforeEntry.ComplexForms, api); + changes += await Sync(afterEntry.Id, afterEntry.ComplexFormTypes, beforeEntry.ComplexFormTypes, api); + return changes + (updateObjectInput is null ? 0 : 1); + } + catch (Exception e) + { + throw new SyncObjectException($"Failed to sync entry {afterEntry}", e); + } } private static async Task Sync(Guid entryId, From 4e6dacda3385ea4f181769fcc2e846b731ea3333 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 12:14:53 +0700 Subject: [PATCH 09/30] add method to SyncService to expose snapshot path for testing --- .../Fixtures/SyncFixture.cs | 6 +++++ .../CrdtFwdataProjectSyncService.cs | 24 +++++++++++-------- .../FwLiteProjectSync.csproj | 4 ++++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index 0b0443e0d..cb82ef31b 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -69,6 +69,12 @@ public async Task DisposeAsync() public CrdtMiniLcmApi CrdtApi { get; set; } = null!; public FwDataMiniLcmApi FwDataApi { get; set; } = null!; + + public void DeleteSyncSnapshot() + { + var snapshotPath = CrdtFwdataProjectSyncService.SnapshotPath(FwDataApi.Project); + if (File.Exists(snapshotPath)) File.Delete(snapshotPath); + } } public class MockProjectContext(CrdtProject? project) : ProjectContext diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 12258dcee..dc50cd4ad 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -1,8 +1,8 @@ using System.Text.Json; +using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.Api; using LcmCrdt; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using MiniLcm; using MiniLcm.Models; using MiniLcm.SyncHelpers; @@ -11,7 +11,7 @@ namespace FwLiteProjectSync; -public class CrdtFwdataProjectSyncService(IOptions lcmCrdtConfig, MiniLcmImport miniLcmImport, ILogger logger) +public class CrdtFwdataProjectSyncService(MiniLcmImport miniLcmImport, ILogger logger) { public record SyncResult(int CrdtChanges, int FwdataChanges); @@ -21,13 +21,13 @@ public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataA { throw new InvalidOperationException($"Project id mismatch, CRDT Id: {crdt.ProjectData.FwProjectId}, FWData Id: {fwdataApi.ProjectId}"); } - var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath); + var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project); SyncResult result = await Sync(crdtApi, fwdataApi, dryRun, fwdataApi.EntryCount, projectSnapshot); fwdataApi.Save(); if (!dryRun) { - await SaveProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath, + await SaveProjectSnapshot(fwdataApi.Project, new ProjectSnapshot( await fwdataApi.GetEntries().ToArrayAsync(), await fwdataApi.GetPartsOfSpeech().ToArrayAsync(), @@ -86,21 +86,25 @@ private void LogDryRun(IMiniLcmApi api, string type) public record ProjectSnapshot(Entry[] Entries, PartOfSpeech[] PartsOfSpeech, SemanticDomain[] SemanticDomains); - private async Task GetProjectSnapshot(string projectName, string? projectPath) + private async Task GetProjectSnapshot(FwDataProject project) { - projectPath ??= lcmCrdtConfig.Value.ProjectPath; - var snapshotPath = Path.Combine(projectPath, $"{projectName}_snapshot.json"); + var snapshotPath = SnapshotPath(project); if (!File.Exists(snapshotPath)) return null; await using var file = File.OpenRead(snapshotPath); return await JsonSerializer.DeserializeAsync(file); } - private async Task SaveProjectSnapshot(string projectName, string? projectPath, ProjectSnapshot projectSnapshot) + private async Task SaveProjectSnapshot(FwDataProject project, ProjectSnapshot projectSnapshot) { - projectPath ??= lcmCrdtConfig.Value.ProjectPath; - var snapshotPath = Path.Combine(projectPath, $"{projectName}_snapshot.json"); + var snapshotPath = SnapshotPath(project); await using var file = File.Create(snapshotPath); await JsonSerializer.SerializeAsync(file, projectSnapshot); } + internal static string SnapshotPath(FwDataProject project) + { + var projectPath = project.ProjectsPath; + var snapshotPath = Path.Combine(projectPath, $"{project.Name}_snapshot.json"); + return snapshotPath; + } } diff --git a/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj b/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj index 7e4c26e04..2d0c452f0 100644 --- a/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj +++ b/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj @@ -19,4 +19,8 @@ + + + + From 98f38d4069b17e9518ba4ce0fbe646d6623d9c68 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 12:16:00 +0700 Subject: [PATCH 10/30] move CanCreateAComplexFormAndItsComponentInOneSync out of EntrySyncTests and into SyncTests --- .../FwLiteProjectSync.Tests/EntrySyncTests.cs | 23 ------------------- .../FwLiteProjectSync.Tests/SyncTests.cs | 20 ++++++++++++++++ 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs index d7a631759..d1b264839 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs @@ -105,27 +105,4 @@ public async Task CanChangeComplexFormTypeViaSync() actual.Should().NotBeNull(); actual.Should().BeEquivalentTo(after, options => options); } - - [Fact] - public async Task CanCreateAComplexFormAndItsComponentInOneSync() - { - //ensure they are synced so a real sync will happen when we want it to - await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); - - var complexFormEntry = await _fixture.CrdtApi.CreateEntry(new() { LexemeForm = { { "en", "complexForm" } } }); - var componentEntry = await _fixture.CrdtApi.CreateEntry(new() - { - LexemeForm = { { "en", "component" } }, - ComplexForms = - [ - new ComplexFormComponent() - { - ComplexFormEntryId = complexFormEntry.Id, ComponentEntryId = Guid.Empty - } - ] - }); - - //one of the entries will be created first, it will try to create the reference to the other but it won't exist yet - await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); - } } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index 752d77fa2..07fc8123e 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -383,4 +383,24 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() options => options.For(e => e.Components).Exclude(c => c.Id) .For(e => e.ComplexForms).Exclude(c => c.Id)); } + + [Fact] + public async Task CanCreateAComplexFormAndItsComponentInOneSync() + { + //ensure they are synced so a real sync will happen when we want it to + await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + + var complexFormEntry = await _fixture.CrdtApi.CreateEntry(new() { LexemeForm = { { "en", "complexForm" } } }); + var componentEntry = await _fixture.CrdtApi.CreateEntry(new() + { + LexemeForm = { { "en", "component" } }, + ComplexForms = + [ + new ComplexFormComponent() { ComplexFormEntryId = complexFormEntry.Id, ComponentEntryId = Guid.Empty } + ] + }); + + //one of the entries will be created first, it will try to create the reference to the other but it won't exist yet + await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + } } From 1e56414141fcb74d54a1a6cf010a036b85ca95a7 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 13:42:22 +0700 Subject: [PATCH 11/30] expose a dry run sync method which returns the dry run records for inspection --- .../CrdtFwdataProjectSyncService.cs | 17 ++++++++++++++++- .../FwLiteProjectSync/DryRunMiniLcmApi.cs | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index dc50cd4ad..a4122e8ed 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -15,6 +15,16 @@ public class CrdtFwdataProjectSyncService(MiniLcmImport miniLcmImport, ILogger CrdtDryRunRecords, + List FwDataDryRunRecords) : SyncResult(CrdtChanges, FwdataChanges); + + public async Task SyncDryRun(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi) + { + return (DryRunSyncResult) await Sync(crdtApi, fwdataApi, true); + } public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi, bool dryRun = false) { if (crdtApi is CrdtMiniLcmApi crdt && crdt.ProjectData.FwProjectId != fwdataApi.ProjectId) @@ -69,7 +79,7 @@ private async Task Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi, LogDryRun(fwdataApi, "fwdata"); //todo push crdt changes to lexbox - + if (dryRun) return new DryRunSyncResult(crdtChanges, fwdataChanges, GetDryRunRecords(crdtApi), GetDryRunRecords(fwdataApi)); return new SyncResult(crdtChanges, fwdataChanges); } @@ -84,6 +94,11 @@ private void LogDryRun(IMiniLcmApi api, string type) logger.LogInformation($"Dry run {type} changes: {dryRunApi.DryRunRecords.Count}"); } + private List GetDryRunRecords(IMiniLcmApi api) + { + return ((DryRunMiniLcmApi)api).DryRunRecords; + } + public record ProjectSnapshot(Entry[] Entries, PartOfSpeech[] PartsOfSpeech, SemanticDomain[] SemanticDomains); private async Task GetProjectSnapshot(FwDataProject project) diff --git a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs index 25c317670..15c17a387 100644 --- a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs +++ b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs @@ -216,7 +216,7 @@ public Task CreateComplexFormComponent(ComplexFormComponen public Task DeleteComplexFormComponent(ComplexFormComponent complexFormComponent) { - DryRunRecords.Add(new DryRunRecord(nameof(DeleteComplexFormComponent), $"Delete complex form component complex entry: {complexFormComponent.ComplexFormHeadword}, component entry: {complexFormComponent.ComponentHeadword}")); + DryRunRecords.Add(new DryRunRecord(nameof(DeleteComplexFormComponent), $"Delete complex form component: {complexFormComponent}")); return Task.CompletedTask; } From 373a5ec43151531225ec8f1d2e7486bd14c38cf5 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 14:00:29 +0700 Subject: [PATCH 12/30] dont track EF queries in miniLcm, should fix weird sync test issues which only show up when running all tests at once --- backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index c39cb33d1..e6e83fb7b 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -18,14 +18,15 @@ public class CrdtMiniLcmApi(DataModel dataModel, CurrentProjectService projectSe private Guid ClientId { get; } = projectService.ProjectData.ClientId; public ProjectData ProjectData => projectService.ProjectData; - private IQueryable Entries => dataModel.QueryLatest(); - private IQueryable ComplexFormComponents => dataModel.QueryLatest(); - private IQueryable ComplexFormTypes => dataModel.QueryLatest(); - private IQueryable Senses => dataModel.QueryLatest(); - private IQueryable ExampleSentences => dataModel.QueryLatest(); - private IQueryable WritingSystems => dataModel.QueryLatest(); - private IQueryable SemanticDomains => dataModel.QueryLatest(); - private IQueryable PartsOfSpeech => dataModel.QueryLatest(); + private IQueryable Entries => dataModel.QueryLatest().AsTracking(false); + private IQueryable ComplexFormComponents => dataModel.QueryLatest() + .AsTracking(false); + private IQueryable ComplexFormTypes => dataModel.QueryLatest().AsTracking(false); + private IQueryable Senses => dataModel.QueryLatest().AsTracking(false); + private IQueryable ExampleSentences => dataModel.QueryLatest().AsTracking(false); + private IQueryable WritingSystems => dataModel.QueryLatest().AsTracking(false); + private IQueryable SemanticDomains => dataModel.QueryLatest().AsTracking(false); + private IQueryable PartsOfSpeech => dataModel.QueryLatest().AsTracking(false); public async Task GetWritingSystems() { From aab136637f2bdd42f570b58fb44566fd0f1883ae Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 14:04:06 +0700 Subject: [PATCH 13/30] don't seed crdt db when creating to avoid sync issues on second sync --- .../Fixtures/SyncFixture.cs | 2 +- .../FwLiteProjectSync.Tests/SyncTests.cs | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index cb82ef31b..c2120c669 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -58,7 +58,7 @@ public async Task InitializeAsync() if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); Directory.CreateDirectory(crdtProjectsFolder); var crdtProject = await _services.ServiceProvider.GetRequiredService() - .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: true)); + .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: false)); CrdtApi = (CrdtMiniLcmApi) await _services.ServiceProvider.OpenCrdtProject(crdtProject); } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index 07fc8123e..d10451967 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -65,6 +65,8 @@ public async Task DisposeAsync() { await _fixture.CrdtApi.DeleteEntry(entry.Id); } + + _fixture.DeleteSyncSnapshot(); } public SyncTests(SyncFixture fixture) @@ -84,7 +86,18 @@ public async Task FirstSyncJustDoesAnImport() var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); crdtEntries.Should().BeEquivalentTo(fwdataEntries, options => options.For(e => e.Components).Exclude(c => c.Id) - .For(e => e.ComplexForms).Exclude(c => c.Id)); + .For(e => e.ComplexForms).Exclude(c => c.Id)); + } + + [Fact] + public async Task SecondSyncDoesNothing() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + var secondSync = await _syncService.SyncDryRun(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0, $"changes were {string.Join(", ", secondSync.CrdtDryRunRecords)}"); + secondSync.FwdataChanges.Should().Be(0, $"changes were {string.Join(", ", secondSync.FwDataDryRunRecords)}"); } [Fact] @@ -273,7 +286,7 @@ await fwdataApi.CreateEntry(new Entry() LexemeForm = { { "en", "Pear" } }, Senses = [ - new Sense() { Gloss = { { "en", "Pear" } }, SemanticDomains = [ semdom3 ] } + new Sense() { Gloss = { { "en", "Pear" } }, SemanticDomains = [semdom3] } ] }); await crdtApi.CreateEntry(new Entry() @@ -281,7 +294,7 @@ await crdtApi.CreateEntry(new Entry() LexemeForm = { { "en", "Banana" } }, Senses = [ - new Sense() { Gloss = { { "en", "Banana" } }, SemanticDomains = [ semdom3 ] } + new Sense() { Gloss = { { "en", "Banana" } }, SemanticDomains = [semdom3] } ] }); await _syncService.Sync(crdtApi, fwdataApi); @@ -365,7 +378,7 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() await _syncService.Sync(crdtApi, fwdataApi); await fwdataApi.CreateSense(_testEntry.Id, new Sense() - { + { Gloss = { { "en", "Fruit" } }, Definition = { { "en", "a round fruit, red or yellow" } }, }); @@ -373,7 +386,7 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() { Gloss = { { "en", "Tree" } }, Definition = { { "en", "a tall, woody plant, which grows fruit" } }, - }); + }); await _syncService.Sync(crdtApi, fwdataApi); From 3bc17fcabd0cb40be4c9f81ac045002cfa61e5a2 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 14:04:59 +0700 Subject: [PATCH 14/30] change test project vernacular ws and change how ComplexFormComponent headwords get set so that they match what we get from fieldworks --- .../Fixtures/SyncFixture.cs | 2 +- .../JsonPatchEntryRewriteTests.cs | 2 -- .../Entries/AddEntryComponentChange.cs | 19 ++++++++++--------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index c2120c669..67cdf0548 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -50,7 +50,7 @@ public async Task InitializeAsync() if (Path.Exists(projectsFolder)) Directory.Delete(projectsFolder, true); Directory.CreateDirectory(projectsFolder); _services.ServiceProvider.GetRequiredService() - .NewProject(new FwDataProject(_projectName, projectsFolder), "en", "fr"); + .NewProject(new FwDataProject(_projectName, projectsFolder), "en", "en"); FwDataApi = _services.ServiceProvider.GetRequiredService().GetFwDataMiniLcmApi(_projectName, false); var crdtProjectsFolder = diff --git a/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs b/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs index b2370c85e..13eb26933 100644 --- a/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs @@ -20,7 +20,6 @@ public void ChangesFromJsonPatch_AddComponentMakesAddEntryComponentChange() changes.Should().ContainSingle().Which.Should().BeOfType().Subject; addEntryComponentChange.ComplexFormEntryId.Should().Be(_entry.Id); addEntryComponentChange.ComponentEntryId.Should().Be(componentEntry.Id); - addEntryComponentChange.ComponentHeadword.Should().Be(componentEntry.Headword()); } [Fact] @@ -84,7 +83,6 @@ public void ChangesFromJsonPatch_AddComplexFormMakesAddEntryComponentChange() changes.Should().ContainSingle().Which.Should().BeOfType().Subject; addEntryComponentChange.ComplexFormEntryId.Should().Be(_entry.Id); addEntryComponentChange.ComponentEntryId.Should().Be(componentEntry.Id); - addEntryComponentChange.ComponentHeadword.Should().Be(componentEntry.Headword()); } [Fact] diff --git a/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs b/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs index a51313198..f93630058 100644 --- a/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs +++ b/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs @@ -10,10 +10,8 @@ namespace LcmCrdt.Changes.Entries; public class AddEntryComponentChange : CreateChange, ISelfNamedType { public Guid ComplexFormEntryId { get; } - public string? ComplexFormHeadword { get; } public Guid ComponentEntryId { get; } public Guid? ComponentSenseId { get; } - public string? ComponentHeadword { get; } [JsonConstructor] public AddEntryComponentChange(Guid entityId, @@ -24,9 +22,7 @@ public AddEntryComponentChange(Guid entityId, Guid? componentSenseId = null) : base(entityId) { ComplexFormEntryId = complexFormEntryId; - ComplexFormHeadword = complexFormHeadword; ComponentEntryId = componentEntryId; - ComponentHeadword = componentHeadword; ComponentSenseId = componentSenseId; } @@ -41,17 +37,22 @@ public AddEntryComponentChange(Guid entityId, public override async ValueTask NewEntity(Commit commit, ChangeContext context) { + var complexFormEntry = await context.GetCurrent(ComplexFormEntryId); + var componentEntry = await context.GetCurrent(ComponentEntryId); + Sense? componentSense = null; + if (ComponentSenseId is not null) + componentSense = await context.GetCurrent(ComponentSenseId.Value); return new ComplexFormComponent { Id = EntityId, ComplexFormEntryId = ComplexFormEntryId, - ComplexFormHeadword = ComplexFormHeadword, + ComplexFormHeadword = complexFormEntry?.Headword(), ComponentEntryId = ComponentEntryId, - ComponentHeadword = ComponentHeadword, + ComponentHeadword = componentEntry?.Headword(), ComponentSenseId = ComponentSenseId, - DeletedAt = (await context.IsObjectDeleted(ComponentEntryId) || - await context.IsObjectDeleted(ComplexFormEntryId) || - ComponentSenseId.HasValue && await context.IsObjectDeleted(ComponentSenseId.Value)) + DeletedAt = (complexFormEntry?.DeletedAt is not null || + componentEntry?.DeletedAt is not null || + (ComponentSenseId.HasValue && componentSense?.DeletedAt is not null)) ? commit.DateTime : (DateTime?)null, }; From a253a8d70300bf300400790e143dd2f81659dd86 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 25 Nov 2024 14:12:20 +0700 Subject: [PATCH 15/30] fix test failing due to no complex form types existing --- backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs index d1b264839..50d9f56f8 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs @@ -95,8 +95,8 @@ public async Task CanChangeComplexFormViaSync_ComplexForms() [Fact] public async Task CanChangeComplexFormTypeViaSync() { + var complexFormType = await _fixture.CrdtApi.CreateComplexFormType(new() { Name = new() { { "en", "complexFormType" } } }); var entry = await _fixture.CrdtApi.CreateEntry(new() { LexemeForm = { { "en", "complexForm1" } } }); - var complexFormType = await _fixture.CrdtApi.GetComplexFormTypes().FirstAsync(); var after = (Entry) entry.Copy(); after.ComplexFormTypes = [complexFormType]; await EntrySync.Sync(after, entry, _fixture.CrdtApi); From cc38fb92a17a04eaa5d17dd4f625f542cf44e7c3 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 09:20:58 +0700 Subject: [PATCH 16/30] remove conflicting test --- backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index d70bdd5b5..b798a9d0c 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -100,17 +100,6 @@ public async Task SecondSyncDoesNothing() secondSync.FwdataChanges.Should().Be(0, $"changes were {string.Join(", ", secondSync.FwDataDryRunRecords)}"); } - [Fact] - public async Task SecondSyncDoesNothing() - { - var crdtApi = _fixture.CrdtApi; - var fwdataApi = _fixture.FwDataApi; - await _syncService.Sync(crdtApi, fwdataApi); - var secondSync = await _syncService.Sync(crdtApi, fwdataApi); - secondSync.CrdtChanges.Should().Be(0); - secondSync.FwdataChanges.Should().Be(0); - } - [Fact] public static async Task SyncFailsWithMismatchedProjectIds() { From 1f7d1ab53f7d99bc351250776b042c093c3e28f1 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 09:36:52 +0700 Subject: [PATCH 17/30] avoid throwing an error when removing complex form types if the type isn't on the entry --- .../Api/FwDataMiniLcmApi.cs | 3 +- .../ComplexFormComponentTestsBase.cs | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index a258593ed..1adb5b67f 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -710,7 +710,8 @@ internal void RemoveComplexFormType(ILexEntry lexEntry, Guid complexFormTypeId) { ILexEntryRef? entryRef = lexEntry.ComplexFormEntryRefs.SingleOrDefault(); if (entryRef is null) return; - var lexEntryType = (ILexEntryType)ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormTypeId); + var lexEntryType = entryRef.ComplexEntryTypesRS.SingleOrDefault(c => c.Guid == complexFormTypeId); + if (lexEntryType is null) return; entryRef.ComplexEntryTypesRS.Remove(lexEntryType); } diff --git a/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs b/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs index 820b660b1..d6f646435 100644 --- a/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs @@ -99,4 +99,43 @@ public async Task CreateComplexFormType_Works() var types = await Api.GetComplexFormTypes().ToArrayAsync(); types.Should().ContainSingle(t => t.Id == complexFormType.Id); } + + [Fact] + public async Task AddComplexFormType_Works() + { + var complexFormType = new ComplexFormType() { Id = Guid.NewGuid(), Name = new() { { "en", "test" } } }; + await Api.CreateComplexFormType(complexFormType); + await Api.AddComplexFormType(_complexFormEntryId, complexFormType.Id); + var entry = await Api.GetEntry(_complexFormEntryId); + entry.Should().NotBeNull(); + entry!.ComplexFormTypes.Should().ContainSingle(c => c.Id == complexFormType.Id); + } + + [Fact] + public async Task RemoveComplexFormType_Works() + { + await AddComplexFormType_Works(); + var entry = await Api.GetEntry(_complexFormEntryId); + await Api.RemoveComplexFormType(_complexFormEntryId, entry!.ComplexFormTypes[0].Id); + entry = await Api.GetEntry(_complexFormEntryId); + entry.Should().NotBeNull(); + entry!.ComplexFormTypes.Should().BeEmpty(); + } + + [Fact] + public async Task RemoveComplexFormType_WorksWhenTypeDoesNotExist() + { + await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + await Api.RemoveComplexFormType(_complexFormEntryId, Guid.NewGuid()); + } + + [Fact] + public async Task RemoveComplexFormType_WorksWhenTypeIsNotOnEntry() + { + //FW projects react differently if an entry has complex forms or not + await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + var complexFormType = new ComplexFormType() { Id = Guid.NewGuid(), Name = new() { { "en", "test" } } }; + await Api.CreateComplexFormType(complexFormType); + await Api.RemoveComplexFormType(_complexFormEntryId, Guid.NewGuid()); + } } From 0db9794e80a1a9d9a353abc8582b04b57942e363 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 09:41:59 +0700 Subject: [PATCH 18/30] fixed bulk import of semantic domains would not import the predefined property --- backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index e6e83fb7b..21223032e 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -135,7 +135,7 @@ public async Task DeleteSemanticDomain(Guid id) public async Task BulkImportSemanticDomains(IEnumerable semanticDomains) { - await dataModel.AddChanges(ClientId, semanticDomains.Select(sd => new CreateSemanticDomainChange(sd.Id, sd.Name, sd.Code))); + await dataModel.AddChanges(ClientId, semanticDomains.Select(sd => new CreateSemanticDomainChange(sd.Id, sd.Name, sd.Code, sd.Predefined))); } public IAsyncEnumerable GetComplexFormTypes() From 68b403fd42f8211f1e90a582712b877172dd003a Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 09:55:22 +0700 Subject: [PATCH 19/30] ensure root service provider is cleaned up in fixtures to avoid issue where test host hangs once tests are done --- .../FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs | 7 ++++++- .../FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs index 5a78db2cf..d1b5ca6bf 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs @@ -6,6 +6,7 @@ using FwDataMiniLcmBridge.LcmUtils; using FwDataMiniLcmBridge.Tests.Fixtures; using LcmCrdt; +using LexCore.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; @@ -24,6 +25,7 @@ public class Sena3Fixture : IAsyncLifetime _services.ServiceProvider.GetRequiredService(); public IServiceProvider Services => _services.ServiceProvider; + private IDisposable _cleanup; private readonly LexboxConfig lexboxConfig; private readonly HttpClient http; public CrdtMiniLcmApi CrdtApi { get; set; } = null!; @@ -45,7 +47,9 @@ public Sena3Fixture() }) .ValidateDataAnnotations() .ValidateOnStart(); - _services = services.BuildServiceProvider().CreateAsyncScope(); + var rootServiceProvider = services.BuildServiceProvider(); + _cleanup = Defer.Action(() => rootServiceProvider.Dispose()); + _services = rootServiceProvider.CreateAsyncScope(); lexboxConfig = Services.GetRequiredService>().Value; var factory = Services.GetRequiredService(); http = factory.CreateClient(nameof(Sena3Fixture)); @@ -79,6 +83,7 @@ public async Task InitializeAsync() public async Task DisposeAsync() { await _services.DisposeAsync(); + _cleanup.Dispose(); } public async Task DownloadProjectBackupStream(string code) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index 13e4ccc11..3cae9d60a 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -4,6 +4,7 @@ using FwDataMiniLcmBridge.LcmUtils; using FwDataMiniLcmBridge.Tests.Fixtures; using LcmCrdt; +using LexCore.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; @@ -20,6 +21,7 @@ public class SyncFixture : IAsyncLifetime _services.ServiceProvider.GetRequiredService(); public IServiceProvider Services => _services.ServiceProvider; private readonly string _projectName; + private readonly IDisposable _cleanup; public static SyncFixture Create([CallerMemberName] string projectName = "") => new(projectName); @@ -28,7 +30,9 @@ private SyncFixture(string projectName) _projectName = projectName; var crdtServices = new ServiceCollection() .AddSyncServices(_projectName); - _services = crdtServices.BuildServiceProvider().CreateAsyncScope(); + var rootServiceProvider = crdtServices.BuildServiceProvider(); + _cleanup = Defer.Action(() => rootServiceProvider.Dispose()); + _services = rootServiceProvider.CreateAsyncScope(); } public SyncFixture(): this("sena-3_" + Guid.NewGuid().ToString("N")) @@ -56,8 +60,8 @@ public async Task InitializeAsync() public async Task DisposeAsync() { - // CAUTION: Do not assume that InitializeAsync() has been called await _services.DisposeAsync(); + _cleanup.Dispose(); } public CrdtMiniLcmApi CrdtApi { get; set; } = null!; From 75bf7a713019f8d408be1b9742cc31ac2428f1e6 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 15:30:23 +0700 Subject: [PATCH 20/30] ensure headword is stable --- .../FwLite/MiniLcm.Tests/Models/EntryTests.cs | 18 ++++++++++++++++++ backend/FwLite/MiniLcm/Models/Entry.cs | 6 ++++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs diff --git a/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs b/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs new file mode 100644 index 000000000..8b680e716 --- /dev/null +++ b/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs @@ -0,0 +1,18 @@ +namespace MiniLcm.Tests.Models; + +public class EntryTests +{ + [Fact] + public void Headword_SameResultForDifferentOrderedMultiStrings() + { + var entry = new Entry() + { + LexemeForm = new MultiString() { Values = { { "en", "test" }, { "fr", "test2" } } } + }; + var entry2 = new Entry() + { + LexemeForm = new MultiString() { Values = { { "fr", "test2" }, { "en", "test" } } } + }; + entry.Headword().Should().Be(entry2.Headword()); + } +} diff --git a/backend/FwLite/MiniLcm/Models/Entry.cs b/backend/FwLite/MiniLcm/Models/Entry.cs index 7e7239ca5..cabb8c9bb 100644 --- a/backend/FwLite/MiniLcm/Models/Entry.cs +++ b/backend/FwLite/MiniLcm/Models/Entry.cs @@ -28,8 +28,10 @@ public record Entry : IObjectWithId public string Headword() { - var word = CitationForm.Values.Values.FirstOrDefault(); - if (string.IsNullOrEmpty(word)) word = LexemeForm.Values.Values.FirstOrDefault(); + //order by code to ensure the headword is stable + //todo choose ws by preference based on ws order/default + var word = CitationForm.Values.OrderBy(kvp => kvp.Key.Code).FirstOrDefault().Value; + if (string.IsNullOrEmpty(word)) word = LexemeForm.Values.OrderBy(kvp => kvp.Key.Code).FirstOrDefault().Value; return word?.Trim() ?? "(Unknown)"; } From 58fc9a41f10e13545b0d9967f40ecdd1afabc7d7 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 16:14:13 +0700 Subject: [PATCH 21/30] flatten Complex Form types from FW so we import all of them, not just the top level --- .../Api/FwDataMiniLcmApi.cs | 13 ++++++------- .../Api/PossibilityExtensions.cs | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index 1adb5b67f..04a0a0297 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -38,6 +38,8 @@ public class FwDataMiniLcmApi(Lazy cacheLazy, bool onCloseSave, ILogge private ICmTranslationFactory CmTranslationFactory => Cache.ServiceLocator.GetInstance(); private ICmPossibilityRepository CmPossibilityRepository => Cache.ServiceLocator.GetInstance(); private ICmPossibilityList ComplexFormTypes => Cache.LangProject.LexDbOA.ComplexEntryTypesOA; + private IEnumerable ComplexFormTypesFlattened => ComplexFormTypes.PossibilitiesOS.Cast().Flatten(); + private ICmPossibilityList VariantTypes => Cache.LangProject.LexDbOA.VariantEntryTypesOA; public void Dispose() @@ -327,12 +329,9 @@ public Task DeleteSemanticDomain(Guid id) public IAsyncEnumerable GetComplexFormTypes() { - return ComplexFormTypes.PossibilitiesOS - .Select(ToComplexFormType) - .ToAsyncEnumerable(); + return ComplexFormTypesFlattened.Select(ToComplexFormType).ToAsyncEnumerable(); } - - private ComplexFormType ToComplexFormType(ICmPossibility t) + private ComplexFormType ToComplexFormType(ILexEntryType t) { return new ComplexFormType() { Id = t.Guid, Name = FromLcmMultiString(t.Name) }; } @@ -351,7 +350,7 @@ public Task CreateComplexFormType(ComplexFormType complexFormTy ComplexFormTypes.PossibilitiesOS.Add(lexComplexFormType); UpdateLcmMultiString(lexComplexFormType.Name, complexFormType.Name); }); - return Task.FromResult(ToComplexFormType(ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormType.Id))); + return Task.FromResult(ToComplexFormType(ComplexFormTypesFlattened.Single(c => c.Guid == complexFormType.Id))); } public IAsyncEnumerable GetVariantTypes() @@ -702,7 +701,7 @@ internal void AddComplexFormType(ILexEntry lexEntry, Guid complexFormTypeId) entryRef.HideMinorEntry = 0; } - var lexEntryType = (ILexEntryType)ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormTypeId); + var lexEntryType = ComplexFormTypesFlattened.Single(c => c.Guid == complexFormTypeId); entryRef.ComplexEntryTypesRS.Add(lexEntryType); } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs new file mode 100644 index 000000000..e5ee6416f --- /dev/null +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs @@ -0,0 +1,18 @@ +using SIL.LCModel; + +namespace FwDataMiniLcmBridge.Api; + +public static class PossibilityExtensions +{ + public static IEnumerable Flatten(this IEnumerable enumerable) where T : ICmPossibility + { + foreach (var cmPossibility in enumerable) + { + yield return cmPossibility; + foreach (var child in Flatten(cmPossibility.SubPossibilitiesOS.Cast())) + { + yield return child; + } + } + } +} From b501099180c1f519b0155394dc805edd009abfdb Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 16:34:43 +0700 Subject: [PATCH 22/30] ensure that all entries are created before complex forms during import to avoid issues where the complex form does not get created because it's referencing an entry which does not exist --- backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index 21223032e..28cc4048e 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -262,7 +262,12 @@ public async Task BulkCreateEntries(IAsyncEnumerable entries) { var semanticDomains = await SemanticDomains.ToDictionaryAsync(sd => sd.Id, sd => sd); var partsOfSpeech = await PartsOfSpeech.ToDictionaryAsync(p => p.Id, p => p); - await dataModel.AddChanges(ClientId, entries.ToBlockingEnumerable().SelectMany(entry => CreateEntryChanges(entry, semanticDomains, partsOfSpeech))); + await dataModel.AddChanges(ClientId, + entries.ToBlockingEnumerable() + .SelectMany(entry => CreateEntryChanges(entry, semanticDomains, partsOfSpeech)) + //force entries to be created first, this avoids issues where references are created before the entry is created + .OrderBy(c => c is CreateEntryChange ? 0 : 1) + ); } private IEnumerable CreateEntryChanges(Entry entry, Dictionary semanticDomains, Dictionary partsOfSpeech) From 10b0994beeae9e27847b3c4a6b8c4b270a498bbb Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 16:35:29 +0700 Subject: [PATCH 23/30] improve how entry assertions are executed to make it simpler to find the data causing the problem --- .../FwLiteProjectSync.Tests/Sena3SyncTests.cs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs index 17b9216b6..1d62312cb 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -1,4 +1,5 @@ using FluentAssertions.Equivalency; +using FluentAssertions.Execution; using FwLiteProjectSync.Tests.Fixtures; using LcmCrdt; using Microsoft.EntityFrameworkCore; @@ -35,11 +36,20 @@ public async Task FirstSena3SyncJustDoesAnImport() var fwdataApi = _fixture.FwDataApi; await _syncService.Sync(crdtApi, fwdataApi); - var crdtEntries = await crdtApi.GetEntries().ToArrayAsync(); - var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); - crdtEntries.Should().BeEquivalentTo(fwdataEntries, - options => options.For(e => e.Components).Exclude(c => c.Id) - .For(e => e.ComplexForms).Exclude(c => c.Id)); + var crdtEntries = await crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await fwdataApi.GetEntries().ToDictionaryAsync(e => e.Id); + crdtEntries.Keys.Should().BeEquivalentTo(fwdataEntries.Keys); + using (new AssertionScope()) + { + foreach (var crdtEntry in crdtEntries.Values) + { + var fwdataEntry = fwdataEntries[crdtEntry.Id]; + crdtEntry.Should().BeEquivalentTo(fwdataEntry, + options => options + .For(e => e.Components).Exclude(c => c.Id) + .For(e => e.ComplexForms).Exclude(c => c.Id), $"entry {crdtEntry.Id} should include fw data"); + } + } } [Fact] From d275bd24731a02a73ffe644a80a3f2ca967f3187 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 26 Nov 2024 16:36:13 +0700 Subject: [PATCH 24/30] use the entry code for headwords when creating complex form components --- .../FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index 04a0a0297..599e4a900 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -390,6 +390,15 @@ private Entry FromLexEntry(ILexEntry entry) }; } + private string LexEntryHeadword(ILexEntry entry) + { + return new Entry() + { + LexemeForm = FromLcmMultiString(entry.LexemeFormOA.Form), + CitationForm = FromLcmMultiString(entry.CitationForm), + }.Headword(); + } + private IList ToComplexFormTypes(ILexEntry entry) { return entry.ComplexFormEntryRefs.SingleOrDefault() @@ -438,9 +447,9 @@ private ComplexFormComponent ToEntryReference(ILexEntry component, ILexEntry com return new ComplexFormComponent { ComponentEntryId = component.Guid, - ComponentHeadword = component.HeadWord.Text, + ComponentHeadword = LexEntryHeadword(component), ComplexFormEntryId = complexEntry.Guid, - ComplexFormHeadword = complexEntry.HeadWord.Text + ComplexFormHeadword = LexEntryHeadword(complexEntry) }; } @@ -452,7 +461,7 @@ private ComplexFormComponent ToSenseReference(ILexSense componentSense, ILexEntr ComponentSenseId = componentSense.Guid, ComponentHeadword = componentSense.Entry.HeadWord.Text, ComplexFormEntryId = complexEntry.Guid, - ComplexFormHeadword = complexEntry.HeadWord.Text + ComplexFormHeadword = LexEntryHeadword(complexEntry) }; } From 977d2e9c11f21683d9434f622bb9386037491c89 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 27 Nov 2024 14:05:19 +0700 Subject: [PATCH 25/30] fix broken build due to duplicate type mismatch --- .../FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 79e1dcfe9..19a00f4db 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -14,8 +14,6 @@ namespace FwLiteProjectSync; public class CrdtFwdataProjectSyncService(MiniLcmImport miniLcmImport, ILogger logger) { - public record SyncResult(int CrdtChanges, int FwdataChanges); - public record DryRunSyncResult( int CrdtChanges, int FwdataChanges, From 75c5a5f0d52e3536441c4446693fd79babaf1c17 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 28 Nov 2024 09:52:14 +0700 Subject: [PATCH 26/30] change error message to follow the expected pattern --- backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs index 1d62312cb..5c08cc3e0 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -47,7 +47,7 @@ public async Task FirstSena3SyncJustDoesAnImport() crdtEntry.Should().BeEquivalentTo(fwdataEntry, options => options .For(e => e.Components).Exclude(c => c.Id) - .For(e => e.ComplexForms).Exclude(c => c.Id), $"entry {crdtEntry.Id} should include fw data"); + .For(e => e.ComplexForms).Exclude(c => c.Id), $"CRDT entry {crdtEntry.Id} was synced with FwData"); } } } From 36ca09a95258e2ad44ad62094be04274a8901247 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 28 Nov 2024 10:24:02 +0700 Subject: [PATCH 27/30] add a number of DryRun tests --- .../FwLiteProjectSync.Tests/Sena3SyncTests.cs | 96 +++++++++++++++++-- .../FwLiteProjectSync.Tests/SyncTests.cs | 41 +++++++- .../CrdtFwdataProjectSyncService.cs | 3 +- 3 files changed, 127 insertions(+), 13 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs index 5c08cc3e0..0d98a93a9 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -29,15 +29,8 @@ public Sena3SyncTests(Sena3Fixture fixture) _syncService = _fixture.SyncService; } - [Fact] - public async Task FirstSena3SyncJustDoesAnImport() + private void ShouldAllBeEquivalentTo(Dictionary crdtEntries, Dictionary fwdataEntries) { - var crdtApi = _fixture.CrdtApi; - var fwdataApi = _fixture.FwDataApi; - await _syncService.Sync(crdtApi, fwdataApi); - - var crdtEntries = await crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); - var fwdataEntries = await fwdataApi.GetEntries().ToDictionaryAsync(e => e.Id); crdtEntries.Keys.Should().BeEquivalentTo(fwdataEntries.Keys); using (new AssertionScope()) { @@ -47,11 +40,96 @@ public async Task FirstSena3SyncJustDoesAnImport() crdtEntry.Should().BeEquivalentTo(fwdataEntry, options => options .For(e => e.Components).Exclude(c => c.Id) - .For(e => e.ComplexForms).Exclude(c => c.Id), $"CRDT entry {crdtEntry.Id} was synced with FwData"); + .For(e => e.ComplexForms).Exclude(c => c.Id), + $"CRDT entry {crdtEntry.Id} was synced with FwData"); } } } + //by default the first sync is an import, this will skip that so that the sync will actually sync data + private async Task BypassImport() + { + await _syncService.SaveProjectSnapshot(_fixture.FwDataApi.Project, new ([], [], [])); + } + + //this lets us query entries when there is no writing system + private async Task WorkaroundMissingWritingSystems() + { + //must have at least one writing system to query for entries + await _fixture.CrdtApi.CreateWritingSystem(WritingSystemType.Vernacular, (await _fixture.FwDataApi.GetWritingSystems()).Vernacular.First()); + + } + + [Fact] + public async Task DryRunImport_MakesNoChanges() + { + var crdtApi = _fixture.CrdtApi; + await WorkaroundMissingWritingSystems(); + crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + var fwdataApi = _fixture.FwDataApi; + await _syncService.SyncDryRun(crdtApi, fwdataApi); + //should still be empty + crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + } + + [Fact] + public async Task DryRunImport_MakesTheSameChangesAsImport() + { + var dryRunSyncResult = await _syncService.SyncDryRun(_fixture.CrdtApi, _fixture.FwDataApi); + var syncResult = await _syncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + dryRunSyncResult.Should().BeEquivalentTo(syncResult); + } + + [Fact] + public async Task DryRunSync_MakesNoChanges() + { + await BypassImport(); + var crdtApi = _fixture.CrdtApi; + await WorkaroundMissingWritingSystems(); + crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + var fwdataApi = _fixture.FwDataApi; + await _syncService.SyncDryRun(crdtApi, fwdataApi); + //should still be empty + crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + } + + [Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")] + public async Task DryRunSync_MakesTheSameChangesAsImport() + { + await BypassImport(); + var dryRunSyncResult = await _syncService.SyncDryRun(_fixture.CrdtApi, _fixture.FwDataApi); + var syncResult = await _syncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + dryRunSyncResult.Should().BeEquivalentTo(syncResult); + } + + [Fact] + public async Task FirstSena3SyncJustDoesAnSync() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + var results = await _syncService.Sync(crdtApi, fwdataApi); + results.FwdataChanges.Should().Be(0); + results.CrdtChanges.Should().BeGreaterThan(fwdataApi.EntryCount); + + var crdtEntries = await crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await fwdataApi.GetEntries().ToDictionaryAsync(e => e.Id); + ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries); + } + + [Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")] + public async Task SyncWithoutImport_CrdtShouldMatchFwdata() + { + await BypassImport(); + + var results = await _syncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + results.FwdataChanges.Should().Be(0); + results.CrdtChanges.Should().BeGreaterThan(_fixture.FwDataApi.EntryCount); + + var crdtEntries = await _fixture.CrdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await _fixture.FwDataApi.GetEntries().ToDictionaryAsync(e => e.Id); + ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries); + } + [Fact] public async Task SecondSena3SyncDoesNothing() { diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index b798a9d0c..36b4084f2 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -95,9 +95,9 @@ public async Task SecondSyncDoesNothing() var crdtApi = _fixture.CrdtApi; var fwdataApi = _fixture.FwDataApi; await _syncService.Sync(crdtApi, fwdataApi); - var secondSync = await _syncService.SyncDryRun(crdtApi, fwdataApi); - secondSync.CrdtChanges.Should().Be(0, $"changes were {string.Join(", ", secondSync.CrdtDryRunRecords)}"); - secondSync.FwdataChanges.Should().Be(0, $"changes were {string.Join(", ", secondSync.FwDataDryRunRecords)}"); + var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); } [Fact] @@ -151,6 +151,41 @@ await crdtApi.CreateEntry(new Entry() .For(e => e.ComplexForms).Exclude(c => c.Id)); } + [Fact] + public async Task SyncDryRun_NoChangesAreSynced() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + var fwDataEntryId = Guid.NewGuid(); + var crdtEntryId = Guid.NewGuid(); + + await fwdataApi.CreateEntry(new Entry() + { + Id = fwDataEntryId, + LexemeForm = { { "en", "Pear" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Pear" } }, } + ] + }); + await crdtApi.CreateEntry(new Entry() + { + Id = crdtEntryId, + LexemeForm = { { "en", "Banana" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Banana" } }, } + ] + }); + await _syncService.SyncDryRun(crdtApi, fwdataApi); + + var crdtEntries = await crdtApi.GetEntries().ToArrayAsync(); + var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); + crdtEntries.Select(e => e.Id).Should().NotContain(fwDataEntryId); + fwdataEntries.Select(e => e.Id).Should().NotContain(crdtEntryId); + } + [Fact] public async Task CreatingAComplexEntryInFwDataSyncsWithoutIssue() { diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 19a00f4db..58685043b 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -58,6 +58,7 @@ private async Task Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi, { await miniLcmImport.ImportProject(crdtApi, fwdataApi, entryCount); LogDryRun(crdtApi, "crdt"); + if (dryRun) return new DryRunSyncResult(entryCount, 0, GetDryRunRecords(crdtApi), []); return new SyncResult(entryCount, 0); } @@ -109,7 +110,7 @@ public record ProjectSnapshot(Entry[] Entries, PartOfSpeech[] PartsOfSpeech, Sem return await JsonSerializer.DeserializeAsync(file); } - private async Task SaveProjectSnapshot(FwDataProject project, ProjectSnapshot projectSnapshot) + internal async Task SaveProjectSnapshot(FwDataProject project, ProjectSnapshot projectSnapshot) { var snapshotPath = SnapshotPath(project); await using var file = File.Create(snapshotPath); From de9f95fca9f11c3de49e715e4b5f64b950e9b0dc Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 28 Nov 2024 11:27:21 +0700 Subject: [PATCH 28/30] don't try to download sena3 from lexbox, just download the zip from google drive --- .../Fixtures/LexboxConfig.cs | 14 -------- .../Fixtures/Sena3SyncFixture.cs | 35 +++++-------------- .../FwLiteProjectSync.Tests/Sena3SyncTests.cs | 3 +- 3 files changed, 10 insertions(+), 42 deletions(-) delete mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Fixtures/LexboxConfig.cs diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/LexboxConfig.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/LexboxConfig.cs deleted file mode 100644 index 7385cbbc8..000000000 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/LexboxConfig.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.ComponentModel.DataAnnotations; - -namespace FwLiteProjectSync.Tests.Fixtures; - -public class LexboxConfig -{ - [Required, Url, RegularExpression(@"^.+/$", ErrorMessage = "Must end with '/'")] - public required string LexboxUrl { get; set; } - public string HgWebUrl => $"{LexboxUrl}hg/"; - [Required] - public required string LexboxUsername { get; set; } - [Required] - public required string LexboxPassword { get; set; } -} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs index d1b5ca6bf..499256a05 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs @@ -26,7 +26,6 @@ public class Sena3Fixture : IAsyncLifetime public IServiceProvider Services => _services.ServiceProvider; private IDisposable _cleanup; - private readonly LexboxConfig lexboxConfig; private readonly HttpClient http; public CrdtMiniLcmApi CrdtApi { get; set; } = null!; public FwDataMiniLcmApi FwDataApi { get; set; } = null!; @@ -36,21 +35,9 @@ public Sena3Fixture() { var services = new ServiceCollection() .AddSyncServices(nameof(Sena3Fixture), false); - services.AddOptions() - .BindConfiguration("LexboxConfig") - .Configure(c => - { - // TODO: How do I set default values if and only if they're not already set (e.g., via environment variables)? - c.LexboxUrl = "http://localhost/"; - c.LexboxUsername = "admin"; - c.LexboxPassword = "pass"; - }) - .ValidateDataAnnotations() - .ValidateOnStart(); var rootServiceProvider = services.BuildServiceProvider(); _cleanup = Defer.Action(() => rootServiceProvider.Dispose()); _services = rootServiceProvider.CreateAsyncScope(); - lexboxConfig = Services.GetRequiredService>().Value; var factory = Services.GetRequiredService(); http = factory.CreateClient(nameof(Sena3Fixture)); } @@ -86,30 +73,24 @@ public async Task DisposeAsync() _cleanup.Dispose(); } - public async Task DownloadProjectBackupStream(string code) + public async Task DownloadSena3ProjectBackupStream() { - var backupUrl = new Uri($"{lexboxConfig.LexboxUrl}api/project/backupProject/{code}"); - var result = await http.GetAsync(backupUrl); + var backupUrl = new Uri("https://drive.google.com/uc?export=download&id=1I-hwc0RHoQqW774gbS5qR-GHa1E7BlsS"); + var result = await http.GetAsync(backupUrl, HttpCompletionOption.ResponseHeadersRead); return await result.Content.ReadAsStreamAsync(); } - - public async Task LoginAs(string lexboxUsername, string lexboxPassword) - { - if (AlreadyLoggedIn) return; - await http.PostAsync($"{lexboxConfig.LexboxUrl}api/login", JsonContent.Create(new { EmailOrUsername=lexboxUsername, Password=lexboxPassword })); - AlreadyLoggedIn = true; - } - public async Task DownloadSena3() { var tempFolder = Path.Combine(Path.GetTempPath(), nameof(Sena3Fixture)); var sena3MasterCopy = Path.Combine(tempFolder, "sena-3"); if (!Directory.Exists(sena3MasterCopy) || !File.Exists(Path.Combine(sena3MasterCopy, "sena-3.fwdata"))) { - await LoginAs(lexboxConfig.LexboxUsername, lexboxConfig.LexboxPassword); Directory.CreateDirectory(sena3MasterCopy); - await using var zipStream = await DownloadProjectBackupStream("sena-3"); - ZipFile.ExtractToDirectory(zipStream, sena3MasterCopy); + await using var zipStream = await DownloadSena3ProjectBackupStream(); + //the zip file is structured like this: /sena-3/.hg + //by extracting it to tempFolder it should merge with sena-3 + ZipFile.ExtractToDirectory(zipStream, tempFolder); + MercurialTestHelper.HgUpdate(sena3MasterCopy, "tip"); LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(new NullProgress(), false, Path.Combine(sena3MasterCopy, "sena-3.fwdata")); MercurialTestHelper.HgClean(sena3MasterCopy, "sena-3.fwdata"); diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs index 0d98a93a9..8963f1a34 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -17,6 +17,7 @@ public class Sena3SyncTests : IClassFixture, IAsyncLifetime public async Task InitializeAsync() { + _fixture.FwDataApi.EntryCount.Should().BeGreaterThan(100, "project should be loaded and have entries"); } public async Task DisposeAsync() @@ -109,7 +110,7 @@ public async Task FirstSena3SyncJustDoesAnSync() var fwdataApi = _fixture.FwDataApi; var results = await _syncService.Sync(crdtApi, fwdataApi); results.FwdataChanges.Should().Be(0); - results.CrdtChanges.Should().BeGreaterThan(fwdataApi.EntryCount); + results.CrdtChanges.Should().BeGreaterThanOrEqualTo(fwdataApi.EntryCount); var crdtEntries = await crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); var fwdataEntries = await fwdataApi.GetEntries().ToDictionaryAsync(e => e.Id); From c3f71aa406dfefa339d64e1a0b0eb3a084fde313 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 28 Nov 2024 12:20:53 +0700 Subject: [PATCH 29/30] isolate sena3 sync tests rather than using the same project for each test --- .../Fixtures/Sena3SyncFixture.cs | 75 +++++++++---------- .../FwLiteProjectSync.Tests/Sena3SyncTests.cs | 74 +++++++++--------- 2 files changed, 71 insertions(+), 78 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs index 499256a05..3bb84e1ca 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs @@ -1,17 +1,10 @@ using System.IO.Compression; -using System.Net.Http.Json; -using System.Runtime.CompilerServices; using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.Api; -using FwDataMiniLcmBridge.LcmUtils; -using FwDataMiniLcmBridge.Tests.Fixtures; using LcmCrdt; using LexCore.Utils; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using MiniLcm; using SIL.IO; using SIL.Progress; @@ -19,67 +12,67 @@ namespace FwLiteProjectSync.Tests.Fixtures; public class Sena3Fixture : IAsyncLifetime { - private readonly AsyncServiceScope _services; + private static readonly HttpClient http = new HttpClient(); - public CrdtFwdataProjectSyncService SyncService => - _services.ServiceProvider.GetRequiredService(); - - public IServiceProvider Services => _services.ServiceProvider; - private IDisposable _cleanup; - private readonly HttpClient http; - public CrdtMiniLcmApi CrdtApi { get; set; } = null!; - public FwDataMiniLcmApi FwDataApi { get; set; } = null!; - private bool AlreadyLoggedIn { get; set; } = false; - - public Sena3Fixture() + public async Task InitializeAsync() { var services = new ServiceCollection() .AddSyncServices(nameof(Sena3Fixture), false); var rootServiceProvider = services.BuildServiceProvider(); - _cleanup = Defer.Action(() => rootServiceProvider.Dispose()); - _services = rootServiceProvider.CreateAsyncScope(); - var factory = Services.GetRequiredService(); - http = factory.CreateClient(nameof(Sena3Fixture)); + var fwProjectsFolder = rootServiceProvider.GetRequiredService>() + .Value + .ProjectsFolder; + if (Path.Exists(fwProjectsFolder)) Directory.Delete(fwProjectsFolder, true); + Directory.CreateDirectory(fwProjectsFolder); + + var crdtProjectsFolder = + rootServiceProvider.GetRequiredService>().Value.ProjectPath; + if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); + rootServiceProvider.Dispose(); + + Directory.CreateDirectory(crdtProjectsFolder); + await DownloadSena3(); } - public async Task InitializeAsync() + public async Task<(CrdtMiniLcmApi CrdtApi, FwDataMiniLcmApi FwDataApi, IServiceProvider services, IDisposable cleanup)> SetupProjects() { var sena3MasterCopy = await DownloadSena3(); + + var rootServiceProvider = new ServiceCollection() + .AddSyncServices(nameof(Sena3Fixture), false) + .BuildServiceProvider(); + var cleanup = Defer.Action(() => rootServiceProvider.Dispose()); + var services = rootServiceProvider.CreateAsyncScope().ServiceProvider; var projectName = "sena-3_" + Guid.NewGuid().ToString("N"); - var projectsFolder = Services.GetRequiredService>().Value + var projectsFolder = services.GetRequiredService>() + .Value .ProjectsFolder; - if (Path.Exists(projectsFolder)) Directory.Delete(projectsFolder, true); - Directory.CreateDirectory(projectsFolder); var fwDataProject = new FwDataProject(projectName, projectsFolder); var fwDataProjectPath = Path.Combine(fwDataProject.ProjectsPath, fwDataProject.Name); DirectoryHelper.Copy(sena3MasterCopy, fwDataProjectPath); File.Move(Path.Combine(fwDataProjectPath, "sena-3.fwdata"), fwDataProject.FilePath); + var fwDataMiniLcmApi = services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); - FwDataApi = Services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); - - var crdtProjectsFolder = - Services.GetRequiredService>().Value.ProjectPath; - if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); - Directory.CreateDirectory(crdtProjectsFolder); - var crdtProject = await Services.GetRequiredService() - .CreateProject(new(projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: false)); - CrdtApi = (CrdtMiniLcmApi) await Services.OpenCrdtProject(crdtProject); + var crdtProject = await services.GetRequiredService() + .CreateProject(new(projectName, FwProjectId: fwDataMiniLcmApi.ProjectId, SeedNewProjectData: false)); + var crdtMiniLcmApi = (CrdtMiniLcmApi)await services.OpenCrdtProject(crdtProject); + return (crdtMiniLcmApi, fwDataMiniLcmApi, services, cleanup); } - public async Task DisposeAsync() + public Task DisposeAsync() { - await _services.DisposeAsync(); - _cleanup.Dispose(); + return Task.CompletedTask; } - public async Task DownloadSena3ProjectBackupStream() + private async Task DownloadSena3ProjectBackupStream() { var backupUrl = new Uri("https://drive.google.com/uc?export=download&id=1I-hwc0RHoQqW774gbS5qR-GHa1E7BlsS"); var result = await http.GetAsync(backupUrl, HttpCompletionOption.ResponseHeadersRead); return await result.Content.ReadAsStreamAsync(); } - public async Task DownloadSena3() + + private async Task DownloadSena3() { var tempFolder = Path.Combine(Path.GetTempPath(), nameof(Sena3Fixture)); var sena3MasterCopy = Path.Combine(tempFolder, "sena-3"); diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs index 8963f1a34..77add42f5 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -1,5 +1,6 @@ using FluentAssertions.Equivalency; using FluentAssertions.Execution; +using FwDataMiniLcmBridge.Api; using FwLiteProjectSync.Tests.Fixtures; using LcmCrdt; using Microsoft.EntityFrameworkCore; @@ -13,21 +14,28 @@ namespace FwLiteProjectSync.Tests; public class Sena3SyncTests : IClassFixture, IAsyncLifetime { private readonly Sena3Fixture _fixture; - private readonly CrdtFwdataProjectSyncService _syncService; + private CrdtFwdataProjectSyncService _syncService = null!; + private CrdtMiniLcmApi _crdtApi = null!; + private FwDataMiniLcmApi _fwDataApi = null!; + private IDisposable? _cleanup; - public async Task InitializeAsync() + + public Sena3SyncTests(Sena3Fixture fixture) { - _fixture.FwDataApi.EntryCount.Should().BeGreaterThan(100, "project should be loaded and have entries"); + _fixture = fixture; } - public async Task DisposeAsync() + public async Task InitializeAsync() { + (_crdtApi, _fwDataApi, var services, _cleanup) = await _fixture.SetupProjects(); + _syncService = services.GetRequiredService(); + _fwDataApi.EntryCount.Should().BeGreaterThan(100, "project should be loaded and have entries"); } - public Sena3SyncTests(Sena3Fixture fixture) + public Task DisposeAsync() { - _fixture = fixture; - _syncService = _fixture.SyncService; + _cleanup?.Dispose(); + return Task.CompletedTask; } private void ShouldAllBeEquivalentTo(Dictionary crdtEntries, Dictionary fwdataEntries) @@ -50,34 +58,32 @@ private void ShouldAllBeEquivalentTo(Dictionary crdtEntries, Dictio //by default the first sync is an import, this will skip that so that the sync will actually sync data private async Task BypassImport() { - await _syncService.SaveProjectSnapshot(_fixture.FwDataApi.Project, new ([], [], [])); + await _syncService.SaveProjectSnapshot(_fwDataApi.Project, new ([], [], [])); } //this lets us query entries when there is no writing system private async Task WorkaroundMissingWritingSystems() { //must have at least one writing system to query for entries - await _fixture.CrdtApi.CreateWritingSystem(WritingSystemType.Vernacular, (await _fixture.FwDataApi.GetWritingSystems()).Vernacular.First()); + await _crdtApi.CreateWritingSystem(WritingSystemType.Vernacular, (await _fwDataApi.GetWritingSystems()).Vernacular.First()); } [Fact] public async Task DryRunImport_MakesNoChanges() { - var crdtApi = _fixture.CrdtApi; await WorkaroundMissingWritingSystems(); - crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); - var fwdataApi = _fixture.FwDataApi; - await _syncService.SyncDryRun(crdtApi, fwdataApi); + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + await _syncService.SyncDryRun(_crdtApi, _fwDataApi); //should still be empty - crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); } [Fact] public async Task DryRunImport_MakesTheSameChangesAsImport() { - var dryRunSyncResult = await _syncService.SyncDryRun(_fixture.CrdtApi, _fixture.FwDataApi); - var syncResult = await _syncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + var dryRunSyncResult = await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + var syncResult = await _syncService.Sync(_crdtApi, _fwDataApi); dryRunSyncResult.Should().BeEquivalentTo(syncResult); } @@ -85,35 +91,31 @@ public async Task DryRunImport_MakesTheSameChangesAsImport() public async Task DryRunSync_MakesNoChanges() { await BypassImport(); - var crdtApi = _fixture.CrdtApi; await WorkaroundMissingWritingSystems(); - crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); - var fwdataApi = _fixture.FwDataApi; - await _syncService.SyncDryRun(crdtApi, fwdataApi); + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + await _syncService.SyncDryRun(_crdtApi, _fwDataApi); //should still be empty - crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); } [Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")] public async Task DryRunSync_MakesTheSameChangesAsImport() { await BypassImport(); - var dryRunSyncResult = await _syncService.SyncDryRun(_fixture.CrdtApi, _fixture.FwDataApi); - var syncResult = await _syncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + var dryRunSyncResult = await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + var syncResult = await _syncService.Sync(_crdtApi, _fwDataApi); dryRunSyncResult.Should().BeEquivalentTo(syncResult); } [Fact] public async Task FirstSena3SyncJustDoesAnSync() { - var crdtApi = _fixture.CrdtApi; - var fwdataApi = _fixture.FwDataApi; - var results = await _syncService.Sync(crdtApi, fwdataApi); + var results = await _syncService.Sync(_crdtApi, _fwDataApi); results.FwdataChanges.Should().Be(0); - results.CrdtChanges.Should().BeGreaterThanOrEqualTo(fwdataApi.EntryCount); + results.CrdtChanges.Should().BeGreaterThanOrEqualTo(_fwDataApi.EntryCount); - var crdtEntries = await crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); - var fwdataEntries = await fwdataApi.GetEntries().ToDictionaryAsync(e => e.Id); + var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id); ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries); } @@ -122,22 +124,20 @@ public async Task SyncWithoutImport_CrdtShouldMatchFwdata() { await BypassImport(); - var results = await _syncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + var results = await _syncService.Sync(_crdtApi, _fwDataApi); results.FwdataChanges.Should().Be(0); - results.CrdtChanges.Should().BeGreaterThan(_fixture.FwDataApi.EntryCount); + results.CrdtChanges.Should().BeGreaterThan(_fwDataApi.EntryCount); - var crdtEntries = await _fixture.CrdtApi.GetEntries().ToDictionaryAsync(e => e.Id); - var fwdataEntries = await _fixture.FwDataApi.GetEntries().ToDictionaryAsync(e => e.Id); + var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id); ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries); } [Fact] public async Task SecondSena3SyncDoesNothing() { - var crdtApi = _fixture.CrdtApi; - var fwdataApi = _fixture.FwDataApi; - await _syncService.Sync(crdtApi, fwdataApi); - var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + await _syncService.Sync(_crdtApi, _fwDataApi); + var secondSync = await _syncService.Sync(_crdtApi, _fwDataApi); secondSync.CrdtChanges.Should().Be(0); secondSync.FwdataChanges.Should().Be(0); } From e00fcfc5b115fd5ceba1660b91c79c6c567cae9e Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 28 Nov 2024 12:21:17 +0700 Subject: [PATCH 30/30] use HgRunner instead of trying to locate mercurial manually --- .../FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs index afccd122a..e0ec3f816 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs @@ -1,3 +1,4 @@ +using Chorus.VcsDrivers.Mercurial; using SIL.CommandLineProcessing; using SIL.PlatformUtilities; using SIL.Progress; @@ -6,12 +7,11 @@ namespace FwLiteProjectSync.Tests.Fixtures; public static class MercurialTestHelper { - public static string HgCommand => - Path.Combine("Mercurial", Platform.IsWindows ? "hg.exe" : "hg"); + private static readonly NullProgress NullProgress = new NullProgress(); private static string RunHgCommand(string repoPath, string args) { - var result = CommandLineRunner.Run(HgCommand, args, repoPath, 120, new NullProgress()); + var result = HgRunner.Run(args, repoPath, 120, NullProgress); if (result.ExitCode == 0) return result.StandardOutput; throw new Exception( $"hg {args} failed.\nStdOut: {result.StandardOutput}\nStdErr: {result.StandardError}");