From 18530d4066e75cfa92f09cd40b1274f9e0ca7727 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 28 Nov 2024 13:40:08 +0700 Subject: [PATCH] Add test for syncing complex forms twice (#1256) * Add integration test for syncing sena-3 * prevent duplicating complex forms which are the same/already exist * catch and throw some exceptions with additional context to help debugging * change test project vernacular ws and change how ComplexFormComponent headwords get set so that they match what we get from fieldworks * ensure root service provider is cleaned up in fixtures to avoid issue where test host hangs once tests are done * ensure headword is stable * flatten Complex Form types from FW so we import all of them, not just the top level * ensure that all entries are created before complex forms during import to avoid issues where the complex form does not get created because it's referencing an entry which does not exist * don't try to download sena3 from lexbox, just download the zip from google drive * isolate sena3 sync tests rather than using the same project for each test * use HgRunner instead of trying to locate mercurial manually --------- Co-authored-by: Kevin Hahn --- .../Fixtures/FwDataTestsKernel.cs | 11 +- .../Api/FwDataMiniLcmApi.cs | 106 +++++++------ .../Api/PossibilityExtensions.cs | 18 +++ .../Api/UpdateProxy/UpdateEntryProxy.cs | 2 +- .../FwLiteProjectSync.Tests/EntrySyncTests.cs | 2 +- .../Fixtures/MercurialTestHelper.cs | 31 ++++ .../Fixtures/Sena3SyncFixture.cs | 93 +++++++++++ .../Fixtures/SyncFixture.cs | 30 ++-- .../Fixtures/TestingKernel.cs | 27 ++++ .../FwLiteProjectSync.Tests.csproj | 11 ++ .../FwLiteProjectSync.Tests/Sena3SyncTests.cs | 144 ++++++++++++++++++ .../FwLiteProjectSync.Tests/SyncTests.cs | 127 ++++++++++++++- .../CrdtFwdataProjectSyncService.cs | 43 ++++-- .../FwLiteProjectSync/DryRunMiniLcmApi.cs | 2 +- .../FwLiteProjectSync.csproj | 4 + ...elSnapshotTests.VerifyDbModel.verified.txt | 9 +- .../JsonPatchEntryRewriteTests.cs | 2 - .../Entries/AddEntryComponentChange.cs | 19 +-- backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs | 31 ++-- backend/FwLite/LcmCrdt/LcmCrdtKernel.cs | 15 ++ .../ComplexFormComponentTestsBase.cs | 55 +++++++ .../FwLite/MiniLcm.Tests/Models/EntryTests.cs | 18 +++ .../Exceptions/CreateObjectException.cs | 12 ++ .../MiniLcm/Exceptions/SyncObjectException.cs | 12 ++ backend/FwLite/MiniLcm/Models/Entry.cs | 13 +- .../MiniLcm/SyncHelpers/DiffCollection.cs | 68 +++++++++ .../FwLite/MiniLcm/SyncHelpers/EntrySync.cs | 32 ++-- 27 files changed, 818 insertions(+), 119 deletions(-) create mode 100644 backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs create mode 100644 backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs create mode 100644 backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs create mode 100644 backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs create mode 100644 backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs index cc951c27d..84a55beee 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs @@ -6,13 +6,16 @@ namespace FwDataMiniLcmBridge.Tests.Fixtures; public static class FwDataTestsKernel { - public static IServiceCollection AddTestFwDataBridge(this IServiceCollection services) + public static IServiceCollection AddTestFwDataBridge(this IServiceCollection services, bool mockProjectLoader = true) { services.AddFwDataBridge(); services.AddSingleton(_ => new ConfigurationRoot([])); - services.AddSingleton(); - services.AddSingleton(sp => sp.GetRequiredService()); - services.AddSingleton(); + if (mockProjectLoader) + { + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + } return services; } } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index 8e132b2e6..599e4a900 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -6,6 +6,7 @@ using FwDataMiniLcmBridge.LcmUtils; using Microsoft.Extensions.Logging; using MiniLcm; +using MiniLcm.Exceptions; using MiniLcm.Models; using MiniLcm.SyncHelpers; using SIL.LCModel; @@ -37,6 +38,8 @@ public class FwDataMiniLcmApi(Lazy cacheLazy, bool onCloseSave, ILogge private ICmTranslationFactory CmTranslationFactory => Cache.ServiceLocator.GetInstance(); private ICmPossibilityRepository CmPossibilityRepository => Cache.ServiceLocator.GetInstance(); private ICmPossibilityList ComplexFormTypes => Cache.LangProject.LexDbOA.ComplexEntryTypesOA; + private IEnumerable ComplexFormTypesFlattened => ComplexFormTypes.PossibilitiesOS.Cast().Flatten(); + private ICmPossibilityList VariantTypes => Cache.LangProject.LexDbOA.VariantEntryTypesOA; public void Dispose() @@ -326,12 +329,9 @@ public Task DeleteSemanticDomain(Guid id) public IAsyncEnumerable GetComplexFormTypes() { - return ComplexFormTypes.PossibilitiesOS - .Select(ToComplexFormType) - .ToAsyncEnumerable(); + return ComplexFormTypesFlattened.Select(ToComplexFormType).ToAsyncEnumerable(); } - - private ComplexFormType ToComplexFormType(ICmPossibility t) + private ComplexFormType ToComplexFormType(ILexEntryType t) { return new ComplexFormType() { Id = t.Guid, Name = FromLcmMultiString(t.Name) }; } @@ -350,7 +350,7 @@ public Task CreateComplexFormType(ComplexFormType complexFormTy ComplexFormTypes.PossibilitiesOS.Add(lexComplexFormType); UpdateLcmMultiString(lexComplexFormType.Name, complexFormType.Name); }); - return Task.FromResult(ToComplexFormType(ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormType.Id))); + return Task.FromResult(ToComplexFormType(ComplexFormTypesFlattened.Single(c => c.Guid == complexFormType.Id))); } public IAsyncEnumerable GetVariantTypes() @@ -390,6 +390,15 @@ private Entry FromLexEntry(ILexEntry entry) }; } + private string LexEntryHeadword(ILexEntry entry) + { + return new Entry() + { + LexemeForm = FromLcmMultiString(entry.LexemeFormOA.Form), + CitationForm = FromLcmMultiString(entry.CitationForm), + }.Headword(); + } + private IList ToComplexFormTypes(ILexEntry entry) { return entry.ComplexFormEntryRefs.SingleOrDefault() @@ -438,9 +447,9 @@ private ComplexFormComponent ToEntryReference(ILexEntry component, ILexEntry com return new ComplexFormComponent { ComponentEntryId = component.Guid, - ComponentHeadword = component.HeadWord.Text, + ComponentHeadword = LexEntryHeadword(component), ComplexFormEntryId = complexEntry.Guid, - ComplexFormHeadword = complexEntry.HeadWord.Text + ComplexFormHeadword = LexEntryHeadword(complexEntry) }; } @@ -452,7 +461,7 @@ private ComplexFormComponent ToSenseReference(ILexSense componentSense, ILexEntr ComponentSenseId = componentSense.Guid, ComponentHeadword = componentSense.Entry.HeadWord.Text, ComplexFormEntryId = complexEntry.Guid, - ComplexFormHeadword = complexEntry.HeadWord.Text + ComplexFormHeadword = LexEntryHeadword(complexEntry) }; } @@ -559,40 +568,48 @@ public IAsyncEnumerable SearchEntries(string query, QueryOptions? options public async Task CreateEntry(Entry entry) { entry.Id = entry.Id == default ? Guid.NewGuid() : entry.Id; - UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create Entry", - "Remove entry", - Cache.ServiceLocator.ActionHandler, - () => - { - var lexEntry = LexEntryFactory.Create(entry.Id, Cache.ServiceLocator.GetInstance().Singleton.LexDbOA); - lexEntry.LexemeFormOA = Cache.ServiceLocator.GetInstance().Create(); - UpdateLcmMultiString(lexEntry.LexemeFormOA.Form, entry.LexemeForm); - UpdateLcmMultiString(lexEntry.CitationForm, entry.CitationForm); - UpdateLcmMultiString(lexEntry.LiteralMeaning, entry.LiteralMeaning); - UpdateLcmMultiString(lexEntry.Comment, entry.Note); - - foreach (var sense in entry.Senses) - { - CreateSense(lexEntry, sense); - } - - //form types should be created before components, otherwise the form type "unspecified" will be added - foreach (var complexFormType in entry.ComplexFormTypes) - { - AddComplexFormType(lexEntry, complexFormType.Id); - } - - foreach (var component in entry.Components) - { - AddComplexFormComponent(lexEntry, component); - } - - foreach (var complexForm in entry.ComplexForms) + try + { + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create Entry", + "Remove entry", + Cache.ServiceLocator.ActionHandler, + () => { - var complexLexEntry = EntriesRepository.GetObject(complexForm.ComplexFormEntryId); - AddComplexFormComponent(complexLexEntry, complexForm); - } - }); + var lexEntry = LexEntryFactory.Create(entry.Id, + Cache.ServiceLocator.GetInstance().Singleton.LexDbOA); + lexEntry.LexemeFormOA = Cache.ServiceLocator.GetInstance().Create(); + UpdateLcmMultiString(lexEntry.LexemeFormOA.Form, entry.LexemeForm); + UpdateLcmMultiString(lexEntry.CitationForm, entry.CitationForm); + UpdateLcmMultiString(lexEntry.LiteralMeaning, entry.LiteralMeaning); + UpdateLcmMultiString(lexEntry.Comment, entry.Note); + + foreach (var sense in entry.Senses) + { + CreateSense(lexEntry, sense); + } + + //form types should be created before components, otherwise the form type "unspecified" will be added + foreach (var complexFormType in entry.ComplexFormTypes) + { + AddComplexFormType(lexEntry, complexFormType.Id); + } + + foreach (var component in entry.Components) + { + AddComplexFormComponent(lexEntry, component); + } + + foreach (var complexForm in entry.ComplexForms) + { + var complexLexEntry = EntriesRepository.GetObject(complexForm.ComplexFormEntryId); + AddComplexFormComponent(complexLexEntry, complexForm); + } + }); + } + catch (Exception e) + { + throw new CreateObjectException($"Failed to create entry {entry}", e); + } return await GetEntry(entry.Id) ?? throw new InvalidOperationException("Entry was not created"); } @@ -693,7 +710,7 @@ internal void AddComplexFormType(ILexEntry lexEntry, Guid complexFormTypeId) entryRef.HideMinorEntry = 0; } - var lexEntryType = (ILexEntryType)ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormTypeId); + var lexEntryType = ComplexFormTypesFlattened.Single(c => c.Guid == complexFormTypeId); entryRef.ComplexEntryTypesRS.Add(lexEntryType); } @@ -701,7 +718,8 @@ internal void RemoveComplexFormType(ILexEntry lexEntry, Guid complexFormTypeId) { ILexEntryRef? entryRef = lexEntry.ComplexFormEntryRefs.SingleOrDefault(); if (entryRef is null) return; - var lexEntryType = (ILexEntryType)ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormTypeId); + var lexEntryType = entryRef.ComplexEntryTypesRS.SingleOrDefault(c => c.Guid == complexFormTypeId); + if (lexEntryType is null) return; entryRef.ComplexEntryTypesRS.Remove(lexEntryType); } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs new file mode 100644 index 000000000..e5ee6416f --- /dev/null +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs @@ -0,0 +1,18 @@ +using SIL.LCModel; + +namespace FwDataMiniLcmBridge.Api; + +public static class PossibilityExtensions +{ + public static IEnumerable Flatten(this IEnumerable enumerable) where T : ICmPossibility + { + foreach (var cmPossibility in enumerable) + { + yield return cmPossibility; + foreach (var child in Flatten(cmPossibility.SubPossibilitiesOS.Cast())) + { + yield return child; + } + } + } +} diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs index 503222fca..d0defe8ad 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs @@ -4,7 +4,7 @@ namespace FwDataMiniLcmBridge.Api.UpdateProxy; -public class UpdateEntryProxy : Entry +public record UpdateEntryProxy : Entry { private readonly ILexEntry _lcmEntry; private readonly FwDataMiniLcmApi _lexboxLcmApi; diff --git a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs index 021587cfd..b2377ae5c 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs @@ -95,8 +95,8 @@ public async Task CanChangeComplexFormViaSync_ComplexForms() [Fact] public async Task CanChangeComplexFormTypeViaSync() { + var complexFormType = await _fixture.CrdtApi.CreateComplexFormType(new() { Name = new() { { "en", "complexFormType" } } }); var entry = await _fixture.CrdtApi.CreateEntry(new() { LexemeForm = { { "en", "complexForm1" } } }); - var complexFormType = await _fixture.CrdtApi.GetComplexFormTypes().FirstAsync(); var after = (Entry) entry.Copy(); after.ComplexFormTypes = [complexFormType]; await EntrySync.Sync(after, entry, _fixture.CrdtApi); diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs new file mode 100644 index 000000000..e0ec3f816 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs @@ -0,0 +1,31 @@ +using Chorus.VcsDrivers.Mercurial; +using SIL.CommandLineProcessing; +using SIL.PlatformUtilities; +using SIL.Progress; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public static class MercurialTestHelper +{ + private static readonly NullProgress NullProgress = new NullProgress(); + + private static string RunHgCommand(string repoPath, string args) + { + var result = HgRunner.Run(args, repoPath, 120, NullProgress); + if (result.ExitCode == 0) return result.StandardOutput; + throw new Exception( + $"hg {args} failed.\nStdOut: {result.StandardOutput}\nStdErr: {result.StandardError}"); + + } + + public static void HgClean(string repoPath, string exclude) + { + RunHgCommand(repoPath, $"purge --no-confirm --exclude {exclude}"); + } + + public static void HgUpdate(string repoPath, string rev) + { + RunHgCommand(repoPath, $"update \"{rev}\""); + } +} + diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs new file mode 100644 index 000000000..3bb84e1ca --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs @@ -0,0 +1,93 @@ +using System.IO.Compression; +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.Api; +using LcmCrdt; +using LexCore.Utils; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using SIL.IO; +using SIL.Progress; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public class Sena3Fixture : IAsyncLifetime +{ + private static readonly HttpClient http = new HttpClient(); + + public async Task InitializeAsync() + { + var services = new ServiceCollection() + .AddSyncServices(nameof(Sena3Fixture), false); + var rootServiceProvider = services.BuildServiceProvider(); + var fwProjectsFolder = rootServiceProvider.GetRequiredService>() + .Value + .ProjectsFolder; + if (Path.Exists(fwProjectsFolder)) Directory.Delete(fwProjectsFolder, true); + Directory.CreateDirectory(fwProjectsFolder); + + var crdtProjectsFolder = + rootServiceProvider.GetRequiredService>().Value.ProjectPath; + if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); + rootServiceProvider.Dispose(); + + Directory.CreateDirectory(crdtProjectsFolder); + await DownloadSena3(); + } + + public async Task<(CrdtMiniLcmApi CrdtApi, FwDataMiniLcmApi FwDataApi, IServiceProvider services, IDisposable cleanup)> SetupProjects() + { + var sena3MasterCopy = await DownloadSena3(); + + var rootServiceProvider = new ServiceCollection() + .AddSyncServices(nameof(Sena3Fixture), false) + .BuildServiceProvider(); + var cleanup = Defer.Action(() => rootServiceProvider.Dispose()); + var services = rootServiceProvider.CreateAsyncScope().ServiceProvider; + var projectName = "sena-3_" + Guid.NewGuid().ToString("N"); + + var projectsFolder = services.GetRequiredService>() + .Value + .ProjectsFolder; + var fwDataProject = new FwDataProject(projectName, projectsFolder); + var fwDataProjectPath = Path.Combine(fwDataProject.ProjectsPath, fwDataProject.Name); + DirectoryHelper.Copy(sena3MasterCopy, fwDataProjectPath); + File.Move(Path.Combine(fwDataProjectPath, "sena-3.fwdata"), fwDataProject.FilePath); + var fwDataMiniLcmApi = services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); + + var crdtProject = await services.GetRequiredService() + .CreateProject(new(projectName, FwProjectId: fwDataMiniLcmApi.ProjectId, SeedNewProjectData: false)); + var crdtMiniLcmApi = (CrdtMiniLcmApi)await services.OpenCrdtProject(crdtProject); + return (crdtMiniLcmApi, fwDataMiniLcmApi, services, cleanup); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + + private async Task DownloadSena3ProjectBackupStream() + { + var backupUrl = new Uri("https://drive.google.com/uc?export=download&id=1I-hwc0RHoQqW774gbS5qR-GHa1E7BlsS"); + var result = await http.GetAsync(backupUrl, HttpCompletionOption.ResponseHeadersRead); + return await result.Content.ReadAsStreamAsync(); + } + + private async Task DownloadSena3() + { + var tempFolder = Path.Combine(Path.GetTempPath(), nameof(Sena3Fixture)); + var sena3MasterCopy = Path.Combine(tempFolder, "sena-3"); + if (!Directory.Exists(sena3MasterCopy) || !File.Exists(Path.Combine(sena3MasterCopy, "sena-3.fwdata"))) + { + Directory.CreateDirectory(sena3MasterCopy); + await using var zipStream = await DownloadSena3ProjectBackupStream(); + //the zip file is structured like this: /sena-3/.hg + //by extracting it to tempFolder it should merge with sena-3 + ZipFile.ExtractToDirectory(zipStream, tempFolder); + + MercurialTestHelper.HgUpdate(sena3MasterCopy, "tip"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(new NullProgress(), false, Path.Combine(sena3MasterCopy, "sena-3.fwdata")); + MercurialTestHelper.HgClean(sena3MasterCopy, "sena-3.fwdata"); + } + return sena3MasterCopy; + } +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index 0b0443e0d..3cae9d60a 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -4,6 +4,7 @@ using FwDataMiniLcmBridge.LcmUtils; using FwDataMiniLcmBridge.Tests.Fixtures; using LcmCrdt; +using LexCore.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; @@ -20,7 +21,7 @@ public class SyncFixture : IAsyncLifetime _services.ServiceProvider.GetRequiredService(); public IServiceProvider Services => _services.ServiceProvider; private readonly string _projectName; - private readonly MockProjectContext _projectContext = new(null); + private readonly IDisposable _cleanup; public static SyncFixture Create([CallerMemberName] string projectName = "") => new(projectName); @@ -28,15 +29,10 @@ private SyncFixture(string projectName) { _projectName = projectName; var crdtServices = new ServiceCollection() - .AddLcmCrdtClient() - .AddSingleton(_projectContext) - .AddTestFwDataBridge() - .AddFwLiteProjectSync() - .Configure(c => c.ProjectsFolder = Path.Combine(".", _projectName, "FwData")) - .Configure(c => c.ProjectPath = Path.Combine(".", _projectName, "LcmCrdt")) - .AddLogging(builder => builder.AddDebug()) - .BuildServiceProvider(); - _services = crdtServices.CreateAsyncScope(); + .AddSyncServices(_projectName); + var rootServiceProvider = crdtServices.BuildServiceProvider(); + _cleanup = Defer.Action(() => rootServiceProvider.Dispose()); + _services = rootServiceProvider.CreateAsyncScope(); } public SyncFixture(): this("sena-3_" + Guid.NewGuid().ToString("N")) @@ -50,7 +46,7 @@ public async Task InitializeAsync() if (Path.Exists(projectsFolder)) Directory.Delete(projectsFolder, true); Directory.CreateDirectory(projectsFolder); _services.ServiceProvider.GetRequiredService() - .NewProject(new FwDataProject(_projectName, projectsFolder), "en", "fr"); + .NewProject(new FwDataProject(_projectName, projectsFolder), "en", "en"); FwDataApi = _services.ServiceProvider.GetRequiredService().GetFwDataMiniLcmApi(_projectName, false); var crdtProjectsFolder = @@ -58,20 +54,22 @@ public async Task InitializeAsync() if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); Directory.CreateDirectory(crdtProjectsFolder); var crdtProject = await _services.ServiceProvider.GetRequiredService() - .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: true)); + .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: false)); CrdtApi = (CrdtMiniLcmApi) await _services.ServiceProvider.OpenCrdtProject(crdtProject); } public async Task DisposeAsync() { await _services.DisposeAsync(); + _cleanup.Dispose(); } public CrdtMiniLcmApi CrdtApi { get; set; } = null!; public FwDataMiniLcmApi FwDataApi { get; set; } = null!; -} -public class MockProjectContext(CrdtProject? project) : ProjectContext -{ - public override CrdtProject? Project { get; set; } = project; + public void DeleteSyncSnapshot() + { + var snapshotPath = CrdtFwdataProjectSyncService.SnapshotPath(FwDataApi.Project); + if (File.Exists(snapshotPath)) File.Delete(snapshotPath); + } } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs new file mode 100644 index 000000000..6eb9f8b40 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs @@ -0,0 +1,27 @@ +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.Tests.Fixtures; +using LcmCrdt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public static class TestingKernel +{ + public static IServiceCollection AddSyncServices(this IServiceCollection services, string projectName, bool mockFwProjectLoader = true) + { + return services.AddLcmCrdtClient() + .AddTestFwDataBridge(mockFwProjectLoader) + .AddFwLiteProjectSync() + .AddSingleton(new MockProjectContext(null)) + .Configure(c => c.ProjectsFolder = Path.Combine(".", projectName, "FwData")) + .Configure(c => c.ProjectPath = Path.Combine(".", projectName, "LcmCrdt")) + .AddLogging(builder => builder.AddDebug()); + } + + public class MockProjectContext(CrdtProject? project) : ProjectContext + { + public override CrdtProject? Project { get; set; } = project; + } + +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj b/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj index c988caadc..0858009d0 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj +++ b/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj @@ -7,6 +7,7 @@ false true + $(MSBuildProjectDirectory) @@ -28,6 +29,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + @@ -41,4 +44,12 @@ + + + + + + + + diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs new file mode 100644 index 000000000..77add42f5 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -0,0 +1,144 @@ +using FluentAssertions.Equivalency; +using FluentAssertions.Execution; +using FwDataMiniLcmBridge.Api; +using FwLiteProjectSync.Tests.Fixtures; +using LcmCrdt; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using MiniLcm; +using MiniLcm.Models; +using SystemTextJsonPatch; + +namespace FwLiteProjectSync.Tests; + +public class Sena3SyncTests : IClassFixture, IAsyncLifetime +{ + private readonly Sena3Fixture _fixture; + private CrdtFwdataProjectSyncService _syncService = null!; + private CrdtMiniLcmApi _crdtApi = null!; + private FwDataMiniLcmApi _fwDataApi = null!; + private IDisposable? _cleanup; + + + public Sena3SyncTests(Sena3Fixture fixture) + { + _fixture = fixture; + } + + public async Task InitializeAsync() + { + (_crdtApi, _fwDataApi, var services, _cleanup) = await _fixture.SetupProjects(); + _syncService = services.GetRequiredService(); + _fwDataApi.EntryCount.Should().BeGreaterThan(100, "project should be loaded and have entries"); + } + + public Task DisposeAsync() + { + _cleanup?.Dispose(); + return Task.CompletedTask; + } + + private void ShouldAllBeEquivalentTo(Dictionary crdtEntries, Dictionary fwdataEntries) + { + crdtEntries.Keys.Should().BeEquivalentTo(fwdataEntries.Keys); + using (new AssertionScope()) + { + foreach (var crdtEntry in crdtEntries.Values) + { + var fwdataEntry = fwdataEntries[crdtEntry.Id]; + crdtEntry.Should().BeEquivalentTo(fwdataEntry, + options => options + .For(e => e.Components).Exclude(c => c.Id) + .For(e => e.ComplexForms).Exclude(c => c.Id), + $"CRDT entry {crdtEntry.Id} was synced with FwData"); + } + } + } + + //by default the first sync is an import, this will skip that so that the sync will actually sync data + private async Task BypassImport() + { + await _syncService.SaveProjectSnapshot(_fwDataApi.Project, new ([], [], [])); + } + + //this lets us query entries when there is no writing system + private async Task WorkaroundMissingWritingSystems() + { + //must have at least one writing system to query for entries + await _crdtApi.CreateWritingSystem(WritingSystemType.Vernacular, (await _fwDataApi.GetWritingSystems()).Vernacular.First()); + + } + + [Fact] + public async Task DryRunImport_MakesNoChanges() + { + await WorkaroundMissingWritingSystems(); + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + //should still be empty + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + } + + [Fact] + public async Task DryRunImport_MakesTheSameChangesAsImport() + { + var dryRunSyncResult = await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + var syncResult = await _syncService.Sync(_crdtApi, _fwDataApi); + dryRunSyncResult.Should().BeEquivalentTo(syncResult); + } + + [Fact] + public async Task DryRunSync_MakesNoChanges() + { + await BypassImport(); + await WorkaroundMissingWritingSystems(); + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + //should still be empty + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + } + + [Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")] + public async Task DryRunSync_MakesTheSameChangesAsImport() + { + await BypassImport(); + var dryRunSyncResult = await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + var syncResult = await _syncService.Sync(_crdtApi, _fwDataApi); + dryRunSyncResult.Should().BeEquivalentTo(syncResult); + } + + [Fact] + public async Task FirstSena3SyncJustDoesAnSync() + { + var results = await _syncService.Sync(_crdtApi, _fwDataApi); + results.FwdataChanges.Should().Be(0); + results.CrdtChanges.Should().BeGreaterThanOrEqualTo(_fwDataApi.EntryCount); + + var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id); + ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries); + } + + [Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")] + public async Task SyncWithoutImport_CrdtShouldMatchFwdata() + { + await BypassImport(); + + var results = await _syncService.Sync(_crdtApi, _fwDataApi); + results.FwdataChanges.Should().Be(0); + results.CrdtChanges.Should().BeGreaterThan(_fwDataApi.EntryCount); + + var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id); + ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries); + } + + [Fact] + public async Task SecondSena3SyncDoesNothing() + { + await _syncService.Sync(_crdtApi, _fwDataApi); + var secondSync = await _syncService.Sync(_crdtApi, _fwDataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); + } +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index 752d77fa2..36b4084f2 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -65,6 +65,8 @@ public async Task DisposeAsync() { await _fixture.CrdtApi.DeleteEntry(entry.Id); } + + _fixture.DeleteSyncSnapshot(); } public SyncTests(SyncFixture fixture) @@ -84,7 +86,18 @@ public async Task FirstSyncJustDoesAnImport() var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); crdtEntries.Should().BeEquivalentTo(fwdataEntries, options => options.For(e => e.Components).Exclude(c => c.Id) - .For(e => e.ComplexForms).Exclude(c => c.Id)); + .For(e => e.ComplexForms).Exclude(c => c.Id)); + } + + [Fact] + public async Task SecondSyncDoesNothing() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); } [Fact] @@ -138,6 +151,90 @@ await crdtApi.CreateEntry(new Entry() .For(e => e.ComplexForms).Exclude(c => c.Id)); } + [Fact] + public async Task SyncDryRun_NoChangesAreSynced() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + var fwDataEntryId = Guid.NewGuid(); + var crdtEntryId = Guid.NewGuid(); + + await fwdataApi.CreateEntry(new Entry() + { + Id = fwDataEntryId, + LexemeForm = { { "en", "Pear" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Pear" } }, } + ] + }); + await crdtApi.CreateEntry(new Entry() + { + Id = crdtEntryId, + LexemeForm = { { "en", "Banana" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Banana" } }, } + ] + }); + await _syncService.SyncDryRun(crdtApi, fwdataApi); + + var crdtEntries = await crdtApi.GetEntries().ToArrayAsync(); + var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); + crdtEntries.Select(e => e.Id).Should().NotContain(fwDataEntryId); + fwdataEntries.Select(e => e.Id).Should().NotContain(crdtEntryId); + } + + [Fact] + public async Task CreatingAComplexEntryInFwDataSyncsWithoutIssue() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + + var hat = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Hat" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Hat" } }, } + ] + }); + var stand = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Stand" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Stand" } }, } + ] + }); + var hatstand = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Hatstand" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Hatstand" } }, } + ], + }); + var component1 = ComplexFormComponent.FromEntries(hatstand, hat); + var component2 = ComplexFormComponent.FromEntries(hatstand, stand); + hatstand.Components = [component1, component2]; + await _syncService.Sync(crdtApi, fwdataApi); + + var crdtEntries = await crdtApi.GetEntries().ToArrayAsync(); + var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); + crdtEntries.Should().BeEquivalentTo(fwdataEntries, + options => options.For(e => e.Components).Exclude(c => c.Id) + .For(e => e.ComplexForms).Exclude(c => c.Id)); + + // Sync again, ensure no problems or changes + var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); + } + + [Fact] public async Task PartsOfSpeechSyncBothWays() { @@ -273,7 +370,7 @@ await fwdataApi.CreateEntry(new Entry() LexemeForm = { { "en", "Pear" } }, Senses = [ - new Sense() { Gloss = { { "en", "Pear" } }, SemanticDomains = [ semdom3 ] } + new Sense() { Gloss = { { "en", "Pear" } }, SemanticDomains = [semdom3] } ] }); await crdtApi.CreateEntry(new Entry() @@ -281,7 +378,7 @@ await crdtApi.CreateEntry(new Entry() LexemeForm = { { "en", "Banana" } }, Senses = [ - new Sense() { Gloss = { { "en", "Banana" } }, SemanticDomains = [ semdom3 ] } + new Sense() { Gloss = { { "en", "Banana" } }, SemanticDomains = [semdom3] } ] }); await _syncService.Sync(crdtApi, fwdataApi); @@ -365,7 +462,7 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() await _syncService.Sync(crdtApi, fwdataApi); await fwdataApi.CreateSense(_testEntry.Id, new Sense() - { + { Gloss = { { "en", "Fruit" } }, Definition = { { "en", "a round fruit, red or yellow" } }, }); @@ -373,7 +470,7 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() { Gloss = { { "en", "Tree" } }, Definition = { { "en", "a tall, woody plant, which grows fruit" } }, - }); + }); await _syncService.Sync(crdtApi, fwdataApi); @@ -383,4 +480,24 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() options => options.For(e => e.Components).Exclude(c => c.Id) .For(e => e.ComplexForms).Exclude(c => c.Id)); } + + [Fact] + public async Task CanCreateAComplexFormAndItsComponentInOneSync() + { + //ensure they are synced so a real sync will happen when we want it to + await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + + var complexFormEntry = await _fixture.CrdtApi.CreateEntry(new() { LexemeForm = { { "en", "complexForm" } } }); + var componentEntry = await _fixture.CrdtApi.CreateEntry(new() + { + LexemeForm = { { "en", "component" } }, + ComplexForms = + [ + new ComplexFormComponent() { ComplexFormEntryId = complexFormEntry.Id, ComponentEntryId = Guid.Empty } + ] + }); + + //one of the entries will be created first, it will try to create the reference to the other but it won't exist yet + await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + } } diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index ee42e1026..58685043b 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -1,9 +1,9 @@ using System.Text.Json; +using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.Api; using LcmCrdt; using LexCore.Sync; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using MiniLcm; using MiniLcm.Models; using MiniLcm.SyncHelpers; @@ -12,21 +12,32 @@ namespace FwLiteProjectSync; -public class CrdtFwdataProjectSyncService(IOptions lcmCrdtConfig, MiniLcmImport miniLcmImport, ILogger logger) +public class CrdtFwdataProjectSyncService(MiniLcmImport miniLcmImport, ILogger logger) { + public record DryRunSyncResult( + int CrdtChanges, + int FwdataChanges, + List CrdtDryRunRecords, + List FwDataDryRunRecords) : SyncResult(CrdtChanges, FwdataChanges); + + public async Task SyncDryRun(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi) + { + return (DryRunSyncResult) await Sync(crdtApi, fwdataApi, true); + } + public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi, bool dryRun = false) { if (crdtApi is CrdtMiniLcmApi crdt && crdt.ProjectData.FwProjectId != fwdataApi.ProjectId) { throw new InvalidOperationException($"Project id mismatch, CRDT Id: {crdt.ProjectData.FwProjectId}, FWData Id: {fwdataApi.ProjectId}"); } - var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath); + var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project); SyncResult result = await Sync(crdtApi, fwdataApi, dryRun, fwdataApi.EntryCount, projectSnapshot); fwdataApi.Save(); if (!dryRun) { - await SaveProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath, + await SaveProjectSnapshot(fwdataApi.Project, new ProjectSnapshot( await fwdataApi.GetEntries().ToArrayAsync(), await fwdataApi.GetPartsOfSpeech().ToArrayAsync(), @@ -47,6 +58,7 @@ private async Task Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi, { await miniLcmImport.ImportProject(crdtApi, fwdataApi, entryCount); LogDryRun(crdtApi, "crdt"); + if (dryRun) return new DryRunSyncResult(entryCount, 0, GetDryRunRecords(crdtApi), []); return new SyncResult(entryCount, 0); } @@ -68,7 +80,7 @@ private async Task Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi, LogDryRun(fwdataApi, "fwdata"); //todo push crdt changes to lexbox - + if (dryRun) return new DryRunSyncResult(crdtChanges, fwdataChanges, GetDryRunRecords(crdtApi), GetDryRunRecords(fwdataApi)); return new SyncResult(crdtChanges, fwdataChanges); } @@ -83,23 +95,32 @@ private void LogDryRun(IMiniLcmApi api, string type) logger.LogInformation($"Dry run {type} changes: {dryRunApi.DryRunRecords.Count}"); } + private List GetDryRunRecords(IMiniLcmApi api) + { + return ((DryRunMiniLcmApi)api).DryRunRecords; + } + public record ProjectSnapshot(Entry[] Entries, PartOfSpeech[] PartsOfSpeech, SemanticDomain[] SemanticDomains); - private async Task GetProjectSnapshot(string projectName, string? projectPath) + private async Task GetProjectSnapshot(FwDataProject project) { - projectPath ??= lcmCrdtConfig.Value.ProjectPath; - var snapshotPath = Path.Combine(projectPath, $"{projectName}_snapshot.json"); + var snapshotPath = SnapshotPath(project); if (!File.Exists(snapshotPath)) return null; await using var file = File.OpenRead(snapshotPath); return await JsonSerializer.DeserializeAsync(file); } - private async Task SaveProjectSnapshot(string projectName, string? projectPath, ProjectSnapshot projectSnapshot) + internal async Task SaveProjectSnapshot(FwDataProject project, ProjectSnapshot projectSnapshot) { - projectPath ??= lcmCrdtConfig.Value.ProjectPath; - var snapshotPath = Path.Combine(projectPath, $"{projectName}_snapshot.json"); + var snapshotPath = SnapshotPath(project); await using var file = File.Create(snapshotPath); await JsonSerializer.SerializeAsync(file, projectSnapshot); } + internal static string SnapshotPath(FwDataProject project) + { + var projectPath = project.ProjectsPath; + var snapshotPath = Path.Combine(projectPath, $"{project.Name}_snapshot.json"); + return snapshotPath; + } } diff --git a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs index 25c317670..15c17a387 100644 --- a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs +++ b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs @@ -216,7 +216,7 @@ public Task CreateComplexFormComponent(ComplexFormComponen public Task DeleteComplexFormComponent(ComplexFormComponent complexFormComponent) { - DryRunRecords.Add(new DryRunRecord(nameof(DeleteComplexFormComponent), $"Delete complex form component complex entry: {complexFormComponent.ComplexFormHeadword}, component entry: {complexFormComponent.ComponentHeadword}")); + DryRunRecords.Add(new DryRunRecord(nameof(DeleteComplexFormComponent), $"Delete complex form component: {complexFormComponent}")); return Task.CompletedTask; } diff --git a/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj b/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj index 7e4c26e04..2d0c452f0 100644 --- a/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj +++ b/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj @@ -19,4 +19,8 @@ + + + + diff --git a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt index a7b2cc8c5..129ee865c 100644 --- a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt @@ -24,6 +24,8 @@ ComponentEntryId (Guid) Required FK Index ComponentHeadword (string) ComponentSenseId (Guid?) FK Index + Annotations: + Relational:ColumnName: ComponentSenseId DeletedAt (DateTimeOffset?) SnapshotId (no field, Guid?) Shadow FK Index Keys: @@ -34,10 +36,15 @@ ComplexFormComponent {'ComponentSenseId'} -> Sense {'Id'} Cascade ComplexFormComponent {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull Indexes: - ComplexFormEntryId ComponentEntryId ComponentSenseId SnapshotId Unique + ComplexFormEntryId, ComponentEntryId Unique + Annotations: + Relational:Filter: ComponentSenseId IS NULL + ComplexFormEntryId, ComponentEntryId, ComponentSenseId Unique + Annotations: + Relational:Filter: ComponentSenseId IS NOT NULL Annotations: DiscriminatorProperty: Relational:FunctionName: diff --git a/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs b/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs index b2370c85e..13eb26933 100644 --- a/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs @@ -20,7 +20,6 @@ public void ChangesFromJsonPatch_AddComponentMakesAddEntryComponentChange() changes.Should().ContainSingle().Which.Should().BeOfType().Subject; addEntryComponentChange.ComplexFormEntryId.Should().Be(_entry.Id); addEntryComponentChange.ComponentEntryId.Should().Be(componentEntry.Id); - addEntryComponentChange.ComponentHeadword.Should().Be(componentEntry.Headword()); } [Fact] @@ -84,7 +83,6 @@ public void ChangesFromJsonPatch_AddComplexFormMakesAddEntryComponentChange() changes.Should().ContainSingle().Which.Should().BeOfType().Subject; addEntryComponentChange.ComplexFormEntryId.Should().Be(_entry.Id); addEntryComponentChange.ComponentEntryId.Should().Be(componentEntry.Id); - addEntryComponentChange.ComponentHeadword.Should().Be(componentEntry.Headword()); } [Fact] diff --git a/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs b/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs index a51313198..f93630058 100644 --- a/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs +++ b/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs @@ -10,10 +10,8 @@ namespace LcmCrdt.Changes.Entries; public class AddEntryComponentChange : CreateChange, ISelfNamedType { public Guid ComplexFormEntryId { get; } - public string? ComplexFormHeadword { get; } public Guid ComponentEntryId { get; } public Guid? ComponentSenseId { get; } - public string? ComponentHeadword { get; } [JsonConstructor] public AddEntryComponentChange(Guid entityId, @@ -24,9 +22,7 @@ public AddEntryComponentChange(Guid entityId, Guid? componentSenseId = null) : base(entityId) { ComplexFormEntryId = complexFormEntryId; - ComplexFormHeadword = complexFormHeadword; ComponentEntryId = componentEntryId; - ComponentHeadword = componentHeadword; ComponentSenseId = componentSenseId; } @@ -41,17 +37,22 @@ public AddEntryComponentChange(Guid entityId, public override async ValueTask NewEntity(Commit commit, ChangeContext context) { + var complexFormEntry = await context.GetCurrent(ComplexFormEntryId); + var componentEntry = await context.GetCurrent(ComponentEntryId); + Sense? componentSense = null; + if (ComponentSenseId is not null) + componentSense = await context.GetCurrent(ComponentSenseId.Value); return new ComplexFormComponent { Id = EntityId, ComplexFormEntryId = ComplexFormEntryId, - ComplexFormHeadword = ComplexFormHeadword, + ComplexFormHeadword = complexFormEntry?.Headword(), ComponentEntryId = ComponentEntryId, - ComponentHeadword = ComponentHeadword, + ComponentHeadword = componentEntry?.Headword(), ComponentSenseId = ComponentSenseId, - DeletedAt = (await context.IsObjectDeleted(ComponentEntryId) || - await context.IsObjectDeleted(ComplexFormEntryId) || - ComponentSenseId.HasValue && await context.IsObjectDeleted(ComponentSenseId.Value)) + DeletedAt = (complexFormEntry?.DeletedAt is not null || + componentEntry?.DeletedAt is not null || + (ComponentSenseId.HasValue && componentSense?.DeletedAt is not null)) ? commit.DateTime : (DateTime?)null, }; diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index 241c8daad..28cc4048e 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -18,14 +18,15 @@ public class CrdtMiniLcmApi(DataModel dataModel, CurrentProjectService projectSe private Guid ClientId { get; } = projectService.ProjectData.ClientId; public ProjectData ProjectData => projectService.ProjectData; - private IQueryable Entries => dataModel.QueryLatest(); - private IQueryable ComplexFormComponents => dataModel.QueryLatest(); - private IQueryable ComplexFormTypes => dataModel.QueryLatest(); - private IQueryable Senses => dataModel.QueryLatest(); - private IQueryable ExampleSentences => dataModel.QueryLatest(); - private IQueryable WritingSystems => dataModel.QueryLatest(); - private IQueryable SemanticDomains => dataModel.QueryLatest(); - private IQueryable PartsOfSpeech => dataModel.QueryLatest(); + private IQueryable Entries => dataModel.QueryLatest().AsTracking(false); + private IQueryable ComplexFormComponents => dataModel.QueryLatest() + .AsTracking(false); + private IQueryable ComplexFormTypes => dataModel.QueryLatest().AsTracking(false); + private IQueryable Senses => dataModel.QueryLatest().AsTracking(false); + private IQueryable ExampleSentences => dataModel.QueryLatest().AsTracking(false); + private IQueryable WritingSystems => dataModel.QueryLatest().AsTracking(false); + private IQueryable SemanticDomains => dataModel.QueryLatest().AsTracking(false); + private IQueryable PartsOfSpeech => dataModel.QueryLatest().AsTracking(false); public async Task GetWritingSystems() { @@ -134,7 +135,7 @@ public async Task DeleteSemanticDomain(Guid id) public async Task BulkImportSemanticDomains(IEnumerable semanticDomains) { - await dataModel.AddChanges(ClientId, semanticDomains.Select(sd => new CreateSemanticDomainChange(sd.Id, sd.Name, sd.Code))); + await dataModel.AddChanges(ClientId, semanticDomains.Select(sd => new CreateSemanticDomainChange(sd.Id, sd.Name, sd.Code, sd.Predefined))); } public IAsyncEnumerable GetComplexFormTypes() @@ -151,6 +152,11 @@ public async Task CreateComplexFormType(ComplexFormType complex public async Task CreateComplexFormComponent(ComplexFormComponent complexFormComponent) { + var existing = await ComplexFormComponents.SingleOrDefaultAsync(c => + c.ComplexFormEntryId == complexFormComponent.ComplexFormEntryId + && c.ComponentEntryId == complexFormComponent.ComponentEntryId + && c.ComponentSenseId == complexFormComponent.ComponentSenseId); + if (existing is not null) return existing; var addEntryComponentChange = new AddEntryComponentChange(complexFormComponent); await dataModel.AddChange(ClientId, addEntryComponentChange); return (await ComplexFormComponents.SingleOrDefaultAsync(c => c.Id == addEntryComponentChange.EntityId)) ?? throw NotFoundException.ForType(); @@ -256,7 +262,12 @@ public async Task BulkCreateEntries(IAsyncEnumerable entries) { var semanticDomains = await SemanticDomains.ToDictionaryAsync(sd => sd.Id, sd => sd); var partsOfSpeech = await PartsOfSpeech.ToDictionaryAsync(p => p.Id, p => p); - await dataModel.AddChanges(ClientId, entries.ToBlockingEnumerable().SelectMany(entry => CreateEntryChanges(entry, semanticDomains, partsOfSpeech))); + await dataModel.AddChanges(ClientId, + entries.ToBlockingEnumerable() + .SelectMany(entry => CreateEntryChanges(entry, semanticDomains, partsOfSpeech)) + //force entries to be created first, this avoids issues where references are created before the entry is created + .OrderBy(c => c is CreateEntryChange ? 0 : 1) + ); } private IEnumerable CreateEntryChanges(Entry entry, Dictionary semanticDomains, Dictionary partsOfSpeech) diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index 9fd13ce3e..74562c440 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -134,7 +134,22 @@ public static void ConfigureCrdt(CrdtConfig config) .Add() .Add(builder => { + const string componentSenseId = "ComponentSenseId"; builder.ToTable("ComplexFormComponents"); + builder.Property(c => c.ComponentSenseId).HasColumnName(componentSenseId); + //these indexes are used to ensure that we don't create duplicate complex form components + //we need the filter otherwise 2 components which are the same and have a null sense id can be created because 2 rows with the same null are not considered duplicates + builder.HasIndex(component => new + { + component.ComplexFormEntryId, + component.ComponentEntryId, + component.ComponentSenseId + }).IsUnique().HasFilter($"{componentSenseId} IS NOT NULL"); + builder.HasIndex(component => new + { + component.ComplexFormEntryId, + component.ComponentEntryId + }).IsUnique().HasFilter($"{componentSenseId} IS NULL"); }); config.ChangeTypeListBuilder.Add>() diff --git a/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs b/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs index 4f51ca275..d6f646435 100644 --- a/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs @@ -50,6 +50,22 @@ public async Task CreateComplexFormComponent_Works() component.ComponentHeadword.Should().Be("component"); } + [Fact] + public async Task CreateComplexFormComponent_UsingTheSameComponentWithNullSenseDoesNothing() + { + var component1 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + var component2 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + component2.Should().BeEquivalentTo(component1); + } + + [Fact] + public async Task CreateComplexFormComponent_UsingTheSameComponentWithSenseDoesNothing() + { + var component1 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry, _componentSenseId1)); + var component2 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry, _componentSenseId1)); + component2.Should().BeEquivalentTo(component1); + } + [Fact] public async Task CreateComplexFormComponent_WorksWithSense() { @@ -83,4 +99,43 @@ public async Task CreateComplexFormType_Works() var types = await Api.GetComplexFormTypes().ToArrayAsync(); types.Should().ContainSingle(t => t.Id == complexFormType.Id); } + + [Fact] + public async Task AddComplexFormType_Works() + { + var complexFormType = new ComplexFormType() { Id = Guid.NewGuid(), Name = new() { { "en", "test" } } }; + await Api.CreateComplexFormType(complexFormType); + await Api.AddComplexFormType(_complexFormEntryId, complexFormType.Id); + var entry = await Api.GetEntry(_complexFormEntryId); + entry.Should().NotBeNull(); + entry!.ComplexFormTypes.Should().ContainSingle(c => c.Id == complexFormType.Id); + } + + [Fact] + public async Task RemoveComplexFormType_Works() + { + await AddComplexFormType_Works(); + var entry = await Api.GetEntry(_complexFormEntryId); + await Api.RemoveComplexFormType(_complexFormEntryId, entry!.ComplexFormTypes[0].Id); + entry = await Api.GetEntry(_complexFormEntryId); + entry.Should().NotBeNull(); + entry!.ComplexFormTypes.Should().BeEmpty(); + } + + [Fact] + public async Task RemoveComplexFormType_WorksWhenTypeDoesNotExist() + { + await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + await Api.RemoveComplexFormType(_complexFormEntryId, Guid.NewGuid()); + } + + [Fact] + public async Task RemoveComplexFormType_WorksWhenTypeIsNotOnEntry() + { + //FW projects react differently if an entry has complex forms or not + await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + var complexFormType = new ComplexFormType() { Id = Guid.NewGuid(), Name = new() { { "en", "test" } } }; + await Api.CreateComplexFormType(complexFormType); + await Api.RemoveComplexFormType(_complexFormEntryId, Guid.NewGuid()); + } } diff --git a/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs b/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs new file mode 100644 index 000000000..8b680e716 --- /dev/null +++ b/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs @@ -0,0 +1,18 @@ +namespace MiniLcm.Tests.Models; + +public class EntryTests +{ + [Fact] + public void Headword_SameResultForDifferentOrderedMultiStrings() + { + var entry = new Entry() + { + LexemeForm = new MultiString() { Values = { { "en", "test" }, { "fr", "test2" } } } + }; + var entry2 = new Entry() + { + LexemeForm = new MultiString() { Values = { { "fr", "test2" }, { "en", "test" } } } + }; + entry.Headword().Should().Be(entry2.Headword()); + } +} diff --git a/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs b/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs new file mode 100644 index 000000000..128cb0515 --- /dev/null +++ b/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs @@ -0,0 +1,12 @@ +namespace MiniLcm.Exceptions; + +public class CreateObjectException: Exception +{ + public CreateObjectException(string? message) : base(message) + { + } + + public CreateObjectException(string? message, Exception? innerException) : base(message, innerException) + { + } +} diff --git a/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs b/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs new file mode 100644 index 000000000..e1f284cde --- /dev/null +++ b/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs @@ -0,0 +1,12 @@ +namespace MiniLcm.Exceptions; + +public class SyncObjectException: Exception +{ + public SyncObjectException(string? message) : base(message) + { + } + + public SyncObjectException(string? message, Exception? innerException) : base(message, innerException) + { + } +} diff --git a/backend/FwLite/MiniLcm/Models/Entry.cs b/backend/FwLite/MiniLcm/Models/Entry.cs index 191db2753..cabb8c9bb 100644 --- a/backend/FwLite/MiniLcm/Models/Entry.cs +++ b/backend/FwLite/MiniLcm/Models/Entry.cs @@ -1,6 +1,6 @@ namespace MiniLcm.Models; -public class Entry : IObjectWithId +public record Entry : IObjectWithId { public Guid Id { get; set; } public DateTimeOffset? DeletedAt { get; set; } @@ -28,8 +28,10 @@ public class Entry : IObjectWithId public string Headword() { - var word = CitationForm.Values.Values.FirstOrDefault(); - if (string.IsNullOrEmpty(word)) word = LexemeForm.Values.Values.FirstOrDefault(); + //order by code to ensure the headword is stable + //todo choose ws by preference based on ws order/default + var word = CitationForm.Values.OrderBy(kvp => kvp.Key.Code).FirstOrDefault().Value; + if (string.IsNullOrEmpty(word)) word = LexemeForm.Values.OrderBy(kvp => kvp.Key.Code).FirstOrDefault().Value; return word?.Trim() ?? "(Unknown)"; } @@ -68,6 +70,11 @@ public Guid[] GetReferences() public void RemoveReference(Guid id, DateTimeOffset time) { } + + public Entry WithoutEntryRefs() + { + return this with { Components = [], ComplexForms = [] }; + } } public class Variants diff --git a/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs b/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs index 2cc251f2b..33bda1e46 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs @@ -4,6 +4,73 @@ namespace MiniLcm.SyncHelpers; public static class DiffCollection { + /// + /// Diffs a list, for new items calls add, it will then call update for the item returned from the add, using that as the before item for the replace call + /// + /// + /// + /// + /// + /// api, value, return value to be used as the before item for the replace call + /// + /// api, before, after is the parameter order + /// + /// + /// + public static async Task DiffAddThenUpdate( + IMiniLcmApi api, + IList before, + IList after, + Func identity, + Func> add, + Func> remove, + Func> replace) where TId : notnull + { + var changes = 0; + var afterEntriesDict = after.ToDictionary(identity); + + foreach (var beforeEntry in before) + { + if (afterEntriesDict.TryGetValue(identity(beforeEntry), out var afterEntry)) + { + changes += await replace(api, beforeEntry, afterEntry); + } + else + { + changes += await remove(api, beforeEntry); + } + + afterEntriesDict.Remove(identity(beforeEntry)); + } + + var postAddUpdates = new List<(T created, T after)>(afterEntriesDict.Values.Count); + foreach (var value in afterEntriesDict.Values) + { + changes++; + postAddUpdates.Add((await add(api, value), value)); + } + foreach ((T createdItem, T afterItem) in postAddUpdates) + { + //todo this may do a lot more work than it needs to, eg sense will be created during add, but they will be checked again here when we know they didn't change + await replace(api, createdItem, afterItem); + } + + return changes; + } + + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// api, before, after is the parameter order + /// + /// + /// public static async Task Diff( IMiniLcmApi api, IList before, @@ -36,6 +103,7 @@ public static async Task Diff( return changes; } + public static async Task Diff( IMiniLcmApi api, IList before, diff --git a/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs b/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs index 50fe78892..c8f2db5a3 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs @@ -10,10 +10,13 @@ public static async Task Sync(Entry[] afterEntries, Entry[] beforeEntries, IMiniLcmApi api) { - Func> add = static async (api, afterEntry) => + Func> add = static async (api, afterEntry) => { - await api.CreateEntry(afterEntry); - return 1; + //create each entry without components. + //After each entry is created, then replace will be called to create those components + var entryWithoutEntryRefs = afterEntry.WithoutEntryRefs(); + await api.CreateEntry(entryWithoutEntryRefs); + return entryWithoutEntryRefs; }; Func> remove = static async (api, beforeEntry) => { @@ -21,19 +24,26 @@ public static async Task Sync(Entry[] afterEntries, return 1; }; Func> replace = static async (api, beforeEntry, afterEntry) => await Sync(afterEntry, beforeEntry, api); - return await DiffCollection.Diff(api, beforeEntries, afterEntries, add, remove, replace); + return await DiffCollection.DiffAddThenUpdate(api, beforeEntries, afterEntries, entry => entry.Id, add, remove, replace); } public static async Task Sync(Entry afterEntry, Entry beforeEntry, IMiniLcmApi api) { - var updateObjectInput = EntryDiffToUpdate(beforeEntry, afterEntry); - if (updateObjectInput is not null) await api.UpdateEntry(afterEntry.Id, updateObjectInput); - var changes = await SensesSync(afterEntry.Id, afterEntry.Senses, beforeEntry.Senses, api); + try + { + var updateObjectInput = EntryDiffToUpdate(beforeEntry, afterEntry); + if (updateObjectInput is not null) await api.UpdateEntry(afterEntry.Id, updateObjectInput); + var changes = await SensesSync(afterEntry.Id, afterEntry.Senses, beforeEntry.Senses, api); - changes += await Sync(afterEntry.Components, beforeEntry.Components, api); - changes += await Sync(afterEntry.ComplexForms, beforeEntry.ComplexForms, api); - changes += await Sync(afterEntry.Id, afterEntry.ComplexFormTypes, beforeEntry.ComplexFormTypes, api); - return changes + (updateObjectInput is null ? 0 : 1); + changes += await Sync(afterEntry.Components, beforeEntry.Components, api); + changes += await Sync(afterEntry.ComplexForms, beforeEntry.ComplexForms, api); + changes += await Sync(afterEntry.Id, afterEntry.ComplexFormTypes, beforeEntry.ComplexFormTypes, api); + return changes + (updateObjectInput is null ? 0 : 1); + } + catch (Exception e) + { + throw new SyncObjectException($"Failed to sync entry {afterEntry}", e); + } } private static async Task Sync(Guid entryId,