diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs index cc951c27d..84a55beee 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs @@ -6,13 +6,16 @@ namespace FwDataMiniLcmBridge.Tests.Fixtures; public static class FwDataTestsKernel { - public static IServiceCollection AddTestFwDataBridge(this IServiceCollection services) + public static IServiceCollection AddTestFwDataBridge(this IServiceCollection services, bool mockProjectLoader = true) { services.AddFwDataBridge(); services.AddSingleton(_ => new ConfigurationRoot([])); - services.AddSingleton(); - services.AddSingleton(sp => sp.GetRequiredService()); - services.AddSingleton(); + if (mockProjectLoader) + { + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + } return services; } } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index 8e132b2e6..599e4a900 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -6,6 +6,7 @@ using FwDataMiniLcmBridge.LcmUtils; using Microsoft.Extensions.Logging; using MiniLcm; +using MiniLcm.Exceptions; using MiniLcm.Models; using MiniLcm.SyncHelpers; using SIL.LCModel; @@ -37,6 +38,8 @@ public class FwDataMiniLcmApi(Lazy cacheLazy, bool onCloseSave, ILogge private ICmTranslationFactory CmTranslationFactory => Cache.ServiceLocator.GetInstance(); private ICmPossibilityRepository CmPossibilityRepository => Cache.ServiceLocator.GetInstance(); private ICmPossibilityList ComplexFormTypes => Cache.LangProject.LexDbOA.ComplexEntryTypesOA; + private IEnumerable ComplexFormTypesFlattened => ComplexFormTypes.PossibilitiesOS.Cast().Flatten(); + private ICmPossibilityList VariantTypes => Cache.LangProject.LexDbOA.VariantEntryTypesOA; public void Dispose() @@ -326,12 +329,9 @@ public Task DeleteSemanticDomain(Guid id) public IAsyncEnumerable GetComplexFormTypes() { - return ComplexFormTypes.PossibilitiesOS - .Select(ToComplexFormType) - .ToAsyncEnumerable(); + return ComplexFormTypesFlattened.Select(ToComplexFormType).ToAsyncEnumerable(); } - - private ComplexFormType ToComplexFormType(ICmPossibility t) + private ComplexFormType ToComplexFormType(ILexEntryType t) { return new ComplexFormType() { Id = t.Guid, Name = FromLcmMultiString(t.Name) }; } @@ -350,7 +350,7 @@ public Task CreateComplexFormType(ComplexFormType complexFormTy ComplexFormTypes.PossibilitiesOS.Add(lexComplexFormType); UpdateLcmMultiString(lexComplexFormType.Name, complexFormType.Name); }); - return Task.FromResult(ToComplexFormType(ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormType.Id))); + return Task.FromResult(ToComplexFormType(ComplexFormTypesFlattened.Single(c => c.Guid == complexFormType.Id))); } public IAsyncEnumerable GetVariantTypes() @@ -390,6 +390,15 @@ private Entry FromLexEntry(ILexEntry entry) }; } + private string LexEntryHeadword(ILexEntry entry) + { + return new Entry() + { + LexemeForm = FromLcmMultiString(entry.LexemeFormOA.Form), + CitationForm = FromLcmMultiString(entry.CitationForm), + }.Headword(); + } + private IList ToComplexFormTypes(ILexEntry entry) { return entry.ComplexFormEntryRefs.SingleOrDefault() @@ -438,9 +447,9 @@ private ComplexFormComponent ToEntryReference(ILexEntry component, ILexEntry com return new ComplexFormComponent { ComponentEntryId = component.Guid, - ComponentHeadword = component.HeadWord.Text, + ComponentHeadword = LexEntryHeadword(component), ComplexFormEntryId = complexEntry.Guid, - ComplexFormHeadword = complexEntry.HeadWord.Text + ComplexFormHeadword = LexEntryHeadword(complexEntry) }; } @@ -452,7 +461,7 @@ private ComplexFormComponent ToSenseReference(ILexSense componentSense, ILexEntr ComponentSenseId = componentSense.Guid, ComponentHeadword = componentSense.Entry.HeadWord.Text, ComplexFormEntryId = complexEntry.Guid, - ComplexFormHeadword = complexEntry.HeadWord.Text + ComplexFormHeadword = LexEntryHeadword(complexEntry) }; } @@ -559,40 +568,48 @@ public IAsyncEnumerable SearchEntries(string query, QueryOptions? options public async Task CreateEntry(Entry entry) { entry.Id = entry.Id == default ? Guid.NewGuid() : entry.Id; - UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create Entry", - "Remove entry", - Cache.ServiceLocator.ActionHandler, - () => - { - var lexEntry = LexEntryFactory.Create(entry.Id, Cache.ServiceLocator.GetInstance().Singleton.LexDbOA); - lexEntry.LexemeFormOA = Cache.ServiceLocator.GetInstance().Create(); - UpdateLcmMultiString(lexEntry.LexemeFormOA.Form, entry.LexemeForm); - UpdateLcmMultiString(lexEntry.CitationForm, entry.CitationForm); - UpdateLcmMultiString(lexEntry.LiteralMeaning, entry.LiteralMeaning); - UpdateLcmMultiString(lexEntry.Comment, entry.Note); - - foreach (var sense in entry.Senses) - { - CreateSense(lexEntry, sense); - } - - //form types should be created before components, otherwise the form type "unspecified" will be added - foreach (var complexFormType in entry.ComplexFormTypes) - { - AddComplexFormType(lexEntry, complexFormType.Id); - } - - foreach (var component in entry.Components) - { - AddComplexFormComponent(lexEntry, component); - } - - foreach (var complexForm in entry.ComplexForms) + try + { + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create Entry", + "Remove entry", + Cache.ServiceLocator.ActionHandler, + () => { - var complexLexEntry = EntriesRepository.GetObject(complexForm.ComplexFormEntryId); - AddComplexFormComponent(complexLexEntry, complexForm); - } - }); + var lexEntry = LexEntryFactory.Create(entry.Id, + Cache.ServiceLocator.GetInstance().Singleton.LexDbOA); + lexEntry.LexemeFormOA = Cache.ServiceLocator.GetInstance().Create(); + UpdateLcmMultiString(lexEntry.LexemeFormOA.Form, entry.LexemeForm); + UpdateLcmMultiString(lexEntry.CitationForm, entry.CitationForm); + UpdateLcmMultiString(lexEntry.LiteralMeaning, entry.LiteralMeaning); + UpdateLcmMultiString(lexEntry.Comment, entry.Note); + + foreach (var sense in entry.Senses) + { + CreateSense(lexEntry, sense); + } + + //form types should be created before components, otherwise the form type "unspecified" will be added + foreach (var complexFormType in entry.ComplexFormTypes) + { + AddComplexFormType(lexEntry, complexFormType.Id); + } + + foreach (var component in entry.Components) + { + AddComplexFormComponent(lexEntry, component); + } + + foreach (var complexForm in entry.ComplexForms) + { + var complexLexEntry = EntriesRepository.GetObject(complexForm.ComplexFormEntryId); + AddComplexFormComponent(complexLexEntry, complexForm); + } + }); + } + catch (Exception e) + { + throw new CreateObjectException($"Failed to create entry {entry}", e); + } return await GetEntry(entry.Id) ?? throw new InvalidOperationException("Entry was not created"); } @@ -693,7 +710,7 @@ internal void AddComplexFormType(ILexEntry lexEntry, Guid complexFormTypeId) entryRef.HideMinorEntry = 0; } - var lexEntryType = (ILexEntryType)ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormTypeId); + var lexEntryType = ComplexFormTypesFlattened.Single(c => c.Guid == complexFormTypeId); entryRef.ComplexEntryTypesRS.Add(lexEntryType); } @@ -701,7 +718,8 @@ internal void RemoveComplexFormType(ILexEntry lexEntry, Guid complexFormTypeId) { ILexEntryRef? entryRef = lexEntry.ComplexFormEntryRefs.SingleOrDefault(); if (entryRef is null) return; - var lexEntryType = (ILexEntryType)ComplexFormTypes.PossibilitiesOS.Single(c => c.Guid == complexFormTypeId); + var lexEntryType = entryRef.ComplexEntryTypesRS.SingleOrDefault(c => c.Guid == complexFormTypeId); + if (lexEntryType is null) return; entryRef.ComplexEntryTypesRS.Remove(lexEntryType); } diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs new file mode 100644 index 000000000..e5ee6416f --- /dev/null +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/PossibilityExtensions.cs @@ -0,0 +1,18 @@ +using SIL.LCModel; + +namespace FwDataMiniLcmBridge.Api; + +public static class PossibilityExtensions +{ + public static IEnumerable Flatten(this IEnumerable enumerable) where T : ICmPossibility + { + foreach (var cmPossibility in enumerable) + { + yield return cmPossibility; + foreach (var child in Flatten(cmPossibility.SubPossibilitiesOS.Cast())) + { + yield return child; + } + } + } +} diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs index 503222fca..d0defe8ad 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateEntryProxy.cs @@ -4,7 +4,7 @@ namespace FwDataMiniLcmBridge.Api.UpdateProxy; -public class UpdateEntryProxy : Entry +public record UpdateEntryProxy : Entry { private readonly ILexEntry _lcmEntry; private readonly FwDataMiniLcmApi _lexboxLcmApi; diff --git a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs index 021587cfd..b2377ae5c 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs @@ -95,8 +95,8 @@ public async Task CanChangeComplexFormViaSync_ComplexForms() [Fact] public async Task CanChangeComplexFormTypeViaSync() { + var complexFormType = await _fixture.CrdtApi.CreateComplexFormType(new() { Name = new() { { "en", "complexFormType" } } }); var entry = await _fixture.CrdtApi.CreateEntry(new() { LexemeForm = { { "en", "complexForm1" } } }); - var complexFormType = await _fixture.CrdtApi.GetComplexFormTypes().FirstAsync(); var after = (Entry) entry.Copy(); after.ComplexFormTypes = [complexFormType]; await EntrySync.Sync(after, entry, _fixture.CrdtApi); diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs new file mode 100644 index 000000000..e0ec3f816 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/MercurialTestHelper.cs @@ -0,0 +1,31 @@ +using Chorus.VcsDrivers.Mercurial; +using SIL.CommandLineProcessing; +using SIL.PlatformUtilities; +using SIL.Progress; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public static class MercurialTestHelper +{ + private static readonly NullProgress NullProgress = new NullProgress(); + + private static string RunHgCommand(string repoPath, string args) + { + var result = HgRunner.Run(args, repoPath, 120, NullProgress); + if (result.ExitCode == 0) return result.StandardOutput; + throw new Exception( + $"hg {args} failed.\nStdOut: {result.StandardOutput}\nStdErr: {result.StandardError}"); + + } + + public static void HgClean(string repoPath, string exclude) + { + RunHgCommand(repoPath, $"purge --no-confirm --exclude {exclude}"); + } + + public static void HgUpdate(string repoPath, string rev) + { + RunHgCommand(repoPath, $"update \"{rev}\""); + } +} + diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs new file mode 100644 index 000000000..3bb84e1ca --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs @@ -0,0 +1,93 @@ +using System.IO.Compression; +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.Api; +using LcmCrdt; +using LexCore.Utils; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using SIL.IO; +using SIL.Progress; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public class Sena3Fixture : IAsyncLifetime +{ + private static readonly HttpClient http = new HttpClient(); + + public async Task InitializeAsync() + { + var services = new ServiceCollection() + .AddSyncServices(nameof(Sena3Fixture), false); + var rootServiceProvider = services.BuildServiceProvider(); + var fwProjectsFolder = rootServiceProvider.GetRequiredService>() + .Value + .ProjectsFolder; + if (Path.Exists(fwProjectsFolder)) Directory.Delete(fwProjectsFolder, true); + Directory.CreateDirectory(fwProjectsFolder); + + var crdtProjectsFolder = + rootServiceProvider.GetRequiredService>().Value.ProjectPath; + if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); + rootServiceProvider.Dispose(); + + Directory.CreateDirectory(crdtProjectsFolder); + await DownloadSena3(); + } + + public async Task<(CrdtMiniLcmApi CrdtApi, FwDataMiniLcmApi FwDataApi, IServiceProvider services, IDisposable cleanup)> SetupProjects() + { + var sena3MasterCopy = await DownloadSena3(); + + var rootServiceProvider = new ServiceCollection() + .AddSyncServices(nameof(Sena3Fixture), false) + .BuildServiceProvider(); + var cleanup = Defer.Action(() => rootServiceProvider.Dispose()); + var services = rootServiceProvider.CreateAsyncScope().ServiceProvider; + var projectName = "sena-3_" + Guid.NewGuid().ToString("N"); + + var projectsFolder = services.GetRequiredService>() + .Value + .ProjectsFolder; + var fwDataProject = new FwDataProject(projectName, projectsFolder); + var fwDataProjectPath = Path.Combine(fwDataProject.ProjectsPath, fwDataProject.Name); + DirectoryHelper.Copy(sena3MasterCopy, fwDataProjectPath); + File.Move(Path.Combine(fwDataProjectPath, "sena-3.fwdata"), fwDataProject.FilePath); + var fwDataMiniLcmApi = services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); + + var crdtProject = await services.GetRequiredService() + .CreateProject(new(projectName, FwProjectId: fwDataMiniLcmApi.ProjectId, SeedNewProjectData: false)); + var crdtMiniLcmApi = (CrdtMiniLcmApi)await services.OpenCrdtProject(crdtProject); + return (crdtMiniLcmApi, fwDataMiniLcmApi, services, cleanup); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + + private async Task DownloadSena3ProjectBackupStream() + { + var backupUrl = new Uri("https://drive.google.com/uc?export=download&id=1I-hwc0RHoQqW774gbS5qR-GHa1E7BlsS"); + var result = await http.GetAsync(backupUrl, HttpCompletionOption.ResponseHeadersRead); + return await result.Content.ReadAsStreamAsync(); + } + + private async Task DownloadSena3() + { + var tempFolder = Path.Combine(Path.GetTempPath(), nameof(Sena3Fixture)); + var sena3MasterCopy = Path.Combine(tempFolder, "sena-3"); + if (!Directory.Exists(sena3MasterCopy) || !File.Exists(Path.Combine(sena3MasterCopy, "sena-3.fwdata"))) + { + Directory.CreateDirectory(sena3MasterCopy); + await using var zipStream = await DownloadSena3ProjectBackupStream(); + //the zip file is structured like this: /sena-3/.hg + //by extracting it to tempFolder it should merge with sena-3 + ZipFile.ExtractToDirectory(zipStream, tempFolder); + + MercurialTestHelper.HgUpdate(sena3MasterCopy, "tip"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(new NullProgress(), false, Path.Combine(sena3MasterCopy, "sena-3.fwdata")); + MercurialTestHelper.HgClean(sena3MasterCopy, "sena-3.fwdata"); + } + return sena3MasterCopy; + } +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index 0b0443e0d..3cae9d60a 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -4,6 +4,7 @@ using FwDataMiniLcmBridge.LcmUtils; using FwDataMiniLcmBridge.Tests.Fixtures; using LcmCrdt; +using LexCore.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; @@ -20,7 +21,7 @@ public class SyncFixture : IAsyncLifetime _services.ServiceProvider.GetRequiredService(); public IServiceProvider Services => _services.ServiceProvider; private readonly string _projectName; - private readonly MockProjectContext _projectContext = new(null); + private readonly IDisposable _cleanup; public static SyncFixture Create([CallerMemberName] string projectName = "") => new(projectName); @@ -28,15 +29,10 @@ private SyncFixture(string projectName) { _projectName = projectName; var crdtServices = new ServiceCollection() - .AddLcmCrdtClient() - .AddSingleton(_projectContext) - .AddTestFwDataBridge() - .AddFwLiteProjectSync() - .Configure(c => c.ProjectsFolder = Path.Combine(".", _projectName, "FwData")) - .Configure(c => c.ProjectPath = Path.Combine(".", _projectName, "LcmCrdt")) - .AddLogging(builder => builder.AddDebug()) - .BuildServiceProvider(); - _services = crdtServices.CreateAsyncScope(); + .AddSyncServices(_projectName); + var rootServiceProvider = crdtServices.BuildServiceProvider(); + _cleanup = Defer.Action(() => rootServiceProvider.Dispose()); + _services = rootServiceProvider.CreateAsyncScope(); } public SyncFixture(): this("sena-3_" + Guid.NewGuid().ToString("N")) @@ -50,7 +46,7 @@ public async Task InitializeAsync() if (Path.Exists(projectsFolder)) Directory.Delete(projectsFolder, true); Directory.CreateDirectory(projectsFolder); _services.ServiceProvider.GetRequiredService() - .NewProject(new FwDataProject(_projectName, projectsFolder), "en", "fr"); + .NewProject(new FwDataProject(_projectName, projectsFolder), "en", "en"); FwDataApi = _services.ServiceProvider.GetRequiredService().GetFwDataMiniLcmApi(_projectName, false); var crdtProjectsFolder = @@ -58,20 +54,22 @@ public async Task InitializeAsync() if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); Directory.CreateDirectory(crdtProjectsFolder); var crdtProject = await _services.ServiceProvider.GetRequiredService() - .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: true)); + .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: false)); CrdtApi = (CrdtMiniLcmApi) await _services.ServiceProvider.OpenCrdtProject(crdtProject); } public async Task DisposeAsync() { await _services.DisposeAsync(); + _cleanup.Dispose(); } public CrdtMiniLcmApi CrdtApi { get; set; } = null!; public FwDataMiniLcmApi FwDataApi { get; set; } = null!; -} -public class MockProjectContext(CrdtProject? project) : ProjectContext -{ - public override CrdtProject? Project { get; set; } = project; + public void DeleteSyncSnapshot() + { + var snapshotPath = CrdtFwdataProjectSyncService.SnapshotPath(FwDataApi.Project); + if (File.Exists(snapshotPath)) File.Delete(snapshotPath); + } } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs new file mode 100644 index 000000000..6eb9f8b40 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/TestingKernel.cs @@ -0,0 +1,27 @@ +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.Tests.Fixtures; +using LcmCrdt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FwLiteProjectSync.Tests.Fixtures; + +public static class TestingKernel +{ + public static IServiceCollection AddSyncServices(this IServiceCollection services, string projectName, bool mockFwProjectLoader = true) + { + return services.AddLcmCrdtClient() + .AddTestFwDataBridge(mockFwProjectLoader) + .AddFwLiteProjectSync() + .AddSingleton(new MockProjectContext(null)) + .Configure(c => c.ProjectsFolder = Path.Combine(".", projectName, "FwData")) + .Configure(c => c.ProjectPath = Path.Combine(".", projectName, "LcmCrdt")) + .AddLogging(builder => builder.AddDebug()); + } + + public class MockProjectContext(CrdtProject? project) : ProjectContext + { + public override CrdtProject? Project { get; set; } = project; + } + +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj b/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj index c988caadc..0858009d0 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj +++ b/backend/FwLite/FwLiteProjectSync.Tests/FwLiteProjectSync.Tests.csproj @@ -7,6 +7,7 @@ false true + $(MSBuildProjectDirectory) @@ -28,6 +29,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + @@ -41,4 +44,12 @@ + + + + + + + + diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs new file mode 100644 index 000000000..77add42f5 --- /dev/null +++ b/backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs @@ -0,0 +1,144 @@ +using FluentAssertions.Equivalency; +using FluentAssertions.Execution; +using FwDataMiniLcmBridge.Api; +using FwLiteProjectSync.Tests.Fixtures; +using LcmCrdt; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using MiniLcm; +using MiniLcm.Models; +using SystemTextJsonPatch; + +namespace FwLiteProjectSync.Tests; + +public class Sena3SyncTests : IClassFixture, IAsyncLifetime +{ + private readonly Sena3Fixture _fixture; + private CrdtFwdataProjectSyncService _syncService = null!; + private CrdtMiniLcmApi _crdtApi = null!; + private FwDataMiniLcmApi _fwDataApi = null!; + private IDisposable? _cleanup; + + + public Sena3SyncTests(Sena3Fixture fixture) + { + _fixture = fixture; + } + + public async Task InitializeAsync() + { + (_crdtApi, _fwDataApi, var services, _cleanup) = await _fixture.SetupProjects(); + _syncService = services.GetRequiredService(); + _fwDataApi.EntryCount.Should().BeGreaterThan(100, "project should be loaded and have entries"); + } + + public Task DisposeAsync() + { + _cleanup?.Dispose(); + return Task.CompletedTask; + } + + private void ShouldAllBeEquivalentTo(Dictionary crdtEntries, Dictionary fwdataEntries) + { + crdtEntries.Keys.Should().BeEquivalentTo(fwdataEntries.Keys); + using (new AssertionScope()) + { + foreach (var crdtEntry in crdtEntries.Values) + { + var fwdataEntry = fwdataEntries[crdtEntry.Id]; + crdtEntry.Should().BeEquivalentTo(fwdataEntry, + options => options + .For(e => e.Components).Exclude(c => c.Id) + .For(e => e.ComplexForms).Exclude(c => c.Id), + $"CRDT entry {crdtEntry.Id} was synced with FwData"); + } + } + } + + //by default the first sync is an import, this will skip that so that the sync will actually sync data + private async Task BypassImport() + { + await _syncService.SaveProjectSnapshot(_fwDataApi.Project, new ([], [], [])); + } + + //this lets us query entries when there is no writing system + private async Task WorkaroundMissingWritingSystems() + { + //must have at least one writing system to query for entries + await _crdtApi.CreateWritingSystem(WritingSystemType.Vernacular, (await _fwDataApi.GetWritingSystems()).Vernacular.First()); + + } + + [Fact] + public async Task DryRunImport_MakesNoChanges() + { + await WorkaroundMissingWritingSystems(); + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + //should still be empty + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + } + + [Fact] + public async Task DryRunImport_MakesTheSameChangesAsImport() + { + var dryRunSyncResult = await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + var syncResult = await _syncService.Sync(_crdtApi, _fwDataApi); + dryRunSyncResult.Should().BeEquivalentTo(syncResult); + } + + [Fact] + public async Task DryRunSync_MakesNoChanges() + { + await BypassImport(); + await WorkaroundMissingWritingSystems(); + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + //should still be empty + _crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty(); + } + + [Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")] + public async Task DryRunSync_MakesTheSameChangesAsImport() + { + await BypassImport(); + var dryRunSyncResult = await _syncService.SyncDryRun(_crdtApi, _fwDataApi); + var syncResult = await _syncService.Sync(_crdtApi, _fwDataApi); + dryRunSyncResult.Should().BeEquivalentTo(syncResult); + } + + [Fact] + public async Task FirstSena3SyncJustDoesAnSync() + { + var results = await _syncService.Sync(_crdtApi, _fwDataApi); + results.FwdataChanges.Should().Be(0); + results.CrdtChanges.Should().BeGreaterThanOrEqualTo(_fwDataApi.EntryCount); + + var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id); + ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries); + } + + [Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")] + public async Task SyncWithoutImport_CrdtShouldMatchFwdata() + { + await BypassImport(); + + var results = await _syncService.Sync(_crdtApi, _fwDataApi); + results.FwdataChanges.Should().Be(0); + results.CrdtChanges.Should().BeGreaterThan(_fwDataApi.EntryCount); + + var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id); + var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id); + ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries); + } + + [Fact] + public async Task SecondSena3SyncDoesNothing() + { + await _syncService.Sync(_crdtApi, _fwDataApi); + var secondSync = await _syncService.Sync(_crdtApi, _fwDataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); + } +} diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index 752d77fa2..36b4084f2 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -65,6 +65,8 @@ public async Task DisposeAsync() { await _fixture.CrdtApi.DeleteEntry(entry.Id); } + + _fixture.DeleteSyncSnapshot(); } public SyncTests(SyncFixture fixture) @@ -84,7 +86,18 @@ public async Task FirstSyncJustDoesAnImport() var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); crdtEntries.Should().BeEquivalentTo(fwdataEntries, options => options.For(e => e.Components).Exclude(c => c.Id) - .For(e => e.ComplexForms).Exclude(c => c.Id)); + .For(e => e.ComplexForms).Exclude(c => c.Id)); + } + + [Fact] + public async Task SecondSyncDoesNothing() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); } [Fact] @@ -138,6 +151,90 @@ await crdtApi.CreateEntry(new Entry() .For(e => e.ComplexForms).Exclude(c => c.Id)); } + [Fact] + public async Task SyncDryRun_NoChangesAreSynced() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + var fwDataEntryId = Guid.NewGuid(); + var crdtEntryId = Guid.NewGuid(); + + await fwdataApi.CreateEntry(new Entry() + { + Id = fwDataEntryId, + LexemeForm = { { "en", "Pear" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Pear" } }, } + ] + }); + await crdtApi.CreateEntry(new Entry() + { + Id = crdtEntryId, + LexemeForm = { { "en", "Banana" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Banana" } }, } + ] + }); + await _syncService.SyncDryRun(crdtApi, fwdataApi); + + var crdtEntries = await crdtApi.GetEntries().ToArrayAsync(); + var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); + crdtEntries.Select(e => e.Id).Should().NotContain(fwDataEntryId); + fwdataEntries.Select(e => e.Id).Should().NotContain(crdtEntryId); + } + + [Fact] + public async Task CreatingAComplexEntryInFwDataSyncsWithoutIssue() + { + var crdtApi = _fixture.CrdtApi; + var fwdataApi = _fixture.FwDataApi; + await _syncService.Sync(crdtApi, fwdataApi); + + var hat = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Hat" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Hat" } }, } + ] + }); + var stand = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Stand" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Stand" } }, } + ] + }); + var hatstand = await fwdataApi.CreateEntry(new Entry() + { + LexemeForm = { { "en", "Hatstand" } }, + Senses = + [ + new Sense() { Gloss = { { "en", "Hatstand" } }, } + ], + }); + var component1 = ComplexFormComponent.FromEntries(hatstand, hat); + var component2 = ComplexFormComponent.FromEntries(hatstand, stand); + hatstand.Components = [component1, component2]; + await _syncService.Sync(crdtApi, fwdataApi); + + var crdtEntries = await crdtApi.GetEntries().ToArrayAsync(); + var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync(); + crdtEntries.Should().BeEquivalentTo(fwdataEntries, + options => options.For(e => e.Components).Exclude(c => c.Id) + .For(e => e.ComplexForms).Exclude(c => c.Id)); + + // Sync again, ensure no problems or changes + var secondSync = await _syncService.Sync(crdtApi, fwdataApi); + secondSync.CrdtChanges.Should().Be(0); + secondSync.FwdataChanges.Should().Be(0); + } + + [Fact] public async Task PartsOfSpeechSyncBothWays() { @@ -273,7 +370,7 @@ await fwdataApi.CreateEntry(new Entry() LexemeForm = { { "en", "Pear" } }, Senses = [ - new Sense() { Gloss = { { "en", "Pear" } }, SemanticDomains = [ semdom3 ] } + new Sense() { Gloss = { { "en", "Pear" } }, SemanticDomains = [semdom3] } ] }); await crdtApi.CreateEntry(new Entry() @@ -281,7 +378,7 @@ await crdtApi.CreateEntry(new Entry() LexemeForm = { { "en", "Banana" } }, Senses = [ - new Sense() { Gloss = { { "en", "Banana" } }, SemanticDomains = [ semdom3 ] } + new Sense() { Gloss = { { "en", "Banana" } }, SemanticDomains = [semdom3] } ] }); await _syncService.Sync(crdtApi, fwdataApi); @@ -365,7 +462,7 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() await _syncService.Sync(crdtApi, fwdataApi); await fwdataApi.CreateSense(_testEntry.Id, new Sense() - { + { Gloss = { { "en", "Fruit" } }, Definition = { { "en", "a round fruit, red or yellow" } }, }); @@ -373,7 +470,7 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() { Gloss = { { "en", "Tree" } }, Definition = { { "en", "a tall, woody plant, which grows fruit" } }, - }); + }); await _syncService.Sync(crdtApi, fwdataApi); @@ -383,4 +480,24 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth() options => options.For(e => e.Components).Exclude(c => c.Id) .For(e => e.ComplexForms).Exclude(c => c.Id)); } + + [Fact] + public async Task CanCreateAComplexFormAndItsComponentInOneSync() + { + //ensure they are synced so a real sync will happen when we want it to + await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + + var complexFormEntry = await _fixture.CrdtApi.CreateEntry(new() { LexemeForm = { { "en", "complexForm" } } }); + var componentEntry = await _fixture.CrdtApi.CreateEntry(new() + { + LexemeForm = { { "en", "component" } }, + ComplexForms = + [ + new ComplexFormComponent() { ComplexFormEntryId = complexFormEntry.Id, ComponentEntryId = Guid.Empty } + ] + }); + + //one of the entries will be created first, it will try to create the reference to the other but it won't exist yet + await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi); + } } diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index ee42e1026..58685043b 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -1,9 +1,9 @@ using System.Text.Json; +using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.Api; using LcmCrdt; using LexCore.Sync; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using MiniLcm; using MiniLcm.Models; using MiniLcm.SyncHelpers; @@ -12,21 +12,32 @@ namespace FwLiteProjectSync; -public class CrdtFwdataProjectSyncService(IOptions lcmCrdtConfig, MiniLcmImport miniLcmImport, ILogger logger) +public class CrdtFwdataProjectSyncService(MiniLcmImport miniLcmImport, ILogger logger) { + public record DryRunSyncResult( + int CrdtChanges, + int FwdataChanges, + List CrdtDryRunRecords, + List FwDataDryRunRecords) : SyncResult(CrdtChanges, FwdataChanges); + + public async Task SyncDryRun(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi) + { + return (DryRunSyncResult) await Sync(crdtApi, fwdataApi, true); + } + public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi, bool dryRun = false) { if (crdtApi is CrdtMiniLcmApi crdt && crdt.ProjectData.FwProjectId != fwdataApi.ProjectId) { throw new InvalidOperationException($"Project id mismatch, CRDT Id: {crdt.ProjectData.FwProjectId}, FWData Id: {fwdataApi.ProjectId}"); } - var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath); + var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project); SyncResult result = await Sync(crdtApi, fwdataApi, dryRun, fwdataApi.EntryCount, projectSnapshot); fwdataApi.Save(); if (!dryRun) { - await SaveProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath, + await SaveProjectSnapshot(fwdataApi.Project, new ProjectSnapshot( await fwdataApi.GetEntries().ToArrayAsync(), await fwdataApi.GetPartsOfSpeech().ToArrayAsync(), @@ -47,6 +58,7 @@ private async Task Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi, { await miniLcmImport.ImportProject(crdtApi, fwdataApi, entryCount); LogDryRun(crdtApi, "crdt"); + if (dryRun) return new DryRunSyncResult(entryCount, 0, GetDryRunRecords(crdtApi), []); return new SyncResult(entryCount, 0); } @@ -68,7 +80,7 @@ private async Task Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi, LogDryRun(fwdataApi, "fwdata"); //todo push crdt changes to lexbox - + if (dryRun) return new DryRunSyncResult(crdtChanges, fwdataChanges, GetDryRunRecords(crdtApi), GetDryRunRecords(fwdataApi)); return new SyncResult(crdtChanges, fwdataChanges); } @@ -83,23 +95,32 @@ private void LogDryRun(IMiniLcmApi api, string type) logger.LogInformation($"Dry run {type} changes: {dryRunApi.DryRunRecords.Count}"); } + private List GetDryRunRecords(IMiniLcmApi api) + { + return ((DryRunMiniLcmApi)api).DryRunRecords; + } + public record ProjectSnapshot(Entry[] Entries, PartOfSpeech[] PartsOfSpeech, SemanticDomain[] SemanticDomains); - private async Task GetProjectSnapshot(string projectName, string? projectPath) + private async Task GetProjectSnapshot(FwDataProject project) { - projectPath ??= lcmCrdtConfig.Value.ProjectPath; - var snapshotPath = Path.Combine(projectPath, $"{projectName}_snapshot.json"); + var snapshotPath = SnapshotPath(project); if (!File.Exists(snapshotPath)) return null; await using var file = File.OpenRead(snapshotPath); return await JsonSerializer.DeserializeAsync(file); } - private async Task SaveProjectSnapshot(string projectName, string? projectPath, ProjectSnapshot projectSnapshot) + internal async Task SaveProjectSnapshot(FwDataProject project, ProjectSnapshot projectSnapshot) { - projectPath ??= lcmCrdtConfig.Value.ProjectPath; - var snapshotPath = Path.Combine(projectPath, $"{projectName}_snapshot.json"); + var snapshotPath = SnapshotPath(project); await using var file = File.Create(snapshotPath); await JsonSerializer.SerializeAsync(file, projectSnapshot); } + internal static string SnapshotPath(FwDataProject project) + { + var projectPath = project.ProjectsPath; + var snapshotPath = Path.Combine(projectPath, $"{project.Name}_snapshot.json"); + return snapshotPath; + } } diff --git a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs index 25c317670..15c17a387 100644 --- a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs +++ b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs @@ -216,7 +216,7 @@ public Task CreateComplexFormComponent(ComplexFormComponen public Task DeleteComplexFormComponent(ComplexFormComponent complexFormComponent) { - DryRunRecords.Add(new DryRunRecord(nameof(DeleteComplexFormComponent), $"Delete complex form component complex entry: {complexFormComponent.ComplexFormHeadword}, component entry: {complexFormComponent.ComponentHeadword}")); + DryRunRecords.Add(new DryRunRecord(nameof(DeleteComplexFormComponent), $"Delete complex form component: {complexFormComponent}")); return Task.CompletedTask; } diff --git a/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj b/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj index 7e4c26e04..2d0c452f0 100644 --- a/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj +++ b/backend/FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj @@ -19,4 +19,8 @@ + + + + diff --git a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt index a7b2cc8c5..129ee865c 100644 --- a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt @@ -24,6 +24,8 @@ ComponentEntryId (Guid) Required FK Index ComponentHeadword (string) ComponentSenseId (Guid?) FK Index + Annotations: + Relational:ColumnName: ComponentSenseId DeletedAt (DateTimeOffset?) SnapshotId (no field, Guid?) Shadow FK Index Keys: @@ -34,10 +36,15 @@ ComplexFormComponent {'ComponentSenseId'} -> Sense {'Id'} Cascade ComplexFormComponent {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull Indexes: - ComplexFormEntryId ComponentEntryId ComponentSenseId SnapshotId Unique + ComplexFormEntryId, ComponentEntryId Unique + Annotations: + Relational:Filter: ComponentSenseId IS NULL + ComplexFormEntryId, ComponentEntryId, ComponentSenseId Unique + Annotations: + Relational:Filter: ComponentSenseId IS NOT NULL Annotations: DiscriminatorProperty: Relational:FunctionName: diff --git a/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs b/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs index b2370c85e..13eb26933 100644 --- a/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/JsonPatchEntryRewriteTests.cs @@ -20,7 +20,6 @@ public void ChangesFromJsonPatch_AddComponentMakesAddEntryComponentChange() changes.Should().ContainSingle().Which.Should().BeOfType().Subject; addEntryComponentChange.ComplexFormEntryId.Should().Be(_entry.Id); addEntryComponentChange.ComponentEntryId.Should().Be(componentEntry.Id); - addEntryComponentChange.ComponentHeadword.Should().Be(componentEntry.Headword()); } [Fact] @@ -84,7 +83,6 @@ public void ChangesFromJsonPatch_AddComplexFormMakesAddEntryComponentChange() changes.Should().ContainSingle().Which.Should().BeOfType().Subject; addEntryComponentChange.ComplexFormEntryId.Should().Be(_entry.Id); addEntryComponentChange.ComponentEntryId.Should().Be(componentEntry.Id); - addEntryComponentChange.ComponentHeadword.Should().Be(componentEntry.Headword()); } [Fact] diff --git a/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs b/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs index a51313198..f93630058 100644 --- a/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs +++ b/backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs @@ -10,10 +10,8 @@ namespace LcmCrdt.Changes.Entries; public class AddEntryComponentChange : CreateChange, ISelfNamedType { public Guid ComplexFormEntryId { get; } - public string? ComplexFormHeadword { get; } public Guid ComponentEntryId { get; } public Guid? ComponentSenseId { get; } - public string? ComponentHeadword { get; } [JsonConstructor] public AddEntryComponentChange(Guid entityId, @@ -24,9 +22,7 @@ public AddEntryComponentChange(Guid entityId, Guid? componentSenseId = null) : base(entityId) { ComplexFormEntryId = complexFormEntryId; - ComplexFormHeadword = complexFormHeadword; ComponentEntryId = componentEntryId; - ComponentHeadword = componentHeadword; ComponentSenseId = componentSenseId; } @@ -41,17 +37,22 @@ public AddEntryComponentChange(Guid entityId, public override async ValueTask NewEntity(Commit commit, ChangeContext context) { + var complexFormEntry = await context.GetCurrent(ComplexFormEntryId); + var componentEntry = await context.GetCurrent(ComponentEntryId); + Sense? componentSense = null; + if (ComponentSenseId is not null) + componentSense = await context.GetCurrent(ComponentSenseId.Value); return new ComplexFormComponent { Id = EntityId, ComplexFormEntryId = ComplexFormEntryId, - ComplexFormHeadword = ComplexFormHeadword, + ComplexFormHeadword = complexFormEntry?.Headword(), ComponentEntryId = ComponentEntryId, - ComponentHeadword = ComponentHeadword, + ComponentHeadword = componentEntry?.Headword(), ComponentSenseId = ComponentSenseId, - DeletedAt = (await context.IsObjectDeleted(ComponentEntryId) || - await context.IsObjectDeleted(ComplexFormEntryId) || - ComponentSenseId.HasValue && await context.IsObjectDeleted(ComponentSenseId.Value)) + DeletedAt = (complexFormEntry?.DeletedAt is not null || + componentEntry?.DeletedAt is not null || + (ComponentSenseId.HasValue && componentSense?.DeletedAt is not null)) ? commit.DateTime : (DateTime?)null, }; diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index 241c8daad..28cc4048e 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -18,14 +18,15 @@ public class CrdtMiniLcmApi(DataModel dataModel, CurrentProjectService projectSe private Guid ClientId { get; } = projectService.ProjectData.ClientId; public ProjectData ProjectData => projectService.ProjectData; - private IQueryable Entries => dataModel.QueryLatest(); - private IQueryable ComplexFormComponents => dataModel.QueryLatest(); - private IQueryable ComplexFormTypes => dataModel.QueryLatest(); - private IQueryable Senses => dataModel.QueryLatest(); - private IQueryable ExampleSentences => dataModel.QueryLatest(); - private IQueryable WritingSystems => dataModel.QueryLatest(); - private IQueryable SemanticDomains => dataModel.QueryLatest(); - private IQueryable PartsOfSpeech => dataModel.QueryLatest(); + private IQueryable Entries => dataModel.QueryLatest().AsTracking(false); + private IQueryable ComplexFormComponents => dataModel.QueryLatest() + .AsTracking(false); + private IQueryable ComplexFormTypes => dataModel.QueryLatest().AsTracking(false); + private IQueryable Senses => dataModel.QueryLatest().AsTracking(false); + private IQueryable ExampleSentences => dataModel.QueryLatest().AsTracking(false); + private IQueryable WritingSystems => dataModel.QueryLatest().AsTracking(false); + private IQueryable SemanticDomains => dataModel.QueryLatest().AsTracking(false); + private IQueryable PartsOfSpeech => dataModel.QueryLatest().AsTracking(false); public async Task GetWritingSystems() { @@ -134,7 +135,7 @@ public async Task DeleteSemanticDomain(Guid id) public async Task BulkImportSemanticDomains(IEnumerable semanticDomains) { - await dataModel.AddChanges(ClientId, semanticDomains.Select(sd => new CreateSemanticDomainChange(sd.Id, sd.Name, sd.Code))); + await dataModel.AddChanges(ClientId, semanticDomains.Select(sd => new CreateSemanticDomainChange(sd.Id, sd.Name, sd.Code, sd.Predefined))); } public IAsyncEnumerable GetComplexFormTypes() @@ -151,6 +152,11 @@ public async Task CreateComplexFormType(ComplexFormType complex public async Task CreateComplexFormComponent(ComplexFormComponent complexFormComponent) { + var existing = await ComplexFormComponents.SingleOrDefaultAsync(c => + c.ComplexFormEntryId == complexFormComponent.ComplexFormEntryId + && c.ComponentEntryId == complexFormComponent.ComponentEntryId + && c.ComponentSenseId == complexFormComponent.ComponentSenseId); + if (existing is not null) return existing; var addEntryComponentChange = new AddEntryComponentChange(complexFormComponent); await dataModel.AddChange(ClientId, addEntryComponentChange); return (await ComplexFormComponents.SingleOrDefaultAsync(c => c.Id == addEntryComponentChange.EntityId)) ?? throw NotFoundException.ForType(); @@ -256,7 +262,12 @@ public async Task BulkCreateEntries(IAsyncEnumerable entries) { var semanticDomains = await SemanticDomains.ToDictionaryAsync(sd => sd.Id, sd => sd); var partsOfSpeech = await PartsOfSpeech.ToDictionaryAsync(p => p.Id, p => p); - await dataModel.AddChanges(ClientId, entries.ToBlockingEnumerable().SelectMany(entry => CreateEntryChanges(entry, semanticDomains, partsOfSpeech))); + await dataModel.AddChanges(ClientId, + entries.ToBlockingEnumerable() + .SelectMany(entry => CreateEntryChanges(entry, semanticDomains, partsOfSpeech)) + //force entries to be created first, this avoids issues where references are created before the entry is created + .OrderBy(c => c is CreateEntryChange ? 0 : 1) + ); } private IEnumerable CreateEntryChanges(Entry entry, Dictionary semanticDomains, Dictionary partsOfSpeech) diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index 9fd13ce3e..74562c440 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -134,7 +134,22 @@ public static void ConfigureCrdt(CrdtConfig config) .Add() .Add(builder => { + const string componentSenseId = "ComponentSenseId"; builder.ToTable("ComplexFormComponents"); + builder.Property(c => c.ComponentSenseId).HasColumnName(componentSenseId); + //these indexes are used to ensure that we don't create duplicate complex form components + //we need the filter otherwise 2 components which are the same and have a null sense id can be created because 2 rows with the same null are not considered duplicates + builder.HasIndex(component => new + { + component.ComplexFormEntryId, + component.ComponentEntryId, + component.ComponentSenseId + }).IsUnique().HasFilter($"{componentSenseId} IS NOT NULL"); + builder.HasIndex(component => new + { + component.ComplexFormEntryId, + component.ComponentEntryId + }).IsUnique().HasFilter($"{componentSenseId} IS NULL"); }); config.ChangeTypeListBuilder.Add>() diff --git a/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs b/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs index 4f51ca275..d6f646435 100644 --- a/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/ComplexFormComponentTestsBase.cs @@ -50,6 +50,22 @@ public async Task CreateComplexFormComponent_Works() component.ComponentHeadword.Should().Be("component"); } + [Fact] + public async Task CreateComplexFormComponent_UsingTheSameComponentWithNullSenseDoesNothing() + { + var component1 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + var component2 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + component2.Should().BeEquivalentTo(component1); + } + + [Fact] + public async Task CreateComplexFormComponent_UsingTheSameComponentWithSenseDoesNothing() + { + var component1 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry, _componentSenseId1)); + var component2 = await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry, _componentSenseId1)); + component2.Should().BeEquivalentTo(component1); + } + [Fact] public async Task CreateComplexFormComponent_WorksWithSense() { @@ -83,4 +99,43 @@ public async Task CreateComplexFormType_Works() var types = await Api.GetComplexFormTypes().ToArrayAsync(); types.Should().ContainSingle(t => t.Id == complexFormType.Id); } + + [Fact] + public async Task AddComplexFormType_Works() + { + var complexFormType = new ComplexFormType() { Id = Guid.NewGuid(), Name = new() { { "en", "test" } } }; + await Api.CreateComplexFormType(complexFormType); + await Api.AddComplexFormType(_complexFormEntryId, complexFormType.Id); + var entry = await Api.GetEntry(_complexFormEntryId); + entry.Should().NotBeNull(); + entry!.ComplexFormTypes.Should().ContainSingle(c => c.Id == complexFormType.Id); + } + + [Fact] + public async Task RemoveComplexFormType_Works() + { + await AddComplexFormType_Works(); + var entry = await Api.GetEntry(_complexFormEntryId); + await Api.RemoveComplexFormType(_complexFormEntryId, entry!.ComplexFormTypes[0].Id); + entry = await Api.GetEntry(_complexFormEntryId); + entry.Should().NotBeNull(); + entry!.ComplexFormTypes.Should().BeEmpty(); + } + + [Fact] + public async Task RemoveComplexFormType_WorksWhenTypeDoesNotExist() + { + await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + await Api.RemoveComplexFormType(_complexFormEntryId, Guid.NewGuid()); + } + + [Fact] + public async Task RemoveComplexFormType_WorksWhenTypeIsNotOnEntry() + { + //FW projects react differently if an entry has complex forms or not + await Api.CreateComplexFormComponent(ComplexFormComponent.FromEntries(_complexFormEntry, _componentEntry)); + var complexFormType = new ComplexFormType() { Id = Guid.NewGuid(), Name = new() { { "en", "test" } } }; + await Api.CreateComplexFormType(complexFormType); + await Api.RemoveComplexFormType(_complexFormEntryId, Guid.NewGuid()); + } } diff --git a/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs b/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs new file mode 100644 index 000000000..8b680e716 --- /dev/null +++ b/backend/FwLite/MiniLcm.Tests/Models/EntryTests.cs @@ -0,0 +1,18 @@ +namespace MiniLcm.Tests.Models; + +public class EntryTests +{ + [Fact] + public void Headword_SameResultForDifferentOrderedMultiStrings() + { + var entry = new Entry() + { + LexemeForm = new MultiString() { Values = { { "en", "test" }, { "fr", "test2" } } } + }; + var entry2 = new Entry() + { + LexemeForm = new MultiString() { Values = { { "fr", "test2" }, { "en", "test" } } } + }; + entry.Headword().Should().Be(entry2.Headword()); + } +} diff --git a/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs b/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs new file mode 100644 index 000000000..128cb0515 --- /dev/null +++ b/backend/FwLite/MiniLcm/Exceptions/CreateObjectException.cs @@ -0,0 +1,12 @@ +namespace MiniLcm.Exceptions; + +public class CreateObjectException: Exception +{ + public CreateObjectException(string? message) : base(message) + { + } + + public CreateObjectException(string? message, Exception? innerException) : base(message, innerException) + { + } +} diff --git a/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs b/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs new file mode 100644 index 000000000..e1f284cde --- /dev/null +++ b/backend/FwLite/MiniLcm/Exceptions/SyncObjectException.cs @@ -0,0 +1,12 @@ +namespace MiniLcm.Exceptions; + +public class SyncObjectException: Exception +{ + public SyncObjectException(string? message) : base(message) + { + } + + public SyncObjectException(string? message, Exception? innerException) : base(message, innerException) + { + } +} diff --git a/backend/FwLite/MiniLcm/Models/Entry.cs b/backend/FwLite/MiniLcm/Models/Entry.cs index 191db2753..cabb8c9bb 100644 --- a/backend/FwLite/MiniLcm/Models/Entry.cs +++ b/backend/FwLite/MiniLcm/Models/Entry.cs @@ -1,6 +1,6 @@ namespace MiniLcm.Models; -public class Entry : IObjectWithId +public record Entry : IObjectWithId { public Guid Id { get; set; } public DateTimeOffset? DeletedAt { get; set; } @@ -28,8 +28,10 @@ public class Entry : IObjectWithId public string Headword() { - var word = CitationForm.Values.Values.FirstOrDefault(); - if (string.IsNullOrEmpty(word)) word = LexemeForm.Values.Values.FirstOrDefault(); + //order by code to ensure the headword is stable + //todo choose ws by preference based on ws order/default + var word = CitationForm.Values.OrderBy(kvp => kvp.Key.Code).FirstOrDefault().Value; + if (string.IsNullOrEmpty(word)) word = LexemeForm.Values.OrderBy(kvp => kvp.Key.Code).FirstOrDefault().Value; return word?.Trim() ?? "(Unknown)"; } @@ -68,6 +70,11 @@ public Guid[] GetReferences() public void RemoveReference(Guid id, DateTimeOffset time) { } + + public Entry WithoutEntryRefs() + { + return this with { Components = [], ComplexForms = [] }; + } } public class Variants diff --git a/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs b/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs index 2cc251f2b..33bda1e46 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/DiffCollection.cs @@ -4,6 +4,73 @@ namespace MiniLcm.SyncHelpers; public static class DiffCollection { + /// + /// Diffs a list, for new items calls add, it will then call update for the item returned from the add, using that as the before item for the replace call + /// + /// + /// + /// + /// + /// api, value, return value to be used as the before item for the replace call + /// + /// api, before, after is the parameter order + /// + /// + /// + public static async Task DiffAddThenUpdate( + IMiniLcmApi api, + IList before, + IList after, + Func identity, + Func> add, + Func> remove, + Func> replace) where TId : notnull + { + var changes = 0; + var afterEntriesDict = after.ToDictionary(identity); + + foreach (var beforeEntry in before) + { + if (afterEntriesDict.TryGetValue(identity(beforeEntry), out var afterEntry)) + { + changes += await replace(api, beforeEntry, afterEntry); + } + else + { + changes += await remove(api, beforeEntry); + } + + afterEntriesDict.Remove(identity(beforeEntry)); + } + + var postAddUpdates = new List<(T created, T after)>(afterEntriesDict.Values.Count); + foreach (var value in afterEntriesDict.Values) + { + changes++; + postAddUpdates.Add((await add(api, value), value)); + } + foreach ((T createdItem, T afterItem) in postAddUpdates) + { + //todo this may do a lot more work than it needs to, eg sense will be created during add, but they will be checked again here when we know they didn't change + await replace(api, createdItem, afterItem); + } + + return changes; + } + + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// api, before, after is the parameter order + /// + /// + /// public static async Task Diff( IMiniLcmApi api, IList before, @@ -36,6 +103,7 @@ public static async Task Diff( return changes; } + public static async Task Diff( IMiniLcmApi api, IList before, diff --git a/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs b/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs index 50fe78892..c8f2db5a3 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs @@ -10,10 +10,13 @@ public static async Task Sync(Entry[] afterEntries, Entry[] beforeEntries, IMiniLcmApi api) { - Func> add = static async (api, afterEntry) => + Func> add = static async (api, afterEntry) => { - await api.CreateEntry(afterEntry); - return 1; + //create each entry without components. + //After each entry is created, then replace will be called to create those components + var entryWithoutEntryRefs = afterEntry.WithoutEntryRefs(); + await api.CreateEntry(entryWithoutEntryRefs); + return entryWithoutEntryRefs; }; Func> remove = static async (api, beforeEntry) => { @@ -21,19 +24,26 @@ public static async Task Sync(Entry[] afterEntries, return 1; }; Func> replace = static async (api, beforeEntry, afterEntry) => await Sync(afterEntry, beforeEntry, api); - return await DiffCollection.Diff(api, beforeEntries, afterEntries, add, remove, replace); + return await DiffCollection.DiffAddThenUpdate(api, beforeEntries, afterEntries, entry => entry.Id, add, remove, replace); } public static async Task Sync(Entry afterEntry, Entry beforeEntry, IMiniLcmApi api) { - var updateObjectInput = EntryDiffToUpdate(beforeEntry, afterEntry); - if (updateObjectInput is not null) await api.UpdateEntry(afterEntry.Id, updateObjectInput); - var changes = await SensesSync(afterEntry.Id, afterEntry.Senses, beforeEntry.Senses, api); + try + { + var updateObjectInput = EntryDiffToUpdate(beforeEntry, afterEntry); + if (updateObjectInput is not null) await api.UpdateEntry(afterEntry.Id, updateObjectInput); + var changes = await SensesSync(afterEntry.Id, afterEntry.Senses, beforeEntry.Senses, api); - changes += await Sync(afterEntry.Components, beforeEntry.Components, api); - changes += await Sync(afterEntry.ComplexForms, beforeEntry.ComplexForms, api); - changes += await Sync(afterEntry.Id, afterEntry.ComplexFormTypes, beforeEntry.ComplexFormTypes, api); - return changes + (updateObjectInput is null ? 0 : 1); + changes += await Sync(afterEntry.Components, beforeEntry.Components, api); + changes += await Sync(afterEntry.ComplexForms, beforeEntry.ComplexForms, api); + changes += await Sync(afterEntry.Id, afterEntry.ComplexFormTypes, beforeEntry.ComplexFormTypes, api); + return changes + (updateObjectInput is null ? 0 : 1); + } + catch (Exception e) + { + throw new SyncObjectException($"Failed to sync entry {afterEntry}", e); + } } private static async Task Sync(Guid entryId,