Skip to content

Commit

Permalink
implement update and delete apis for ComplexFormTypes and sync them i…
Browse files Browse the repository at this point in the history 
…n CrdtFwdataProjectSyncService (#1295)

* implement update and delete apis for ComplexFormTypes and sync them in CrdtFwdataProjectSyncService

* also sync writing system changes

* enable test which imports all data as a sync instead of the dedicated import code

* test creating writing systems, make create WS throw if it already exists

* fix issue with conflicts between the multiple sync tests running at once. Also fixes issues where previous projects would not be cleaned up
  • Loading branch information
@hahn-kev
hahn-kev authored Dec 4, 2024
1 parent 62e2c19 commit effc25b
Show file tree
Hide file tree
Showing 27 changed files with 405 additions and 151 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/fw-lite.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ on:
jobs:
build-and-test:
name: Build FW Lite and run tests
timeout-minutes: 20
timeout-minutes: 30
runs-on: windows-latest
outputs:
version: ${{ steps.setVersion.outputs.VERSION }}
Expand Down
52 changes: 49 additions & 3 deletions backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,11 @@ internal void CompleteExemplars(WritingSystems writingSystems)

public Task<WritingSystem> CreateWritingSystem(WritingSystemType type, WritingSystem writingSystem)
{
var exitingWs = type == WritingSystemType.Analysis ? Cache.ServiceLocator.WritingSystems.AnalysisWritingSystems : Cache.ServiceLocator.WritingSystems.VernacularWritingSystems;
if (exitingWs.Any(ws => ws.Id == writingSystem.WsId))
{
throw new DuplicateObjectException($"Writing system {writingSystem.WsId.Code} already exists");
}
CoreWritingSystemDefinition? ws = null;
UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create Writing System",
"Remove writing system",
Expand Down Expand Up @@ -377,6 +382,14 @@ public IAsyncEnumerable<ComplexFormType> GetComplexFormTypes()
{
return ComplexFormTypesFlattened.Select(ToComplexFormType).ToAsyncEnumerable();
}

public Task<ComplexFormType?> GetComplexFormType(Guid id)
{
var lexEntryType = ComplexFormTypesFlattened.SingleOrDefault(c => c.Guid == id);
if (lexEntryType is null) return Task.FromResult<ComplexFormType?>(null);
return Task.FromResult<ComplexFormType?>(ToComplexFormType(lexEntryType));
}

private ComplexFormType ToComplexFormType(ILexEntryType t)
{
return new ComplexFormType() { Id = t.Guid, Name = FromLcmMultiString(t.Name) };
Expand All @@ -400,6 +413,40 @@ public async Task<ComplexFormType> CreateComplexFormType(ComplexFormType complex
return ToComplexFormType(ComplexFormTypesFlattened.Single(c => c.Guid == complexFormType.Id));
}

public Task<ComplexFormType> UpdateComplexFormType(Guid id, UpdateObjectInput<ComplexFormType> update)
{
var type = ComplexFormTypesFlattened.SingleOrDefault(c => c.Guid == id);
if (type is null) throw new NullReferenceException($"unable to find complex form type with id {id}");
UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Update Complex Form Type",
"Revert Complex Form Type",
Cache.ServiceLocator.ActionHandler,
() =>
{
var updateProxy = new UpdateComplexFormTypeProxy(type, null, this);
update.Apply(updateProxy);
});
return Task.FromResult(ToComplexFormType(type));
}

public async Task<ComplexFormType> UpdateComplexFormType(ComplexFormType before, ComplexFormType after)
{
await ComplexFormTypeSync.Sync(before, after, this);
return ToComplexFormType(ComplexFormTypesFlattened.Single(c => c.Guid == after.Id));
}

public async Task DeleteComplexFormType(Guid id)
{
var type = ComplexFormTypesFlattened.SingleOrDefault(c => c.Guid == id);
if (type is null) return;
await Cache.DoUsingNewOrCurrentUOW("Delete Complex Form Type",
"Revert delete",
() =>
{
type.Delete();
return ValueTask.CompletedTask;
});
}

public IAsyncEnumerable<VariantType> GetVariantTypes()
{
return VariantTypes.PossibilitiesOS
Expand Down Expand Up @@ -591,9 +638,8 @@ public IAsyncEnumerable<Entry> GetEntries(
string? text = e.CitationForm.get_String(sortWs).Text;
text ??= e.LexemeFormOA.Form.get_String(sortWs).Text;
return text?.Trim(LcmHelpers.WhitespaceChars);
})
.Skip(options.Offset)
.Take(options.Count);
});
entries = options.ApplyPaging(entries);

return entries.ToAsyncEnumerable().Select(FromLexEntry);
}
Expand Down
16 changes: 13 additions & 3 deletions backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateComplexFormTypeProxy.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,35 @@ namespace FwDataMiniLcmBridge.Api.UpdateProxy;
public record UpdateComplexFormTypeProxy : ComplexFormType
{
private readonly ILexEntryType _lexEntryType;
private readonly ILexEntry _lcmEntry;
private readonly ILexEntry? _lcmEntry;
private readonly FwDataMiniLcmApi _lexboxLcmApi;

[SetsRequiredMembers]
public UpdateComplexFormTypeProxy(ILexEntryType lexEntryType, ILexEntry lcmEntry, FwDataMiniLcmApi lexboxLcmApi)
public UpdateComplexFormTypeProxy(ILexEntryType lexEntryType, ILexEntry? lcmEntry, FwDataMiniLcmApi lexboxLcmApi)
{
_lexEntryType = lexEntryType;
_lcmEntry = lcmEntry;
_lexboxLcmApi = lexboxLcmApi;
Name = new();
Name = base.Name = new();
}

public override Guid Id
{
get => _lexEntryType.Guid;
set
{
if (_lcmEntry is null)
throw new InvalidOperationException("Cannot update complex form type Id on a null entry");
_lexboxLcmApi.RemoveComplexFormType(_lcmEntry, _lexEntryType.Guid);
_lexboxLcmApi.AddComplexFormType(_lcmEntry, value);
}
}

public override required MultiString Name
{
get => new UpdateMultiStringProxy(_lexEntryType.Name, _lexboxLcmApi);
set
{
}
}
}
2 changes: 1 addition & 1 deletion backend/FwLite/FwLiteProjectSync.Tests/Fixtures/Sena3SyncFixture.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public async Task InitializeAsync()
var crdtProjectsFolder =
rootServiceProvider.GetRequiredService<IOptions<LcmCrdtConfig>>().Value.ProjectPath;
if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true);
rootServiceProvider.Dispose();
await rootServiceProvider.DisposeAsync();

Directory.CreateDirectory(crdtProjectsFolder);
await DownloadSena3();
Expand Down
12 changes: 4 additions & 8 deletions backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,10 @@
using FwDataMiniLcmBridge;
using FwDataMiniLcmBridge.Api;
using FwDataMiniLcmBridge.LcmUtils;
using FwDataMiniLcmBridge.Tests.Fixtures;
using LcmCrdt;
using LexCore.Utils;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MiniLcm;

namespace FwLiteProjectSync.Tests.Fixtures;

Expand All @@ -23,19 +19,19 @@ public class SyncFixture : IAsyncLifetime
private readonly string _projectName;
private readonly IDisposable _cleanup;

public static SyncFixture Create([CallerMemberName] string projectName = "") => new(projectName);
public static SyncFixture Create([CallerMemberName] string projectName = "", [CallerMemberName] string projectFolder = "") => new(projectName, projectFolder);

private SyncFixture(string projectName)
private SyncFixture(string projectName, string projectFolder)
{
_projectName = projectName;
var crdtServices = new ServiceCollection()
.AddSyncServices(_projectName);
.AddSyncServices(projectFolder);
var rootServiceProvider = crdtServices.BuildServiceProvider();
_cleanup = Defer.Action(() => rootServiceProvider.Dispose());
_services = rootServiceProvider.CreateAsyncScope();
}

public SyncFixture(): this("sena-3_" + Guid.NewGuid().ToString("N"))
public SyncFixture(): this("sena-3_" + Guid.NewGuid().ToString("N"), "FwLiteSyncFixture")
{
}

Expand Down
36 changes: 25 additions & 11 deletions backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class Sena3SyncTests : IClassFixture<Sena3Fixture>, IAsyncLifetime
private CrdtMiniLcmApi _crdtApi = null!;
private FwDataMiniLcmApi _fwDataApi = null!;
private IDisposable? _cleanup;
private MiniLcmImport _miniLcmImport = null!;


public Sena3SyncTests(Sena3Fixture fixture)
Expand All @@ -29,6 +30,7 @@ public async Task InitializeAsync()
{
(_crdtApi, _fwDataApi, var services, _cleanup) = await _fixture.SetupProjects();
_syncService = services.GetRequiredService<CrdtFwdataProjectSyncService>();
_miniLcmImport = services.GetRequiredService<MiniLcmImport>();
_fwDataApi.EntryCount.Should().BeGreaterThan(100, "project should be loaded and have entries");
}

Expand Down Expand Up @@ -56,9 +58,11 @@ private void ShouldAllBeEquivalentTo(Dictionary<Guid, Entry> crdtEntries, Dictio
}

//by default the first sync is an import, this will skip that so that the sync will actually sync data
private async Task BypassImport()
private async Task BypassImport(bool wsImported = false)
{
await _syncService.SaveProjectSnapshot(_fwDataApi.Project, new ([], [], []));
var snapshot = CrdtFwdataProjectSyncService.ProjectSnapshot.Empty;
if (wsImported) snapshot = snapshot with { WritingSystems = await _fwDataApi.GetWritingSystems() };
await _syncService.SaveProjectSnapshot(_fwDataApi.Project, snapshot);
}

//this lets us query entries when there is no writing system
Expand Down Expand Up @@ -98,28 +102,37 @@ public async Task DryRunSync_MakesNoChanges()
_crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty();
}

[Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")]
public async Task DryRunSync_MakesTheSameChangesAsImport()
[Fact]
public async Task DryRunSync_MakesTheSameChangesAsSync()
{
await BypassImport();
//syncing requires querying entries, which fails if there are no writing systems, so we import those first
await _miniLcmImport.ImportWritingSystems(_crdtApi, _fwDataApi);
await BypassImport(true);

var dryRunSyncResult = await _syncService.SyncDryRun(_crdtApi, _fwDataApi);
var syncResult = await _syncService.Sync(_crdtApi, _fwDataApi);
dryRunSyncResult.Should().BeEquivalentTo(syncResult);
dryRunSyncResult.CrdtChanges.Should().Be(syncResult.CrdtChanges);
//can't test fwdata changes as they will not work correctly since the sync code expects Crdts to contain data from FWData
//this throws off the algorithm and it will try to delete everything in fwdata since there's no data in the crdt since it was a dry run
}

[Fact]
public async Task FirstSena3SyncJustDoesAnSync()
{
_fwDataApi.EntryCount.Should().BeGreaterThan(1000,
"projects with less than 1000 entries don't trip over the default query limit");

var results = await _syncService.Sync(_crdtApi, _fwDataApi);
results.FwdataChanges.Should().Be(0);
results.CrdtChanges.Should().BeGreaterThanOrEqualTo(_fwDataApi.EntryCount);

var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id);
var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id);
var crdtEntries = await _crdtApi.GetAllEntries().ToDictionaryAsync(e => e.Id);
var fwdataEntries = await _fwDataApi.GetAllEntries().ToDictionaryAsync(e => e.Id);
fwdataEntries.Count.Should().Be(_fwDataApi.EntryCount);
ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries);
}

[Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")]
[Fact]
public async Task SyncWithoutImport_CrdtShouldMatchFwdata()
{
await BypassImport();
Expand All @@ -128,8 +141,9 @@ public async Task SyncWithoutImport_CrdtShouldMatchFwdata()
results.FwdataChanges.Should().Be(0);
results.CrdtChanges.Should().BeGreaterThan(_fwDataApi.EntryCount);

var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id);
var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id);
var crdtEntries = await _crdtApi.GetAllEntries().ToDictionaryAsync(e => e.Id);
var fwdataEntries = await _fwDataApi.GetAllEntries().ToDictionaryAsync(e => e.Id);
fwdataEntries.Count.Should().Be(_fwDataApi.EntryCount);
ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries);
}

Expand Down
52 changes: 32 additions & 20 deletions backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,14 @@ await _fixture.FwDataApi.CreateEntry(new Entry()

public async Task DisposeAsync()
{
await foreach (var entry in _fixture.FwDataApi.GetEntries())
await foreach (var entry in _fixture.FwDataApi.GetAllEntries())
{
await _fixture.FwDataApi.DeleteEntry(entry.Id);
}
foreach (var entry in await _fixture.CrdtApi.GetEntries().ToArrayAsync())
foreach (var entry in await _fixture.CrdtApi.GetAllEntries().ToArrayAsync())
{
await _fixture.CrdtApi.DeleteEntry(entry.Id);
}

_fixture.DeleteSyncSnapshot();
}

public SyncTests(SyncFixture fixture)
Expand All @@ -82,8 +80,8 @@ public async Task FirstSyncJustDoesAnImport()
var fwdataApi = _fixture.FwDataApi;
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand Down Expand Up @@ -144,8 +142,8 @@ await crdtApi.CreateEntry(new Entry()
});
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand Down Expand Up @@ -180,8 +178,8 @@ await crdtApi.CreateEntry(new Entry()
});
await _syncService.SyncDryRun(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Select(e => e.Id).Should().NotContain(fwDataEntryId);
fwdataEntries.Select(e => e.Id).Should().NotContain(crdtEntryId);
}
Expand Down Expand Up @@ -222,8 +220,8 @@ public async Task CreatingAComplexEntryInFwDataSyncsWithoutIssue()
hatstand.Components = [component1, component2];
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand Down Expand Up @@ -304,8 +302,8 @@ await crdtApi.CreateEntry(new Entry()
});
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand Down Expand Up @@ -383,8 +381,8 @@ await crdtApi.CreateEntry(new Entry()
});
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand All @@ -406,8 +404,8 @@ public async Task UpdatingAnEntryInEachProjectSyncsAcrossBoth()
results.CrdtChanges.Should().Be(1);
results.FwdataChanges.Should().Be(1);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options
.For(e => e.Components).Exclude(c => c.Id)
Expand Down Expand Up @@ -474,8 +472,8 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth()

await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand All @@ -500,4 +498,18 @@ public async Task CanCreateAComplexFormAndItsComponentInOneSync()
//one of the entries will be created first, it will try to create the reference to the other but it won't exist yet
await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi);
}

[Fact]
public async Task CanCreateAComplexFormTypeAndSyncsIt()
{
//ensure they are synced so a real sync will happen when we want it to
await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi);

var complexFormEntry = await _fixture.CrdtApi.CreateComplexFormType(new() { Name = new() { { "en", "complexFormType" } } });

//one of the entries will be created first, it will try to create the reference to the other but it won't exist yet
await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi);

_fixture.FwDataApi.GetComplexFormTypes().ToBlockingEnumerable().Should().ContainEquivalentOf(complexFormEntry);
}
}
Loading

0 comments on commit effc25b

Please sign in to comment.