Add fwddata bridge Semantic domains and Parts of speech (#886)

* define parts of speech models in MiniLcm, implement tests and updates for the FwDataBridge

pull part of speech helper methods into their own utility class

rewrite semantic domain implementation to support them being objects that are referenced, instead of just being a string.

prevent json patch changes from having index references, this will avoid conflicts where the index changes due to merges.

introduce json patch rewriting to convert patches into specific changes.
Rewrite changes to Sense.PartOfSpeechId into SetPartOfSpeechChange.

allow creating parts of speech as CRDTs and setup a PoC of pre seeding them.

allow creating semantic domains and referencing them in senses, rewrite json patch to change semantic domains of senses. Add tests for creating senses with and without semantic domains, and with and without part of speeches.

* remove duplicate field, fix overloaded GetEntries function causing build error due to 2 overloads not requiring parameters.

* add WritingSystemTests.cs, and make the project loader fixture a collection fixture so it's shared by all tests in the project.

* reduce allocations in ContributeExemplars by ~50% by using spans of chars instead of strings.

* use string contains with StringComparison instead of calling ToLowerInvariant on each string.

* optimize ws lookup when populating exemplars by using a frozen dictionary, and set the MultiString capacity when converting a LcmMultiString to a MiniLcmMultiString

* Populate UI with parts-of-speech

* Search improvements

* Index dictionary by first grapheme instead of first char

* Revert "Index dictionary by first grapheme instead of first char", because first-char sounds like a better way to go (https://sil-lt.slack.com/archives/C806BLR42/p1718797173733449)

This reverts commit 614dbd5.

* Fix exemplar lookup not always working

* Add loading indicators

* Improve index character overlay size

* Option type cleanup

* Add Sandbox to reproduce open bug

* Populate semantic domain dropdown (WIP)

* Make entry list always fully visible.

* Prevent unnecessary change events

* Fix type error

* Use less peculiar icon

* Display search keyboard shortcut

* Redesign home page

* Resize keys

* Fix Part of speech changes not being persisted

* Uppercase index exemplars to prevent duplicates

* Store selected-entry, selected-index-char and search in URL

* Add transition so increased debounce is not so noticeable.

* Add save status indicator

* Layout and mobile fixes

* Cheap fix for semantic domain list being way to big

* Format selected semantic domains

* gracefully handle null order by text

* rework CrdtMultiOptionField to support editing object lists

* add parts of speech to CrdtLexboxApi.cs to fix failing tests

* add parts of speech and semantic domains to the lf classic api

* disable server garbage collection to reduce memory usage. Disable single file until sillsdev/icu-dotnet#201 is merged.

* apply some suggested feedback from review.

* normalize exemplar to NFD before using it for comparison

---------

Co-authored-by: Tim Haasdyk <[email protected]>

LexBox.sln

@@ -39,6 +39,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Crdt.Core", "backend\harmon
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FwDataMiniLcmBridge", "backend\FwDataMiniLcmBridge\FwDataMiniLcmBridge.csproj", "{279197B6-EC06-4DE0-94F8-625379C3AD83}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FwDataMiniLcmBridge.Tests", "backend\FwDataMiniLcmBridge.Tests\FwDataMiniLcmBridge.Tests.csproj", "{B0299A49-C0B2-4553-A72E-1670D4CB5138}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -101,6 +103,10 @@ Global
 		{279197B6-EC06-4DE0-94F8-625379C3AD83}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{279197B6-EC06-4DE0-94F8-625379C3AD83}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{279197B6-EC06-4DE0-94F8-625379C3AD83}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B0299A49-C0B2-4553-A72E-1670D4CB5138}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B0299A49-C0B2-4553-A72E-1670D4CB5138}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B0299A49-C0B2-4553-A72E-1670D4CB5138}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B0299A49-C0B2-4553-A72E-1670D4CB5138}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{E8BB768B-C3DC-4BE6-9B9F-82319E05AF86} = {7B6E21C4-5AF4-4505-B7D9-59A3886C5090}
@@ -111,5 +117,6 @@ Global
 		{740C8FF5-8006-4047-8C52-53873C2DD7C4} = {7B6E21C4-5AF4-4505-B7D9-59A3886C5090}
 		{8B54FFB5-0BDF-403E-83CC-A3B3861EC507} = {7B6E21C4-5AF4-4505-B7D9-59A3886C5090}
 		{279197B6-EC06-4DE0-94F8-625379C3AD83} = {7B6E21C4-5AF4-4505-B7D9-59A3886C5090}
+		{B0299A49-C0B2-4553-A72E-1670D4CB5138} = {7B6E21C4-5AF4-4505-B7D9-59A3886C5090}
 	EndGlobalSection
 EndGlobal

backend/FwDataMiniLcmBridge.Tests/Fixtures/ProjectLoaderFixture.cs

@@ -0,0 +1,34 @@
+using FwDataMiniLcmBridge.Api;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FwDataMiniLcmBridge.Tests.Fixtures;
+
+public class ProjectLoaderFixture : IDisposable
+{
+    public const string Name = "ProjectLoaderCollection";
+    private readonly FwDataFactory _fwDataFactory;
+    private readonly ServiceProvider _serviceProvider;
+
+    public ProjectLoaderFixture()
+    {
+        //todo make mock of IProjectLoader so we can load from test projects
+        var provider = new ServiceCollection().AddFwDataBridge().BuildServiceProvider();
+        _serviceProvider = provider;
+        _fwDataFactory = provider.GetRequiredService<FwDataFactory>();
+    }
+
+    public FwDataMiniLcmApi CreateApi(string projectName)
+    {
+        return _fwDataFactory.GetFwDataMiniLcmApi(projectName, false);
+    }
+
+    public void Dispose()
+    {
+        _serviceProvider.Dispose();
+    }
+}
+
+[CollectionDefinition(ProjectLoaderFixture.Name)]
+public class ProjectLoaderCollection : ICollectionFixture<ProjectLoaderFixture>
+{
+}

backend/FwDataMiniLcmBridge.Tests/FwDataMiniLcmBridge.Tests.csproj

@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+        <IsTestProject>true</IsTestProject>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="coverlet.collector" Version="6.0.0"/>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0"/>
+        <PackageReference Include="FluentAssertions" Version="6.12.0"/>
+        <PackageReference Include="xunit" Version="2.5.3"/>
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3"/>
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <Using Include="Xunit"/>
+        <Using Include="FluentAssertions"/>
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\FwDataMiniLcmBridge\FwDataMiniLcmBridge.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Folder Include="TestData\" />
+    </ItemGroup>
+
+</Project>

backend/FwDataMiniLcmBridge.Tests/PartOfSpeechTests.cs

@@ -0,0 +1,47 @@
+using FwDataMiniLcmBridge.Tests.Fixtures;
+using MiniLcm;
+
+namespace FwDataMiniLcmBridge.Tests;
+
+[Collection(ProjectLoaderFixture.Name)]
+public class PartOfSpeechTests(ProjectLoaderFixture fixture)
+{
+    [Fact]
+    public async Task GetPartsOfSpeech_ReturnsAllPartsOfSpeech()
+    {
+        var api = fixture.CreateApi("sena-3");
+        var partOfSpeeches = await api.GetPartsOfSpeech().ToArrayAsync();
+        partOfSpeeches.Should().AllSatisfy(po => po.Id.Should().NotBe(Guid.Empty));
+    }
+
+    [Fact]
+    public async Task Sense_HasPartOfSpeech()
+    {
+        var api = fixture.CreateApi("sena-3");
+        var entry = await api.GetEntries().FirstAsync(e => e.Senses.Any(s => !string.IsNullOrEmpty(s.PartOfSpeech)));
+        var sense = entry.Senses.First(s => !string.IsNullOrEmpty(s.PartOfSpeech));
+        sense.PartOfSpeech.Should().NotBeNullOrEmpty();
+        sense.PartOfSpeechId.Should().NotBeNull();
+    }
+
+    [Fact]
+    public async Task Sense_UpdatesPartOfSpeech()
+    {
+        var api = fixture.CreateApi("sena-3");
+        var entry = await api.GetEntries().FirstAsync(e => e.Senses.Any(s => !string.IsNullOrEmpty(s.PartOfSpeech)));
+        var sense = entry.Senses.First(s => !string.IsNullOrEmpty(s.PartOfSpeech));
+        var newPartOfSpeech = await api.GetPartsOfSpeech().FirstAsync(po => po.Id != sense.PartOfSpeechId);
+
+        var update = api.CreateUpdateBuilder<Sense>()
+            .Set(s => s.PartOfSpeech, newPartOfSpeech.Name["en"])//this won't actually update the part of speech, but it shouldn't cause an issue either.
+            .Set(s => s.PartOfSpeechId, newPartOfSpeech.Id)
+            .Build();
+        await api.UpdateSense(entry.Id, sense.Id, update);
+
+        entry = await api.GetEntry(entry.Id);
+        ArgumentNullException.ThrowIfNull(entry);
+        var updatedSense = entry.Senses.First(s => s.Id == sense.Id);
+        updatedSense.PartOfSpeechId.Should().Be(newPartOfSpeech.Id);
+        updatedSense.PartOfSpeech.Should().NotBe(sense.PartOfSpeech);//the part of speech here is whatever the default is for the project, not english.
+    }
+}

backend/FwDataMiniLcmBridge.Tests/SemanticDomainTests.cs

@@ -0,0 +1,75 @@
+using FwDataMiniLcmBridge.Tests.Fixtures;
+using MiniLcm;
+
+namespace FwDataMiniLcmBridge.Tests;
+
+[Collection(ProjectLoaderFixture.Name)]
+public class SemanticDomainTests(ProjectLoaderFixture fixture)
+{
+    [Fact]
+    public async Task GetSemanticDomains_ReturnsAllSemanticDomains()
+    {
+        var api = fixture.CreateApi("sena-3");
+        var semanticDomains = await api.GetSemanticDomains().ToArrayAsync();
+        semanticDomains.Should().AllSatisfy(sd =>
+        {
+            sd.Id.Should().NotBe(Guid.Empty);
+            sd.Name.Values.Should().NotBeEmpty();
+            sd.Code.Should().NotBeEmpty();
+        });
+    }
+
+    [Fact]
+    public async Task Sense_HasSemanticDomains()
+    {
+        var api = fixture.CreateApi("sena-3");
+        var entry = await api.GetEntries().FirstAsync(e => e.Senses.Any(s => s.SemanticDomains.Any()));
+        var sense = entry.Senses.First(s => s.SemanticDomains.Any());
+        sense.SemanticDomains.Should().NotBeEmpty();
+        sense.SemanticDomains.Should().AllSatisfy(sd =>
+        {
+            sd.Id.Should().NotBe(Guid.Empty);
+            sd.Name.Values.Should().NotBeEmpty();
+            sd.Code.Should().NotBeEmpty();
+        });
+    }
+
+    [Fact]
+    public async Task Sense_AddSemanticDomain()
+    {
+        var api = fixture.CreateApi("sena-3");
+        var entry = await api.GetEntries().FirstAsync(e => e.Senses.Any(s => s.SemanticDomains.Any()));
+        var sense = entry.Senses.First(s => s.SemanticDomains.Any());
+        var currentSemanticDomain = sense.SemanticDomains.First();
+        var newSemanticDomain = await api.GetSemanticDomains().FirstAsync(sd => sd.Id != currentSemanticDomain.Id);
+
+        var update = api.CreateUpdateBuilder<Sense>()
+            .Add(s => s.SemanticDomains, newSemanticDomain)
+            .Build();
+        await api.UpdateSense(entry.Id, sense.Id, update);
+
+        entry = await api.GetEntry(entry.Id);
+        ArgumentNullException.ThrowIfNull(entry);
+        var updatedSense = entry.Senses.First(s => s.Id == sense.Id);
+        updatedSense.SemanticDomains.Select(sd => sd.Id).Should().Contain(newSemanticDomain.Id);
+    }
+
+    [Fact]
+    public async Task Sense_RemoveSemanticDomain()
+    {
+        var api = fixture.CreateApi("sena-3");
+        var entry = await api.GetEntries().FirstAsync(e => e.Senses.Any(s => s.SemanticDomains.Any()));
+        var sense = entry.Senses.First(s => s.SemanticDomains.Any());
+        var domainToRemove = sense.SemanticDomains[0];
+
+        var update = api.CreateUpdateBuilder<Sense>()
+            .Remove(s => s.SemanticDomains, 0)
+            .Build();
+        await api.UpdateSense(entry.Id, sense.Id, update);
+
+        entry = await api.GetEntry(entry.Id);
+        ArgumentNullException.ThrowIfNull(entry);
+        var updatedSense = entry.Senses.First(s => s.Id == sense.Id);
+        updatedSense.SemanticDomains.Select(sd => sd.Id).Should().NotContain(domainToRemove.Id);
+    }
+}

backend/FwDataMiniLcmBridge.Tests/WritingSystemTests.cs

@@ -0,0 +1,22 @@
+using FwDataMiniLcmBridge.Tests.Fixtures;
+
+namespace FwDataMiniLcmBridge.Tests;
+
+[Collection(ProjectLoaderFixture.Name)]
+public class WritingSystemTests(ProjectLoaderFixture fixture)
+{
+    [Fact]
+    public async Task GetWritingSystems_DoesNotReturnNullOrEmpty()
+    {
+        var writingSystems = await fixture.CreateApi("sena-3").GetWritingSystems();
+        writingSystems.Vernacular.Should().NotBeNullOrEmpty();
+        writingSystems.Analysis.Should().NotBeNullOrEmpty();
+    }
+
+    [Fact]
+    public async Task GetWritingSystems_ReturnsExemplars()
+    {
+        var writingSystems = await fixture.CreateApi("sena-3").GetWritingSystems();
+        writingSystems.Vernacular.Should().Contain(ws => ws.Exemplars.Any());
+    }
+}