diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index beb449c27..4cb2468b5 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -58,6 +58,7 @@ internal int GetWritingSystemHandle(WritingSystemId ws, WritingSystemType? type return lcmWs.Handle; } + internal CoreWritingSystemDefinition? GetLcmWritingSystem(WritingSystemId ws, WritingSystemType? type = null) { if (ws == "default") @@ -156,6 +157,11 @@ public async IAsyncEnumerable GetPartsOfSpeech() } } + public async Task CreatePartOfSpeech(PartOfSpeech partOfSpeech) + { + throw new NotImplementedException(); + } + public async IAsyncEnumerable GetSemanticDomains() { foreach (var semanticDomain in SemanticDomainRepository.AllInstances().OrderBy(p => p.Name.BestAnalysisAlternative.Text)) @@ -164,11 +170,16 @@ public async IAsyncEnumerable GetSemanticDomains() { Id = semanticDomain.Guid, Name = FromLcmMultiString(semanticDomain.Name), - Code = semanticDomain.OcmCodes + Code = semanticDomain.OcmCodes ?? "" }; } } + public async Task CreateSemanticDomain(SemanticDomain semanticDomain) + { + throw new NotImplementedException(); + } + internal ICmSemanticDomain GetLcmSemanticDomain(Guid semanticDomainId) { return SemanticDomainRepository.GetObject(semanticDomainId); diff --git a/backend/FwLite/LcmCrdt/CrdtLexboxApi.cs b/backend/FwLite/LcmCrdt/CrdtLexboxApi.cs index daa0db55a..c6c4895b2 100644 --- a/backend/FwLite/LcmCrdt/CrdtLexboxApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtLexboxApi.cs @@ -73,11 +73,21 @@ public IAsyncEnumerable GetPartsOfSpeech() return PartsOfSpeech.AsAsyncEnumerable(); } + public async Task CreatePartOfSpeech(PartOfSpeech partOfSpeech) + { + await dataModel.AddChange(ClientId, new CreatePartOfSpeechChange(partOfSpeech.Id, partOfSpeech.Name, false)); + } + public IAsyncEnumerable GetSemanticDomains() { return SemanticDomains.AsAsyncEnumerable(); } + public async Task CreateSemanticDomain(MiniLcm.SemanticDomain semanticDomain) + { + await dataModel.AddChange(ClientId, new CreateSemanticDomainChange(semanticDomain.Id, semanticDomain.Name, semanticDomain.Code)); + } + public IAsyncEnumerable GetEntries(QueryOptions? options = null) { return GetEntriesAsyncEnum(predicate: null, options); @@ -188,6 +198,35 @@ await dataModel.AddChanges(ClientId, ], deferCommit: true); } + public async Task BulkCreateEntries(IAsyncEnumerable entries) + { + var semanticDomains = await SemanticDomains.ToDictionaryAsync(sd => sd.Id, sd => sd); + var partsOfSpeech = await PartsOfSpeech.ToDictionaryAsync(p => p.Id, p => p); + await dataModel.AddChanges(ClientId, entries.ToBlockingEnumerable().SelectMany(entry => CreateEntryChanges(entry, semanticDomains, partsOfSpeech))); + } + + private IEnumerable CreateEntryChanges(MiniLcm.Entry entry, Dictionary semanticDomains, Dictionary partsOfSpeech) + { + yield return new CreateEntryChange(entry); + foreach (var sense in entry.Senses) + { + sense.SemanticDomains = sense.SemanticDomains + .Select(sd => semanticDomains.TryGetValue(sd.Id, out var selectedSd) ? selectedSd : null) + .OfType() + .ToList(); + if (sense.PartOfSpeechId is not null && partsOfSpeech.TryGetValue(sense.PartOfSpeechId.Value, out var partOfSpeech)) + { + sense.PartOfSpeechId = partOfSpeech.Id; + sense.PartOfSpeech = partOfSpeech.Name["en"] ?? string.Empty; + } + yield return new CreateSenseChange(sense, entry.Id); + foreach (var exampleSentence in sense.ExampleSentences) + { + yield return new CreateExampleSentenceChange(exampleSentence, sense.Id); + } + } + } + public async Task CreateEntry(MiniLcm.Entry entry) { await dataModel.AddChanges(ClientId, @@ -226,7 +265,6 @@ private async IAsyncEnumerable CreateSenseChanges(Guid entryId, MiniLcm sense.PartOfSpeech = partOfSpeech?.Name["en"] ?? string.Empty; } - yield return new CreateSenseChange(sense, entryId); foreach (var change in sense.ExampleSentences.Select(sentence => new CreateExampleSentenceChange(sentence, sense.Id))) diff --git a/backend/FwLite/LocalWebApp/Hubs/CrdtMiniLcmApiHub.cs b/backend/FwLite/LocalWebApp/Hubs/CrdtMiniLcmApiHub.cs index 759067d7c..386569189 100644 --- a/backend/FwLite/LocalWebApp/Hubs/CrdtMiniLcmApiHub.cs +++ b/backend/FwLite/LocalWebApp/Hubs/CrdtMiniLcmApiHub.cs @@ -37,6 +37,16 @@ public async Task UpdateWritingSystem(WritingSystemId id, Writing return writingSystem; } + public IAsyncEnumerable GetPartsOfSpeech() + { + return lexboxApi.GetPartsOfSpeech(); + } + + public IAsyncEnumerable GetSemanticDomains() + { + return lexboxApi.GetSemanticDomains(); + } + public IAsyncEnumerable GetEntriesForExemplar(string exemplar, QueryOptions? options = null) { throw new NotImplementedException(); diff --git a/backend/FwLite/LocalWebApp/LocalWebApp.csproj b/backend/FwLite/LocalWebApp/LocalWebApp.csproj index 13ad4392f..b60f0832b 100644 --- a/backend/FwLite/LocalWebApp/LocalWebApp.csproj +++ b/backend/FwLite/LocalWebApp/LocalWebApp.csproj @@ -16,6 +16,7 @@ + diff --git a/backend/FwLite/LocalWebApp/Services/ImportFwdataService.cs b/backend/FwLite/LocalWebApp/Services/ImportFwdataService.cs index 5b9cb3b0a..0826b3d84 100644 --- a/backend/FwLite/LocalWebApp/Services/ImportFwdataService.cs +++ b/backend/FwLite/LocalWebApp/Services/ImportFwdataService.cs @@ -1,6 +1,6 @@ -using FwDataMiniLcmBridge; -using FwDataMiniLcmBridge.Api; -using FwDataMiniLcmBridge.LcmUtils; +using System.Diagnostics; +using FwDataMiniLcmBridge; +using Humanizer; using LcmCrdt; using MiniLcm; @@ -10,6 +10,7 @@ public class ImportFwdataService(ProjectsService projectsService, ILogger Import(string projectName) { + var startTime = Stopwatch.GetTimestamp(); var fwDataProject = FieldWorksProjectList.GetProject(projectName); if (fwDataProject is null) { @@ -22,11 +23,12 @@ public async Task Import(string projectName) var crdtApi = provider.GetRequiredService(); await ImportProject(crdtApi, fwDataApi, fwDataApi.EntryCount); }); - logger.LogInformation("Import of {ProjectName} complete!", fwDataApi.Project.Name); + var timeSpent = Stopwatch.GetElapsedTime(startTime); + logger.LogInformation("Import of {ProjectName} complete, took {TimeSpend}", fwDataApi.Project.Name, timeSpent.Humanize()); return project; } - async Task ImportProject(ILexboxApi importTo, ILexboxApi importFrom, int entryCount) + private async Task ImportProject(ILexboxApi importTo, ILexboxApi importFrom, int entryCount) { var writingSystems = await importFrom.GetWritingSystems(); foreach (var ws in writingSystems.Analysis) @@ -41,19 +43,31 @@ async Task ImportProject(ILexboxApi importTo, ILexboxApi importFrom, int entryCo logger.LogInformation("Imported ws {WsId}", ws.Id); } - var index = 0; - await foreach (var entry in importFrom.GetEntries(new QueryOptions(Count: 100_000, Offset: 0))) + await foreach (var semanticDomain in importFrom.GetSemanticDomains()) { - if (importTo is CrdtLexboxApi crdtLexboxApi) - { - await crdtLexboxApi.CreateEntryLite(entry); - } - else + await importTo.CreateSemanticDomain(semanticDomain); + logger.LogTrace("Imported semantic domain {Id}", semanticDomain.Id); + } + await foreach (var partOfSpeech in importFrom.GetPartsOfSpeech()) + { + await importTo.CreatePartOfSpeech(partOfSpeech); + logger.LogInformation("Imported part of speech {Id}", partOfSpeech.Id); + } + + var entries = importFrom.GetEntries(new QueryOptions(Count: 100_000, Offset: 0)); + if (importTo is CrdtLexboxApi crdtLexboxApi) + { + await crdtLexboxApi.BulkCreateEntries(entries); + } + else + { + var index = 0; + await foreach (var entry in entries) { await importTo.CreateEntry(entry); + logger.LogTrace("Imported entry, {Index} of {Count} {Id}", index++, entryCount, entry.Id); } - - logger.LogInformation("Imported entry, {Index} of {Count} {Id}", index++, entryCount, entry.Id); } + logger.LogInformation("Imported {Count} entries", entryCount); } } diff --git a/backend/FwLite/MiniLcm/ILexboxApi.cs b/backend/FwLite/MiniLcm/ILexboxApi.cs index 971840434..e85d5531f 100644 --- a/backend/FwLite/MiniLcm/ILexboxApi.cs +++ b/backend/FwLite/MiniLcm/ILexboxApi.cs @@ -16,10 +16,15 @@ IAsyncEnumerable GetPartsOfSpeech() { throw new NotImplementedException(); } + Task CreatePartOfSpeech(PartOfSpeech partOfSpeech); IAsyncEnumerable GetSemanticDomains() { throw new NotImplementedException(); } + + Task CreateSemanticDomain(SemanticDomain semanticDomain); + + IAsyncEnumerable GetEntries(QueryOptions? options = null); IAsyncEnumerable SearchEntries(string query, QueryOptions? options = null); Task GetEntry(Guid id); diff --git a/backend/FwLite/MiniLcm/InMemoryApi.cs b/backend/FwLite/MiniLcm/InMemoryApi.cs index 07ffec4ae..9bacdfcd5 100644 --- a/backend/FwLite/MiniLcm/InMemoryApi.cs +++ b/backend/FwLite/MiniLcm/InMemoryApi.cs @@ -164,6 +164,15 @@ public Task CreateSense(Guid entryId, Sense sense) return Task.FromResult(sense); } + public async Task CreatePartOfSpeech(PartOfSpeech partOfSpeech) + { + throw new NotImplementedException(); + } + + public async Task CreateSemanticDomain(SemanticDomain semanticDomain) + { + throw new NotImplementedException(); + } public Task DeleteEntry(Guid id) { diff --git a/frontend/viewer/src/HomeView.svelte b/frontend/viewer/src/HomeView.svelte index 34ef4e02e..9d6545c7d 100644 --- a/frontend/viewer/src/HomeView.svelte +++ b/frontend/viewer/src/HomeView.svelte @@ -40,16 +40,16 @@ projectsPromise = fetchProjects(); } - let loading = ''; + let importing = ''; async function importFwDataProject(name: string) { - loading = name; + importing = name; await fetch(`/api/import/fwdata/${name}`, { method: 'POST', }); projectsPromise = fetchProjects(); await projectsPromise; - loading = ''; + importing = ''; } let downloading = ''; @@ -194,6 +194,7 @@ {:else if rowData.fwdata}