#460 Make the Resource Viewer editable as a proof-of-concept Scripture editor

.gitignore

@@ -40,3 +40,4 @@ launchSettings.json
 
 # Test development user appdata files
 dev-appdata/
+c-sharp/**/*.BAK

c-sharp/NetworkObjects/UsfmDataProvider.cs

@@ -1,6 +1,7 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using System.Xml;
+using System.Xml.XPath;
 using Paranext.DataProvider.JsonUtils;
 using Paranext.DataProvider.MessageHandlers;
 using Paranext.DataProvider.MessageTransports;
@@ -42,8 +43,9 @@ protected override ResponseToRequest HandleRequest(string functionName, JsonArra
             {
                 "getBookNames" => GetBookNames(),
                 "getChapter" => GetChapter(args[0]!.ToJsonString()),
-                "getChapterUsx" => GetChapterUsx(args[0]!.ToJsonString()),
-                "getBookUsx" => GetBookUsx(args[0]!.ToJsonString()),
+                "getChapterUsx" => GetUsx(args[0]!.ToJsonString()),
+                "setChapterUsx" => SetUsx(args[0]!.ToJsonString(), args[1]!.ToString()),
+                "getBookUsx" => GetUsx(args[0]!.ToJsonString()),
                 "getVerse" => GetVerse(args[0]!.ToJsonString()),
                 _ => ResponseToRequest.Failed($"Unexpected function: {functionName}")
             };
@@ -67,17 +69,17 @@ private ResponseToRequest GetChapter(string args)
             : ResponseToRequest.Failed(errorMsg);
     }
 
-    private ResponseToRequest GetChapterUsx(string args)
+    private ResponseToRequest GetUsx(string args)
     {
         return VerseRefConverter.TryCreateVerseRef(args, out var verseRef, out string errorMsg)
-            ? ResponseToRequest.Succeeded(GetUsxForChapter(verseRef))
+            ? ResponseToRequest.Succeeded(GetUsx(verseRef))
             : ResponseToRequest.Failed(errorMsg);
     }
 
-    private ResponseToRequest GetBookUsx(string args)
+    private ResponseToRequest SetUsx(string argVref, string argNewUsx)
     {
-        return VerseRefConverter.TryCreateVerseRef(args, out var verseRef, out string errorMsg)
-            ? ResponseToRequest.Succeeded(GetUsxForBook(verseRef))
+        return VerseRefConverter.TryCreateVerseRef(argVref, out var verseRef, out string errorMsg)
+            ? SetUsx(verseRef, argNewUsx)
             : ResponseToRequest.Failed(errorMsg);
     }
 
@@ -88,18 +90,39 @@ private ResponseToRequest GetVerse(string args)
             : ResponseToRequest.Failed(errorMsg);
     }
 
-    private string GetUsxForChapter(VerseRef vref)
+    private string GetUsx(VerseRef vref)
     {
         XmlDocument usx = ConvertUsfmToUsx(GetUsfm(vref.BookNum, vref.ChapterNum), vref.BookNum);
         string contents = usx.OuterXml ?? string.Empty;
         return contents;
     }
 
-    private string GetUsxForBook(VerseRef vref)
+    public ResponseToRequest SetUsx(VerseRef vref, string newUsx)
     {
-        XmlDocument usx = ConvertUsfmToUsx(GetUsfm(vref.BookNum), vref.BookNum);
-        string contents = usx.OuterXml ?? string.Empty;
-        return contents;
+        try
+        {
+            XmlDocument doc = new() { PreserveWhitespace = true };
+            doc.LoadXml(newUsx);
+            if (doc.FirstChild?.Name != "usx")
+                return ResponseToRequest.Failed("Invalid USX");
+
+            UsxFragmenter.FindFragments(
+                _scrText!.ScrStylesheet(vref.BookNum),
+                doc.CreateNavigator(),
+                XPathExpression.Compile("*[false()]"),
+                out string usfm
+            );
+
+            usfm = UsfmToken.NormalizeUsfm(_scrText, vref.BookNum, usfm);
+            _scrText.PutText(vref.BookNum, vref.ChapterNum, false, usfm, null);
+            SendDataUpdateEvent("*");
+        }
+        catch (Exception e)
+        {
+            return ResponseToRequest.Failed(e.Message);
+        }
+
+        return ResponseToRequest.Succeeded();
     }
 
     /// <summary>

c-sharp/ParatextUtils/ParatextGlobals.cs

@@ -34,6 +34,7 @@ public static void Initialize(string dataFolderPath)
                 ICUDllLocator.Initialize(false, false);
 
                 // Now tell Paratext.Data to use the specified folder
+                dataFolderPath = Path.GetFullPath(dataFolderPath); // Make sure path is rooted
                 ParatextData.Initialize(dataFolderPath, false);
                 s_initialized = true;
             }

c-sharp/Projects/ParatextProjectDataProvider.cs

@@ -20,6 +20,9 @@
         Getters.Add("getChapterUSFM", GetChapterUSFM);
         Setters.Add("setChapterUSFM", SetChapterUSFM);
         Getters.Add("getVerseUSFM", GetVerseUSFM);
+
+        Getters.Add("getChapterUSX", GetChapterUSX);
+        Setters.Add("setChapterUSX", SetChapterUSX);
     }
 
     protected override Task StartDataProvider()
@@ -34,7 +37,7 @@
 
     protected override ResponseToRequest SetExtensionData(ProjectDataScope dataScope, string data)
     {
         return _paratextPsi.SetExtensionData(dataScope, data);
     }
 
     private ResponseToRequest Get(string dataType, string dataQualifier)
@@ -58,9 +61,10 @@
                 DataType = dataType,
                 DataQualifier = dataQualifier,
             };
         return _paratextPsi.SetProjectData(scope, data);
     }
 
+    #region USFM handling methods
     private ResponseToRequest GetBookUSFM(string jsonString)
     {
         return Get(ParatextProjectStorageInterpreter.BookUSFM, jsonString);
@@ -80,4 +84,17 @@
     {
         return Set(ParatextProjectStorageInterpreter.ChapterUSFM, dataQualifier, data);
     }
+    #endregion
+
+    #region USX handling methods
+    private ResponseToRequest GetChapterUSX(string jsonString)
+    {
+        return Get(ParatextProjectStorageInterpreter.ChapterUSX, jsonString);
+    }
+
+    private ResponseToRequest SetChapterUSX(string dataQualifier, string data)
+    {
+        return Set(ParatextProjectStorageInterpreter.ChapterUSX, dataQualifier, data);
+    }
+    #endregion
 }

c-sharp/Projects/ParatextProjectStorageInterpreter.cs

@@ -1,9 +1,18 @@
+using System.Net.Http.Json;
 using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.RegularExpressions;
+using System.Xml;
+using System.Xml.Linq;
+using System.Xml.XPath;
 using Newtonsoft.Json;
 using Paranext.DataProvider.JsonUtils;
 using Paranext.DataProvider.MessageHandlers;
 using Paranext.DataProvider.MessageTransports;
 using Paratext.Data;
+using PtxUtils;
+using SIL.Scripture;
 
 namespace Paranext.DataProvider.Projects;
 
@@ -12,6 +21,7 @@ internal class ParatextProjectStorageInterpreter : ProjectStorageInterpreter
     public const string BookUSFM = "BookUSFM";
     public const string ChapterUSFM = "ChapterUSFM";
     public const string VerseUSFM = "VerseUSFM";
+    public const string ChapterUSX = "ChapterUSX";
 
     public ParatextProjectStorageInterpreter(PapiClient papiClient)
         : base(ProjectStorageType.ParatextFolders, new[] { ProjectType.Paratext }, papiClient) { }
@@ -88,21 +98,25 @@ public override ResponseToRequest GetProjectData(ProjectDataScope scope)
         {
             BookUSFM
                 => string.IsNullOrEmpty(error)
-                    ? ResponseToRequest.Succeeded(scrText.GetText(verseRef, false, false))
+                    ? ResponseToRequest.Succeeded(scrText.GetText(verseRef, false, true))
                     : ResponseToRequest.Failed(error),
             ChapterUSFM
                 => string.IsNullOrEmpty(error)
-                    ? ResponseToRequest.Succeeded(scrText.GetText(verseRef, true, false))
+                    ? ResponseToRequest.Succeeded(scrText.GetText(verseRef, true, true))
                     : ResponseToRequest.Failed(error),
             VerseUSFM
                 => string.IsNullOrEmpty(error)
                     ? ResponseToRequest.Succeeded(scrText.Parser.GetVerseUsfmText(verseRef))
                     : ResponseToRequest.Failed(error),
+            ChapterUSX
+                => string.IsNullOrEmpty(error)
+                    ? ResponseToRequest.Succeeded(GetChapterUsx(scrText, verseRef))
+                    : ResponseToRequest.Failed(error),
             _ => ResponseToRequest.Failed($"Unknown data type: {scope.DataType}")
         };
     }
 
-    public override ResponseToRequest SetProjectData(ProjectDataScope scope, string data)
+    public override ResponseToRequest SetProjectData(ProjectDataScope scope, JsonNode data)
     {
         if (scope.ProjectID == null)
             return ResponseToRequest.Failed("Must provide a project ID");
@@ -128,13 +142,23 @@ public override ResponseToRequest SetProjectData(ProjectDataScope scope, string
                             verseRef.BookNum,
                             verseRef.ChapterNum,
                             false,
-                            data,
+                            data.ToString(),
                             writeLock
                         );
                     }
                 );
                 // The value of returned string is case sensitive and cannot change unless data provider subscriptions change
                 return ResponseToRequest.Succeeded(ChapterUSFM);
+            case ChapterUSX:
+                if (!string.IsNullOrEmpty(error))
+                    return ResponseToRequest.Failed(error);
+                ResponseToRequest? response = null;
+                RunWithinLock(
+                    WriteScope.ProjectText(scrText, verseRef.BookNum, verseRef.ChapterNum),
+                    writeLock =>
+                        response = SetChapterUsx(scrText, verseRef, data.ToString(), writeLock)
+                );
+                return response ?? ResponseToRequest.Failed("Unknown error occurred");
             default:
                 return ResponseToRequest.Failed($"Unknown data type: {scope.DataType}");
         }
@@ -157,7 +181,7 @@ public override ResponseToRequest GetExtensionData(ProjectDataScope scope)
         return ResponseToRequest.Succeeded(textReader.ReadToEnd());
     }
 
-    public override ResponseToRequest SetExtensionData(ProjectDataScope scope, string data)
+    public override ResponseToRequest SetExtensionData(ProjectDataScope scope, JsonNode data)
     {
         if (scope.ProjectID == null)
             return ResponseToRequest.Failed("Must provide a project ID");
@@ -181,7 +205,7 @@ public override ResponseToRequest SetExtensionData(ProjectDataScope scope, strin
                         throw new Exception("Write lock is not active");
                     dataStream.SetLength(0);
                     using TextWriter textWriter = new StreamWriter(dataStream, Encoding.UTF8);
-                    textWriter.Write(data);
+                    textWriter.Write(data.ToString());
                     textWriter.Flush();
                 }
             );
@@ -204,6 +228,66 @@ public override ResponseToRequest SetExtensionData(ProjectDataScope scope, strin
         );
     }
 
+    private string GetChapterUsx(ScrText scrText, VerseRef vref)
+    {
+        XmlDocument usx = ConvertUsfmToUsx(
+            scrText,
+            scrText.GetText(vref, true, true),
+            vref.BookNum
+        );
+        return usx.OuterXml ?? string.Empty;
+    }
+
+    private ResponseToRequest SetChapterUsx(
+        ScrText scrText,
+        VerseRef vref,
+        string newUsx,
+        WriteLock writeLock
+    )
+    {
+        try
+        {
+            XDocument doc;
+            using (TextReader reader = new StringReader(newUsx))
+                doc = XDocument.Load(reader, LoadOptions.PreserveWhitespace);
+
+            if (doc.Root?.Name != "usx")
+                return ResponseToRequest.Failed("Invalid USX");
+
+            UsxFragmenter.FindFragments(
+                scrText.ScrStylesheet(vref.BookNum),
+                doc.CreateNavigator(),
+                XPathExpression.Compile("*[false()]"),
+                out string usfm
+            );
+
+            usfm = UsfmToken.NormalizeUsfm(scrText, vref.BookNum, usfm);
+            scrText.PutText(vref.BookNum, vref.ChapterNum, true, usfm, writeLock);
+        }
+        catch (Exception e)
+        {
+            return ResponseToRequest.Failed(e.ToString());
+        }
+
+        return ResponseToRequest.Succeeded(ChapterUSX);
+    }
+
+    private XmlDocument ConvertUsfmToUsx(ScrText scrText, string usfm, int bookNum)
+    {
+        ScrStylesheet scrStylesheet = scrText.ScrStylesheet(bookNum);
+        // Tokenize usfm
+        List<UsfmToken> tokens = UsfmToken.Tokenize(scrStylesheet, usfm ?? string.Empty, true);
+
+        XmlDocument doc = new XmlDocument();
+        using (XmlWriter xmlw = doc.CreateNavigator()!.AppendChild())
+        {
+            // Convert to XML
+            UsfmToUsx.ConvertToXmlWriter(scrStylesheet, tokens, xmlw, false);
+            xmlw.Flush();
+        }
+        return doc;
+    }
+
     private static void RunWithinLock(WriteScope writeScope, Action<WriteLock> action)
     {
         var myLock =

c-sharp/Projects/ProjectStorageInterpreter.cs

@@ -55,13 +55,13 @@ out string errorMessage
             case "getProjectData":
                 return GetProjectData(dataScope);
             case "setProjectData":
-                var setProjectReturn = SetProjectData(dataScope, args[1]!.ToJsonString());
+                var setProjectReturn = SetProjectData(dataScope, args[1]!);
                 SendDataUpdateEvent(setProjectReturn.Contents);
                 return setProjectReturn;
             case "getExtensionData":
                 return GetExtensionData(dataScope);
             case "setExtensionData":
-                var setExtensionReturn = SetExtensionData(dataScope, args[1]!.ToJsonString());
+                var setExtensionReturn = SetExtensionData(dataScope, args[1]!);
                 SendDataUpdateEvent(setExtensionReturn.Contents);
                 return setExtensionReturn;
             default:
@@ -106,7 +106,7 @@ private ResponseToRequest GetSupportedTypes()
     /// <summary>
     /// Set data in a project identified by <param name="scope"></param>.
     /// </summary>
-    public abstract ResponseToRequest SetProjectData(ProjectDataScope scope, string jsonData);
+    public abstract ResponseToRequest SetProjectData(ProjectDataScope scope, JsonNode data);
 
     /// <summary>
     /// Get an extension's data in a project identified by <param name="scope"></param>.
@@ -116,5 +116,5 @@ private ResponseToRequest GetSupportedTypes()
     /// <summary>
     /// Set an extension's data in a project identified by <param name="scope"></param>.
     /// </summary>
-    public abstract ResponseToRequest SetExtensionData(ProjectDataScope scope, string jsonData);
+    public abstract ResponseToRequest SetExtensionData(ProjectDataScope scope, JsonNode data);
 }

extensions/src/resource-viewer/resource-viewer.web-view.tsx

@@ -123,6 +123,7 @@ const usxEditorCharMap = Object.fromEntries(
 
 interface ScriptureTextPanelUsxProps {
   usx: string;
+  onChanged?: (newUsx: string) => void;
 }
 
 const defaultScrRef: ScriptureReference = {
@@ -135,16 +136,17 @@ const defaultScrRef: ScriptureReference = {
  * Scripture text panel that displays a read only version of a usx editor that displays the current
  * chapter
  */
-function ScriptureTextPanelUsxEditor({ usx }: ScriptureTextPanelUsxProps) {
+function ScriptureTextPanelUsxEditor({ usx, onChanged }: ScriptureTextPanelUsxProps) {
   return (
     // eslint-disable-next-line jsx-a11y/no-static-element-interactions
     <div className="text-panel">
       <UsxEditor
         usx={usx}
         paraMap={usxEditorParaMap}
         charMap={usxEditorCharMap}
-        onUsxChanged={() => {
-          /* Read only */
+        onUsxChanged={(newUsx) => {
+          // TODO: Check if the project is editable
+          if (onChanged) onChanged(newUsx);
         }}
       />
     </div>
@@ -155,11 +157,19 @@ globalThis.webViewComponent = function ResourceViewer(): JSX.Element {
   logger.info('Preparing to display the Resource Viewer');
 
   const [scrRef] = useSetting('platform.verseRef', defaultScrRef);
-  const [usx, , isLoading] = useData.ChapterUsx<UsfmProviderDataTypes, 'ChapterUsx'>(
+  const [usx, setUsx, isLoading] = useData.ChapterUsx<UsfmProviderDataTypes, 'ChapterUsx'>(
     'usfm',
     useMemo(() => new VerseRef(scrRef.bookNum, scrRef.chapterNum, scrRef.verseNum), [scrRef]),
     'Loading Scripture...',
   );
 
-  return <div>{isLoading ? 'Loading' : <ScriptureTextPanelUsxEditor usx={usx ?? '<usx/>'} />}</div>;
+  return (
+    <div>
+      {isLoading ? (
+        'Loading'
+      ) : (
+        <ScriptureTextPanelUsxEditor usx={usx ?? '<usx/>'} onChanged={setUsx} />
+      )}
+    </div>
+  );
 };