Improve XML sanitization in advanced mode to fix broken entities.

src/main/java/com/github/dbmdz/solrocr/lucene/filters/SanitizingXmlFilter.java

@@ -1,5 +1,6 @@
 package com.github.dbmdz.solrocr.lucene.filters;
 
+import com.ctc.wstx.io.WstxInputData;
 import com.github.dbmdz.solrocr.util.SourceAwareReader;
 import com.google.common.collect.ImmutableSet;
 import java.io.IOException;
@@ -99,8 +100,30 @@ public int read(char[] cbuf, int off, int len) throws IOException {
           break;
         }
         int entityEnd = multiIndexOf(cbuf, match + 1, '<', ';');
-        if (entityEnd < 0 || entityEnd == match + 1 || cbuf[entityEnd] == '<') {
-          // Illegal entity declaration (doesn't end before next element opening), mask
+
+        if (entityEnd < match + 1) {
+          // Not enough data to determine entity end, we may have to carry over
+          // FIXME: This code is largely identical to the carry-over case below, find a way to
+          //        deduplicate
+          int carryOverSize = numRead - (match + 1);
+          if (carryOverSize == numRead) {
+            // Can't carry-over the whole buffer, since the read would return 0
+            // We bail out of validation since there's no way we can force the caller to request
+            // a bigger read, and doing it ourselves and reading ahead would be too complicated
+            idx = (off + numRead);
+            continue outer;
+          }
+          // Reduce read size and carry over remainder, so the element is fully part of the next read
+          truncated = true;
+          this.carryOver = new char[carryOverSize];
+          System.arraycopy(cbuf, match + 1, this.carryOver, 0, this.carryOver.length);
+          this.carryOverIdx = 0;
+          numRead -= this.carryOver.length;
+          break outer;
+        }
+
+        if (!isLegalEntity(cbuf, match, entityEnd)) {
+          // Illegal entity, mask
           cbuf[match] = '_';
         }
         idx = match + 1;
@@ -290,6 +313,54 @@ private static int multiIndexOf(final char[] array, int startIndex, final char..
     return -1;
   }
 
+  private static boolean isLegalEntity(char[] cbuf, int startIdx, int endIdx) {
+    if (cbuf[startIdx + 1] == '#') {
+      // Check if we have a valid hexadecimal/decimal entity
+      if (cbuf[startIdx + 2] == 'x') {
+        // Hexadecimal entity
+        for (int i = startIdx + 3; i < endIdx; i++) {
+          char c = cbuf[i];
+          if (!Character.isDigit(c) && !(c >= 'a' && c <= 'f') && !(c >= 'A' && c <= 'F')) {
+            return false;
+          }
+        }
+        long value = Long.parseLong(new String(cbuf, startIdx + 3, endIdx - startIdx - 3), 16);
+        if (value > 0x10FFFF) {
+          // Not a legal unicode codepoint
+          return false;
+        }
+      } else {
+        // Numeric entity
+        for (int i = startIdx + 2; i < endIdx; i++) {
+          char c = cbuf[i];
+          if (!Character.isDigit(c)) {
+            return false;
+          }
+        }
+      }
+      // Legal numeric/hexadecimal entity if it's terminated correctly
+      return cbuf[endIdx] == ';';
+    } else if (!Character.isLetter(cbuf[startIdx + 1])) {
+      // Only numeric/hexadecimal entities can start with a non-letter character
+      return false;
+    }
+
+    // If the entity candidate doesn't end on a semicolon, it was terminated by a tag, i.e. not a
+    // legal entity
+    if (cbuf[endIdx] != ';') {
+      return false;
+    }
+
+    // Use the same logic Woodstox uses to validate entity names
+    for (int i = startIdx + 1; i < endIdx; i++) {
+      char c = cbuf[i];
+      if (!WstxInputData.isNameChar(c, false, false)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   @Override
   public Optional<String> getSource() {
     if (this.input instanceof SourceAwareReader) {

src/test/java/com/github/dbmdz/solrocr/solr/HocrTest.java

@@ -531,4 +531,32 @@ public void testIssue288() throws IOException {
             "ocr_text_stored");
     assertQ(req, "count(.//lst[@name=\"87371\"]//arr[@name='snippets']/lst)=19");
   }
+
+  public void testBrokenEntities() throws IOException {
+    Path ocrPath = Paths.get("src/test/resources/data/hocr_broken_entities.html");
+    assertU(
+        adoc(
+            "ocr_text_stored",
+            new String(Files.readAllBytes(ocrPath), StandardCharsets.UTF_8),
+            "id",
+            "87372"));
+    assertU(commit());
+    SolrQueryRequest req =
+        xmlQ(
+            "q",
+            "Gerichtsvollzieher",
+            "hl.snippets",
+            "4096",
+            "hl.weightMatches",
+            "true",
+            "hl.ocr.contextSize",
+            "4",
+            "df",
+            "ocr_text_stored",
+            "hl.ocr.fl",
+            "ocr_text_stored");
+    assertQ(
+        req,
+        "contains(.//lst[@name='87372']//arr[@name='snippets']/lst/str[@name='text']/text(), 'dt_HiFi-i?cBßflpedx1ttonI-iii;_;ikW')");
+  }
 }