Improve XML sanitization in advanced mode to fix broken entities.

Previously we'd only check if a candidate entity was properly terminated
by a semicolon, which missed some broken XML we have in legacy projects.
The check is now much more thorough and mimicks much of the process the
parser itself does when parsing entities.

src/main/java/com/github/dbmdz/solrocr/lucene/filters/SanitizingXmlFilter.java

@@ -1,5 +1,6 @@
 package com.github.dbmdz.solrocr.lucene.filters;
 
+import com.ctc.wstx.io.WstxInputData;
 import com.github.dbmdz.solrocr.util.SourceAwareReader;
 import com.google.common.collect.ImmutableSet;
 import java.io.IOException;
@@ -99,8 +100,30 @@ public int read(char[] cbuf, int off, int len) throws IOException {
           break;
         }
         int entityEnd = multiIndexOf(cbuf, match + 1, '<', ';');
-        if (entityEnd < 0 || entityEnd == match + 1 || cbuf[entityEnd] == '<') {
-          // Illegal entity declaration (doesn't end before next element opening), mask
+
+        if (entityEnd < match + 1) {
+          // Not enough data to determine entity end, we may have to carry over
+          // FIXME: This code is largely identical to the carry-over case below, find a way to
+          //        deduplicate
+          int carryOverSize = numRead - (match + 1);
+          if (carryOverSize == numRead) {
+            // Can't carry-over the whole buffer, since the read would return 0
+            // We bail out of validation since there's no way we can force the caller to request
+            // a bigger read, and doing it ourselves and reading ahead would be too complicated
+            idx = (off + numRead);
+            continue outer;
+          }
+          // Reduce read size and carry over remainder, so the element is fully part of the next read
+          truncated = true;
+          this.carryOver = new char[carryOverSize];
+          System.arraycopy(cbuf, match + 1, this.carryOver, 0, this.carryOver.length);
+          this.carryOverIdx = 0;
+          numRead -= this.carryOver.length;
+          break outer;
+        }
+
+        if (!isLegalEntity(cbuf, match, entityEnd)) {
+          // Illegal entity, mask
           cbuf[match] = '_';
         }
         idx = match + 1;
@@ -290,6 +313,54 @@ private static int multiIndexOf(final char[] array, int startIndex, final char..
     return -1;
   }
 
+  private static boolean isLegalEntity(char[] cbuf, int startIdx, int endIdx) {
+    if (cbuf[startIdx + 1] == '#') {
+      // Check if we have a valid hexadecimal/decimal entity
+      if (cbuf[startIdx + 2] == 'x') {
+        // Hexadecimal entity
+        for (int i = startIdx + 3; i < endIdx; i++) {
+          char c = cbuf[i];
+          if (!Character.isDigit(c) && !(c >= 'a' && c <= 'f') && !(c >= 'A' && c <= 'F')) {
+            return false;
+          }
+        }
+        long value = Long.parseLong(new String(cbuf, startIdx + 3, endIdx - startIdx - 3), 16);
+        if (value > 0x10FFFF) {
+          // Not a legal unicode codepoint
+          return false;
+        }
+      } else {
+        // Numeric entity
+        for (int i = startIdx + 2; i < endIdx; i++) {
+          char c = cbuf[i];
+          if (!Character.isDigit(c)) {
+            return false;
+          }
+        }
+      }
+      // Legal numeric/hexadecimal entity if it's terminated correctly
+      return cbuf[endIdx] == ';';
+    } else if (!Character.isLetter(cbuf[startIdx + 1])) {
+      // Only numeric/hexadecimal entities can start with a non-letter character
+      return false;
+    }
+
+    // If the entity candidate doesn't end on a semicolon, it was terminated by a tag, i.e. not a
+    // legal entity
+    if (cbuf[endIdx] != ';') {
+      return false;
+    }
+
+    // Use the same logic Woodstox uses to validate entity names
+    for (int i = startIdx + 1; i < endIdx; i++) {
+      char c = cbuf[i];
+      if (!WstxInputData.isNameChar(c, false, false)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   @Override
   public Optional<String> getSource() {
     if (this.input instanceof SourceAwareReader) {

src/test/java/com/github/dbmdz/solrocr/solr/HocrTest.java

@@ -531,4 +531,32 @@ public void testIssue288() throws IOException {
             "ocr_text_stored");
     assertQ(req, "count(.//lst[@name=\"87371\"]//arr[@name='snippets']/lst)=19");
   }
+
+  public void testBrokenEntities() throws IOException {
+    Path ocrPath = Paths.get("src/test/resources/data/hocr_broken_entities.html");
+    assertU(
+        adoc(
+            "ocr_text_stored",
+            new String(Files.readAllBytes(ocrPath), StandardCharsets.UTF_8),
+            "id",
+            "87372"));
+    assertU(commit());
+    SolrQueryRequest req =
+        xmlQ(
+            "q",
+            "Gerichtsvollzieher",
+            "hl.snippets",
+            "4096",
+            "hl.weightMatches",
+            "true",
+            "hl.ocr.contextSize",
+            "4",
+            "df",
+            "ocr_text_stored",
+            "hl.ocr.fl",
+            "ocr_text_stored");
+    assertQ(
+        req,
+        "contains(.//lst[@name='87372']//arr[@name='snippets']/lst/str[@name='text']/text(), 'dt_HiFi-i?cBßflpedx1ttonI-iii;_;ikW')");
+  }
 }