Upgrading spark for to 3.5.4 to fix the issue with SQL plan optimisat…

…ion for reverseResolve() in where.

fhirpath/src/main/java/au/csiro/pathling/fhirpath/execution/MultiFhirpathEvaluator.java

@@ -124,7 +124,7 @@ public static JoinSet fromContextPaths(@Nonnull final ResourceType subjectResour
     final Set<DataRoot> joinRoots = dataRootResolver.findDataRoots(contextPaths);
     log.debug("Join roots: {}", joinRoots);
     final JoinSet joinSet = JoinSet.mergeRoots(joinRoots).iterator().next();
-    log.debug("Join set: {}", joinSet.toTreeString());
+    log.debug("Join set:\n{}", joinSet.toTreeString());
     return joinSet;
   }
 }

fhirpath/src/test/java/au/csiro/pathling/fhirpathe/execution/FhirpathTest.java

@@ -96,6 +96,33 @@ void accessParentResourceInJoinedExpression() {
         );
   }
 
+  @Test
+  void reverseResolveSubsumesInWhere() {
+    TerminologyServiceHelpers.setupSubsumes(terminologyService);
+    final ObjectDataSource dataSource = getPatientsWithConditions();
+
+    // THIS seems to be a simpler way to reproduce the same issue with spark 3.5.0
+    // works: 
+    // "where(reverseResolve(Condition.subject).code.exists()).id"
+    // fails:
+    // "where(reverseResolve(Condition.subject).code.text.exists()).id"
+
+    final Dataset<Row> resultDataset = evalExpression(dataSource, ResourceType.PATIENT,
+        "where($this.reverseResolve(Condition.subject).code"
+            + ".subsumedBy(system-x|code-xx) contains true).gender"
+    );
+    System.out.println(resultDataset.logicalPlan());
+    System.out.println(resultDataset.queryExecution().executedPlan().toString());
+    resultDataset.show();
+    new DatasetAssert(resultDataset)
+        .hasRowsUnordered(
+            RowFactory.create("1", "female"),
+            RowFactory.create("2", null),
+            RowFactory.create("3", null)
+        );
+  }
+
+
   @Test
   void simpleReverseResolveToSingularValue() {
     final ObjectDataSource dataSource = getPatientsWithConditions();
@@ -767,4 +794,5 @@ void doubleNestedReverseResolveToTheSameResource() {
         );
   }
 
+
 }

pom.xml

@@ -72,7 +72,7 @@
     <project.majorVersion>7</project.majorVersion>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
-    <pathling.sparkVersion>3.5.0</pathling.sparkVersion>
+    <pathling.sparkVersion>3.5.4</pathling.sparkVersion>
     <pathling.scalaVersion>2.12</pathling.scalaVersion>
     <pathling.scalaFullVersion>2.12.17</pathling.scalaFullVersion>
     <pathling.deltaVersion>3.2.0</pathling.deltaVersion>