add in actual JSON schema validation to the automated tests

models/ivoa/vo-dml/ivoa_base.vodml-binding.xml

@@ -7,6 +7,7 @@
   <python-package>org.ivoa.dm</python-package>
   <xml-targetnamespace prefix="ivoa" >http://ivoa.net/vodml/ivoa</xml-targetnamespace>
   <xmllegacy-targetnamespace prefix="ivoa" schemaFilename="ivoa-legacy.xsd" schemaLocation="https://ivoa.net/xml/IVOA-v1.xsd">http://ivoa.net/dm/models/vo-dml/xsd/ivoa</xmllegacy-targetnamespace>
+  <json lax="true"/>
   <type-mapping>
   <vodml-id>real</vodml-id>
   <java-type jpa-atomic="true">Double</java-type>

models/sample/test/lifecycleTest.vo-dml.xml

@@ -9,10 +9,9 @@
       <title></title>
       <author>pharriso</author>
       <version>0.1</version>
-      <lastModified>2024-11-21T13:44:44Z</lastModified>
+      <lastModified>2024-12-19T13:11:08Z</lastModified>
       <import>
-        <name>ivoa</name>
-        <version>1.0</version>
+        <name>null</name><!--should not be needed in modern vo-dml -->
         <url>IVOA-v1.0.vo-dml.xml</url>
         <documentationURL>not known</documentationURL>
       </import>
@@ -66,18 +65,6 @@
                    <maxOccurs>1</maxOccurs>
                  </multiplicity>
                </attribute>
-               <reference>
-                 <vodml-id>Contained.refbad2</vodml-id>
-                 <name>refbad2</name>
-                 <description></description>	    
-                 <datatype>
-                   <vodml-ref>lifecycleTest:ReferredLifeCycle</vodml-ref>
-                 </datatype>
-                 <multiplicity>
-                   <minOccurs>1</minOccurs>
-                   <maxOccurs>1</maxOccurs>
-                 </multiplicity>
-               </reference>
             </objectType>
             <objectType>
                <vodml-id>ATest</vodml-id>

models/sample/test/lifecycleTest.vodsl

@@ -16,7 +16,12 @@
 
  otype Contained "" {
  	test2: ivoa:string "";
- 	refbad2 references ReferredLifeCycle "";/* this is bad because one of the places that Contained could be contained - i.e Atest3 is not in the ReferredLifeCycle containment hierarchy */
+/* below is *really* bad (i.e. breaks the generated code json serialization intention as the
+   it them moves where the referred to things are apparently contained - as the JSON referencing mechanism
+   currently in place will output the real object first and then the reference - so it need to come across the "contained"
+   object first - TODO try to express this restriction in schematron.
+*/
+// 	refbad2 references ReferredLifeCycle "";/* this is bad because one of the places that Contained could be contained - i.e Atest3 is not in the ReferredLifeCycle containment hierarchy */
  }
 
 otype ATest { //this does things as per current rules

runtime/java/build.gradle.kts

@@ -16,7 +16,7 @@ dependencies {
 //    implementation("org.glassfish.jaxb:jaxb-runtime:2.3.6")
     implementation("jakarta.persistence:jakarta.persistence-api:3.1.0")
     implementation("com.fasterxml.jackson.core:jackson-databind:2.17.0")
-    implementation("com.networknt:json-schema-validator:1.5.3")
+    implementation("com.networknt:json-schema-validator:1.5.4")
     implementation("org.hibernate.orm:hibernate-core:6.5.3.Final")
 
     implementation("org.slf4j:slf4j-api:1.7.36")

runtime/java/src/main/java/org/ivoa/vodml/ModelDescription.java

@@ -38,6 +38,12 @@ public interface ModelDescription  {
      */
     String xmlNamespace();
 
+    /**
+     * The json schema.
+     * @return the jsonSchema at the head of the model
+     */
+    String jsonSchema();
+
     /**
      * Get the classes that make up content.
      * @return the list of classes.

runtime/java/src/main/java/org/ivoa/vodml/validation/AbstractBaseValidation.java

@@ -9,6 +9,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import com.networknt.schema.output.OutputUnit;
 import jakarta.persistence.EntityManager;
 import jakarta.xml.bind.JAXBContext;
 import jakarta.xml.bind.JAXBElement;
@@ -61,8 +62,13 @@ protected  <T> RoundTripResult<T> roundTripJSON(VodmlModel<T> m) throws JsonProc
         String json = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(model);
         System.out.println("JSON output");
         System.out.println(json);
+        JSONValidator jsonValidator = new JSONValidator(m.management());
+        OutputUnit vresult = jsonValidator.validate(json);
+        if (!vresult.isValid()) {
+            System.err.println(vresult.toString());
+        }
         T retval = mapper.readValue(json, clazz);
-        return new RoundTripResult<T>(true, retval);
+        return new RoundTripResult<T>(vresult.isValid(), retval);
 
     }
 

runtime/java/src/main/java/org/ivoa/vodml/validation/JSONValidator.java

@@ -0,0 +1,65 @@
+package org.ivoa.vodml.validation;
+
+
+/*
+ * Created on 17/12/2024 by Paul Harrison ([email protected]).
+ */
+
+import com.networknt.schema.*;
+import com.networknt.schema.output.OutputUnit;
+import org.ivoa.vodml.ModelManagement;
+
+import java.util.Set;
+
+/**
+ * Validate JSON against the generated schema.
+ */
+public class JSONValidator {
+
+   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
+         .getLogger(JSONValidator.class);
+   private final ModelManagement<?> modelManagement;
+   private final SchemaValidatorsConfig config;
+   private JsonSchemaFactory jsonSchemaFactory;
+
+   /**
+    *  create a validator for a particular model.
+    * @param modelManagement the model against which the validator should be created.
+    */
+   public JSONValidator(ModelManagement<?> modelManagement) {
+
+      this.modelManagement = modelManagement;
+      // This creates a schema factory that will use Draft 2020-12 as the default if $schema is not specified
+// in the schema data. If $schema is specified in the schema data then that schema dialect will be used
+// instead and this version is ignored.
+
+      jsonSchemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012, builder ->
+            // This creates a mapping from $id which starts with https://www.example.org/ to the retrieval URI classpath:
+            builder.schemaMappers(schemaMappers -> schemaMappers.mapPrefix("https://ivoa.net/dm/", "classpath:"))
+      );
+
+      SchemaValidatorsConfig.Builder builder = SchemaValidatorsConfig.builder();
+// By default the JDK regular expression implementation which is not ECMA 262 compliant is used
+// Note that setting this requires including optional dependencies
+// builder.regularExpressionFactory(GraalJSRegularExpressionFactory.getInstance());
+// builder.regularExpressionFactory(JoniRegularExpressionFactory.getInstance());
+      config = builder.build();
+
+   }
+
+   /**
+    * validate some JSON against the model schema.
+    * @param json the input JSON as a string.
+    * @return the validation result.
+    */
+   public OutputUnit validate(String json) {
+      JsonSchema schema = jsonSchemaFactory.getSchema(SchemaLocation.of(modelManagement.description().jsonSchema()), config);
+
+      OutputUnit ou = schema.validate(json, InputFormat.JSON, OutputFormat.HIERARCHICAL, executionContext -> {
+         // By default since Draft 2019-09 the format keyword only generates annotations and not assertions
+         executionContext.getExecutionConfig().setFormatAssertionsEnabled(true);
+      });
+      return ou;
+   }
+}
+

tools/gradletooling/TODO.md

@@ -120,6 +120,7 @@ VODML Tooling TODO
   * allow refs to be serialized/deserialized as ids always.... - for use in APIs.... https://stackoverflow.com/questions/51172496/how-to-dynamically-ignore-a-property-on-jackson-serialization
   * perhaps have custom written ivoa base schema.... express some better rules... e.g. non neg integer...
   * modern usage https://blogs.oracle.com/javamagazine/post/java-json-serialization-jackson
+  * It might be best to generate schema even for imported models as it is difficult to decide what to do for `"additionalProperties": false` without context of how a particular type is being derived from.
 
 
 # Python production

tools/gradletooling/sample/src/test/java/org/ivoa/dm/lifecycle/LifeCycleDetailedTest.java

@@ -47,7 +47,7 @@ void setUp() throws Exception {
     List<ReferredLifeCycle> refcont =
         Arrays.asList(new ReferredLifeCycle("rc1"), new ReferredLifeCycle("rc2"));
     List<Contained> contained =
-        Arrays.asList(new Contained("firstcontained", refcont.get(0)), new Contained("secondContained", refcont.get(1)));
+        Arrays.asList(new Contained("firstcontained"), new Contained("secondContained"));
 
     atest =
         ATest.createATest(

tools/gradletooling/sample/src/test/java/org/ivoa/dm/lifecycle/LifecycleTestModelTest.java

@@ -59,7 +59,7 @@ public LifecycleTestModel createModel() {
      List<ReferredLifeCycle> refcont =
         Arrays.asList(new ReferredLifeCycle("rc1"), new ReferredLifeCycle("rc2"));
     List<Contained> contained =
-        Arrays.asList(new Contained("firstcontained", refcont.get(0)), new Contained("secondContained", refcont.get(1)));
+        Arrays.asList(new Contained("firstcontained"), new Contained("secondContained"));
 
     atest =
         ATest.createATest(

tools/xslt/jaxb.xsl

@@ -563,6 +563,11 @@
 
         }
 
+        @Override
+        public String jsonSchema() {
+        return "<xsl:value-of select="vf:jsonBaseURI(current()/name)"/>";
+        }
+
         /**
         * Return a list of content classes for this model.
         * @return the list.

tools/xslt/vo-dml2jsonschema.xsl

@@ -35,9 +35,11 @@ that allow for successful JSON round tripping.
   <!-- Input parameters -->
   <xsl:param name="lastModifiedText"/>
   <xsl:param name="binding"/>
-  <xsl:param name="strict" select="false()"/> <!-- TODO not really sure if strict working in the sense wanted -i.e. picking up on "undefined" properties, but that might be a function of the java verifier -->
   <xsl:include href="binding_setup.xsl"/>
 
+  <xsl:variable name="strict" as="xsd:boolean">
+      <xsl:sequence select="not($mapping/bnd:mappedModels/model[name=current()/vo-dml:model/name]/json/@lax='true')"/>
+  </xsl:variable>
  <!-- main pattern : processes for root node model -->
   <xsl:template match="/">
     <xsl:message >Generating JSON <xsl:value-of select="document-uri(.) "/> - considering models <xsl:value-of select="string-join($models/vo-dml:model/name,' and ')" /></xsl:message>
@@ -61,7 +63,7 @@ that allow for successful JSON round tripping.
                 <xsl:apply-templates select="current()" mode="refs"/>
                 <xsl:apply-templates select="current()" mode="content"/>
              }
-             <xsl:call-template name="makeStrict"/>
+             ,"additionalProperties": false
             }
 
          }
@@ -86,16 +88,19 @@ that allow for successful JSON round tripping.
                "type" : "object"
                ,"properties" : {
             "$comment" : "placeholder to make commas easier!"
-            <xsl:for-each select="$references-vodmlref">
+            <xsl:for-each select="$references-vodmlref"><!-- IMPL mostly expecting the actual reference object mostly in the refs array - but could be a ref to an object if it has occurred as contained reference in a preceding reference object -->
                 ,"<xsl:value-of select="current()"/>" : {
                    "type": "array"
                    ,"items" : {
-                         <xsl:value-of select="vf:jsonType(current())"/>
+                      "oneOf" : [
+                         {<xsl:value-of select="vf:jsonType(current())"/>},
+                         {<xsl:value-of select="vf:jsonReferenceType(current())"/>}
+                       ]
                    }
                 }
             </xsl:for-each>
             }
-            <xsl:call-template name="makeStrict"/>
+            ,"additionalProperties": false
             }
         </xsl:if>
     </xsl:template>
@@ -111,7 +116,7 @@ that allow for successful JSON round tripping.
                 </xsl:for-each>
             ]
         }
-           <xsl:call-template name="makeStrict"/>
+        ,"additionalProperties": false
         }
     </xsl:template>
 
@@ -139,7 +144,15 @@ that allow for successful JSON round tripping.
 
   <xsl:template name="makeStrict">
     <xsl:if test="$strict">
-      ,"additionalProperties": false
+      <xsl:choose>
+          <xsl:when test="not(current()/extends) and not(vf:hasSubTypes(vf:asvodmlref(current())))">
+              ,"additionalProperties" : false
+          </xsl:when>
+          <xsl:when test="not(vf:hasSubTypes(vf:asvodmlref(current())))">
+              ,"unevaluatedProperties" : false
+          </xsl:when>
+      </xsl:choose>
+
     </xsl:if>
   </xsl:template>
 
@@ -155,9 +168,9 @@ that allow for successful JSON round tripping.
     ,<xsl:apply-templates select="description"/>
     ,"properties" : {
     "$comment" : "placeholder to make commas easier!"
-    <xsl:if test="not(extends)"> <!-- impl perhaps vf:hasSubTypes(vf:asvodmlref(current())) what we really want and then do special things for the "content" types -->
+      <!--  as properties optional by default - just define this  <xsl:if test="not(extends)"> &lt;!&ndash; impl perhaps vf:hasSubTypes(vf:asvodmlref(current())) what we really want and then do special things for the "content" types &ndash;&gt;-->
     ,"@type" : { "type": "string"}
-    </xsl:if>
+<!--    </xsl:if>-->
     <xsl:apply-templates select="attribute"/>
     <xsl:apply-templates select="composition"/>
     <xsl:apply-templates select="reference"/>
@@ -186,6 +199,7 @@ that allow for successful JSON round tripping.
         ,<xsl:apply-templates select="description"/>
         ,"properties" : {
         "$comment" : "placeholder to make commas easier!"
+        ,"@type" : { "type": "string"}
         <xsl:apply-templates select="attribute"/>
         <xsl:apply-templates select="reference"/>
         }
@@ -239,12 +253,22 @@ that allow for successful JSON round tripping.
 
 
 
-  <xsl:template match="attribute" >
+  <xsl:template match="attribute[multiplicity/maxOccurs=1]" >
       , "<xsl:value-of select="name"/>" : {
        <xsl:value-of select="vf:jsonType(datatype/vodml-ref)"/>
        ,<xsl:apply-templates select="description"/>
     }
   </xsl:template>
+    <!-- allow attributes with multiplicity > 1 -->
+    <xsl:template match="attribute[multiplicity/maxOccurs!=1]" >
+        , "<xsl:value-of select="name"/>" : {
+        "type":"array"
+        ,"items": {
+        <xsl:value-of select="vf:jsonType(datatype/vodml-ref)"/>
+        }
+        ,<xsl:apply-templates select="description"/>
+        }
+    </xsl:template>
 
   <xsl:template match="multiplicity">
     <!--  only legal values: 0..1   1   0..*   1..* -->
@@ -265,6 +289,21 @@ that allow for successful JSON round tripping.
     </xsl:if>
   </xsl:template>
 
+    <xsl:template match="composition[multiplicity/maxOccurs=1]" >
+        ,"<xsl:value-of select="name"/>" : {
+        "type":"object"
+        <xsl:choose>
+            <xsl:when test="vf:hasSubTypes(datatype/vodml-ref)">
+                ,"anyOf": [
+                <xsl:value-of select="string-join(for $v in vf:subTypeIds(datatype/vodml-ref) return concat('{',vf:jsonType($v),'}') ,',')"/>
+                ]
+            </xsl:when>
+            <xsl:otherwise>
+                ,<xsl:value-of select="vf:jsonType(datatype/vodml-ref)"/>
+            </xsl:otherwise>
+        </xsl:choose>
+        }
+    </xsl:template>
 
   <xsl:template match="composition" >
       ,"<xsl:value-of select="name"/>" : {
@@ -284,7 +323,20 @@ that allow for successful JSON round tripping.
       }
   </xsl:template>
 
-  <xsl:template match="reference" > <!-- IMPL normally this will be just an integer reference to an existing instance - apart from first occurance of contained reference -->
+    <!-- allow for "plain aggregation" of references-->
+    <xsl:template match="reference[multiplicity/maxOccurs!=1]" >
+        , "<xsl:value-of select="name"/>" : {
+        "type":"array"
+        ,"items": {
+        <xsl:value-of select="vf:jsonReferenceType(datatype/vodml-ref)"/>
+        }
+        ,<xsl:apply-templates select="description"/>
+        }
+    </xsl:template>
+
+
+    <xsl:template match="reference" > <!-- IMPL normally this will be just an integer reference to an existing instance - apart from first occurrence of contained reference
+  IMPL - ideally this would just be a reference - however if there is a reference within a reference then the first instance might be the object-->
       , "<xsl:value-of select="name"/>" : {
       "oneOf" : [
       {<xsl:value-of select="vf:jsonReferenceType(datatype/vodml-ref)"/>},