Validation upgrade

- (valid) validation tests are now separated from example. Rather than
  running the examples directly, we have dedicated config files per
  test. This allows for declaring individual root classes per test.
  One test configuration can reference multiple applicable examples

- Valid examples are now also test-converted to JSON, and verified for
  no-modification via `git diff`. When developing it is sufficient to
  stage changes of intentional differences.

We are not yet ready to also convert examples to TTL (things fail in
fundamental ways). I think we should explore that. I fear missing pieces
in our general concept.

.github/workflows/model-checks.yml

@@ -24,4 +24,4 @@ jobs:
 
       - name: Run checks
         run: |
-          make check
+          make check-models

.github/workflows/validate-examples.yaml

@@ -1,4 +1,4 @@
-name: Validate examples
+name: Validate examples and verify unmodified conversion
 
 on:
   push:
@@ -24,4 +24,4 @@ jobs:
 
       - name: Validate
         run: |
-          make validate-examples
+          make check-validation

Makefile

@@ -46,11 +46,11 @@ build/mkdocs-site: build/linkml-docs src/extra-docs/*.md
 	mkdocs build
 
 # add additional schemas to lint here
-check: \
-	check-data-access-schema \
-	check-git-provenance-schema \
-	check-ontology
-check-%: src/linkml/%.yaml
+check-models: \
+	check-model-data-access-schema \
+	check-model-git-provenance-schema \
+	check-model-ontology
+check-model-%: src/linkml/%.yaml
 	@echo [Check $<]
 	@echo "Run linter"
 	@linkml-lint \
@@ -70,31 +70,52 @@ check-%: src/linkml/%.yaml
 	@echo Generate OWL
 	@${FAILIF_STDERR} gen-owl $< > /dev/null
 
-# add additional schemas to validate examples for here.
-# each one needs to provide valid examples at
-# src/examples/<schema-name>/* and invalid examples at
-# src/counter-examples/<schema-name>/*.
-#
-# In particular the valid examples should follow the
-# naming schema <class>-<example-name>.yaml to be
-# usable as documentation examples for `gen-doc`
-validate-examples: \
-	validate-examples-data-access-schema \
-	validate-examples-git-provenance-schema
-validate-examples-%:
-	$(MAKE) validate-valid-examples-$* validate-invalid-examples-$*
-validate-valid-examples-%: src/linkml/%.yaml src/examples/%
-	linkml-validate -s $^/*
-validate-invalid-examples-%: src/linkml/%.yaml src/counter-examples/%
-	# loop over all counter-examples, skip the schema file itself
-	echo "Verify EXPECTED validation failures"
-	@for ex in $^/*; do \
+check-validation: \
+	check-validation-data-access-schema \
+	convert-examples-data-access-schema \
+	check-validation-git-provenance-schema \
+	convert-examples-git-provenance-schema
+check-validation-%:
+	$(MAKE) check-valid-validation-$* check-invalid-validation-$*
+check-valid-validation-%: tests/%/validation src/linkml/%.yaml
+	@for ex in $</*.valid.cfg.yaml; do \
+		echo "Validate $$ex" ; \
+		linkml-validate --config "$$ex" ; \
+	done
+check-invalid-validation-%: tests/%/validation src/linkml/%.yaml
+	@for ex in $</*.invalid.cfg.yaml; do \
+		echo "(In)validate $$ex" ; \
+		linkml-validate --config "$$ex" && UNEXPECTEDLY VALID || true; \
+	done
+
+convert-examples: \
+	convert-examples-data-access-schema \
+	convert-examples-git-provenance-schema
+convert-examples-%: src/linkml/%.yaml src/examples/%
+	# loop over all examples, skip the schema file itself
+	for ex in $^/*.yaml; do \
 		[ "$$ex" = "$<" ] && continue; \
-		linkml-validate -s $< $$ex && UNEXPECTEDLY VALID || true; \
+		echo "Converting $$ex" ; \
+		for outf in json ; do \
+			linkml-convert \
+				-s "$<" \
+				--target-class-from-path \
+				-t "$$outf" \
+				"$$ex" \
+				> $${ex%.yaml}.$${outf}.tmp && \
+			mv $${ex%.yaml}.$${outf}.tmp $${ex%.yaml}.$${outf} ; \
+		done \
 	done
+	@git --no-pager diff -- $^
+	@if [ -n "$$(git diff -- $^)" ]; then \
+		echo -n 'ERROR: Unexpected modification of example output. ' ; \
+   		echo 'Inspect and commit changes shown above!' ; \
+		exit 22 ; \
+	fi
+
 
 clean:
 	rm -rf build
 	rm -f *-stamp
 
-.PHONY: clean check validate-examples
+.PHONY: clean check-models check-examples convert-examples

src/examples/data-access-schema/ObjectAvailability-annexwebdav.json

@@ -0,0 +1,11 @@
+{
+  "access_method": {
+    "metatype": "dlco:GitAnnexSpecialRemote",
+    "special_remote_type": "webdav",
+    "has_parameter": [
+      "some"
+    ]
+  },
+  "object_id": "MD5E-s4--ba1f2511fc30423bdbb183fe33f3dd0f.txt",
+  "@type": "ObjectAvailability"
+}

src/examples/data-access-schema/ObjectAvailability-annexwebdav.yaml

@@ -0,0 +1,6 @@
+access_method:
+  metatype: dlco:GitAnnexSpecialRemote
+  special_remote_type: webdav
+  has_parameter:
+    - some
+object_id: MD5E-s4--ba1f2511fc30423bdbb183fe33f3dd0f.txt

src/examples/git-provenance-schema/Commit-simple.json

@@ -0,0 +1,9 @@
+{
+  "id": "7726424f50c6e9a70ba31e8d44c5d86fc46170da",
+  "was_generated_by": {
+    "was_associated_with": {
+      "name": "Michael Hanke"
+    }
+  },
+  "@type": "Commit"
+}

src/examples/git-provenance-schema/Commit-simple.yaml

@@ -0,0 +1,4 @@
+id: 7726424f50c6e9a70ba31e8d44c5d86fc46170da
+was_generated_by:
+  was_associated_with:
+    name: Michael Hanke

tests/data-access-schema/validation/annexremote.valid.cfg.yaml

@@ -0,0 +1,8 @@
+schema: src/linkml/data-access-schema.yaml
+target_class: ObjectAvailability
+data_sources:
+  - src/examples/data-access-schema/ObjectAvailability-annexwebdav.yaml
+plugins:
+  JsonschemaValidationPlugin:
+    closed: true
+  RecommendedSlotsPlugin:

tests/data-access-schema/validation/annexremote_missing_type.invalid.cfg.yaml

@@ -0,0 +1,8 @@
+schema: src/linkml/data-access-schema.yaml
+target_class: DataObject
+data_sources:
+  - tests/data-access-schema/validation/GitAnnexSpecialRemote-missing_type.yaml
+plugins:
+  JsonschemaValidationPlugin:
+    closed: true
+  RecommendedSlotsPlugin:

tests/git-provenance-schema/validation/basic.valid.cfg.yaml

@@ -0,0 +1,9 @@
+schema: src/linkml/git-provenance-schema.yaml
+target_class: Commit
+data_sources:
+  - src/examples/git-provenance-schema/Commit-simple.yaml
+plugins:
+  JsonschemaValidationPlugin:
+    closed: true
+    include_range_class_descendants: true
+  RecommendedSlotsPlugin: