diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e81ec8c6..d90f6fb7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -20,7 +20,7 @@ jobs: with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Build and Run Tests - run: mvn clean install + run: make install unit-tests - name: current dir run: pwd - name: current dir diff --git a/.gitignore b/.gitignore index cc938d59..c2e099ce 100644 --- a/.gitignore +++ b/.gitignore @@ -325,3 +325,4 @@ lib64 pyvenv.cfg /build/ /node_modules/ +.temp diff --git a/Makefile b/Makefile index 80df4822..4b64a0c8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ # Model2owl directory MODEL2OWL_FOLDER?=. +ABSOLUTE_MODEL2OWL_FOLDER?=$(shell realpath "${MODEL2OWL_FOLDER}") # rdflib version RDF_LIB_VERSION?=6.2.0 #Saxon path @@ -33,6 +34,12 @@ TURTLE_FILELIST=$(shell ls ${ONTOLOGY_FOLDER_PATH}/*.ttl) # Widoco variables WIDOCO_RDF_INPUT_FILE_PATH?=test/reasoning-investigation/model-2020-12-16/ePO_restrictions.rdf WIDOCO_OUTPUT_FOLDER_PATH?=output/widoco +NAMESPACES_USER_XML_FILE_PATH?=${MODEL2OWL_FOLDER}/test/ePO-default-config/namespaces.xml +INTERM_FOLDER_PATH?=${ABSOLUTE_MODEL2OWL_FOLDER}/.temp +ENRICHED_NAMESPACES_XML_PATH:=${INTERM_FOLDER_PATH}/enriched-namespaces.xml +NAMESPACES_AS_RDFPIPE_ARGS=$(shell ${MODEL2OWL_FOLDER}/scripts/get_namespaces.sh ${ENRICHED_NAMESPACES_XML_PATH}) +RDF_XML_MIME_TYPE:='application/rdf+xml' +TURTLE_MIME_TYPE:='turtle' # download saxon library get-saxon: @@ -61,37 +68,69 @@ get-widoco: ###################################################################################### # Download, install saxon, xspec, rdflib and other dependencies ###################################################################################### -install: get-saxon get-rdflib get-widoco +install: get-saxon create-virtual-env get-rdflib get-widoco ############################ Main tasks ############################################## # Run unit_tests unit-tests: - @mvn install + @make test-prerequisites + @mvn install -Dsaxon.options.enrichedNamespacesPath=${ENRICHED_NAMESPACES_XML_PATH} + +# Actions required in order to setup the environment for testing purposes. +# Usage (`[]` denotes an optional argument; if omited, default value will be used): +# make test-prerequisites [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user +test-prerequisites: + @make gen-enriched-ns-file create-virtual-env: @python -m venv model2owl-venv # Generate the glossary from an input file +# Usage (`[]` denotes an optional argument; if omited, default value will be used): +# make generate-glossary [XMI_INPUT_FILE_PATH=/path/to/cm.xmi] +# [OUTPUT_GLOSSARY_PATH=/output/directory] +# [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user +# # Example when not using the default variables # make generate-glossary XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_GLOSSARY_PATH=/home/mypc/work/model2owl/glossary generate-glossary: @mkdir -p "${OUTPUT_GLOSSARY_PATH}" + @make gen-enriched-ns-file @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_GLOSSARY_PATH}" - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/html-model-glossary.xsl -o:${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \ + -xsl:${MODEL2OWL_FOLDER}/src/html-model-glossary.xsl \ + -o:${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @echo The glossary is located at the following location: @echo @ls -lh ${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html @echo +# Usage of the convention report generation recipes (generate-convention-report | generate-convention-SVRL-report). +# `[]` denotes an optional argument; if omited, default value will be used: +# make (generate-convention-report | generate-convention-SVRL-report) +# [XMI_INPUT_FILE_PATH=/path/to/cm.xmi] +# [OUTPUT_CONVENTION_REPORT_PATH=/output/directory] +# [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user generate-convention-report: @mkdir -p "${OUTPUT_CONVENTION_REPORT_PATH}" + @make gen-enriched-ns-file @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_CONVENTION_REPORT_PATH}" - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/html-conventions-report.xsl -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \ + -xsl:${MODEL2OWL_FOLDER}/src/html-conventions-report.xsl \ + -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @echo The convention report is located at the following location: @echo @ls -lh ${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html @@ -99,29 +138,79 @@ generate-convention-report: generate-convention-SVRL-report: @mkdir -p "${OUTPUT_CONVENTION_REPORT_PATH}" + @make gen-enriched-ns-file @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_CONVENTION_REPORT_PATH}" - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/svrl-conventions-report.xsl -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \ + -xsl:${MODEL2OWL_FOLDER}/src/svrl-conventions-report.xsl \ + -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @echo The convention report is located at the following location: @echo @ls -lh ${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml @echo -#Example how to run transformation commands : + + +# Usage of the transformation recipes (owl-core | owl-restrictions | shacl). +# `[]` denotes an optional argument; if omited, default value will be used: +# make (owl-core | owl-restrictions | shacl) [XMI_INPUT_FILE_PATH=/path/to/cm.xmi] +# [OUTPUT_FOLDER_PATH=/output/directory] +# [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user +# +# Example: # make owl-core XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_FOLDER_PATH=./my-folder owl-core: - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-core.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf + @make gen-enriched-ns-file + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-core.xsl \ + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" + @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf \ + OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf @echo Output owl core file: @ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf + @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf + owl-restrictions: - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-restrictions.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf + @make gen-enriched-ns-file + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-restrictions.xsl \ + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" + @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf \ + OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf @echo Output owl restrictions file: @ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf + @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf + shacl: - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/shacl-shapes.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf + @make gen-enriched-ns-file + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/shacl-shapes.xsl \ + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" + @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf \ + OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf @echo Output shacl file location: @ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf + @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf +# Generate enriched namespaces XML file which contains user namespaces (defined +# in namespaces.xml) and internal namespaces (such as core-shape) +# Usage (`[]` denotes an optional argument; if omited, default value will be used): +# make gen-enriched-ns-file [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user +gen-enriched-ns-file: + @mkdir -p ${INTERM_FOLDER_PATH} + @java -jar ${SAXON} -s:${NAMESPACES_USER_XML_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/xml/enriched-namespaces.xsl \ + -o:${ENRICHED_NAMESPACES_XML_PATH} # Combine xmi UML files # all files for combine should be in test/test-multi-xmi (or in XMI_MERGED_OUTPUT_FOLDER_PATH) @@ -137,14 +226,25 @@ merge-xmi: -#Example how to run converting commands : -# make convert-to-turtle ONTOLOGY_FOLDER_PATH=./my-folder -# ONTOLOGY_FOLDER_PATH is the the path to the folder containing .rdf files for converting to turtle or .ttl files to convert to rdf +# Example how to run converting commands (`[]` denotes an optional argument; +# if omited, default value will be used): +# make convert-to-turtle [ONTOLOGY_FOLDER_PATH=./my-folder] +# [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# ONTOLOGY_FOLDER_PATH: the path to the folder containing .rdf files for +# converting to turtle or .ttl files to convert to rdf +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user convert-rdf-to-turtle: + @make gen-enriched-ns-file @for FILE_PATH in ${RDF_FILELIST}; do \ echo Converting $${FILE_PATH} into Turtle; \ source model2owl-venv/bin/activate; \ - rdfpipe -i application/rdf+xml -o turtle $${FILE_PATH} > $${FILE_PATH%.*}.ttl; \ + make convert-between-serialization-formats \ + INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + OUTPUT_FORMAT=${TURTLE_MIME_TYPE} \ + FILE_PATH=$${FILE_PATH} \ + OUTPUT_FILE_PATH=$${FILE_PATH%.*}.ttl \ + USE_NAMESPACES=1; \ echo Input in RDF/XML format; \ echo $${FILE_PATH}; \ echo " ==> Output in Turtle format"; \ @@ -182,6 +282,36 @@ convert-rdf-to-rdf: echo " ==> Output in RDF/XML format"; \ ls -lh $${FILE_PATH%.*}.rdf; \ done + +# A generic recipe for converting RDF data from one serialization format to +# another. It can also be used to regenerate a file using the same format. +# +# Arguments: +# FILE_PATH: Input RDF file in any allowed serialization format +# OUTPUT_FILE_PATH: Path for the output file +# INPUT_FORMAT: a MIME type of the given input RDF file +# OUTPUT_FORMAT: a MIME type of any of the valid RDF serializations +# USE_NAMESPACES: optional; if non-empty then namespaces (from the +# enriched-namespaces.xml file). This can be used if the input +# (FILE_PATH) doesn't include namespaces we want to be applied +# (e.g. to have compact instead of full URIs in the output +# file). +# +# Supported MIME types: https://rdflib.readthedocs.io/en/7.0.0/plugin_serializers.html +# +# Example: +# make convert-between-serialization-formats +# INPUT_FORMAT='application/rdf+xml' +# OUTPUT_FORMAT='application/rdf+xml' +# FILE_PATH=output/ePO_core.tmp.rdf +# OUTPUT_FILE_PATH=output/ePO_core.rdf +# USE_NAMESPACES=1 +convert-between-serialization-formats: + @source model2owl-venv/bin/activate; \ + rdfpipe -i ${INPUT_FORMAT} -o ${OUTPUT_FORMAT} \ + $(if $(USE_NAMESPACES),${NAMESPACES_AS_RDFPIPE_ARGS}) \ + ${FILE_PATH} > ${OUTPUT_FILE_PATH} + # make validate-rdf-file FILE_TO_VALIDATE_PATH=./output/eFulfilment.rdf validate-rdf-file: @$(JENA_RIOT_TOOL) --validate $(FILE_TO_VALIDATE_PATH) diff --git a/pom.xml b/pom.xml index 61bb1c03..d6e8e564 100644 --- a/pom.xml +++ b/pom.xml @@ -25,6 +25,7 @@ true false + enrichedNamespacesPath=${saxon.options.enrichedNamespacesPath} diff --git a/scripts/get_namespaces.sh b/scripts/get_namespaces.sh new file mode 100755 index 00000000..f22d1d99 --- /dev/null +++ b/scripts/get_namespaces.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Gets namespaces from an XML file with the model2owl project namespaces and +# prepares argument list from them to be used with `rdfpipe` tool. Uses Saxon +# installed in the project main directory. +# +# USAGE: get_namespaces.sh NAMESPACES_XML_FILE_PATH +PROJECT_DIR=$(dirname $(dirname $(realpath ${BASH_SOURCE[0]}))) +SAXON=${PROJECT_DIR}/saxon/saxon.jar + +if [ -z "$1" ]; then + echo "ERROR: path to *.xml file with namespaces not given." + exit 1 +fi +namespaces_file_path="$1" + +namespaces_file_dir=$(dirname $(realpath $namespaces_file_path)) +namespaces_file_name=$(basename $namespaces_file_path) + +cd ${namespaces_file_dir} +namespaces=$( + java -cp $SAXON net.sf.saxon.Query -s:${namespaces_file_name} \ + -qs:'for $x in /*:prefixes/*:prefix return concat(string($x/@name), "=", string($x/@value))' \ + \!method=text +) +ns_args=$( \ + echo "$namespaces" | tr ' ' '\n' | awk '{printf("--ns='\''%s'\'' ", $0)}' +) + +echo "$ns_args" diff --git a/src/common/checkers.xsl b/src/common/checkers.xsl index b6ad192a..982063e7 100644 --- a/src/common/checkers.xsl +++ b/src/common/checkers.xsl @@ -512,15 +512,15 @@ - This function will check if a given list of namespaces are defined in - namespaces.xml file. If not all the namespaces were defined it will return a list with - those namespaces + This function will check if a given list of namespaces are + defined in enriched-namespaces.xml file. If not all the namespaces were + defined it will return a list with those namespaces + select="($internalNamespacePrefixes/*:prefixes/*:prefix/@name)"/> + + + Lookup a data-type in the xsd and rdf accepted data-type document (usually an external file with xsd and rdf data-types definitions) and return false or the data-type @@ -61,7 +68,7 @@ - + + + + + + diff --git a/src/owl-core.xsl b/src/owl-core.xsl index a2fb4041..f4d029be 100644 --- a/src/owl-core.xsl +++ b/src/owl-core.xsl @@ -46,9 +46,7 @@ - - - + @@ -63,7 +61,7 @@ - + diff --git a/src/owl-restrictions.xsl b/src/owl-restrictions.xsl index 22772ac9..fbcf6b2c 100644 --- a/src/owl-restrictions.xsl +++ b/src/owl-restrictions.xsl @@ -40,11 +40,7 @@ - - - - - + @@ -58,7 +54,7 @@ - + diff --git a/src/shacl-shapes.xsl b/src/shacl-shapes.xsl index 00c0349c..7892e87a 100644 --- a/src/shacl-shapes.xsl +++ b/src/shacl-shapes.xsl @@ -44,13 +44,7 @@ - - - - - - - + @@ -62,7 +56,7 @@ - + diff --git a/src/xml/enriched-namespaces.xsl b/src/xml/enriched-namespaces.xsl new file mode 100644 index 00000000..b3d9c88e --- /dev/null +++ b/src/xml/enriched-namespaces.xsl @@ -0,0 +1,36 @@ + + + + + + + + + A template for generating enriched namespaces XML file that + contains internal model2owl namespaces which are constructed + based on the model2owl configuration. + + + + + + + + + + + + + + + + + + diff --git a/test/ePO-default-config/namespaces.xml b/test/ePO-default-config/namespaces.xml index 0cb964ac..4f44ddae 100644 --- a/test/ePO-default-config/namespaces.xml +++ b/test/ePO-default-config/namespaces.xml @@ -12,13 +12,17 @@ + + - - + + + + @@ -26,12 +30,12 @@ + - diff --git a/test/unitTests/test-xml/test-enriched-namespaces.xspec b/test/unitTests/test-xml/test-enriched-namespaces.xspec new file mode 100644 index 00000000..ff422095 --- /dev/null +++ b/test/unitTests/test-xml/test-enriched-namespaces.xspec @@ -0,0 +1,23 @@ + + + + + + + + + + + + + +