diff --git a/Dockerfile b/Dockerfile index e9ffaf5c..5154165e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,7 @@ ARG ONTPUB_FAMILY=fibo ARG HYGIENE_TEST_PARAMETER_VALUE=edmcouncil ARG ONTPUB_IS_DARK_MODE=1 ARG RDFTOOLKIT_VERSION +ARG CONSISTENCY_CHECK_TIMEOUT ENV \ BASH_ENV=/etc/profile \ diff --git a/README.md b/README.md index 25e65d67..39c513ab 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ List of allowed `` (and defaults for ``): - `HYGIENE_TEST_PARAMETER_VALUE` :- filter pattern, e.g. `example` (default: `edmcouncil`); this variable is also possible to set during build time (see `--rebuildimage` option) - `HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME` :- the name of the file (inside the directory named `${ONTPUB FAMILY}`for which the "warning" level consistency check test will be performed (i.e. in the case of a lack of consistency, the ontology building process is not terminated), e.g. `AboutFIBODev.rdf` (no default - in the absence of a value, the tests will not be run) - `HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME` :- the name of the file (inside the directory named `${ONTPUB FAMILY}` for which the "error" level consistency check test will be performed (i.e. in the case of a lack of consistency, the ontology building process is terminated with an error message), e.g. `AboutFIBOProd.rdf` (no default - in the absence of a value, the tests will not be run) +- `CONSISTENCY_CHECK_TIMEOUT` :- maximum duration of a single consistency check process - at the "warning" or "error" level. After this time (default: 1 hour), the process will be killed (`kill -s KILL ...`), an error message will be displayed and the `hygiene` stage will continue - `ONTPUB_MERGED_INFIX` :- infix for "merged" files, e.g. `-Merged` (no default - in the absence of a value, the merged files will not be created) - `ONTPUB_SHACL_INFIX` :- infix for SHACL shapes, e.g. `_SHACL` (no default - in the absence of a value, the merged files will not be created) The current process of creating SHACL shapes requires merged files, so if you send in `ONTPUB_SHACL_INFIX`, you also need to set up `ONTPUB_MERGED_INFIX`. diff --git a/docker-run.sh b/docker-run.sh index 566ab21c..b00b7e42 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -476,6 +476,10 @@ function run() { opts+=('--env') opts+=("DATADICTIONARY_COLUMNS=${DATADICTIONARY_COLUMNS}") fi + if [ -n "${CONSISTENCY_CHECK_TIMEOUT}" ] ; then + opts+=('--env') + opts+=("CONSISTENCY_CHECK_TIMEOUT=${CONSISTENCY_CHECK_TIMEOUT}") + fi logVar ONTPUB_FAMILY diff --git a/publisher/product/ontology/build.sh b/publisher/product/ontology/build.sh index a5f74ab8..21a17924 100644 --- a/publisher/product/ontology/build.sh +++ b/publisher/product/ontology/build.sh @@ -101,60 +101,6 @@ function runHygieneTests() { --value "${HYGIENE_TEST_PARAMETER_VALUE}" \ --output_folder "${TMPDIR}/hygiene/" - # - # Run consistency-check for DEV and PROD ontologies - # - - rm -f "${hygiene_product_tag_root}/consistency-check.log" &>/dev/null - - test -n "${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME}" && logRule "run consistency check at level: warning" && \ - for SPEC in ${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME} ; do - if [ -s "${source_family_root}/${SPEC}" ] && [ ! -d "${source_family_root}/${SPEC}" ] ; then - rm -f ${TMPDIR}/output.json - logItem "${SPEC}" "$(getOntologyIRI < "${source_family_root}/${SPEC}")" - if ${ONTOVIEWER_TOOLKIT_JAVA} --data "${source_family_root}/${SPEC}" \ - --output ${TMPDIR}/output.json $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping ${source_family_root}/catalog-v001.xml") \ - --goal consistency-check &> "${hygiene_product_tag_root}/consistency-check.log" && jq -e "." &>/dev/null < "${TMPDIR}/output.json" ; then - displayMissingImports "${TMPDIR}/output.json" - if [ "$(jq -r ".consistent" < "${TMPDIR}/output.json")" = "true" ] ; then - echo -e "\t\x1b\x5b\x33\x32\x6d$(echo "Ontology \"${SPEC}\" is consistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d" - else - echo -e "\t\x1b\x5b\x33\x31\x6d$(echo "Ontology \"${SPEC}\" is inconsistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d" - return 1 - fi - else - echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: running consistency-check - see 'consistency-check.log'" - return 1 - fi - fi - done - - test -n "${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME}" && logRule "run consistency check at level: error" && \ - for SPEC in ${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME} ; do - if [ -s "${source_family_root}/${SPEC}" ] && [ ! -d "${source_family_root}/${SPEC}" ] ; then - rm -f ${TMPDIR}/output.json - logItem "${SPEC}" "$(getOntologyIRI < "${source_family_root}/${SPEC}")" - if ${ONTOVIEWER_TOOLKIT_JAVA} --data "${source_family_root}/${SPEC}" \ - --output ${TMPDIR}/output.json $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping ${source_family_root}/catalog-v001.xml") \ - --goal consistency-check &>> "${hygiene_product_tag_root}/consistency-check.log" && jq -e "." &>/dev/null < "${TMPDIR}/output.json" ; then - displayMissingImports "${TMPDIR}/output.json" - if [ "$(jq -r ".consistent" < "${TMPDIR}/output.json")" = "true" ] ; then - echo -e "\t\x1b\x5b\x33\x32\x6d$(echo "Ontology \"${SPEC}\" is consistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d" - else - echo -e "\t\x1b\x5b\x33\x31\x6d$(echo "Ontology \"${SPEC}\" is inconsistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d" - return 1 - fi - else - echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: running consistency-check - see 'consistency-check.log'" - return 1 - fi - fi - done - - rm -f ${TMPDIR}/output.json &>/dev/null - - test -n "${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME}${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME}" && logRule "consistency-check: end" - # # Get ontologies for Dev # @@ -180,7 +126,7 @@ function runHygieneTests() { success=$? if [ "${success}" != 0 ] ; then log "Merging prod ontologies encountered problem(s), so hygiene test results may be incomplete." ; fi - + logRule "Will run the following tests:" while read -r hygieneTestSparqlFile ; do @@ -245,11 +191,62 @@ function runHygieneTests() { allerrorscount=$((${DEVerrorscount} + ${PRODerrorscount})) test ${allerrorscount} -gt 0 && logItem "$(echo -e '\n\x1b\x5b\x33\x32\x6dall errors count\x1b\x5b\x30\x6d ')" ${allerrorscount} && return 1 - cp -avf "${hygiene_product_tag_root}"/hygiene-test.DEV.log "${hygiene_product_tag_root}"/hygiene-test.DEV.tsv - cp -avf "${hygiene_product_tag_root}"/hygiene-test.PROD.log "${hygiene_product_tag_root}"/hygiene-test.PROD.tsv + cp -af "${hygiene_product_tag_root}"/hygiene-test.DEV.log "${hygiene_product_tag_root}"/hygiene-test.DEV.tsv + cp -af "${hygiene_product_tag_root}"/hygiene-test.PROD.log "${hygiene_product_tag_root}"/hygiene-test.PROD.tsv logRule "Passed all the hygiene tests" + # + # Run consistency-check for DEV and PROD ontologies + # + + rm -f "${hygiene_product_tag_root}/consistency-check.log" &>/dev/null + + test -n "${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME}" && \ + HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME="${HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME:+${HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME} }${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME}" + declare -A HYGIENE_INCONSISTENCY_SPEC_FILE_NAME + test -n "${HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME}" && \ + HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["info"]="${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["info"]:+${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["info"]} }${HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME}" + test -n "${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME}" && \ + HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["warning"]="${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["warning"]:+${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["warning"]} }${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME}" + test -n "${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME}" && \ + HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["error"]="${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["error"]:+${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["error"]} }${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME}" + + for level in ${!HYGIENE_INCONSISTENCY_SPEC_FILE_NAME[*]} ; do + test -n "${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME[${level}]}" && logRule "run consistency check at level: ${level}" && \ + for SPEC in ${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME[${level}]} ; do + if [ -s "${source_family_root}/${SPEC}" ] && [ ! -d "${source_family_root}/${SPEC}" ] ; then + rm -f ${TMPDIR}/output.json + logItem "${SPEC}" "$(getOntologyIRI < "${source_family_root}/${SPEC}")" + timeout --foreground -s KILL ${CONSISTENCY_CHECK_TIMEOUT:-1h} ${ONTOVIEWER_TOOLKIT_JAVA} --data "${source_family_root}/${SPEC}" \ + --output ${TMPDIR}/output.json $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping ${source_family_root}/catalog-v001.xml") \ + --goal consistency-check &>> "${hygiene_product_tag_root}/consistency-check.log" + local ret=$? + if [ ${ret} -eq $((128+9)) ] ; then + echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: consistency check at level \"${level}\" was timed out after \"${CONSISTENCY_CHECK_TIMEOUT:-1h}\"" + elif [ ${ret} -ne 0 ] ; then + echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: running consistency-check - see 'consistency-check.log'" + return 1 + elif jq -e "." &>/dev/null < "${TMPDIR}/output.json" ; then + displayMissingImports "${TMPDIR}/output.json" + if [ "$(jq -r ".consistent" < "${TMPDIR}/output.json")" = "true" ] ; then + echo -e "\t\x1b\x5b\x33\x32\x6d$(echo "Ontology \"${SPEC}\" is consistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d" + else + echo -e "\t\x1b\x5b\x33\x31\x6d$(echo "Ontology \"${SPEC}\" is inconsistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d" + [[ "${level}" == "error" ]] && return 1 + fi + else + echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: consistency check at level \"${level}\" did not produce the correct result" + fi + echo "--------------" >> "${hygiene_product_tag_root}/consistency-check.log" + fi + done && echo "==============" >> "${hygiene_product_tag_root}/consistency-check.log" + done + + rm -f "${TMPDIR}"/output.json &>/dev/null + + test -e "${hygiene_product_tag_root}/consistency-check.log" && logRule "consistency-check: end" + return 0 }