Merge pull request #160 from edmcouncil/155-time-limits-for-consisten…

…cy-check Add time limits for consistency check
edmcouncil · Apr 19, 2024 · 6d15002 · 6d15002
2 parents 84710de + e731bd7
commit 6d15002
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 57 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -34,6 +34,7 @@ ARG ONTPUB_FAMILY=fibo
 ARG HYGIENE_TEST_PARAMETER_VALUE=edmcouncil
 ARG ONTPUB_IS_DARK_MODE=1
 ARG RDFTOOLKIT_VERSION
+ARG CONSISTENCY_CHECK_TIMEOUT
 
 ENV \
   BASH_ENV=/etc/profile \

diff --git a/README.md b/README.md
@@ -96,6 +96,7 @@ List of allowed `<VARIABLE_NAME>` (and defaults for `<VARIABLE_VALUE>`):
 - `HYGIENE_TEST_PARAMETER_VALUE` :- filter pattern, e.g. `example` (default: `edmcouncil`); this variable is also possible to set during build time (see `--rebuildimage` option)
 - `HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME` :- the name of the file (inside the directory named `${ONTPUB FAMILY}`for which the "warning" level consistency check test will be performed (i.e. in the case of a lack of consistency, the ontology building process is not terminated), e.g. `AboutFIBODev.rdf` (no default - in the absence of a value, the tests will not be run)
 - `HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME` :- the name of the file (inside the directory named `${ONTPUB FAMILY}` for which the "error" level consistency check test will be performed (i.e. in the case of a lack of consistency, the ontology building process is terminated with an error message), e.g. `AboutFIBOProd.rdf` (no default - in the absence of a value, the tests will not be run)
+- `CONSISTENCY_CHECK_TIMEOUT` :- maximum duration of a single consistency check process - at the "warning" or "error" level. After this time (default: 1 hour), the process will be killed (`kill -s KILL ...`), an error message will be displayed and the `hygiene` stage will continue
 - `ONTPUB_MERGED_INFIX` :- infix for "merged" files, e.g. `-Merged` (no default - in the absence of a value, the merged files will not be created)
 - `ONTPUB_SHACL_INFIX` :- infix for SHACL shapes, e.g. `_SHACL` (no default - in the absence of a value, the merged files will not be created)
   The current process of creating SHACL shapes requires merged files, so if you send in `ONTPUB_SHACL_INFIX`, you also need to set up `ONTPUB_MERGED_INFIX`.

diff --git a/docker-run.sh b/docker-run.sh
@@ -476,6 +476,10 @@ function run() {
     opts+=('--env')
     opts+=("DATADICTIONARY_COLUMNS=${DATADICTIONARY_COLUMNS}")
   fi
+  if [ -n "${CONSISTENCY_CHECK_TIMEOUT}" ] ; then
+    opts+=('--env')
+    opts+=("CONSISTENCY_CHECK_TIMEOUT=${CONSISTENCY_CHECK_TIMEOUT}")
+  fi
 
   logVar ONTPUB_FAMILY
 

diff --git a/publisher/product/ontology/build.sh b/publisher/product/ontology/build.sh
@@ -101,60 +101,6 @@ function runHygieneTests() {
   --value "${HYGIENE_TEST_PARAMETER_VALUE}" \
   --output_folder "${TMPDIR}/hygiene/"
 
-  #
-  # Run consistency-check for DEV and PROD ontologies
-  #
-
-  rm -f "${hygiene_product_tag_root}/consistency-check.log" &>/dev/null
-
-  test -n "${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME}" && logRule "run consistency check at level: warning" && \
-  for SPEC in ${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME} ; do
-   if [ -s "${source_family_root}/${SPEC}" ] && [ ! -d "${source_family_root}/${SPEC}" ] ; then
-    rm -f ${TMPDIR}/output.json
-    logItem "${SPEC}" "$(getOntologyIRI < "${source_family_root}/${SPEC}")"
-    if ${ONTOVIEWER_TOOLKIT_JAVA} --data "${source_family_root}/${SPEC}" \
-        --output ${TMPDIR}/output.json $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping ${source_family_root}/catalog-v001.xml") \
-        --goal consistency-check &> "${hygiene_product_tag_root}/consistency-check.log" && jq -e "." &>/dev/null < "${TMPDIR}/output.json" ; then
-      displayMissingImports "${TMPDIR}/output.json"
-      if [ "$(jq -r ".consistent" < "${TMPDIR}/output.json")" = "true" ] ; then
-        echo -e "\t\x1b\x5b\x33\x32\x6d$(echo "Ontology \"${SPEC}\" is consistent."   | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d"
-      else
-        echo -e "\t\x1b\x5b\x33\x31\x6d$(echo "Ontology \"${SPEC}\" is inconsistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d"
-        return 1
-      fi
-    else
-      echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: running consistency-check - see 'consistency-check.log'"
-      return 1
-    fi
-   fi
-  done
-
-  test -n "${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME}" && logRule "run consistency check at level: error" && \
-  for SPEC in ${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME} ; do
-   if [ -s "${source_family_root}/${SPEC}" ] && [ ! -d "${source_family_root}/${SPEC}" ] ; then
-    rm -f ${TMPDIR}/output.json
-    logItem "${SPEC}" "$(getOntologyIRI < "${source_family_root}/${SPEC}")"
-    if ${ONTOVIEWER_TOOLKIT_JAVA} --data "${source_family_root}/${SPEC}" \
-        --output ${TMPDIR}/output.json $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping ${source_family_root}/catalog-v001.xml") \
-        --goal consistency-check &>> "${hygiene_product_tag_root}/consistency-check.log" && jq -e "." &>/dev/null < "${TMPDIR}/output.json" ; then
-      displayMissingImports "${TMPDIR}/output.json"
-      if [ "$(jq -r ".consistent" < "${TMPDIR}/output.json")" = "true" ] ; then
-        echo -e "\t\x1b\x5b\x33\x32\x6d$(echo "Ontology \"${SPEC}\" is consistent."   | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d"
-      else
-        echo -e "\t\x1b\x5b\x33\x31\x6d$(echo "Ontology \"${SPEC}\" is inconsistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d"
-        return 1
-      fi
-    else
-      echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: running consistency-check - see 'consistency-check.log'"
-      return 1
-    fi
-   fi
-  done
-
-  rm -f ${TMPDIR}/output.json &>/dev/null
-
-  test -n "${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME}${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME}" && logRule "consistency-check: end"
-
   #
   # Get ontologies for Dev
   #
@@ -180,7 +126,7 @@ function runHygieneTests() {
 
   success=$?
   if [ "${success}" != 0 ] ; then log "Merging prod ontologies encountered problem(s), so hygiene test results may be incomplete." ; fi
-    
+
   logRule "Will run the following tests:"
 
   while read -r hygieneTestSparqlFile ; do
@@ -245,11 +191,62 @@ function runHygieneTests() {
   allerrorscount=$((${DEVerrorscount} + ${PRODerrorscount}))
   test ${allerrorscount} -gt 0 && logItem "$(echo -e '\n\x1b\x5b\x33\x32\x6dall errors count\x1b\x5b\x30\x6d  ')" ${allerrorscount} && return 1
 
-  cp -avf "${hygiene_product_tag_root}"/hygiene-test.DEV.log "${hygiene_product_tag_root}"/hygiene-test.DEV.tsv
-  cp -avf "${hygiene_product_tag_root}"/hygiene-test.PROD.log "${hygiene_product_tag_root}"/hygiene-test.PROD.tsv
+  cp -af "${hygiene_product_tag_root}"/hygiene-test.DEV.log "${hygiene_product_tag_root}"/hygiene-test.DEV.tsv
+  cp -af "${hygiene_product_tag_root}"/hygiene-test.PROD.log "${hygiene_product_tag_root}"/hygiene-test.PROD.tsv
 
   logRule "Passed all the hygiene tests"
 
+  #
+  # Run consistency-check for DEV and PROD ontologies
+  #
+
+  rm -f "${hygiene_product_tag_root}/consistency-check.log" &>/dev/null
+
+  test -n "${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME}" && \
+   HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME="${HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME:+${HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME} }${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME}"
+  declare -A HYGIENE_INCONSISTENCY_SPEC_FILE_NAME
+  test -n "${HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME}" && \
+   HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["info"]="${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["info"]:+${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["info"]} }${HYGIENE_INFO_INCONSISTENCY_SPEC_FILE_NAME}"
+  test -n "${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME}" && \
+   HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["warning"]="${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["warning"]:+${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["warning"]} }${HYGIENE_WARN_INCONSISTENCY_SPEC_FILE_NAME}"
+  test -n "${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME}" && \
+   HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["error"]="${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["error"]:+${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME["error"]} }${HYGIENE_ERROR_INCONSISTENCY_SPEC_FILE_NAME}"
+
+  for level in ${!HYGIENE_INCONSISTENCY_SPEC_FILE_NAME[*]} ; do
+   test -n "${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME[${level}]}" && logRule "run consistency check at level: ${level}" && \
+    for SPEC in ${HYGIENE_INCONSISTENCY_SPEC_FILE_NAME[${level}]} ; do
+     if [ -s "${source_family_root}/${SPEC}" ] && [ ! -d "${source_family_root}/${SPEC}" ] ; then
+      rm -f ${TMPDIR}/output.json
+      logItem "${SPEC}" "$(getOntologyIRI < "${source_family_root}/${SPEC}")"
+      timeout --foreground -s KILL ${CONSISTENCY_CHECK_TIMEOUT:-1h} ${ONTOVIEWER_TOOLKIT_JAVA} --data "${source_family_root}/${SPEC}" \
+        --output ${TMPDIR}/output.json $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping ${source_family_root}/catalog-v001.xml") \
+        --goal consistency-check &>> "${hygiene_product_tag_root}/consistency-check.log"
+      local ret=$?
+      if [ ${ret} -eq $((128+9)) ] ; then
+       echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: consistency check at level \"${level}\" was timed out after \"${CONSISTENCY_CHECK_TIMEOUT:-1h}\""
+      elif [ ${ret} -ne 0 ] ; then
+       echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: running consistency-check - see 'consistency-check.log'"
+       return 1
+      elif jq -e "." &>/dev/null < "${TMPDIR}/output.json" ; then
+       displayMissingImports "${TMPDIR}/output.json"
+       if [ "$(jq -r ".consistent" < "${TMPDIR}/output.json")" = "true" ] ; then
+        echo -e "\t\x1b\x5b\x33\x32\x6d$(echo "Ontology \"${SPEC}\" is consistent."   | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d"
+       else
+        echo -e "\t\x1b\x5b\x33\x31\x6d$(echo "Ontology \"${SPEC}\" is inconsistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d"
+        [[ "${level}" == "error" ]] && return 1
+       fi
+      else
+       echo -e "\t\x1b\x5b\x33\x31\x6dERROR\x1b\x5b\x30\x6d: consistency check at level \"${level}\" did not produce the correct result"
+      fi
+      echo "--------------" >> "${hygiene_product_tag_root}/consistency-check.log"
+     fi
+    done && echo "==============" >> "${hygiene_product_tag_root}/consistency-check.log"
+  done
+
+  rm -f "${TMPDIR}"/output.json &>/dev/null
+
+  test -e "${hygiene_product_tag_root}/consistency-check.log" && logRule "consistency-check: end"
+
   return 0
 }