Merge pull request #154 from edmcouncil/revert_to_old_ontology_creation

Revert to 21e562f
edmcouncil · Feb 8, 2024 · c459b99 · c459b99
2 parents b10cd02 + 84d162a
commit c459b99
Show file tree

Hide file tree

Showing 3 changed files with 130 additions and 44 deletions.
diff --git a/publisher/product/datadictionary/build.sh b/publisher/product/datadictionary/build.sh
@@ -58,38 +58,35 @@ function publishProductDataDictionaryContent() {
 
   local extractDataColumn="${DATADICTIONARY_COLUMNS:+ --extract-data-column $(echo "${DATADICTIONARY_COLUMNS}" | sed 's/|/ --extract-data-column /g')}"
 
-  local dev_suffix=${DEV_SPEC/.rdf/}
-  local prod_suffix=${PROD_SPEC/.rdf/}
-
   logRule "Running OntoViewer Toolkit to generate CSV files containing data from ontologies - see logs \"$(logFileName datadictionary.DEV.log)\""
-  logItem "$(basename "${DEV_SPEC}")" "$(logFileName "${tag_root_url}/$(basename "${datadictionaryBaseName}-${dev_suffix}.csv")")"
+  logItem "$(basename "${DEV_SPEC}")" "$(logFileName "${tag_root_url}/$(basename "${datadictionaryBaseName}-dev.csv")")"
   debug=false ${ONTOVIEWER_TOOLKIT_JAVA} --goal extract-data ${maturityLevel} ${maturityLevelProperty} ${extractDataColumn} --data "${source_family_root}/${DEV_SPEC}" \
-    --output "${datadictionaryBaseName}-${dev_suffix}.csv" \
+    --output "${datadictionaryBaseName}-dev.csv" \
     --filter-pattern "${HYGIENE_TEST_PARAMETER_VALUE}" \
     --ontology-mapping "${source_family_root}/catalog-v001.xml" > "${datadictionary_product_tag_root}/datadictionary.DEV.log" 2>&1
 
   logRule "Running OntoViewer Toolkit to generate CSV files containing data from ontologies - see logs \"$(logFileName datadictionary.PROD.log)\""
-  logItem "$(basename "${PROD_SPEC}")" "$(logFileName "${tag_root_url}/$(basename "${datadictionaryBaseName}-${prod_suffix}.csv")")"
+  logItem "$(basename "${PROD_SPEC}")" "$(logFileName "${tag_root_url}/$(basename "${datadictionaryBaseName}-prod.csv")")"
   debug=false ${ONTOVIEWER_TOOLKIT_JAVA} --goal extract-data ${maturityLevel} ${maturityLevelProperty} ${extractDataColumn} --data "${source_family_root}/${PROD_SPEC}" \
-    --output "${datadictionaryBaseName}-${prod_suffix}.csv" \
+    --output "${datadictionaryBaseName}-prod.csv" \
     --filter-pattern "${HYGIENE_TEST_PARAMETER_VALUE}" \
     --ontology-mapping "${source_family_root}/catalog-v001.xml" > "${datadictionary_product_tag_root}/datadictionary.PROD.log" 2>&1
 
   logRule "Writing from csv files to xlsx files for prod"
 
-  echo -e "==== $(basename "${datadictionaryBaseName}-${prod_suffix}.xlsx")" > "${datadictionary_product_tag_root}/datadictionary.log"
+  echo -e "==== $(basename "${datadictionaryBaseName}-prod.xlsx")" > "${datadictionary_product_tag_root}/datadictionary.log"
 
   ${PYTHON3} ${SCRIPT_DIR}/lib/csv-to-xlsx.py \
-    "${datadictionaryBaseName}-${prod_suffix}.csv" \
-    "${datadictionaryBaseName}-${prod_suffix}.xlsx" \
+    "${datadictionaryBaseName}-prod.csv" \
+    "${datadictionaryBaseName}-prod.xlsx" \
     "${datadictionary_script_dir}/csvconfig" 2>&1 | tee -a "${datadictionary_product_tag_root}/datadictionary.log"
 
   logRule "Writing from csv files to xlsx files for dev"
 
-  echo -e "\n==== $(basename "${datadictionaryBaseName}-${dev_suffix}.xlsx")" >> "${datadictionary_product_tag_root}/datadictionary.log"
+  echo -e "\n==== $(basename "${datadictionaryBaseName}-dev.xlsx")" >> "${datadictionary_product_tag_root}/datadictionary.log"
   ${PYTHON3} ${SCRIPT_DIR}/lib/csv-to-xlsx.py \
-    "${datadictionaryBaseName}-${dev_suffix}.csv" \
-    "${datadictionaryBaseName}-${dev_suffix}.xlsx" \
+    "${datadictionaryBaseName}-dev.csv" \
+    "${datadictionaryBaseName}-dev.xlsx" \
     "${datadictionary_script_dir}/csvconfig" 2>&1 | tee -a "${datadictionary_product_tag_root}/datadictionary.log"
 
   return 0

diff --git a/publisher/product/ontology/build-about.sh b/publisher/product/ontology/build-about.sh
@@ -22,21 +22,18 @@ function ontologyCreateAboutFiles () {
 
   rm -f "${TMPDIR}/err.tmp"
 
-  local dev_suffix=${DEV_SPEC/.rdf/}
-  local prod_suffix=${PROD_SPEC/.rdf/}
-
   # use "owl:imports" from ${PROD_SPEC} if exists
   test -n "${PROD_SPEC}" && test -r "${tag_root:?}/${PROD_SPEC}" && (
     cd "${tag_root:?}" || return $?
 
-    logItem "<owl:imports> from \"${PROD_SPEC}\"$(echo -en '\t')" "$(logFileName "${tag_root}/Load${prod_suffix}.rdf")"
+    logItem "<owl:imports> from \"${PROD_SPEC}\"$(echo -en '\t')" "$(logFileName "${tag_root}/Load${ONTPUB_FAMILY^^}Prod.rdf")"
 
     cat > "${tmpAboutFileProd}" << __HERE__
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
 @prefix owl: <http://www.w3.org/2002/07/owl#> 
 @prefix xsd: <http://www.w3.org/2001/XMLSchema#>
-<${product_root_url}/Load${prod_suffix}/> a owl:Ontology;
+<${product_root_url}/Load${ONTPUB_FAMILY^^}Prod/> a owl:Ontology;
 __HERE__
 
     getOwlImports < "${tag_root}/${PROD_SPEC}" 2>/dev/null | \
@@ -46,10 +43,10 @@ __HERE__
     "${JENA_ARQ}" \
       --data="${tmpAboutFileProd}" \
       --query="${SCRIPT_DIR}/lib/echo.sparql" \
-      --results=RDF > "${tag_root}/Load${prod_suffix}.rdf" 2> "${TMPDIR}/err.tmp"
+      --results=RDF > "${tag_root}/Load${ONTPUB_FAMILY^^}Prod.rdf" 2> "${TMPDIR}/err.tmp"
 
     if [ -s "${TMPDIR}/err.tmp" ] ; then
-      warning "no RDF XML output generated.  Use Load${prod_suffix}.ttl file instead"
+      warning "no RDF XML output generated.  Use Load${ONTPUB_FAMILY^^}Prod.ttl file instead"
     fi
     rm -f "${TMPDIR}/err.tmp"
   )
@@ -58,14 +55,14 @@ __HERE__
   test -n "${DEV_SPEC}" && test -r "${tag_root:?}/${PROD_SPEC}" && (
     cd "${tag_root}" || return $?
 
-    logItem "<owl:imports> from \"${DEV_SPEC}\"$(echo -en '\t')" "$(logFileName "${tag_root}/Load${dev_suffix}.rdf")"
+    logItem "<owl:imports> from \"${DEV_SPEC}\"$(echo -en '\t')" "$(logFileName "${tag_root}/Load${ONTPUB_FAMILY^^}Dev.rdf")"
 
     cat > "${tmpAboutFileDev}" << __HERE__
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
 @prefix owl: <http://www.w3.org/2002/07/owl#> 
 @prefix xsd: <http://www.w3.org/2001/XMLSchema#>
-<${product_root_url}/Load${dev_suffix}/> a owl:Ontology;
+<${product_root_url}/Load${ONTPUB_FAMILY^^}Dev/> a owl:Ontology;
 __HERE__
 
     getOwlImports < "${tag_root}/${DEV_SPEC}" 2>/dev/null | \
@@ -75,10 +72,10 @@ __HERE__
     "${JENA_ARQ}" \
       --data="${tmpAboutFileDev}" \
       --query="${SCRIPT_DIR}/lib/echo.sparql" \
-      --results=RDF > "${tag_root}/Load${dev_suffix}.rdf" 2> "${TMPDIR}/err.tmp"
+      --results=RDF > "${tag_root}/Load${ONTPUB_FAMILY^^}Dev.rdf" 2> "${TMPDIR}/err.tmp"
 
     if [ -s "${TMPDIR}/err.tmp" ] ; then
-      warning "no RDF XML output generated.  Use Load${dev_suffix}.ttl file instead"
+      warning "no RDF XML output generated.  Use Load${ONTPUB_FAMILY^^}Dev.ttl file instead"
     fi
     rm -f "${TMPDIR}/err.tmp"
   )

diff --git a/publisher/product/ontology/build.sh b/publisher/product/ontology/build.sh
@@ -48,6 +48,13 @@ function publishProductOntology() {
   test -z "${ONTPUB_SHACL_INFIX}" || ontologyCreateSHACLFiles || return $?
 
   ontologyZipFiles > "${tag_root}/ontology-zips.log" || return $?
+
+  if ((speedy)) ; then
+    log "speedy=true -> Not doing quads because they are slow"
+  else
+    buildquads || return $?
+  fi
+
   return 0
 }
 
@@ -698,6 +705,97 @@ function ontologyZipFiles () {
   return 0
 }
 
+function buildquads () {
+
+  local ProdQuadsFile="${tag_root}/prod.${ONTPUB_FAMILY}.nq"
+  local DevQuadsFile="${tag_root}/dev.${ONTPUB_FAMILY}.nq"
+
+  local ProdFlatNT="${tag_root}/old_prod.${ONTPUB_FAMILY}-quickstart.nt"
+  local DevFlatNT="${tag_root}/old_dev.${ONTPUB_FAMILY}-quickstart.nt"
+
+  local ProdFlatTTL="${tag_root}/old_prod.${ONTPUB_FAMILY}-quickstart.ttl"
+  local DevFlatTTL="${tag_root}/old_dev.${ONTPUB_FAMILY}-quickstart.ttl"
+
+  local ProdTMPTTL="$(mktemp ${TMPDIR}/prod.temp.XXXXXX.ttl)"
+  local DevTMPTTL="$(mktemp ${TMPDIR}/dev.temp.XXXXXX.ttl)"
+
+  local CSVPrefixes="${tag_root}/prefixes.${ONTPUB_FAMILY}.csv"
+  local TTLPrefixes="${tag_root}/prefixes.${ONTPUB_FAMILY}.ttl"
+  local SPARQLPrefixes="${tag_root}/prefixes.${ONTPUB_FAMILY}.sq"
+
+
+  local tmpflat="$(mktemp ${TMPDIR}/flatten.XXXXXX.sq)"
+  cat >"${tmpflat}" << __HERE__
+PREFIX owl: <http://www.w3.org/2002/07/owl#> 
+
+CONSTRUCT {?s ?p ?o}
+WHERE {GRAPH ?g {?s ?p ?o
+FILTER NOT EXISTS {?s a owl:Ontology}
+}
+}
+__HERE__
+
+  local tmpflatecho="$(mktemp ${TMPDIR}/flattecho.XXXXXX.sq)"
+  cat >"${tmpflatecho}" << __HERE__
+PREFIX owl: <http://www.w3.org/2002/07/owl#> 
+
+CONSTRUCT {?s ?p ?o}
+WHERE {?s ?p ?o
+FILTER NOT EXISTS {?s a owl:Ontology}
+}
+__HERE__
+
+
+  local tmppx="$(mktemp ${TMPDIR}/px.XXXXXX.sq)"
+  cat >"${tmppx}" << __HERE__
+prefix sm: <http://www.omg.org/techprocess/ab/SpecificationMetadata/>
+prefix owl: <http://www.w3.org/2002/07/owl#>
+prefix xsd: <http://www.w3.org/2001/XMLSchema#>
+
+SELECT ?line
+WHERE {graph ?g {?o a owl:Ontology ;
+sm:fileAbbreviation ?px
+BIND (CONCAT ("prefix ", ?px, ": <", xsd:string(?o), ">") AS ?line)
+}
+}
+__HERE__
+
+  local tmpecho="$(mktemp ${TMPDIR}/echo.XXXXXX.sq)"
+  cat >"${tmpecho}" << __HERE__
+CONSTRUCT {?s ?p ?o}
+WHERE {?s ?p ?o}
+__HERE__
+
+
+  local prefixes="$(mktemp ${TMPDIR}/prefixes.XXXXXX)"
+
+  local tmpmodule="$(mktemp ${TMPDIR}/module.XXXXXX.nt)"
+
+
+  log "starting buildquads with the new quadify"
+
+
+
+  (
+    cd ${ontology_product_tag_root}
+	  echo "starting dev"
+
+	  ${FIND} . -mindepth 2 -name '*.ttl' -print | while read file; do quadify "$file"; done > "${DevQuadsFile}"
+	  echo "starting prod"
+	  ${GREP} -rl 'fibo-fnd-utl-av:hasMaturityLevel fibo-fnd-utl-av:Release' | \
+	      while read file ; do quadify $file ; done > ${ProdQuadsFile}
+     set -x
+	  ${FIND} ${INPUT} -name "Metadata*.rdf" -exec \
+               ${JENA_RIOT} \
+                 --syntax=RDF/XML {} \; \
+		 > ${tmpmodule}
+
+  )
+
+  log "finished buildquads"
+
+  return 0
+  }
 
 function createQuickVersions() {
   require source_family_root || return $?
@@ -707,22 +805,16 @@ function createQuickVersions() {
 
   setProduct ontology || return $?
 
-  logStep "createQuickVersions"
-
   #
   # Get ontologies for Dev
   #
-
   log "Merging all dev ontologies into one RDF file"
 
-  local quick_dev_spec=${DEV_SPEC/.rdf/}-quickstart
-  local quick_prod_spec=${PROD_SPEC/.rdf/}-quickstart
-
   ${ONTOVIEWER_TOOLKIT_JAVA} \
     --goal merge-imports \
     --data "${source_family_root}/${DEV_SPEC}" $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping \"${source_family_root}/catalog-v001.xml\"") \
-    --ontology-iri "${product_root_url}/${quick_dev_spec}/" --ontology-version-iri "${tag_root_url}/${quick_dev_spec}/" \
-    --output "${tag_root}/${quick_dev_spec}.rdf" &>/dev/null
+    --ontology-iri "${product_root_url}/Quick${ONTPUB_FAMILY^^}Dev/" --ontology-version-iri "${tag_root_url}/Quick${ONTPUB_FAMILY^^}Dev/" \
+    --output "${tag_root}/Quick${ONTPUB_FAMILY^^}Dev.rdf" &>/dev/null
 
   #
   # Get ontologies for Prod
@@ -731,20 +823,20 @@ function createQuickVersions() {
   ${ONTOVIEWER_TOOLKIT_JAVA} \
     --goal merge-imports \
     --data "${source_family_root}/${PROD_SPEC}" $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping \"${source_family_root}/catalog-v001.xml\"") \
-    --ontology-iri "${product_root_url}/${quick_prod_spec}/" --ontology-version-iri "${tag_root_url}/${quick_prod_spec}/" \
-    --output "${tag_root}/${quick_prod_spec}.rdf" &>/dev/null
-
-  log "Converting quick ontologies to other formats"
+    --ontology-iri "${product_root_url}/Quick${ONTPUB_FAMILY^^}Prod/" --ontology-version-iri "${tag_root_url}/Quick${ONTPUB_FAMILY^^}Prod/" \
+    --output "${tag_root}/Quick${ONTPUB_FAMILY^^}Prod.rdf" &>/dev/null
 
-  ${SCRIPT_DIR}/utils/convertRdfFile.sh rdf-xml "${tag_root}/${quick_dev_spec}.rdf" "turtle"
-  ${SCRIPT_DIR}/utils/convertRdfFile.sh rdf-xml "${tag_root}/${quick_prod_spec}.rdf" "turtle"
+  ${SCRIPT_DIR}/utils/convertRdfFile.sh rdf-xml "${tag_root}/Quick${ONTPUB_FAMILY^^}Dev.rdf" "turtle" && \
+    mv "${tag_root}/Quick${ONTPUB_FAMILY^^}Dev.ttl" ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.ttl
+  ${SCRIPT_DIR}/utils/convertRdfFile.sh rdf-xml "${tag_root}/Quick${ONTPUB_FAMILY^^}Prod.rdf" "turtle" && \
+    mv "${tag_root}/Quick${ONTPUB_FAMILY^^}Prod.ttl" ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.ttl
 
-  ${JENA_ARQ} --data=${tag_root}/${quick_dev_spec}.ttl --query=/publisher/lib/echo.sparql --results=NT > ${tag_root}/${quick_dev_spec}.nt
-  ${JENA_ARQ} --data=${tag_root}/${quick_prod_spec}.ttl --query=/publisher/lib/echo.sparql --results=NT > ${tag_root}/${quick_prod_spec}.nt
+  ${JENA_ARQ} --data=${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.ttl --query=/publisher/lib/echo.sparql --results=NT > ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.nt
+  ${JENA_ARQ} --data=${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.ttl --query=/publisher/lib/echo.sparql --results=NT > ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.nt
 
-  zip ${tag_root}/${quick_dev_spec}.ttl.zip ${tag_root}/${quick_dev_spec}.ttl
-  zip ${tag_root}/${quick_prod_spec}.ttl.zip ${tag_root}/${quick_prod_spec}.ttl
-  zip ${tag_root}/${quick_dev_spec}.nt.zip ${tag_root}/${quick_dev_spec}.nt
-  zip ${tag_root}/${quick_prod_spec}.nt.zip ${tag_root}/${quick_prod_spec}.nt
+  zip ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.ttl.zip ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.ttl
+  zip ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.ttl.zip ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.ttl
+  zip ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.nt.zip ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.nt
+  zip ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.nt.zip ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.nt
 
 }