Write untagged nodes by default (#101)

* remove option --add-way-node-geometry, write all nodes per default * change E2E test dataset to liechtenstein --------- Co-authored-by: Axel Lehmann <[email protected]>
ad-freiburg · Nov 15, 2024 · 3769c8e · 3769c8e
1 parent 0cfbb53
commit 3769c8e
Show file tree

Hide file tree

Showing 15 changed files with 68 additions and 200 deletions.
diff --git a/.github/workflows/docker-build-bz2.yml b/.github/workflows/docker-build-bz2.yml
@@ -32,22 +32,22 @@ jobs:
           docker build -f ${{matrix.dockerfile}} -t osm2rdf .
           docker run --rm osm2rdf --help
 
-      - name: Build TTL for Malta and check its validity
+      - name: Build TTL for Liechtenstein and check its validity
         run: |
           set -v
-          mkdir osm-malta && cd $_
-          curl -L -o osm-malta.pbf https://download.geofabrik.de/europe/malta-latest.osm.pbf
-          ls -l osm-malta.pbf
-          docker run --rm -v $(pwd):/data osm2rdf /data/osm-malta.pbf -o /data/osm-malta.ttl
-          ls -l osm-malta.pbf osm-malta.ttl.bz2
-          docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-malta.ttl.bz2
+          mkdir osm-liechtenstein && cd $_
+          curl -L -o osm-liechtenstein.pbf https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf
+          ls -l osm-liechtenstein.pbf
+          docker run --rm -v $(pwd):/data osm2rdf /data/osm-liechtenstein.pbf -o /data/osm-liechtenstein.ttl
+          ls -l osm-liechtenstein.pbf osm-liechtenstein.ttl.bz2
+          docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-liechtenstein.ttl.bz2
 
       - name: Build QLever index and count the number of geometries
         run: |
           set -v
-          cd osm-malta
-          docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "bzcat osm-malta.ttl.bz2 | IndexBuilderMain -F ttl -f - -i osm-malta"
-          docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-malta -p 7000"
+          cd osm-liechtenstein
+          docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "bzcat osm-liechtenstein.ttl.bz2 | IndexBuilderMain -F ttl -f - -i osm-liechtenstein"
+          docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-liechtenstein -p 7000"
           sleep 5
           docker logs qlever
           RESULT_JSON=$(curl http://localhost:7000 --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT (COUNT(?geometry) AS ?count) WHERE { ?osm_id geo:hasGeometry ?geometry }")

diff --git a/.github/workflows/docker-build-gz.yml b/.github/workflows/docker-build-gz.yml
@@ -32,22 +32,22 @@ jobs:
           docker build -f ${{matrix.dockerfile}} -t osm2rdf .
           docker run --rm osm2rdf --help
 
-      - name: Build TTL for Malta and check its validity
+      - name: Build TTL for Liechtenstein and check its validity
         run: |
           set -v
-          mkdir osm-malta && cd $_
-          curl -L -o osm-malta.pbf https://download.geofabrik.de/europe/malta-latest.osm.pbf
-          ls -l osm-malta.pbf
-          docker run --rm -v $(pwd):/data osm2rdf /data/osm-malta.pbf -o /data/osm-malta.ttl --output-compression gz
-          ls -l osm-malta.pbf osm-malta.ttl.gz
-          docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-malta.ttl.gz
+          mkdir osm-liechtenstein && cd $_
+          curl -L -o osm-liechtenstein.pbf https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf
+          ls -l osm-liechtenstein.pbf
+          docker run --rm -v $(pwd):/data osm2rdf /data/osm-liechtenstein.pbf -o /data/osm-liechtenstein.ttl --output-compression gz
+          ls -l osm-liechtenstein.pbf osm-liechtenstein.ttl.gz
+          docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-liechtenstein.ttl.gz
 
       - name: Build QLever index and count the number of geometries
         run: |
           set -v
-          cd osm-malta
-          docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "zcat osm-malta.ttl.gz | IndexBuilderMain -F ttl -f - -i osm-malta"
-          docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-malta -p 7000"
+          cd osm-liechtenstein
+          docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "zcat osm-liechtenstein.ttl.gz | IndexBuilderMain -F ttl -f - -i osm-liechtenstein"
+          docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-liechtenstein -p 7000"
           sleep 5
           docker logs qlever
           RESULT_JSON=$(curl http://localhost:7000 --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT (COUNT(?geometry) AS ?count) WHERE { ?osm_id geo:hasGeometry ?geometry }")

diff --git a/.github/workflows/docker-build-stdout.yml b/.github/workflows/docker-build-stdout.yml
@@ -32,22 +32,22 @@ jobs:
           docker build -f ${{matrix.dockerfile}} -t osm2rdf .
           docker run --rm osm2rdf --help
 
-      - name: Build TTL for Malta and check its validity
+      - name: Build TTL for Liechtenstein and check its validity
         run: |
           set -v
-          mkdir osm-malta && cd $_
-          curl -L -o osm-malta.pbf https://download.geofabrik.de/europe/malta-latest.osm.pbf
-          ls -l osm-malta.pbf
-          docker run --rm -v $(pwd):/data osm2rdf /data/osm-malta.pbf > osm-malta.ttl
-          ls -l osm-malta.pbf osm-malta.ttl
-          docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-malta.ttl
+          mkdir osm-liechtenstein && cd $_
+          curl -L -o osm-liechtenstein.pbf https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf
+          ls -l osm-liechtenstein.pbf
+          docker run --rm -v $(pwd):/data osm2rdf /data/osm-liechtenstein.pbf > osm-liechtenstein.ttl
+          ls -l osm-liechtenstein.pbf osm-liechtenstein.ttl
+          docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-liechtenstein.ttl
 
       - name: Build QLever index and count the number of geometries
         run: |
           set -v
-          cd osm-malta
-          docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "cat osm-malta.ttl | IndexBuilderMain -F ttl -f - -i osm-malta"
-          docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-malta -p 7000"
+          cd osm-liechtenstein
+          docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "cat osm-liechtenstein.ttl | IndexBuilderMain -F ttl -f - -i osm-liechtenstein"
+          docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-liechtenstein -p 7000"
           sleep 5
           docker logs qlever
           RESULT_JSON=$(curl http://localhost:7000 --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT (COUNT(?geometry) AS ?count) WHERE { ?osm_id geo:hasGeometry ?geometry }")

diff --git a/.github/workflows/docker-build-uncompressed.yml b/.github/workflows/docker-build-uncompressed.yml
@@ -32,22 +32,22 @@ jobs:
           docker build -f ${{matrix.dockerfile}} -t osm2rdf .
           docker run --rm osm2rdf --help
 
-      - name: Build TTL for Malta and check its validity
+      - name: Build TTL for Liechtenstein and check its validity
         run: |
           set -v
-          mkdir osm-malta && cd $_
-          curl -L -o osm-malta.pbf https://download.geofabrik.de/europe/malta-latest.osm.pbf
-          ls -l osm-malta.pbf
-          docker run --rm -v $(pwd):/data osm2rdf /data/osm-malta.pbf --output-compression none -o /data/osm-malta.ttl
-          ls -l osm-malta.pbf osm-malta.ttl
-          docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-malta.ttl
+          mkdir osm-liechtenstein && cd $_
+          curl -L -o osm-liechtenstein.pbf https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf
+          ls -l osm-liechtenstein.pbf
+          docker run --rm -v $(pwd):/data osm2rdf /data/osm-liechtenstein.pbf --output-compression none -o /data/osm-liechtenstein.ttl
+          ls -l osm-liechtenstein.pbf osm-liechtenstein.ttl
+          docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-liechtenstein.ttl
 
       - name: Build QLever index and count the number of geometries
         run: |
           set -v
-          cd osm-malta
-          docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "cat osm-malta.ttl | IndexBuilderMain -F ttl -f - -i osm-malta"
-          docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-malta -p 7000"
+          cd osm-liechtenstein
+          docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "cat osm-liechtenstein.ttl | IndexBuilderMain -F ttl -f - -i osm-liechtenstein"
+          docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-liechtenstein -p 7000"
           sleep 5
           docker logs qlever
           RESULT_JSON=$(curl http://localhost:7000 --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT (COUNT(?geometry) AS ?count) WHERE { ?osm_id geo:hasGeometry ?geometry }")

diff --git a/include/osm2rdf/config/Config.h b/include/osm2rdf/config/Config.h
@@ -71,7 +71,6 @@ struct Config {
   bool addAreaWayLinestrings = false;
   bool addCentroids = true;
   bool addWayMetadata = false;
-  bool addWayNodeGeometry = false;
   bool addWayNodeOrder = false;
   bool addWayNodeSpatialMetadata = false;
   bool skipWikiLinks = false;

diff --git a/include/osm2rdf/config/Constants.h b/include/osm2rdf/config/Constants.h
@@ -197,14 +197,6 @@ const static inline std::string ADD_WAY_METADATA_OPTION_LONG =
 const static inline std::string ADD_WAY_METADATA_OPTION_HELP =
     "Add information about the way structure";
 
-const static inline std::string ADD_WAY_NODE_GEOMETRY_INFO =
-    "Adding way member geometries";
-const static inline std::string ADD_WAY_NODE_GEOMETRY_OPTION_SHORT = "";
-const static inline std::string ADD_WAY_NODE_GEOMETRY_OPTION_LONG =
-    "add-way-node-geometry";
-const static inline std::string ADD_WAY_NODE_GEOMETRY_OPTION_HELP =
-    "Add explicit geometry for every node member";
-
 const static inline std::string ADD_WAY_NODE_ORDER_INFO =
     "Adding way node order";
 const static inline std::string ADD_WAY_NODE_ORDER_OPTION_SHORT = "";

diff --git a/src/config/Config.cpp b/src/config/Config.cpp
@@ -72,10 +72,6 @@ std::string osm2rdf::config::Config::getInfo(std::string_view prefix) const {
         oss << "\n"
             << prefix << osm2rdf::config::constants::ADD_WAY_METADATA_INFO;
       }
-      if (addWayNodeGeometry) {
-        oss << "\n"
-            << prefix << osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_INFO;
-      }
       if (addWayNodeOrder) {
         oss << "\n"
             << prefix << osm2rdf::config::constants::ADD_WAY_NODE_ORDER_INFO;
@@ -273,10 +269,6 @@ void osm2rdf::config::Config::fromArgs(int argc, char** argv) {
       osm2rdf::config::constants::ADD_WAY_METADATA_OPTION_SHORT,
       osm2rdf::config::constants::ADD_WAY_METADATA_OPTION_LONG,
       osm2rdf::config::constants::ADD_WAY_METADATA_OPTION_HELP);
-  auto addWayNodeGeometryOp = parser.add<popl::Switch>(
-      osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_OPTION_SHORT,
-      osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_OPTION_LONG,
-      osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_OPTION_HELP);
   auto addWayNodeOrderOp = parser.add<popl::Switch>(
       osm2rdf::config::constants::ADD_WAY_NODE_ORDER_OPTION_SHORT,
       osm2rdf::config::constants::ADD_WAY_NODE_ORDER_OPTION_LONG,
@@ -444,7 +436,6 @@ void osm2rdf::config::Config::fromArgs(int argc, char** argv) {
     addAreaWayLinestrings = addAreaWayLinestringsOp->is_set();
     addCentroids = !noAddCentroidsOp->is_set();
     addWayMetadata = addWayMetadataOp->is_set();
-    addWayNodeGeometry = addWayNodeGeometryOp->is_set();
     addWayNodeOrder = addWayNodeOrderOp->is_set();
     addWayNodeSpatialMetadata = addWayNodeSpatialMetadataOp->is_set();
     skipWikiLinks = skipWikiLinksOp->is_set();
@@ -453,7 +444,6 @@ void osm2rdf::config::Config::fromArgs(int argc, char** argv) {
     wktDeviation = wktDeviationOp->value();
     wktPrecision = wktPrecisionOp->value();
 
-    addWayNodeOrder |= addWayNodeGeometry;
     addWayNodeOrder |= addWayNodeSpatialMetadata;
 
     if (semicolonTagKeysOp->is_set()) {

diff --git a/src/osm/CountHandler.cpp b/src/osm/CountHandler.cpp
@@ -29,7 +29,7 @@ void osm2rdf::osm::CountHandler::prepare_for_lookup() {
 void osm2rdf::osm::CountHandler::node(const osmium::Node& node){
   if (node.positive_id() < _minId) _minId = node.positive_id();
   if (node.positive_id() > _maxId) _maxId = node.positive_id();
-  if (_firstPassDone || node.tags().empty())  {
+  if (_firstPassDone)  {
     return;
   }
   _numNodes++;

diff --git a/src/osm/FactHandler.cpp b/src/osm/FactHandler.cpp
@@ -266,20 +266,6 @@ void osm2rdf::osm::FactHandler<W>::way(const osm2rdf::osm::Way& way) {
                                         std::to_string(wayOrder++),
                                         "^^" + IRI__XSD_INTEGER);
 
-      if (_config.addWayNodeGeometry) {
-        const std::string& subj = _writer->generateIRI(
-            NODE_NAMESPACE[_config.sourceDataset], node.id());
-
-        _writer->writeTriple(subj, IRI__RDF_TYPE, IRI__OSM_NODE);
-
-        const std::string& geomObj = _writer->generateIRIUnsafe(
-            NAMESPACE__OSM2RDF_GEOM, DATASET_ID[_config.sourceDataset] +
-                                         "_node_" + std::to_string(node.id()));
-
-        _writer->writeTriple(subj, IRI__GEOSPARQL__HAS_GEOMETRY, geomObj);
-        writeGeometry(geomObj, IRI__GEOSPARQL__AS_WKT, node.geom());
-      }
-
       if (_config.addWayNodeSpatialMetadata && !lastBlankNode.empty()) {
         _writer->writeTriple(
             lastBlankNode, IRI__OSMWAY_NEXT_NODE,

diff --git a/src/osm/OsmiumHandler.cpp b/src/osm/OsmiumHandler.cpp
@@ -186,9 +186,6 @@ void osm2rdf::osm::OsmiumHandler<W>::area(const osmium::Area& area) {
 template <typename W>
 void osm2rdf::osm::OsmiumHandler<W>::node(const osmium::Node& node) {
   _nodesSeen++;
-  if (node.tags().empty()) {
-    return;
-  }
 
   try {
     const auto& osmNode = osm2rdf::osm::Node(node);

diff --git a/tests/E2E.cpp b/tests/E2E.cpp
@@ -89,7 +89,7 @@ TEST(E2E, singleNode) {
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("areas seen:0 dumped: 0 geometry: 0\n"));
   ASSERT_THAT(printedState,
-              ::testing::HasSubstr("nodes seen:1 dumped: 0 geometry: 0\n"));
+              ::testing::HasSubstr("nodes seen:1 dumped: 1 geometry: 1\n"));
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
   ASSERT_THAT(printedState,
@@ -415,7 +415,7 @@ TEST(E2E, osmWikiExample) {
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("areas seen:0 dumped: 0 geometry: 0\n"));
   ASSERT_THAT(printedState,
-              ::testing::HasSubstr("nodes seen:4 dumped: 1 geometry: 1\n"));
+              ::testing::HasSubstr("nodes seen:4 dumped: 4 geometry: 4\n"));
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("relations seen:1 dumped: 1 geometry: 0\n"));
   ASSERT_THAT(printedState,
@@ -503,7 +503,7 @@ TEST(E2E, building51NT) {
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("areas seen:1 dumped: 1 geometry: 1\n"));
   ASSERT_THAT(printedState,
-              ::testing::HasSubstr("nodes seen:19 dumped: 3 geometry: 3\n"));
+              ::testing::HasSubstr("nodes seen:19 dumped: 19 geometry: 19\n"));
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
   ASSERT_THAT(printedState,
@@ -676,7 +676,7 @@ TEST(E2E, building51TTL) {
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("areas seen:1 dumped: 1 geometry: 1\n"));
   ASSERT_THAT(printedState,
-              ::testing::HasSubstr("nodes seen:19 dumped: 3 geometry: 3\n"));
+              ::testing::HasSubstr("nodes seen:19 dumped: 19 geometry: 19\n"));
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
   ASSERT_THAT(printedState,
@@ -812,7 +812,7 @@ TEST(E2E, building51QLEVER) {
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("areas seen:1 dumped: 1 geometry: 1\n"));
   ASSERT_THAT(printedState,
-              ::testing::HasSubstr("nodes seen:19 dumped: 3 geometry: 3\n"));
+              ::testing::HasSubstr("nodes seen:19 dumped: 19 geometry: 19\n"));
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
   ASSERT_THAT(printedState,
@@ -947,7 +947,7 @@ TEST(E2E, tf) {
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("areas seen:1 dumped: 1 geometry: 1\n"));
   ASSERT_THAT(printedState,
-              ::testing::HasSubstr("nodes seen:45 dumped: 0 geometry: 0\n"));
+              ::testing::HasSubstr("nodes seen:45 dumped: 45 geometry: 45\n"));
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
   ASSERT_THAT(printedState,
@@ -1048,7 +1048,7 @@ TEST(E2E, building51inTF) {
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("areas seen:2 dumped: 2 geometry: 2\n"));
   ASSERT_THAT(printedState,
-              ::testing::HasSubstr("nodes seen:64 dumped: 3 geometry: 3\n"));
+              ::testing::HasSubstr("nodes seen:64 dumped: 64 geometry: 64\n"));
   ASSERT_THAT(printedState,
               ::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
   ASSERT_THAT(printedState,

diff --git a/tests/config/Config.cpp b/tests/config/Config.cpp
@@ -42,7 +42,6 @@ void assertDefaultConfig(const osm2rdf::config::Config& config) {
   ASSERT_FALSE(config.noWayGeometricRelations);
 
   ASSERT_FALSE(config.addAreaWayLinestrings);
-  ASSERT_FALSE(config.addWayNodeGeometry);
   ASSERT_FALSE(config.addWayNodeOrder);
   ASSERT_FALSE(config.addWayNodeSpatialMetadata);
   ASSERT_FALSE(config.addWayMetadata);
@@ -578,22 +577,6 @@ TEST(CONFIG_Config, fromArgsAddWayMetadataLong) {
   ASSERT_TRUE(config.addWayMetadata);
 }
 
-// ____________________________________________________________________________
-TEST(CONFIG_Config, fromArgsAddWayNodeGeomentryLong) {
-  osm2rdf::config::Config config;
-  assertDefaultConfig(config);
-
-  osm2rdf::util::CacheFile cf("/tmp/dummyInput");
-  const auto arg =
-      "--" + osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_OPTION_LONG;
-  const int argc = 3;
-  char* argv[argc] = {const_cast<char*>(""), const_cast<char*>(arg.c_str()),
-                      const_cast<char*>("/tmp/dummyInput")};
-  config.fromArgs(argc, argv);
-  ASSERT_EQ("", config.output.string());
-  ASSERT_TRUE(config.addWayNodeGeometry);
-}
-
 // ____________________________________________________________________________
 TEST(CONFIG_Config, fromArgsAddWayNodeOrderLong) {
   osm2rdf::config::Config config;
@@ -890,17 +873,6 @@ TEST(CONFIG_Config, getInfoAddWayMetadata) {
                        osm2rdf::config::constants::ADD_WAY_METADATA_INFO));
 }
 
-// ____________________________________________________________________________
-TEST(CONFIG_Config, getInfoAddWayNodeGeometry) {
-  osm2rdf::config::Config config;
-  assertDefaultConfig(config);
-  config.addWayNodeGeometry = true;
-
-  const std::string res = config.getInfo("");
-  ASSERT_THAT(res, ::testing::HasSubstr(
-                       osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_INFO));
-}
-
 // ____________________________________________________________________________
 TEST(CONFIG_Config, getInfoAddWayNodeOrder) {
   osm2rdf::config::Config config;