Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Write untagged nodes by default #101

Merged
merged 3 commits into from
Nov 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions .github/workflows/docker-build-bz2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,22 @@ jobs:
docker build -f ${{matrix.dockerfile}} -t osm2rdf .
docker run --rm osm2rdf --help

- name: Build TTL for Malta and check its validity
- name: Build TTL for Liechtenstein and check its validity
run: |
set -v
mkdir osm-malta && cd $_
curl -L -o osm-malta.pbf https://download.geofabrik.de/europe/malta-latest.osm.pbf
ls -l osm-malta.pbf
docker run --rm -v $(pwd):/data osm2rdf /data/osm-malta.pbf -o /data/osm-malta.ttl
ls -l osm-malta.pbf osm-malta.ttl.bz2
docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-malta.ttl.bz2
mkdir osm-liechtenstein && cd $_
curl -L -o osm-liechtenstein.pbf https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf
ls -l osm-liechtenstein.pbf
docker run --rm -v $(pwd):/data osm2rdf /data/osm-liechtenstein.pbf -o /data/osm-liechtenstein.ttl
ls -l osm-liechtenstein.pbf osm-liechtenstein.ttl.bz2
docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-liechtenstein.ttl.bz2

- name: Build QLever index and count the number of geometries
run: |
set -v
cd osm-malta
docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "bzcat osm-malta.ttl.bz2 | IndexBuilderMain -F ttl -f - -i osm-malta"
docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-malta -p 7000"
cd osm-liechtenstein
docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "bzcat osm-liechtenstein.ttl.bz2 | IndexBuilderMain -F ttl -f - -i osm-liechtenstein"
docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-liechtenstein -p 7000"
sleep 5
docker logs qlever
RESULT_JSON=$(curl http://localhost:7000 --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT (COUNT(?geometry) AS ?count) WHERE { ?osm_id geo:hasGeometry ?geometry }")
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/docker-build-gz.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,22 @@ jobs:
docker build -f ${{matrix.dockerfile}} -t osm2rdf .
docker run --rm osm2rdf --help

- name: Build TTL for Malta and check its validity
- name: Build TTL for Liechtenstein and check its validity
run: |
set -v
mkdir osm-malta && cd $_
curl -L -o osm-malta.pbf https://download.geofabrik.de/europe/malta-latest.osm.pbf
ls -l osm-malta.pbf
docker run --rm -v $(pwd):/data osm2rdf /data/osm-malta.pbf -o /data/osm-malta.ttl --output-compression gz
ls -l osm-malta.pbf osm-malta.ttl.gz
docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-malta.ttl.gz
mkdir osm-liechtenstein && cd $_
curl -L -o osm-liechtenstein.pbf https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf
ls -l osm-liechtenstein.pbf
docker run --rm -v $(pwd):/data osm2rdf /data/osm-liechtenstein.pbf -o /data/osm-liechtenstein.ttl --output-compression gz
ls -l osm-liechtenstein.pbf osm-liechtenstein.ttl.gz
docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-liechtenstein.ttl.gz

- name: Build QLever index and count the number of geometries
run: |
set -v
cd osm-malta
docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "zcat osm-malta.ttl.gz | IndexBuilderMain -F ttl -f - -i osm-malta"
docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-malta -p 7000"
cd osm-liechtenstein
docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "zcat osm-liechtenstein.ttl.gz | IndexBuilderMain -F ttl -f - -i osm-liechtenstein"
docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-liechtenstein -p 7000"
sleep 5
docker logs qlever
RESULT_JSON=$(curl http://localhost:7000 --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT (COUNT(?geometry) AS ?count) WHERE { ?osm_id geo:hasGeometry ?geometry }")
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/docker-build-stdout.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,22 @@ jobs:
docker build -f ${{matrix.dockerfile}} -t osm2rdf .
docker run --rm osm2rdf --help

- name: Build TTL for Malta and check its validity
- name: Build TTL for Liechtenstein and check its validity
run: |
set -v
mkdir osm-malta && cd $_
curl -L -o osm-malta.pbf https://download.geofabrik.de/europe/malta-latest.osm.pbf
ls -l osm-malta.pbf
docker run --rm -v $(pwd):/data osm2rdf /data/osm-malta.pbf > osm-malta.ttl
ls -l osm-malta.pbf osm-malta.ttl
docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-malta.ttl
mkdir osm-liechtenstein && cd $_
curl -L -o osm-liechtenstein.pbf https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf
ls -l osm-liechtenstein.pbf
docker run --rm -v $(pwd):/data osm2rdf /data/osm-liechtenstein.pbf > osm-liechtenstein.ttl
ls -l osm-liechtenstein.pbf osm-liechtenstein.ttl
docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-liechtenstein.ttl

- name: Build QLever index and count the number of geometries
run: |
set -v
cd osm-malta
docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "cat osm-malta.ttl | IndexBuilderMain -F ttl -f - -i osm-malta"
docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-malta -p 7000"
cd osm-liechtenstein
docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "cat osm-liechtenstein.ttl | IndexBuilderMain -F ttl -f - -i osm-liechtenstein"
docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-liechtenstein -p 7000"
sleep 5
docker logs qlever
RESULT_JSON=$(curl http://localhost:7000 --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT (COUNT(?geometry) AS ?count) WHERE { ?osm_id geo:hasGeometry ?geometry }")
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/docker-build-uncompressed.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,22 @@ jobs:
docker build -f ${{matrix.dockerfile}} -t osm2rdf .
docker run --rm osm2rdf --help

- name: Build TTL for Malta and check its validity
- name: Build TTL for Liechtenstein and check its validity
run: |
set -v
mkdir osm-malta && cd $_
curl -L -o osm-malta.pbf https://download.geofabrik.de/europe/malta-latest.osm.pbf
ls -l osm-malta.pbf
docker run --rm -v $(pwd):/data osm2rdf /data/osm-malta.pbf --output-compression none -o /data/osm-malta.ttl
ls -l osm-malta.pbf osm-malta.ttl
docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-malta.ttl
mkdir osm-liechtenstein && cd $_
curl -L -o osm-liechtenstein.pbf https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf
ls -l osm-liechtenstein.pbf
docker run --rm -v $(pwd):/data osm2rdf /data/osm-liechtenstein.pbf --output-compression none -o /data/osm-liechtenstein.ttl
ls -l osm-liechtenstein.pbf osm-liechtenstein.ttl
docker run --rm -v $(pwd):/data stain/jena riot --validate /data/osm-liechtenstein.ttl

- name: Build QLever index and count the number of geometries
run: |
set -v
cd osm-malta
docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "cat osm-malta.ttl | IndexBuilderMain -F ttl -f - -i osm-malta"
docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-malta -p 7000"
cd osm-liechtenstein
docker run -u $(id -u):$(id -g) -v $(pwd):/data -w /data --entrypoint bash adfreiburg/qlever -c "cat osm-liechtenstein.ttl | IndexBuilderMain -F ttl -f - -i osm-liechtenstein"
docker run -d -p 7000:7000 -v $(pwd):/data -w /data --entrypoint bash --name qlever adfreiburg/qlever -c "ServerMain -i /data/osm-liechtenstein -p 7000"
sleep 5
docker logs qlever
RESULT_JSON=$(curl http://localhost:7000 --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT (COUNT(?geometry) AS ?count) WHERE { ?osm_id geo:hasGeometry ?geometry }")
Expand Down
1 change: 0 additions & 1 deletion include/osm2rdf/config/Config.h
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ struct Config {
bool addAreaWayLinestrings = false;
bool addCentroids = true;
bool addWayMetadata = false;
bool addWayNodeGeometry = false;
bool addWayNodeOrder = false;
bool addWayNodeSpatialMetadata = false;
bool skipWikiLinks = false;
Expand Down
8 changes: 0 additions & 8 deletions include/osm2rdf/config/Constants.h
Original file line number Diff line number Diff line change
Expand Up @@ -197,14 +197,6 @@ const static inline std::string ADD_WAY_METADATA_OPTION_LONG =
const static inline std::string ADD_WAY_METADATA_OPTION_HELP =
"Add information about the way structure";

const static inline std::string ADD_WAY_NODE_GEOMETRY_INFO =
"Adding way member geometries";
const static inline std::string ADD_WAY_NODE_GEOMETRY_OPTION_SHORT = "";
const static inline std::string ADD_WAY_NODE_GEOMETRY_OPTION_LONG =
"add-way-node-geometry";
const static inline std::string ADD_WAY_NODE_GEOMETRY_OPTION_HELP =
"Add explicit geometry for every node member";

const static inline std::string ADD_WAY_NODE_ORDER_INFO =
"Adding way node order";
const static inline std::string ADD_WAY_NODE_ORDER_OPTION_SHORT = "";
Expand Down
10 changes: 0 additions & 10 deletions src/config/Config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,6 @@ std::string osm2rdf::config::Config::getInfo(std::string_view prefix) const {
oss << "\n"
<< prefix << osm2rdf::config::constants::ADD_WAY_METADATA_INFO;
}
if (addWayNodeGeometry) {
oss << "\n"
<< prefix << osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_INFO;
}
if (addWayNodeOrder) {
oss << "\n"
<< prefix << osm2rdf::config::constants::ADD_WAY_NODE_ORDER_INFO;
Expand Down Expand Up @@ -273,10 +269,6 @@ void osm2rdf::config::Config::fromArgs(int argc, char** argv) {
osm2rdf::config::constants::ADD_WAY_METADATA_OPTION_SHORT,
osm2rdf::config::constants::ADD_WAY_METADATA_OPTION_LONG,
osm2rdf::config::constants::ADD_WAY_METADATA_OPTION_HELP);
auto addWayNodeGeometryOp = parser.add<popl::Switch>(
osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_OPTION_SHORT,
osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_OPTION_LONG,
osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_OPTION_HELP);
auto addWayNodeOrderOp = parser.add<popl::Switch>(
osm2rdf::config::constants::ADD_WAY_NODE_ORDER_OPTION_SHORT,
osm2rdf::config::constants::ADD_WAY_NODE_ORDER_OPTION_LONG,
Expand Down Expand Up @@ -444,7 +436,6 @@ void osm2rdf::config::Config::fromArgs(int argc, char** argv) {
addAreaWayLinestrings = addAreaWayLinestringsOp->is_set();
addCentroids = !noAddCentroidsOp->is_set();
addWayMetadata = addWayMetadataOp->is_set();
addWayNodeGeometry = addWayNodeGeometryOp->is_set();
addWayNodeOrder = addWayNodeOrderOp->is_set();
addWayNodeSpatialMetadata = addWayNodeSpatialMetadataOp->is_set();
skipWikiLinks = skipWikiLinksOp->is_set();
Expand All @@ -453,7 +444,6 @@ void osm2rdf::config::Config::fromArgs(int argc, char** argv) {
wktDeviation = wktDeviationOp->value();
wktPrecision = wktPrecisionOp->value();

addWayNodeOrder |= addWayNodeGeometry;
addWayNodeOrder |= addWayNodeSpatialMetadata;

if (semicolonTagKeysOp->is_set()) {
Expand Down
2 changes: 1 addition & 1 deletion src/osm/CountHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ void osm2rdf::osm::CountHandler::prepare_for_lookup() {
void osm2rdf::osm::CountHandler::node(const osmium::Node& node){
if (node.positive_id() < _minId) _minId = node.positive_id();
if (node.positive_id() > _maxId) _maxId = node.positive_id();
if (_firstPassDone || node.tags().empty()) {
if (_firstPassDone) {
return;
}
_numNodes++;
Expand Down
14 changes: 0 additions & 14 deletions src/osm/FactHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -266,20 +266,6 @@ void osm2rdf::osm::FactHandler<W>::way(const osm2rdf::osm::Way& way) {
std::to_string(wayOrder++),
"^^" + IRI__XSD_INTEGER);

if (_config.addWayNodeGeometry) {
const std::string& subj = _writer->generateIRI(
NODE_NAMESPACE[_config.sourceDataset], node.id());

_writer->writeTriple(subj, IRI__RDF_TYPE, IRI__OSM_NODE);

const std::string& geomObj = _writer->generateIRIUnsafe(
NAMESPACE__OSM2RDF_GEOM, DATASET_ID[_config.sourceDataset] +
"_node_" + std::to_string(node.id()));

_writer->writeTriple(subj, IRI__GEOSPARQL__HAS_GEOMETRY, geomObj);
writeGeometry(geomObj, IRI__GEOSPARQL__AS_WKT, node.geom());
}

if (_config.addWayNodeSpatialMetadata && !lastBlankNode.empty()) {
_writer->writeTriple(
lastBlankNode, IRI__OSMWAY_NEXT_NODE,
Expand Down
3 changes: 0 additions & 3 deletions src/osm/OsmiumHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,9 +186,6 @@ void osm2rdf::osm::OsmiumHandler<W>::area(const osmium::Area& area) {
template <typename W>
void osm2rdf::osm::OsmiumHandler<W>::node(const osmium::Node& node) {
_nodesSeen++;
if (node.tags().empty()) {
return;
}

try {
const auto& osmNode = osm2rdf::osm::Node(node);
Expand Down
14 changes: 7 additions & 7 deletions tests/E2E.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ TEST(E2E, singleNode) {
ASSERT_THAT(printedState,
::testing::HasSubstr("areas seen:0 dumped: 0 geometry: 0\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("nodes seen:1 dumped: 0 geometry: 0\n"));
::testing::HasSubstr("nodes seen:1 dumped: 1 geometry: 1\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
ASSERT_THAT(printedState,
Expand Down Expand Up @@ -415,7 +415,7 @@ TEST(E2E, osmWikiExample) {
ASSERT_THAT(printedState,
::testing::HasSubstr("areas seen:0 dumped: 0 geometry: 0\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("nodes seen:4 dumped: 1 geometry: 1\n"));
::testing::HasSubstr("nodes seen:4 dumped: 4 geometry: 4\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("relations seen:1 dumped: 1 geometry: 0\n"));
ASSERT_THAT(printedState,
Expand Down Expand Up @@ -503,7 +503,7 @@ TEST(E2E, building51NT) {
ASSERT_THAT(printedState,
::testing::HasSubstr("areas seen:1 dumped: 1 geometry: 1\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("nodes seen:19 dumped: 3 geometry: 3\n"));
::testing::HasSubstr("nodes seen:19 dumped: 19 geometry: 19\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
ASSERT_THAT(printedState,
Expand Down Expand Up @@ -676,7 +676,7 @@ TEST(E2E, building51TTL) {
ASSERT_THAT(printedState,
::testing::HasSubstr("areas seen:1 dumped: 1 geometry: 1\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("nodes seen:19 dumped: 3 geometry: 3\n"));
::testing::HasSubstr("nodes seen:19 dumped: 19 geometry: 19\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
ASSERT_THAT(printedState,
Expand Down Expand Up @@ -812,7 +812,7 @@ TEST(E2E, building51QLEVER) {
ASSERT_THAT(printedState,
::testing::HasSubstr("areas seen:1 dumped: 1 geometry: 1\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("nodes seen:19 dumped: 3 geometry: 3\n"));
::testing::HasSubstr("nodes seen:19 dumped: 19 geometry: 19\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
ASSERT_THAT(printedState,
Expand Down Expand Up @@ -947,7 +947,7 @@ TEST(E2E, tf) {
ASSERT_THAT(printedState,
::testing::HasSubstr("areas seen:1 dumped: 1 geometry: 1\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("nodes seen:45 dumped: 0 geometry: 0\n"));
::testing::HasSubstr("nodes seen:45 dumped: 45 geometry: 45\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
ASSERT_THAT(printedState,
Expand Down Expand Up @@ -1048,7 +1048,7 @@ TEST(E2E, building51inTF) {
ASSERT_THAT(printedState,
::testing::HasSubstr("areas seen:2 dumped: 2 geometry: 2\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("nodes seen:64 dumped: 3 geometry: 3\n"));
::testing::HasSubstr("nodes seen:64 dumped: 64 geometry: 64\n"));
ASSERT_THAT(printedState,
::testing::HasSubstr("relations seen:0 dumped: 0 geometry: 0\n"));
ASSERT_THAT(printedState,
Expand Down
28 changes: 0 additions & 28 deletions tests/config/Config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ void assertDefaultConfig(const osm2rdf::config::Config& config) {
ASSERT_FALSE(config.noWayGeometricRelations);

ASSERT_FALSE(config.addAreaWayLinestrings);
ASSERT_FALSE(config.addWayNodeGeometry);
ASSERT_FALSE(config.addWayNodeOrder);
ASSERT_FALSE(config.addWayNodeSpatialMetadata);
ASSERT_FALSE(config.addWayMetadata);
Expand Down Expand Up @@ -578,22 +577,6 @@ TEST(CONFIG_Config, fromArgsAddWayMetadataLong) {
ASSERT_TRUE(config.addWayMetadata);
}

// ____________________________________________________________________________
TEST(CONFIG_Config, fromArgsAddWayNodeGeomentryLong) {
osm2rdf::config::Config config;
assertDefaultConfig(config);

osm2rdf::util::CacheFile cf("/tmp/dummyInput");
const auto arg =
"--" + osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_OPTION_LONG;
const int argc = 3;
char* argv[argc] = {const_cast<char*>(""), const_cast<char*>(arg.c_str()),
const_cast<char*>("/tmp/dummyInput")};
config.fromArgs(argc, argv);
ASSERT_EQ("", config.output.string());
ASSERT_TRUE(config.addWayNodeGeometry);
}

// ____________________________________________________________________________
TEST(CONFIG_Config, fromArgsAddWayNodeOrderLong) {
osm2rdf::config::Config config;
Expand Down Expand Up @@ -890,17 +873,6 @@ TEST(CONFIG_Config, getInfoAddWayMetadata) {
osm2rdf::config::constants::ADD_WAY_METADATA_INFO));
}

// ____________________________________________________________________________
TEST(CONFIG_Config, getInfoAddWayNodeGeometry) {
osm2rdf::config::Config config;
assertDefaultConfig(config);
config.addWayNodeGeometry = true;

const std::string res = config.getInfo("");
ASSERT_THAT(res, ::testing::HasSubstr(
osm2rdf::config::constants::ADD_WAY_NODE_GEOMETRY_INFO));
}

// ____________________________________________________________________________
TEST(CONFIG_Config, getInfoAddWayNodeOrder) {
osm2rdf::config::Config config;
Expand Down
Loading
Loading