From 4ef1b25da6501aedf5b89b301abe8a6458938c87 Mon Sep 17 00:00:00 2001 From: Eliot Jordan Date: Wed, 6 Sep 2023 11:19:22 -0500 Subject: [PATCH] Use orginfo json formatted output --- .../geo_derivatives/processors/vector/info.rb | 35 +-- .../vector_spec.rb | 4 +- .../processors/vector/info_spec.rb | 16 +- spec/fixtures/files/gdal/ogrinfo_line.json | 254 ++++++++++++++++++ spec/fixtures/files/gdal/ogrinfo_line.txt | 14 - spec/fixtures/files/gdal/ogrinfo_polygon.json | 143 ++++++++++ spec/fixtures/files/gdal/ogrinfo_polygon.txt | 22 -- 7 files changed, 427 insertions(+), 61 deletions(-) create mode 100644 spec/fixtures/files/gdal/ogrinfo_line.json delete mode 100644 spec/fixtures/files/gdal/ogrinfo_line.txt create mode 100644 spec/fixtures/files/gdal/ogrinfo_polygon.json delete mode 100644 spec/fixtures/files/gdal/ogrinfo_polygon.txt diff --git a/app/derivative_services/geo_derivatives/processors/vector/info.rb b/app/derivative_services/geo_derivatives/processors/vector/info.rb index 6cb7e7d584..47d7c614b6 100644 --- a/app/derivative_services/geo_derivatives/processors/vector/info.rb +++ b/app/derivative_services/geo_derivatives/processors/vector/info.rb @@ -3,11 +3,21 @@ module GeoDerivatives module Processors module Vector class Info - attr_accessor :doc + attr_accessor :path attr_writer :name, :driver def initialize(path) - @doc = ogrinfo(path) + @path = path + end + + # Parsed json out from ogrinfo + # @return [Hash] + def doc + @doc ||= begin + JSON.parse(ogrinfo(path)) + rescue + {} + end end # Returns the vector dataset name @@ -40,7 +50,7 @@ def bounds # @param path [String] path to vector file or shapefile directory # @return [String] output of ogrinfo def ogrinfo(path) - stdout, stderr, status = Open3.capture3("ogrinfo", "-ro", "-so", "-al", path.to_s) + stdout, stderr, status = Open3.capture3("ogrinfo", "-json", "-ro", "-so", "-al", path.to_s) raise(GeoDerivatives::OgrError, stderr) unless status.success? stdout end @@ -49,27 +59,25 @@ def ogrinfo(path) # the vector dataset name. # @return [String] vector dataset name def vector_name - match = /(?<=Layer name:\s).*?(?=\n)/.match(doc) - match ? match[0] : "" + doc.dig("layers", 0, "name") || "" end # Given an output string from the ogrinfo command, returns # the ogr driver used to read dataset. # @return [String] ogr driver name def driver_name - match = /(?<=driver\s`).*?(?=')/.match(doc) - match ? match[0] : "" + doc.fetch("driverShortName", "") end # Given an output string from the ogrinfo command, returns # the vector geometry type. # @return [String] vector geom def vector_geom - match = /(?<=Geometry:\s).*?(?=\n)/.match(doc) - geom = match ? match[0] : "" + geom = doc.dig("layers", 0, "geometryFields", 0, "type") || "" + # Transform OGR-style 'Line String' into GeoJSON 'Line' # Transform OGR-style '3D Multi Polygon' into GeoJSON 'Polygon' - if geom == "Line String" + if geom == "LineString" geom = "Line" elsif geom == "3D Multi Polygon" geom = "Polygon" @@ -82,12 +90,9 @@ def vector_geom # the vector bounding box. # @return [Hash] vector bounds def vector_bounds - match = /(?<=Extent:\s).*?(?=\n)/.match(doc) - extent = match ? match[0] : "" + extent = doc.dig("layers", 0, "geometryFields", 0, "extent").map { |c| c.truncate(6) } - # remove parens and spaces, split into array, and assign elements to variables - w, s, e, n = extent.delete(" ").gsub(")-(", ",").delete("(").delete(")").split(",") - { north: n.to_f, east: e.to_f, south: s.to_f, west: w.to_f } + { north: extent[3].to_f, east: extent[2].to_f, south: extent[1].to_f, west: extent[0].to_f } end end end diff --git a/spec/characterization_services/gdal_characterization_service/vector_spec.rb b/spec/characterization_services/gdal_characterization_service/vector_spec.rb index 74b379add0..6e70b43303 100644 --- a/spec/characterization_services/gdal_characterization_service/vector_spec.rb +++ b/spec/characterization_services/gdal_characterization_service/vector_spec.rb @@ -24,7 +24,7 @@ file_set = valid_file_set new_file_set = described_class.new(file_set: file_set, persister: persister).characterize(save: false) expect(new_file_set.original_file.mime_type).to eq ["application/vnd.geo+json"] - expect(new_file_set.original_file.geometry).to eq ["Multi Polygon"] + expect(new_file_set.original_file.geometry).to eq ["MultiPolygon"] end end @@ -36,7 +36,7 @@ file_set = valid_file_set new_file_set = described_class.new(file_set: file_set, persister: persister).characterize(save: false) expect(new_file_set.original_file.mime_type).to eq ["application/vnd.geo+json"] - expect(new_file_set.original_file.geometry).to eq ["Multi Polygon"] + expect(new_file_set.original_file.geometry).to eq ["MultiPolygon"] end end diff --git a/spec/derivative_services/geo_derivatives/processors/vector/info_spec.rb b/spec/derivative_services/geo_derivatives/processors/vector/info_spec.rb index b8bed1ab64..399ed3ebf4 100644 --- a/spec/derivative_services/geo_derivatives/processors/vector/info_spec.rb +++ b/spec/derivative_services/geo_derivatives/processors/vector/info_spec.rb @@ -4,9 +4,9 @@ RSpec.describe GeoDerivatives::Processors::Vector::Info do let(:processor) { described_class.new(path) } - let(:path) { "test.tif" } - let(:polygon_info_doc) { file_fixture("files/gdal/ogrinfo_polygon.txt").read } - let(:line_info_doc) { file_fixture("files/gdal/ogrinfo_line.txt").read } + let(:path) { "test.geojson" } + let(:polygon_info_doc) { file_fixture("files/gdal/ogrinfo_polygon.json").read } + let(:line_info_doc) { file_fixture("files/gdal/ogrinfo_line.json").read } let(:status) { double(success?: true) } before do @@ -15,14 +15,14 @@ context "when initializing a new info class" do it "shells out to ogrinfo and sets the doc variable to the output string" do - expect(processor.doc).to eq(polygon_info_doc) - expect(Open3).to have_received(:capture3).with("ogrinfo", "-ro", "-so", "-al", path.to_s) + expect(processor.doc).to eq(JSON.parse(polygon_info_doc)) + expect(Open3).to have_received(:capture3).with("ogrinfo", "-json", "-ro", "-so", "-al", path.to_s) end end context "with a polygon vector" do before do - allow(processor).to receive(:doc).and_return(polygon_info_doc) + allow(Open3).to receive(:capture3).and_return([polygon_info_doc, "", status]) end describe "#name" do @@ -46,7 +46,7 @@ describe "#bounds" do it "returns bounds hash" do expect(processor.bounds).to eq(north: 42.408249, - east: -71.052853, + east: -71.052852, south: 42.347654, west: -71.163867) end @@ -55,7 +55,7 @@ context "with a line vector" do before do - allow(processor).to receive(:doc).and_return(line_info_doc) + allow(Open3).to receive(:capture3).and_return([line_info_doc, "", status]) end describe "#geom" do diff --git a/spec/fixtures/files/gdal/ogrinfo_line.json b/spec/fixtures/files/gdal/ogrinfo_line.json new file mode 100644 index 0000000000..6a2cd883aa --- /dev/null +++ b/spec/fixtures/files/gdal/ogrinfo_line.json @@ -0,0 +1,254 @@ +{ + "description":"/Users/eliotj/Downloads/railroad2004", + "driverShortName":"ESRI Shapefile", + "driverLongName":"ESRI Shapefile", + "layers":[ + { + "name":"railroad", + "metadata":{ + "":{ + "DBF_DATE_LAST_UPDATE":"2004-05-21" + }, + "SHAPEFILE":{ + "ENCODING_FROM_LDID":"ISO-8859-1", + "LDID_VALUE":"87", + "SOURCE_ENCODING":"ISO-8859-1" + } + }, + "geometryFields":[ + { + "name":"", + "type":"LineString", + "nullable":true, + "extent":[ + 913557.58879884903, + 125930.64707553876, + 1060140.8159581227, + 272873.05330644269 + ], + "coordinateSystem":{ + "wkt":"PROJCRS[\"NAD83 / New York Long Island (ftUS)\",\n BASEGEOGCRS[\"NAD83\",\n DATUM[\"North American Datum 1983\",\n ELLIPSOID[\"GRS 1980\",6378137,298.257222101,\n LENGTHUNIT[\"metre\",1]]],\n PRIMEM[\"Greenwich\",0,\n ANGLEUNIT[\"degree\",0.0174532925199433]],\n ID[\"EPSG\",4269]],\n CONVERSION[\"SPCS83 New York Long Island zone (US Survey feet)\",\n METHOD[\"Lambert Conic Conformal (2SP)\",\n ID[\"EPSG\",9802]],\n PARAMETER[\"Latitude of false origin\",40.1666666666667,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8821]],\n PARAMETER[\"Longitude of false origin\",-74,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8822]],\n PARAMETER[\"Latitude of 1st standard parallel\",41.0333333333333,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8823]],\n PARAMETER[\"Latitude of 2nd standard parallel\",40.6666666666667,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8824]],\n PARAMETER[\"Easting at false origin\",984250,\n LENGTHUNIT[\"US survey foot\",0.304800609601219],\n ID[\"EPSG\",8826]],\n PARAMETER[\"Northing at false origin\",0,\n LENGTHUNIT[\"US survey foot\",0.304800609601219],\n ID[\"EPSG\",8827]]],\n CS[Cartesian,2],\n AXIS[\"easting (X)\",east,\n ORDER[1],\n LENGTHUNIT[\"US survey foot\",0.304800609601219]],\n AXIS[\"northing (Y)\",north,\n ORDER[2],\n LENGTHUNIT[\"US survey foot\",0.304800609601219]],\n USAGE[\n SCOPE[\"Engineering survey, topographic mapping.\"],\n AREA[\"United States (USA) - New York - counties of Bronx; Kings; Nassau; New York; Queens; Richmond; Suffolk.\"],\n BBOX[40.47,-74.26,41.3,-71.8]],\n ID[\"EPSG\",2263]]", + "projjson":{ + "$schema":"https://proj.org/schemas/v0.7/projjson.schema.json", + "type":"ProjectedCRS", + "name":"NAD83 / New York Long Island (ftUS)", + "base_crs":{ + "name":"NAD83", + "datum":{ + "type":"GeodeticReferenceFrame", + "name":"North American Datum 1983", + "ellipsoid":{ + "name":"GRS 1980", + "semi_major_axis":6378137, + "inverse_flattening":298.257222101 + } + }, + "coordinate_system":{ + "subtype":"ellipsoidal", + "axis":[ + { + "name":"Geodetic latitude", + "abbreviation":"Lat", + "direction":"north", + "unit":"degree" + }, + { + "name":"Geodetic longitude", + "abbreviation":"Lon", + "direction":"east", + "unit":"degree" + } + ] + }, + "id":{ + "authority":"EPSG", + "code":4269 + } + }, + "conversion":{ + "name":"SPCS83 New York Long Island zone (US Survey feet)", + "method":{ + "name":"Lambert Conic Conformal (2SP)", + "id":{ + "authority":"EPSG", + "code":9802 + } + }, + "parameters":[ + { + "name":"Latitude of false origin", + "value":40.1666666666667, + "unit":"degree", + "id":{ + "authority":"EPSG", + "code":8821 + } + }, + { + "name":"Longitude of false origin", + "value":-74, + "unit":"degree", + "id":{ + "authority":"EPSG", + "code":8822 + } + }, + { + "name":"Latitude of 1st standard parallel", + "value":41.0333333333333, + "unit":"degree", + "id":{ + "authority":"EPSG", + "code":8823 + } + }, + { + "name":"Latitude of 2nd standard parallel", + "value":40.6666666666667, + "unit":"degree", + "id":{ + "authority":"EPSG", + "code":8824 + } + }, + { + "name":"Easting at false origin", + "value":984250, + "unit":{ + "type":"LinearUnit", + "name":"US survey foot", + "conversion_factor":0.304800609601219 + }, + "id":{ + "authority":"EPSG", + "code":8826 + } + }, + { + "name":"Northing at false origin", + "value":0, + "unit":{ + "type":"LinearUnit", + "name":"US survey foot", + "conversion_factor":0.304800609601219 + }, + "id":{ + "authority":"EPSG", + "code":8827 + } + } + ] + }, + "coordinate_system":{ + "subtype":"Cartesian", + "axis":[ + { + "name":"Easting", + "abbreviation":"X", + "direction":"east", + "unit":{ + "type":"LinearUnit", + "name":"US survey foot", + "conversion_factor":0.304800609601219 + } + }, + { + "name":"Northing", + "abbreviation":"Y", + "direction":"north", + "unit":{ + "type":"LinearUnit", + "name":"US survey foot", + "conversion_factor":0.304800609601219 + } + } + ] + }, + "scope":"Engineering survey, topographic mapping.", + "area":"United States (USA) - New York - counties of Bronx; Kings; Nassau; New York; Queens; Richmond; Suffolk.", + "bbox":{ + "south_latitude":40.47, + "west_longitude":-74.26, + "north_latitude":41.3, + "east_longitude":-71.8 + }, + "id":{ + "authority":"EPSG", + "code":2263 + } + }, + "dataAxisToSRSAxisMapping":[ + 1, + 2 + ] + } + } + ], + "featureCount":11610, + "fields":[ + { + "name":"SOURCEDATE", + "type":"String", + "width":25, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"NAME", + "type":"String", + "width":32, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"SOURCETYPE", + "type":"String", + "width":15, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"SOURCE_ID", + "type":"Integer64", + "width":11, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"MODIFIED", + "type":"String", + "width":25, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"FEAT_CODE", + "type":"Integer", + "width":4, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"FEAT_DESC", + "type":"String", + "width":50, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"EXPORTED", + "type":"String", + "width":12, + "nullable":true, + "uniqueConstraint":false + } + ] + } + ], + "metadata":{ + }, + "domains":{ + }, + "relationships":{ + } +} \ No newline at end of file diff --git a/spec/fixtures/files/gdal/ogrinfo_line.txt b/spec/fixtures/files/gdal/ogrinfo_line.txt deleted file mode 100644 index acca8e5da2..0000000000 --- a/spec/fixtures/files/gdal/ogrinfo_line.txt +++ /dev/null @@ -1,14 +0,0 @@ -INFO: Open of `line/' - using driver `ESRI Shapefile' successful. - -Layer name: GraticuleDegrees -Geometry: Line String -Feature Count: 180 -Extent: (-74.000000, 37.000000) - (-65.000000, 46.000000) -Layer SRS WKT: -GEOGCS["GCS_North_American_1983", - DATUM["North_American_Datum_1983", - SPHEROID["GRS_1980",6378137.0,298.257222101]], - PRIMEM["Greenwich",0.0], - UNIT["Degree",0.0174532925199433]] -Shape_Leng: Real (19.11) \ No newline at end of file diff --git a/spec/fixtures/files/gdal/ogrinfo_polygon.json b/spec/fixtures/files/gdal/ogrinfo_polygon.json new file mode 100644 index 0000000000..2044925407 --- /dev/null +++ b/spec/fixtures/files/gdal/ogrinfo_polygon.json @@ -0,0 +1,143 @@ +{ + "description":"tufts-cambridgegrid100-04-shapefile", + "driverShortName":"ESRI Shapefile", + "driverLongName":"ESRI Shapefile", + "layers":[ + { + "name":"tufts-cambridgegrid100-04", + "metadata":{ + "":{ + "DBF_DATE_LAST_UPDATE":"1923-09-06" + }, + "SHAPEFILE":{ + "SOURCE_ENCODING":"" + } + }, + "geometryFields":[ + { + "name":"", + "type":"Polygon", + "nullable":true, + "extent":[ + -71.163867241977428, + 42.347654173288426, + -71.052852699155395, + 42.40824928097171 + ], + "coordinateSystem":{ + "wkt":"GEOGCRS[\"WGS 84\",\n DATUM[\"World Geodetic System 1984\",\n ELLIPSOID[\"WGS 84\",6378137,298.257223563,\n LENGTHUNIT[\"metre\",1]],\n ID[\"EPSG\",6326]],\n PRIMEM[\"Greenwich\",0,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8901]],\n CS[ellipsoidal,2],\n AXIS[\"geodetic longitude\",east,\n ORDER[1],\n ANGLEUNIT[\"degree\",0.0174532925199433]],\n AXIS[\"geodetic latitude\",north,\n ORDER[2],\n ANGLEUNIT[\"degree\",0.0174532925199433]]]", + "projjson":{ + "$schema":"https://proj.org/schemas/v0.7/projjson.schema.json", + "type":"GeographicCRS", + "name":"WGS 84", + "datum":{ + "type":"GeodeticReferenceFrame", + "name":"World Geodetic System 1984", + "ellipsoid":{ + "name":"WGS 84", + "semi_major_axis":6378137, + "inverse_flattening":298.257223563 + }, + "id":{ + "authority":"EPSG", + "code":6326 + } + }, + "coordinate_system":{ + "subtype":"ellipsoidal", + "axis":[ + { + "name":"Geodetic longitude", + "abbreviation":"", + "direction":"east", + "unit":"degree" + }, + { + "name":"Geodetic latitude", + "abbreviation":"", + "direction":"north", + "unit":"degree" + } + ] + } + }, + "dataAxisToSRSAxisMapping":[ + 1, + 2 + ] + } + } + ], + "featureCount":60, + "fields":[ + { + "name":"perimeter", + "type":"Real", + "width":33, + "precision":15, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"grid100_", + "type":"Real", + "width":33, + "precision":15, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"grid100_id", + "type":"Real", + "width":33, + "precision":15, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"gr_tile", + "type":"Integer", + "width":9, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"x_coord", + "type":"Real", + "width":33, + "precision":15, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"y_coord", + "type":"Real", + "width":33, + "precision":15, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"gr_map", + "type":"Integer", + "width":9, + "nullable":true, + "uniqueConstraint":false + }, + { + "name":"tile", + "type":"String", + "width":254, + "nullable":true, + "uniqueConstraint":false + } + ] + } + ], + "metadata":{ + }, + "domains":{ + }, + "relationships":{ + } +} diff --git a/spec/fixtures/files/gdal/ogrinfo_polygon.txt b/spec/fixtures/files/gdal/ogrinfo_polygon.txt deleted file mode 100644 index da41b628cc..0000000000 --- a/spec/fixtures/files/gdal/ogrinfo_polygon.txt +++ /dev/null @@ -1,22 +0,0 @@ -INFO: Open of `/var/folders/hm/srr0rtcd14v99xw_k0zlfj000000gq/T/sufia20161021-45314-1thdj5b_1477031854961' - using driver `ESRI Shapefile' successful. - -Layer name: tufts-cambridgegrid100-04 -Geometry: Polygon -Feature Count: 60 -Extent: (-71.163867, 42.347654) - (-71.052853, 42.408249) -Layer SRS WKT: -GEOGCS["GCS_WGS_1984", - DATUM["WGS_1984", - SPHEROID["WGS_84",6378137,298.257223563]], - PRIMEM["Greenwich",0], - UNIT["Degree",0.017453292519943295]] -AREA: Real (33.31) -PERIMETER: Real (33.31) -GRID100_: Real (33.31) -GRID100_ID: Real (33.31) -GR_TILE: Integer (9.0) -X_COORD: Real (33.31) -Y_COORD: Real (33.31) -GR_MAP: Integer (9.0) -TILE: String (254.0) \ No newline at end of file