Merge pull request #252 from koopjs/refactor/projections

[DNM] Uses SpatialReference for wkid2wkt translation
koopjs · Sep 17, 2015 · 315fce7 · 315fce7
2 parents 50cbf91 + f605b5b
commit 315fce7
Show file tree

Hide file tree

Showing 5 changed files with 174 additions and 118 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,11 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ## Unreleased
 ### Fixed
 * Don't include vrt files in shapefile zips
+* OGR2OGR calls with projections use double quotes for Windows compatibility
+
+### Changed
+* Refactored projection support to use [spatialreference](http://github.com/koopjs/spatialreference)
+* Refactored export logic for csv and shapefiles into separate functions
 
 ## [2.7.2] - 2015-09-16 
 ### Fixed

diff --git a/lib/ExportWorker.js b/lib/ExportWorker.js
@@ -132,7 +132,10 @@ jobs.process('exports', concurrency, function (job, done) {
                   format: job.data.format
                 }
 
-                job.data.options.large = true
+                var options = job.data.options
+                options.large = true
+                options.db = koopLib.Cache.db
+                options.logger = koopLib.log
 
                 koopLib.Exporter.callOgr(params, null, job.data.options, function (err, formatFile) {
                   if (err) return done(err)

diff --git a/lib/Exporter.js b/lib/Exporter.js
@@ -4,11 +4,11 @@ var fs = require('fs')
 var mkdirp = require('mkdirp')
 var async = require('async')
 var crypto = require('crypto')
-var projCodes = require('esri-proj-codes')
 var mv = require('mv')
 var formatSpatialRef = require('format-spatial-ref')
 var rm = require('rimraf')
 var exec = require('child_process').exec
+var SR = require('spatialreference')
 var ogrFormats = {
   kml: 'KML',
   zip: '"ESRI Shapefile"',
@@ -340,43 +340,45 @@ function callOgr (params, geojson, options, callback) {
   // we already have the file, just return it
   if (fs.existsSync(outFile)) return callback(null, outFile)
 
-  var ogrParams = getOgrParams(format, inFile, outFile, geojson, options)
-  exec(ogrParams, function (err) {
-    if (err) {
-      callback(err.message + ' ' + ogrParams, null)
-    } else {
-      if (format === 'zip') {
-        // mkdir for base path (dir + key) to store shp
-        mkdirp(paths.base, function () {
-          if (!options.name) {
-            options.name = paths.tmpName
-          }
-
-          // cp each file into dir with new name
-          function _createZip () {
-            var newZipTmp = paths.base + '/' + options.name + paths.tmpName + '.zip'
-            var newZip = paths.base + '/' + options.name + '.zip'
-            var cmd = ['zip', '-rj', '"' + newZipTmp + '"', paths.base + '/', '--exclude=*.json*', '--exclude=*.vrt'].join(' ')
-            exec(cmd, function (err) {
-              if (err) return callback(err)
-
-              moveFile(newZipTmp, newZip, callback)
-              removeShapeFile(paths.base, options.name, function (err) {
-                if (err) console.log('error removing shpfile ' + err)
+  getOgrParams(format, inFile, outFile, geojson, options, function (err, cmd) {
+    if (err) return callback(err)
+    exec(cmd, function (err) {
+      if (err) {
+        callback(err.message + ': ' + cmd, null)
+      } else {
+        if (format === 'zip') {
+          // mkdir for base path (dir + key) to store shp
+          mkdirp(paths.base, function () {
+            if (!options.name) {
+              options.name = paths.tmpName
+            }
+
+            // cp each file into dir with new name
+            function _createZip () {
+              var newZipTmp = paths.base + '/' + options.name + paths.tmpName + '.zip'
+              var newZip = paths.base + '/' + options.name + '.zip'
+              var cmd = ['zip', '-rj', '"' + newZipTmp + '"', paths.base + '/', '--exclude=*.json*', '--exclude=*.vrt'].join(' ')
+              exec(cmd, function (err) {
+                if (err) return callback(err)
+
+                moveFile(newZipTmp, newZip, callback)
+                removeShapeFile(paths.base, options.name, function (err) {
+                  if (err) console.log('error removing shpfile ' + err)
+                })
               })
-            })
-          }
+            }
 
-          if (metadata) {
-            fs.writeFileSync(paths.base + '/' + options.name + '.xml', metadata)
-          }
+            if (metadata) {
+              fs.writeFileSync(paths.base + '/' + options.name + '.xml', metadata)
+            }
 
-          moveShapeFile(outFile, paths.base, options.name, _createZip)
-        })
-      } else {
-        moveFile(outFile, paths.rootNewFile, callback)
+            moveShapeFile(outFile, paths.base, options.name, _createZip)
+          })
+        } else {
+          moveFile(outFile, paths.rootNewFile, callback)
+        }
       }
-    }
+    })
   })
 }
 
@@ -437,89 +439,85 @@ function moveFile (inFile, newFile, callback) {
   })
 }
 
-function fixWkt (wkt, wkid) {
-  // always replace Lambert_Conformal_Conic with Lambert_Conformal_Conic_1SP
-  // open ogr2ogr bug: http://trac.osgeo.org/gdal/ticket/2072
-
-  // if we are using LCC we need to apply the datum transformation
-  if (wkt.match(/Lambert_Conformal_Conic/)) {
-    wkt = wkt.replace('Lambert_Conformal_Conic', 'Lambert_Conformal_Conic_2SP')
-  } else if (wkt.match(/UTM/)) {
-    wkt = wkt.replace('],PRIMEM', ',TOWGS84[-0.9956,1.9013,0.5215,0.025915,0.009426,0.011599,-0.00062]],PRIMEM')
-  }
-
+function fixWkt (inWkt, wkid) {
   // we have issue projecting this WKID w/o a datum xform
   // FYI there may be other proj codes needed here
-  if (wkid && wkid === 2927) {
-    wkt = wkt.replace('],PRIMEM', ',TOWGS84[-0.9956,1.9013,0.5215,0.025915,0.009426,0.011599,-0.00062]],PRIMEM')
-  } else if (wkid && wkid === 3078) {
-    wkt = '+proj=omerc +lat_0=45.30916666666666 +lonc=-86 +alpha=337.25556 +k=0.9996 +x_0=2546731.496 +y_0=-4354009.816 +  ellps=GRS80 +datum=NAD83 +units=m +no_defs'
+  var wkt
+  switch (wkid) {
+    case 3078:
+      wkt = '+proj=omerc +lat_0=45.30916666666666 +lonc=-86 +alpha=337.25556 +k=0.9996 +x_0=2546731.496 +y_0=-4354009.816 + ellps=GRS80 +datum=NAD83 +units=m +no_defs'
+      break
+    case 2927:
+      wkt = 'PROJCS["NAD83(HARN) / Washington South (ftUS)",GEOGCS["NAD83(HARN)",DATUM["NAD83_High_Accuracy_Regional_Network",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[-0.9956,1.9013,0.5215,0.025915,0.009426,0.011599,-0.00062],AUTHORITY["EPSG","6152"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4152"]],UNIT["US survey foot",0.3048006096012192,AUTHORITY["EPSG","9003"]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["standard_parallel_1",47.33333333333334],PARAMETER["standard_parallel_2",45.83333333333334],PARAMETER["latitude_of_origin",45.33333333333334],PARAMETER["central_meridian",-120.5],PARAMETER["false_easting",1640416.667],PARAMETER["false_northing",0],AUTHORITY["EPSG","2927"],AXIS["X",EAST],AXIS["Y",NORTH]]'
+      break
+    case 28992:
+      wkt = '+title=Amersfoort/Amersfoort +proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.999908 +x_0=155000 +y_0=463000 +ellps=bessel +units=m +no_defs +towgs84=565.2369,50.0087,465.658,-0.406857330322398,0.350732676542563,-1.8703473836068,4.0812'
+      break
+    default:
+      wkt = inWkt
   }
+  if (wkt.match(/UTM/)) return wkt.replace('TOWGS84[0,0,0,0,0,0,0]', 'TOWGS84[-0.9956,1.9013,0.5215,0.025915,0.009426,0.011599,-0.00062]')
   return wkt
 }
 
-function getOgrParams (format, inFile, outFile, geojson, options) {
+function getOgrParams (format, inFile, outFile, geojson, options, callback) {
   // escape quotes in file names
   inFile = inFile.replace(/"/g, '"')
   outFile = outFile.replace(/"/g, '"')
 
   // replace the format extension for zip/shp so the shp gets created as a dir
   outFile = outFile.replace('\.zip', '')
 
-  var cmd = [
-    'ogr2ogr',
-    '--config',
-    'SHAPE_ENCODING',
-    'UTF-8',
-    '-f',
-    ogrFormats[format],
-    outFile,
-    inFile
-  ]
+  var cmd = ['ogr2ogr', '--config', 'SHAPE_ENCODING', 'UTF-8', '-f', ogrFormats[format], outFile, inFile]
 
   if (format === 'csv') {
-    if (geojson &&
-      geojson.features &&
-      geojson.features.length &&
-      geojson.features[0].geometry &&
-      geojson.features[0].geometry.type === 'Point' &&
-      ((!geojson.features[0].properties['x'] && !geojson.features[0].properties['y']) ||
-      (!geojson.features[0].properties['X'] && !geojson.features[0].properties['Y']))) {
-      cmd.push('-lco')
-      cmd.push('WRITE_BOM=YES')
-      cmd.push('-lco')
-      cmd.push('GEOMETRY=AS_XY')
-    }
+    cmd = csvParams(geojson, cmd)
+    callback(null, finishOgrParams(cmd))
   } else if (format === 'zip' || format === 'shp') {
     // only project features for shp when wkid != 4326 or 3857 or 102100
-    var sr = {}
-    var wkt
-
-    if (options.outSR) {
-      sr = formatSpatialRef(options.outSR)
-    }
+    shapefileParams(cmd, options, function (err, cmd) {
+      if (err) callback(err)
+      callback(null, finishOgrParams(cmd))
+    })
+  } else {
+    callback(null, finishOgrParams(cmd))
+  }
+}
 
-    var wkid = sr.latestWkid || sr.wkid || options.wkid
-    var proj = projCodes.lookup(sr.latestWkid) || projCodes.lookup(sr.wkid) || projCodes.lookup(options.wkid)
-    wkt = sr.wkt || options.wkt
+function csvParams (geojson, cmd) {
+  // how would we even get here without geojson?? leaving this in only to prevent regressions
+  if (!geojson) return cmd
+  var hasPointGeom = geojson.features && geojson.features.length & geojson.features[0].geometry && geojson.features[0].geometry.type === 'Point'
+  var hasXY = (geojson.features[0].properties['x'] && geojson.features[0].properties['y']) || (geojson.features[0].properties['X'] && geojson.features[0].properties['Y'])
+  if (hasPointGeom && !hasXY) {
+    cmd.push('-lco')
+    cmd.push('WRITE_BOM=YES')
+    cmd.push('-lco')
+    cmd.push('GEOMETRY=AS_XY')
+  }
+  return cmd
+}
 
-    if (proj) {
-      if (proj.wkt) {
-        cmd.push('-t_srs')
-        cmd.push("'" + fixWkt(proj.wkt, wkid) + "'")
-      } else {
-        console.log('No proj info found for WKID', proj.wkid, outFile)
-      }
-    } else if (wkt) {
+function shapefileParams (cmd, options, callback) {
+  if (options.outSR) options.sr = formatSpatialRef(options.outSR)
+  if (options.sr || options.wkid) {
+    addProjection(options, function (err, wkt) {
+      if (err) return callback(err)
       cmd.push('-t_srs')
-      cmd.push("'" + fixWkt(wkt) + "'")
-    }
-
-    // make sure field names are not truncated multiple times
+      cmd.push('"' + wkt + '"')
+      // make sure field names are not truncated multiple times
+      cmd.push('-fieldmap')
+      cmd.push('identity')
+      callback(null, cmd)
+    })
+  } else {
     cmd.push('-fieldmap')
     cmd.push('identity')
+    callback(null, cmd)
   }
+}
 
+function finishOgrParams (cmd) {
   cmd.push('-update')
   cmd.push('-append')
   cmd.push('-skipfailures')
@@ -528,6 +526,17 @@ function getOgrParams (format, inFile, outFile, geojson, options) {
   return cmd.join(' ')
 }
 
+function addProjection (options, callback) {
+  // if there is a passed in WKT just use that
+  if (!options.sr) options.sr = {}
+  if (options.sr.wkt) return callback(null, options.sr.wkt)
+  var sr = new SR({db: options.db, logger: options.logger})
+  var wkid = options.sr.latestWkid || options.sr.wkid || options.wkid
+  sr.wkidToWkt(wkid, function (err, wkt) {
+    callback(err, fixWkt(wkt, wkid))
+  })
+}
+
 exports.createIdFilter = createIdFilter
 exports.createPaths = createPaths
 exports.callOgr = callOgr

diff --git a/package.json b/package.json
@@ -20,7 +20,6 @@
     "connect-multiparty": "^1.2.5",
     "ejs": "^2.3.3",
     "esri-extent": "^1.1.0",
-    "esri-proj-codes": "^1.0.0",
     "express": "~4.12.3",
     "format-spatial-ref": "^1.0.0",
     "kue": "~0.9.4",
@@ -30,6 +29,7 @@
     "node-fs": "~0.1.7",
     "request": "~2.54.0",
     "rimraf": "^2.3.2",
+    "spatialreference": "^1.0.0",
     "sphericalmercator": "~1.0.3",
     "sql-parser": "~0.5.0",
     "terraformer": "1.0.5",
@@ -46,6 +46,7 @@
     "jshint": "~2.6.3",
     "jshint-stylish": "^1.0.1",
     "mocha": "^2.2.1",
+    "nock": "^2.12.0",
     "should": "~5.2.0",
     "standard": "^5.2.2"
   },