From 4486f982fde1f324702c6e1e4a5ea56e6ea094f7 Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 14 Apr 2015 15:31:58 -0600 Subject: [PATCH 1/6] adding update to ogr2ogr calls; adding some docs on index methods; fixed idFilter clause --- index.js | 34 +++++++++++++++++++--------------- lib/Exporter.js | 10 ++++++---- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/index.js b/index.js index d73ea62b8..47baf5916 100644 --- a/index.js +++ b/index.js @@ -55,8 +55,11 @@ module.exports = function( config ) { }) }); - // register providers into the app - // sets up models, routes -> controllers/handlers + /** + * Register providers into the main Koop app + * exposes the provider's routes, controller, and models + * @param {object} the provider to be registered + */ app.register = function(provider){ if (provider.type && provider.type == 'plugin'){ return app.registerPlugin( provider ); @@ -87,6 +90,12 @@ module.exports = function( config ) { } }; + /** + * Registers a koop plugin + * Plugins can be any function that you want to have global access to + * within koop provider models + * @param {object} any koop plugin + */ app.registerPlugin = function( plugin ){ koop[plugin.name] = plugin; }; @@ -216,17 +225,6 @@ module.exports = function( config ) { koop.Cache.db = koop.LocalDB; - // registers a DB modules - app.registerCache = function( adapter ){ - if ( config.db && config.db.conn ) { - koop.Cache.db = adapter.connect( config.db.conn, koop ); - } - else { - console.log('Cannot register this cache, missing db connection in config'); - } - return; - }; - // remove the x powered by header from all responses app.use(function (req, res, next) { res.removeHeader("X-Powered-By"); @@ -240,9 +238,15 @@ module.exports = function( config ) { if (!config.db || !config.db.conn){ console.warn('Warning koop w/o persistent cache means no data will be cached across server sessions.'); } + // the defaul cache is the local in-mem cache + // to persist data you must call registerCache with db adapter koop.Cache.db = koop.LocalDB; - - // registers a DB modules + + /** + * Register DB adapters into the main Koop app + * overwrites any existing koop.Cache.db + * @param {object} a koop db adapter + */ app.registerCache = function( adapter ){ if ( config.db && config.db.conn ) { koop.Cache.db = adapter.connect( config.db.conn, koop ); diff --git a/lib/Exporter.js b/lib/Exporter.js index 603ece4ac..5c4c6908d 100644 --- a/lib/Exporter.js +++ b/lib/Exporter.js @@ -97,10 +97,9 @@ exports.exportLarge = function( koop, format, id, key, type, options, finish, do var q = async.queue(function (task, cb) { // instead of passing a limit and offset // we use a WHERE clause - var idFilter = ' id >= '+ options.offset + ' AND id < ' + options.offset + options.limit; - var idFilter = ' id >= '+ task.offset + ' AND id < ' + (parseInt(task.offset) + parseInt(task.options.limit)); + var idFilter = ' id >= '+ options.offset + ' AND id < ' + parseInt(options.offset) + parseInt(options.limit); var opts = { - ifFilter: idFilter, + idFilter: idFilter, layer: options.layer, where: options.where, geometry: options.geometry, @@ -344,7 +343,9 @@ function createPaths(dir, key, format, options){ paths.jsonFile = (options.name || key) + paths.tmpName+ '.json'; paths.jsonFileTmp = (options.name || key) + paths.tmpName + '.json'; - paths.vrtFile = (options.name || key) + '.vrt'; + // the VRT file must use the key to support large filters + // the file has to be unique to the filter + paths.vrtFile = key + '.vrt'; paths.newFileTmp = key + paths.tmpName + '.' + format; paths.newFile = (options.name || key) + '.' + format; @@ -458,6 +459,7 @@ function getOgrParams( format, inFile, outFile, geojson, options ){ } + cmd.push('-update'); cmd.push('-append'); cmd.push('-skipfailures'); cmd.push('-lco'); From 7af8600f34704de3f2d8178f35128c891a21b475 Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 14 Apr 2015 15:34:15 -0600 Subject: [PATCH 2/6] updating the changelog package for version 2.1.1 --- CHANGELOG.md | 9 +++++++++ package.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b0c54be..59ab990de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [2.1.1] - 2015-04-14 +### Changed +* sending -update to ogr2ogr to support large file exports +* fixed the idFilter clause for normal, non-worker exports + +### Added +* added a few jsdocs to the index, the goal is go through all public methods and do this + ## [2.1.0] - 2015-04-09 ### Added * a new route `/export-workers` to inspect the current backlog of export workers if configured to run in worker mode @@ -186,6 +194,7 @@ Koop is now just a node module that exposes an express middleware app with hooks - koop-server is no more; all central code is in the koop project - to use Koop you must use it as middleware in an app that boots up an http server +[2.1.1]: https://github.com/Esri/koop/compare/v2.1.0...v2.1.1 [2.1.0]: https://github.com/Esri/koop/compare/v2.0.4...v2.1.0 [2.0.4]: https://github.com/Esri/koop/compare/v2.0.3...v2.0.4 [2.0.3]: https://github.com/Esri/koop/compare/v2.0.2...v2.0.3 diff --git a/package.json b/package.json index db67b224e..4331f0249 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "koop", - "version": "2.1.0", + "version": "2.1.1", "description": "A node module/express middleware for converting GeoJSON to Esri Feature Services.", "main": "index.js", "scripts": { From 3a0a73c97e3efc41163004378a0b6388545b2f84 Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 14 Apr 2015 15:35:06 -0600 Subject: [PATCH 3/6] updating the changelog with a note about the vrt file name change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59ab990de..0b9c4eaf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Changed * sending -update to ogr2ogr to support large file exports * fixed the idFilter clause for normal, non-worker exports +* using the dataset hash key for VRT files to fix a bug with exports ### Added * added a few jsdocs to the index, the goal is go through all public methods and do this From 326d324b37a2f4c541ab82de5d55dea410ed7405 Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 14 Apr 2015 15:37:48 -0600 Subject: [PATCH 4/6] idFilter offset needs to be in parens --- lib/Exporter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Exporter.js b/lib/Exporter.js index 5c4c6908d..34c263b40 100644 --- a/lib/Exporter.js +++ b/lib/Exporter.js @@ -97,7 +97,7 @@ exports.exportLarge = function( koop, format, id, key, type, options, finish, do var q = async.queue(function (task, cb) { // instead of passing a limit and offset // we use a WHERE clause - var idFilter = ' id >= '+ options.offset + ' AND id < ' + parseInt(options.offset) + parseInt(options.limit); + var idFilter = ' id >= '+ options.offset + ' AND id < ' + (parseInt(options.offset) + parseInt(options.limit)); var opts = { idFilter: idFilter, layer: options.layer, From 370e3cc698a6e79d023f334fa85b25270b3db59b Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 14 Apr 2015 16:19:55 -0600 Subject: [PATCH 5/6] caught a bug with datasets over 5k and filtered to less than 5k --- lib/ExportWorker.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/ExportWorker.js b/lib/ExportWorker.js index cde5629d0..30f9b99d4 100644 --- a/lib/ExportWorker.js +++ b/lib/ExportWorker.js @@ -87,7 +87,20 @@ jobs.process('exports', 2, function(job, done){ job.data.options.key, job.data.options, result, - done + function(err, path){ + koop.Cache.getInfo(job.data.table, function(err, info){ + console.log('got info') + delete info.status; + delete info.generating; + delete info.export_lock; + koop.Cache.updateInfo(job.data.table, info, function(e, res){ + if (e) { + return done(e); + } + return done(); + }); + }); + } ); }); }); From bbf7bf1938fe4c1dbeea9c11194eb3a27522cac2 Mon Sep 17 00:00:00 2001 From: chelm Date: Tue, 14 Apr 2015 16:20:37 -0600 Subject: [PATCH 6/6] changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b9c4eaf5..5b074a9cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). * sending -update to ogr2ogr to support large file exports * fixed the idFilter clause for normal, non-worker exports * using the dataset hash key for VRT files to fix a bug with exports +* fixed an issue with large files filtered to less 5k ending in stuck datasets ### Added * added a few jsdocs to the index, the goal is go through all public methods and do this