diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b0c54be..5b074a9cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [2.1.1] - 2015-04-14 +### Changed +* sending -update to ogr2ogr to support large file exports +* fixed the idFilter clause for normal, non-worker exports +* using the dataset hash key for VRT files to fix a bug with exports +* fixed an issue with large files filtered to less 5k ending in stuck datasets + +### Added +* added a few jsdocs to the index, the goal is go through all public methods and do this + ## [2.1.0] - 2015-04-09 ### Added * a new route `/export-workers` to inspect the current backlog of export workers if configured to run in worker mode @@ -186,6 +196,7 @@ Koop is now just a node module that exposes an express middleware app with hooks - koop-server is no more; all central code is in the koop project - to use Koop you must use it as middleware in an app that boots up an http server +[2.1.1]: https://github.com/Esri/koop/compare/v2.1.0...v2.1.1 [2.1.0]: https://github.com/Esri/koop/compare/v2.0.4...v2.1.0 [2.0.4]: https://github.com/Esri/koop/compare/v2.0.3...v2.0.4 [2.0.3]: https://github.com/Esri/koop/compare/v2.0.2...v2.0.3 diff --git a/index.js b/index.js index d73ea62b8..47baf5916 100644 --- a/index.js +++ b/index.js @@ -55,8 +55,11 @@ module.exports = function( config ) { }) }); - // register providers into the app - // sets up models, routes -> controllers/handlers + /** + * Register providers into the main Koop app + * exposes the provider's routes, controller, and models + * @param {object} the provider to be registered + */ app.register = function(provider){ if (provider.type && provider.type == 'plugin'){ return app.registerPlugin( provider ); @@ -87,6 +90,12 @@ module.exports = function( config ) { } }; + /** + * Registers a koop plugin + * Plugins can be any function that you want to have global access to + * within koop provider models + * @param {object} any koop plugin + */ app.registerPlugin = function( plugin ){ koop[plugin.name] = plugin; }; @@ -216,17 +225,6 @@ module.exports = function( config ) { koop.Cache.db = koop.LocalDB; - // registers a DB modules - app.registerCache = function( adapter ){ - if ( config.db && config.db.conn ) { - koop.Cache.db = adapter.connect( config.db.conn, koop ); - } - else { - console.log('Cannot register this cache, missing db connection in config'); - } - return; - }; - // remove the x powered by header from all responses app.use(function (req, res, next) { res.removeHeader("X-Powered-By"); @@ -240,9 +238,15 @@ module.exports = function( config ) { if (!config.db || !config.db.conn){ console.warn('Warning koop w/o persistent cache means no data will be cached across server sessions.'); } + // the defaul cache is the local in-mem cache + // to persist data you must call registerCache with db adapter koop.Cache.db = koop.LocalDB; - - // registers a DB modules + + /** + * Register DB adapters into the main Koop app + * overwrites any existing koop.Cache.db + * @param {object} a koop db adapter + */ app.registerCache = function( adapter ){ if ( config.db && config.db.conn ) { koop.Cache.db = adapter.connect( config.db.conn, koop ); diff --git a/lib/ExportWorker.js b/lib/ExportWorker.js index cde5629d0..30f9b99d4 100644 --- a/lib/ExportWorker.js +++ b/lib/ExportWorker.js @@ -87,7 +87,20 @@ jobs.process('exports', 2, function(job, done){ job.data.options.key, job.data.options, result, - done + function(err, path){ + koop.Cache.getInfo(job.data.table, function(err, info){ + console.log('got info') + delete info.status; + delete info.generating; + delete info.export_lock; + koop.Cache.updateInfo(job.data.table, info, function(e, res){ + if (e) { + return done(e); + } + return done(); + }); + }); + } ); }); }); diff --git a/lib/Exporter.js b/lib/Exporter.js index 603ece4ac..34c263b40 100644 --- a/lib/Exporter.js +++ b/lib/Exporter.js @@ -97,10 +97,9 @@ exports.exportLarge = function( koop, format, id, key, type, options, finish, do var q = async.queue(function (task, cb) { // instead of passing a limit and offset // we use a WHERE clause - var idFilter = ' id >= '+ options.offset + ' AND id < ' + options.offset + options.limit; - var idFilter = ' id >= '+ task.offset + ' AND id < ' + (parseInt(task.offset) + parseInt(task.options.limit)); + var idFilter = ' id >= '+ options.offset + ' AND id < ' + (parseInt(options.offset) + parseInt(options.limit)); var opts = { - ifFilter: idFilter, + idFilter: idFilter, layer: options.layer, where: options.where, geometry: options.geometry, @@ -344,7 +343,9 @@ function createPaths(dir, key, format, options){ paths.jsonFile = (options.name || key) + paths.tmpName+ '.json'; paths.jsonFileTmp = (options.name || key) + paths.tmpName + '.json'; - paths.vrtFile = (options.name || key) + '.vrt'; + // the VRT file must use the key to support large filters + // the file has to be unique to the filter + paths.vrtFile = key + '.vrt'; paths.newFileTmp = key + paths.tmpName + '.' + format; paths.newFile = (options.name || key) + '.' + format; @@ -458,6 +459,7 @@ function getOgrParams( format, inFile, outFile, geojson, options ){ } + cmd.push('-update'); cmd.push('-append'); cmd.push('-skipfailures'); cmd.push('-lco'); diff --git a/package.json b/package.json index db67b224e..4331f0249 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "koop", - "version": "2.1.0", + "version": "2.1.1", "description": "A node module/express middleware for converting GeoJSON to Esri Feature Services.", "main": "index.js", "scripts": {