Merge pull request #150 from Esri/bug-fixes-for-2.1.1

Bug fixes for 2.1.1

CHANGELOG.md

@@ -2,6 +2,16 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.1.1] - 2015-04-14
+### Changed
+* sending -update to ogr2ogr to support large file exports
+* fixed the idFilter clause for normal, non-worker exports 
+* using the dataset hash key for VRT files to fix a bug with exports
+* fixed an issue with large files filtered to less 5k ending in stuck datasets 
+
+### Added 
+* added a few jsdocs to the index, the goal is go through all public methods and do this
+
 ## [2.1.0] - 2015-04-09
 ### Added 
 * a new route `/export-workers` to inspect the current backlog of export workers if configured to run in worker mode
@@ -186,6 +196,7 @@ Koop is now just a node module that exposes an express middleware app with hooks
   - koop-server is no more; all central code is in the koop project
   - to use Koop you must use it as middleware in an app that boots up an http server
 
+[2.1.1]: https://github.com/Esri/koop/compare/v2.1.0...v2.1.1
 [2.1.0]: https://github.com/Esri/koop/compare/v2.0.4...v2.1.0
 [2.0.4]: https://github.com/Esri/koop/compare/v2.0.3...v2.0.4
 [2.0.3]: https://github.com/Esri/koop/compare/v2.0.2...v2.0.3

index.js

@@ -55,8 +55,11 @@ module.exports = function( config ) {
     })
   });
 
-  // register providers into the app
-  // sets up models, routes -> controllers/handlers
+  /** 
+   * Register providers into the main Koop app
+   * exposes the provider's routes, controller, and models 
+   * @param {object} the provider to be registered
+   */ 
   app.register = function(provider){
     if (provider.type && provider.type == 'plugin'){
       return app.registerPlugin( provider );
@@ -87,6 +90,12 @@ module.exports = function( config ) {
     }
   };
 
+  /** 
+   * Registers a koop plugin
+   * Plugins can be any function that you want to have global access to 
+   * within koop provider models  
+   * @param {object} any koop plugin
+   */
   app.registerPlugin = function( plugin ){
     koop[plugin.name] = plugin;
   };
@@ -216,17 +225,6 @@ module.exports = function( config ) {
 
   koop.Cache.db = koop.LocalDB; 
 
-  // registers a DB modules  
-  app.registerCache = function( adapter ){
-    if ( config.db && config.db.conn ) {
-      koop.Cache.db = adapter.connect( config.db.conn, koop );
-    } 
-    else {
-      console.log('Cannot register this cache, missing db connection in config');
-    }
-    return;
-  };
-
   // remove the x powered by header from all responses
   app.use(function (req, res, next) {
       res.removeHeader("X-Powered-By");
@@ -240,9 +238,15 @@ module.exports = function( config ) {
   if (!config.db || !config.db.conn){
     console.warn('Warning koop w/o persistent cache means no data will be cached across server sessions.');
   }
+  // the defaul cache is the local in-mem cache 
+  // to persist data you must call registerCache with db adapter 
   koop.Cache.db = koop.LocalDB; 
-
-  // registers a DB modules  
+
+  /** 
+   * Register DB adapters into the main Koop app
+   * overwrites any existing koop.Cache.db 
+   * @param {object} a koop db adapter
+   */
   app.registerCache = function( adapter ){
     if ( config.db && config.db.conn ) {
       koop.Cache.db = adapter.connect( config.db.conn, koop );

lib/ExportWorker.js

@@ -87,7 +87,20 @@ jobs.process('exports', 2, function(job, done){
                       job.data.options.key, 
                       job.data.options, 
                       result, 
-                      done
+                      function(err, path){
+                        koop.Cache.getInfo(job.data.table, function(err, info){
+                          console.log('got info')
+                          delete info.status;
+                          delete info.generating;
+                          delete info.export_lock;
+                          koop.Cache.updateInfo(job.data.table, info, function(e, res){
+                            if (e) {
+                              return done(e);
+                            }
+                            return done();
+                          });      
+                        });      
+                      }
                     );            
                 });
               });

lib/Exporter.js

@@ -97,10 +97,9 @@ exports.exportLarge = function( koop, format, id, key, type, options, finish, do
   var q = async.queue(function (task, cb) {
     // instead of passing a limit and offset 
     // we use a WHERE clause 
-    var idFilter = ' id >= '+ options.offset + ' AND id < ' + options.offset + options.limit;
-    var idFilter = ' id >= '+ task.offset + ' AND id < ' + (parseInt(task.offset) + parseInt(task.options.limit));
+    var idFilter = ' id >= '+ options.offset + ' AND id < ' + (parseInt(options.offset) + parseInt(options.limit));
     var opts = {
-      ifFilter: idFilter,
+      idFilter: idFilter,
       layer: options.layer,
       where: options.where,
       geometry: options.geometry,
@@ -344,7 +343,9 @@ function createPaths(dir, key, format, options){
 
   paths.jsonFile    = (options.name || key) + paths.tmpName+ '.json';
   paths.jsonFileTmp = (options.name || key) + paths.tmpName + '.json';
-  paths.vrtFile     = (options.name || key) + '.vrt';
+  // the VRT file must use the key to support large filters
+  // the file has to be unique to the filter
+  paths.vrtFile     = key + '.vrt';
   paths.newFileTmp  = key + paths.tmpName + '.' + format;
   paths.newFile     = (options.name || key) + '.' + format;
 
@@ -458,6 +459,7 @@ function getOgrParams( format, inFile, outFile, geojson, options ){
 
   }
 
+  cmd.push('-update');
   cmd.push('-append');
   cmd.push('-skipfailures');
   cmd.push('-lco');

package.json

@@ -1,6 +1,6 @@
 {
   "name": "koop",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "A node module/express middleware for converting GeoJSON to Esri Feature Services.",
   "main": "index.js",
   "scripts": {