diff --git a/CHANGELOG.md b/CHANGELOG.md index 363a91afe..f959cedc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [2.0.2] - 2015-04-02 +### Changed +* Small data exports in worker mode now use the regular exportToFormat workflow in Exporter.js + ## [2.0.1] - 2015-04-01 ### Changed * Forcing status processing on progress reporting from export worker jobs @@ -162,6 +166,7 @@ Koop is now just a node module that exposes an express middleware app with hooks - koop-server is no more; all central code is in the koop project - to use Koop you must use it as middleware in an app that boots up an http server +[2.0.2]: https://github.com/Esri/koop/compare/v2.0.1...v2.0.2 [2.0.1]: https://github.com/Esri/koop/compare/v2.0.0...v2.0.1 [2.0.0]: https://github.com/Esri/koop/compare/v1.1.2...v2.0.0 [1.1.2]: https://github.com/Esri/koop/compare/v1.1.1...v1.1.2 diff --git a/lib/ExportWorker.js b/lib/ExportWorker.js index f38725035..0bf561a23 100644 --- a/lib/ExportWorker.js +++ b/lib/ExportWorker.js @@ -45,7 +45,7 @@ jobs = kue.createQueue({ // - ie can this be distributed across more machines? // - does it need to on the same server always? jobs.process('exports', 2, function(job, done){ - console.log('starting export job', job.id ); + console.log('starting export job', job.id); // Jobs for removing files from the local FS if (job.data.remove){ @@ -57,72 +57,102 @@ jobs.process('exports', 2, function(job, done){ }); // if we have VRT file we can use that to create a new export - } else if (fs.existsSync(job.data.files.rootVrtFile) && !job.data.options.ignore_cache){ - // since we have a VRT file locally - // we just want to create the export and complete the job - koop.Cache.getInfo(job.data.table, function(err, info){ - if (err){ - console.log('vrt file exists, but no data in the db', job.data.files.rootVrtFile) - return done('failed to generate export' + err); + } else { + koop.Cache.getCount(job.data.table, job.data.options, function(err, count){ + + // if we can handle the data in one page + if (count < job.data.options.limit) { + koop.Cache.db.select(job.data.dbkey, job.data.options, function(err, geojson){ + delete geojson[0].info; + koop.Exporter.exportToFormat( + job.data.options.format, + job.data.options.dir, + job.data.options.key, + geojson[0], + job.data.options, + function(err, result ){ + if (err){ + return done( err ); + } + finishExport( + job.data.format, + job.data.options.key, + job.data.options, + result, + done + ); + }); + }); } - info.status = 'processing'; - info.generating = { - progress: 100+'%' - }; - koop.Cache.updateInfo(job.data.table, info, function(err, res){ - // create large file from vrt - var cmd = ['ogr2ogr', '-f', job.data.ogrFormat, '-update', '-append', ( job.data.format == 'zip' ) ? job.data.files.rootNewFileTmp.replace('.zip','') : job.data.files.rootNewFileTmp, job.data.files.rootVrtFile]; - - try { - - var params = { - inFile: job.data.files.rootVrtFile, - outFile: job.data.files.rootNewFileTmp, - paths: job.data.files, - format: job.data.format + // else if we already have a VRT + else if (fs.existsSync(job.data.files.rootVrtFile) && !job.data.options.ignore_cache){ + // since we have a VRT file locally + // we just want to create the export and complete the job + koop.Cache.getInfo(job.data.table, function(err, info){ + if (err){ + console.log('vrt file exists, but no data in the db', job.data.files.rootVrtFile) + return done('failed to generate export' + err); + } + info.status = 'processing'; + info.generating = { + progress: 100+'%' }; - - koop.Exporter.callOgr(params, null, job.data.options, function(err, formatFile){ - if (err){ - return done( err ); - } - // remove the processing state and return the job - delete info.status; - delete info.generating; - delete info.export_lock; - finishExport( job.data.format, job.data.options.key, job.data.options, { paths: job.data.files, file: formatFile }, function(){ - koop.Cache.updateInfo(job.data.table, info, function(e, res){ - if (err) { - return done(err); - } else { - if (typeof gc === 'function') { - gc(); - } - return done(); + koop.Cache.updateInfo(job.data.table, info, function(err, res){ + // create large file from vrt + var cmd = ['ogr2ogr', '-f', job.data.ogrFormat, '-update', '-append', ( job.data.format == 'zip' ) ? job.data.files.rootNewFileTmp.replace('.zip','') : job.data.files.rootNewFileTmp, job.data.files.rootVrtFile]; + + try { + + var params = { + inFile: job.data.files.rootVrtFile, + outFile: job.data.files.rootNewFileTmp, + paths: job.data.files, + format: job.data.format + }; + + koop.Exporter.callOgr(params, null, job.data.options, function(err, formatFile){ + if (err){ + return done( err ); } + // remove the processing state and return the job + delete info.status; + delete info.generating; + delete info.export_lock; + finishExport( job.data.format, job.data.options.key, job.data.options, { paths: job.data.files, file: formatFile }, function(){ + koop.Cache.updateInfo(job.data.table, info, function(e, res){ + if (err) { + return done(err); + } else { + if (typeof gc === 'function') { + gc(); + } + return done(); + } + }); + }); }); - }); + } catch (e){ + console.log('error calling org', e); + info.generating = { error: e }; + koop.Cache.updateInfo( job.data.table, info, function(err, res){ + done('failed to generate file ' + e); + }); + } }); - } catch (e){ - console.log('error calling org', e); - info.generating = { error: e }; - koop.Cache.updateInfo( job.data.table, info, function(err, res){ - done('failed to generate file ' + e); + }); + } else { + fs.mkdir( job.data.files.base, '0777', true, function(){ + koop.Cache.getCount(job.data.table, job.data.options, function(err, count){ + var pages = Math.ceil(count / job.data.options.limit); + for (var i=0; i < pages; i++){ + var offset = i * (job.data.options.limit); + var chunk = { file: job.data.files.base+'/part.' + i + '.json', offset: offset }; + job.data.pages.push( chunk ); + } + createFiles( job, done ); }); - } - }); - }); - } else { - fs.mkdir( job.data.files.base, '0777', true, function(){ - koop.Cache.getCount(job.data.table, job.data.options, function(err, count){ - var pages = Math.ceil(count / job.data.options.limit); - for (var i=0; i < pages; i++){ - var offset = i * (job.data.options.limit); - var chunk = { file: job.data.files.base+'/part.' + i + '.json', offset: offset }; - job.data.pages.push( chunk ); - } - createFiles( job, done ); - }); + }); + } }); } }); diff --git a/lib/Exporter.js b/lib/Exporter.js index 885706634..ad70c5d89 100644 --- a/lib/Exporter.js +++ b/lib/Exporter.js @@ -21,6 +21,7 @@ var ogrFormats = { // exports large data via multi part file strategy exports.exportLarge = function( koop, format, id, key, type, options, finish, done ){ + // exports large data via multi part file strategy if (!format){ return done('No format provided', null); @@ -135,6 +136,7 @@ exports.exportLarge = function( koop, format, id, key, type, options, finish, do var task = {}; task.options = options; task.options.key = key; + task.options.dir = dir; task.dbkey = dbkey; task.table = table; task.format = format; diff --git a/package.json b/package.json index 5affc380e..d174efa14 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "koop", - "version": "2.0.1", + "version": "2.0.2", "description": "A node module/express middleware for converting GeoJSON to Esri Feature Services.", "main": "index.js", "scripts": {