Skip to content

Commit

Permalink
Merge pull request #140 from Esri/small-exports
Browse files Browse the repository at this point in the history 
Small exports
  • Loading branch information
@ungoldman
ungoldman committed Apr 2, 2015
2 parents cf87ec4 + 5819bf4 commit 2483884
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 62 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](http://semver.org/).

## [2.0.2] - 2015-04-02
### Changed
* Small data exports in worker mode now use the regular exportToFormat workflow in Exporter.js

## [2.0.1] - 2015-04-01
### Changed
* Forcing status processing on progress reporting from export worker jobs
Expand Down Expand Up @@ -162,6 +166,7 @@ Koop is now just a node module that exposes an express middleware app with hooks
- koop-server is no more; all central code is in the koop project
- to use Koop you must use it as middleware in an app that boots up an http server

[2.0.2]: https://github.com/Esri/koop/compare/v2.0.1...v2.0.2
[2.0.1]: https://github.com/Esri/koop/compare/v2.0.0...v2.0.1
[2.0.0]: https://github.com/Esri/koop/compare/v1.1.2...v2.0.0
[1.1.2]: https://github.com/Esri/koop/compare/v1.1.1...v1.1.2
Expand Down
152 changes: 91 additions & 61 deletions lib/ExportWorker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs = kue.createQueue({
// - ie can this be distributed across more machines?
// - does it need to on the same server always?
jobs.process('exports', 2, function(job, done){
console.log('starting export job', job.id );
console.log('starting export job', job.id);

// Jobs for removing files from the local FS
if (job.data.remove){
Expand All @@ -57,72 +57,102 @@ jobs.process('exports', 2, function(job, done){
});

// if we have VRT file we can use that to create a new export
} else if (fs.existsSync(job.data.files.rootVrtFile) && !job.data.options.ignore_cache){
// since we have a VRT file locally
// we just want to create the export and complete the job
koop.Cache.getInfo(job.data.table, function(err, info){
if (err){
console.log('vrt file exists, but no data in the db', job.data.files.rootVrtFile)
return done('failed to generate export' + err);
} else {
koop.Cache.getCount(job.data.table, job.data.options, function(err, count){

// if we can handle the data in one page
if (count < job.data.options.limit) {
koop.Cache.db.select(job.data.dbkey, job.data.options, function(err, geojson){
delete geojson[0].info;
koop.Exporter.exportToFormat(
job.data.options.format,
job.data.options.dir,
job.data.options.key,
geojson[0],
job.data.options,
function(err, result ){
if (err){
return done( err );
}
finishExport(
job.data.format,
job.data.options.key,
job.data.options,
result,
done
);
});
});
}
info.status = 'processing';
info.generating = {
progress: 100+'%'
};
koop.Cache.updateInfo(job.data.table, info, function(err, res){
// create large file from vrt
var cmd = ['ogr2ogr', '-f', job.data.ogrFormat, '-update', '-append', ( job.data.format == 'zip' ) ? job.data.files.rootNewFileTmp.replace('.zip','') : job.data.files.rootNewFileTmp, job.data.files.rootVrtFile];

try {

var params = {
inFile: job.data.files.rootVrtFile,
outFile: job.data.files.rootNewFileTmp,
paths: job.data.files,
format: job.data.format
// else if we already have a VRT
else if (fs.existsSync(job.data.files.rootVrtFile) && !job.data.options.ignore_cache){
// since we have a VRT file locally
// we just want to create the export and complete the job
koop.Cache.getInfo(job.data.table, function(err, info){
if (err){
console.log('vrt file exists, but no data in the db', job.data.files.rootVrtFile)
return done('failed to generate export' + err);
}
info.status = 'processing';
info.generating = {
progress: 100+'%'
};

koop.Exporter.callOgr(params, null, job.data.options, function(err, formatFile){
if (err){
return done( err );
}
// remove the processing state and return the job
delete info.status;
delete info.generating;
delete info.export_lock;
finishExport( job.data.format, job.data.options.key, job.data.options, { paths: job.data.files, file: formatFile }, function(){
koop.Cache.updateInfo(job.data.table, info, function(e, res){
if (err) {
return done(err);
} else {
if (typeof gc === 'function') {
gc();
}
return done();
koop.Cache.updateInfo(job.data.table, info, function(err, res){
// create large file from vrt
var cmd = ['ogr2ogr', '-f', job.data.ogrFormat, '-update', '-append', ( job.data.format == 'zip' ) ? job.data.files.rootNewFileTmp.replace('.zip','') : job.data.files.rootNewFileTmp, job.data.files.rootVrtFile];

try {

var params = {
inFile: job.data.files.rootVrtFile,
outFile: job.data.files.rootNewFileTmp,
paths: job.data.files,
format: job.data.format
};

koop.Exporter.callOgr(params, null, job.data.options, function(err, formatFile){
if (err){
return done( err );
}
// remove the processing state and return the job
delete info.status;
delete info.generating;
delete info.export_lock;
finishExport( job.data.format, job.data.options.key, job.data.options, { paths: job.data.files, file: formatFile }, function(){
koop.Cache.updateInfo(job.data.table, info, function(e, res){
if (err) {
return done(err);
} else {
if (typeof gc === 'function') {
gc();
}
return done();
}
});
});
});
});
} catch (e){
console.log('error calling org', e);
info.generating = { error: e };
koop.Cache.updateInfo( job.data.table, info, function(err, res){
done('failed to generate file ' + e);
});
}
});
} catch (e){
console.log('error calling org', e);
info.generating = { error: e };
koop.Cache.updateInfo( job.data.table, info, function(err, res){
done('failed to generate file ' + e);
});
} else {
fs.mkdir( job.data.files.base, '0777', true, function(){
koop.Cache.getCount(job.data.table, job.data.options, function(err, count){
var pages = Math.ceil(count / job.data.options.limit);
for (var i=0; i < pages; i++){
var offset = i * (job.data.options.limit);
var chunk = { file: job.data.files.base+'/part.' + i + '.json', offset: offset };
job.data.pages.push( chunk );
}
createFiles( job, done );
});
}
});
});
} else {
fs.mkdir( job.data.files.base, '0777', true, function(){
koop.Cache.getCount(job.data.table, job.data.options, function(err, count){
var pages = Math.ceil(count / job.data.options.limit);
for (var i=0; i < pages; i++){
var offset = i * (job.data.options.limit);
var chunk = { file: job.data.files.base+'/part.' + i + '.json', offset: offset };
job.data.pages.push( chunk );
}
createFiles( job, done );
});
});
}
});
}
});
Expand Down
2 changes: 2 additions & 0 deletions lib/Exporter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ var ogrFormats = {

// exports large data via multi part file strategy
exports.exportLarge = function( koop, format, id, key, type, options, finish, done ){

// exports large data via multi part file strategy
if (!format){
return done('No format provided', null);
Expand Down Expand Up @@ -135,6 +136,7 @@ exports.exportLarge = function( koop, format, id, key, type, options, finish, do
var task = {};
task.options = options;
task.options.key = key;
task.options.dir = dir;
task.dbkey = dbkey;
task.table = table;
task.format = format;
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "koop",
"version": "2.0.1",
"version": "2.0.2",
"description": "A node module/express middleware for converting GeoJSON to Esri Feature Services.",
"main": "index.js",
"scripts": {
Expand Down

0 comments on commit 2483884

Please sign in to comment.