adds error logging to export worker

koopjs · Nov 18, 2015 · a84bc58 · a84bc58
1 parent aaa854c
commit a84bc58
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+### Changed
+* Increase logging of errors in export worker
+
 ## [2.10.3] - 2015-11-10
 ### Fixed
 * Remove 3 causes of unhandled exceptions in lib/geojson

diff --git a/lib/ExportWorker.js b/lib/ExportWorker.js
@@ -52,6 +52,7 @@ jobs.process('exports', concurrency, function (job, done) {
   // All runtime errors will be funnelled through this function.
   // That's why we throw errors instead of calling the callback
   domain.on('error', function (err) {
+    koop.log.error('Error caught in domain', err)
     handleError(job.data.table, job.data.key, err)
     done(err)
   })
@@ -60,7 +61,10 @@ jobs.process('exports', concurrency, function (job, done) {
     // Jobs for removing files from the local FS
     if (job.data.remove) return remove(job, done)
     cache.getCount(job.data.table, job.data.options, function (err, count) {
-      if (err) throw err
+      if (err) {
+        koop.log.error('Error getting count from the DB', err)
+        throw err
+      }
       // if we can handle the data in one page
       if (count <= job.data.options.limit) {
         singlePage(job, done)
@@ -84,10 +88,10 @@ jobs.process('exports', concurrency, function (job, done) {
  */
 function handleError (table, key, error) {
   cache.getInfo(table, function (err, info) {
-    if (err) koop.log.error(err)
+    if (err) koop.log.error('Error getting info from the db', table, err)
     info.generating[key].error = {message: error.message}
     cache.updateInfo(table, info, function (err) {
-      if (err) koop.log.error(err)
+      if (err) koop.log.error('Error updating info in the db', table, info, err)
     })
   })
 }
@@ -105,7 +109,10 @@ function remove (job, done) {
   var dir = path.join(files.localDir, 'files', (job.data.itemId + '_' + job.data.layerId))
 
   rimraf(dir, function (err) {
-    if (err) throw err
+    if (err) {
+      koop.log.error('Error removing directories from file system', dir)
+      throw err
+    }
     return done()
   })
 }
@@ -123,9 +130,15 @@ function singlePage (job, done) {
   var options = job.data.options
 
   cache.db.select(job.data.dbKey, options, function (err, geojson) {
-    if (err) throw err
+    if (err) {
+      koop.log.error('Error selecting rows from the DB', job.data.dbKey, options, err)
+      throw err
+    }
     koop.Exporter.exportToFormat(format, job.data.dir, key, geojson[0], options, function (err, result) {
-      if (err) throw err
+      if (err) {
+        koop.log.error('Error during single page export', format, key, options, err)
+        throw err
+      }
       done()
       // just send over the pageLength in the third parameter
       // this API is wonky so we just calc the progess on our own
@@ -188,13 +201,16 @@ function createFromVRT (job, done) {
 
   try {
     koop.Exporter.callOgr(params, geojson, job.data.options, function (err, formatFile) {
-      if (err) throw err
+      if (err) {
+        koop.log.error('Error calling OGR with VRT', params, job.data.options, err)
+        throw err
+      }
       var paths = { paths: job.data.paths, file: formatFile }
       done()
       saveFile(job.data.table, job.data.format, job.data.key, job.data.options, paths)
     })
   } catch (e) {
-    koop.log.error('error calling OGR2OGR', e)
+    koop.log.error('error calling OGR2OGR with VRT', params, job.data.options, e)
     throw e
   }
 }
@@ -252,7 +268,10 @@ var workerQ = async.queue(function (task, cb) {
   var filePart = task.file
   // TODO in koop-pgcache 2.0 this will be a geojson feature collection and not and array
   cache.db.select(task.dbKey, opts, function (err, entry) {
-    if (err) throw err
+    if (err) {
+      koop.log.error('Error selecting rows from the DB', task.dbKey, opts, err)
+      throw err
+    }
     if (fs.existsSync(filePart)) fs.unlinkSync(filePart)
 
     fs.writeFile(filePart, JSON.stringify(entry[0]), function () {
@@ -291,13 +310,16 @@ workerQ.drain = function () {
 
   try {
     koop.Exporter.callOgr(params, geojson, job.data.options, function (err, formatFile) {
-      if (err) throw err
+      if (err) {
+        koop.log.error('Error calling OGR with VRT', params, job.data.options, err)
+        throw err
+      }
       var paths = {paths: task.files, file: formatFile}
       done()
       saveFile(task.table, task.format, job.data.key, task.options, paths)
     })
   } catch (e) {
-    koop.log.error('error calling OGR2OGR', e)
+    koop.log.error('error calling OGR2OGR', params, job.data.options, e)
     throw e
   }
 }
@@ -321,7 +343,7 @@ function saveFile (table, format, key, options, result, retried) {
     if (err) {
       // on the first failure retry tthis operation
       if (!retried) return saveFile(table, format, key, options, result, true)
-      koop.log.error('Failed to write file to S3', err)
+      koop.log.error('Failed to write file to S3', table, format, key, err)
       handleError(table, key, new Error('Error while writing to S3'))
     }
     try {
@@ -339,7 +361,7 @@ function saveFile (table, format, key, options, result, retried) {
       }
       files.copy(copyOpts, function (err) {
         if (err) {
-          koop.log.error('Error while writing Latest file to S3', err)
+          koop.log.error('Error while writing Latest file to S3', copyOpts, err)
         }
       })
     }