initial demo of new runner-adapter logic with fs adapter

Makefile

@@ -26,12 +26,11 @@ help:
 	@echo '  clean-docker        - Remove docker marker files (docker directory)'
 	@echo ''
 	@echo 'Test targets:'
-	@echo '  test                - Run all tests that do not require docker'
-	@echo '                        images (unit + e2e + blackbox)'
+	@echo '  test                - Run unit and integration tests'
+	@echo '  test-full           - Run unit, integration and system tests'
 	@echo '  unit-test           - Run unit tests'
-	@echo '  e2e-test            - Run end-to-end tests'
-	@echo '  blackbox-test       - Run blackbox tests'
-	@echo '  docker-test         - Run tests requiring docker containers'
+	@echo '  integration-test    - Run integration tests'
+	@echo '  system-test         - Run system tests'
 	@echo '  coverage            - Generate cobertura test coverage reports'
 	@echo '                        (runs all tests)'
 	@echo '  html-coverage       - Generate cobertura html reports'
@@ -51,19 +50,23 @@ help:
 
 .PHONY: test
 test: node_modules
-	$(MOCHA) $(MOCHA_PARAMS) test/e2e test/blackbox test/unit
+	$(MOCHA) $(MOCHA_PARAMS) test/*/integration* test/*/unit*
+
+.PHONY: test-full
+test-full: node_modules
+	$(MOCHA) $(MOCHA_PARAMS) test
 
 .PHONY: unit-test
 unit-test: node_modules
-	$(MOCHA) $(MOCHA_PARAMS) test/unit
+	$(MOCHA) $(MOCHA_PARAMS) test/*/unit*
 
-.PHONY: e2e-test
-e2e-test: node_modules
-	$(MOCHA) $(MOCHA_PARAMS) test/e2e
+.PHONY: integration-test
+integration-test: node_modules
+	$(MOCHA) $(MOCHA_PARAMS) test/*/integration*
 
-.PHONY: blackbox-test
-blackbox-test: node_modules
-	$(MOCHA) $(MOCHA_PARAMS) test/blackbox
+.PHONY: system-test
+system-test: node_modules
+	$(MOCHA) $(MOCHA_PARAMS) test/system
 
 .PHONY: log-delayed
 log-delayed: node_modules

bin/gearslothd

@@ -2,21 +2,15 @@
 
 var config = require('../lib/config');
 var daemon = require('../lib/daemon');
-var log = require('../lib/log');
 
 try {
   var conf = config.initialize(process.argv);
   if (!conf) {
     process.exit(0);
   }
-  conf = config.resolveController(conf);
-  config.initializeDb(conf, function(err, conf) {
-    if (err)
-      throw new Error('Error initializing database adapter:', err.message);
-    daemon.initialize(conf);
-  });
-} catch (err) {
+  daemon.initialize(conf);
+}
+catch (err) {
   console.error('gearslothd:', err.message);
   process.exit(1);
 }
-

debian/changelog

@@ -1,3 +1,34 @@
+gearsloth (0.4.1-1) unstable; urgency=low
+
+  * Show complete payload in debug log
+
+ -- Tuomas Starck <[email protected]>  Tue, 11 Nov 2014 17:10:17 +0200
+
+gearsloth (0.4.0-1) unstable; urgency=low
+
+  * Many fixes for fs-adapter
+  * A little bit less horrible logging
+
+ -- Tuomas Starck <[email protected]>  Fri, 31 Oct 2014 15:25:35 +0000
+
+gearsloth (0.3.6-1) unstable; urgency=low
+
+  * New release after fixes
+
+ -- Tuomas Starck <[email protected]>  Wed, 17 Sep 2014 08:29:29 +0000
+
+gearsloth (0.3.5-1) unstable; urgency=low
+
+  * Another test build for fs-adapter
+
+ -- Tuomas Starck <[email protected]>  Tue, 16 Sep 2014 06:07:08 +0000
+
+gearsloth (0.3.3-1) unstable; urgency=low
+
+  * Test build for fs-adapter deployment
+
+ -- Tuomas Starck <[email protected]>  Fri, 12 Sep 2014 13:18:15 +0000
+
 gearsloth (0.2.4-1) unstable; urgency=low
 
   * Example configurations

debian/gearsloth.postinst

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 
 USERNAME=gearsloth
+FS_ADAPTER_DIR=/usr/lib/gearsloth/fs-adapter
 
 if [ "$1" != "configure" ]; then
     exit 0
@@ -10,4 +11,9 @@ if ! getent passwd $USERNAME >/dev/null; then
     adduser --system --no-create-home $USERNAME
 fi
 
+if [ ! -d $FS_ADAPTER_DIR ]; then
+    mkdir -p $FS_ADAPTER_DIR
+    chown $USERNAME:root $FS_ADAPTER_DIR
+fi
+
 exit 0

debian/gearsloth.postrm

@@ -1,9 +1,12 @@
 #!/bin/sh -e
 
+FS_ADAPTER_DIR=/usr/lib/gearsloth/fs-adapter
+
 case $1 in
     purge)
         rm -f /etc/gearsloth/gearsloth.json
         rmdir --ignore-fail-on-non-empty /etc/gearsloth
+        rmdir --ignore-fail-on-non-empty $FS_ADAPTER_DIR
         deluser --quiet gearsloth >/dev/null 2>&1 || true
     ;;
 esac

debian/gearsloth.upstart

@@ -18,7 +18,7 @@ script
   set -e
   FIFO=`mktemp --dry-run`
   mkfifo $FIFO
-  (logger -t gearsloth -p local7.info <$FIFO &)
+  (logger -t sloth -p local7.info <$FIFO &)
   exec 2>$FIFO
   exec >$FIFO
   rm $FIFO

debian/install

@@ -3,9 +3,8 @@ package.json                 usr/lib/nodejs/gearsloth
 bin                          usr/lib/nodejs/gearsloth
 lib                          usr/lib/nodejs/gearsloth
 
+node_modules/async           usr/lib/nodejs/gearsloth/node_modules
+node_modules/underscore      usr/lib/nodejs/gearsloth/node_modules
 node_modules/gearman-coffee  usr/lib/nodejs/gearsloth/node_modules
-node_modules/mysql           usr/lib/nodejs/gearsloth/node_modules
 
-examples/initialize-mysql.sql         etc/gearsloth
-examples/dual-mysql-and-gearman.json  etc/gearsloth
 examples/simple-sqlite.json           etc/gearsloth

examples/sqlite-adapter-example.js

@@ -17,30 +17,30 @@ function afterInit(err, dbconn) {
     log.debug('sqlite-example', err);
     return;
   }
-	
-	var example_task = {
-//	  at: new Date(),
-	  after: 0,
-	  func_name: 'log',
-	  payload: 'kittehs',
-	  controller:'special',
-	  strategy_options:{
-	    retry:true,
-	    times:3
-	  }
-	};
+
+  var example_task = {
+    // at: new Date(),
+    after: 0,
+    func_name: 'log',
+    payload: 'kittehs',
+    controller:'special',
+    strategy_options:{
+      retry:true,
+      times:3
+    }
+  };
 
   dbconn.saveTask(example_task, function(err) {
-  log.debug('sqlite-example', 'Error:', err.message); 
+  log.debug('sqlite-example', 'Error:', err.message);
     log.debug('sqlite-example', "Task saved: " + new Date());
   });
-  
+
   var stop = dbconn.listenTask(function (err, task) {
       if (err) {
         log.debug('sqlite-example', err);
         return;
       }
-      
+
       log.debug('sqlite-example', "Recieved task:");
       console.log(task);
       console.log(new Date());

lib/adapters/composite.js

@@ -19,19 +19,20 @@ function Compositeadapter(config, databases) {
  *
  * @param {Object} config - Configuration object as passed from config.js
  * @param {Function} callback
- * @param {Object} config_helper - A helper to use for initializing adapters (for testing)
+ * @param {Object} config_helper - A testing helper to use for initializing
  */
 
 function initialize(config, callback, config_helper) {
   config_helper = config_helper || require('../config/index');
 
-  if(!checkConfSanity(config)) {
-    callback(new Error('conf.dbopt is not sane, can\'t initialize composite adapter'));
+  if (!checkConfSanity(config)) {
+    var msg = 'conf.dbopt is not sane, can\'t initialize composite adapter';
+    callback(new Error(msg));
     return;
   }
 
   populateDatabasesArray(config, config_helper, function(err, databases) {
-    if(err)
+    if (err)
       return callback(err);
     var adapter = new Compositeadapter(config, databases);
     callback(null, adapter);
@@ -52,19 +53,22 @@ Compositeadapter.prototype.saveTask = function(task, callback, failcounter) {
   failcounter = isNaN(failcounter) ? 0 : failcounter;
 
   // this prevents the function from being called recursively indefinitely :P
-  if(failcounter >= this._databaseCount()) {
+  if (failcounter >= this._databaseCount()) {
     callback(new Error('No databases available (maybe they are all down?)'));
     return;
   }
 
   var db = this._pickDb();
   db.saveTask(task, function(err, row_id) {
-    if(err) {
-      log.err('composite-adapter', 'Unreachable database \''+ db.db_id +'\': ' + err.message);
-      _this.saveTask(task, callback, ++failcounter);       // Call saveTask recursively until a working
-      return;                                              // database is found.
+    if (err) {
+      var msg = 'Unreachable database ' + db.db_id + ':\n';
+      log.err('composite-adapter', msg + err.message);
+
+      // call saveTask recursively until a working database is found
+      _this.saveTask(task, callback, ++failcounter);
+      return;
     }
-    callback(null, row_id); 
+    callback(null, row_id);
   });
 };
 
@@ -77,7 +81,7 @@ Compositeadapter.prototype.saveTask = function(task, callback, failcounter) {
 Compositeadapter.prototype.listenTask = function(callback) {
   _.each(this._databases, function(db, db_id) {
     db.listenTask(function(err, task) {
-      if(err) callback(err);
+      if (err) callback(err);
       else {
         var task_id = task.id;
         task.id = {
@@ -100,8 +104,8 @@ Compositeadapter.prototype.listenTask = function(callback) {
 
 Compositeadapter.prototype.updateTask = function(task, callback) {
   var db = this._findDbById(task.id.db_id);
-  if(!db) {
-    callback(new Error('There is no database available with id ' + task.id.db_id));
+  if (!db) {
+    callback(new Error('No database available with id ' + task.id.db_id));
     return;
   }
   task.id = task.id.task_id;
@@ -117,8 +121,8 @@ Compositeadapter.prototype.updateTask = function(task, callback) {
 
 Compositeadapter.prototype.completeTask = function(task, callback) {
   var db = this._findDbById(task.id.db_id);
-  if(!db) {
-    callback(new Error('There is no database available with id ' + task.id.db_id));
+  if (!db) {
+    callback(new Error('No database available with id ' + task.id.db_id));
     return;
   }
   task.id = task.id.task_id;
@@ -127,8 +131,8 @@ Compositeadapter.prototype.completeTask = function(task, callback) {
 
 Compositeadapter.prototype.disableTask = function(task, callback) {
   var db = this._findDbById(task.id.db_id);
-  if(!db) {
-    callback(new Error('There is no database available with id ' + task.id.db_id));
+  if (!db) {
+    callback(new Error('No database available with id ' + task.id.db_id));
     return;
   }
   task.id = task.id.task_id;
@@ -154,26 +158,26 @@ Compositeadapter.prototype._databaseCount = function() {
 
 function checkConfSanity(config) {
   return (config.dbopt &&
-           (config.dbopt instanceof Array) &&
-           config.dbopt.length > 0);
+          (config.dbopt instanceof Array) &&
+          config.dbopt.length > 0);
 }
 
 function populateDatabasesArray(config, config_helper, callback) {
   var databases = {};
 
   async.each(config.dbopt, function(dbconf, callback) {
-    if(!dbconf.db || dbconf.db_id === null) {
+    if (!dbconf.db ||  dbconf.db_id === null) {
       callback(new Error('Invalid database config: db or db_id missing'));
     }
     config_helper.initializeDb(dbconf, function(err, augmented_conf) {
-      if(err) {
-        return callback(new Error('Could not initialize a db connection: ' + err.message));
+      if (err) {
+        var msg = 'Could not initialize database connection';
+        return callback(new Error(msg + '\n' + err.message));
       }
       databases[dbconf.db_id] = augmented_conf.dbconn;
       callback(null);
     });
   }, function(err) {
-    callback(err, databases);    
+    callback(err, databases);
   });
 }
-