From 0609b91b01ff56ed59c3b3e7df095470e2ac3d8d Mon Sep 17 00:00:00 2001 From: jottinger Date: Tue, 11 Oct 2016 10:30:27 -0400 Subject: [PATCH 01/11] Ignore Webstorm directories --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ca1e2acbf..4c95cfa33 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ node_modules/ .session.vim coverage.html lib-cov/ + +# Ignore webstorm directories +.idea/ From 061a5c0c87414f7ce125542cb3ee168f5af11ec6 Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Tue, 11 Oct 2016 20:04:22 -0400 Subject: [PATCH 02/11] Initial attempt at index sync, no options for control yet --- .gitignore | 0 lib/Table.js | 67 ++++++++++++++++++++++++++++++++++++++++++++-------- package.json | 3 ++- 3 files changed, 59 insertions(+), 11 deletions(-) mode change 100644 => 100755 .gitignore diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/lib/Table.js b/lib/Table.js index 82e784e80..f55928dec 100644 --- a/lib/Table.js +++ b/lib/Table.js @@ -2,7 +2,7 @@ var Q = require('q'); var debug = require('debug')('dynamoose:table'); var util = require('util'); - +var _ = require('underscore'); function Table(name, schema, options, base) { debug('new Table (%s)', name, schema); @@ -16,18 +16,62 @@ function Table(name, schema, options, base) { } } - Table.prototype.init = function(next) { debug('initializing table, %s, %j', this.name, this.options); var deferred = Q.defer(); var table = this; + var ddb = this.base.ddb(); if (this.options.create) { this.describe() .then(function (data) { debug('table exist -- initialization done'); - // TODO verify table keys and index's match + // verify table keys and index's match + var localTableReq=buildTableReq(table.name, table.schema); + var remoteTableReq=data.Table; + + var localIndexes=localTableReq.GlobalSecondaryIndexes; + var remoteIndexes=remoteTableReq.GlobalSecondaryIndexes; + + for(var i=0;i Date: Wed, 12 Oct 2016 09:07:37 -0400 Subject: [PATCH 03/11] updating to include 'update' option --- Readme.md | 3 +- lib/Table.js | 225 ++++++++++++++++++++++++++------------------------- 2 files changed, 118 insertions(+), 110 deletions(-) mode change 100644 => 100755 Readme.md mode change 100644 => 100755 lib/Table.js diff --git a/Readme.md b/Readme.md old mode 100644 new mode 100755 index 73686d9a0..8d96e408f --- a/Readme.md +++ b/Readme.md @@ -60,7 +60,8 @@ Default `options`: ```js { - create: true, // Create table in DB, if it does not exist + create: true, // Create table in DB, if it does not exist, + update: false, // Update remote indexes if they do not match local index structure waitForActive: true, // Wait for table to be created before trying to us it waitForActiveTimeout: 180000 // wait 3 minutes for table to activate } diff --git a/lib/Table.js b/lib/Table.js old mode 100644 new mode 100755 index f55928dec..a0ac646a6 --- a/lib/Table.js +++ b/lib/Table.js @@ -11,12 +11,12 @@ function Table(name, schema, options, base) { this.options = options || {}; this.base = base; - if(this.options.create === undefined || this.options.create === null) { + if (this.options.create === undefined || this.options.create === null) { this.options.create = true; } } -Table.prototype.init = function(next) { +Table.prototype.init = function (next) { debug('initializing table, %s, %j', this.name, this.options); var deferred = Q.defer(); @@ -25,77 +25,82 @@ Table.prototype.init = function(next) { if (this.options.create) { this.describe() - .then(function (data) { - debug('table exist -- initialization done'); - // verify table keys and index's match - var localTableReq=buildTableReq(table.name, table.schema); - var remoteTableReq=data.Table; - - var localIndexes=localTableReq.GlobalSecondaryIndexes; - var remoteIndexes=remoteTableReq.GlobalSecondaryIndexes; - - for(var i=0;i timeoutAt) { + if (Date.now() > timeoutAt) { return deferred.reject( new Error('Wait for Active timed out after ' + timeout + ' ms.') ); } - if(!table.initialized) { + if (!table.initialized) { return setTimeout(waitForActive, 10); } table.describe() - .then(function (data) { - if(data.Table.TableStatus !== 'ACTIVE'){ - debug('Waiting for Active - %s', table.name); - setTimeout(waitForActive, 1000); - } else { - // TODO verify table keys and index's match - table.active = true; - deferred.resolve(); - } - }, function (err) { - return deferred.reject(err); - }); + .then(function (data) { + if (data.Table.TableStatus !== 'ACTIVE') { + debug('Waiting for Active - %s', table.name); + setTimeout(waitForActive, 1000); + } else { + // TODO verify table keys and index's match + table.active = true; + deferred.resolve(); + } + }, function (err) { + return deferred.reject(err); + }); } waitForActive(); @@ -152,7 +158,7 @@ Table.prototype.waitForActive = function(timeout, next) { return deferred.promise.nodeify(next); }; -Table.prototype.describe = function(next) { +Table.prototype.describe = function (next) { var describeTableReq = { TableName: this.name }; @@ -162,8 +168,8 @@ Table.prototype.describe = function(next) { var deferred = Q.defer(); var ddb = this.base.ddb(); - ddb.describeTable(describeTableReq, function(err, data) { - if(err) { + ddb.describeTable(describeTableReq, function (err, data) { + if (err) { debug('error describing table', err); return deferred.reject(err); } @@ -175,30 +181,31 @@ Table.prototype.describe = function(next) { return deferred.promise.nodeify(next); }; -var buildTableReq=function buildTableReq(name, schema) { +var buildTableReq = function buildTableReq(name, schema) { var attrDefs = []; var keyAttr = {}; - function addKeyAttr (attr) { - if(attr) { + + function addKeyAttr(attr) { + if (attr) { keyAttr[attr.name] = attr; } } addKeyAttr(schema.hashKey); addKeyAttr(schema.rangeKey); - for(var globalIndexName in schema.indexes.global) { + for (var globalIndexName in schema.indexes.global) { addKeyAttr(schema.indexes.global[globalIndexName]); // add the range key to the attribute definitions if specified var rangeKeyName = schema.indexes.global[globalIndexName].indexes[globalIndexName].rangeKey; addKeyAttr(schema.attributes[rangeKeyName]); } - for(var indexName in schema.indexes.local) { + for (var indexName in schema.indexes.local) { addKeyAttr(schema.indexes.local[indexName]); } - for(var keyAttrName in keyAttr) { + for (var keyAttrName in keyAttr) { attrDefs.push({ AttributeName: keyAttrName, AttributeType: keyAttr[keyAttrName].type.dynamo @@ -210,7 +217,7 @@ var buildTableReq=function buildTableReq(name, schema) { AttributeName: schema.hashKey.name, KeyType: 'HASH' }]; - if(schema.rangeKey) { + if (schema.rangeKey) { keySchema.push({ AttributeName: schema.rangeKey.name, KeyType: 'RANGE' @@ -231,7 +238,7 @@ var buildTableReq=function buildTableReq(name, schema) { debug('Creating table local indexes', schema.indexes.local); var localSecIndexes, index; - for(var localSecIndexName in schema.indexes.local) { + for (var localSecIndexName in schema.indexes.local) { localSecIndexes = localSecIndexes || []; var indexAttr = schema.indexes.local[localSecIndexName]; @@ -247,8 +254,8 @@ var buildTableReq=function buildTableReq(name, schema) { }] }; - if(index.project) { - if(util.isArray(index.project)){ + if (index.project) { + if (util.isArray(index.project)) { localSecIndex.Projection = { ProjectionType: 'INCLUDE', NonKeyAttributes: index.project @@ -269,7 +276,7 @@ var buildTableReq=function buildTableReq(name, schema) { var globalSecIndexes; - for(var globalSecIndexName in schema.indexes.global) { + for (var globalSecIndexName in schema.indexes.global) { globalSecIndexes = globalSecIndexes || []; var globalIndexAttr = schema.indexes.global[globalSecIndexName]; @@ -288,15 +295,15 @@ var buildTableReq=function buildTableReq(name, schema) { }; - if(index.rangeKey) { + if (index.rangeKey) { globalSecIndex.KeySchema.push({ AttributeName: index.rangeKey, KeyType: 'RANGE' }); } - if(index.project) { - if(util.isArray(index.project)){ + if (index.project) { + if (util.isArray(index.project)) { globalSecIndex.Projection = { ProjectionType: 'INCLUDE', NonKeyAttributes: index.project @@ -315,18 +322,18 @@ var buildTableReq=function buildTableReq(name, schema) { globalSecIndexes.push(globalSecIndex); } - if(localSecIndexes) { + if (localSecIndexes) { createTableReq.LocalSecondaryIndexes = localSecIndexes; } - if(globalSecIndexes) { + if (globalSecIndexes) { createTableReq.GlobalSecondaryIndexes = globalSecIndexes; } return createTableReq; }; -Table.prototype.create = function(next) { +Table.prototype.create = function (next) { var ddb = this.base.ddb(); var schema = this.schema; var createTableReq = buildTableReq(this.name, schema); @@ -335,8 +342,8 @@ Table.prototype.create = function(next) { var deferred = Q.defer(); - ddb.createTable(createTableReq, function(err, data) { - if(err) { + ddb.createTable(createTableReq, function (err, data) { + if (err) { debug('error creating table', err); return deferred.reject(err); } @@ -347,7 +354,7 @@ Table.prototype.create = function(next) { }; -Table.prototype.delete = function(next) { +Table.prototype.delete = function (next) { var deleteTableReq = { TableName: this.name }; @@ -358,8 +365,8 @@ Table.prototype.delete = function(next) { var deferred = Q.defer(); - ddb.deleteTable(deleteTableReq, function(err, data) { - if(err) { + ddb.deleteTable(deleteTableReq, function (err, data) { + if (err) { debug('error deleting table', err); return deferred.reject(err); } @@ -371,7 +378,7 @@ Table.prototype.delete = function(next) { }; -Table.prototype.update = function(next) { +Table.prototype.update = function (next) { // var ddb = this.base.ddb(); // ddb.updateTable(); var deferred = Q.defer(); From b22d8740b932183e0a3a1fd0eba70c634e74eceb Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Wed, 12 Oct 2016 09:45:55 -0400 Subject: [PATCH 04/11] Correcting detection of index, trimming unnecessary field from remote --- lib/Table.js | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/Table.js b/lib/Table.js index a0ac646a6..2a617bf08 100755 --- a/lib/Table.js +++ b/lib/Table.js @@ -35,16 +35,27 @@ Table.prototype.init = function (next) { var remoteIndexes = remoteTableReq.GlobalSecondaryIndexes; for (var i = 0; i < localIndexes.length; i++) { + debug('checking index %s', localIndexes[i].IndexName); var remoteMatches = false; for (var j = 0; j < remoteIndexes.length; j++) { if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { // let's see if the core data matches. if it doesn't, // we may need to delete the remote GSI and rebuild. - if (_.isEqual(_.pick(remoteIndexes[j], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'), - _.pick(localIndexes[i], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'))) { - if (this.options.update) { + var localIndex = _.pick(localIndexes[i], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); + debug('local index: %j', localIndex); + var remoteIndex = _.pick(remoteIndexes[j], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); + if(remoteIndex.hasOwnProperty('ProvisionedThroughput')) { + delete remoteIndex.ProvisionedThroughput.NumberOfDecreasesToday; + } + debug('remote index: %j', remoteIndex); + + debug('is equal: %s', _.isEqual(remoteIndex, localIndex)); + if (!_.isEqual(remoteIndex, localIndex)) { + debug('remote and local index do not match'); + if (table.options.update) { // need to delete the other index if update is set ddb.UpdateTable({ + TableName: table.name, GlobalSecondaryIndexUpdates: [ { Delete: { @@ -62,8 +73,10 @@ Table.prototype.init = function (next) { } if (!remoteMatches) { // need to create remote index - if (this.options.update) { + if (table.options.update) { + debug('creating index %s', localIndexes[i].IndexName) ddb.UpdateTable({ + TableName: table.name, GlobalSecondaryIndexUpdates: [ { Create: localIndexes[i] From 8c4c0d91ce99374d1685ada216f0179a6365e50b Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Wed, 12 Oct 2016 15:59:17 -0400 Subject: [PATCH 05/11] Updates for Table.js, including test --- lib/Table.js | 162 +++++++++++++++++++++++++++++++------------------- test/Table.js | 133 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+), 60 deletions(-) mode change 100644 => 100755 test/Table.js diff --git a/lib/Table.js b/lib/Table.js index 2a617bf08..8fb1cc245 100755 --- a/lib/Table.js +++ b/lib/Table.js @@ -16,6 +16,86 @@ function Table(name, schema, options, base) { } } +var compareIndexes = function compareIndexes(local, remote) { + var indexes = { + delete: [], + create: [] + }; + var localTableReq = local; + var remoteTableReq = remote; + + var localIndexes = localTableReq.GlobalSecondaryIndexes; + var remoteIndexes = remoteTableReq.GlobalSecondaryIndexes; + + for (var i = 0; i < localIndexes.length; i++) { + var remoteMatches = false; + for (var j = 0; j < remoteIndexes.length; j++) { + if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { + // let's see if the core data matches. if it doesn't, + // we may need to delete the remote GSI and rebuild. + var localIndex = _.pick(localIndexes[i], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); + var remoteIndex = _.pick(remoteIndexes[j], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); + if (remoteIndex.hasOwnProperty('ProvisionedThroughput')) { + delete remoteIndex.ProvisionedThroughput.NumberOfDecreasesToday; + } + + if (!_.isEqual(remoteIndex, localIndex)) { + indexes.delete.push(localIndex); + } else { + remoteMatches = true; + } + } + } + if (!remoteMatches) { + indexes.create.push(localIndexes[i]); + } + } + return indexes; +}; + +var deleteIndex = function deleteIndex(table, ddb, tableName, indexName) { + debug('deleting index %s.%s', tableName, indexName); + var params = { + TableName: tableName, + GlobalSecondaryIndexUpdates: [ + { + Delete: { + IndexName: indexName + } + } + ] + }; + ddb.updateTable(params, function (err) { + if (err) { + console.log(JSON.stringify(err)); + } + else { + table.waitForActive(); + } + }); +}; + +var createIndex = function createIndex(table, ddb, tableName, attributes, indexSpec) { + debug('creating index %s.%s', tableName, indexSpec.IndexName); + var params = { + TableName: tableName, + AttributeDefinitions: attributes, + GlobalSecondaryIndexUpdates: [ + { + Create: indexSpec + } + ] + }; + ddb.updateTable(params, function (err) { + if (err) { + console.log(JSON.stringify(err)); + } + else { + table.waitForActive(); + } + }); +}; + Table.prototype.init = function (next) { debug('initializing table, %s, %j', this.name, this.options); var deferred = Q.defer(); @@ -27,66 +107,18 @@ Table.prototype.init = function (next) { this.describe() .then(function (data) { debug('table exist -- initialization done'); - // verify table keys and index's match var localTableReq = buildTableReq(table.name, table.schema); - var remoteTableReq = data.Table; - - var localIndexes = localTableReq.GlobalSecondaryIndexes; - var remoteIndexes = remoteTableReq.GlobalSecondaryIndexes; - - for (var i = 0; i < localIndexes.length; i++) { - debug('checking index %s', localIndexes[i].IndexName); - var remoteMatches = false; - for (var j = 0; j < remoteIndexes.length; j++) { - if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { - // let's see if the core data matches. if it doesn't, - // we may need to delete the remote GSI and rebuild. - var localIndex = _.pick(localIndexes[i], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); - debug('local index: %j', localIndex); - var remoteIndex = _.pick(remoteIndexes[j], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); - if(remoteIndex.hasOwnProperty('ProvisionedThroughput')) { - delete remoteIndex.ProvisionedThroughput.NumberOfDecreasesToday; - } - debug('remote index: %j', remoteIndex); - - debug('is equal: %s', _.isEqual(remoteIndex, localIndex)); - if (!_.isEqual(remoteIndex, localIndex)) { - debug('remote and local index do not match'); - if (table.options.update) { - // need to delete the other index if update is set - ddb.UpdateTable({ - TableName: table.name, - GlobalSecondaryIndexUpdates: [ - { - Delete: { - IndexName: remoteIndexes[j].IndexName - } - } - ] - }); - waitForActive(); - } - } else { - remoteMatches = true; - } - } + var indexes = compareIndexes(localTableReq, data.Table); + if (table.options.update) { + for (var deleteIdx in indexes.delete) { + deleteIndex(table, ddb, table.name, indexes.delete[deleteIdx].IndexName); } - if (!remoteMatches) { - // need to create remote index - if (table.options.update) { - debug('creating index %s', localIndexes[i].IndexName) - ddb.UpdateTable({ - TableName: table.name, - GlobalSecondaryIndexUpdates: [ - { - Create: localIndexes[i] - } - ] - }); - waitForActive(); - } else { - debug('Local index %s does not match remote index; update is not turned on', localIndexes[i].IndexName); - } + for (var createIdx in indexes.create) { + createIndex(table, ddb, table.name, localTableReq.AttributeDefinitions, indexes.create[createIdx]); + } + } else { + if (indexes.delete.length > 0 || indexes.create.length > 0) { + debug('indexes are not synchronized and update flag is set to false'); } } @@ -151,13 +183,23 @@ Table.prototype.waitForActive = function (timeout, next) { if (!table.initialized) { return setTimeout(waitForActive, 10); } + var localTableReq = buildTableReq(table.name, table.schema); + table.describe() .then(function (data) { - if (data.Table.TableStatus !== 'ACTIVE') { + var active = (data.Table.TableStatus === 'ACTIVE'); + if(table.options.update) { + var indexes = compareIndexes(localTableReq, data.Table); + debug('%j', indexes); + if (indexes.delete.length > 0 || indexes.create.length > 0) { + debug('indexes are not synchronized and update flag is set to false'); + active = false; + } + } + if (!active) { debug('Waiting for Active - %s', table.name); setTimeout(waitForActive, 1000); } else { - // TODO verify table keys and index's match table.active = true; deferred.resolve(); } diff --git a/test/Table.js b/test/Table.js old mode 100644 new mode 100755 index 9a0ea818b..3b3bf7da0 --- a/test/Table.js +++ b/test/Table.js @@ -110,4 +110,137 @@ describe('Table tests', function (){ done(); }); }); + + it('Support index migration', function (done) { + var Song = dynamoose.model('DMSong', { + id: { + type: Number, + required: true, + hashKey: true, + }, + band: { + type: String, + required: true, + trim: true + }, + album: { + type: String, + required: true, + trim: true, + index: { + global: true, + rangeKey: 'id', + name: 'albumIndex', + project: ['band', 'album'], + throughput: 5 // read and write are both 5 + } + }, + song: { + type: String, + required: true, + trim: true, + index: { + global: true, + rangeKey: 'id', + name: 'songIndex', + project: true, // ProjectionType: ALL + throughput: 5 // read and write are both 5 + } + }, + track: { + type: Number, + required: false, + } + }, + { + create: true, update: true + }); + var tom_sawyer = new Song({id: 1, band: 'Rush', album: 'Moving Pictures', song: 'Tom Sawyer', track: 1}); + tom_sawyer.save(); + var params = {TableName: 'DMSong'}; + dynamoose.ddb().describeTable(params, function (err, data) { + if (err) { + done(err); + } + else { + var found=false; + for(var i in data.Table.GlobalSecondaryIndexes) { + var gsi=data.Table.GlobalSecondaryIndexes[i]; + if(gsi.IndexName === 'albumIndex') { + should.equal(gsi.Projection.ProjectionType, 'INCLUDE'); + found=true; + } + } + should.equal(found, true); + delete dynamoose.models['DMSong']; + var NewSong = dynamoose.model('DMSong', { + id: { + type: Number, + required: true, + hashKey: true, + }, + band: { + type: String, + required: true, + trim: true + }, + album: { + type: String, + required: true, + trim: true, + index: { + global: true, + rangeKey: 'id', + name: 'albumIndex', + project: true, // ProjectionType: ALL + throughput: 5 // read and write are both 5 + } + }, + song: { + type: String, + required: true, + trim: true, + index: { + global: true, + rangeKey: 'id', + name: 'songIndex', + project: true, // ProjectionType: ALL + throughput: 5 // read and write are both 5 + } + }, + track: { + type: Number, + required: false, + } + }, + { + create: true, + update: true, + waitForActive: true + }); + + var red_barchetta = new NewSong({id: 2, band: 'Rush', album: 'Moving Pictures', song: 'Red Barchetta', track: 2}); + red_barchetta.save(); + dynamoose.ddb().describeTable(params, function (err, data) { + if (err) { + done(err); + } + else { + console.log("---------------------REVISED TABLE"); + console.log(JSON.stringify(data, null, 2)); + found=false; + for(var i in data.Table.GlobalSecondaryIndexes) { + var gsi=data.Table.GlobalSecondaryIndexes[i]; + if(gsi.IndexName === 'albumIndex') { + should.equal(gsi.Projection.ProjectionType, 'ALL'); + found=true; + } + } + should.equal(found, true); + done(); + } + }); + } + }); + }); }); From 6da21efa18d59b8e0518a3f52c173ce01d160471 Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Thu, 13 Oct 2016 13:01:28 -0400 Subject: [PATCH 06/11] Converting to promises --- lib/Table.js | 116 ++++++++++++++---------- test/Table.js | 246 ++++++++++++++++++++++++++------------------------ 2 files changed, 194 insertions(+), 168 deletions(-) diff --git a/lib/Table.js b/lib/Table.js index 8fb1cc245..4496727eb 100755 --- a/lib/Table.js +++ b/lib/Table.js @@ -17,6 +17,7 @@ function Table(name, schema, options, base) { } var compareIndexes = function compareIndexes(local, remote) { + var q = Q.defer(); var indexes = { delete: [], create: [] @@ -27,6 +28,7 @@ var compareIndexes = function compareIndexes(local, remote) { var localIndexes = localTableReq.GlobalSecondaryIndexes; var remoteIndexes = remoteTableReq.GlobalSecondaryIndexes; + debug('compareIndexes'); for (var i = 0; i < localIndexes.length; i++) { var remoteMatches = false; for (var j = 0; j < remoteIndexes.length; j++) { @@ -50,11 +52,12 @@ var compareIndexes = function compareIndexes(local, remote) { indexes.create.push(localIndexes[i]); } } - return indexes; + q.resolve(indexes); + return q.promise; }; -var deleteIndex = function deleteIndex(table, ddb, tableName, indexName) { - debug('deleting index %s.%s', tableName, indexName); +var deleteIndex = function deleteIndex(table, ddb, tableName, indexName, next) { + var q = Q.defer(); var params = { TableName: tableName, GlobalSecondaryIndexUpdates: [ @@ -65,18 +68,20 @@ var deleteIndex = function deleteIndex(table, ddb, tableName, indexName) { } ] }; + debug('deleting index %s.%s', tableName, indexName); ddb.updateTable(params, function (err) { if (err) { - console.log(JSON.stringify(err)); + q.reject(err); } else { - table.waitForActive(); + q.resolve(); } }); + return q.promise.nodeify(next); }; -var createIndex = function createIndex(table, ddb, tableName, attributes, indexSpec) { - debug('creating index %s.%s', tableName, indexSpec.IndexName); +var createIndex = function createIndex(table, ddb, tableName, attributes, indexSpec, next) { + var q = Q.defer(); var params = { TableName: tableName, AttributeDefinitions: attributes, @@ -86,14 +91,16 @@ var createIndex = function createIndex(table, ddb, tableName, attributes, indexS } ] }; + debug('creating index %s.%s', tableName, indexSpec.IndexName); ddb.updateTable(params, function (err) { if (err) { - console.log(JSON.stringify(err)); + q.reject(err); } else { - table.waitForActive(); + q.resolve(); } }); + return q.promise.nodeify(next); }; Table.prototype.init = function (next) { @@ -102,50 +109,63 @@ Table.prototype.init = function (next) { var table = this; var ddb = this.base.ddb(); - + var localTableReq; + var remoteData; if (this.options.create) { this.describe() .then(function (data) { - debug('table exist -- initialization done'); - var localTableReq = buildTableReq(table.name, table.schema); - var indexes = compareIndexes(localTableReq, data.Table); - if (table.options.update) { - for (var deleteIdx in indexes.delete) { - deleteIndex(table, ddb, table.name, indexes.delete[deleteIdx].IndexName); - } - for (var createIdx in indexes.create) { - createIndex(table, ddb, table.name, localTableReq.AttributeDefinitions, indexes.create[createIdx]); - } - } else { - if (indexes.delete.length > 0 || indexes.create.length > 0) { - debug('indexes are not synchronized and update flag is set to false'); - } + debug('table exist -- initialization done'); + localTableReq = buildTableReq(table.name, table.schema); + remoteData=data; + return compareIndexes(localTableReq, data.Table); + }) + .then(function (indexes) { + debug('%j', indexes); + var updates = []; + if (table.options.update) { + debug('checking indexes'); + for (var deleteIdx in indexes.delete) { + debug('need to delete %s', indexes.delete[deleteIdx].IndexName); + updates.push(deleteIndex(table, ddb, table.name, indexes.delete[deleteIdx].IndexName)); + updates.push(table.waitForActive()); } - - table.active = data.Table.TableStatus === 'ACTIVE'; - table.initialized = true; - return deferred.resolve(); - }, - function (err) { - if (err && err.code === 'ResourceNotFoundException') { - debug('table does not exist -- creating'); - return deferred.resolve( - table.create() - .then(function () { - table.initialized = true; - }) - // .then(function() { - // if(table.options.waitForActive) { - // return table.waitForActive(); - // } - // }) - ); + for (var createIdx in indexes.create) { + debug('need to create %s', indexes.create[createIdx].IndexName); + updates.push(createIndex(table, ddb, table.name, localTableReq.AttributeDefinitions, indexes.create[createIdx])); + updates.push(table.waitForActive()); } - if (err) { - debug('error initializing', err); - return deferred.reject(err); + } else { + if (indexes.delete.length > 0 || indexes.create.length > 0) { + debug('indexes are not synchronized and update flag is set to false'); + deferred.reject('indexes are not synchronized and update flag is set to false'); } - }); + } + + table.active = remoteData.Table.TableStatus === 'ACTIVE'; + table.initialized = true; + debug('updates: %j', updates); + return deferred.resolve(Q.all(updates)); + }) + .catch(function (err) { + if (err && err.code === 'ResourceNotFoundException') { + debug('table does not exist -- creating'); + return deferred.resolve( + table.create() + .then(function () { + table.initialized = true; + }) + // .then(function() { + // if(table.options.waitForActive) { + // return table.waitForActive(); + // } + // }) + ); + } + if (err) { + debug('error initializing', err); + return deferred.reject(err); + } + }); } else { table.initialized = true; return deferred.resolve(); @@ -188,7 +208,7 @@ Table.prototype.waitForActive = function (timeout, next) { table.describe() .then(function (data) { var active = (data.Table.TableStatus === 'ACTIVE'); - if(table.options.update) { + if (table.options.update) { var indexes = compareIndexes(localTableReq, data.Table); debug('%j', indexes); if (indexes.delete.length > 0 || indexes.create.length > 0) { diff --git a/test/Table.js b/test/Table.js index 3b3bf7da0..07f3226bb 100755 --- a/test/Table.js +++ b/test/Table.js @@ -16,14 +16,16 @@ var Table = dynamoose.Table; var should = require('should'); -describe('Table tests', function (){ +describe('Table tests', function () { this.timeout(5000); - var schema = new Schema({ id: Number, name: String, childern: [Number], address: { street:String,city:String} }); + var schema = new Schema({id: Number, name: String, childern: [Number], address: {street: String, city: String}}); var globalIndexSchema = new Schema({ ownerId: { type: Number, - validate: function(v) { return v > 0; }, + validate: function (v) { + return v > 0; + }, hashKey: true }, breed: { @@ -58,7 +60,7 @@ describe('Table tests', function (){ it('Create simple table', function (done) { - table.create(function(err) { + table.create(function (err) { should.not.exist(err); done(); }); @@ -66,7 +68,7 @@ describe('Table tests', function (){ it('Describe simple table', function (done) { - table.describe(function(err, data) { + table.describe(function (err, data) { should.not.exist(err); should.exist(data); done(); @@ -75,7 +77,7 @@ describe('Table tests', function (){ it('Delete simple table', function (done) { - table.delete(function(err, data) { + table.delete(function (err, data) { should.not.exist(err); should.exist(data); done(); @@ -86,7 +88,7 @@ describe('Table tests', function (){ var missing = new Table('missing', schema, null, dynamoose); - missing.describe(function(err, data) { + missing.describe(function (err, data) { should.exist(err); should.not.exist(data); err.code.should.eql('ResourceNotFoundException'); @@ -96,7 +98,7 @@ describe('Table tests', function (){ it('Create table with global index with non indexed range key', function (done) { - globalIndexTable.create(function(err) { + globalIndexTable.create(function (err) { should.not.exist(err); done(); }); @@ -104,57 +106,57 @@ describe('Table tests', function (){ it('Delete table with global index', function (done) { - globalIndexTable.delete(function(err, data) { + globalIndexTable.delete(function (err, data) { should.not.exist(err); should.exist(data); done(); }); }); - it('Support index migration', function (done) { + it('create DMSong with limited projection', function (done) { var Song = dynamoose.model('DMSong', { - id: { - type: Number, - required: true, - hashKey: true, - }, - band: { - type: String, - required: true, - trim: true - }, - album: { - type: String, - required: true, - trim: true, - index: { - global: true, - rangeKey: 'id', - name: 'albumIndex', - project: ['band', 'album'], - throughput: 5 // read and write are both 5 - } - }, - song: { - type: String, - required: true, - trim: true, - index: { - global: true, - rangeKey: 'id', - name: 'songIndex', - project: true, // ProjectionType: ALL - throughput: 5 // read and write are both 5 - } - }, - track: { - type: Number, - required: false, + id: { + type: Number, + required: true, + hashKey: true, + }, + band: { + type: String, + required: true, + trim: true + }, + album: { + type: String, + required: true, + trim: true, + index: { + global: true, + rangeKey: 'id', + name: 'albumIndex', + project: ['band', 'album'], + throughput: 5 // read and write are both 5 + } + }, + song: { + type: String, + required: true, + trim: true, + index: { + global: true, + rangeKey: 'id', + name: 'songIndex', + project: true, // ProjectionType: ALL + throughput: 5 // read and write are both 5 } }, - { - create: true, update: true - }); + track: { + type: Number, + required: false, + } + }, + { + create: true, update: true + }); var tom_sawyer = new Song({id: 1, band: 'Rush', album: 'Moving Pictures', song: 'Tom Sawyer', track: 1}); tom_sawyer.save(); var params = {TableName: 'DMSong'}; @@ -163,83 +165,87 @@ describe('Table tests', function (){ done(err); } else { - var found=false; - for(var i in data.Table.GlobalSecondaryIndexes) { - var gsi=data.Table.GlobalSecondaryIndexes[i]; - if(gsi.IndexName === 'albumIndex') { + var found = false; + for (var i in data.Table.GlobalSecondaryIndexes) { + var gsi = data.Table.GlobalSecondaryIndexes[i]; + if (gsi.IndexName === 'albumIndex') { should.equal(gsi.Projection.ProjectionType, 'INCLUDE'); - found=true; + found = true; } } should.equal(found, true); delete dynamoose.models['DMSong']; - var NewSong = dynamoose.model('DMSong', { - id: { - type: Number, - required: true, - hashKey: true, - }, - band: { - type: String, - required: true, - trim: true - }, - album: { - type: String, - required: true, - trim: true, - index: { - global: true, - rangeKey: 'id', - name: 'albumIndex', - project: true, // ProjectionType: ALL - throughput: 5 // read and write are both 5 - } - }, - song: { - type: String, - required: true, - trim: true, - index: { - global: true, - rangeKey: 'id', - name: 'songIndex', - project: true, // ProjectionType: ALL - throughput: 5 // read and write are both 5 - } - }, - track: { - type: Number, - required: false, - } - }, - { - create: true, - update: true, - waitForActive: true - }); - - var red_barchetta = new NewSong({id: 2, band: 'Rush', album: 'Moving Pictures', song: 'Red Barchetta', track: 2}); - red_barchetta.save(); - dynamoose.ddb().describeTable(params, function (err, data) { - if (err) { - done(err); + done(); + } + }); + }); + it('update DMSong with broader projection', function (done) { + var params = {TableName: 'DMSong'}; + var Song = dynamoose.model('DMSong', { + id: { + type: Number, + required: true, + hashKey: true, + }, + band: { + type: String, + required: true, + trim: true + }, + album: { + type: String, + required: true, + trim: true, + index: { + global: true, + rangeKey: 'id', + name: 'albumIndex', + project: true, // ProjectionType: ALL + throughput: 5 // read and write are both 5 } - else { - console.log("---------------------REVISED TABLE"); - console.log(JSON.stringify(data, null, 2)); - found=false; - for(var i in data.Table.GlobalSecondaryIndexes) { - var gsi=data.Table.GlobalSecondaryIndexes[i]; - if(gsi.IndexName === 'albumIndex') { - should.equal(gsi.Projection.ProjectionType, 'ALL'); - found=true; - } - } - should.equal(found, true); - done(); + }, + song: { + type: String, + required: true, + trim: true, + index: { + global: true, + rangeKey: 'id', + name: 'songIndex', + project: true, // ProjectionType: ALL + throughput: 5 // read and write are both 5 } - }); + }, + track: { + type: Number, + required: false, + } + }, + { + create: true, + update: true, + waitForActive: true + }); + + var red_barchetta = new Song({id: 2, band: 'Rush', album: 'Moving Pictures', song: 'Red Barchetta', track: 2}); + red_barchetta.save(); + dynamoose.ddb().describeTable(params, function (err, data) { + if (err) { + done(err); + } + else { + console.log("---------------------REVISED TABLE"); + console.log(JSON.stringify(data, null, 2)); + var found = false; + for (var i in data.Table.GlobalSecondaryIndexes) { + var gsi = data.Table.GlobalSecondaryIndexes[i]; + if (gsi.IndexName === 'albumIndex') { + should.equal(gsi.Projection.ProjectionType, 'ALL'); + found = true; + } + } + should.equal(found, true); + done(); } }); }); From 2704dd9798ab885a6b60f1b566875a42c29f4ac5 Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Thu, 13 Oct 2016 13:19:49 -0400 Subject: [PATCH 07/11] removing promises from compareIndexes --- lib/Table.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/Table.js b/lib/Table.js index 4496727eb..9beeb6859 100755 --- a/lib/Table.js +++ b/lib/Table.js @@ -17,7 +17,6 @@ function Table(name, schema, options, base) { } var compareIndexes = function compareIndexes(local, remote) { - var q = Q.defer(); var indexes = { delete: [], create: [] @@ -52,8 +51,7 @@ var compareIndexes = function compareIndexes(local, remote) { indexes.create.push(localIndexes[i]); } } - q.resolve(indexes); - return q.promise; + return indexes; }; var deleteIndex = function deleteIndex(table, ddb, tableName, indexName, next) { @@ -110,16 +108,14 @@ Table.prototype.init = function (next) { var table = this; var ddb = this.base.ddb(); var localTableReq; - var remoteData; + if (this.options.create) { this.describe() .then(function (data) { debug('table exist -- initialization done'); localTableReq = buildTableReq(table.name, table.schema); - remoteData=data; - return compareIndexes(localTableReq, data.Table); - }) - .then(function (indexes) { + var indexes=compareIndexes(localTableReq, data.Table); + debug('%j', indexes); var updates = []; if (table.options.update) { @@ -141,7 +137,7 @@ Table.prototype.init = function (next) { } } - table.active = remoteData.Table.TableStatus === 'ACTIVE'; + table.active = data.Table.TableStatus === 'ACTIVE'; table.initialized = true; debug('updates: %j', updates); return deferred.resolve(Q.all(updates)); From 2f93dcec41cdc092e187b4d9d1bb3c6a65512187 Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Fri, 14 Oct 2016 15:40:31 -0400 Subject: [PATCH 08/11] trying to reorganize such that it all makes sense and ... works --- lib/Table.js | 128 ++++++++++++++++++++++++++++++-------------------- test/Table.js | 73 ++++++++++++++-------------- 2 files changed, 116 insertions(+), 85 deletions(-) diff --git a/lib/Table.js b/lib/Table.js index 9beeb6859..d23bfc8b2 100755 --- a/lib/Table.js +++ b/lib/Table.js @@ -1,6 +1,7 @@ 'use strict'; var Q = require('q'); var debug = require('debug')('dynamoose:table'); +var trace = require('debug')('dynamoose:index'); var util = require('util'); var _ = require('underscore'); @@ -19,7 +20,8 @@ function Table(name, schema, options, base) { var compareIndexes = function compareIndexes(local, remote) { var indexes = { delete: [], - create: [] + create: [], + both: [] }; var localTableReq = local; var remoteTableReq = remote; @@ -28,8 +30,9 @@ var compareIndexes = function compareIndexes(local, remote) { var remoteIndexes = remoteTableReq.GlobalSecondaryIndexes; debug('compareIndexes'); + // let's see what remote indexes we need to sync or create for (var i = 0; i < localIndexes.length; i++) { - var remoteMatches = false; + var remoteIndexFound = false; for (var j = 0; j < remoteIndexes.length; j++) { if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { // let's see if the core data matches. if it doesn't, @@ -41,23 +44,41 @@ var compareIndexes = function compareIndexes(local, remote) { } if (!_.isEqual(remoteIndex, localIndex)) { - indexes.delete.push(localIndex); + indexes.both.push(localIndex); + remoteIndexFound = true; } else { - remoteMatches = true; + remoteIndexFound = true; } } } - if (!remoteMatches) { + if (!remoteIndexFound) { indexes.create.push(localIndexes[i]); } } + // now let's see what remote indexes exist that shouldn't exist + for (j = 0; j < remoteIndexes.length; j++) { + var localExists = false; + for (i = 0; i < localIndexes.length; i++) { + if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { + localExists = true; + } + } + if (!localExists) { + indexes.delete.push(remoteIndexes[j]); + } + } + return indexes; }; -var deleteIndex = function deleteIndex(table, ddb, tableName, indexName, next) { - var q = Q.defer(); +Table.prototype.deleteIndex = function deleteIndex(indexName) { + debug('deleting index %s', indexName); + var deferred = Q.defer(); + var table = this; + table.active = false; + var ddb = this.base.ddb(); var params = { - TableName: tableName, + TableName: table.name, GlobalSecondaryIndexUpdates: [ { Delete: { @@ -66,22 +87,30 @@ var deleteIndex = function deleteIndex(table, ddb, tableName, indexName, next) { } ] }; - debug('deleting index %s.%s', tableName, indexName); - ddb.updateTable(params, function (err) { + ddb.updateTable(params, function (err, data) { + debug('deleteIndex handler running'); if (err) { - q.reject(err); + deferred.reject(err); } else { - q.resolve(); + setTimeout(function () { + table.waitForActive(); + deferred.resolve(data); + }, 500); } }); - return q.promise.nodeify(next); + return deferred.promise; }; -var createIndex = function createIndex(table, ddb, tableName, attributes, indexSpec, next) { - var q = Q.defer(); +Table.prototype.createIndex = function createIndex(attributes, indexSpec) { + debug('creating index on %s, named %s', this.name, indexSpec.IndexName); + var deferred = Q.defer(); + var table = this; + table.active = false; + var ddb = this.base.ddb(); + var params = { - TableName: tableName, + TableName: this.name, AttributeDefinitions: attributes, GlobalSecondaryIndexUpdates: [ { @@ -89,16 +118,16 @@ var createIndex = function createIndex(table, ddb, tableName, attributes, indexS } ] }; - debug('creating index %s.%s', tableName, indexSpec.IndexName); - ddb.updateTable(params, function (err) { + ddb.updateTable(params, function (err, data) { + debug('in createTable handler'); if (err) { - q.reject(err); + deferred.reject(err); } else { - q.resolve(); + deferred.resolve(data); } }); - return q.promise.nodeify(next); + return deferred.promise; }; Table.prototype.init = function (next) { @@ -114,21 +143,22 @@ Table.prototype.init = function (next) { .then(function (data) { debug('table exist -- initialization done'); localTableReq = buildTableReq(table.name, table.schema); - var indexes=compareIndexes(localTableReq, data.Table); + var indexes = compareIndexes(localTableReq, data.Table); - debug('%j', indexes); - var updates = []; + debug('%s', JSON.stringify(indexes, null, 2)); if (table.options.update) { debug('checking indexes'); for (var deleteIdx in indexes.delete) { - debug('need to delete %s', indexes.delete[deleteIdx].IndexName); - updates.push(deleteIndex(table, ddb, table.name, indexes.delete[deleteIdx].IndexName)); - updates.push(table.waitForActive()); + table.deleteIndex(indexes.delete[deleteIdx].IndexName); + } + for (var bothIdx in indexes.both) { + table.deleteIndex(indexes.both[bothIdx].IndexName) + .then(function () { + table.createIndex(localTableReq.AttributeDefinitions, indexes.both[bothIdx]) + }); } for (var createIdx in indexes.create) { - debug('need to create %s', indexes.create[createIdx].IndexName); - updates.push(createIndex(table, ddb, table.name, localTableReq.AttributeDefinitions, indexes.create[createIdx])); - updates.push(table.waitForActive()); + table.createIndex(localTableReq.AttributeDefinitions, indexes.create[createIdx]); } } else { if (indexes.delete.length > 0 || indexes.create.length > 0) { @@ -136,11 +166,11 @@ Table.prototype.init = function (next) { deferred.reject('indexes are not synchronized and update flag is set to false'); } } - - table.active = data.Table.TableStatus === 'ACTIVE'; + table.waitForActive(); + //table.active = data.Table.TableStatus === 'ACTIVE'; table.initialized = true; - debug('updates: %j', updates); - return deferred.resolve(Q.all(updates)); + + return deferred.resolve(); }) .catch(function (err) { if (err && err.code === 'ResourceNotFoundException') { @@ -187,10 +217,11 @@ Table.prototype.waitForActive = function (timeout, next) { var timeoutAt = Date.now() + timeout; function waitForActive() { - if (table.active) { - debug('Table is Active - %s', table.name); - return deferred.resolve(); - } + /* + if (table.active) { + debug('Table flag is set to Active - %s', table.name); + return deferred.resolve(); + }*/ if (Date.now() > timeoutAt) { return deferred.reject( new Error('Wait for Active timed out after ' + timeout + ' ms.') @@ -199,27 +230,25 @@ Table.prototype.waitForActive = function (timeout, next) { if (!table.initialized) { return setTimeout(waitForActive, 10); } - var localTableReq = buildTableReq(table.name, table.schema); - table.describe() .then(function (data) { var active = (data.Table.TableStatus === 'ACTIVE'); - if (table.options.update) { - var indexes = compareIndexes(localTableReq, data.Table); - debug('%j', indexes); - if (indexes.delete.length > 0 || indexes.create.length > 0) { - debug('indexes are not synchronized and update flag is set to false'); + data.Table.GlobalSecondaryIndexes.forEach(function (gsi) { + //debug('waitForActive Index Check: %s', JSON.stringify(gsi, null, 2)); + debug('index %s.IndexStatus is %s', gsi.IndexName, gsi.IndexStatus); + if (gsi.IndexStatus !== 'ACTIVE') { active = false; } - } + }); if (!active) { - debug('Waiting for Active - %s', table.name); + debug('Waiting for Active again - %s', table.name); setTimeout(waitForActive, 1000); } else { table.active = true; deferred.resolve(); } - }, function (err) { + }) + .catch(function (err) { return deferred.reject(err); }); } @@ -234,8 +263,6 @@ Table.prototype.describe = function (next) { TableName: this.name }; - debug('ddb.describeTable request: %j', describeTableReq); - var deferred = Q.defer(); var ddb = this.base.ddb(); @@ -244,7 +271,6 @@ Table.prototype.describe = function (next) { debug('error describing table', err); return deferred.reject(err); } - debug('got table description: %j', data); deferred.resolve(data); }); diff --git a/test/Table.js b/test/Table.js index 07f3226bb..978d35866 100755 --- a/test/Table.js +++ b/test/Table.js @@ -160,27 +160,28 @@ describe('Table tests', function () { var tom_sawyer = new Song({id: 1, band: 'Rush', album: 'Moving Pictures', song: 'Tom Sawyer', track: 1}); tom_sawyer.save(); var params = {TableName: 'DMSong'}; - dynamoose.ddb().describeTable(params, function (err, data) { - if (err) { - done(err); - } - else { - var found = false; - for (var i in data.Table.GlobalSecondaryIndexes) { - var gsi = data.Table.GlobalSecondaryIndexes[i]; - if (gsi.IndexName === 'albumIndex') { - should.equal(gsi.Projection.ProjectionType, 'INCLUDE'); - found = true; + setTimeout(function() { + dynamoose.ddb().describeTable(params, function (err, data) { + if (err) { + done(err); + } + else { + var found = false; + for (var i in data.Table.GlobalSecondaryIndexes) { + var gsi = data.Table.GlobalSecondaryIndexes[i]; + if (gsi.IndexName === 'albumIndex') { + should.equal(gsi.Projection.ProjectionType, 'INCLUDE'); + found = true; + } } + should.equal(found, true); + delete dynamoose.models['DMSong']; + done(); } - should.equal(found, true); - delete dynamoose.models['DMSong']; - done(); - } - }); + }); + }, 4000); }); it('update DMSong with broader projection', function (done) { - var params = {TableName: 'DMSong'}; var Song = dynamoose.model('DMSong', { id: { type: Number, @@ -229,24 +230,28 @@ describe('Table tests', function () { var red_barchetta = new Song({id: 2, band: 'Rush', album: 'Moving Pictures', song: 'Red Barchetta', track: 2}); red_barchetta.save(); - dynamoose.ddb().describeTable(params, function (err, data) { - if (err) { - done(err); - } - else { - console.log("---------------------REVISED TABLE"); - console.log(JSON.stringify(data, null, 2)); - var found = false; - for (var i in data.Table.GlobalSecondaryIndexes) { - var gsi = data.Table.GlobalSecondaryIndexes[i]; - if (gsi.IndexName === 'albumIndex') { - should.equal(gsi.Projection.ProjectionType, 'ALL'); - found = true; + + var params = {TableName: 'DMSong'}; + setTimeout(function() { + dynamoose.ddb().describeTable(params, function (err, data) { + if (err) { + done(err); + } + else { + console.log("---------------------REVISED TABLE"); + console.log(JSON.stringify(data, null, 2)); + var found = false; + for (var i in data.Table.GlobalSecondaryIndexes) { + var gsi = data.Table.GlobalSecondaryIndexes[i]; + if (gsi.IndexName === 'albumIndex') { + should.equal(gsi.Projection.ProjectionType, 'ALL'); + found = true; + } } + should.equal(found, true); + done(); } - should.equal(found, true); - done(); - } - }); + }); + }, 4000); }); }); From e3e9081b20d9acbbb5004423ea9c0425c9d2b1f8 Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Fri, 14 Oct 2016 16:01:44 -0400 Subject: [PATCH 09/11] minor code cleanup for jslint --- lib/Table.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/Table.js b/lib/Table.js index d23bfc8b2..9561a0fc5 100755 --- a/lib/Table.js +++ b/lib/Table.js @@ -1,7 +1,6 @@ 'use strict'; var Q = require('q'); var debug = require('debug')('dynamoose:table'); -var trace = require('debug')('dynamoose:index'); var util = require('util'); var _ = require('underscore'); @@ -18,6 +17,8 @@ function Table(name, schema, options, base) { } var compareIndexes = function compareIndexes(local, remote) { + var i; + var j; var indexes = { delete: [], create: [], @@ -31,10 +32,10 @@ var compareIndexes = function compareIndexes(local, remote) { debug('compareIndexes'); // let's see what remote indexes we need to sync or create - for (var i = 0; i < localIndexes.length; i++) { + for (i = 0; i < localIndexes.length; i++) { var remoteIndexFound = false; - for (var j = 0; j < remoteIndexes.length; j++) { - if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { + for (j = 0; j < remoteIndexes.length; j++) { + if (remoteIndexes[j].IndexName === localIndexes[i].IndexName) { // let's see if the core data matches. if it doesn't, // we may need to delete the remote GSI and rebuild. var localIndex = _.pick(localIndexes[i], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); @@ -59,7 +60,7 @@ var compareIndexes = function compareIndexes(local, remote) { for (j = 0; j < remoteIndexes.length; j++) { var localExists = false; for (i = 0; i < localIndexes.length; i++) { - if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { + if (remoteIndexes[j].IndexName === localIndexes[i].IndexName) { localExists = true; } } @@ -135,7 +136,6 @@ Table.prototype.init = function (next) { var deferred = Q.defer(); var table = this; - var ddb = this.base.ddb(); var localTableReq; if (this.options.create) { @@ -154,8 +154,9 @@ Table.prototype.init = function (next) { for (var bothIdx in indexes.both) { table.deleteIndex(indexes.both[bothIdx].IndexName) .then(function () { - table.createIndex(localTableReq.AttributeDefinitions, indexes.both[bothIdx]) + table.createIndex(localTableReq.AttributeDefinitions, indexes.both[bothIdx]); }); + } for (var createIdx in indexes.create) { table.createIndex(localTableReq.AttributeDefinitions, indexes.create[createIdx]); @@ -242,7 +243,7 @@ Table.prototype.waitForActive = function (timeout, next) { }); if (!active) { debug('Waiting for Active again - %s', table.name); - setTimeout(waitForActive, 1000); + setTimeout(waitForActive, 500); } else { table.active = true; deferred.resolve(); From 8b1f5f98f5103e9baa323c82834a2c89b857a0be Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Mon, 17 Oct 2016 10:36:34 -0400 Subject: [PATCH 10/11] fixing waitForActive promises --- lib/Table.js | 46 +++++++++++++++++++++++----------------------- test/Table.js | 4 ++-- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/Table.js b/lib/Table.js index 9561a0fc5..1811e452e 100755 --- a/lib/Table.js +++ b/lib/Table.js @@ -1,6 +1,7 @@ 'use strict'; var Q = require('q'); var debug = require('debug')('dynamoose:table'); +var trace = require('debug')('dynamoose:index'); var util = require('util'); var _ = require('underscore'); @@ -17,8 +18,6 @@ function Table(name, schema, options, base) { } var compareIndexes = function compareIndexes(local, remote) { - var i; - var j; var indexes = { delete: [], create: [], @@ -32,10 +31,10 @@ var compareIndexes = function compareIndexes(local, remote) { debug('compareIndexes'); // let's see what remote indexes we need to sync or create - for (i = 0; i < localIndexes.length; i++) { + for (var i = 0; i < localIndexes.length; i++) { var remoteIndexFound = false; - for (j = 0; j < remoteIndexes.length; j++) { - if (remoteIndexes[j].IndexName === localIndexes[i].IndexName) { + for (var j = 0; j < remoteIndexes.length; j++) { + if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { // let's see if the core data matches. if it doesn't, // we may need to delete the remote GSI and rebuild. var localIndex = _.pick(localIndexes[i], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); @@ -56,11 +55,10 @@ var compareIndexes = function compareIndexes(local, remote) { indexes.create.push(localIndexes[i]); } } - // now let's see what remote indexes exist that shouldn't exist - for (j = 0; j < remoteIndexes.length; j++) { + for (var j = 0; j < remoteIndexes.length; j++) { var localExists = false; - for (i = 0; i < localIndexes.length; i++) { - if (remoteIndexes[j].IndexName === localIndexes[i].IndexName) { + for (var i = 0; i < localIndexes.length; i++) { + if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { localExists = true; } } @@ -68,16 +66,15 @@ var compareIndexes = function compareIndexes(local, remote) { indexes.delete.push(remoteIndexes[j]); } } + // now let's see what remote indexes exist that shouldn't exist return indexes; }; Table.prototype.deleteIndex = function deleteIndex(indexName) { - debug('deleting index %s', indexName); var deferred = Q.defer(); var table = this; table.active = false; - var ddb = this.base.ddb(); var params = { TableName: table.name, GlobalSecondaryIndexUpdates: [ @@ -88,28 +85,27 @@ Table.prototype.deleteIndex = function deleteIndex(indexName) { } ] }; - ddb.updateTable(params, function (err, data) { + table.base.ddb().updateTable(params, function (err, data) { debug('deleteIndex handler running'); if (err) { deferred.reject(err); } else { setTimeout(function () { - table.waitForActive(); - deferred.resolve(data); - }, 500); + table.waitForActive() + .then(function () { + deferred.resolve(data); + }); + }, 300); } }); return deferred.promise; }; Table.prototype.createIndex = function createIndex(attributes, indexSpec) { - debug('creating index on %s, named %s', this.name, indexSpec.IndexName); var deferred = Q.defer(); var table = this; table.active = false; - var ddb = this.base.ddb(); - var params = { TableName: this.name, AttributeDefinitions: attributes, @@ -119,13 +115,17 @@ Table.prototype.createIndex = function createIndex(attributes, indexSpec) { } ] }; - ddb.updateTable(params, function (err, data) { - debug('in createTable handler'); + this.base.ddb().updateTable(params, function (err, data) { if (err) { deferred.reject(err); } else { - deferred.resolve(data); + setTimeout(function () { + table.waitForActive() + .then(function () { + deferred.resolve(data); + }); + }, 300); } }); return deferred.promise; @@ -136,6 +136,7 @@ Table.prototype.init = function (next) { var deferred = Q.defer(); var table = this; + var ddb = this.base.ddb(); var localTableReq; if (this.options.create) { @@ -154,9 +155,8 @@ Table.prototype.init = function (next) { for (var bothIdx in indexes.both) { table.deleteIndex(indexes.both[bothIdx].IndexName) .then(function () { - table.createIndex(localTableReq.AttributeDefinitions, indexes.both[bothIdx]); + table.createIndex(localTableReq.AttributeDefinitions, indexes.both[bothIdx]) }); - } for (var createIdx in indexes.create) { table.createIndex(localTableReq.AttributeDefinitions, indexes.create[createIdx]); diff --git a/test/Table.js b/test/Table.js index 978d35866..e45955cd5 100755 --- a/test/Table.js +++ b/test/Table.js @@ -179,7 +179,7 @@ describe('Table tests', function () { done(); } }); - }, 4000); + }, 2000); }); it('update DMSong with broader projection', function (done) { var Song = dynamoose.model('DMSong', { @@ -252,6 +252,6 @@ describe('Table tests', function () { done(); } }); - }, 4000); + }, 2000); }); }); From cc5a22903fbb70fb463a7ad6689a5b3fb45311c4 Mon Sep 17 00:00:00 2001 From: "Joseph B. Ottinger" Date: Tue, 18 Oct 2016 10:07:10 -0400 Subject: [PATCH 11/11] jslint cleanup --- lib/Table.js | 19 ++++++++++--------- test/Table.js | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/Table.js b/lib/Table.js index 1811e452e..2902cdc50 100755 --- a/lib/Table.js +++ b/lib/Table.js @@ -1,7 +1,6 @@ 'use strict'; var Q = require('q'); var debug = require('debug')('dynamoose:table'); -var trace = require('debug')('dynamoose:index'); var util = require('util'); var _ = require('underscore'); @@ -25,16 +24,18 @@ var compareIndexes = function compareIndexes(local, remote) { }; var localTableReq = local; var remoteTableReq = remote; + var i; + var j; var localIndexes = localTableReq.GlobalSecondaryIndexes; var remoteIndexes = remoteTableReq.GlobalSecondaryIndexes; debug('compareIndexes'); // let's see what remote indexes we need to sync or create - for (var i = 0; i < localIndexes.length; i++) { + for (i = 0; i < localIndexes.length; i++) { var remoteIndexFound = false; - for (var j = 0; j < remoteIndexes.length; j++) { - if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { + for (j = 0; j < remoteIndexes.length; j++) { + if (remoteIndexes[j].IndexName === localIndexes[i].IndexName) { // let's see if the core data matches. if it doesn't, // we may need to delete the remote GSI and rebuild. var localIndex = _.pick(localIndexes[i], 'IndexName', 'KeySchema', 'Projection', 'ProvisionedThroughput'); @@ -55,10 +56,10 @@ var compareIndexes = function compareIndexes(local, remote) { indexes.create.push(localIndexes[i]); } } - for (var j = 0; j < remoteIndexes.length; j++) { + for (j = 0; j < remoteIndexes.length; j++) { var localExists = false; - for (var i = 0; i < localIndexes.length; i++) { - if (remoteIndexes[j].IndexName == localIndexes[i].IndexName) { + for (i = 0; i < localIndexes.length; i++) { + if (remoteIndexes[j].IndexName === localIndexes[i].IndexName) { localExists = true; } } @@ -136,7 +137,6 @@ Table.prototype.init = function (next) { var deferred = Q.defer(); var table = this; - var ddb = this.base.ddb(); var localTableReq; if (this.options.create) { @@ -153,9 +153,10 @@ Table.prototype.init = function (next) { table.deleteIndex(indexes.delete[deleteIdx].IndexName); } for (var bothIdx in indexes.both) { + /*jshint loopfunc: true */ table.deleteIndex(indexes.both[bothIdx].IndexName) .then(function () { - table.createIndex(localTableReq.AttributeDefinitions, indexes.both[bothIdx]) + table.createIndex(localTableReq.AttributeDefinitions, indexes.both[bothIdx]); }); } for (var createIdx in indexes.create) { diff --git a/test/Table.js b/test/Table.js index e45955cd5..a64763522 100755 --- a/test/Table.js +++ b/test/Table.js @@ -175,7 +175,7 @@ describe('Table tests', function () { } } should.equal(found, true); - delete dynamoose.models['DMSong']; + delete dynamoose.models.DMSong; done(); } });