diff --git a/.gitignore b/.gitignore index e19efb2..446cc6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /node_modules/ /.idea/ +/.history *.iml test/helper/config.bk.js package-lock.json \ No newline at end of file diff --git a/lib/connection.js b/lib/connection.js index 7f9a453..9e9ebf1 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -3,7 +3,9 @@ var _ = require('lodash'), Promise = require('bluebird'), Retry = require('./retry'), errors = require('./errors'), + getNextPageFn = require('./nextPageFn'), util = require('util'), + helper = require('./util'), EventEmitter = require('events').EventEmitter, log = require('./util').logger(), restlerMethodArgCount = { @@ -43,24 +45,6 @@ function Connection(options) { util.inherits(Connection, EventEmitter); -function getNextPageFn(conn, method, args) { - var options = _.clone(_.last(args) || {}); - args = _.clone(args); - args.pop(); - - return function nextPage(nextPageToken) { - var params = options; - if (method === 'get') { - params = options.query = options.query || {}; - } else if (method === 'post' || method === 'put') { - params = options.data = options.data || {}; - } - params.nextPageToken = nextPageToken; - - return conn._request.apply(conn, _.flatten([method, args, options], true)); - } -} - /** * This function is just a helper function that delegates everything to * restler, but returns a Promise instead of going with the existing event @@ -98,12 +82,22 @@ Connection.prototype._request = function(method) { log.debug('Request failed: ', data); defer.reject(new errors.HttpError(resp.statusCode, data)); } else { - if (_.has(data, 'nextPageToken')) { - Object.defineProperty(data, 'nextPage', { - enumerable: false, - value: _.partial(nextPageFn, data.nextPageToken) - }); + if(helper.nextPageType(args[0]) === 'offset') { + if(!_.isEmpty(data.result)) { + Object.defineProperty(data, 'nextPage', { + enumerable: false, + value: _.partial(nextPageFn) + }); + } + } else { + if (_.has(data, 'nextPageToken')) { + Object.defineProperty(data, 'nextPage', { + enumerable: false, + value: _.partial(nextPageFn, data.nextPageToken) + }); + } } + defer.resolve(data); } }) diff --git a/lib/nextPageFn/index.js b/lib/nextPageFn/index.js new file mode 100644 index 0000000..a58ada2 --- /dev/null +++ b/lib/nextPageFn/index.js @@ -0,0 +1,20 @@ +var offset = require('./offset'); +var token = require('./token'); +var util = require('../util'); +var _ = require('lodash'); + +module.exports = getNextPageFn; + +function getNextPageFn(conn, method, args) { + var assetStrategy = { + offset, + token + }; + + var options = _.clone(_.last(args) || {}); + args = _.clone(args); + args.pop(); + + var selectedStrategy = assetStrategy[util.nextPageType(args[0])] + return selectedStrategy(conn, method, args, options); +} diff --git a/lib/nextPageFn/offset.js b/lib/nextPageFn/offset.js new file mode 100644 index 0000000..f471ebb --- /dev/null +++ b/lib/nextPageFn/offset.js @@ -0,0 +1,16 @@ +var _ = require('lodash'); + +module.exports = (conn, method, args, options) => { + return () => { + var params = options; + if (method === 'get') { + params = options.query = options.query || {}; + } else if (method === 'post' || method === 'put') { + params = options.data = options.data || {}; + } + + params.offset = (params.offset || 0) + (params.maxReturn || 20); + + return conn._request.apply(conn, _.flatten([method, args, options], true)); + } +}; diff --git a/lib/nextPageFn/token.js b/lib/nextPageFn/token.js new file mode 100644 index 0000000..4756d30 --- /dev/null +++ b/lib/nextPageFn/token.js @@ -0,0 +1,16 @@ +var _ = require('lodash'); + +module.exports = (conn, method, args, options) => { + return (nextPageToken) => { + var params = options; + if (method === 'get') { + params = options.query = options.query || {}; + } else if (method === 'post' || method === 'put') { + params = options.data = options.data || {}; + } + + params.nextPageToken = nextPageToken; + + return conn._request.apply(conn, _.flatten([method, args, options], true)); + } +}; diff --git a/lib/stream.js b/lib/stream.js index 6599229..0d3e4e3 100644 --- a/lib/stream.js +++ b/lib/stream.js @@ -67,7 +67,7 @@ MarketoStream.prototype._read = function () { this.fetch() } else if (this._pushNext()) { return; - } else if (this._data.moreResult || this._data.nextPageToken) { + } else if (this._data.moreResult || this._data.nextPage) { this.fetch(this._data.nextPage()) } else { // No data left in the batch and no more data from marketo, end the stream this.push(null); diff --git a/lib/util.js b/lib/util.js index 0582297..c4c3ede 100644 --- a/lib/util.js +++ b/lib/util.js @@ -64,5 +64,13 @@ module.exports = { } return this.arrayToCSV(options, ['fields', 'filterValues']); - } + }, + + nextPageType: function(requestedUrl) { + var type = 'token'; + if(requestedUrl.indexOf('asset') !== -1) { + type = 'offset'; + } + return type; + }, }; diff --git a/package.json b/package.json index 8ea434d..2909a66 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-marketo-rest", - "version": "0.7.7", + "version": "0.7.8", "description": "marketo rest client", "repository": { "type": "git", @@ -23,6 +23,8 @@ }, "devDependencies": { "mocha": "5.0.x", - "replay": "2.1.x" + "nock": "^11.7.0", + "replay": "2.1.x", + "rewire": "^4.0.1" } } diff --git a/test/connection.test.js b/test/connection.test.js new file mode 100644 index 0000000..086c102 --- /dev/null +++ b/test/connection.test.js @@ -0,0 +1,101 @@ +const _ = require('lodash'), + Promise = require('bluebird'), + assert = require('assert'), + util = require('../lib/util'), + nock = require('nock'), + rewire = require('rewire'); + +const Connection = rewire("../lib/connection.js") +Connection.__set__("getNextPageFn", getNextPageFn) +function getNextPageFn(conn, method, args) { + var assetStrategy = { + offset: () => { + return () => 'offset' + }, + token: () => { + return () => 'token' + }, + }; + + var options = _.clone(_.last(args) || {}); + args = _.clone(args); + args.pop(); + + var selectedStrategy = assetStrategy[util.nextPageType(args[0])] + return selectedStrategy(conn, method, args, options); +} + +const assetScope = nock('http://localhost-mock') + .filteringRequestBody(body => { + console.log(body); + return true; + }) + .post('/asset') + .reply(200, { + "success": true, + "errors": [], + "requestId": "6efc#16c8967a21f", + "warnings": [], + "result": [ + { + "id": 4363, + "name": "Smart List Test 01" + } + ] + }); + +const tokenScope = nock('http://localhost-mock') + .filteringRequestBody(body => { + console.log(body); + return true; + }) + .post('/rest/v1') + .reply(200, { + "moreResult": true, + "nextPageToken": "string", + "requestId": "string", + "result": [ + { + "id": 0, + "status": "string" + } + ], + "success": true + }); + +const ASSET_URL = 'asset', + TOKEN_URL = 'rest/v1', + IDENTITY_URL = 'identity'; + +Connection.prototype.getOAuthToken = function() { + return Promise.resolve({access_token: 'test_token'}); +} + +function getUrl(path) { + return 'http://localhost-mock/' + path; +} + +function getConnection() { + var options = { + endpoint: getUrl(''), + identity: getUrl(IDENTITY_URL), + clientId: 'someId', + clientSecret: 'someSecret' + }; + return new Connection(options); +} + +describe('Connection', function() { + it('token type pagination', function() { + return getConnection().post(TOKEN_URL, {data: {_method: 'GET', maxReturn: 200}}).then(resp => { + assert.equal(resp.nextPage(), 'token') + }); + }); + + it('offset type pagination', function() { + return getConnection().post(ASSET_URL, {data: {_method: 'GET', maxReturn: 200}}).then(resp => { + assert.equal(resp.nextPage(), 'offset') + }); + }); +}); + \ No newline at end of file