From 4086519cac82f4b5ecb85d786433682115f37fbf Mon Sep 17 00:00:00 2001 From: "MD. Jahidul Islam" Date: Sat, 23 Nov 2019 08:39:46 +0100 Subject: [PATCH 1/2] pagination for asset type url using offset to paginate all pages for asset type as it doesn't provide nextpagetoken in the response --- .gitignore | 1 + lib/api/smartList.js | 31 ++++++++++++ lib/connection.js | 43 +++++++---------- lib/marketo.js | 2 + lib/nextPageFn/index.js | 20 ++++++++ lib/nextPageFn/offset.js | 16 +++++++ lib/nextPageFn/token.js | 16 +++++++ lib/stream.js | 2 +- lib/util.js | 10 +++- package.json | 6 ++- test/activities.test.js | 2 +- test/connection.test.js | 101 +++++++++++++++++++++++++++++++++++++++ test/smartlists.test.js | 31 ++++++++++++ 13 files changed, 251 insertions(+), 30 deletions(-) create mode 100644 lib/api/smartList.js create mode 100644 lib/nextPageFn/index.js create mode 100644 lib/nextPageFn/offset.js create mode 100644 lib/nextPageFn/token.js create mode 100644 test/connection.test.js create mode 100644 test/smartlists.test.js diff --git a/.gitignore b/.gitignore index e19efb2..446cc6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /node_modules/ /.idea/ +/.history *.iml test/helper/config.bk.js package-lock.json \ No newline at end of file diff --git a/lib/api/smartList.js b/lib/api/smartList.js new file mode 100644 index 0000000..c5e255d --- /dev/null +++ b/lib/api/smartList.js @@ -0,0 +1,31 @@ +var _ = require('lodash'), + util = require('../util'); + +var SMARTLISTS = util.createAssetPath('smartLists.json'); + +function SmartList(marketo, connection) { + this._marketo = marketo; + this._connection = connection; +} + +SmartList.prototype = { + find: function(options) { + var arrayFields = []; + options = _.extend({}, options, { + _method: 'GET', + }); + options = util.arrayToCSV(options, arrayFields); + return this._connection.post(SMARTLISTS, {data: options}); + }, + byId: function(smartListId, options) { + var SMARTLISTS_ID = util.createAssetPath('smartList', `${smartListId}.json`); + var arrayFields = []; + options = _.extend({}, options, { + _method: 'GET', + }); + options = util.arrayToCSV(options, arrayFields); + return this._connection.post(SMARTLISTS_ID, {data: options}); + } +}; + +module.exports = SmartList; \ No newline at end of file diff --git a/lib/connection.js b/lib/connection.js index 6773bff..9e9ebf1 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -3,7 +3,9 @@ var _ = require('lodash'), Promise = require('bluebird'), Retry = require('./retry'), errors = require('./errors'), + getNextPageFn = require('./nextPageFn'), util = require('util'), + helper = require('./util'), EventEmitter = require('events').EventEmitter, log = require('./util').logger(), restlerMethodArgCount = { @@ -43,28 +45,10 @@ function Connection(options) { util.inherits(Connection, EventEmitter); -function getNextPageFn(conn, method, args) { - var options = _.clone(_.last(args) || {}); - args = _.clone(args); - args.pop(); - - return function nextPage(nextPageToken) { - var params = options; - if (method === 'get') { - params = options.query = options.query || {}; - } else if (method === 'post' || method === 'put') { - params = options.data = options.data || {}; - } - params.nextPageToken = nextPageToken; - - return conn._request.apply(conn, _.flatten([method, args, options], true)); - } -} - /** * This function is just a helper function that delegates everything to * restler, but returns a Promise instead of going with the existing event - * based resposnes. + * based responses. */ Connection.prototype._request = function(method) { var args = Array.prototype.slice.call(arguments, 1), @@ -92,19 +76,28 @@ Connection.prototype._request = function(method) { 'Authorization': 'Bearer ' + token.access_token }); args.push(options); - rest[method].apply(rest, args) .on('success', function(data, resp) { if (data.success === false && _.has(data, 'errors')) { log.debug('Request failed: ', data); defer.reject(new errors.HttpError(resp.statusCode, data)); } else { - if (_.has(data, 'nextPageToken')) { - Object.defineProperty(data, 'nextPage', { - enumerable: false, - value: _.partial(nextPageFn, data.nextPageToken) - }); + if(helper.nextPageType(args[0]) === 'offset') { + if(!_.isEmpty(data.result)) { + Object.defineProperty(data, 'nextPage', { + enumerable: false, + value: _.partial(nextPageFn) + }); + } + } else { + if (_.has(data, 'nextPageToken')) { + Object.defineProperty(data, 'nextPage', { + enumerable: false, + value: _.partial(nextPageFn, data.nextPageToken) + }); + } } + defer.resolve(data); } }) diff --git a/lib/marketo.js b/lib/marketo.js index 381ca61..d30c60c 100644 --- a/lib/marketo.js +++ b/lib/marketo.js @@ -8,6 +8,7 @@ var _ = require('lodash'), LandingPage = require('./api/landingPage'), Lead = require('./api/lead'), List = require('./api/list'), + SmartList = require('./api/smartList'), Stats = require('./api/stats'), BulkLeadExtract = require('./api/bulkLeadExtract'), BulkActivityExtract = require('./api/bulkActivityExtract'), @@ -26,6 +27,7 @@ function Marketo(options) { this.email = new Email(this, this._connection); this.landingPage = new LandingPage(this, this._connection); this.list = new List(this, this._connection); + this.smartList = new SmartList(this, this._connection); this.lead = new Lead(this, this._connection); this.stats = new Stats(this, this._connection); this.activities = new Activities(this, this._connection); diff --git a/lib/nextPageFn/index.js b/lib/nextPageFn/index.js new file mode 100644 index 0000000..a58ada2 --- /dev/null +++ b/lib/nextPageFn/index.js @@ -0,0 +1,20 @@ +var offset = require('./offset'); +var token = require('./token'); +var util = require('../util'); +var _ = require('lodash'); + +module.exports = getNextPageFn; + +function getNextPageFn(conn, method, args) { + var assetStrategy = { + offset, + token + }; + + var options = _.clone(_.last(args) || {}); + args = _.clone(args); + args.pop(); + + var selectedStrategy = assetStrategy[util.nextPageType(args[0])] + return selectedStrategy(conn, method, args, options); +} diff --git a/lib/nextPageFn/offset.js b/lib/nextPageFn/offset.js new file mode 100644 index 0000000..ef79391 --- /dev/null +++ b/lib/nextPageFn/offset.js @@ -0,0 +1,16 @@ +var _ = require('lodash'); + +module.exports = (conn, method, args, options) => { + return () => { + var params = options; + if (method === 'get') { + params = options.query = options.query || {}; + } else if (method === 'post' || method === 'put') { + params = options.data = options.data || {}; + } + + params.offset = (params.offset || 0) + params.maxReturn; + + return conn._request.apply(conn, _.flatten([method, args, options], true)); + } +}; diff --git a/lib/nextPageFn/token.js b/lib/nextPageFn/token.js new file mode 100644 index 0000000..4756d30 --- /dev/null +++ b/lib/nextPageFn/token.js @@ -0,0 +1,16 @@ +var _ = require('lodash'); + +module.exports = (conn, method, args, options) => { + return (nextPageToken) => { + var params = options; + if (method === 'get') { + params = options.query = options.query || {}; + } else if (method === 'post' || method === 'put') { + params = options.data = options.data || {}; + } + + params.nextPageToken = nextPageToken; + + return conn._request.apply(conn, _.flatten([method, args, options], true)); + } +}; diff --git a/lib/stream.js b/lib/stream.js index 6599229..0d3e4e3 100644 --- a/lib/stream.js +++ b/lib/stream.js @@ -67,7 +67,7 @@ MarketoStream.prototype._read = function () { this.fetch() } else if (this._pushNext()) { return; - } else if (this._data.moreResult || this._data.nextPageToken) { + } else if (this._data.moreResult || this._data.nextPage) { this.fetch(this._data.nextPage()) } else { // No data left in the batch and no more data from marketo, end the stream this.push(null); diff --git a/lib/util.js b/lib/util.js index 0582297..c4c3ede 100644 --- a/lib/util.js +++ b/lib/util.js @@ -64,5 +64,13 @@ module.exports = { } return this.arrayToCSV(options, ['fields', 'filterValues']); - } + }, + + nextPageType: function(requestedUrl) { + var type = 'token'; + if(requestedUrl.indexOf('asset') !== -1) { + type = 'offset'; + } + return type; + }, }; diff --git a/package.json b/package.json index 8188b74..2909a66 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-marketo-rest", - "version": "0.7.5", + "version": "0.7.8", "description": "marketo rest client", "repository": { "type": "git", @@ -23,6 +23,8 @@ }, "devDependencies": { "mocha": "5.0.x", - "replay": "2.1.x" + "nock": "^11.7.0", + "replay": "2.1.x", + "rewire": "^4.0.1" } } diff --git a/test/activities.test.js b/test/activities.test.js index 82d6c07..d01ef12 100644 --- a/test/activities.test.js +++ b/test/activities.test.js @@ -4,7 +4,7 @@ var assert = require('assert'), describe('Activities', function() { describe('list activity types', function() { - it('lists activity types', function(done){ + it('lists activity types', function(done) { marketo.activities.getActivityTypes().then(function(resp) { var activity = resp.result[0]; assert.equal(activity.id, 1); diff --git a/test/connection.test.js b/test/connection.test.js new file mode 100644 index 0000000..086c102 --- /dev/null +++ b/test/connection.test.js @@ -0,0 +1,101 @@ +const _ = require('lodash'), + Promise = require('bluebird'), + assert = require('assert'), + util = require('../lib/util'), + nock = require('nock'), + rewire = require('rewire'); + +const Connection = rewire("../lib/connection.js") +Connection.__set__("getNextPageFn", getNextPageFn) +function getNextPageFn(conn, method, args) { + var assetStrategy = { + offset: () => { + return () => 'offset' + }, + token: () => { + return () => 'token' + }, + }; + + var options = _.clone(_.last(args) || {}); + args = _.clone(args); + args.pop(); + + var selectedStrategy = assetStrategy[util.nextPageType(args[0])] + return selectedStrategy(conn, method, args, options); +} + +const assetScope = nock('http://localhost-mock') + .filteringRequestBody(body => { + console.log(body); + return true; + }) + .post('/asset') + .reply(200, { + "success": true, + "errors": [], + "requestId": "6efc#16c8967a21f", + "warnings": [], + "result": [ + { + "id": 4363, + "name": "Smart List Test 01" + } + ] + }); + +const tokenScope = nock('http://localhost-mock') + .filteringRequestBody(body => { + console.log(body); + return true; + }) + .post('/rest/v1') + .reply(200, { + "moreResult": true, + "nextPageToken": "string", + "requestId": "string", + "result": [ + { + "id": 0, + "status": "string" + } + ], + "success": true + }); + +const ASSET_URL = 'asset', + TOKEN_URL = 'rest/v1', + IDENTITY_URL = 'identity'; + +Connection.prototype.getOAuthToken = function() { + return Promise.resolve({access_token: 'test_token'}); +} + +function getUrl(path) { + return 'http://localhost-mock/' + path; +} + +function getConnection() { + var options = { + endpoint: getUrl(''), + identity: getUrl(IDENTITY_URL), + clientId: 'someId', + clientSecret: 'someSecret' + }; + return new Connection(options); +} + +describe('Connection', function() { + it('token type pagination', function() { + return getConnection().post(TOKEN_URL, {data: {_method: 'GET', maxReturn: 200}}).then(resp => { + assert.equal(resp.nextPage(), 'token') + }); + }); + + it('offset type pagination', function() { + return getConnection().post(ASSET_URL, {data: {_method: 'GET', maxReturn: 200}}).then(resp => { + assert.equal(resp.nextPage(), 'offset') + }); + }); +}); + \ No newline at end of file diff --git a/test/smartlists.test.js b/test/smartlists.test.js new file mode 100644 index 0000000..0976aa5 --- /dev/null +++ b/test/smartlists.test.js @@ -0,0 +1,31 @@ +var assert = require('assert'), + _ = require('lodash'), + marketo = require('./helper/connection'); + +describe('SmartLists', function () { + var smartListId; + describe('#find', function () { + it('displays list of smart lists', function (done) { + marketo.smartList.find({maxReturn: 100}).then(function (response) { + assert.equal(response.success, true); + assert.equal(response.errors.length, 0); + assert(_.has(response.result[0], 'name')); + assert(_.has(response.result[0], 'workspace')); + smartListId = response.result[0].id; + done(); + }).catch(done); + }); + }); + describe('#byId', function () { + it('finds smart list by id', function (done) { + marketo.smartList.byId(smartListId, {includeRules: true}).then(function (response) { + assert.equal(response.success, true); + assert.equal(response.errors.length, 0); + assert.equal(response.result.length, 1); + assert(_.has(response.result[0], 'name')); + assert(_.has(response.result[0], 'workspace')); + done(); + }).catch(done); + }); + }); +}); \ No newline at end of file From 8ca8dc83c41b175edabf3b1ad3e8c931c87b2f47 Mon Sep 17 00:00:00 2001 From: MD Jahidul Islam Date: Sat, 23 Nov 2019 10:20:03 +0100 Subject: [PATCH 2/2] added default value for maxReturn --- lib/nextPageFn/offset.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/nextPageFn/offset.js b/lib/nextPageFn/offset.js index ef79391..f471ebb 100644 --- a/lib/nextPageFn/offset.js +++ b/lib/nextPageFn/offset.js @@ -9,7 +9,7 @@ module.exports = (conn, method, args, options) => { params = options.data = options.data || {}; } - params.offset = (params.offset || 0) + params.maxReturn; + params.offset = (params.offset || 0) + (params.maxReturn || 20); return conn._request.apply(conn, _.flatten([method, args, options], true)); }