From bf921de1391ca76b88b23cf91c55ac804f2f68c4 Mon Sep 17 00:00:00 2001 From: Jason Buckner Date: Thu, 13 Aug 2015 14:49:30 -0700 Subject: [PATCH 1/2] =?UTF-8?q?Add=20an=20option=20to=20configure=20concur?= =?UTF-8?q?rent=20transfers=20for=20async=20requests=20Startling=20only=20?= =?UTF-8?q?allows=2050=20concurrent=20transfers=20and=20the=20current=20de?= =?UTF-8?q?fault=20(10)=20causes=20the=20API=20to=20hit=20the=20limit.=20I?= =?UTF-8?q?=E2=80=99ve=20found=20changing=20it=20to=205=20fixes=20it,=20bu?= =?UTF-8?q?t=20this=20will=20allow=20the=20user=20to=20customize=20that=20?= =?UTF-8?q?number.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 18 ++++++++++++++++++ tasks/config_defaults.js | 3 +++ tasks/smartling_delete.js | 10 ++++++---- tasks/smartling_get.js | 15 +++++++++------ tasks/smartling_rename.js | 12 +++++++----- tasks/smartling_status.js | 10 ++++++---- tasks/smartling_upload.js | 10 ++++++---- 7 files changed, 55 insertions(+), 23 deletions(-) create mode 100644 tasks/config_defaults.js diff --git a/README.md b/README.md index 948c407..22a1593 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,24 @@ smartling_status: { } ``` +#### options.concurrentTransfers +Type: `Integer` +Default value: `10` + +When `concurrentTransfers` is provided, it will limit the number of current transfers to the provided number. This is useful to avoid hitting Smartling's API limits by requesting too many files at a time. + +```js +smartling_get: { + options: { + smartling: { ... } + concurrentTransfers: 5 + }, + default: { + src: 'path/to/translations/*.json' + } +} +``` + ### Usage Examples See Gruntfile.js for usage examples diff --git a/tasks/config_defaults.js b/tasks/config_defaults.js new file mode 100644 index 0000000..1758921 --- /dev/null +++ b/tasks/config_defaults.js @@ -0,0 +1,3 @@ +module.exports = { + concurrentTransfers: 10 +} diff --git a/tasks/smartling_delete.js b/tasks/smartling_delete.js index 805345b..fd5d6a2 100644 --- a/tasks/smartling_delete.js +++ b/tasks/smartling_delete.js @@ -11,17 +11,19 @@ 'use strict'; module.exports = function (grunt) { - var asyncUtil = require('async'), - SmartlingTask = require('../lib/smartling-task'), - DeleteStats = require('../lib/delete-stats'); + var asyncUtil = require('async'), + SmartlingTask = require('../lib/smartling-task'), + DeleteStats = require('../lib/delete-stats'), + configDefaults = require('./config_defaults'); grunt.registerMultiTask('smartling_delete', 'Delete files from Smartling', SmartlingTask.make(function (task, options, sdk, done, logJson) { var stats = new DeleteStats(); + var concurrentTransfers = options.concurrentTransfers || configDefaults.concurrentTransfers; var fileUris = task.getFileUris(); - asyncUtil.eachLimit(fileUris, 10, function (fileUri, callback) { + asyncUtil.eachLimit(fileUris, concurrentTransfers, function (fileUri, callback) { sdk.delete(fileUri) .then(function () { //file deleted successfully diff --git a/tasks/smartling_get.js b/tasks/smartling_get.js index 5273d39..38fddf4 100644 --- a/tasks/smartling_get.js +++ b/tasks/smartling_get.js @@ -11,16 +11,19 @@ 'use strict'; module.exports = function (grunt) { - var SmartlingTask = require('../lib/smartling-task'), - asyncUtil = require('async'), - _ = require('lodash'), - path = require('path'), - GetStats = require('../lib/get-stats'); + var SmartlingTask = require('../lib/smartling-task'), + asyncUtil = require('async'), + _ = require('lodash'), + path = require('path'), + GetStats = require('../lib/get-stats'), + configDefaults = require('./config_defaults'); + grunt.registerMultiTask('smartling_get', 'Get files from Smartling', SmartlingTask.make(function (task, options, sdk, done, logJson) { var stats = new GetStats(); var outputDirectory = options.dest; + var concurrentTransfers = options.concurrentTransfers || configDefaults.concurrentTransfers; var fileUris = task.getFileUris(); var destFileUriFunc = options.destFileUriFunc || function(fileUri) { @@ -44,7 +47,7 @@ module.exports = function (grunt) { } asyncUtil.each(locales, function(locale, localeCallback) { - asyncUtil.eachLimit(fileUris, 10, function (fileUri, fileCallback) { + asyncUtil.eachLimit(fileUris, concurrentTransfers, function (fileUri, fileCallback) { var destFilepath; if (outputDirectory) { //check to modufy the destination fileUri diff --git a/tasks/smartling_rename.js b/tasks/smartling_rename.js index 6e07adf..417d14b 100644 --- a/tasks/smartling_rename.js +++ b/tasks/smartling_rename.js @@ -11,18 +11,20 @@ 'use strict'; module.exports = function (grunt) { - var SmartlingTask = require('../lib/smartling-task'), - asyncUtil = require('async'), - path = require('path'), - RenameStats = require('../lib/rename-stats'); + var SmartlingTask = require('../lib/smartling-task'), + asyncUtil = require('async'), + path = require('path'), + RenameStats = require('../lib/rename-stats'), + configDefaults = require('./config_defaults'); grunt.registerMultiTask('smartling_rename', 'Rename files in Smartling', SmartlingTask.make(function (task, options, sdk, done, logJson) { var stats = new RenameStats(); + var concurrentTransfers = options.concurrentTransfers || configDefaults.concurrentTransfers; var fileUris = task.getFileUris(); - asyncUtil.eachLimit(fileUris, 10, + asyncUtil.eachLimit(fileUris, concurrentTransfers, function (fileUri, callback) { var newFileUri = options.newFileUriFunc(fileUri); diff --git a/tasks/smartling_status.js b/tasks/smartling_status.js index 0f999be..2f43fef 100644 --- a/tasks/smartling_status.js +++ b/tasks/smartling_status.js @@ -11,17 +11,19 @@ 'use strict'; module.exports = function (grunt) { - var asyncUtil = require('async'), - SmartlingTask = require('../lib/smartling-task'), - StatusStats = require('../lib/status-stats'); + var asyncUtil = require('async'), + SmartlingTask = require('../lib/smartling-task'), + StatusStats = require('../lib/status-stats'), + configDefaults = require('./config_defaults'); grunt.registerMultiTask('smartling_status', 'Get status of files in Smartling', SmartlingTask.make(function (task, options, sdk, done, logJson) { var stats = new StatusStats(); + var concurrentTransfers = options.concurrentTransfers || configDefaults.concurrentTransfers; var fileUris = task.getFileUris(); - asyncUtil.eachLimit(fileUris, 10, function(fileUri, callback) { + asyncUtil.eachLimit(fileUris, concurrentTransfers, function(fileUri, callback) { sdk.status(fileUri, options.operation.locale) .then(function(statusInfo) { if (options.verbose) { diff --git a/tasks/smartling_upload.js b/tasks/smartling_upload.js index 6b044bc..ebe0594 100644 --- a/tasks/smartling_upload.js +++ b/tasks/smartling_upload.js @@ -11,19 +11,21 @@ 'use strict'; module.exports = function (grunt) { - var asyncUtil = require('async'), - SmartlingTask = require('../lib/smartling-task'), - UploadStats = require('../lib/upload-stats'); + var asyncUtil = require('async'), + SmartlingTask = require('../lib/smartling-task'), + UploadStats = require('../lib/upload-stats'), + configDefaults = require('./config_defaults'); grunt.registerMultiTask('smartling_upload', 'Upload files to Smartling', SmartlingTask.make(function (task, options, sdk, done, logJson) { var stats = new UploadStats(); + var concurrentTransfers = options.concurrentTransfers || configDefaults.concurrentTransfers; if (task.files) { task.files.forEach(function (file) { //logJson(file); - asyncUtil.eachLimit(file.src, 10, function (filepath, callback) { + asyncUtil.eachLimit(file.src, concurrentTransfers, function (filepath, callback) { var fileUri = options.fileUriFunc(filepath); sdk.upload(filepath, fileUri, options.operation.fileType, options.operation) From 77930251fe6e0ec7ae90507896d94ffc9ecce772 Mon Sep 17 00:00:00 2001 From: Jason Buckner Date: Thu, 13 Aug 2015 14:58:37 -0700 Subject: [PATCH 2/2] =?UTF-8?q?don=E2=80=99t=20forget=20the=20semicolon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tasks/config_defaults.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/config_defaults.js b/tasks/config_defaults.js index 1758921..bd25fd4 100644 --- a/tasks/config_defaults.js +++ b/tasks/config_defaults.js @@ -1,3 +1,3 @@ module.exports = { concurrentTransfers: 10 -} +};