From 84a06a28c736316116af7b4e5a2e6af86be848bb Mon Sep 17 00:00:00 2001 From: Ravi Gairola Date: Mon, 4 Apr 2016 14:24:59 -0700 Subject: [PATCH] Added html encode option --- index.js | 103 +++++++++++++++++++++++++---------------- package.json | 3 +- readme.md | 32 +++++++++---- test/index.test.js | 15 +++++- test/locales/de_DE.ini | 1 + 5 files changed, 102 insertions(+), 52 deletions(-) diff --git a/index.js b/index.js index 513e719..b577c91 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,7 @@ var path = require('path'); var _ = require('lodash'); var flat = require('flat'); var gutil = require('gulp-util'); +var he = require('he'); var through = require('through2'); @@ -38,49 +39,67 @@ var defaults = { ignoreErrors: false, dryRun: false, includeOriginal: false, - ignoreTokens: false + ignoreTokens: false, + encodeEntities: true }; +/** + * A helper function to test whether an option was set to true or the value matches the options regular expression. + * @param {boolean|string|RegExp} needle + * @param {String} haystack + * @returns {boolean} + */ +function trueOrMatch(needle, haystack) { + if (needle === true) { + return true; + } + if (needle instanceof RegExp && needle.test(haystack)) { + return true; + } + return !!(needle instanceof String && haystack.indexOf(needle) !== -1); + +} + + /** * Loads the dictionaries from the locale directory. * @param {Object} options */ function load(options) { if (cache[options.locales]) { - dictionaries = cache[options.locales]; - } else { - try { - var files = fs.readdirSync(options.locales); - for (var i in files) { - var file = files[i]; - switch (path.extname(file)) { - case '.json': - case '.js': - dictionaries[path.basename(file, path.extname(file))] = flat(require(path.join(process.cwd(), options.locales, file))); - break; - case '.ini': - var iniData = fs.readFileSync(path.join(process.cwd(), options.locales, file)); - dictionaries[path.basename(file, path.extname(file))] = flat(ini2json(iniData)); - break; - case '.csv': - var csvData = fs.readFileSync(path.join(process.cwd(), options.locales, file)); - dictionaries[path.basename(file, path.extname(file))] = csv2json(csvData); - break; - } - } - if (options.cache) { - cache[options.locales] = dictionaries; + return dictionaries = cache[options.locales]; + } + try { + var files = fs.readdirSync(options.locales); + for (var i in files) { + var file = files[i]; + switch (path.extname(file)) { + case '.json': + case '.js': + dictionaries[path.basename(file, path.extname(file))] = flat(require(path.join(process.cwd(), options.locales, file))); + break; + case '.ini': + var iniData = fs.readFileSync(path.join(process.cwd(), options.locales, file)); + dictionaries[path.basename(file, path.extname(file))] = flat(ini2json(iniData)); + break; + case '.csv': + var csvData = fs.readFileSync(path.join(process.cwd(), options.locales, file)); + dictionaries[path.basename(file, path.extname(file))] = csv2json(csvData); + break; } - } catch (e) { - throw new Error('No translation dictionaries have been found!'); } + if (options.cache) { + cache[options.locales] = dictionaries; + } + } catch (e) { + throw new Error('No translation dictionaries have been found!'); } } /** * Splits a line from an ini file into 2. Any subsequent '=' are ignored. - * @param {string} line - * @returns {string[]} + * @param {String} line + * @returns {String[]} */ function splitIniLine(line) { var separator = line.indexOf('='); @@ -95,7 +114,7 @@ function splitIniLine(line) { /** * Simple conversion helper to get a json file from an ini file. - * @param {string} iniData + * @param {String} iniData * @returns {{}} */ function ini2json(iniData) { @@ -127,8 +146,8 @@ function ini2json(iniData) { /** * Converts a line of a CSV file to an array of strings, omitting empty fields. - * @param {string} line - * @returns {string[]} + * @param {String} line + * @returns {String[]} */ function splitCsvLine(line) { if (!line.trim().length) { @@ -163,7 +182,7 @@ function splitCsvLine(line) { /** * Simple conversion helper to get a json file from a csv file. - * @param {string} csvData + * @param {String} csvData * @returns {Object} */ function csv2json(csvData) { @@ -187,9 +206,9 @@ function csv2json(csvData) { /** * Performs the actual translation from a tokenized source to the final content. * @param {Object} options - * @param {string} contents + * @param {String} contents * @param {number} copied - * @param {string} filePath + * @param {String} filePath * @returns {Object} */ function translate(options, contents, copied, filePath) { @@ -205,7 +224,7 @@ function translate(options, contents, copied, filePath) { throw new Error('No translation dictionaries available to create any files!'); } var i = contents.indexOf(options.delimiter.prefix); - if (!(options.ignoreTokens === true || options.ignoreTokens instanceof RegExp && options.ignoreTokens.test(filePath))) { + if (!trueOrMatch(options.ignoreTokens, filePath)) { while ((i !== -1)) { var endMatch, length, token, key; var tail = contents.substr(i); @@ -226,8 +245,12 @@ function translate(options, contents, copied, filePath) { for (var lang in processed) { processed[lang] += contents.substring(copied, i); if (dictionaries[lang][key] !== undefined) { - processed[lang] += dictionaries[lang][key]; - } else if (options.warn) { + if (trueOrMatch(options.encodeEntities, filePath)) { + processed[lang] += he.encode(dictionaries[lang][key], { useNamedReferences: true }); + } else { + processed[lang] += dictionaries[lang][key]; + } + } else if (trueOrMatch(options.warn, filePath)) { gutil.log('Missing translation of language', lang, 'for key', key, 'in file', filePath); } processed[lang] += contents.substring(i + length, next == -1 ? contents.length : next); @@ -315,10 +338,10 @@ module.exports = function(options) { try { var files = replace(file, options); - if (options.dryRun === true || options.dryRun instanceof RegExp && options.dryRun.test(file.path)) { + if (trueOrMatch(options.dryRun, file.path)) { this.push(file); } else { - if (options.includeOriginal === true || options.includeOriginal instanceof RegExp && options.includeOriginal.test(file.path)) { + if (trueOrMatch(options.includeOriginal, file.path)) { this.push(file); } for (var i in files) { @@ -326,7 +349,7 @@ module.exports = function(options) { } } } catch (err) { - if (!options.ignoreErrors) { + if (!trueOrMatch(options.ignoreErrors, file.path)) { this.emit('error', new gutil.PluginError('gulp-international', err)); } } diff --git a/package.json b/package.json index 4ccfd25..3857a22 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gulp-international", - "version": "1.0.7", + "version": "1.0.8", "description": "A gulp plugin that creates multi language versions of your source files", "license": "Apache-2.0", "homepage": "http://github.com/mallocator/gulp-international", @@ -46,6 +46,7 @@ "dependencies": { "flat": "^2.0.0", "gulp-util": "^3.0.1", + "he": "^0.5.0", "lodash": "^4.6.1", "through2": "^2.0.1" }, diff --git a/readme.md b/readme.md index 75967c6..5a3e1ad 100644 --- a/readme.md +++ b/readme.md @@ -155,12 +155,23 @@ The opposite of the whitelist. Any language specified here will be ignored durin with a string or multiple with an array of strings. +### encodeEntities + +Type: boolean|RegExp|String +Default: ```true``` + +Any non utf8 characters that do not map to html are replaced with html entities if this option is enabled. A translation such as "über" +would be replaced with "über". If the setting is a regular expression then only files with a matching (original) filename will be +escaped. If the option is instead a string then the filename is searched for this substring. + + ### warn -Type: boolean +Type: boolean|RegExp|String Default: ```true``` -This enables warnings to be printed out if any tokens are missing. +This enables warnings to be printed out if any tokens are missing. If the setting is a regular expression then only files with a matching +(original) filename will throw warnings. If the option is instead a string then the filename is searched for this substring. ### cache @@ -174,20 +185,22 @@ on your configuration you might want to disable caching based on how your livere ### ignoreErrors -Type: boolean +Type: boolean|RegExp|String Default: ```false``` -Allows to disable throwing of any errors that might occur so that the pipe will continue executing. +Allows to disable throwing of any errors that might occur so that the pipe will continue executing. If the setting is a regular expression +then only files with a matching (original) filename will ignore errors. If the option is instead a string then the filename is searched for +this substring. ### dryRun -Type: boolean|RegExp +Type: boolean|RegExp|String Default: ```false``` When set to true the plugin will perform all operations for a translation, but will pass on the original file along the pipe instead of the newly generated ones. If the setting is a regular expression then only files with a matching (original) filename will be -ignored. +ignored. If the option is instead a string then the filename is searched for this substring. Flow graph: ``` @@ -202,7 +215,7 @@ Default: ```false``` When set to true the plugin will ignore all tokens, but still create new files as if they were different for each language. This differs from a dryRun, which would instead pass on the original file. If the setting is a regular expression then only files -with a matching (original) filename will be ignored. +with a matching (original) filename will be ignored. If the option is instead a string then the filename is searched for this substring. Flow graph: ``` @@ -215,8 +228,9 @@ source.file -> source-lang1.file -> original content Type: boolean|RegExp Default: ```false``` -When set to true the original file is passed along the pipe along with all translated files. If the setting is a -regular expression then only files with a matching (original) filename will be included. +When set to true the original file is passed along the pipe along with all translated files. If the setting is a regular expression then +only files with a matching (original) filename will be included. If the option is instead a string then the filename is searched for this +substring. Flow graph: ``` diff --git a/test/index.test.js b/test/index.test.js index e66d403..be739e7 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -53,7 +53,7 @@ describe('gulp-international', () => { it('should not process languages on the blacklist', done => { - var options = { locales: 'test/locales', blacklist: 'en_US' }; + var options = { locales: 'test/locales', blacklist: 'en_US', encodeEntities: false }; helper(options, files => { expect(files.length).to.equal(3); expect(files[0].contents.toString('utf8')).to.equal('

Inhalt1

'); @@ -173,6 +173,16 @@ describe('gulp-international', () => { done(); }); }); + + + it('should translate html entities if the option is enabled (default)', done => { + var content = '

R.entity

'; + var options = { locales: 'test/locales', whitelist: 'de_DE' }; + helper(options, content, files => { + expect(files[0].contents.toString()).to.equal('

Süpär Späßö!

'); + done(); + }); + }); }); describe('Error cases', () => { @@ -345,7 +355,8 @@ describe('gulp-international', () => { `; var options = { locales: 'test/locales', - whitelist: ['en_US', 'pt-BR'] + whitelist: ['en_US', 'pt-BR'], + encodeEntities: false }; helper(options, content, files => { expect(files.length).to.equal(2); diff --git a/test/locales/de_DE.ini b/test/locales/de_DE.ini index 451db17..453b340 100644 --- a/test/locales/de_DE.ini +++ b/test/locales/de_DE.ini @@ -1,4 +1,5 @@ token1=Inhalt1 +entity=Süpär Späßö! [section1] token2=Inhalt2