Skip to content

Commit

Permalink
prevent the "invalid byte sequence for encoding UTF8: 0x00" issue
Browse files Browse the repository at this point in the history
  • Loading branch information
AnatolyUss committed Mar 25, 2017
1 parent 6e32b73 commit c97a026
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 5 deletions.
24 changes: 20 additions & 4 deletions migration/fmtp/CsvStringifyModified.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,23 @@
util = require('util');

module.exports = function() {
var callback, chunks, data, options, stringifier;
var callback, chunks, data, options, stringifier, encoding;
if (arguments.length === 3) {
data = arguments[0];
// Original version.
/*data = arguments[0];
options = arguments[1];
callback = arguments[2];
callback = arguments[2];*/
if (Array.isArray(arguments[0])) {
data = arguments[0];
} else {
options = arguments[0];
}
if (typeof arguments[1] === 'function') {
callback = arguments[1];
} else {
options = arguments[1];
}
encoding = arguments[2];
} else if (arguments.length === 2) {
if (Array.isArray(arguments[0])) {
data = arguments[0];
Expand All @@ -36,7 +48,10 @@
if (options == null) {
options = {};
}

stringifier = new Stringifier(options);
stringifier._encoding = encoding;

if (data) {
process.nextTick(function() {
var d, j, len;
Expand Down Expand Up @@ -254,6 +269,7 @@
field = JSON.stringify(field);
}
if (field) {
field = this._encoding === 'utf8' ? field.replace(/\u0000/g, ' ') : field;
containsdelimiter = field.indexOf(delimiter) >= 0;
containsQuote = field.indexOf(quote) >= 0;
containsLinebreak = field.indexOf('\r') >= 0 || field.indexOf('\n') >= 0;
Expand All @@ -268,7 +284,7 @@
} else if (this.options.quotedEmpty || ((this.options.quotedEmpty == null) && line[i] === '' && this.options.quotedString)) {
newLine += quote + quote;
}

if (i !== line.length - 1) {
newLine += delimiter;
}
Expand Down
2 changes: 1 addition & 1 deletion migration/fmtp/DataLoader.js
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ const populateTableWorker = (self, tableName, strSelectFieldList, offset, rowsIn
}
});
}
});
}, self._encoding);
}
});
}
Expand Down

0 comments on commit c97a026

Please sign in to comment.