From fda3fc74c3d110c625d46f77583b8fd06c829ddc Mon Sep 17 00:00:00 2001 From: Pete Lawrence Date: Thu, 9 Feb 2017 12:21:15 +0000 Subject: [PATCH 1/2] Added handler to convert numeric fields in Cloudfront logs to numbers, so that they can be handled as numbers in ES --- handlers/convertCloudfrontFieldTypes.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 handlers/convertCloudfrontFieldTypes.js diff --git a/handlers/convertCloudfrontFieldTypes.js b/handlers/convertCloudfrontFieldTypes.js new file mode 100644 index 0000000..001ab5e --- /dev/null +++ b/handlers/convertCloudfrontFieldTypes.js @@ -0,0 +1,16 @@ +exports.process = function(config) { + console.log('convertCloudfrontFieldTypes'); + + var records = config.data; + + for (var i = 0; i < records.length; i++) { + var record = records[i]; + + record['sc-bytes'] = parseInt(record['sc-bytes']); + record['sc-status'] = parseInt(record['sc-status']); + record['cs-bytes'] = parseInt(record['cs-bytes']); + record['time-taken'] = parseFloat(record['time-taken']); + } + + return Promise.resolve(config); +}; From 49535ebf5fb36c2b1238e2baeabaf708c7a0621f Mon Sep 17 00:00:00 2001 From: Pete Lawrence Date: Wed, 3 May 2017 10:22:24 +0100 Subject: [PATCH 2/2] Made elasticsearch requestTimeout configurable --- handlers/shipElasticsearch.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/handlers/shipElasticsearch.js b/handlers/shipElasticsearch.js index 0365907..54df51d 100644 --- a/handlers/shipElasticsearch.js +++ b/handlers/shipElasticsearch.js @@ -7,6 +7,10 @@ exports.process = function(config) { host: config.elasticsearch.host }; + if (config.elasticsearch.requestTimeout) { + esConfig.requestTimeout = config.elasticsearch.requestTimeout; + } + if (config.elasticsearch.useAWS) { esConfig.connectionClass = require('http-aws-es'); esConfig.amazonES = {