Merge pull request #23 from arithmetric/add_elb_support

Adds support for Amazon Elastic Load Balancer (ELB) logs
arithmetric · May 15, 2018 · ae32d90 · ae32d90
2 parents e3cf44c + 9306619
commit ae32d90
Show file tree

Hide file tree

Showing 12 changed files with 497 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -10,13 +10,13 @@ data sources to data stores, like Elasticsearch.
 ## Features
 
 - Supported input formats: AWS Cloudfront access logs, AWS Cloudtrail API logs,
-AWS CloudWatch logs, AWS Config change logs, and other formats by implementing
-a custom handler.
+AWS CloudWatch logs, AWS Config change logs, AWS Elastic Load Balancer access
+logs, and other formats by implementing a custom handler.
 
 - Supported output formats: JSON, plain text key-value pairs.
 
 - Supported shipping methods: Elasticsearch (includes signing for AWS domains),
-HTTP POST, TCP socket.
+HTTP/S POST, TCP socket.
 
 ## Set Up
 
@@ -25,7 +25,7 @@ with your configuration. See the example included (under `example/`) and the
 configuration documentation below to get started.
 
 2. Use the AWS Management Console to create a Lambda function using the Node.js
-4.3 runtime. Upload your package, configure it with event sources as desired
+6.10 runtime. Upload your package, configure it with event sources as desired
 (S3 buckets or CloudWatch logs). Be sure the Lambda function has an IAM role
 with any necessary permissions, like getting data from an S3 bucket or accessing
 an AWS Elasticsearch domain.
@@ -169,6 +169,46 @@ contain the log's date time value.
 
 - `config.data` is transformed to an array of objects with log data.
 
+### formatELBv1
+
+Processes parsed data from AWS Elastic Load Balancer (ELB) Classic Load Balancer
+(ELB version 1) logs and normalizes it in key-value objects.
+
+Raw ELB log files in S3 should be processed with `parseSpaces` before using this
+format handler.
+
+**Inputs**:
+
+- `config.data` should be an array (one item per log record) of arrays (one item
+per record field).
+
+- `config.dateField` (optional) is the key name in the output that should
+contain the log's date time value.
+
+**Outputs**:
+
+- `config.data` is transformed to an array of objects with log data.
+
+### formatELBv2
+
+Processes parsed data from AWS Elastic Load Balancer (ELB) Application Load
+Balancer (ELB version 2) logs and normalizes it in key-value objects.
+
+Raw ELB log files in S3 should be processed with `parseSpaces` before using this
+format handler.
+
+**Inputs**:
+
+- `config.data` should be an array (one item per log record) of arrays (one item
+per record field).
+
+- `config.dateField` (optional) is the key name in the output that should
+contain the log's date time value.
+
+**Outputs**:
+
+- `config.data` is transformed to an array of objects with log data.
+
 ### getS3Object
 
 Fetches an object from S3.
@@ -273,9 +313,9 @@ signing the request.
 - `config.elasticsearch.maxChunkSize` (optional) is the maximum number of log
 items to ship in one request. By default, set to 1000.
 
-- `config.elasticsearch.requestTimeout` (optional) is the elasticsearch client
-timeout in milliseconds.  Elasticsearch's default is 30000 (30 seconds). Can
-also be set to 'Infinity'.
+- `config.elasticsearch.requestTimeout` (optional) is the Elasticsearch client
+timeout in milliseconds or can be set to `Infinity` for no timeout. The
+Elasticsearch client's default is 30000 (30 seconds).
 
 ### shipHttp
 

diff --git a/handlers/formatELBv1.js b/handlers/formatELBv1.js
@@ -0,0 +1,53 @@
+exports.process = function(config) {
+  console.log('formatELB for Classic Load Balancers');
+  if (!config.data ||
+      (!config.data.length && config.data.length !== 0)) {
+    return Promise.reject('Received unexpected ELB log format:' +
+      JSON.stringify(config.data));
+  }
+
+  var output = [];
+  var fields = [
+    'timestamp',
+    'elb',
+    'client',
+    'backend',
+    'request_processing_time',
+    'backend_processing_time',
+    'response_processing_time',
+    'elb_status_code',
+    'backend_status_code',
+    'received_bytes',
+    'sent_bytes',
+    'request',
+    'user_agent',
+    'ssl_cipher',
+    'ssl_protocol'
+  ];
+  var numRows = config.data.length;
+  var numCols;
+  var i;
+  var j;
+  var row;
+  var item;
+
+  for (i = 0; i < numRows; i++) {
+    row = config.data[i];
+    if (row.length !== 15) {
+      console.log('Expected 15 columns in row. Found ' + row.length + ': ' +
+        config.data[i]);
+    }
+
+    item = {};
+    numCols = Math.min(row.length, fields.length);
+    for (j = 0; j < numCols; j++) {
+      item[fields[j]] = row[j];
+    }
+    if (config.dateField && config.dateField !== 'timestamp') {
+      item[config.dateField] = item.timestamp;
+    }
+    output.push(item);
+  }
+  config.data = output;
+  return Promise.resolve(config);
+};
diff --git a/handlers/formatELBv2.js b/handlers/formatELBv2.js
@@ -0,0 +1,59 @@
+exports.process = function(config) {
+  console.log('formatELBv2 for Application Load Balancers');
+  if (!config.data ||
+      (!config.data.length && config.data.length !== 0)) {
+    return Promise.reject('Received unexpected ELB log format:' +
+      JSON.stringify(config.data));
+  }
+
+  var output = [];
+  var fields = [
+    'type',
+    'timestamp',
+    'elb',
+    'client',
+    'target',
+    'request_processing_time',
+    'target_processing_time',
+    'response_processing_time',
+    'elb_status_code',
+    'target_status_code',
+    'received_bytes',
+    'sent_bytes',
+    'request',
+    'user_agent',
+    'ssl_cipher',
+    'ssl_protocol',
+    'target_group_arn',
+    'trace_id',
+    'domain_name',
+    'chosen_cert_arn',
+    'matched_rule_priority'
+  ];
+  var numRows = config.data.length;
+  var numCols;
+  var i;
+  var j;
+  var row;
+  var item;
+
+  for (i = 0; i < numRows; i++) {
+    row = config.data[i];
+    if (row.length !== 21) {
+      console.log('Expected 21 columns in row. Found ' + row.length + ': ' +
+        config.data[i]);
+    }
+
+    item = {};
+    numCols = Math.min(row.length, fields.length);
+    for (j = 0; j < numCols; j++) {
+      item[fields[j]] = row[j];
+    }
+    if (config.dateField && config.dateField !== 'timestamp') {
+      item[config.dateField] = item.timestamp;
+    }
+    output.push(item);
+  }
+  config.data = output;
+  return Promise.resolve(config);
+};
diff --git a/handlers/parseSpaces.js b/handlers/parseSpaces.js
@@ -0,0 +1,18 @@
+var parse = require('csv-parse');
+
+exports.process = function(config) {
+  console.log('parseSpaces');
+  return new Promise(function(resolve, reject) {
+    parse(config.data, {
+      delimiter: ' ',
+      relax_column_count: true, // eslint-disable-line camelcase
+      trim: true
+    }, function(err, data) {
+      if (err) {
+        return reject(err);
+      }
+      config.data = data;
+      resolve(config);
+    });
+  });
+};
diff --git a/test/assets/elbv1.format.json b/test/assets/elbv1.format.json
@@ -0,0 +1,38 @@
+[
+  {
+    "timestamp": "2018-05-09T12:20:13.464900Z",
+    "elb": "elb-test",
+    "client": "12.23.34.45:5678",
+    "backend": "10.0.10.1:8080",
+    "request_processing_time": "0.000039",
+    "backend_processing_time": "0.000905",
+    "response_processing_time": "0.000022",
+    "elb_status_code": "301",
+    "backend_status_code": "301",
+    "received_bytes": "0",
+    "sent_bytes": "178",
+    "request": "GET http://elb-test-1234567890.us-east-1.elb.amazonaws.com:80/ HTTP/1.1",
+    "user_agent": "Amazon Route 53 Health Check Service; ref:1234567890;",
+    "ssl_cipher": "-",
+    "ssl_protocol": "-",
+    "date": "2018-05-09T12:20:13.464900Z"
+  },
+  {
+    "timestamp": "2018-05-09T12:20:14.464900Z",
+    "elb": "elb-test",
+    "client": "12.23.34.45:5678",
+    "backend": "10.0.10.1:8080",
+    "request_processing_time": "0.000039",
+    "backend_processing_time": "0.000905",
+    "response_processing_time": "0.000022",
+    "elb_status_code": "301",
+    "backend_status_code": "301",
+    "received_bytes": "0",
+    "sent_bytes": "178",
+    "request": "GET http://elb-test-1234567890.us-east-1.elb.amazonaws.com:80/ HTTP/1.1",
+    "user_agent": "Amazon Route 53 Health Check Service; ref:1234567890;",
+    "ssl_cipher": "-",
+    "ssl_protocol": "-",
+    "date": "2018-05-09T12:20:14.464900Z"
+  }
+]
diff --git a/test/assets/elbv1.parse.json b/test/assets/elbv1.parse.json
@@ -0,0 +1,37 @@
+[
+  [
+    "2018-05-09T12:20:13.464900Z",
+    "elb-test",
+    "12.23.34.45:5678",
+    "10.0.10.1:8080",
+    "0.000039",
+    "0.000905",
+    "0.000022",
+    "301",
+    "301",
+    "0",
+    "178",
+    "GET http://elb-test-1234567890.us-east-1.elb.amazonaws.com:80/ HTTP/1.1",
+    "Amazon Route 53 Health Check Service; ref:1234567890;",
+    "-",
+    "-"
+  ],
+  [
+    "2018-05-09T12:20:14.464900Z",
+    "elb-test",
+    "12.23.34.45:5678",
+    "10.0.10.1:8080",
+    "0.000039",
+    "0.000905",
+    "0.000022",
+    "301",
+    "301",
+    "0",
+    "178",
+    "GET http://elb-test-1234567890.us-east-1.elb.amazonaws.com:80/ HTTP/1.1",
+    "Amazon Route 53 Health Check Service; ref:1234567890;",
+    "-",
+    "-",
+    "new field"
+  ]
+]
diff --git a/test/assets/elbv2.format.json b/test/assets/elbv2.format.json
@@ -0,0 +1,50 @@
+[
+  {
+    "type": "https",
+    "timestamp": "2016-08-10T23:39:43.065466Z",
+    "elb": "app/my-loadbalancer/50dc6c495c0c9188",
+    "client": "192.168.131.39:2817",
+    "target": "10.0.0.1:80",
+    "request_processing_time": "0.086",
+    "target_processing_time": "0.048",
+    "response_processing_time": "0.037",
+    "elb_status_code": "200",
+    "target_status_code": "200",
+    "received_bytes": "0",
+    "sent_bytes": "57",
+    "request": "GET https://www.example.com:443/ HTTP/1.1",
+    "user_agent": "curl/7.46.0",
+    "ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+    "ssl_protocol": "TLSv1.2",
+    "target_group_arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+    "trace_id": "Root=1-58337281-1d84f3d73c47ec4e58577259",
+    "domain_name": "www.example.com",
+    "chosen_cert_arn": "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012",
+    "matched_rule_priority": "0",
+    "date": "2016-08-10T23:39:43.065466Z"
+  },
+  {
+    "type": "https",
+    "timestamp": "2016-08-10T23:39:44.065466Z",
+    "elb": "app/my-loadbalancer/50dc6c495c0c9188",
+    "client": "192.168.131.39:2817",
+    "target": "10.0.0.1:80",
+    "request_processing_time": "0.086",
+    "target_processing_time": "0.048",
+    "response_processing_time": "0.037",
+    "elb_status_code": "200",
+    "target_status_code": "200",
+    "received_bytes": "0",
+    "sent_bytes": "57",
+    "request": "GET https://www.example.com:443/ HTTP/1.1",
+    "user_agent": "curl/7.46.0",
+    "ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+    "ssl_protocol": "TLSv1.2",
+    "target_group_arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+    "trace_id": "Root=1-58337281-1d84f3d73c47ec4e58577259",
+    "domain_name": "www.example.com",
+    "chosen_cert_arn": "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012",
+    "matched_rule_priority": "0",
+    "date": "2016-08-10T23:39:44.065466Z"
+  }
+]
diff --git a/test/assets/elbv2.parse.json b/test/assets/elbv2.parse.json
@@ -0,0 +1,49 @@
+[
+  [
+    "https",
+    "2016-08-10T23:39:43.065466Z",
+    "app/my-loadbalancer/50dc6c495c0c9188",
+    "192.168.131.39:2817",
+    "10.0.0.1:80",
+    "0.086",
+    "0.048",
+    "0.037",
+    "200",
+    "200",
+    "0",
+    "57",
+    "GET https://www.example.com:443/ HTTP/1.1",
+    "curl/7.46.0",
+    "ECDHE-RSA-AES128-GCM-SHA256",
+    "TLSv1.2",
+    "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+    "Root=1-58337281-1d84f3d73c47ec4e58577259",
+    "www.example.com",
+    "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012",
+    "0"
+  ],
+  [
+    "https",
+    "2016-08-10T23:39:44.065466Z",
+    "app/my-loadbalancer/50dc6c495c0c9188",
+    "192.168.131.39:2817",
+    "10.0.0.1:80",
+    "0.086",
+    "0.048",
+    "0.037",
+    "200",
+    "200",
+    "0",
+    "57",
+    "GET https://www.example.com:443/ HTTP/1.1",
+    "curl/7.46.0",
+    "ECDHE-RSA-AES128-GCM-SHA256",
+    "TLSv1.2",
+    "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+    "Root=1-58337281-1d84f3d73c47ec4e58577259",
+    "www.example.com",
+    "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012",
+    "0",
+    "new field"
+  ]
+]
diff --git a/test/assets/elbv2.source.txt b/test/assets/elbv2.source.txt
@@ -0,0 +1,2 @@
+https 2016-08-10T23:39:43.065466Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" www.example.com arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012 0
+https 2016-08-10T23:39:44.065466Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" www.example.com arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012 0 "new field"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		https 2016-08-10T23:39:43.065466Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" www.example.com arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012 0
		https 2016-08-10T23:39:44.065466Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" www.example.com arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012 0 "new field"