[Cloud Security] POC change aws securityhub data stream into full posture

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.32.0"
+  changes:
+    - description: Turn SecurityHub Findings data stream into full posture data stream
+      type: enhancement
+      link: tbd
 - version: "2.31.3"
   changes:
     - description: Removed the `reducedTimeRange` filter from the AWS Billing Total Estimated Charges lens to ensure value is displayed.

packages/aws/data_stream/securityhub_findings/agent/stream/httpjson.yml.hbs

@@ -1,5 +1,5 @@
 config_version: 2
-interval: {{interval}}
+interval: 24h
 {{#if enable_request_tracer}}
 request.tracer.filename: "../../logs/httpjson/http-request-trace-*.ndjson"
 request.tracer.maxbackups: 5
@@ -24,13 +24,12 @@ request.transforms:
       value: 100
       value_type: int
   - set:
-      target: body.SortCriteria
-      value: '[{"Field":"UpdatedAt","SortOrder":"asc"}]'
+      target: body.Filters.RecordState 
+      value: '[{"Comparison":"NOT_EQUALS","Value":"ARCHIVED"}]'
       value_type: json
   - set:
-      target: body.Filters.UpdatedAt
-      value: '[{ "Start": "[[formatDate (parseDate .cursor.last_execution_datetime "RFC3339") "2006-01-02T15"]]", "End": "[[formatDate (now) "2006-01-02T15"]]" }]'
-      default: '[{ "Start": "[[formatDate (now (parseDuration "-{{initial_interval}}")) "2006-01-02T15"]]", "End": "[[formatDate (now) "2006-01-02T15"]]" }]'
+      target: body.Filters.WorkflowStatus 
+      value: '[{"Comparison":"NOT_EQUALS","Value":"SUPPRESSED"},{"Comparison":"NOT_EQUALS","Value":"RESOLVED"}]'
       value_type: json
   - set:
       target: header.Authorization
@@ -46,9 +45,6 @@ response.pagination:
   - set:
       target: header.Authorization
       value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/securityhub/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "securityhub")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/securityhub/aws4_request") (hash "sha256" "POST\n" "/findings\n" "\n" "host:securityhub.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]'
-cursor:
-  last_execution_datetime:
-    value: '[[if (ne (len .last_response.body.Findings) 100)]][[.last_event.UpdatedAt]][[end]]'
 response.split:
   target: body.Findings
   ignore_empty_value: true

packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml

@@ -584,9 +584,8 @@ processors:
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
       field: '@timestamp'
-      copy_from: aws.securityhub_findings.updated_at
+      value: "{{{_ingest.timestamp}}}"
       tag: set_timestamp
-      ignore_empty_value: true
   - convert:
       field: json.Criticality
       target_field: aws.securityhub_findings.criticality

packages/aws/data_stream/securityhub_findings/manifest.yml

@@ -6,29 +6,13 @@ streams:
     description: Collect AWS Security Hub Findings from AWS.
     template_path: httpjson.yml.hbs
     vars:
-      - name: interval
-        type: text
-        title: Interval
-        description: Interval to fetch AWS Security Hub Findings from AWS. (Interval should be greater than 1 hour.)
-        multi: false
-        required: true
-        show_user: true
-        default: 1h
       - name: enable_request_tracer
         type: bool
         title: Enable request tracing
         multi: false
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
-      - name: initial_interval
-        type: text
-        title: Initial Interval
-        description: How far back to pull the AWS Security Hub Findings from AWS. (Initial Interval should be greater than 1 hour.)
-        default: 24h
-        multi: false
-        required: true
-        show_user: true
       - name: tags
         type: text
         title: Tags

packages/aws/elasticsearch/transform/latest_cdr_misconfigurations/transform.yml

@@ -2,7 +2,7 @@ source:
   index:
     - "logs-aws.securityhub_findings-*"
 dest:
-  index: "security_solution-aws.misconfiguration_latest-v1"
+  index: "security_solution-aws.misconfiguration_latest-v2"
   aliases:
     - alias: "security_solution-aws.misconfiguration_latest"
       move_on_creation: true
@@ -20,11 +20,11 @@ sync:
 retention_policy:
   time:
     field: "@timestamp"
-    max_age: 90d
+    max_age: 24h
 settings:
   unattended: true
 _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.2.0

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: aws
 title: AWS
-version: 2.31.3
+version: 2.32.0
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories: