From 9b3c45e6bb11753fc9bee13b499e9dcdf631db21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 12 Sep 2024 17:47:49 +0200 Subject: [PATCH] Add stateless index template definition (#395) * Add stateless index template definition Event generator is pending * Update to 8.11.0 * Update ECS generator * Remove event generator for stateless ECS module * Remove commented code * Fix typo --- ecs/README.md | 12 +- ecs/alerts/fields/mapping-settings.json | 4 + ecs/alerts/fields/subset.yml | 596 ++++++++++++++++++ .../fields/template-settings-legacy.json | 18 + ecs/alerts/fields/template-settings.json | 18 + ecs/generate.sh | 18 +- 6 files changed, 656 insertions(+), 10 deletions(-) create mode 100644 ecs/alerts/fields/mapping-settings.json create mode 100644 ecs/alerts/fields/subset.yml create mode 100644 ecs/alerts/fields/template-settings-legacy.json create mode 100644 ecs/alerts/fields/template-settings.json diff --git a/ecs/README.md b/ecs/README.md index d4897318585e2..a3818271a5e53 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -45,25 +45,25 @@ files to generate the mappings. These are the inputs for the ECS generator. * INDEXER_SRC: Path to the wazuh-indexer repository * MODULE: Module to generate mappings for * --upload : Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200 - Example: generate.sh v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200 + Example: generate.sh v8.11.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200 ``` 3. Use the `generate.sh` script to generate the mappings for a module. The script takes 3 arguments, plus 2 optional arguments to upload the mappings to the `wazuh-indexer`. Both, composable and legacy mappings are generated. For example, to generate the mappings for the `vulnerability-detector` module using the - ECS version `v8.10.0` and assuming that path of this repository is `~/wazuh/wazuh-indexer`: + ECS version `v8.11.0` and assuming that path of this repository is `~/wazuh/wazuh-indexer`: ```bash - ./generate.sh v8.10.0 ~/wazuh/wazuh-indexer vulnerability-detector + ./generate.sh v8.11.0 ~/wazuh/wazuh-indexer vulnerability-detector ``` The tool will output the folder where they have been generated. ```console - Loading schemas from git ref v8.10.0 - Running generator. ECS version 8.10.0 + Loading schemas from git ref v8.11.0 + Running generator. ECS version 8.11.0 Replacing "match_only_text" type with "text" - Mappings saved to ~/wazuh/wazuh-indexer/ecs/vulnerability-detector/mappings/v8.10.0 + Mappings saved to ~/wazuh/wazuh-indexer/ecs/vulnerability-detector/mappings/v8.11.0 ``` 4. When you are done. Exit the virtual environment. diff --git a/ecs/alerts/fields/mapping-settings.json b/ecs/alerts/fields/mapping-settings.json new file mode 100644 index 0000000000000..0ad2b48fcc1be --- /dev/null +++ b/ecs/alerts/fields/mapping-settings.json @@ -0,0 +1,4 @@ +{ + "dynamic": "strict", + "date_detection": false +} \ No newline at end of file diff --git a/ecs/alerts/fields/subset.yml b/ecs/alerts/fields/subset.yml new file mode 100644 index 0000000000000..fa784b9806d6c --- /dev/null +++ b/ecs/alerts/fields/subset.yml @@ -0,0 +1,596 @@ +--- +name: main +fields: + base: + fields: "*" + agent: + fields: "*" + as: + fields: "*" + client: + fields: + address: {} + as: + fields: "*" + bytes: {} + domain: {} + geo: + fields: "*" + ip: {} + mac: {} + nat: + fields: + ip: {} + port: {} + packets: {} + port: {} + subdomain: {} + registered_domain: {} + top_level_domain: {} + user: + fields: + domain: {} + email: {} + full_name: {} + group: + fields: "*" + hash: {} + id: {} + name: {} + roles: {} + cloud: + fields: "*" + code_signature: + fields: "*" + container: + fields: "*" + data_stream: + fields: "*" + destination: + fields: + address: {} + as: + fields: "*" + bytes: {} + domain: {} + geo: + fields: "*" + ip: {} + mac: {} + nat: + fields: + ip: {} + port: {} + packets: {} + port: {} + subdomain: {} + registered_domain: {} + top_level_domain: {} + user: + fields: + domain: {} + email: {} + full_name: {} + group: + fields: "*" + hash: {} + id: {} + name: {} + roles: {} + device: + fields: "*" + dll: + fields: "*" + dns: + fields: "*" + ecs: + fields: "*" + elf: + fields: "*" + email: + fields: "*" + error: + fields: "*" + event: + fields: "*" + faas: + fields: "*" + file: + fields: "*" + geo: + fields: "*" + group: + fields: "*" + hash: + fields: "*" + host: + fields: "*" + http: + fields: "*" + interface: + fields: "*" + log: + fields: "*" + macho: + fields: "*" + network: + fields: "*" + observer: + fields: "*" + orchestrator: + fields: "*" + organization: + fields: "*" + os: + fields: "*" + package: + fields: "*" + pe: + fields: "*" + process: + fields: + args: {} + args_count: {} + code_signature: + fields: "*" + command_line: {} + elf: + fields: "*" + end: {} + entity_id: {} + entry_leader: + fields: + args: {} + args_count: {} + command_line: {} + entity_id: {} + entry_meta: + fields: + type: {} + source: + fields: + ip: {} + executable: {} + interactive: {} + name: {} + parent: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + session_leader: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + pid: {} + vpid: {} + same_as_process: {} + start: {} + tty: + fields: + char_device: + fields: + major: {} + minor: {} + working_directory: {} + user: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + group: + fields: + id: {} + name: {} + real_group: + fields: + id: {} + name: {} + saved_group: + fields: + id: {} + name: {} + supplemental_groups: + fields: + id: {} + name: {} + attested_user: + fields: + id: {} + name: {} + attested_groups: + fields: + name: {} + entry_meta: + fields: + type: + docs_only: True + env_vars: {} + executable: {} + exit_code: {} + group_leader: + fields: + args: {} + args_count: {} + command_line: {} + entity_id: {} + executable: {} + interactive: {} + name: {} + pid: {} + vpid: {} + same_as_process: {} + start: {} + tty: + fields: + char_device: + fields: + major: {} + minor: {} + working_directory: {} + user: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + group: + fields: + id: {} + name: {} + real_group: + fields: + id: {} + name: {} + saved_group: + fields: + id: {} + name: {} + supplemental_groups: + fields: + id: {} + name: {} + hash: + fields: "*" + interactive: {} + io: + fields: "*" + macho: + fields: "*" + name: {} + parent: + fields: + args: {} + args_count: {} + code_signature: + fields: "*" + command_line: {} + elf: + fields: "*" + end: {} + entity_id: {} + executable: {} + exit_code: {} + group_leader: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + hash: + fields: "*" + interactive: {} + macho: + fields: "*" + name: {} + pe: + fields: "*" + pgid: {} + pid: {} + vpid: {} + start: {} + thread: + fields: + id: {} + name: {} + capabilities: + fields: + effective: {} + permitted: {} + title: {} + tty: + fields: + char_device: + fields: + major: {} + minor: {} + uptime: {} + working_directory: {} + user: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + group: + fields: + id: {} + name: {} + real_group: + fields: + id: {} + name: {} + saved_group: + fields: + id: {} + name: {} + supplemental_groups: + fields: + id: {} + name: {} + pe: + fields: "*" + pgid: {} + pid: {} + vpid: {} + previous: + fields: + args: {} + args_count: {} + executable: {} + real_group: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + same_as_process: + docs_only: True + saved_group: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + start: {} + supplemental_groups: + fields: + id: {} + name: {} + session_leader: + fields: + args: {} + args_count: {} + command_line: {} + entity_id: {} + executable: {} + interactive: {} + name: {} + pid: {} + vpid: {} + same_as_process: {} + start: {} + tty: + fields: + char_device: + fields: + major: {} + minor: {} + working_directory: {} + parent: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + session_leader: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + user: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + group: + fields: + id: {} + name: {} + real_group: + fields: + id: {} + name: {} + saved_group: + fields: + id: {} + name: {} + supplemental_groups: + fields: + id: {} + name: {} + thread: + fields: + id: {} + name: {} + capabilities: + fields: + effective: {} + permitted: {} + title: {} + tty: + fields: "*" + uptime: {} + user: + fields: + id: {} + name: {} + working_directory: {} + registry: + fields: "*" + related: + fields: "*" + risk: + fields: "*" + rule: + fields: "*" + server: + fields: + address: {} + as: + fields: "*" + bytes: {} + domain: {} + geo: + fields: "*" + ip: {} + mac: {} + nat: + fields: + ip: {} + port: {} + packets: {} + port: {} + subdomain: {} + registered_domain: {} + top_level_domain: {} + user: + fields: + domain: {} + email: {} + full_name: {} + group: + fields: "*" + hash: {} + id: {} + name: {} + roles: {} + service: + fields: "*" + source: + fields: + address: {} + as: + fields: "*" + bytes: {} + domain: {} + geo: + fields: "*" + ip: {} + mac: {} + nat: + fields: + ip: {} + port: {} + packets: {} + port: {} + subdomain: {} + registered_domain: {} + top_level_domain: {} + user: + fields: + domain: {} + email: {} + full_name: {} + group: + fields: "*" + hash: {} + id: {} + name: {} + roles: {} + threat: + fields: "*" + tls: + fields: "*" + tracing: + fields: "*" + url: + fields: "*" + user_agent: + fields: "*" + user: + fields: + changes: + fields: + domain: {} + email: {} + group: + fields: "*" + full_name: {} + hash: {} + id: {} + name: {} + roles: {} + domain: {} + effective: + fields: + domain: {} + email: {} + group: + fields: "*" + full_name: {} + hash: {} + id: {} + name: {} + roles: {} + email: {} + group: + fields: "*" + full_name: {} + hash: {} + id: {} + name: {} + risk: + fields: "*" + roles: {} + target: + fields: + domain: {} + email: {} + group: + fields: "*" + full_name: {} + hash: {} + id: {} + name: {} + roles: {} + vlan: + fields: "*" + vulnerability: + fields: "*" + x509: + fields: "*" \ No newline at end of file diff --git a/ecs/alerts/fields/template-settings-legacy.json b/ecs/alerts/fields/template-settings-legacy.json new file mode 100644 index 0000000000000..54aac2ceaf55c --- /dev/null +++ b/ecs/alerts/fields/template-settings-legacy.json @@ -0,0 +1,18 @@ +{ + "index_patterns": [ + "wazuh-alerts-5.x-*" + ], + "order": 1, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "0", + "refresh_interval": "5s", + "mapping": { + "total_fields": { + "limit": 2500 + } + } + } + } +} \ No newline at end of file diff --git a/ecs/alerts/fields/template-settings.json b/ecs/alerts/fields/template-settings.json new file mode 100644 index 0000000000000..9982494c55ca2 --- /dev/null +++ b/ecs/alerts/fields/template-settings.json @@ -0,0 +1,18 @@ +{ + "index_patterns": [ + "wazuh-alerts-5.x-*" + ], + "priority": 1, + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": 2500 + } + }, + "refresh_interval": "5s" + } + } + } +} \ No newline at end of file diff --git a/ecs/generate.sh b/ecs/generate.sh index 3d96dd446284c..7b860256f0936 100755 --- a/ecs/generate.sh +++ b/ecs/generate.sh @@ -43,9 +43,19 @@ generate_mappings() { --mapping-settings "$IN_FILES_DIR/mapping-settings.json" \ --out "$OUT_DIR" || exit 1 - # Replace "match_only_text" type (not supported by OpenSearch) with "text" - echo "Replacing \"match_only_text\" type with \"text\"" - find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \; + # Replace "constant_keyword" type (not supported by OpenSearch) with "keyword" + echo "Replacing \"constant_keyword\" type with \"keyword\"" + find "$OUT_DIR" -type f -exec sed -i 's/constant_keyword/keyword/g' {} \; + + # Replace "flattened" type (not supported by OpenSearch) with "flat_object" + echo "Replacing \"flattened\" type with \"flat_object\"" + find "$OUT_DIR" -type f -exec sed -i 's/flattened/flat_object/g' {} \; + + # Replace "scaled_float" type with "float" + echo "Replacing \"scaled_float\" type with \"float\"" + find "$OUT_DIR" -type f -exec sed -i 's/scaled_float/float/g' {} \; + echo "Removing scaling_factor lines" + find "$OUT_DIR" -type f -exec sed -i '/scaling_factor/d' {} \; local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json" local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json" @@ -105,4 +115,4 @@ UPLOAD="${4:-false}" URL="${5:-https://localhost:9200}" # Generate mappings -generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL" \ No newline at end of file +generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL"