From a71f524f1647388d06c90ee325419f918fb55031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Mon, 2 Dec 2024 17:56:36 +0100 Subject: [PATCH] =?UTF-8?q?Revert=20"Remove=20host=20fields=20at=20root=20?= =?UTF-8?q?level=20from=20stateful=20indices=20definitions=20(#=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 6f5398df669e21f7be03e7dbd711283e5c4e66b3. --- ecs/docs/README.md | 22 -- ecs/docs/inventory-hardware.md | 97 +++++++-- ecs/docs/inventory-hotfixes.md | 50 +++-- ecs/docs/inventory-networks.md | 193 ++++++++++++++++-- ecs/docs/inventory-packages.md | 11 +- ecs/docs/inventory-ports.md | 17 +- ecs/docs/inventory-processes.md | 51 +---- ecs/docs/states-fim.md | 12 +- ecs/docs/states-vulnerability.md | 61 +++--- .../event-generator/event_generator.py | 1 + ecs/states-fim/fields/subset.yml | 4 + .../event-generator/event_generator.py | 1 + .../fields/subset.yml | 2 + .../event-generator/event_generator.py | 1 + .../fields/subset.yml | 2 + .../event-generator/event_generator.py | 1 + .../fields/subset.yml | 2 + .../event-generator/event_generator.py | 1 + .../fields/subset.yml | 2 + .../event-generator/event_generator.py | 1 + ecs/states-inventory-ports/fields/subset.yml | 2 + .../event-generator/event_generator.py | 1 + .../fields/custom/host.yml | 2 +- .../fields/custom/os.yml | 2 +- .../fields/custom/risk.yml | 2 +- .../fields/subset.yml | 2 + .../event-generator/event_generator.py | 1 + ecs/states-vulnerabilities/fields/subset.yml | 2 + 28 files changed, 378 insertions(+), 168 deletions(-) delete mode 100644 ecs/docs/README.md diff --git a/ecs/docs/README.md b/ecs/docs/README.md deleted file mode 100644 index a94635cbce67b..0000000000000 --- a/ecs/docs/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Wazuh Common Schema - -The Wazuh Common Schema is a derivation of the [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) (ECS) providing a common data schema for the different central components of Wazuh. - -- [agent](./agent.md) -- [alerts](alerts.md) -- [command](commands.md) -- [states-fim](states-fim.md) -- [states-inventory-hardware](states-inventory-hardware.md) -- [states-inventory-hotfixes](states-inventory-hotfixes.md) -- [states-inventory-networks](states-inventory-networks.md) -- [states-inventory-packages](states-inventory-packages.md) -- [states-inventory-ports](states-inventory-ports.md) -- [states-inventory-processes](states-inventory-processes.md) -- [states-inventory-system](states-inventory-system.md) -- [states-vulnerabilities](states-vulnerabilities.md) - ---- - -### Useful resources -For more information and additional resources, please refer to the following links: -- [ECS schemas repository](https://github.com/elastic/ecs/tree/main/schemas) diff --git a/ecs/docs/inventory-hardware.md b/ecs/docs/inventory-hardware.md index 832bacfbb1ae4..75baa484b83d1 100644 --- a/ecs/docs/inventory-hardware.md +++ b/ecs/docs/inventory-hardware.md @@ -34,17 +34,25 @@ fields: "@timestamp": {} agent: fields: - groups: {} id: {} - name: {} - type: {} - version: {} - host: - fields: "*" + groups: {} observer: fields: serial_number: {} - + host: + fields: + memory: + fields: + total: {} + free: {} + used: + fields: + percentage: {} + cpu: + fields: + name: {} + cores: {} + speed: {} ``` ### Index settings @@ -56,12 +64,77 @@ fields: "template": { "settings": { "index": { - "number_of_shards": "1", "number_of_replicas": "0", - "refresh_interval": "5s", - "query.default_field": [ - "observer.board_serial" - ] + "number_of_shards": "1", + "query.default_field": ["observer.board_serial"], + "refresh_interval": "5s" + } + }, + "mappings": { + "date_detection": false, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "groups": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "cpu": { + "properties": { + "cores": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "speed": { + "type": "long" + } + }, + "type": "object" + }, + "memory": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "percentage": { + "type": "long" + } + }, + "type": "object" + } + }, + "type": "object" + } + } + }, + "observer": { + "properties": { + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } } } } diff --git a/ecs/docs/inventory-hotfixes.md b/ecs/docs/inventory-hotfixes.md index 17606d9dba4ee..fadc5377da19c 100644 --- a/ecs/docs/inventory-hotfixes.md +++ b/ecs/docs/inventory-hotfixes.md @@ -27,19 +27,13 @@ fields: "@timestamp": {} agent: fields: - groups: {} id: {} - name: {} - type: {} - version: {} - host: - fields: "*" + groups: {} package: fields: hotfix: fields: name: {} - ``` ### Index settings @@ -51,12 +45,44 @@ fields: "template": { "settings": { "index": { - "number_of_shards": "1", "number_of_replicas": "0", - "refresh_interval": "5s", - "query.default_field": [ - "package.hotfix.name" - ] + "number_of_shards": "1", + "query.default_field": ["package.hotfix.name"], + "refresh_interval": "5s" + } + }, + "mappings": { + "date_detection": false, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "groups": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "package": { + "properties": { + "hotfix": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + } + } + } } } } diff --git a/ecs/docs/inventory-networks.md b/ecs/docs/inventory-networks.md index 87115fdc87608..6459cde110aac 100644 --- a/ecs/docs/inventory-networks.md +++ b/ecs/docs/inventory-networks.md @@ -51,25 +51,34 @@ fields: "@timestamp": {} agent: fields: + id: {} groups: {} + destination: + fields: + ip: {} + port: {} + device: + fields: id: {} - name: {} - type: {} - version: {} - host: - fields: "*" - interface: + file: fields: - mtu: {} - state: {} - type: {} + inode: {} + host: + fields: + ip: {} + mac: {} + network: + fields: + egress: + fields: + bytes: {} + packets: {} + ingress: + fields: + bytes: {} + packets: {} network: fields: - broadcast: {} - dhcp: {} - gateway: {} - metric: {} - netmask: {} protocol: {} type: {} observer: @@ -80,7 +89,14 @@ fields: fields: alias: {} name: {} - + process: + fields: + name: {} + pid: {} + source: + fields: + ip: {} + port: {} ``` ### Index settings @@ -92,9 +108,8 @@ fields: "template": { "settings": { "index": { - "number_of_shards": "1", "number_of_replicas": "0", - "refresh_interval": "5s", + "number_of_shards": "1", "query.default_field": [ "agent.id", "agent.groups", @@ -104,7 +119,149 @@ fields: "observer.ingress.interface.name", "observer.ingress.interface.alias", "process.name" - ] + ], + "refresh_interval": "5s" + } + }, + "mappings": { + "date_detection": false, + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "groups": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "device": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "file": { + "properties": { + "inode": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "network": { + "properties": { + "egress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "ingress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + } + } + }, + "network": { + "properties": { + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "observer": { + "properties": { + "ingress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + } + } + }, + "process": { + "properties": { + "name": { + "fields": { + "text": { + "type": "match_only_text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + } } } } diff --git a/ecs/docs/inventory-packages.md b/ecs/docs/inventory-packages.md index 417b7bced0bc5..8091da88b85fa 100644 --- a/ecs/docs/inventory-packages.md +++ b/ecs/docs/inventory-packages.md @@ -47,16 +47,10 @@ fields: base: fields: "@timestamp": {} - tags: [] agent: fields: - groups: {} id: {} - name: {} - type: {} - version: {} - host: - fields: "*" + groups: {} package: fields: architecture: "" @@ -67,7 +61,6 @@ fields: size: {} type: "" version: "" - ``` ### Index settings @@ -85,7 +78,7 @@ fields: "query.default_field": [ "agent.id", "agent.groups", - "package.architecture", + "package.architecture" "package.name", "package.version", "package.type" diff --git a/ecs/docs/inventory-ports.md b/ecs/docs/inventory-ports.md index bbad0b8842f52..863d2a000ac41 100644 --- a/ecs/docs/inventory-ports.md +++ b/ecs/docs/inventory-ports.md @@ -40,13 +40,8 @@ fields: "@timestamp": {} agent: fields: - groups: {} id: {} - name: {} - type: {} - version: {} - host: - fields: "*" + groups: {} destination: fields: ip: {} @@ -57,6 +52,16 @@ fields: file: fields: inode: {} + host: + fields: + network: + fields: + egress: + fields: + queue: {} + ingress: + fields: + queue: {} network: fields: protocol: {} diff --git a/ecs/docs/inventory-processes.md b/ecs/docs/inventory-processes.md index 81572b8979705..087838f7f9c46 100644 --- a/ecs/docs/inventory-processes.md +++ b/ecs/docs/inventory-processes.md @@ -66,16 +66,10 @@ fields: base: fields: "@timestamp": {} - tags: [] agent: fields: - groups: {} id: {} - name: {} - type: {} - version: {} - host: - fields: "*" + groups: {} process: fields: pid: {} @@ -107,48 +101,7 @@ fields: thread: fields: id: "" -``` - -```yml ---- -- name: agent - title: Wazuh Agents - short: Wazuh Inc. custom fields. - type: group - group: 2 - fields: - - name: groups - type: keyword - level: custom - description: > - List of groups the agent belong to. -``` - -```yml ---- -- name: host - reusable: - top_level: true - expected: - - { at: agent, as: host } -``` - -```yml ---- -- name: os - reusable: - top_level: false - expected: - - agent.host -``` - -```yml ---- -- name: risk - reusable: - top_level: false - expected: - - agent.host + tty: {} ``` ### Index settings diff --git a/ecs/docs/states-fim.md b/ecs/docs/states-fim.md index af48052fdfff2..129fcf9ec94a6 100644 --- a/ecs/docs/states-fim.md +++ b/ecs/docs/states-fim.md @@ -38,20 +38,12 @@ Based on ECS: ```yml --- -name: wazuh-states-fim +name: fim fields: - base: - fields: - tags: [] agent: fields: - groups: {} id: {} - name: {} - type: {} - version: {} - host: - fields: "*" + groups: {} file: fields: attributes: {} diff --git a/ecs/docs/states-vulnerability.md b/ecs/docs/states-vulnerability.md index c40a6e0709524..61718d1419873 100644 --- a/ecs/docs/states-vulnerability.md +++ b/ecs/docs/states-vulnerability.md @@ -67,19 +67,21 @@ Based on ECS: name: wazuh-states-vulnerabilities fields: base: - fields: - tags: [] + tags: [] agent: - fields: - groups: {} - id: {} - name: {} - type: {} - version: {} - host: - fields: "*" + fields: "*" package: fields: "*" + host: + fields: + os: + fields: + full: "" + kernel: "" + name: "" + platform: "" + type: "" + version: "" vulnerability: fields: "*" wazuh: @@ -87,6 +89,7 @@ fields: ``` ```yml +--- - name: vulnerability title: Vulnerability group: 2 @@ -120,6 +123,7 @@ fields: ```yml --- +--- - name: wazuh title: Wazuh description: > @@ -147,23 +151,26 @@ fields: ```json { "index_patterns": ["wazuh-states-vulnerabilities*"], - "settings": { - "index": { - "number_of_shards": "1", - "number_of_replicas": "0", - "refresh_interval": "5s", - "query.default_field": [ - "agent.id", - "agent.groups", - "host.os.full", - "host.os.version", - "package.name", - "package.version", - "vulnerability.id", - "vulnerability.description", - "vulnerability.severity", - "wazuh.cluster.name" - ] + "priority": 1, + "template": { + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "0", + "refresh_interval": "5s", + "query.default_field": [ + "agent.id", + "agent.groups", + "host.os.full", + "host.os.version", + "package.name", + "package.version", + "vulnerability.id", + "vulnerability.description", + "vulnerability.severity", + "wazuh.cluster.name" + ] + } } } } diff --git a/ecs/states-fim/event-generator/event_generator.py b/ecs/states-fim/event-generator/event_generator.py index 5cd14d0b389c2..9c733c286bd43 100644 --- a/ecs/states-fim/event-generator/event_generator.py +++ b/ecs/states-fim/event-generator/event_generator.py @@ -155,6 +155,7 @@ def generate_random_data(number): event_data = { 'agent': generate_random_agent(), 'file': generate_random_file(), + 'host': generate_random_host(), 'registry': generate_random_registry() } data.append(event_data) diff --git a/ecs/states-fim/fields/subset.yml b/ecs/states-fim/fields/subset.yml index 7eab78e238a21..a9e6f01ce45b0 100644 --- a/ecs/states-fim/fields/subset.yml +++ b/ecs/states-fim/fields/subset.yml @@ -33,7 +33,11 @@ fields: type: {} uid: {} owner: {} + host: + fields: "*" registry: fields: key: {} value: {} + + diff --git a/ecs/states-inventory-hardware/event-generator/event_generator.py b/ecs/states-inventory-hardware/event-generator/event_generator.py index a04151219aa3c..779272592da66 100644 --- a/ecs/states-inventory-hardware/event-generator/event_generator.py +++ b/ecs/states-inventory-hardware/event-generator/event_generator.py @@ -145,6 +145,7 @@ def generate_random_data(number): event_data = { '@timestamp': generate_random_date(), 'agent': generate_random_agent(), + 'host': generate_random_host(), 'observer': generate_random_observer() } data.append(event_data) diff --git a/ecs/states-inventory-hardware/fields/subset.yml b/ecs/states-inventory-hardware/fields/subset.yml index 609d4a0050acd..da5a194e26ddf 100644 --- a/ecs/states-inventory-hardware/fields/subset.yml +++ b/ecs/states-inventory-hardware/fields/subset.yml @@ -17,3 +17,5 @@ fields: observer: fields: serial_number: {} + host: + fields: "*" diff --git a/ecs/states-inventory-hotfixes/event-generator/event_generator.py b/ecs/states-inventory-hotfixes/event-generator/event_generator.py index 048315afb8313..88cfdd0c76d82 100644 --- a/ecs/states-inventory-hotfixes/event-generator/event_generator.py +++ b/ecs/states-inventory-hotfixes/event-generator/event_generator.py @@ -137,6 +137,7 @@ def generate_random_data(number): event_data = { '@timestamp': generate_random_date(), 'agent': generate_random_agent(), + 'host': generate_random_host(), 'package': generate_random_package() } data.append(event_data) diff --git a/ecs/states-inventory-hotfixes/fields/subset.yml b/ecs/states-inventory-hotfixes/fields/subset.yml index 3cbf6f38f132f..7bb4f66950326 100644 --- a/ecs/states-inventory-hotfixes/fields/subset.yml +++ b/ecs/states-inventory-hotfixes/fields/subset.yml @@ -14,6 +14,8 @@ fields: version: {} host: fields: "*" + host: + fields: "*" package: fields: hotfix: diff --git a/ecs/states-inventory-networks/event-generator/event_generator.py b/ecs/states-inventory-networks/event-generator/event_generator.py index bc8e681329c1d..c9ec2c2fd363f 100644 --- a/ecs/states-inventory-networks/event-generator/event_generator.py +++ b/ecs/states-inventory-networks/event-generator/event_generator.py @@ -157,6 +157,7 @@ def generate_random_data(number): event_data = { '@timestamp': generate_random_date(), 'agent': generate_random_agent(), + 'host': generate_random_host(), 'network': generate_random_network(), 'observer': generate_random_observer() } diff --git a/ecs/states-inventory-networks/fields/subset.yml b/ecs/states-inventory-networks/fields/subset.yml index 48d90261e03ac..24392a19582a2 100644 --- a/ecs/states-inventory-networks/fields/subset.yml +++ b/ecs/states-inventory-networks/fields/subset.yml @@ -14,6 +14,8 @@ fields: version: {} host: fields: "*" + host: + fields: "*" interface: fields: mtu: {} diff --git a/ecs/states-inventory-packages/event-generator/event_generator.py b/ecs/states-inventory-packages/event-generator/event_generator.py index 77034d735931e..fda9227d7c826 100644 --- a/ecs/states-inventory-packages/event-generator/event_generator.py +++ b/ecs/states-inventory-packages/event-generator/event_generator.py @@ -142,6 +142,7 @@ def generate_random_data(number): event_data = { '@timestamp': generate_random_date(), 'agent': generate_random_agent(), + 'host': generate_random_host(), 'package': generate_random_package() } data.append(event_data) diff --git a/ecs/states-inventory-packages/fields/subset.yml b/ecs/states-inventory-packages/fields/subset.yml index 00ebd0b231be4..f2fdfb2fad9a0 100644 --- a/ecs/states-inventory-packages/fields/subset.yml +++ b/ecs/states-inventory-packages/fields/subset.yml @@ -14,6 +14,8 @@ fields: version: {} host: fields: "*" + host: + fields: "*" package: fields: architecture: "" diff --git a/ecs/states-inventory-ports/event-generator/event_generator.py b/ecs/states-inventory-ports/event-generator/event_generator.py index e409999521bb3..bede09340b104 100644 --- a/ecs/states-inventory-ports/event-generator/event_generator.py +++ b/ecs/states-inventory-ports/event-generator/event_generator.py @@ -171,6 +171,7 @@ def generate_random_data(number): 'destination': generate_random_destination(), 'device': generate_random_device(), 'file': generate_random_file(), + 'host': generate_random_host(), 'network': { 'protocol': random.choice(['TCP', 'UDP', 'ICMP']) }, diff --git a/ecs/states-inventory-ports/fields/subset.yml b/ecs/states-inventory-ports/fields/subset.yml index 54a87eef42b81..549917083aaa8 100644 --- a/ecs/states-inventory-ports/fields/subset.yml +++ b/ecs/states-inventory-ports/fields/subset.yml @@ -24,6 +24,8 @@ fields: file: fields: inode: {} + host: + fields: "*" network: fields: protocol: {} diff --git a/ecs/states-inventory-processes/event-generator/event_generator.py b/ecs/states-inventory-processes/event-generator/event_generator.py index 3da0e29cd07a2..3395616d104c9 100644 --- a/ecs/states-inventory-processes/event-generator/event_generator.py +++ b/ecs/states-inventory-processes/event-generator/event_generator.py @@ -163,6 +163,7 @@ def generate_random_data(number): event_data = { '@timestamp': generate_random_date(), 'agent': generate_random_agent(), + 'host': generate_random_host(), 'process': generate_random_process() } data.append(event_data) diff --git a/ecs/states-inventory-processes/fields/custom/host.yml b/ecs/states-inventory-processes/fields/custom/host.yml index 5bf50e3e3f675..a0356d13da657 100644 --- a/ecs/states-inventory-processes/fields/custom/host.yml +++ b/ecs/states-inventory-processes/fields/custom/host.yml @@ -3,4 +3,4 @@ reusable: top_level: true expected: - - { at: agent, as: host } + - { at: agent, as: host } \ No newline at end of file diff --git a/ecs/states-inventory-processes/fields/custom/os.yml b/ecs/states-inventory-processes/fields/custom/os.yml index 0181d44d62751..952c2d6e93a40 100644 --- a/ecs/states-inventory-processes/fields/custom/os.yml +++ b/ecs/states-inventory-processes/fields/custom/os.yml @@ -3,4 +3,4 @@ reusable: top_level: false expected: - - agent.host + - agent.host \ No newline at end of file diff --git a/ecs/states-inventory-processes/fields/custom/risk.yml b/ecs/states-inventory-processes/fields/custom/risk.yml index 599a04a4f9d17..1c06213bc6205 100644 --- a/ecs/states-inventory-processes/fields/custom/risk.yml +++ b/ecs/states-inventory-processes/fields/custom/risk.yml @@ -3,4 +3,4 @@ reusable: top_level: false expected: - - agent.host + - agent.host \ No newline at end of file diff --git a/ecs/states-inventory-processes/fields/subset.yml b/ecs/states-inventory-processes/fields/subset.yml index 16ccccb2dfb9a..55693facfee71 100644 --- a/ecs/states-inventory-processes/fields/subset.yml +++ b/ecs/states-inventory-processes/fields/subset.yml @@ -14,6 +14,8 @@ fields: version: {} host: fields: "*" + host: + fields: "*" process: fields: pid: {} diff --git a/ecs/states-vulnerabilities/event-generator/event_generator.py b/ecs/states-vulnerabilities/event-generator/event_generator.py index f3e0704ff1b3a..de80c8bf49e92 100644 --- a/ecs/states-vulnerabilities/event-generator/event_generator.py +++ b/ecs/states-vulnerabilities/event-generator/event_generator.py @@ -173,6 +173,7 @@ def generate_random_data(number): for _ in range(number): event_data = { 'agent': generate_random_agent(), + 'host': generate_random_host(), 'package': generate_random_package(), 'vulnerability': generate_random_vulnerability() } diff --git a/ecs/states-vulnerabilities/fields/subset.yml b/ecs/states-vulnerabilities/fields/subset.yml index 2981f226f774d..d0b44d3a712f1 100644 --- a/ecs/states-vulnerabilities/fields/subset.yml +++ b/ecs/states-vulnerabilities/fields/subset.yml @@ -15,6 +15,8 @@ fields: fields: "*" package: fields: "*" + host: + fields: "*" vulnerability: fields: "*" wazuh: