From b6555acbfe97ce94efd6ad336d6a4a7c7b0c8bb1 Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Mon, 2 Dec 2024 08:19:13 -0300 Subject: [PATCH 01/15] Update stateles indices templates (#156) Remove host ECS fields from root level --- .../main/resources/index-template-fim.json | 202 --------------- .../resources/index-template-hardware.json | 232 ------------------ .../resources/index-template-hotfixes.json | 202 --------------- .../resources/index-template-networks.json | 214 ---------------- .../resources/index-template-packages.json | 202 --------------- .../main/resources/index-template-ports.json | 208 ---------------- .../resources/index-template-processes.json | 202 --------------- .../index-template-vulnerabilities.json | 202 --------------- 8 files changed, 1664 deletions(-) diff --git a/plugins/setup/src/main/resources/index-template-fim.json b/plugins/setup/src/main/resources/index-template-fim.json index f873565b..4bcd4650 100644 --- a/plugins/setup/src/main/resources/index-template-fim.json +++ b/plugins/setup/src/main/resources/index-template-fim.json @@ -322,208 +322,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "registry": { "properties": { "key": { diff --git a/plugins/setup/src/main/resources/index-template-hardware.json b/plugins/setup/src/main/resources/index-template-hardware.json index 7d28fe8b..36481872 100644 --- a/plugins/setup/src/main/resources/index-template-hardware.json +++ b/plugins/setup/src/main/resources/index-template-hardware.json @@ -275,238 +275,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "cores": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "speed": { - "type": "long" - }, - "usage": { - "type": "float" - } - }, - "type": "object" - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "memory": { - "properties": { - "free": { - "type": "long" - }, - "total": { - "type": "long" - }, - "used": { - "properties": { - "percentage": { - "type": "long" - } - }, - "type": "object" - } - }, - "type": "object" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "observer": { "properties": { "serial_number": { diff --git a/plugins/setup/src/main/resources/index-template-hotfixes.json b/plugins/setup/src/main/resources/index-template-hotfixes.json index 654d8430..52d66fe9 100644 --- a/plugins/setup/src/main/resources/index-template-hotfixes.json +++ b/plugins/setup/src/main/resources/index-template-hotfixes.json @@ -245,208 +245,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "package": { "properties": { "hotfix": { diff --git a/plugins/setup/src/main/resources/index-template-networks.json b/plugins/setup/src/main/resources/index-template-networks.json index e8d27e21..9bc9fed3 100644 --- a/plugins/setup/src/main/resources/index-template-networks.json +++ b/plugins/setup/src/main/resources/index-template-networks.json @@ -257,220 +257,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "drops": { - "type": "long" - }, - "errors": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "drops": { - "type": "long" - }, - "errors": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "network": { "properties": { "broadcast": { diff --git a/plugins/setup/src/main/resources/index-template-packages.json b/plugins/setup/src/main/resources/index-template-packages.json index 79657dbb..353de124 100644 --- a/plugins/setup/src/main/resources/index-template-packages.json +++ b/plugins/setup/src/main/resources/index-template-packages.json @@ -245,208 +245,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "package": { "properties": { "architecture": { diff --git a/plugins/setup/src/main/resources/index-template-ports.json b/plugins/setup/src/main/resources/index-template-ports.json index 8d2598cb..70dbc338 100644 --- a/plugins/setup/src/main/resources/index-template-ports.json +++ b/plugins/setup/src/main/resources/index-template-ports.json @@ -277,214 +277,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - }, - "queue": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - }, - "queue": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "network": { "properties": { "protocol": { diff --git a/plugins/setup/src/main/resources/index-template-processes.json b/plugins/setup/src/main/resources/index-template-processes.json index 74cfce49..dbb8dde3 100644 --- a/plugins/setup/src/main/resources/index-template-processes.json +++ b/plugins/setup/src/main/resources/index-template-processes.json @@ -245,208 +245,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "process": { "properties": { "args": { diff --git a/plugins/setup/src/main/resources/index-template-vulnerabilities.json b/plugins/setup/src/main/resources/index-template-vulnerabilities.json index a4597b28..940a42fe 100644 --- a/plugins/setup/src/main/resources/index-template-vulnerabilities.json +++ b/plugins/setup/src/main/resources/index-template-vulnerabilities.json @@ -242,208 +242,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "package": { "properties": { "architecture": { From 39d222829bc62b1eecce0cb760537aed34f9cbf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Mon, 2 Dec 2024 19:38:00 +0100 Subject: [PATCH 02/15] Update index templates (#159) --- .../resources/index-template-hardware.json | 41 ++++ .../resources/index-template-networks.json | 214 ++++++++++++++++++ .../main/resources/index-template-ports.json | 22 ++ .../index-template-vulnerabilities.json | 202 +++++++++++++++++ 4 files changed, 479 insertions(+) diff --git a/plugins/setup/src/main/resources/index-template-hardware.json b/plugins/setup/src/main/resources/index-template-hardware.json index 36481872..1ade0c83 100644 --- a/plugins/setup/src/main/resources/index-template-hardware.json +++ b/plugins/setup/src/main/resources/index-template-hardware.json @@ -275,6 +275,47 @@ } } }, + "host": { + "properties": { + "cpu": { + "properties": { + "cores": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "speed": { + "type": "long" + }, + "usage": { + "type": "float" + } + }, + "type": "object" + }, + "memory": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "percentage": { + "type": "long" + } + }, + "type": "object" + } + }, + "type": "object" + } + } + }, "observer": { "properties": { "serial_number": { diff --git a/plugins/setup/src/main/resources/index-template-networks.json b/plugins/setup/src/main/resources/index-template-networks.json index 9bc9fed3..e8d27e21 100644 --- a/plugins/setup/src/main/resources/index-template-networks.json +++ b/plugins/setup/src/main/resources/index-template-networks.json @@ -257,6 +257,220 @@ } } }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "boot": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cpu": { + "properties": { + "usage": { + "type": "float" + } + } + }, + "disk": { + "properties": { + "read": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "write": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "network": { + "properties": { + "egress": { + "properties": { + "bytes": { + "type": "long" + }, + "drops": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "ingress": { + "properties": { + "bytes": { + "type": "long" + }, + "drops": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pid_ns_ino": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk": { + "properties": { + "calculated_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "calculated_score": { + "type": "float" + }, + "calculated_score_norm": { + "type": "float" + }, + "static_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "static_score": { + "type": "float" + }, + "static_score_norm": { + "type": "float" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + } + } + }, "network": { "properties": { "broadcast": { diff --git a/plugins/setup/src/main/resources/index-template-ports.json b/plugins/setup/src/main/resources/index-template-ports.json index 70dbc338..a37680f8 100644 --- a/plugins/setup/src/main/resources/index-template-ports.json +++ b/plugins/setup/src/main/resources/index-template-ports.json @@ -277,6 +277,28 @@ } } }, + "host": { + "properties": { + "network": { + "properties": { + "egress": { + "properties": { + "queue": { + "type": "long" + } + } + }, + "ingress": { + "properties": { + "queue": { + "type": "long" + } + } + } + } + } + } + }, "network": { "properties": { "protocol": { diff --git a/plugins/setup/src/main/resources/index-template-vulnerabilities.json b/plugins/setup/src/main/resources/index-template-vulnerabilities.json index 940a42fe..a4597b28 100644 --- a/plugins/setup/src/main/resources/index-template-vulnerabilities.json +++ b/plugins/setup/src/main/resources/index-template-vulnerabilities.json @@ -242,6 +242,208 @@ } } }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "boot": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cpu": { + "properties": { + "usage": { + "type": "float" + } + } + }, + "disk": { + "properties": { + "read": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "write": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "network": { + "properties": { + "egress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "ingress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pid_ns_ino": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk": { + "properties": { + "calculated_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "calculated_score": { + "type": "float" + }, + "calculated_score_norm": { + "type": "float" + }, + "static_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "static_score": { + "type": "float" + }, + "static_score_norm": { + "type": "float" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + } + } + }, "package": { "properties": { "architecture": { From 9a484788a47a80ae17ed8668b235f67224b0eb8f Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Mon, 2 Dec 2024 15:59:53 -0300 Subject: [PATCH 03/15] Update inventory-networks index template (#158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add interface at root-level Co-authored-by: Álex Ruiz --- .../main/resources/index-template-networks.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plugins/setup/src/main/resources/index-template-networks.json b/plugins/setup/src/main/resources/index-template-networks.json index e8d27e21..9c5c1886 100644 --- a/plugins/setup/src/main/resources/index-template-networks.json +++ b/plugins/setup/src/main/resources/index-template-networks.json @@ -471,6 +471,21 @@ } } }, + "interface": { + "properties": { + "mtu": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "network": { "properties": { "broadcast": { From 8e544289de1b3568ad1d0fef6d7f4aaa2be23f95 Mon Sep 17 00:00:00 2001 From: Malena Casas Date: Tue, 3 Dec 2024 12:43:05 -0300 Subject: [PATCH 04/15] Change CM's `POST /commands` endpoint to receive an array (#152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add bulk creation in CommandIndex. Modify handlePost to process a list of Command * Edit expected body in POST of commands. First try to modify the test * Add refresh and count commands test * Delete old comment code * Delete test resource, wazuh-indexer.keystore.json, not used. * Improve the solution by separating the logic into reusable methods * Add log error when the commands are indexed * Update openapi.yml --------- Co-authored-by: Álex Ruiz --- plugins/command-manager/openapi.yml | 21 +++-- .../commandmanager/index/CommandIndex.java | 79 +++++++++++++++++-- .../wazuh/commandmanager/model/Command.java | 11 +++ .../wazuh/commandmanager/model/Documents.java | 75 ++++++++++++++++++ .../rest/RestPostCommandAction.java | 68 ++++++++++------ .../resources/wazuh-indexer.keystore.json | 11 --- .../rest-api-spec/test/20_create.yml | 75 ++++++++++-------- 7 files changed, 259 insertions(+), 81 deletions(-) create mode 100644 plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java delete mode 100644 plugins/command-manager/src/test/resources/wazuh-indexer.keystore.json diff --git a/plugins/command-manager/openapi.yml b/plugins/command-manager/openapi.yml index a5706102..a2d44918 100644 --- a/plugins/command-manager/openapi.yml +++ b/plugins/command-manager/openapi.yml @@ -9,8 +9,8 @@ paths: post: tags: - "authentication" - summary: Add a new command to the queue. - description: Add a new command to the queue. + summary: Mock of the Wazuh Server M_API authentication endpoint. + description: Returns a JWT. responses: "200": description: OK @@ -18,20 +18,31 @@ paths: post: tags: - "commands" - summary: Add a new command to the queue. - description: Add a new command to the queue. + summary: Add commands. + description: Receives and processes an array of commands. requestBody: required: true content: "application/json": schema: - $ref: "#/components/schemas/Command" + $ref: "#/components/schemas/Commands" responses: "200": description: OK + "400": + description: parsing_exception + "500": + description: Internal server error (boom!) components: schemas: + Commands: + type: object + properties: + commands: + type: array + items: + $ref: '#/components/schemas/Command' Command: type: object properties: diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java index ea8f2c27..9281a3c8 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; @@ -24,6 +25,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -59,6 +61,7 @@ public CommandIndex(Client client, ClusterService clusterService, ThreadPool thr * @param document instance of the document model to persist in the index. * @return A CompletableFuture with the RestStatus response from the operation */ + @Deprecated public CompletableFuture asyncCreate(Document document) { CompletableFuture future = new CompletableFuture<>(); ExecutorService executor = this.threadPool.executor(ThreadPool.Names.WRITE); @@ -74,14 +77,7 @@ public CompletableFuture asyncCreate(Document document) { log.info("Indexing command with id [{}]", document.getId()); try { - IndexRequest request = - new IndexRequest() - .index(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME) - .source( - document.toXContent( - XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .id(document.getId()) - .create(true); + IndexRequest request = createIndexRequest(document); executor.submit( () -> { try (ThreadContext.StoredContext ignored = @@ -89,6 +85,10 @@ public CompletableFuture asyncCreate(Document document) { RestStatus restStatus = client.index(request).actionGet().status(); future.complete(restStatus); } catch (Exception e) { + log.error( + "Error indexing command with id [{}] due to {}", + document.getId(), + e.getMessage()); future.completeExceptionally(e); } }); @@ -98,6 +98,50 @@ public CompletableFuture asyncCreate(Document document) { return future; } + /** + * @param documents list of instances of the document model to persist in the index. + * @return A CompletableFuture with the RestStatus response from the operation + */ + public CompletableFuture asyncBulkCreate(ArrayList documents) { + CompletableFuture future = new CompletableFuture<>(); + ExecutorService executor = this.threadPool.executor(ThreadPool.Names.WRITE); + + // Create index template if it does not exist. + if (!indexTemplateExists(CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { + putIndexTemplate(CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } else { + log.info( + "Index template {} already exists. Skipping creation.", + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } + + BulkRequest bulkRequest = new BulkRequest(); + for (Document document : documents) { + log.info("Adding command with id [{}] to the bulk request", document.getId()); + try { + bulkRequest.add(createIndexRequest(document)); + } catch (IOException e) { + log.error( + "Error creating IndexRequest with document id [{}] due to {}", + document.getId(), + e); + } + } + + executor.submit( + () -> { + try (ThreadContext.StoredContext ignored = + this.threadPool.getThreadContext().stashContext()) { + RestStatus restStatus = client.bulk(bulkRequest).actionGet().status(); + future.complete(restStatus); + } catch (Exception e) { + log.error("Error indexing commands with bulk due to {}", e.getMessage()); + future.completeExceptionally(e); + } + }); + return future; + } + /** * Checks for the existence of the given index template in the cluster. * @@ -147,4 +191,23 @@ public void putIndexTemplate(String templateName) { log.error("Error reading index template [{}] from filesystem", templateName); } } + + /** + * Create an IndexRequest object from a Document object. + * + * @param document the document to create the IndexRequest for COMMAND_MANAGER_INDEX + * @return an IndexRequest object + * @throws IOException thrown by XContentFactory.jsonBuilder() + */ + private IndexRequest createIndexRequest(Document document) throws IOException { + IndexRequest request = + new IndexRequest() + .index(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME) + .source( + document.toXContent( + XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(document.getId()) + .create(true); + return request; + } } diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java index 57b7ec5a..798d1182 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.List; import reactor.util.annotation.NonNull; @@ -127,6 +128,16 @@ public static Command parse(XContentParser parser) } } + public static List parseToArray(XContentParser parser) + throws IOException, IllegalArgumentException { + List commands = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + Command command = Command.parse(parser); + commands.add(command); + } + return commands; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(COMMAND); diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java new file mode 100644 index 00000000..b41fc802 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.model; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; + +import com.wazuh.commandmanager.CommandManagerPlugin; + +public class Documents implements ToXContentObject { + private ArrayList documents; + + public Documents() { + this.documents = new ArrayList<>(); + } + + /** + * Default constructor + * + * @param documents + */ + public Documents(ArrayList documents) { + this.documents = documents; + } + + /** + * Get the list of Document objects. + * + * @return the list of documents. + */ + public ArrayList getDocuments() { + return documents; + } + + /** + * Set the list of Document objects. + * + * @param documents the list of documents to set. + */ + public void setDocuments(ArrayList documents) { + this.documents = documents; + } + + /** + * Adds a document to the list of documents. + * + * @param document The document to add to the list. + */ + public void addDocument(Document document) { + this.documents.add(document); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("_index", CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); + builder.startArray("_documents"); + for (Document document : this.documents) { + builder.startObject(); + builder.field("_id", document.getId()); + builder.endObject(); + } + builder.endArray(); + return builder; + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java index 277ae153..92f6b765 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java @@ -22,6 +22,7 @@ import org.opensearch.rest.RestRequest; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -30,6 +31,7 @@ import com.wazuh.commandmanager.model.Agent; import com.wazuh.commandmanager.model.Command; import com.wazuh.commandmanager.model.Document; +import com.wazuh.commandmanager.model.Documents; import com.wazuh.commandmanager.utils.httpclient.HttpRestClientDemo; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; @@ -94,42 +96,60 @@ private RestChannelConsumer handlePost(RestRequest request) throws IOException { request.uri(), request.getRequestId(), request.header("Host")); + // Get request details + if (!request.hasContent()) { + // Bad request if body doesn't exist + return channel -> { + channel.sendResponse( + new BytesRestResponse(RestStatus.BAD_REQUEST, "Body content is required")); + }; + } + XContentParser parser = request.contentParser(); + List commands = new ArrayList<>(); ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + // The array of commands is inside the "commands" JSON object. + // This line moves the parser pointer into this object. + parser.nextToken(); + if (parser.nextToken() == XContentParser.Token.START_ARRAY) { + commands = Command.parseToArray(parser); + } else { + log.error("Token does not match {}", parser.currentToken()); + } + + Documents documents = new Documents(); + for (Command command : commands) { + Document document = + new Document( + new Agent(List.of("groups000")), // TODO read agent from .agents index + command); + documents.addDocument(document); - Command command = Command.parse(parser); - Document document = - new Document( - new Agent(List.of("groups000")), // TODO read agent from .agents index - command); - - // Commands delivery to the Management API. - // Note: needs to be decoupled from the Rest handler (job scheduler task). - try { - String payload = - document.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) - .toString(); - SimpleHttpResponse response = - HttpRestClientDemo.runWithResponse(payload, document.getId()); - log.info("Received response to POST request with code [{}]", response.getCode()); - log.info("Raw response:\n{}", response.getBodyText()); - } catch (Exception e) { - log.error("Error reading response: {}", e.getMessage()); + // Commands delivery to the Management API. + // Note: needs to be decoupled from the Rest handler (job scheduler task). + try { + String payload = + documents + .toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) + .toString(); + SimpleHttpResponse response = + HttpRestClientDemo.runWithResponse(payload, document.getId()); + log.info("Received response to POST request with code [{}]", response.getCode()); + log.info("Raw response:\n{}", response.getBodyText()); + } catch (Exception e) { + log.error("Error reading response: {}", e.getMessage()); + } } // Send response return channel -> { this.commandIndex - .asyncCreate(document) + .asyncBulkCreate(documents.getDocuments()) .thenAccept( restStatus -> { try (XContentBuilder builder = channel.newBuilder()) { - builder.startObject(); - builder.field( - "_index", - CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); - builder.field("_id", document.getId()); + documents.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.field("result", restStatus.name()); builder.endObject(); channel.sendResponse( diff --git a/plugins/command-manager/src/test/resources/wazuh-indexer.keystore.json b/plugins/command-manager/src/test/resources/wazuh-indexer.keystore.json deleted file mode 100644 index c93a5df3..00000000 --- a/plugins/command-manager/src/test/resources/wazuh-indexer.keystore.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "name" : "wazuh-api", - "properties" : { - "uri" : "http://localhost:9090", - "auth.type" : "basicauth", - "auth.username" : "admin", - "auth.password" : "type" - } - } -] \ No newline at end of file diff --git a/plugins/command-manager/src/yamlRestTest/resources/rest-api-spec/test/20_create.yml b/plugins/command-manager/src/yamlRestTest/resources/rest-api-spec/test/20_create.yml index f081ec51..ccc467c8 100644 --- a/plugins/command-manager/src/yamlRestTest/resources/rest-api-spec/test/20_create.yml +++ b/plugins/command-manager/src/yamlRestTest/resources/rest-api-spec/test/20_create.yml @@ -1,37 +1,46 @@ --- "Create command": - - do: - _plugins._command_manager: - body: - source: "Users/Services" - user: "user13" - target: { - id: "target4", - type: "agent" - } - action: { - name: "change_group", - args: [ "/path/to/executable/arg8" ], - version: "v4" - } - timeout: 100 + - do: + _plugins._command_manager: + body: + commands: + [ + { + source: "Users/Services", + user: "user13", + target: { id: "target4", type: "agent" }, + action: + { + name: "change_group", + args: ["/path/to/executable/arg8"], + version: "v4", + }, + timeout: 100, + }, - - set: { _id: document_id } - - match: { _index: .commands } + { + source: "Users/Services", + user: "user54", + target: { id: "target5", type: "agent" }, + action: + { + name: "stop", + args: ["/path/to/executable/arg7"], + version: "v4", + }, + timeout: 30, + }, + ] - - do: - get: - index: .commands - id: $document_id - - match: { _source.command.source: "Users/Services" } - - match: { _source.command.user: "user13" } - - match: { _source.command.target.type: "agent" } - - match: { _source.command.target.id: "target4" } - - match: { _source.command.action: - { - name: "change_group", - args: [ "/path/to/executable/arg8" ], - version: "v4" - } - } - - match: { _source.command.timeout: 100 } + - match: { _index: .commands } + - match: { result: "OK" } + + - do: + indices.refresh: + index: [.commands] + + - do: + count: + index: .commands + + - match: { count: 2 } From 7f2420978fbbf638065921ea5896fa6571b1b3c9 Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Tue, 3 Dec 2024 17:23:00 -0300 Subject: [PATCH 05/15] Implement timeseries index model to command-manager plugin (#153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update commands index template Add @timestamp and delivery_timestamp fields * Add timestamp and delivery_timestamp attributes to Command model * Move timestamp and deliveryTimestamp from Command to Document Add getter function for the Command attrbiture timeout Use native System.currentTimeMillis() to get current timestamp * Update command index template Move delivery_timestamp to top-level * Fix typo on field names * Implement timestamp and delivery_timestamp constants attributes for Document model Add missing 'this' on timeout getter * Update timestamp and deliveryTimestamp to be of type ZonedDateTime Implement OpenSearch DateUtils * Implement OpensSearch date_time_no_millis date format pattern * Fix errors --------- Co-authored-by: Álex Ruiz --- .../wazuh/commandmanager/model/Command.java | 9 ++++ .../wazuh/commandmanager/model/Document.java | 25 ++++++++- .../wazuh/commandmanager/model/Documents.java | 15 +++--- .../rest/RestPostCommandAction.java | 52 +++++++++++-------- .../resources/index-template-commands.json | 6 +++ .../resources/index-template-commands.json | 6 +++ 6 files changed, 83 insertions(+), 30 deletions(-) diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java index 798d1182..c5221248 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java @@ -62,6 +62,15 @@ public Command( this.status = Status.PENDING; } + /** + * Retrieves the timeout value for this command. + * + * @return the timeout value in milliseconds. + */ + public Integer getTimeout() { + return this.timeout; + } + /** * Parses the request's payload into the Command model. * diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Document.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Document.java index b9ec3850..6a9eef76 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Document.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Document.java @@ -9,18 +9,28 @@ package com.wazuh.commandmanager.model; import org.opensearch.common.UUIDs; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateUtils; +import org.opensearch.common.time.FormatNames; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; +import java.time.ZonedDateTime; import java.util.List; /** Command's target fields. */ public class Document implements ToXContentObject { + private static final String DATE_FORMAT = FormatNames.DATE_TIME_NO_MILLIS.getSnakeCaseName(); + private static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern(DATE_FORMAT); + public static final String TIMESTAMP = "@timestamp"; + public static final String DELIVERY_TIMESTAMP = "delivery_timestamp"; private final Agent agent; private final Command command; private final String id; + private final ZonedDateTime timestamp; + private final ZonedDateTime deliveryTimestamp; /** * Default constructor @@ -32,6 +42,8 @@ public Document(Agent agent, Command command) { this.agent = agent; this.command = command; this.id = UUIDs.base64UUID(); + this.timestamp = DateUtils.nowWithMillisResolution(); + this.deliveryTimestamp = timestamp.plusSeconds(command.getTimeout()); } /** @@ -68,11 +80,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); this.agent.toXContent(builder, ToXContentObject.EMPTY_PARAMS); this.command.toXContent(builder, ToXContentObject.EMPTY_PARAMS); + builder.field(TIMESTAMP, DATE_FORMATTER.format(this.timestamp)); + builder.field(DELIVERY_TIMESTAMP, DATE_FORMATTER.format(this.deliveryTimestamp)); return builder.endObject(); } @Override public String toString() { - return "Document{" + "agent=" + agent + ", command=" + command + '}'; + return "Document{" + + "@timestamp=" + + timestamp + + ", delivery_timestamp=" + + deliveryTimestamp + + ", agent=" + + agent + + ", command=" + + command + + '}'; } } diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java index b41fc802..36ecd068 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java @@ -14,8 +14,6 @@ import java.io.IOException; import java.util.ArrayList; -import com.wazuh.commandmanager.CommandManagerPlugin; - public class Documents implements ToXContentObject { private ArrayList documents; @@ -59,17 +57,22 @@ public void addDocument(Document document) { this.documents.add(document); } + /** + * Fit this object into a XContentBuilder parser, preparing it for the reply of POST /commands. + * + * @param builder XContentBuilder builder + * @param params ToXContent.EMPTY_PARAMS + * @return XContentBuilder builder with the representation of this object. + * @throws IOException parsing error. + */ @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("_index", CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); builder.startArray("_documents"); for (Document document : this.documents) { builder.startObject(); builder.field("_id", document.getId()); builder.endObject(); } - builder.endArray(); - return builder; + return builder.endArray(); } } diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java index 92f6b765..fe786d67 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java @@ -8,11 +8,9 @@ */ package com.wazuh.commandmanager.rest; -import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -25,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.concurrent.CompletableFuture; import com.wazuh.commandmanager.CommandManagerPlugin; import com.wazuh.commandmanager.index.CommandIndex; @@ -32,7 +31,6 @@ import com.wazuh.commandmanager.model.Command; import com.wazuh.commandmanager.model.Document; import com.wazuh.commandmanager.model.Documents; -import com.wazuh.commandmanager.utils.httpclient.HttpRestClientDemo; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.rest.RestRequest.Method.POST; @@ -97,7 +95,9 @@ private RestChannelConsumer handlePost(RestRequest request) throws IOException { request.getRequestId(), request.header("Host")); - // Get request details + /// Request validation + /// ================== + /// Fail fast. if (!request.hasContent()) { // Bad request if body doesn't exist return channel -> { @@ -106,11 +106,14 @@ private RestChannelConsumer handlePost(RestRequest request) throws IOException { }; } + /// Request parsing + /// =============== + /// Retrieves and generates an array list of commands. XContentParser parser = request.contentParser(); List commands = new ArrayList<>(); ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); // The array of commands is inside the "commands" JSON object. - // This line moves the parser pointer into this object. + // This line moves the parser pointer to this object. parser.nextToken(); if (parser.nextToken() == XContentParser.Token.START_ARRAY) { commands = Command.parseToArray(parser); @@ -118,6 +121,13 @@ private RestChannelConsumer handlePost(RestRequest request) throws IOException { log.error("Token does not match {}", parser.currentToken()); } + /// Commands expansion + /// ================== + /// Transforms the array of commands to orders. + /// While commands can be targeted to groups of agents, orders are targeted to individual + // agents. + /// Given a group of agents A with N agents, a total of N orders are generated. One for each + // agent. Documents documents = new Documents(); for (Command command : commands) { Document document = @@ -125,30 +135,26 @@ private RestChannelConsumer handlePost(RestRequest request) throws IOException { new Agent(List.of("groups000")), // TODO read agent from .agents index command); documents.addDocument(document); - - // Commands delivery to the Management API. - // Note: needs to be decoupled from the Rest handler (job scheduler task). - try { - String payload = - documents - .toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) - .toString(); - SimpleHttpResponse response = - HttpRestClientDemo.runWithResponse(payload, document.getId()); - log.info("Received response to POST request with code [{}]", response.getCode()); - log.info("Raw response:\n{}", response.getBodyText()); - } catch (Exception e) { - log.error("Error reading response: {}", e.getMessage()); - } } - // Send response + /// Orders indexing + /// ================== + /// The orders are inserted into the index. + CompletableFuture bulkRequestFuture = + this.commandIndex.asyncBulkCreate(documents.getDocuments()); + + /// Send response + /// ================== + /// Reply to the request. return channel -> { - this.commandIndex - .asyncBulkCreate(documents.getDocuments()) + bulkRequestFuture .thenAccept( restStatus -> { try (XContentBuilder builder = channel.newBuilder()) { + builder.startObject(); + builder.field( + "_index", + CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); documents.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.field("result", restStatus.name()); builder.endObject(); diff --git a/plugins/command-manager/src/main/resources/index-template-commands.json b/plugins/command-manager/src/main/resources/index-template-commands.json index 3614c17b..6c834803 100644 --- a/plugins/command-manager/src/main/resources/index-template-commands.json +++ b/plugins/command-manager/src/main/resources/index-template-commands.json @@ -6,6 +6,9 @@ "date_detection": false, "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "agent": { "properties": { "groups": { @@ -83,6 +86,9 @@ "type": "keyword" } } + }, + "delivery_timestamp": { + "type": "date" } } }, diff --git a/plugins/setup/src/main/resources/index-template-commands.json b/plugins/setup/src/main/resources/index-template-commands.json index 3614c17b..6c834803 100644 --- a/plugins/setup/src/main/resources/index-template-commands.json +++ b/plugins/setup/src/main/resources/index-template-commands.json @@ -6,6 +6,9 @@ "date_detection": false, "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "agent": { "properties": { "groups": { @@ -83,6 +86,9 @@ "type": "keyword" } } + }, + "delivery_timestamp": { + "type": "date" } } }, From 270e9b543bd333adbeb8e7a71dceb1561889c7ca Mon Sep 17 00:00:00 2001 From: Fede Galland <99492720+f-galland@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:46:53 -0300 Subject: [PATCH 06/15] Implement Job Scheduler logic (#103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add PoC extending the Job-Scheduler plugin Not functional yet * Modify build.gradle to load job scheduler on gradlew run * Add PoC extending the Job-Scheduler plugin Not functional yet * Modify build.gradle to load job scheduler on gradlew run * Reformat code * Rebuild build.gradle * Move JobSchedulerExtension to the right resources directory * Fix logger class name * Expose a POST endpoint to schedule tasks. * Init jobScheduler * Check if runner interface is being called. * Try getting runjob() executed * Correct META-INF directory contents * Create Job on plugin startup * Move create() to its own JobDocument class. * Fix switch statement * Schedule job on cluster startup. * Make sure to schedule the job only on new clusters. * Split job indexing to a separate method * Create a job that logs the search results on the .commands index. * Make the command manager run searches on the commands index. * Check if index exists before running search. * Wrap searchResponse in CompletableFuture * Retrieve commands sorted by ascending timeout time. * Fix sort field * Adding scroll logic * Start implementing PIT search * Send the Search Response to a test REST endpoint * Change the command.status of submitted commands to DONE * Fix build.gradle after merging master * Remove old RestPostCommandAction class after master merge * Tidy up after master merge * Fix gradlew run error * Apply spotless * Switch to synchronous code * Adding pagination through PointInTime classes * Adding pagination through PointInTime classes * Only set sort fields for the first page. * Fix empty search response errors * Improve while loop * Make PointInTime a non-singleton class * Simplificar get instancia. Agregar cabecera de licencia faltante. * Fix linter warnings * Block until PointInTimeBuilder is obtained * Create one pit per search * Make SearchJob a non-singleton class * Indexing operation timeout should be set in ms * Indexing operation timeout should be set in ms * Remove mockito explicit dependency * Removing settings object * Removing unneeded CommandManagerJobRunner object * JavaDocs for scheduleCommandJob method * Use fqn for job index name constant * Add javadocs for parseInstantValue * Add javadocs for CommandManagerJobParameter * Add javadocs for CommandManagerJobRunner * Remove unneeded parameter * Add javadocs to JobDocument class * Add javadocs to JobDocument's create() method * Rename SearchJob to SearchThread and make it implement Runnable * Improve exception handling * Improve exception handling * Use URI and credentials in settings * Rename updateStatusField to setSentStatus and improve it * Make "command" a constant * Make queries constant * Refactor totalHits() * Remove unneeded getters * Improve buildPit() exception handling * Load settings from environment * Fix JavaDocs * Improve searchAfter logic * Fixing errors after merge * Fix build.gradle errors Applies Spotless * Fix settings related errors * Fix http client permission issues * Send orders array with proper authentication * Only set status to SENT when a non-error response has been received * Add missing import * Use https for the Management API mock --------- Signed-off-by: Álex Ruiz Signed-off-by: Fede Galland <99492720+f-galland@users.noreply.github.com> Co-authored-by: Álex Ruiz Co-authored-by: Malena Casas --- plugins/command-manager/build.gradle | 51 ++- .../commandmanager/CommandManagerPlugin.java | 152 ++++++++- .../CommandManagerJobParameter.java | 113 +++++++ .../jobscheduler/CommandManagerJobRunner.java | 93 ++++++ .../jobscheduler/JobDocument.java | 80 +++++ .../jobscheduler/SearchThread.java | 314 ++++++++++++++++++ ...rch.jobscheduler.spi.JobSchedulerExtension | 6 + 7 files changed, 790 insertions(+), 19 deletions(-) create mode 100644 plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobParameter.java create mode 100644 plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobRunner.java create mode 100644 plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java create mode 100644 plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java create mode 100644 plugins/command-manager/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension diff --git a/plugins/command-manager/build.gradle b/plugins/command-manager/build.gradle index 69655c32..6b5864d1 100644 --- a/plugins/command-manager/build.gradle +++ b/plugins/command-manager/build.gradle @@ -1,9 +1,13 @@ import org.opensearch.gradle.test.RestIntegTestTask +import java.util.concurrent.Callable buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "2.18.0-SNAPSHOT") + opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","") + opensearch_build = opensearch_no_snapshot + ".0" + job_scheduler_version = System.getProperty("job_scheduler.version", opensearch_build) wazuh_version = System.getProperty("version", "5.0.0") revision = System.getProperty("revision", "0") } @@ -24,7 +28,7 @@ apply plugin: 'java' apply plugin: 'idea' apply plugin: 'eclipse' apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.pluginzip' def pluginName = 'wazuh-indexer-command-manager' @@ -67,6 +71,7 @@ opensearchplugin { name pluginName description pluginDescription classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + extendedPlugins = ['opensearch-job-scheduler'] licenseFile rootProject.file('LICENSE.txt') noticeFile rootProject.file('NOTICE.txt') } @@ -88,6 +93,10 @@ def versions = [ imposter: "4.1.2" ] +configurations { + zipArchive +} + dependencies { implementation "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" implementation "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" @@ -99,6 +108,11 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + // Job Scheduler stuff + zipArchive group: 'org.opensearch.plugin', name: 'opensearch-job-scheduler', version: "${opensearch_build}" + // implementation "org.opensearch:opensearch:${opensearch_version}" + compileOnly "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}" + testImplementation "junit:junit:${versions.junit}" // imposter @@ -134,30 +148,37 @@ test { jvmArgs "-Djava.security.policy=./plugins/command-manager/src/main/plugin-metadata/plugin-security.policy/plugin-security.policy" } -task integTest(type: RestIntegTestTask) { - description = "Run tests against a cluster" - testClassesDirs = sourceSets.test.output.classesDirs - classpath = sourceSets.test.runtimeClasspath +def getJobSchedulerPlugin() { + provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile + } + } + } + }) } -tasks.named("check").configure { dependsOn(integTest) } -integTest { +testClusters.integTest { + plugin(getJobSchedulerPlugin()) + testDistribution = "ARCHIVE" + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable if (System.getProperty("test.debug") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' } -} - -testClusters.integTest { - testDistribution = "INTEG_TEST" - //testDistribution = "ARCHIVE" - // This installs our plugin into the testClusters - plugin(project.tasks.bundlePlugin.archiveFile) // add customized keystore keystore 'm_api.auth.username', 'admin' keystore 'm_api.auth.password', 'test' - keystore 'm_api.uri', 'http://127.0.0.1:55000' // base URI of the M_API + keystore 'm_api.uri', 'https://127.0.0.1:55000' // base URI of the M_API } run { diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java index 71b805fd..66444ff7 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java @@ -8,33 +8,52 @@ */ package com.wazuh.commandmanager; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.UUIDs; import org.opensearch.common.settings.*; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; +import org.opensearch.jobscheduler.spi.JobSchedulerExtension; +import org.opensearch.jobscheduler.spi.ScheduledJobParser; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.ReloadablePlugin; import org.opensearch.repositories.RepositoriesService; -import org.opensearch.rest.RestController; -import org.opensearch.rest.RestHandler; +import org.opensearch.rest.*; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import com.wazuh.commandmanager.index.CommandIndex; +import com.wazuh.commandmanager.jobscheduler.CommandManagerJobParameter; +import com.wazuh.commandmanager.jobscheduler.CommandManagerJobRunner; +import com.wazuh.commandmanager.jobscheduler.JobDocument; import com.wazuh.commandmanager.rest.RestPostCommandAction; import com.wazuh.commandmanager.settings.PluginSettings; import com.wazuh.commandmanager.utils.httpclient.HttpRestClient; @@ -42,15 +61,30 @@ /** * The Command Manager plugin exposes an HTTP API with a single endpoint to receive raw commands * from the Wazuh Server. These commands are processed, indexed and sent back to the Server for its - * delivery to, in most cases, the Agents. + * delivery to, in most cases, the Agents. The Command Manager plugin exposes an HTTP API with a + * single endpoint to receive raw commands from the Wazuh Server. These commands are processed, + * indexed and sent back to the Server for its delivery to, in most cases, the Agents. + * + *

The Command Manager plugin is also a JobScheduler extension plugin. */ -public class CommandManagerPlugin extends Plugin implements ActionPlugin, ReloadablePlugin { +public class CommandManagerPlugin extends Plugin + implements ActionPlugin, ReloadablePlugin, JobSchedulerExtension { public static final String COMMAND_MANAGER_BASE_URI = "/_plugins/_command_manager"; public static final String COMMANDS_URI = COMMAND_MANAGER_BASE_URI + "/commands"; public static final String COMMAND_MANAGER_INDEX_NAME = ".commands"; public static final String COMMAND_MANAGER_INDEX_TEMPLATE_NAME = "index-template-commands"; + public static final String COMMAND_DOCUMENT_PARENT_OBJECT_NAME = "command"; + public static final String JOB_INDEX_NAME = ".scheduled-commands"; + public static final Integer JOB_PERIOD_MINUTES = 1; + public static final Integer PAGE_SIZE = 2; + public static final Long DEFAULT_TIMEOUT_SECONDS = 20L; + public static final TimeValue PIT_KEEPALIVE_SECONDS = TimeValue.timeValueSeconds(30L); + + private static final Logger log = LogManager.getLogger(CommandManagerPlugin.class); + public static final String JOB_TYPE = "command_manager_scheduler_extension"; private CommandIndex commandIndex; + private JobDocument jobDocument; @Override public Collection createComponents( @@ -68,9 +102,50 @@ public Collection createComponents( this.commandIndex = new CommandIndex(client, clusterService, threadPool); PluginSettings.getInstance(environment.settings()); + // JobSchedulerExtension stuff + CommandManagerJobRunner.getInstance() + .setThreadPool(threadPool) + .setClient(client) + .setClusterService(clusterService) + .setEnvironment(environment); + + scheduleCommandJob(client, clusterService, threadPool); + return Collections.emptyList(); } + /** + * Indexes a document into the jobs index, so that JobScheduler plugin can run it + * + * @param client: The cluster client, used for indexing + * @param clusterService: Provides the addListener method. We use it to determine if this is a + * new cluster. + * @param threadPool: Used by jobDocument to create the document in a thread. + */ + private void scheduleCommandJob( + Client client, ClusterService clusterService, ThreadPool threadPool) { + clusterService.addListener( + event -> { + if (event.localNodeClusterManager() && event.isNewCluster()) { + jobDocument = JobDocument.getInstance(); + CompletableFuture indexResponseCompletableFuture = + jobDocument.create( + client, + threadPool, + UUIDs.base64UUID(), + getJobType(), + JOB_PERIOD_MINUTES); + indexResponseCompletableFuture.thenAccept( + indexResponse -> { + log.info( + "Scheduled task successfully, response: {}", + indexResponse.getResult().toString()); + }); + } + }); + } + + @Override public List getRestHandlers( Settings settings, RestController restController, @@ -100,6 +175,75 @@ public void reload(Settings settings) { // xxxService.refreshAndClearCache(commandManagerSettings); } + @Override + public String getJobType() { + return CommandManagerPlugin.JOB_TYPE; + } + + @Override + public String getJobIndex() { + return CommandManagerPlugin.JOB_INDEX_NAME; + } + + @Override + public ScheduledJobRunner getJobRunner() { + log.info("getJobRunner() executed"); + return CommandManagerJobRunner.getInstance(); + } + + @Override + public ScheduledJobParser getJobParser() { + log.info("getJobParser() executed"); + return (parser, id, jobDocVersion) -> { + CommandManagerJobParameter jobParameter = new CommandManagerJobParameter(); + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + + while (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case CommandManagerJobParameter.NAME_FIELD: + jobParameter.setJobName(parser.text()); + break; + case CommandManagerJobParameter.ENABLED_FIELD: + jobParameter.setEnabled(parser.booleanValue()); + break; + case CommandManagerJobParameter.ENABLED_TIME_FIELD: + jobParameter.setEnabledTime(parseInstantValue(parser)); + break; + case CommandManagerJobParameter.LAST_UPDATE_TIME_FIELD: + jobParameter.setLastUpdateTime(parseInstantValue(parser)); + break; + case CommandManagerJobParameter.SCHEDULE_FIELD: + jobParameter.setSchedule(ScheduleParser.parse(parser)); + break; + default: + XContentParserUtils.throwUnknownToken( + parser.currentToken(), parser.getTokenLocation()); + } + } + return jobParameter; + }; + } + + /** + * Returns the proper Instant object with milliseconds from the Unix epoch when the current + * token actually holds a value. + * + * @param parser: The parser as provided by JobScheduler + */ + private Instant parseInstantValue(XContentParser parser) throws IOException { + if (XContentParser.Token.VALUE_NULL.equals(parser.currentToken())) { + return null; + } + if (parser.currentToken().isValue()) { + return Instant.ofEpochMilli(parser.longValue()); + } + XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); + return null; + } + /** * Close the resources opened by this plugin. * diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobParameter.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobParameter.java new file mode 100644 index 00000000..c6da74a3 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobParameter.java @@ -0,0 +1,113 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.jobscheduler; + +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.Schedule; + +import java.io.IOException; +import java.time.Instant; + +/** A model for the parameters and schema to be indexed to the jobs index. */ +public class CommandManagerJobParameter implements ScheduledJobParameter { + public static final String NAME_FIELD = "name"; + public static final String ENABLED_FIELD = "enabled"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String ENABLED_TIME_FIELD_READABLE = "enabled_time_field"; + + private String jobName; + private Instant lastUpdateTime; + private Instant enabledTime; + private boolean isEnabled; + private Schedule schedule; + + public CommandManagerJobParameter() {} + + public CommandManagerJobParameter(String jobName, Schedule schedule) { + this.jobName = jobName; + this.schedule = schedule; + + Instant now = Instant.now(); + this.isEnabled = true; + this.enabledTime = now; + this.lastUpdateTime = now; + } + + @Override + public String getName() { + return this.jobName; + } + + @Override + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + + @Override + public Instant getEnabledTime() { + return this.enabledTime; + } + + @Override + public Schedule getSchedule() { + return this.schedule; + } + + @Override + public boolean isEnabled() { + return this.isEnabled; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + + public void setEnabled(boolean enabled) { + isEnabled = enabled; + } + + public void setSchedule(Schedule schedule) { + this.schedule = schedule; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD, this.jobName); + builder.field(ENABLED_FIELD, this.isEnabled); + builder.field(SCHEDULE_FIELD, this.schedule); + if (this.enabledTime != null) { + builder.timeField( + ENABLED_TIME_FIELD, + ENABLED_TIME_FIELD_READABLE, + this.enabledTime.toEpochMilli()); + } + if (this.lastUpdateTime != null) { + builder.timeField( + LAST_UPDATE_TIME_FIELD, + LAST_UPDATE_TIME_FIELD_READABLE, + this.lastUpdateTime.toEpochMilli()); + } + builder.endObject(); + + return builder; + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobRunner.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobRunner.java new file mode 100644 index 00000000..3f2e32e0 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobRunner.java @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.env.Environment; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.threadpool.ThreadPool; + +import com.wazuh.commandmanager.CommandManagerPlugin; + +/** + * Implements the ScheduledJobRunner interface, which exposes the runJob() method, which executes + * the job's logic in its own thread. + */ +public class CommandManagerJobRunner implements ScheduledJobRunner { + + private static final Logger log = LogManager.getLogger(CommandManagerJobRunner.class); + private static CommandManagerJobRunner INSTANCE; + private ThreadPool threadPool; + private ClusterService clusterService; + + private Client client; + private Environment environment; + + private CommandManagerJobRunner() { + // Singleton class, use getJobRunner method instead of constructor + } + + public static CommandManagerJobRunner getInstance() { + log.info("Getting Job Runner Instance"); + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (CommandManagerJobRunner.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new CommandManagerJobRunner(); + return INSTANCE; + } + } + + private boolean commandManagerIndexExists() { + return this.clusterService + .state() + .routingTable() + .hasIndex(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); + } + + @Override + public void runJob(ScheduledJobParameter jobParameter, JobExecutionContext context) { + if (!commandManagerIndexExists()) { + log.info( + "{} index not yet created, not running command manager jobs", + CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); + return; + } + SearchThread searchThread = new SearchThread(this.client); + threadPool.generic().submit(searchThread); + } + + public CommandManagerJobRunner setClusterService(ClusterService clusterService) { + this.clusterService = clusterService; + return getInstance(); + } + + public CommandManagerJobRunner setClient(Client client) { + this.client = client; + return getInstance(); + } + + public CommandManagerJobRunner setEnvironment(Environment environment) { + this.environment = environment; + return getInstance(); + } + + public CommandManagerJobRunner setThreadPool(ThreadPool threadPool) { + this.threadPool = threadPool; + return getInstance(); + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java new file mode 100644 index 00000000..cf776d39 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.client.Client; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +import com.wazuh.commandmanager.CommandManagerPlugin; + +/** Indexes the command job to the Jobs index. */ +public class JobDocument { + private static final Logger log = LogManager.getLogger(JobDocument.class); + private static final JobDocument INSTANCE = new JobDocument(); + + private JobDocument() {} + + public static JobDocument getInstance() { + log.info("Getting JobDocument Instance"); + return INSTANCE; + } + + /** + * Writes a CommandManagerJobParameter type document to the jobs index + * + * @param client: The cluster's client + * @param threadPool: The cluster's threadPool + * @param id: The job ID to be used + * @param jobName: The name of the job + * @param interval: The interval the action is expected to run at + * @return a CompletableFuture that will hold the IndexResponse. + */ + public CompletableFuture create( + Client client, ThreadPool threadPool, String id, String jobName, Integer interval) { + CompletableFuture completableFuture = new CompletableFuture<>(); + ExecutorService executorService = threadPool.executor(ThreadPool.Names.WRITE); + CommandManagerJobParameter jobParameter = + new CommandManagerJobParameter( + jobName, new IntervalSchedule(Instant.now(), interval, ChronoUnit.MINUTES)); + try { + IndexRequest indexRequest = + new IndexRequest() + .index(CommandManagerPlugin.JOB_INDEX_NAME) + .id(id) + .source(jobParameter.toXContent(JsonXContent.contentBuilder(), null)) + .create(true); + executorService.submit( + () -> { + try (ThreadContext.StoredContext ignored = + threadPool.getThreadContext().stashContext()) { + IndexResponse indexResponse = client.index(indexRequest).actionGet(); + completableFuture.complete(indexResponse); + } catch (Exception e) { + completableFuture.completeExceptionally(e); + } + }); + } catch (IOException e) { + log.error("Failed to index command with ID {}: {}", id, e); + } + return completableFuture; + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java new file mode 100644 index 00000000..052386b3 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java @@ -0,0 +1,314 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.jobscheduler; + +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.CreatePitRequest; +import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Client; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortOrder; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.*; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import com.wazuh.commandmanager.CommandManagerPlugin; +import com.wazuh.commandmanager.model.Command; +import com.wazuh.commandmanager.model.Status; +import com.wazuh.commandmanager.settings.PluginSettings; +import com.wazuh.commandmanager.utils.httpclient.AuthHttpRestClient; + +/** + * The class in charge of searching and managing commands in {@link + * com.wazuh.commandmanager.model.Status#PENDING} status and of submitting them to the destination + * client. + */ +public class SearchThread implements Runnable { + public static final String COMMAND_STATUS_FIELD = + Command.COMMAND + "." + Command.STATUS + ".keyword"; + public static final String COMMAND_ORDER_ID_FIELD = + Command.COMMAND + "." + Command.ORDER_ID + ".keyword"; + public static final String COMMAND_TIMEOUT_FIELD = Command.COMMAND + "." + Command.TIMEOUT; + private static final Logger log = LogManager.getLogger(SearchThread.class); + public static final String ORDERS_OBJECT = "/orders"; + private final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + private final Client client; + private SearchResponse currentPage = null; + + public SearchThread(Client client) { + this.client = client; + } + + /** + * Retrieves a nested value from a {@code Map} in a (somewhat) safe way. + * + * @param map The parent map to look at. + * @param key The key our nested object is found under. + * @param type The type we expect the nested object to be of. + * @param The type of the nested object. + * @return the nested object cast into the proper type. + */ + public static T getNestedObject(Map map, String key, Class type) { + Object value = map.get(key); + if (type.isInstance(value)) { + return type.cast(value); + } else { + throw new ClassCastException( + "Expected " + + type + + " but found " + + (value != null ? value.getClass() : "null")); + } + } + + /** + * Iterates over search results, updating their status field and submitting them to the + * destination + * + * @param searchResponse The search results page + * @throws IllegalStateException Rethrown from setSentStatus() + */ + public void handlePage(SearchResponse searchResponse) throws IllegalStateException { + SearchHits searchHits = searchResponse.getHits(); + for (SearchHit hit : searchHits) { + SimpleHttpResponse response = deliverOrders(hit); + if (response == null) { + return; + } + if (RestStatus.fromCode(response.getCode()) == RestStatus.CREATED + | RestStatus.fromCode(response.getCode()) == RestStatus.ACCEPTED + | RestStatus.fromCode(response.getCode()) == RestStatus.OK) { + setSentStatus(hit); + } + } + } + + /** + * Send the command order over HTTP + * + * @param hit The command order + */ + @SuppressWarnings("unchecked") + private SimpleHttpResponse deliverOrders(SearchHit hit) { + try (XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()) { + PluginSettings settings = PluginSettings.getInstance(); + String orders = + xContentBuilder + .map( + Collections.singletonMap( + "orders", + new Object[] { + getNestedObject( + hit.getSourceAsMap(), + Command.COMMAND, + Map.class) + })) + .toString(); + URI uri = new URIBuilder(settings.getUri() + SearchThread.ORDERS_OBJECT).build(); + return AccessController.doPrivileged( + (PrivilegedAction) + () -> AuthHttpRestClient.getInstance().post(uri, orders, hit.getId())); + } catch (IOException e) { + log.error("Error parsing hit contents: {}", e.getMessage()); + } catch (URISyntaxException e) { + log.error("Invalid URI: {}", e.getMessage()); + } + return null; + } + + /** + * Retrieves the hit's contents and updates the {@link com.wazuh.commandmanager.model.Status} + * field to {@link com.wazuh.commandmanager.model.Status#SENT}. + * + * @param hit The page's result we are to update. + * @throws IllegalStateException Raised by {@link + * org.opensearch.common.action.ActionFuture#actionGet(long)}. + */ + @SuppressWarnings("unchecked") + private void setSentStatus(SearchHit hit) throws IllegalStateException { + Map commandMap = + getNestedObject( + hit.getSourceAsMap(), + CommandManagerPlugin.COMMAND_DOCUMENT_PARENT_OBJECT_NAME, + Map.class); + commandMap.put(Command.STATUS, Status.SENT); + hit.getSourceAsMap() + .put(CommandManagerPlugin.COMMAND_DOCUMENT_PARENT_OBJECT_NAME, commandMap); + IndexRequest indexRequest = + new IndexRequest() + .index(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME) + .source(hit.getSourceAsMap()) + .id(hit.getId()); + this.client + .index(indexRequest) + .actionGet(CommandManagerPlugin.DEFAULT_TIMEOUT_SECONDS * 1000); + } + + /** + * Runs a PIT style query against the Commands index. + * + * @param pointInTimeBuilder A pit builder object used to run the query. + * @param searchAfter An array of objects containing the last page's values of the sort fields. + * @return The search response. + * @throws IllegalStateException Raised by {@link + * org.opensearch.common.action.ActionFuture#actionGet(long)}. + */ + public SearchResponse pitQuery(PointInTimeBuilder pointInTimeBuilder, Object[] searchAfter) + throws IllegalStateException { + SearchRequest searchRequest = + new SearchRequest(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); + TermQueryBuilder termQueryBuilder = + QueryBuilders.termQuery(SearchThread.COMMAND_STATUS_FIELD, Status.PENDING); + TimeValue timeout = + TimeValue.timeValueSeconds(CommandManagerPlugin.DEFAULT_TIMEOUT_SECONDS); + this.searchSourceBuilder + .query(termQueryBuilder) + .size(CommandManagerPlugin.PAGE_SIZE) + .trackTotalHits(true) + .timeout(timeout) + .pointInTimeBuilder(pointInTimeBuilder); + if (this.searchSourceBuilder.sorts() == null) { + this.searchSourceBuilder + .sort(SearchThread.COMMAND_ORDER_ID_FIELD, SortOrder.ASC) + .sort(SearchThread.COMMAND_TIMEOUT_FIELD, SortOrder.ASC); + } + if (searchAfter.length > 0) { + this.searchSourceBuilder.searchAfter(searchAfter); + } + searchRequest.source(this.searchSourceBuilder); + return this.client.search(searchRequest).actionGet(timeout); + } + + @Override + public void run() { + long consumableHits = 0L; + boolean firstPage = true; + PointInTimeBuilder pointInTimeBuilder = buildPit(); + try { + do { + this.currentPage = + pitQuery( + pointInTimeBuilder, + getSearchAfter(this.currentPage).orElse(new Object[0])); + if (firstPage) { + consumableHits = totalHits(); + firstPage = false; + } + if (consumableHits > 0) { + handlePage(this.currentPage); + consumableHits -= getPageLength(); + } + } while (consumableHits > 0); + } catch (ArrayIndexOutOfBoundsException e) { + log.error("ArrayIndexOutOfBoundsException retrieving page: {}", e.getMessage()); + } catch (IllegalStateException e) { + log.error("IllegalStateException retrieving page: {}", e.getMessage()); + } catch (Exception e) { + log.error("Generic exception retrieving page: {}", e.getMessage()); + } + } + + private long getPageLength() { + return this.currentPage.getHits().getHits().length; + } + + private long totalHits() { + if (this.currentPage.getHits().getTotalHits() != null) { + return this.currentPage.getHits().getTotalHits().value; + } else { + return 0; + } + } + + /** + * Gets the sort values of the last hit of a page. It is used by a PIT search to get the next + * page of results. + * + * @param searchResponse The current page + * @return The values of the sort fields of the last hit of a page whenever present. Otherwise, + * an empty Optional. + */ + private Optional getSearchAfter(SearchResponse searchResponse) { + if (searchResponse == null) { + return Optional.empty(); + } + try { + List hits = Arrays.asList(searchResponse.getHits().getHits()); + if (hits.isEmpty()) { + log.warn("Empty hits page, not getting searchAfter values"); + return Optional.empty(); + } + return Optional.ofNullable(hits.get(hits.size() - 1).getSortValues()); + } catch (NullPointerException | NoSuchElementException e) { + log.error("Could not get the page's searchAfter values: {}", e.getMessage()); + return Optional.empty(); + } + } + + /** + * Prepares a PointInTimeBuilder object to be used in a search. + * + * @return a PointInTimeBuilder or null. + */ + private PointInTimeBuilder buildPit() { + CompletableFuture future = new CompletableFuture<>(); + ActionListener actionListener = + new ActionListener<>() { + @Override + public void onResponse(CreatePitResponse createPitResponse) { + future.complete(createPitResponse); + } + + @Override + public void onFailure(Exception e) { + log.error(e.getMessage()); + future.completeExceptionally(e); + } + }; + this.client.createPit( + new CreatePitRequest( + CommandManagerPlugin.PIT_KEEPALIVE_SECONDS, + false, + CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME), + actionListener); + try { + return new PointInTimeBuilder(future.get().getId()); + } catch (CancellationException e) { + log.error("Building PIT was cancelled: {}", e.getMessage()); + } catch (ExecutionException e) { + log.error("Error building PIT: {}", e.getMessage()); + } catch (InterruptedException e) { + log.error("Building PIT was interrupted: {}", e.getMessage()); + } + return null; + } +} diff --git a/plugins/command-manager/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension b/plugins/command-manager/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension new file mode 100644 index 00000000..69bad7f1 --- /dev/null +++ b/plugins/command-manager/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension @@ -0,0 +1,6 @@ +# +# Copyright Wazuh Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +com.wazuh.commandmanager.CommandManagerPlugin \ No newline at end of file From 65bd2267ce7a172ed4190c39643dacba93fe5427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 4 Dec 2024 17:47:22 +0100 Subject: [PATCH 07/15] Refactor orders delivery to the `M_API` (#165) * Add document_id to the reply to the M_API * Add batching to orders delivery --- .../commandmanager/CommandManagerPlugin.java | 4 +- .../jobscheduler/SearchThread.java | 65 ++++++++++--------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java index 66444ff7..e60340ce 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java @@ -76,9 +76,9 @@ public class CommandManagerPlugin extends Plugin public static final String COMMAND_DOCUMENT_PARENT_OBJECT_NAME = "command"; public static final String JOB_INDEX_NAME = ".scheduled-commands"; public static final Integer JOB_PERIOD_MINUTES = 1; - public static final Integer PAGE_SIZE = 2; + public static final Integer PAGE_SIZE = 100; public static final Long DEFAULT_TIMEOUT_SECONDS = 20L; - public static final TimeValue PIT_KEEPALIVE_SECONDS = TimeValue.timeValueSeconds(30L); + public static final TimeValue PIT_KEEP_ALIVE_SECONDS = TimeValue.timeValueSeconds(30L); private static final Logger log = LogManager.getLogger(CommandManagerPlugin.class); public static final String JOB_TYPE = "command_manager_scheduler_extension"; diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java index 052386b3..6e268a94 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java @@ -18,6 +18,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; +import org.opensearch.common.action.ActionFuture; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; @@ -48,9 +49,8 @@ import com.wazuh.commandmanager.utils.httpclient.AuthHttpRestClient; /** - * The class in charge of searching and managing commands in {@link - * com.wazuh.commandmanager.model.Status#PENDING} status and of submitting them to the destination - * client. + * The class in charge of searching and managing commands in {@link Status#PENDING} status and of + * submitting them to the destination client. */ public class SearchThread implements Runnable { public static final String COMMAND_STATUS_FIELD = @@ -97,17 +97,38 @@ public static T getNestedObject(Map map, String key, Class orders = new ArrayList<>(); + for (SearchHit hit : searchHits) { - SimpleHttpResponse response = deliverOrders(hit); + // Create a JSON representation of each hit and add it to the orders array. + Map orderMap = + getNestedObject(hit.getSourceAsMap(), Command.COMMAND, Map.class); + // Add document id to the object. + orderMap.put("document_id", hit.getId()); + orders.add(orderMap); + } + + String payload = null; + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + payload = builder.map(Collections.singletonMap("orders", orders)).toString(); + } catch (IOException e) { + log.error("Error parsing hit contents: {}", e.getMessage()); + } + + if (payload != null) { + SimpleHttpResponse response = deliverOrders(payload); if (response == null) { return; } if (RestStatus.fromCode(response.getCode()) == RestStatus.CREATED | RestStatus.fromCode(response.getCode()) == RestStatus.ACCEPTED | RestStatus.fromCode(response.getCode()) == RestStatus.OK) { - setSentStatus(hit); + for (SearchHit hit : searchHits) { + setSentStatus(hit); + } } } } @@ -115,30 +136,15 @@ public void handlePage(SearchResponse searchResponse) throws IllegalStateExcepti /** * Send the command order over HTTP * - * @param hit The command order + * @param orders The list of order to send. */ - @SuppressWarnings("unchecked") - private SimpleHttpResponse deliverOrders(SearchHit hit) { - try (XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()) { + private SimpleHttpResponse deliverOrders(String orders) { + try { PluginSettings settings = PluginSettings.getInstance(); - String orders = - xContentBuilder - .map( - Collections.singletonMap( - "orders", - new Object[] { - getNestedObject( - hit.getSourceAsMap(), - Command.COMMAND, - Map.class) - })) - .toString(); URI uri = new URIBuilder(settings.getUri() + SearchThread.ORDERS_OBJECT).build(); return AccessController.doPrivileged( (PrivilegedAction) - () -> AuthHttpRestClient.getInstance().post(uri, orders, hit.getId())); - } catch (IOException e) { - log.error("Error parsing hit contents: {}", e.getMessage()); + () -> AuthHttpRestClient.getInstance().post(uri, orders, null)); } catch (URISyntaxException e) { log.error("Invalid URI: {}", e.getMessage()); } @@ -146,12 +152,10 @@ private SimpleHttpResponse deliverOrders(SearchHit hit) { } /** - * Retrieves the hit's contents and updates the {@link com.wazuh.commandmanager.model.Status} - * field to {@link com.wazuh.commandmanager.model.Status#SENT}. + * Retrieves the hit's contents and updates the {@link Status} field to {@link Status#SENT}. * * @param hit The page's result we are to update. - * @throws IllegalStateException Raised by {@link - * org.opensearch.common.action.ActionFuture#actionGet(long)}. + * @throws IllegalStateException Raised by {@link ActionFuture#actionGet(long)}. */ @SuppressWarnings("unchecked") private void setSentStatus(SearchHit hit) throws IllegalStateException { @@ -179,8 +183,7 @@ private void setSentStatus(SearchHit hit) throws IllegalStateException { * @param pointInTimeBuilder A pit builder object used to run the query. * @param searchAfter An array of objects containing the last page's values of the sort fields. * @return The search response. - * @throws IllegalStateException Raised by {@link - * org.opensearch.common.action.ActionFuture#actionGet(long)}. + * @throws IllegalStateException Raised by {@link ActionFuture#actionGet(long)}. */ public SearchResponse pitQuery(PointInTimeBuilder pointInTimeBuilder, Object[] searchAfter) throws IllegalStateException { @@ -296,7 +299,7 @@ public void onFailure(Exception e) { }; this.client.createPit( new CreatePitRequest( - CommandManagerPlugin.PIT_KEEPALIVE_SECONDS, + CommandManagerPlugin.PIT_KEEP_ALIVE_SECONDS, false, CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME), actionListener); From cc7c4b19d5eea9def13d6e6049348ce0edb103a9 Mon Sep 17 00:00:00 2001 From: Malena Casas Date: Thu, 5 Dec 2024 13:19:01 -0300 Subject: [PATCH 08/15] Spotles apply (#170) --- .../commandmanager/index/CommandIndex.java | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java index 9281a3c8..4024c182 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java @@ -66,15 +66,6 @@ public CompletableFuture asyncCreate(Document document) { CompletableFuture future = new CompletableFuture<>(); ExecutorService executor = this.threadPool.executor(ThreadPool.Names.WRITE); - // Create index template if it does not exist. - if (!indexTemplateExists(CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { - putIndexTemplate(CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); - } else { - log.info( - "Index template {} already exists. Skipping creation.", - CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); - } - log.info("Indexing command with id [{}]", document.getId()); try { IndexRequest request = createIndexRequest(document); @@ -82,6 +73,17 @@ public CompletableFuture asyncCreate(Document document) { () -> { try (ThreadContext.StoredContext ignored = this.threadPool.getThreadContext().stashContext()) { + // Create index template if it does not exist. + if (!indexTemplateExists( + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { + putIndexTemplate( + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } else { + log.info( + "Index template {} already exists. Skipping creation.", + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } + RestStatus restStatus = client.index(request).actionGet().status(); future.complete(restStatus); } catch (Exception e) { @@ -106,15 +108,6 @@ public CompletableFuture asyncBulkCreate(ArrayList documen CompletableFuture future = new CompletableFuture<>(); ExecutorService executor = this.threadPool.executor(ThreadPool.Names.WRITE); - // Create index template if it does not exist. - if (!indexTemplateExists(CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { - putIndexTemplate(CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); - } else { - log.info( - "Index template {} already exists. Skipping creation.", - CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); - } - BulkRequest bulkRequest = new BulkRequest(); for (Document document : documents) { log.info("Adding command with id [{}] to the bulk request", document.getId()); @@ -132,6 +125,17 @@ public CompletableFuture asyncBulkCreate(ArrayList documen () -> { try (ThreadContext.StoredContext ignored = this.threadPool.getThreadContext().stashContext()) { + // Create index template if it does not exist. + if (!indexTemplateExists( + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { + putIndexTemplate( + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } else { + log.info( + "Index template {} already exists. Skipping creation.", + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } + RestStatus restStatus = client.bulk(bulkRequest).actionGet().status(); future.complete(restStatus); } catch (Exception e) { @@ -174,18 +178,11 @@ public void putIndexTemplate(String templateName) { .name(templateName) .patterns((List) template.get("index_patterns")); - executor.submit( - () -> { - AcknowledgedResponse acknowledgedResponse = - this.client - .admin() - .indices() - .putTemplate(putIndexTemplateRequest) - .actionGet(); - if (acknowledgedResponse.isAcknowledged()) { - log.info("Index template [{}] created successfully", templateName); - } - }); + AcknowledgedResponse acknowledgedResponse = + this.client.admin().indices().putTemplate(putIndexTemplateRequest).actionGet(); + if (acknowledgedResponse.isAcknowledged()) { + log.info("Index template [{}] created successfully", templateName); + } } catch (IOException e) { log.error("Error reading index template [{}] from filesystem", templateName); From 6d1b2d9f177e9976bfbb247864e5b8a31bd82253 Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Fri, 6 Dec 2024 10:23:29 -0300 Subject: [PATCH 09/15] Update ports index template adding interface field at top level (#174) LGTM! Merging --- .../setup/src/main/resources/index-template-ports.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/setup/src/main/resources/index-template-ports.json b/plugins/setup/src/main/resources/index-template-ports.json index a37680f8..e96eb56c 100644 --- a/plugins/setup/src/main/resources/index-template-ports.json +++ b/plugins/setup/src/main/resources/index-template-ports.json @@ -299,6 +299,14 @@ } } }, + "interface": { + "properties": { + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "network": { "properties": { "protocol": { From ed6d08afd5269d45a9bade4d45fb099e31e1d140 Mon Sep 17 00:00:00 2001 From: Fede Galland <99492720+f-galland@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:23:54 -0300 Subject: [PATCH 10/15] Add .scheduled-commands template (#173) * Add .scheduled-commands template * Creating scheduled commands index template upon job startup --- .../commandmanager/CommandManagerPlugin.java | 2 + .../commandmanager/index/CommandIndex.java | 15 +++--- .../jobscheduler/JobDocument.java | 13 ++++- .../utils/IndexTemplateUtils.java | 52 +++++++++++++++++++ .../index-template-scheduled-commands.json | 51 ++++++++++++++++++ .../com/wazuh/setup/index/WazuhIndices.java | 1 + .../index-template-scheduled-commands.json | 51 ++++++++++++++++++ 7 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 plugins/command-manager/src/main/resources/index-template-scheduled-commands.json create mode 100644 plugins/setup/src/main/resources/index-template-scheduled-commands.json diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java index e60340ce..d52301ba 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java @@ -75,6 +75,7 @@ public class CommandManagerPlugin extends Plugin public static final String COMMAND_MANAGER_INDEX_TEMPLATE_NAME = "index-template-commands"; public static final String COMMAND_DOCUMENT_PARENT_OBJECT_NAME = "command"; public static final String JOB_INDEX_NAME = ".scheduled-commands"; + public static final String JOB_INDEX_TEMPLATE_NAME = "index-template-scheduled-commands"; public static final Integer JOB_PERIOD_MINUTES = 1; public static final Integer PAGE_SIZE = 100; public static final Long DEFAULT_TIMEOUT_SECONDS = 20L; @@ -130,6 +131,7 @@ private void scheduleCommandJob( jobDocument = JobDocument.getInstance(); CompletableFuture indexResponseCompletableFuture = jobDocument.create( + clusterService, client, threadPool, UUIDs.base64UUID(), diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java index 4024c182..8db3fd56 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java @@ -74,10 +74,12 @@ public CompletableFuture asyncCreate(Document document) { try (ThreadContext.StoredContext ignored = this.threadPool.getThreadContext().stashContext()) { // Create index template if it does not exist. - if (!indexTemplateExists( + if (!IndexTemplateUtils.indexTemplateExists( + this.clusterService, CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { - putIndexTemplate( - CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + IndexTemplateUtils.putIndexTemplate( + this.client, + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); } else { log.info( "Index template {} already exists. Skipping creation.", @@ -126,9 +128,11 @@ public CompletableFuture asyncBulkCreate(ArrayList documen try (ThreadContext.StoredContext ignored = this.threadPool.getThreadContext().stashContext()) { // Create index template if it does not exist. - if (!indexTemplateExists( + if (!IndexTemplateUtils.indexTemplateExists( + this.clusterService, CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { - putIndexTemplate( + IndexTemplateUtils.putIndexTemplate( + this.client, CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); } else { log.info( @@ -166,7 +170,6 @@ public boolean indexTemplateExists(String template_name) { * @param templateName : The name if the index template to load */ public void putIndexTemplate(String templateName) { - ExecutorService executor = this.threadPool.executor(ThreadPool.Names.WRITE); try { // @throws IOException Map template = IndexTemplateUtils.fromFile(templateName + ".json"); diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java index cf776d39..5522c9c9 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java @@ -8,11 +8,13 @@ */ package com.wazuh.commandmanager.jobscheduler; +import com.wazuh.commandmanager.utils.IndexTemplateUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; @@ -49,7 +51,7 @@ public static JobDocument getInstance() { * @return a CompletableFuture that will hold the IndexResponse. */ public CompletableFuture create( - Client client, ThreadPool threadPool, String id, String jobName, Integer interval) { + ClusterService clusterService, Client client, ThreadPool threadPool, String id, String jobName, Integer interval) { CompletableFuture completableFuture = new CompletableFuture<>(); ExecutorService executorService = threadPool.executor(ThreadPool.Names.WRITE); CommandManagerJobParameter jobParameter = @@ -65,7 +67,14 @@ public CompletableFuture create( executorService.submit( () -> { try (ThreadContext.StoredContext ignored = - threadPool.getThreadContext().stashContext()) { + threadPool.getThreadContext().stashContext()) { + if (!IndexTemplateUtils.indexTemplateExists(clusterService,CommandManagerPlugin.JOB_INDEX_TEMPLATE_NAME)) { + IndexTemplateUtils.putIndexTemplate(client, CommandManagerPlugin.JOB_INDEX_TEMPLATE_NAME); + } else { + log.info( + "Index template {} already exists. Skipping creation.", + CommandManagerPlugin.JOB_INDEX_NAME); + } IndexResponse indexResponse = client.index(indexRequest).actionGet(); completableFuture.complete(indexResponse); } catch (Exception e) { diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/IndexTemplateUtils.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/IndexTemplateUtils.java index 409b133c..6af5616a 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/IndexTemplateUtils.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/IndexTemplateUtils.java @@ -8,6 +8,14 @@ */ package com.wazuh.commandmanager.utils; +import com.wazuh.commandmanager.index.CommandIndex; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -15,12 +23,14 @@ import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Map; import reactor.util.annotation.NonNull; /** Util functions to parse and manage index templates files. */ public class IndexTemplateUtils { + private static final Logger log = LogManager.getLogger(IndexTemplateUtils.class); /** Default constructor */ public IndexTemplateUtils() {} @@ -70,4 +80,46 @@ public static Map toMap(InputStream is) throws IOException { public static Map get(Map map, String key) { return (Map) map.get(key); } + + /** + * Checks for the existence of the given index template in the cluster. + * + * @param clusterService The cluster service used to check the node's existence + * @param templateName index template name within the resources folder + * @return whether the index template exists. + */ + public static boolean indexTemplateExists(ClusterService clusterService, String templateName) { + Map templates = + clusterService.state().metadata().templates(); + log.debug("Existing index templates: {} ", templates); + + return templates.containsKey(templateName); + } + + /** + * Inserts an index template + * @param templateName : The name if the index template to load + */ + public static void putIndexTemplate(Client client, String templateName) { + try { + // @throws IOException + Map template = IndexTemplateUtils.fromFile(templateName + ".json"); + + PutIndexTemplateRequest putIndexTemplateRequest = + new PutIndexTemplateRequest() + .mapping(IndexTemplateUtils.get(template, "mappings")) + .settings(IndexTemplateUtils.get(template, "settings")) + .name(templateName) + .patterns((List) template.get("index_patterns")); + + AcknowledgedResponse acknowledgedResponse = + client.admin().indices().putTemplate(putIndexTemplateRequest).actionGet(); + if (acknowledgedResponse.isAcknowledged()) { + log.info("Index template [{}] created successfully", templateName); + } + + } catch (IOException e) { + log.error("Error reading index template [{}] from filesystem", templateName); + } + } } diff --git a/plugins/command-manager/src/main/resources/index-template-scheduled-commands.json b/plugins/command-manager/src/main/resources/index-template-scheduled-commands.json new file mode 100644 index 00000000..232dbe73 --- /dev/null +++ b/plugins/command-manager/src/main/resources/index-template-scheduled-commands.json @@ -0,0 +1,51 @@ +{ + "index_patterns": [ + ".scheduled-commands" + ], + "mappings": { + "dynamic": "strict", + "properties": { + "name": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "schedule": { + "properties": { + "interval": { + "properties": { + "start_time": { + "type": "date", + "format": "epoch_millis" + }, + "period": { + "type": "integer" + }, + "unit": { + "type": "keyword" + } + } + } + } + }, + "enabled_time": { + "type": "date", + "format": "epoch_millis" + }, + "last_update_time": { + "type": "date", + "format": "epoch_millis" + } + } + }, + "order": 1, + "settings": { + "index": { + "hidden": true, + "number_of_replicas": "0", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } +} diff --git a/plugins/setup/src/main/java/com/wazuh/setup/index/WazuhIndices.java b/plugins/setup/src/main/java/com/wazuh/setup/index/WazuhIndices.java index a0073efa..b7dd165d 100644 --- a/plugins/setup/src/main/java/com/wazuh/setup/index/WazuhIndices.java +++ b/plugins/setup/src/main/java/com/wazuh/setup/index/WazuhIndices.java @@ -50,6 +50,7 @@ public WazuhIndices(Client client, ClusterService clusterService) { this.indexTemplates.put("index-template-agent", ".agents"); this.indexTemplates.put("index-template-alerts", "wazuh-alerts-5.x-0001"); this.indexTemplates.put("index-template-commands", ".commands"); + this.indexTemplates.put("index-template-scheduled-commands", ".scheduled-commands"); this.indexTemplates.put("index-template-fim", "wazuh-states-fim"); this.indexTemplates.put("index-template-hardware", "wazuh-states-inventory-hardware"); this.indexTemplates.put("index-template-hotfixes", "wazuh-states-inventory-hotfixes"); diff --git a/plugins/setup/src/main/resources/index-template-scheduled-commands.json b/plugins/setup/src/main/resources/index-template-scheduled-commands.json new file mode 100644 index 00000000..232dbe73 --- /dev/null +++ b/plugins/setup/src/main/resources/index-template-scheduled-commands.json @@ -0,0 +1,51 @@ +{ + "index_patterns": [ + ".scheduled-commands" + ], + "mappings": { + "dynamic": "strict", + "properties": { + "name": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "schedule": { + "properties": { + "interval": { + "properties": { + "start_time": { + "type": "date", + "format": "epoch_millis" + }, + "period": { + "type": "integer" + }, + "unit": { + "type": "keyword" + } + } + } + } + }, + "enabled_time": { + "type": "date", + "format": "epoch_millis" + }, + "last_update_time": { + "type": "date", + "format": "epoch_millis" + } + } + }, + "order": 1, + "settings": { + "index": { + "hidden": true, + "number_of_replicas": "0", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } +} From efc1675587782850b5db19259adc0729f21536f3 Mon Sep 17 00:00:00 2001 From: Fede Galland <99492720+f-galland@users.noreply.github.com> Date: Mon, 9 Dec 2024 06:49:41 -0300 Subject: [PATCH 11/15] Fix all shards failed error when delivering command orders (#176) * Fix hit.getSourceAsMap() dereferencing * Cleanup code --- .../jobscheduler/SearchThread.java | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java index 6e268a94..309d4679 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java @@ -54,9 +54,9 @@ */ public class SearchThread implements Runnable { public static final String COMMAND_STATUS_FIELD = - Command.COMMAND + "." + Command.STATUS + ".keyword"; + Command.COMMAND + "." + Command.STATUS; public static final String COMMAND_ORDER_ID_FIELD = - Command.COMMAND + "." + Command.ORDER_ID + ".keyword"; + Command.COMMAND + "." + Command.ORDER_ID; public static final String COMMAND_TIMEOUT_FIELD = Command.COMMAND + "." + Command.TIMEOUT; private static final Logger log = LogManager.getLogger(SearchThread.class); public static final String ORDERS_OBJECT = "/orders"; @@ -79,14 +79,24 @@ public SearchThread(Client client) { */ public static T getNestedObject(Map map, String key, Class type) { Object value = map.get(key); + if (value == null) { + return null; + } if (type.isInstance(value)) { + // Make a defensive copy for supported types like Map or List + if (value instanceof Map) { + return type.cast(new HashMap<>((Map) value)); + } else if (value instanceof List) { + return type.cast(new ArrayList<>((List) value)); + } + // Return the value directly if it is immutable (e.g., String, Integer) return type.cast(value); } else { throw new ClassCastException( - "Expected " - + type - + " but found " - + (value != null ? value.getClass() : "null")); + "Expected " + + type.getName() + + " but found " + + value.getClass().getName()); } } @@ -101,16 +111,13 @@ public static T getNestedObject(Map map, String key, Class orders = new ArrayList<>(); - for (SearchHit hit : searchHits) { - // Create a JSON representation of each hit and add it to the orders array. - Map orderMap = - getNestedObject(hit.getSourceAsMap(), Command.COMMAND, Map.class); - // Add document id to the object. - orderMap.put("document_id", hit.getId()); - orders.add(orderMap); + Map orderMap = getNestedObject(hit.getSourceAsMap(), Command.COMMAND, Map.class); + if (orderMap != null) { + orderMap.put("document_id", hit.getId()); + orders.add(orderMap); + } } - String payload = null; try (XContentBuilder builder = XContentFactory.jsonBuilder()) { payload = builder.map(Collections.singletonMap("orders", orders)).toString(); @@ -160,21 +167,21 @@ private SimpleHttpResponse deliverOrders(String orders) { @SuppressWarnings("unchecked") private void setSentStatus(SearchHit hit) throws IllegalStateException { Map commandMap = - getNestedObject( - hit.getSourceAsMap(), - CommandManagerPlugin.COMMAND_DOCUMENT_PARENT_OBJECT_NAME, - Map.class); + getNestedObject( + hit.getSourceAsMap(), + CommandManagerPlugin.COMMAND_DOCUMENT_PARENT_OBJECT_NAME, + Map.class); commandMap.put(Command.STATUS, Status.SENT); hit.getSourceAsMap() - .put(CommandManagerPlugin.COMMAND_DOCUMENT_PARENT_OBJECT_NAME, commandMap); + .put(CommandManagerPlugin.COMMAND_DOCUMENT_PARENT_OBJECT_NAME, commandMap); IndexRequest indexRequest = - new IndexRequest() - .index(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME) - .source(hit.getSourceAsMap()) - .id(hit.getId()); + new IndexRequest() + .index(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME) + .source(hit.getSourceAsMap()) + .id(hit.getId()); this.client - .index(indexRequest) - .actionGet(CommandManagerPlugin.DEFAULT_TIMEOUT_SECONDS * 1000); + .index(indexRequest) + .actionGet(CommandManagerPlugin.DEFAULT_TIMEOUT_SECONDS * 1000); } /** From a17d08a34370143c6f57828221f7a6580d084a2d Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Mon, 9 Dec 2024 06:50:54 -0300 Subject: [PATCH 12/15] Update processes index template with new tty fields (#175) --- .../src/main/resources/index-template-processes.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/setup/src/main/resources/index-template-processes.json b/plugins/setup/src/main/resources/index-template-processes.json index dbb8dde3..b1fd3f18 100644 --- a/plugins/setup/src/main/resources/index-template-processes.json +++ b/plugins/setup/src/main/resources/index-template-processes.json @@ -328,6 +328,18 @@ } } }, + "tty": { + "properties": { + "char_device": { + "properties": { + "major": { + "type": "long" + } + } + } + }, + "type": "object" + }, "user": { "properties": { "id": { From 419369b05e4c8f6e46b5f810727f0e867d5206d7 Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Mon, 9 Dec 2024 08:10:28 -0300 Subject: [PATCH 13/15] Update command-manager to sort documents by delivery timeout (#169) * Update handlePage to sort hits by its delivery timeout * Use OpenSearch qeury to sort commands by its delivery timestamp * Remove info logs --------- Co-authored-by: Fede Galland <99492720+f-galland@users.noreply.github.com> --- .../com/wazuh/commandmanager/jobscheduler/SearchThread.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java index 309d4679..b8e52fe9 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java @@ -44,6 +44,7 @@ import com.wazuh.commandmanager.CommandManagerPlugin; import com.wazuh.commandmanager.model.Command; +import com.wazuh.commandmanager.model.Document; import com.wazuh.commandmanager.model.Status; import com.wazuh.commandmanager.settings.PluginSettings; import com.wazuh.commandmanager.utils.httpclient.AuthHttpRestClient; @@ -58,6 +59,7 @@ public class SearchThread implements Runnable { public static final String COMMAND_ORDER_ID_FIELD = Command.COMMAND + "." + Command.ORDER_ID; public static final String COMMAND_TIMEOUT_FIELD = Command.COMMAND + "." + Command.TIMEOUT; + public static final String DELIVERY_TIMESTAMP_FIELD = Document.DELIVERY_TIMESTAMP; private static final Logger log = LogManager.getLogger(SearchThread.class); public static final String ORDERS_OBJECT = "/orders"; private final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -207,9 +209,7 @@ public SearchResponse pitQuery(PointInTimeBuilder pointInTimeBuilder, Object[] s .timeout(timeout) .pointInTimeBuilder(pointInTimeBuilder); if (this.searchSourceBuilder.sorts() == null) { - this.searchSourceBuilder - .sort(SearchThread.COMMAND_ORDER_ID_FIELD, SortOrder.ASC) - .sort(SearchThread.COMMAND_TIMEOUT_FIELD, SortOrder.ASC); + this.searchSourceBuilder.sort(SearchThread.DELIVERY_TIMESTAMP_FIELD, SortOrder.ASC); } if (searchAfter.length > 0) { this.searchSourceBuilder.searchAfter(searchAfter); From 86e2608377271761b528d6f8cf7872ffc921feea Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Wed, 11 Dec 2024 06:15:20 -0300 Subject: [PATCH 14/15] Update alerts index template with dynamic fields enabled (#178) --- plugins/setup/src/main/resources/index-template-alerts.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/setup/src/main/resources/index-template-alerts.json b/plugins/setup/src/main/resources/index-template-alerts.json index d50aba75..1358a430 100644 --- a/plugins/setup/src/main/resources/index-template-alerts.json +++ b/plugins/setup/src/main/resources/index-template-alerts.json @@ -4,7 +4,7 @@ ], "mappings": { "date_detection": false, - "dynamic": "strict", + "dynamic": true, "properties": { "@timestamp": { "type": "date" From 5dd5396b8e9b2c8c5fecb891b4631ee80149260c Mon Sep 17 00:00:00 2001 From: Fede Galland <99492720+f-galland@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:23:15 -0300 Subject: [PATCH 15/15] Remove multi-fields and change wildcard to keyword in process template (#182) --- .../resources/index-template-processes.json | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/plugins/setup/src/main/resources/index-template-processes.json b/plugins/setup/src/main/resources/index-template-processes.json index b1fd3f18..c6bced0c 100644 --- a/plugins/setup/src/main/resources/index-template-processes.json +++ b/plugins/setup/src/main/resources/index-template-processes.json @@ -155,11 +155,6 @@ "type": "keyword" }, "full": { - "fields": { - "text": { - "type": "match_only_text" - } - }, "ignore_above": 1024, "type": "keyword" }, @@ -168,11 +163,6 @@ "type": "keyword" }, "name": { - "fields": { - "text": { - "type": "match_only_text" - } - }, "ignore_above": 1024, "type": "keyword" }, @@ -252,12 +242,7 @@ "type": "keyword" }, "command_line": { - "fields": { - "text": { - "type": "match_only_text" - } - }, - "type": "wildcard" + "type": "keyword" }, "group": { "properties": { @@ -268,11 +253,6 @@ } }, "name": { - "fields": { - "text": { - "type": "match_only_text" - } - }, "ignore_above": 1024, "type": "keyword" },