From fd0f18b9f016e2631c7071f8eb242e939eb0933d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 4 Sep 2024 13:23:03 +0200 Subject: [PATCH] Add tooling to generate the agents index template (#370) --- ecs/agent/event-generator/event_generator.py | 114 ++++++++++++++++++ ecs/agent/fields/custom/wazuh-agent.yml | 27 +++++ ecs/agent/fields/mapping-settings.json | 4 + ecs/agent/fields/subset.yml | 22 ++++ .../fields/template-settings-legacy.json | 23 ++++ ecs/agent/fields/template-settings.json | 25 ++++ ecs/generate.sh | 30 ++++- 7 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 ecs/agent/event-generator/event_generator.py create mode 100644 ecs/agent/fields/custom/wazuh-agent.yml create mode 100644 ecs/agent/fields/mapping-settings.json create mode 100644 ecs/agent/fields/subset.yml create mode 100644 ecs/agent/fields/template-settings-legacy.json create mode 100644 ecs/agent/fields/template-settings.json diff --git a/ecs/agent/event-generator/event_generator.py b/ecs/agent/event-generator/event_generator.py new file mode 100644 index 0000000000000..f676f0176d444 --- /dev/null +++ b/ecs/agent/event-generator/event_generator.py @@ -0,0 +1,114 @@ +#!/bin/python3 + +import datetime +import random +import json +import requests +import warnings +import logging + +# Constants and Configuration +LOG_FILE = 'generate_data.log' +GENERATED_DATA_FILE = 'generatedData.json' +DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + +# Configure logging +logging.basicConfig(filename=LOG_FILE, level=logging.INFO) + +# Suppress warnings +warnings.filterwarnings("ignore") + + +def generate_random_date(): + start_date = datetime.datetime.now() + end_date = start_date - datetime.timedelta(days=10) + random_date = start_date + (end_date - start_date) * random.random() + return random_date.strftime(DATE_FORMAT) + + +def generate_random_agent(): + agent = { + 'id': f'agent{random.randint(0, 99)}', + 'name': f'Agent{random.randint(0, 99)}', + 'type': random.choice(['filebeat', 'windows', 'linux', 'macos']), + 'version': f'v{random.randint(0, 9)}-stable', + 'is_connected': random.choice([True, False]), + 'last_login': generate_random_date(), + 'groups': [f'group{random.randint(0, 99)}', f'group{random.randint(0, 99)}'], + 'key': f'key{random.randint(0, 999)}' + } + return agent + + +def generate_random_host(): + family = random.choice(['debian', 'ubuntu', 'macos', 'ios', 'android', 'RHEL']) + version = f'{random.randint(0, 99)}.{random.randint(0, 99)}' + host = { + 'ip': f'{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}', + 'os': { + 'full': f'{family} {version}', + } + } + return host + + +def generate_random_data(number): + data = [] + for _ in range(number): + event_data = { + 'agent': generate_random_agent(), + 'host': generate_random_host(), + } + data.append(event_data) + return data + + +def inject_events(ip, port, index, username, password, data): + url = f'https://{ip}:{port}/{index}/_doc' + session = requests.Session() + session.auth = (username, password) + session.verify = False + headers = {'Content-Type': 'application/json'} + + try: + for event_data in data: + response = session.post(url, json=event_data, headers=headers) + if response.status_code != 201: + logging.error(f'Error: {response.status_code}') + logging.error(response.text) + break + logging.info('Data injection completed successfully.') + except Exception as e: + logging.error(f'Error: {str(e)}') + + +def main(): + try: + number = int(input("How many events do you want to generate? ")) + except ValueError: + logging.error("Invalid input. Please enter a valid number.") + return + + logging.info(f"Generating {number} events...") + data = generate_random_data(number) + + with open(GENERATED_DATA_FILE, 'a') as outfile: + for event_data in data: + json.dump(event_data, outfile) + outfile.write('\n') + + logging.info('Data generation completed.') + + inject = input( + "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower() + if inject == 'y': + ip = input("Enter the IP of your Indexer: ") + port = input("Enter the port of your Indexer: ") + index = input("Enter the index name: ") + username = input("Username: ") + password = input("Password: ") + inject_events(ip, port, index, username, password, data) + + +if __name__ == "__main__": + main() diff --git a/ecs/agent/fields/custom/wazuh-agent.yml b/ecs/agent/fields/custom/wazuh-agent.yml new file mode 100644 index 0000000000000..0492778271095 --- /dev/null +++ b/ecs/agent/fields/custom/wazuh-agent.yml @@ -0,0 +1,27 @@ +--- +- name: agent + title: Wazuh Agents + short: Wazuh Inc. custom fields. + type: group + group: 2 + fields: + - name: groups + type: keyword + level: custom + description: > + The groups the agent belongs to. + - name: key + type: keyword + level: custom + description: > + The agent's registration key. + - name: last_login + type: date + level: custom + description: > + The agent's last login. + - name: is_connected + type: boolean + level: custom + description: > + Agents' interpreted connection status depending on `agent.last_login`. diff --git a/ecs/agent/fields/mapping-settings.json b/ecs/agent/fields/mapping-settings.json new file mode 100644 index 0000000000000..0ad2b48fcc1be --- /dev/null +++ b/ecs/agent/fields/mapping-settings.json @@ -0,0 +1,4 @@ +{ + "dynamic": "strict", + "date_detection": false +} \ No newline at end of file diff --git a/ecs/agent/fields/subset.yml b/ecs/agent/fields/subset.yml new file mode 100644 index 0000000000000..2d24cd20429f2 --- /dev/null +++ b/ecs/agent/fields/subset.yml @@ -0,0 +1,22 @@ +--- +name: agent +fields: + base: + fields: + tags: [] + agent: + fields: + id: {} + name: {} + type: {} + version: {} + groups: {} + key: {} + last_login: {} + is_connected: {} + host: + fields: + ip: {} + os: + fields: + full: {} \ No newline at end of file diff --git a/ecs/agent/fields/template-settings-legacy.json b/ecs/agent/fields/template-settings-legacy.json new file mode 100644 index 0000000000000..157c89196df07 --- /dev/null +++ b/ecs/agent/fields/template-settings-legacy.json @@ -0,0 +1,23 @@ +{ + "index_patterns": [ + ".agents*" + ], + "order": 1, + "settings": { + "index": { + "hidden": true, + "number_of_shards": "1", + "number_of_replicas": "0", + "refresh_interval": "5s", + "query.default_field": [ + "agent.id", + "agent.name", + "agent.type", + "agent.version", + "agent.name", + "host.os.full", + "host.ip" + ] + } + } +} \ No newline at end of file diff --git a/ecs/agent/fields/template-settings.json b/ecs/agent/fields/template-settings.json new file mode 100644 index 0000000000000..30c94f204d38c --- /dev/null +++ b/ecs/agent/fields/template-settings.json @@ -0,0 +1,25 @@ +{ + "index_patterns": [ + ".agents*" + ], + "priority": 1, + "template": { + "settings": { + "index": { + "hidden": true, + "number_of_shards": "1", + "number_of_replicas": "0", + "refresh_interval": "5s", + "query.default_field": [ + "agent.id", + "agent.name", + "agent.type", + "agent.version", + "agent.name", + "host.os.full", + "host.ip" + ] + } + } + } +} \ No newline at end of file diff --git a/ecs/generate.sh b/ecs/generate.sh index 4b747c0c9a0cb..3d96dd446284c 100755 --- a/ecs/generate.sh +++ b/ecs/generate.sh @@ -13,6 +13,19 @@ show_usage() { echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200" } +# Function to remove multi-fields from the generated index template +remove_multi_fields() { + local IN_FILE="$1" + local OUT_FILE="$2" + + jq 'del( + .mappings.properties.host.properties.os.properties.full.fields, + .mappings.properties.host.properties.os.properties.name.fields, + .mappings.properties.vulnerability.properties.description.fields + )' "$IN_FILE" > "$OUT_FILE" +} + + # Function to generate mappings generate_mappings() { local IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields" @@ -34,8 +47,21 @@ generate_mappings() { echo "Replacing \"match_only_text\" type with \"text\"" find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \; + local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json" + local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json" + + # Delete the "tags" field from the index template + echo "Deleting the \"tags\" field from the index template" + jq 'del(.mappings.properties.tags)' "$IN_FILE" > "$OUT_FILE" + mv "$OUT_FILE" "$IN_FILE" + + # Remove multi-fields from the generated index template + echo "Removing multi-fields from the index template" + remove_multi_fields "$IN_FILE" "$OUT_FILE" + mv "$OUT_FILE" "$IN_FILE" + # Transform legacy index template for OpenSearch compatibility - cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{ + cat "$IN_FILE" | jq '{ "index_patterns": .index_patterns, "priority": .order, "template": { @@ -79,4 +105,4 @@ UPLOAD="${4:-false}" URL="${5:-https://localhost:9200}" # Generate mappings -generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL" +generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL" \ No newline at end of file