Add tooling to generate the agents index template (#370)

ecs/agent/event-generator/event_generator.py

@@ -0,0 +1,114 @@
+#!/bin/python3
+
+import datetime
+import random
+import json
+import requests
+import warnings
+import logging
+
+# Constants and Configuration
+LOG_FILE = 'generate_data.log'
+GENERATED_DATA_FILE = 'generatedData.json'
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
+# Configure logging
+logging.basicConfig(filename=LOG_FILE, level=logging.INFO)
+
+# Suppress warnings
+warnings.filterwarnings("ignore")
+
+
+def generate_random_date():
+    start_date = datetime.datetime.now()
+    end_date = start_date - datetime.timedelta(days=10)
+    random_date = start_date + (end_date - start_date) * random.random()
+    return random_date.strftime(DATE_FORMAT)
+
+
+def generate_random_agent():
+    agent = {
+        'id': f'agent{random.randint(0, 99)}',
+        'name': f'Agent{random.randint(0, 99)}',
+        'type': random.choice(['filebeat', 'windows', 'linux', 'macos']),
+        'version': f'v{random.randint(0, 9)}-stable',
+        'is_connected': random.choice([True, False]),
+        'last_login': generate_random_date(),
+        'groups': [f'group{random.randint(0, 99)}', f'group{random.randint(0, 99)}'],
+        'key': f'key{random.randint(0, 999)}'
+    }
+    return agent
+
+
+def generate_random_host():
+    family = random.choice(['debian', 'ubuntu', 'macos', 'ios', 'android', 'RHEL'])
+    version = f'{random.randint(0, 99)}.{random.randint(0, 99)}'
+    host = {
+        'ip': f'{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}',
+        'os': {
+            'full': f'{family} {version}',
+        }
+    }
+    return host
+
+
+def generate_random_data(number):
+    data = []
+    for _ in range(number):
+        event_data = {
+            'agent': generate_random_agent(),
+            'host': generate_random_host(),
+        }
+        data.append(event_data)
+    return data
+
+
+def inject_events(ip, port, index, username, password, data):
+    url = f'https://{ip}:{port}/{index}/_doc'
+    session = requests.Session()
+    session.auth = (username, password)
+    session.verify = False
+    headers = {'Content-Type': 'application/json'}
+
+    try:
+        for event_data in data:
+            response = session.post(url, json=event_data, headers=headers)
+            if response.status_code != 201:
+                logging.error(f'Error: {response.status_code}')
+                logging.error(response.text)
+                break
+        logging.info('Data injection completed successfully.')
+    except Exception as e:
+        logging.error(f'Error: {str(e)}')
+
+
+def main():
+    try:
+        number = int(input("How many events do you want to generate? "))
+    except ValueError:
+        logging.error("Invalid input. Please enter a valid number.")
+        return
+
+    logging.info(f"Generating {number} events...")
+    data = generate_random_data(number)
+
+    with open(GENERATED_DATA_FILE, 'a') as outfile:
+        for event_data in data:
+            json.dump(event_data, outfile)
+            outfile.write('\n')
+
+    logging.info('Data generation completed.')
+
+    inject = input(
+        "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower()
+    if inject == 'y':
+        ip = input("Enter the IP of your Indexer: ")
+        port = input("Enter the port of your Indexer: ")
+        index = input("Enter the index name: ")
+        username = input("Username: ")
+        password = input("Password: ")
+        inject_events(ip, port, index, username, password, data)
+
+
+if __name__ == "__main__":
+    main()

ecs/agent/fields/custom/wazuh-agent.yml

@@ -0,0 +1,27 @@
+---
+- name: agent
+  title: Wazuh Agents
+  short: Wazuh Inc. custom fields.
+  type: group
+  group: 2
+  fields:
+    - name: groups
+      type: keyword
+      level: custom
+      description: >
+        The groups the agent belongs to.
+    - name: key
+      type: keyword
+      level: custom
+      description: >
+        The agent's registration key.
+    - name: last_login
+      type: date
+      level: custom
+      description: >
+        The agent's last login.
+    - name: is_connected
+      type: boolean
+      level: custom
+      description: >
+        Agents' interpreted connection status depending on `agent.last_login`.

ecs/agent/fields/mapping-settings.json

@@ -0,0 +1,4 @@
+{
+    "dynamic": "strict",
+    "date_detection": false
+}

ecs/agent/fields/subset.yml

@@ -0,0 +1,22 @@
+---
+name: agent
+fields:
+  base:
+    fields:
+      tags: []
+  agent:
+    fields:
+      id: {}
+      name: {}
+      type: {}
+      version: {}
+      groups: {}
+      key: {}
+      last_login: {}
+      is_connected: {}
+  host:
+    fields:
+      ip: {}
+      os:
+        fields:
+          full: {}

ecs/agent/fields/template-settings-legacy.json

@@ -0,0 +1,23 @@
+{
+  "index_patterns": [
+    ".agents*"
+  ],
+  "order": 1,
+  "settings": {
+    "index": {
+      "hidden": true,
+      "number_of_shards": "1",
+      "number_of_replicas": "0",
+      "refresh_interval": "5s",
+      "query.default_field": [
+        "agent.id",
+        "agent.name",
+        "agent.type",
+        "agent.version",
+        "agent.name",
+        "host.os.full",
+        "host.ip"
+      ]
+    }
+  }
+}

ecs/agent/fields/template-settings.json

@@ -0,0 +1,25 @@
+{
+  "index_patterns": [
+    ".agents*"
+  ],
+  "priority": 1,
+  "template": {
+    "settings": {
+      "index": {
+        "hidden": true,
+        "number_of_shards": "1",
+        "number_of_replicas": "0",
+        "refresh_interval": "5s",
+        "query.default_field": [
+          "agent.id",
+          "agent.name",
+          "agent.type",
+          "agent.version",
+          "agent.name",
+          "host.os.full",
+          "host.ip"
+        ]
+      }
+    }
+  }
+}

ecs/generate.sh

@@ -13,6 +13,19 @@ show_usage() {
   echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200"
 }
 
+# Function to remove multi-fields from the generated index template
+remove_multi_fields() {
+  local IN_FILE="$1"
+  local OUT_FILE="$2"
+
+  jq 'del(
+    .mappings.properties.host.properties.os.properties.full.fields,
+    .mappings.properties.host.properties.os.properties.name.fields,
+    .mappings.properties.vulnerability.properties.description.fields
+  )' "$IN_FILE" > "$OUT_FILE"
+}
+
+
 # Function to generate mappings
 generate_mappings() {
   local IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields"
@@ -34,8 +47,21 @@ generate_mappings() {
   echo "Replacing \"match_only_text\" type with \"text\""
   find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \;
 
+  local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json"
+  local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json"
+
+  # Delete the "tags" field from the index template
+  echo "Deleting the \"tags\" field from the index template"
+  jq 'del(.mappings.properties.tags)' "$IN_FILE" > "$OUT_FILE"
+  mv "$OUT_FILE" "$IN_FILE"
+
+  # Remove multi-fields from the generated index template
+  echo "Removing multi-fields from the index template"
+  remove_multi_fields "$IN_FILE" "$OUT_FILE"
+  mv "$OUT_FILE" "$IN_FILE"
+
   # Transform legacy index template for OpenSearch compatibility
-  cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{
+  cat "$IN_FILE" | jq '{
     "index_patterns": .index_patterns,
     "priority": .order,
     "template": {
@@ -79,4 +105,4 @@ UPLOAD="${4:-false}"
 URL="${5:-https://localhost:9200}"
 
 # Generate mappings
-generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL"
+generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL"