Implement workflow for automatic ECS templates generation

.github/workflows/generate-ecs-mappings.yml

@@ -0,0 +1,154 @@
+name: ECS Generator
+
+on:
+  push:
+    paths:
+      - 'ecs/**'
+
+jobs:
+  run-ecs-generator:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up Docker Compose
+        run: sudo apt-get install docker-compose
+
+      - name: Extract ECS Modules and Run ECS Generator
+        id: run-ecs-generator
+        run: |
+          # Fetch base branch
+          git fetch origin +refs/heads/master:refs/remotes/origin/master
+
+          # Extract the ECS module names from the modified files
+          modified_files=$(git diff --name-only origin/master)
+          updated_modules=()
+          for file in $modified_files; do
+            if [[ $file == ecs/* ]]; then
+              ecs_module=$(echo $file | cut -d'/' -f2)
+              if [[ ! " ${updated_modules[*]} " =~ " ${ecs_module} " ]]; then
+                updated_modules+=("$ecs_module")
+              fi
+            fi
+          done
+
+          # Filter out modules that do not have corresponding JSON files
+          declare -A module_to_file=(
+            [agent]="index-template-agent.json"
+            [alerts]="index-template-alerts.json"
+            [commands]="index-template-commands.json"
+            [hardware]="index-template-hardware.json"
+            [hotfixes]="index-template-hotfixes.json"
+            [fim]="index-template-fim.json"
+            [networks]="index-template-networks.json"
+            [packages]="index-template-packages.json"
+            [ports]="index-template-ports.json"
+            [processes]="index-template-processes.json"
+            [scheduled-commands]="index-template-scheduled-commands.json"
+            [system]="index-template-system.json"
+            [vulnerabilities]="index-template-vulnerabilities.json"
+          )
+
+          relevant_modules=()
+          for ecs_module in "${updated_modules[@]}"; do
+            if [[ -n "${module_to_file[$ecs_module]}" ]]; then
+              relevant_modules+=("$ecs_module")
+            fi
+          done
+
+          if [[ ${#relevant_modules[@]} -gt 0 ]]; then
+            export REPO_PATH=$(pwd)
+            for ecs_module in "${relevant_modules[@]}"; do
+              # Run the ECS generator script for each relevant module
+              bash docker/ecs/mapping-generator.sh run "$ecs_module"
+              echo "Processed ECS module: $ecs_module"
+            done
+            echo "relevant_modules=${relevant_modules[*]}" >> $GITHUB_ENV
+          else
+            echo "No relevant modifications detected in ecs/ directory."
+            exit 0
+          fi
+
+      - name: Tear down ECS Generator
+        if: always()
+        run: bash docker/ecs/mapping-generator.sh down
+
+      - name: Upload artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ecs-template
+          path: ecs/**/mappings/v8.11.0/generated/elasticsearch/legacy/template.json
+
+      - name: Checkout target repository
+        uses: actions/checkout@v4
+        with:
+          repository: wazuh/wazuh-indexer-plugins
+          token: ${{ secrets.GITHUB_TOKEN }}
+          path: wazuh-indexer-plugins
+
+      - name: Copy generated files to target repository
+        run: |
+          # Map ECS modules to target JSON filenames
+          declare -A module_to_file=(
+            [agent]="index-template-agent.json"
+            [alerts]="index-template-alerts.json"
+            [commands]="index-template-commands.json"
+            [hardware]="index-template-hardware.json"
+            [hotfixes]="index-template-hotfixes.json"
+            [fim]="index-template-fim.json"
+            [networks]="index-template-networks.json"
+            [packages]="index-template-packages.json"
+            [ports]="index-template-ports.json"
+            [processes]="index-template-processes.json"
+            [scheduled-commands]="index-template-scheduled-commands.json"
+            [system]="index-template-system.json"
+            [vulnerabilities]="index-template-vulnerabilities.json"
+          )
+
+          for ecs_module in ${relevant_modules[@]}; do
+            target_file=${module_to_file[$ecs_module]}
+            if [[ -z "$target_file" ]]; then
+              echo "No corresponding file for module $ecs_module"
+              continue
+            fi
+
+            mkdir -p wazuh-indexer-plugins/plugins/setup/src/main/resources/
+            cp ecs/$ecs_module/mappings/v8.11.0/generated/elasticsearch/legacy/template.json wazuh-indexer-plugins/plugins/setup/src/main/resources/$target_file
+          done
+
+      - name: Commit and push changes
+        run: |
+          cd wazuh-indexer-plugins
+          git config --global user.email "[email protected]"
+          git config --global user.name "GitHub Actions"
+
+          branch_name="update-ecs-templates"
+
+          # Check if branch exists
+          if git ls-remote --heads origin $branch_name | grep $branch_name; then
+            git checkout $branch_name
+          else
+            git checkout -b $branch_name
+          fi
+
+          git add .
+          git commit -m "Update ECS templates for modified modules: $relevant_modules"
+          git push origin $branch_name
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "Update ECS templates for modified modules: $relevant_modules"
+          branch: update-ecs-templates
+          title: "Update ECS templates for modified modules: $relevant_modules"
+          body: "This PR updates the ECS templates for the following modules: $relevant_modules."
+          base: master

docker/ecs/images/Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.10
+
+# Update the package list and upgrade all packages
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    # Install dependencies
+    apt-get install  -y git jq && \
+    # Cleanup
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    # Clone elastic ECS repository and install required Python libraries
+    git clone https://github.com/elastic/ecs.git && \
+    pip install -r ecs/scripts/requirements.txt && \
+    # Create the directory for the ecs definitions (this will be used as a volume)
+    mkdir -p /source/ecs
+
+# Ensure the generate.sh script is in the correct location
+ADD docker/ecs/images/generator.sh /ecs/generator.sh
+
+# Define the directory as a volume to allow for external mounting
+VOLUME /source/ecs
+
+# Ensure the generate.sh script is executable
+RUN chmod +x /ecs/generator.sh
+
+# Set the working directory to the ECS repository
+WORKDIR /ecs
+
+# Define the entry point for the container to execute the generate.sh script
+ENTRYPOINT ["/bin/bash", "/ecs/generator.sh"]

docker/ecs/images/generator.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# SPDX-License-Identifier: Apache-2.0
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# Default values
+ECS_VERSION="${ECS_VERSION:-v8.11.0}"
+ECS_SOURCE=/source
+
+# Function to display usage information
+show_usage() {
+  echo "Usage: $0"
+  echo "Environment Variables:"
+  echo "  * ECS_MODULE:   Module to generate mappings for"
+  echo "  * ECS_VERSION:  (Optional) ECS version to generate mappings for (default: v8.11.0)"
+  echo "Example: docker run -e ECS_MODULE=alerts -e ECS_VERSION=v8.11.0 ecs-generator"
+}
+
+# Ensure ECS_MODULE is provided
+if [ -z "${ECS_MODULE:-}" ]; then
+  show_usage
+  exit 1
+fi
+
+# Function to remove multi-fields from the generated index template
+remove_multi_fields() {
+  local in_file="$1"
+  local out_file="$2"
+
+  jq 'del(
+    .mappings.properties.host.properties.os.properties.full.fields,
+    .mappings.properties.host.properties.os.properties.name.fields,
+    .mappings.properties.vulnerability.properties.description.fields
+  )' "$in_file" > "$out_file"
+}
+
+# Function to generate mappings
+generate_mappings() {
+  local ecs_module="$1"
+  local indexer_path="$2"
+  local ecs_version="$3"
+
+  local in_files_dir="$indexer_path/ecs/$ecs_module/fields"
+  local out_dir="$indexer_path/ecs/$ecs_module/mappings/$ecs_version"
+
+  # Ensure the output directory exists
+  mkdir -p "$out_dir"
+
+  # Generate mappings
+  python scripts/generator.py --strict --ref "$ecs_version" \
+    --include "$in_files_dir/custom/" \
+    --subset "$in_files_dir/subset.yml" \
+    --template-settings "$in_files_dir/template-settings.json" \
+    --template-settings-legacy "$in_files_dir/template-settings-legacy.json" \
+    --mapping-settings "$in_files_dir/mapping-settings.json" \
+    --out "$out_dir"
+
+  # Replace unsupported types
+  echo "Replacing unsupported types in generated mappings"
+  find "$out_dir" -type f -exec sed -i 's/constant_keyword/keyword/g' {} \;
+  find "$out_dir" -type f -exec sed -i 's/flattened/flat_object/g' {} \;
+  find "$out_dir" -type f -exec sed -i 's/scaled_float/float/g' {} \;
+  find "$out_dir" -type f -exec sed -i '/scaling_factor/d' {} \;
+
+  local in_file="$out_dir/generated/elasticsearch/legacy/template.json"
+  local out_file="$out_dir/generated/elasticsearch/legacy/template-tmp.json"
+
+  # Delete the "tags" field from the index template
+  echo "Deleting the \"tags\" field from the index template"
+  jq 'del(.mappings.properties.tags)' "$in_file" > "$out_file"
+  mv "$out_file" "$in_file"
+
+  # Remove multi-fields from the generated index template
+  echo "Removing multi-fields from the index template"
+  remove_multi_fields "$in_file" "$out_file"
+  mv "$out_file" "$in_file"
+
+  # Transform legacy index template for OpenSearch compatibility
+  jq '{
+    "index_patterns": .index_patterns,
+    "priority": .order,
+    "template": {
+      "settings": .settings,
+      "mappings": .mappings
+    }
+  }' "$in_file" > "$out_dir/generated/elasticsearch/legacy/opensearch-template.json"
+
+  echo "Mappings saved to $out_dir"
+}
+
+# Generate mappings
+generate_mappings "$ECS_MODULE" "$ECS_SOURCE" "$ECS_VERSION"

docker/ecs/mapping-generator.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Run the ECS generator tool container.
+# Requirements:
+#   - Docker
+#   - Docker Compose
+
+set -e
+
+# The container is built only if needed, the tool can be executed several times
+# for different modules in the same build since the script runs as entrypoint
+
+
+
+# ====
+# Checks that the script is run from the intended location
+# ====
+function navigate_to_project_root() {
+    local repo_root_marker
+    local script_path
+    repo_root_marker=".github"
+    script_path=$(dirname "$(realpath "$0")")
+
+    while [[ "$script_path" != "/" ]] && [[ ! -d "$script_path/$repo_root_marker" ]]; do
+        script_path=$(dirname "$script_path")
+    done
+
+    if [[ "$script_path" == "/" ]]; then
+        echo "Error: Unable to find the repository root."
+        exit 1
+    fi
+
+    cd "$script_path"
+}
+
+# ====
+# Displays usage information
+# ====
+function usage() {
+    echo "Usage: $0 {run|down|stop} <ECS_MODULE> [REPO_PATH]"
+    exit 1
+}
+
+function main() {
+    local compose_filename="docker/ecs/mapping-generator.yml"
+    local compose_command
+    local module
+    local repo_path
+
+    navigate_to_project_root
+
+    compose_command="docker compose -f $compose_filename"
+
+    case $1 in
+        run)
+            if [[ "$#" -lt 2 || "$#" -gt 3 ]]; then
+                usage
+            fi
+            module="$2"
+            repo_path="${3:-$(pwd)}"
+
+            # Start the container with the required env variables
+            ECS_MODULE="$module" REPO_PATH="$repo_path" $compose_command up
+            # The containers are stopped after each execution
+            $compose_command stop
+            ;;
+        down)
+            $compose_command down
+            ;;
+        stop)
+            $compose_command stop
+            ;;
+        *)
+            usage
+            ;;
+    esac
+}
+
+main "$@"

docker/ecs/mapping-generator.yml

@@ -0,0 +1,11 @@
+services:
+  ecs-mapping-generator:
+    image: wazuh-ecs-generator
+    container_name: wazuh-ecs-generator
+    build:
+      context: ./../..
+      dockerfile: ${REPO_PATH:-.}/docker/ecs/images/Dockerfile
+    volumes:
+      - ${REPO_PATH:-.}/ecs:/source/ecs
+    environment:
+      - ECS_MODULE=${ECS_MODULE:-default_module}

ecs/alerts/fields/mapping-settings.json

@@ -1,4 +1,4 @@
 {
     "dynamic": "strict",
     "date_detection": false
-}
+}