diff --git a/.github/workflows/generate-ecs-mappings.yml b/.github/workflows/generate-ecs-mappings.yml new file mode 100644 index 0000000000000..5452872bd79b3 --- /dev/null +++ b/.github/workflows/generate-ecs-mappings.yml @@ -0,0 +1,154 @@ +name: ECS Generator + +on: + push: + paths: + - 'ecs/**' + +jobs: + run-ecs-generator: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up Docker Compose + run: sudo apt-get install docker-compose + + - name: Extract ECS Modules and Run ECS Generator + id: run-ecs-generator + run: | + # Fetch base branch + git fetch origin +refs/heads/master:refs/remotes/origin/master + + # Extract the ECS module names from the modified files + modified_files=$(git diff --name-only origin/master) + updated_modules=() + for file in $modified_files; do + if [[ $file == ecs/* ]]; then + ecs_module=$(echo $file | cut -d'/' -f2) + if [[ ! " ${updated_modules[*]} " =~ " ${ecs_module} " ]]; then + updated_modules+=("$ecs_module") + fi + fi + done + + # Filter out modules that do not have corresponding JSON files + declare -A module_to_file=( + [agent]="index-template-agent.json" + [alerts]="index-template-alerts.json" + [commands]="index-template-commands.json" + [hardware]="index-template-hardware.json" + [hotfixes]="index-template-hotfixes.json" + [fim]="index-template-fim.json" + [networks]="index-template-networks.json" + [packages]="index-template-packages.json" + [ports]="index-template-ports.json" + [processes]="index-template-processes.json" + [scheduled-commands]="index-template-scheduled-commands.json" + [system]="index-template-system.json" + [vulnerabilities]="index-template-vulnerabilities.json" + ) + + relevant_modules=() + for ecs_module in "${updated_modules[@]}"; do + if [[ -n "${module_to_file[$ecs_module]}" ]]; then + relevant_modules+=("$ecs_module") + fi + done + + if [[ ${#relevant_modules[@]} -gt 0 ]]; then + export REPO_PATH=$(pwd) + for ecs_module in "${relevant_modules[@]}"; do + # Run the ECS generator script for each relevant module + bash docker/ecs/mapping-generator.sh run "$ecs_module" + echo "Processed ECS module: $ecs_module" + done + echo "relevant_modules=${relevant_modules[*]}" >> $GITHUB_ENV + else + echo "No relevant modifications detected in ecs/ directory." + exit 0 + fi + + - name: Tear down ECS Generator + if: always() + run: bash docker/ecs/mapping-generator.sh down + + - name: Upload artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: ecs-template + path: ecs/**/mappings/v8.11.0/generated/elasticsearch/legacy/template.json + + - name: Checkout target repository + uses: actions/checkout@v4 + with: + repository: wazuh/wazuh-indexer-plugins + token: ${{ secrets.GITHUB_TOKEN }} + path: wazuh-indexer-plugins + + - name: Copy generated files to target repository + run: | + # Map ECS modules to target JSON filenames + declare -A module_to_file=( + [agent]="index-template-agent.json" + [alerts]="index-template-alerts.json" + [commands]="index-template-commands.json" + [hardware]="index-template-hardware.json" + [hotfixes]="index-template-hotfixes.json" + [fim]="index-template-fim.json" + [networks]="index-template-networks.json" + [packages]="index-template-packages.json" + [ports]="index-template-ports.json" + [processes]="index-template-processes.json" + [scheduled-commands]="index-template-scheduled-commands.json" + [system]="index-template-system.json" + [vulnerabilities]="index-template-vulnerabilities.json" + ) + + for ecs_module in ${relevant_modules[@]}; do + target_file=${module_to_file[$ecs_module]} + if [[ -z "$target_file" ]]; then + echo "No corresponding file for module $ecs_module" + continue + fi + + mkdir -p wazuh-indexer-plugins/plugins/setup/src/main/resources/ + cp ecs/$ecs_module/mappings/v8.11.0/generated/elasticsearch/legacy/template.json wazuh-indexer-plugins/plugins/setup/src/main/resources/$target_file + done + + - name: Commit and push changes + run: | + cd wazuh-indexer-plugins + git config --global user.email "github-actions@github.com" + git config --global user.name "GitHub Actions" + + branch_name="update-ecs-templates" + + # Check if branch exists + if git ls-remote --heads origin $branch_name | grep $branch_name; then + git checkout $branch_name + else + git checkout -b $branch_name + fi + + git add . + git commit -m "Update ECS templates for modified modules: $relevant_modules" + git push origin $branch_name + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "Update ECS templates for modified modules: $relevant_modules" + branch: update-ecs-templates + title: "Update ECS templates for modified modules: $relevant_modules" + body: "This PR updates the ECS templates for the following modules: $relevant_modules." + base: master diff --git a/docker/ecs/images/Dockerfile b/docker/ecs/images/Dockerfile new file mode 100644 index 0000000000000..0153810699146 --- /dev/null +++ b/docker/ecs/images/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.10 + +# Update the package list and upgrade all packages +RUN apt-get update && \ + apt-get upgrade -y && \ + # Install dependencies + apt-get install -y git jq && \ + # Cleanup + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + # Clone elastic ECS repository and install required Python libraries + git clone https://github.com/elastic/ecs.git && \ + pip install -r ecs/scripts/requirements.txt && \ + # Create the directory for the ecs definitions (this will be used as a volume) + mkdir -p /source/ecs + +# Ensure the generate.sh script is in the correct location +ADD docker/ecs/images/generator.sh /ecs/generator.sh + +# Define the directory as a volume to allow for external mounting +VOLUME /source/ecs + +# Ensure the generate.sh script is executable +RUN chmod +x /ecs/generator.sh + +# Set the working directory to the ECS repository +WORKDIR /ecs + +# Define the entry point for the container to execute the generate.sh script +ENTRYPOINT ["/bin/bash", "/ecs/generator.sh"] diff --git a/docker/ecs/images/generator.sh b/docker/ecs/images/generator.sh new file mode 100755 index 0000000000000..65a59e47367e3 --- /dev/null +++ b/docker/ecs/images/generator.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +set -euo pipefail + +# SPDX-License-Identifier: Apache-2.0 +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# Default values +ECS_VERSION="${ECS_VERSION:-v8.11.0}" +ECS_SOURCE=/source + +# Function to display usage information +show_usage() { + echo "Usage: $0" + echo "Environment Variables:" + echo " * ECS_MODULE: Module to generate mappings for" + echo " * ECS_VERSION: (Optional) ECS version to generate mappings for (default: v8.11.0)" + echo "Example: docker run -e ECS_MODULE=alerts -e ECS_VERSION=v8.11.0 ecs-generator" +} + +# Ensure ECS_MODULE is provided +if [ -z "${ECS_MODULE:-}" ]; then + show_usage + exit 1 +fi + +# Function to remove multi-fields from the generated index template +remove_multi_fields() { + local in_file="$1" + local out_file="$2" + + jq 'del( + .mappings.properties.host.properties.os.properties.full.fields, + .mappings.properties.host.properties.os.properties.name.fields, + .mappings.properties.vulnerability.properties.description.fields + )' "$in_file" > "$out_file" +} + +# Function to generate mappings +generate_mappings() { + local ecs_module="$1" + local indexer_path="$2" + local ecs_version="$3" + + local in_files_dir="$indexer_path/ecs/$ecs_module/fields" + local out_dir="$indexer_path/ecs/$ecs_module/mappings/$ecs_version" + + # Ensure the output directory exists + mkdir -p "$out_dir" + + # Generate mappings + python scripts/generator.py --strict --ref "$ecs_version" \ + --include "$in_files_dir/custom/" \ + --subset "$in_files_dir/subset.yml" \ + --template-settings "$in_files_dir/template-settings.json" \ + --template-settings-legacy "$in_files_dir/template-settings-legacy.json" \ + --mapping-settings "$in_files_dir/mapping-settings.json" \ + --out "$out_dir" + + # Replace unsupported types + echo "Replacing unsupported types in generated mappings" + find "$out_dir" -type f -exec sed -i 's/constant_keyword/keyword/g' {} \; + find "$out_dir" -type f -exec sed -i 's/flattened/flat_object/g' {} \; + find "$out_dir" -type f -exec sed -i 's/scaled_float/float/g' {} \; + find "$out_dir" -type f -exec sed -i '/scaling_factor/d' {} \; + + local in_file="$out_dir/generated/elasticsearch/legacy/template.json" + local out_file="$out_dir/generated/elasticsearch/legacy/template-tmp.json" + + # Delete the "tags" field from the index template + echo "Deleting the \"tags\" field from the index template" + jq 'del(.mappings.properties.tags)' "$in_file" > "$out_file" + mv "$out_file" "$in_file" + + # Remove multi-fields from the generated index template + echo "Removing multi-fields from the index template" + remove_multi_fields "$in_file" "$out_file" + mv "$out_file" "$in_file" + + # Transform legacy index template for OpenSearch compatibility + jq '{ + "index_patterns": .index_patterns, + "priority": .order, + "template": { + "settings": .settings, + "mappings": .mappings + } + }' "$in_file" > "$out_dir/generated/elasticsearch/legacy/opensearch-template.json" + + echo "Mappings saved to $out_dir" +} + +# Generate mappings +generate_mappings "$ECS_MODULE" "$ECS_SOURCE" "$ECS_VERSION" diff --git a/docker/ecs/mapping-generator.sh b/docker/ecs/mapping-generator.sh new file mode 100644 index 0000000000000..c2cb0e6bbf51d --- /dev/null +++ b/docker/ecs/mapping-generator.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Run the ECS generator tool container. +# Requirements: +# - Docker +# - Docker Compose + +set -e + +# The container is built only if needed, the tool can be executed several times +# for different modules in the same build since the script runs as entrypoint + + + +# ==== +# Checks that the script is run from the intended location +# ==== +function navigate_to_project_root() { + local repo_root_marker + local script_path + repo_root_marker=".github" + script_path=$(dirname "$(realpath "$0")") + + while [[ "$script_path" != "/" ]] && [[ ! -d "$script_path/$repo_root_marker" ]]; do + script_path=$(dirname "$script_path") + done + + if [[ "$script_path" == "/" ]]; then + echo "Error: Unable to find the repository root." + exit 1 + fi + + cd "$script_path" +} + +# ==== +# Displays usage information +# ==== +function usage() { + echo "Usage: $0 {run|down|stop} [REPO_PATH]" + exit 1 +} + +function main() { + local compose_filename="docker/ecs/mapping-generator.yml" + local compose_command + local module + local repo_path + + navigate_to_project_root + + compose_command="docker compose -f $compose_filename" + + case $1 in + run) + if [[ "$#" -lt 2 || "$#" -gt 3 ]]; then + usage + fi + module="$2" + repo_path="${3:-$(pwd)}" + + # Start the container with the required env variables + ECS_MODULE="$module" REPO_PATH="$repo_path" $compose_command up + # The containers are stopped after each execution + $compose_command stop + ;; + down) + $compose_command down + ;; + stop) + $compose_command stop + ;; + *) + usage + ;; + esac +} + +main "$@" diff --git a/docker/ecs/mapping-generator.yml b/docker/ecs/mapping-generator.yml new file mode 100644 index 0000000000000..e0ae24f468e98 --- /dev/null +++ b/docker/ecs/mapping-generator.yml @@ -0,0 +1,11 @@ +services: + ecs-mapping-generator: + image: wazuh-ecs-generator + container_name: wazuh-ecs-generator + build: + context: ./../.. + dockerfile: ${REPO_PATH:-.}/docker/ecs/images/Dockerfile + volumes: + - ${REPO_PATH:-.}/ecs:/source/ecs + environment: + - ECS_MODULE=${ECS_MODULE:-default_module} diff --git a/ecs/alerts/fields/mapping-settings.json b/ecs/alerts/fields/mapping-settings.json index 0ad2b48fcc1be..43be8693577e8 100644 --- a/ecs/alerts/fields/mapping-settings.json +++ b/ecs/alerts/fields/mapping-settings.json @@ -1,4 +1,4 @@ { "dynamic": "strict", "date_detection": false -} \ No newline at end of file +}