diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9afe6c40034fe..7a941964cbec6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,6 +35,10 @@ on: description: "ID used to identify the workflow uniquely." type: string required: false + plugins_reference: + description: "Branch, commit or tag for the plugins repository" + type: string + default: "master" workflow_call: inputs: revision: @@ -64,6 +68,10 @@ on: id: type: string required: false + plugins_reference: + description: "Branch, commit or tag for the plugins repository" + type: string + default: "master" secrets: CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: required: true @@ -95,68 +103,117 @@ jobs: - id: setup run: | matrix=$(jq -cn \ - --argjson distribution '${{ inputs.distribution }}' \ - --argjson architecture '${{ inputs.architecture }}' \ + --argjson distribution '${{ github.event.inputs.distribution }}' \ + --argjson architecture '${{ github.event.inputs.architecture }}' \ '{distribution: $distribution, architecture: $architecture}' ) echo "matrix=$matrix" >> $GITHUB_OUTPUT + build-plugins: + if: ${{ github.event.inputs.plugins_reference != '' }} + strategy: + fail-fast: false + matrix: + plugins: ["wazuh-indexer-setup"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: wazuh/wazuh-indexer-plugins + ref: ${{ github.event.inputs.plugins_reference }} + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Get version + id: version + run: echo "version=$(> "$GITHUB_OUTPUT" + + - name: Build with Gradle + working-directory: .//plugins/${{ matrix.plugins }} + run: | + ./gradlew build -Dversion=${{ steps.version.outputs.version }} -Drevision=${{ github.event.inputs.revision }} + ls -lR build/distributions/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.plugins }} + path: "./plugins/${{ matrix.plugins }}/build/distributions/${{ matrix.plugins }}-${{ steps.version.outputs.version }}.${{ github.event.inputs.revision }}.zip" + if-no-files-found: error + build: - needs: [matrix] + needs: [matrix, build-plugins] runs-on: ubuntu-latest strategy: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} steps: - uses: actions/checkout@v4 + + # Download plugins + - uses: actions/download-artifact@v4 + if: ${{ github.event.inputs.plugins_reference != '' }} + with: + path: ./artifacts/plugins + merge-multiple: true + + - name: Display structure of downloaded files + if: ${{ github.event.inputs.plugins_reference != '' }} + run: ls -lR ./artifacts/plugins + - uses: actions/setup-java@v4 with: distribution: temurin - java-version: 11 + java-version: 21 - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 + uses: gradle/actions/setup-gradle@v4 - name: Provision if: ${{ matrix.distribution == 'deb' }} run: | - sudo bash packaging_scripts/provision.sh + sudo bash build-scripts/provision.sh - name: Run `baptizer.sh` (min) run: | - name=$(bash packaging_scripts/baptizer.sh -m \ + name=$(bash build-scripts/baptizer.sh -m \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ - -r ${{ inputs.revision }} \ - ${{ inputs.is_stage && '-x' || '' }} \ + -r ${{ github.event.inputs.revision }} \ + ${{ github.event.inputs.is_stage && '-x' || '' }} \ ) echo "name=$name" >> $GITHUB_OUTPUT id: min_package - name: Run `baptizer.sh` run: | - name=$(bash packaging_scripts/baptizer.sh \ + name=$(bash build-scripts/baptizer.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ - -r ${{ inputs.revision }} \ - ${{ inputs.is_stage && '-x' || '' }} \ + -r ${{ github.event.inputs.revision }} \ + ${{ github.event.inputs.is_stage && '-x' || '' }} \ ) echo "name=$name" >> $GITHUB_OUTPUT id: package - name: Run `build.sh` run: | - bash packaging_scripts/build.sh \ + bash build-scripts/build.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -n ${{ steps.min_package.outputs.name }} - name: Run `assemble.sh` run: | - bash packaging_scripts/assemble.sh \ + bash build-scripts/assemble.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ - -r ${{ inputs.revision }} + -r ${{ github.event.inputs.revision }} - name: Test RPM package if: ${{ matrix.distribution == 'rpm' }} @@ -180,7 +237,7 @@ jobs: if-no-files-found: error - name: Set up AWS CLI - if: ${{ inputs.upload }} + if: ${{ github.event.inputs.upload }} uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} @@ -188,7 +245,7 @@ jobs: aws-region: ${{ secrets.CI_AWS_REGION }} - name: Upload package to S3 - if: ${{ inputs.upload }} + if: ${{ github.event.inputs.upload }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" @@ -197,7 +254,7 @@ jobs: echo "S3 URI: ${s3uri}" - name: Upload checksum to S3 - if: ${{ inputs.upload && inputs.checksum }} + if: ${{ github.event.inputs.upload && github.event.inputs.checksum }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}.sha512" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" diff --git a/.github/workflows/version_check.yml b/.github/workflows/version_check.yml new file mode 100644 index 0000000000000..060b2be3332dc --- /dev/null +++ b/.github/workflows/version_check.yml @@ -0,0 +1,15 @@ +name: Version check + +on: + push: + paths: + - "VERSION" + +jobs: + check-version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check version + run: bash build-scripts/check-version.sh diff --git a/build-scripts/README.md b/build-scripts/README.md new file mode 100644 index 0000000000000..e21c1df556659 --- /dev/null +++ b/build-scripts/README.md @@ -0,0 +1,271 @@ +# How to Build `wazuh-indexer` DEB and RPM Packages + +> [!CAUTION] +> +> Be aware that there might be some problems while following the steps in this guide due to outdated information. +> This document is pending a review. Let us know if you find any issues. + +The packages' generation process consists on 2 steps: + +- **Build**: compiles the Java application and bundles it into a package. +- **Assembly**: uses the package from the previous step and inflates it with plugins and + configuration files, ready for production deployment. + +We usually generate the packages using GitHub Actions, however, the process is designed to +be independent enough for maximum portability. GitHub Actions provides infrastructure, while +the building process is self-contained in the application code. + +Each section includes instructions to generate packages locally, using Act or Docker. + +- [Install Act](https://github.com/nektos/act) + +The names of the packages are managed by the `baptizer.sh` script. + +## Building the plugins + +Follow the [DEVELOPER_GUIDE.md](https://github.com/wazuh/wazuh-indexer-plugins/blob/master/DEVELOPER_GUIDE.md) instructions to build the plugins. The build scripts expect the plugins in the Maven local repository or under the `artifacts/plugins` folder. + +1. Build the plugins. +2. Publish the plugins to the local Maven repository: run `./gradlew publishToMavenLocal`. + - Alternatively, copy the generated zip files to the `artifacts/plugins` folder. +3. Build and Assemble the `wazuh-indexer` package. + +## Build and Assemble in Act + +Use Act to run the `build.yml` workflow locally. The `act.input.env` file contains the inputs +for the workflow. As the workflow clones the `wazuh-indexer-plugins` repository, the `GITHUB_TOKEN` +is required. You can use the `gh` CLI to authenticate, as seen in the example below. + +```console +act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts --input-file build-scripts/act.input.env -s GITHUB_TOKEN="$(gh auth token)" +``` + +## Build + +For local package generation, use the `build.sh` script. Take a look at the `build.yml` +workflow file for an example of usage. + +```bash +bash build-scripts/build.sh -a x64 -d tar -n $(bash build-scripts/baptizer.sh -a x64 -d tar -m) +``` + +#### Act (GitHub Workflow locally) + +```console +act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts + +[Build slim packages/build] 🏁 Job succeeded +``` + +#### Running in Docker + +Using the [Docker environment](../docker): + +```console +docker exec -it wi-build_$( By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. + > The script takes care of creating the required folder structure, copying also the min package + > and the Makefile. + + Current folder loadout at this stage: + + ``` + artifacts/ + |-- dist + | |-- wazuh-indexer-min_4.9.1_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + `-- wazuh-indexer-min_4.9.1_amd64.deb + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. + `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + +2. Install the plugins using the `opensearch-plugin` CLI tool. +3. Set up configuration files. + + > Included in `min-package`. Default files are overwritten. + +4. Bundle a DEB file with `debmake` and the `Makefile`. + + > `debmake` and other dependencies can be installed using the `provision.sh` script. + > The script is invoked by the GitHub Workflow. + + Current folder loadout at this stage: + + ``` + artifacts/ + |-- artifact_name.txt + |-- dist + | |-- wazuh-indexer-min_4.9.1_amd64.deb + | `-- wazuh-indexer_4.9.1_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + |-- wazuh-indexer-min_4.9.1_amd64.deb + `-- debian/ + | -- control + | -- copyright + | -- rules + | -- preinst + | -- prerm + | -- postinst + ``` + +#### Running in Docker + +Pre-requisites: + +- Current directory: `wazuh-indexer/` +- Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. +- Using the [Docker environment](../docker): + +```console +docker exec -it wi-assemble_$( By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`. The script takes care of creating the required folder structure, copying also the min package and the SPEC file. + + Current folder loadout at this stage: + + ``` + /rpm/$ARCH + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. + `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + +2. Install the plugins using the `opensearch-plugin` CLI tool. +3. Set up configuration files. + + > Included in `min-package`. Default files are overwritten. + +4. Bundle an RPM file with `rpmbuild` and the SPEC file `wazuh-indexer.rpm.spec`. + + > `rpmbuild` is part of the `rpm` OS package. + + > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. + + Current folder loadout at this stage: + + ``` + /rpm/$ARCH + /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` + +#### Running in Docker + +Pre-requisites: + +- Current directory: `wazuh-indexer/` +- Existing rpm package in `wazuh-indexer/artifacts/dist/rpm`, as a result of the _Build_ stage. +- Using the [Docker environment](../docker): + +```console +docker exec -it wi-assemble_$(> "${1}/opensearch-performance-analyzer/opensearch_security.policy" + } >>"${1}/opensearch-performance-analyzer/opensearch_security.policy" } # ==== @@ -219,13 +223,30 @@ function enable_performance_analyzer_rca() { # Install plugins # ==== function install_plugins() { - echo "Install plugins" - maven_repo_local="$HOME/maven" + echo "Installing OpenSearch plugins" + local maven_repo_local="$HOME/.m2" for plugin in "${plugins[@]}"; do - plugin_from_maven="org.opensearch.plugin:${plugin}:${VERSION}.0" + local plugin_from_maven="org.opensearch.plugin:${plugin}:${VERSION}.0" mvn -Dmaven.repo.local="${maven_repo_local}" org.apache.maven.plugins:maven-dependency-plugin:2.1:get -DrepoUrl=https://repo1.maven.org/maven2 -Dartifact="${plugin_from_maven}:zip" OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "file:${maven_repo_local}/org/opensearch/plugin/${plugin}/${VERSION}.0/${plugin}-${VERSION}.0.zip" done + + echo "Installing Wazuh plugins" + local indexer_plugin_version="${1}.${REVISION}" + for plugin_name in "${wazuh_plugins[@]}"; do + # Check if the plugin is in the local maven repository. This is usually + # case for local executions. + local plugin_path="${maven_repo_local}/repository/org/wazuh/${plugin_name}-plugin/${indexer_plugin_version}/${plugin_name}-${indexer_plugin_version}.zip" + + # Otherwise, search for the plugins in the output folder. + if [ -z "${plugin_from_maven_local}" ]; then + echo "Plugin ${plugin_name} not found in local maven repository. Searching on ./${OUTPUT}/plugins" + # Working directory at this point is: wazuh-indexer/artifacts/tmp/{rpm|deb|tar} + plugin_path="$(pwd)/../../plugins/${plugin_name}-${indexer_plugin_version}.zip" + fi + + OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "file:${plugin_path}" + done } # ==== @@ -255,7 +276,7 @@ function assemble_tar() { version=$(cat VERSION) # Install plugins - install_plugins + install_plugins "${version}" fix_log_rotation ${PATH_CONF} # Swap configuration files add_configuration_files @@ -295,7 +316,7 @@ function assemble_rpm() { version=$(cat ./usr/share/wazuh-indexer/VERSION) # Install plugins - install_plugins + install_plugins "${version}" fix_log_rotation ${PATH_CONF} enable_performance_analyzer_rca ${src_path} # Swap configuration files @@ -349,7 +370,7 @@ function assemble_deb() { version=$(cat ./usr/share/wazuh-indexer/VERSION) # Install plugins - install_plugins + install_plugins "${version}" fix_log_rotation ${PATH_CONF} enable_performance_analyzer_rca ${src_path} # Swap configuration files @@ -389,7 +410,7 @@ function main() { echo "Assembling wazuh-indexer for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" - VERSION=$(bash packaging_scripts/upstream_version.sh) + VERSION=$(bash build-scripts/upstream-version.sh) ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min.*$SUFFIX.*\.$EXT") ARTIFACT_PACKAGE_NAME=${ARTIFACT_BUILD_NAME/-min/} @@ -411,7 +432,7 @@ function main() { esac # Create checksum - sha512sum "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME" > "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME".sha512 + sha512sum "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME" >"${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME".sha512 } main "${@}" diff --git a/packaging_scripts/baptizer.sh b/build-scripts/baptizer.sh similarity index 100% rename from packaging_scripts/baptizer.sh rename to build-scripts/baptizer.sh diff --git a/packaging_scripts/build.sh b/build-scripts/build.sh old mode 100755 new mode 100644 similarity index 100% rename from packaging_scripts/build.sh rename to build-scripts/build.sh diff --git a/build-scripts/check-version.sh b/build-scripts/check-version.sh new file mode 100644 index 0000000000000..dd9781c0707f3 --- /dev/null +++ b/build-scripts/check-version.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Check if VERSION file has exactly one line and that line is non-empty +if [ "$(wc -l By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. - > The script takes care of creating the required folder structure, copying also the min package - > and the Makefile. - - Current folder loadout at this stage: - - ``` - artifacts/ - |-- dist - | |-- wazuh-indexer-min_4.9.1_amd64.deb - `-- tmp - `-- deb - |-- Makefile - |-- data.tar.gz - |-- debmake_install.sh - |-- etc - |-- usr - |-- var - `-- wazuh-indexer-min_4.9.1_amd64.deb - ``` - - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. - `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. - -2. Install the plugins using the `opensearch-plugin` CLI tool. -3. Set up configuration files. - - > Included in `min-package`. Default files are overwritten. - -4. Bundle a DEB file with `debmake` and the `Makefile`. - - > `debmake` and other dependencies can be installed using the `provision.sh` script. - > The script is invoked by the GitHub Workflow. - - Current folder loadout at this stage: - - ``` - artifacts/ - |-- artifact_name.txt - |-- dist - | |-- wazuh-indexer-min_4.9.1_amd64.deb - | `-- wazuh-indexer_4.9.1_amd64.deb - `-- tmp - `-- deb - |-- Makefile - |-- data.tar.gz - |-- debmake_install.sh - |-- etc - |-- usr - |-- var - |-- wazuh-indexer-min_4.9.1_amd64.deb - `-- debian/ - | -- control - | -- copyright - | -- rules - | -- preinst - | -- prerm - | -- postinst - ``` - -#### Running in Act - -```console -act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 - -[Build slim packages/build] 🏁 Job succeeded -``` - -#### Running in Docker - -Pre-requisites: - -- Current directory: `wazuh-indexer/` -- Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. -- Using the [Docker environment](../docker): - -```console -docker exec -it wi-assemble_$( By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`.The script takes care of creating the required folder structure, copying also the min package and the SPEC file. - - Current folder loadout at this stage: - - ``` - /rpm/$ARCH - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` - - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. - `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. - -2. Install the plugins using the `opensearch-plugin` CLI tool. -3. Set up configuration files. - - > Included in `min-package`. Default files are overwritten. - -4. Bundle an RPM file with `rpmbuild` and the SPEC file `wazuh-indexer.rpm.spec`. - - > `rpmbuild` is part of the `rpm` OS package. - - > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. - - Current folder loadout at this stage: - - ``` - /rpm/$ARCH - /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` - -#### Running in Act - -```console -act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:rpm --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 - -[Build slim packages/build] 🏁 Job succeeded -``` - -#### Running in Docker - -Pre-requisites: - -- Current directory: `wazuh-indexer/` -- Existing rpm package in `wazuh-indexer/artifacts/dist/rpm`, as a result of the _Build_ stage. -- Using the [Docker environment](../docker): - -```console -docker exec -it wi-assemble_$(