From 1229b530f27eeef9a99b3698e2c99d25f0507354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Tue, 26 Mar 2024 13:25:03 +0100 Subject: [PATCH 1/2] Update wazuh-states-vulnerabilities index mapping (#191) * Update wazuh-states-vulnerabilities index mapping * Extend ECS Vulnerability fields --- ecs/generate.sh | 2 +- .../event-generator/event_generator.py | 8 +++++--- .../fields/custom/vulnerability.yml | 19 +++++++++++++++++++ .../fields/custom/wazuh.yml | 2 +- ecs/vulnerability-detector/fields/subset.yml | 1 - 5 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 ecs/vulnerability-detector/fields/custom/vulnerability.yml diff --git a/ecs/generate.sh b/ecs/generate.sh index c819c72a8ca76..4b747c0c9a0cb 100755 --- a/ecs/generate.sh +++ b/ecs/generate.sh @@ -23,7 +23,7 @@ generate_mappings() { # Generate mappings python scripts/generator.py --strict --ref "$ECS_VERSION" \ - --include "$IN_FILES_DIR/custom/wazuh.yml" \ + --include "$IN_FILES_DIR/custom/" \ --subset "$IN_FILES_DIR/subset.yml" \ --template-settings "$IN_FILES_DIR/template-settings.json" \ --template-settings-legacy "$IN_FILES_DIR/template-settings-legacy.json" \ diff --git a/ecs/vulnerability-detector/event-generator/event_generator.py b/ecs/vulnerability-detector/event-generator/event_generator.py index 0ed75ec9600a1..73f8e8be71205 100755 --- a/ecs/vulnerability-detector/event-generator/event_generator.py +++ b/ecs/vulnerability-detector/event-generator/event_generator.py @@ -163,7 +163,9 @@ def generate_random_vulnerability(): 'temporal': round(random.uniform(0, 10), 1), 'version': round(random.uniform(0, 10), 1) }, - 'severity': random.choice(['Low', 'Medium', 'High', 'Critical']) + 'severity': random.choice(['Low', 'Medium', 'High', 'Critical']), + 'detected_at': generate_random_date(), + 'published_at': generate_random_date(), } return vulnerability @@ -179,7 +181,7 @@ def generate_random_wazuh(): }, 'schema': { 'version': '1.7.0' - } + }, } return wazuh @@ -188,7 +190,7 @@ def generate_random_data(number): data = [] for _ in range(number): event_data = { - '@timestamp': generate_random_date(), + # '@timestamp': generate_random_date(), 'agent': generate_random_agent(), # 'ecs': {'version': '1.7.0'}, # 'event': generate_random_event(), diff --git a/ecs/vulnerability-detector/fields/custom/vulnerability.yml b/ecs/vulnerability-detector/fields/custom/vulnerability.yml new file mode 100644 index 0000000000000..51be3282cc161 --- /dev/null +++ b/ecs/vulnerability-detector/fields/custom/vulnerability.yml @@ -0,0 +1,19 @@ +- name: vulnerability + title: Vulnerability + group: 2 + short: Fields to describe the vulnerability relevant to an event. + description: > + The vulnerability fields describe information about a vulnerability that is + relevant to an event. + type: group + fields: + - name: detected_at + type: date + level: custom + description: > + Vulnerability's detection date. + - name: published_at + type: date + level: custom + description: > + Vulnerability's publication date. \ No newline at end of file diff --git a/ecs/vulnerability-detector/fields/custom/wazuh.yml b/ecs/vulnerability-detector/fields/custom/wazuh.yml index f7bcf4f897c07..235a746758812 100644 --- a/ecs/vulnerability-detector/fields/custom/wazuh.yml +++ b/ecs/vulnerability-detector/fields/custom/wazuh.yml @@ -23,4 +23,4 @@ type: keyword level: custom description: > - Wazuh schema version. \ No newline at end of file + Wazuh schema version. diff --git a/ecs/vulnerability-detector/fields/subset.yml b/ecs/vulnerability-detector/fields/subset.yml index f5b0d60757794..951f7e492d1c2 100644 --- a/ecs/vulnerability-detector/fields/subset.yml +++ b/ecs/vulnerability-detector/fields/subset.yml @@ -3,7 +3,6 @@ name: vulnerability_detector fields: base: fields: - "@timestamp": {} tags: [] message: "" agent: From 518f36ea776c9e5ed61fe1ba31dc63c6c4297f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 27 Mar 2024 17:03:48 +0100 Subject: [PATCH 2/2] Add pipeline to generate release packages (#193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add script to get the version of OpenSearch * Set revision to 0 by default. - Reduce inputs for scripts. - Add script to generate packages' naming convention. - Make scripts self-aware of the OpenSearch version. * Fix assemble * Smoke test new pipeline to build packages * Fix syntax errors * Update build.yml Signed-off-by: Álex Ruiz * Add workflow to build packages on push * Run actionlint * Fix jq argjson * Fix set matrix output ? * Try new approach using a single workflow * Fix GITHUB_OUTPUT * Fix baptizer invocation * Add testing and upload to new approach * Fix hard coded revision number on RPM assembly * New attempt * Skip upload unless specified * Install plugins on RPM * Promote new approach Removes previous workflows to generate packages * Fix workflow name * Attempt to fix release package naming * Fix build.sh invocation from workflow * Use min package name in workflow * Use min package name for release naming convention in workflow * Attemtp to fix regex * Upgrade to aws-actions/configure-aws-credentials@v4 Clean up * Apply latest requirements Add workflow with single matrix for QA use. Rename inputs. Add checksum input. * Add checksum generation and upload * Use choice as input types for system and architecture * Invoke build single packages with upload option * Add documentation and clean up * Rename scripts folder to packaging_scripts --------- Signed-off-by: Álex Ruiz --- .github/workflows/build.yml | 221 +++++++++++---- .github/workflows/build_on_push.yml | 14 + .github/workflows/build_single.yml | 46 +++ .github/workflows/r_assemble.yml | 61 ---- .github/workflows/r_build.yml | 60 ---- .github/workflows/r_commit_sha.yml | 22 -- .github/workflows/r_test.yml | 88 ------ .github/workflows/r_upload.yml | 41 --- .github/workflows/r_version.yml | 22 -- .../packages/src/rpm/wazuh-indexer.rpm.spec | 2 +- packaging_scripts/README.md | 264 ++++++++++++++++++ {scripts => packaging_scripts}/assemble.sh | 39 ++- packaging_scripts/baptizer.sh | 155 ++++++++++ packaging_scripts/build.sh | 209 ++++++++++++++ {scripts => packaging_scripts}/provision.sh | 0 packaging_scripts/upstream_version.sh | 6 + scripts/README.md | 202 -------------- scripts/build.sh | 203 -------------- 18 files changed, 876 insertions(+), 779 deletions(-) create mode 100644 .github/workflows/build_on_push.yml create mode 100644 .github/workflows/build_single.yml delete mode 100644 .github/workflows/r_assemble.yml delete mode 100644 .github/workflows/r_build.yml delete mode 100644 .github/workflows/r_commit_sha.yml delete mode 100644 .github/workflows/r_test.yml delete mode 100644 .github/workflows/r_upload.yml delete mode 100644 .github/workflows/r_version.yml create mode 100644 packaging_scripts/README.md rename {scripts => packaging_scripts}/assemble.sh (91%) create mode 100644 packaging_scripts/baptizer.sh create mode 100755 packaging_scripts/build.sh rename {scripts => packaging_scripts}/provision.sh (100%) create mode 100644 packaging_scripts/upstream_version.sh delete mode 100644 scripts/README.md delete mode 100755 scripts/build.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f68708a1197fb..456ad172bd5f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,23 +1,68 @@ -name: Build packages +name: Build packages (on demand) # This workflow runs when any of the following occur: # - Run manually +# - Invoked from another workflow on: - push: - # Sequence of patterns matched against refs/heads - branches: - - "ci/*" workflow_dispatch: inputs: revision: description: "Revision" type: string - required: true - default: "1" + default: "0" upload: description: "Upload ?" - type: bool + type: boolean + default: false + is_stage: + description: "Is stage ?" + type: boolean + default: false + distribution: + description: '[ "tar", "rpm", "deb" ]' + type: string + default: '[ "rpm", "deb" ]' + architecture: + description: '[ "x64", "arm64" ]' + type: string + default: '[ "x64" ]' + checksum: + description: "Checksum ?" + type: boolean default: false + workflow_call: + inputs: + revision: + description: "Revision" + type: string + default: "0" + upload: + description: "Upload ?" + type: boolean + default: false + is_stage: + description: "Is stage ?" + type: boolean + default: false + distribution: + description: '[ "tar", "rpm", "deb" ]' + type: string + default: '[ "rpm", "deb" ]' + architecture: + description: '[ "x64", "arm64" ]' + type: string + default: '[ "x64" ]' + checksum: + description: "Checksum ?" + type: boolean + default: false + secrets: + CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: + required: true + description: "AWS user access key" + CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY: + required: true + description: "AWS user secret key" # ========================== # Bibliography @@ -33,57 +78,117 @@ on: # | https://docs.github.com/en/actions/learn-github-actions/expressions#example jobs: - version: - uses: ./.github/workflows/r_version.yml - - commit_sha: - uses: ./.github/workflows/r_commit_sha.yml + matrix: + name: Set up matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.setup.outputs.matrix }} + steps: + - id: setup + run: | + matrix=$(jq -cn \ + --argjson distribution '${{ inputs.distribution }}' \ + --argjson architecture '${{ inputs.architecture }}' \ + '{distribution: $distribution, architecture: $architecture}' + ) + echo "matrix=$matrix" >> $GITHUB_OUTPUT build: - needs: [version, commit_sha] - strategy: - matrix: - distribution: [tar, rpm, deb] - architecture: [x64, arm64] - uses: ./.github/workflows/r_build.yml - with: - architecture: ${{ matrix.architecture }} - distribution: ${{ matrix.distribution }} - revision: ${{ github.event_name == 'push' && '1' || inputs.revision }} - name: wazuh-indexer-min_${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '1' || inputs.revision }}-${{ matrix.architecture }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.distribution }} - - assemble: - needs: [version, commit_sha, build] - strategy: - matrix: - distribution: [tar, rpm, deb] - architecture: [x64, arm64] - exclude: - # skip arm64 until we have arm runners - - architecture: arm64 - - distribution: tar - - uses: ./.github/workflows/r_assemble.yml - with: - architecture: ${{ matrix.architecture }} - distribution: ${{ matrix.distribution }} - min: wazuh-indexer-min_${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '1' || inputs.revision }}-${{ matrix.architecture }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.distribution }} - - test: - needs: [version, commit_sha, assemble] + needs: [matrix] + runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - os: [{ suffix: "amd64", ext: "deb" }, { suffix: "x86_64", ext: "rpm" }] - uses: ./.github/workflows/r_test.yml - with: - package: wazuh-indexer-${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '1' || inputs.revision }}_${{ matrix.os.suffix }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.os.ext }} - - upload: - needs: [version, commit_sha, test] - # Upload only on 'workflow_dispatch' event and if 'upload=true' - if: ${{ github.event_name == 'push' && inputs.upload }} - uses: ./.github/workflows/r_upload.yml - with: - package: wazuh-indexer-${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '1' || inputs.revision }}_${{ matrix.os.suffix }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.os.ext }} - secrets: inherit + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Provision + if: ${{ matrix.distribution == 'deb' }} + run: | + sudo bash packaging_scripts/provision.sh + + - name: Run `baptizer.sh` (min) + run: | + name=$(bash packaging_scripts/baptizer.sh -m \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} \ + ${{ inputs.is_stage && '-x' || '' }} \ + ) + echo "name=$name" >> $GITHUB_OUTPUT + id: min_package + + - name: Run `baptizer.sh` + run: | + name=$(bash packaging_scripts/baptizer.sh \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} \ + ${{ inputs.is_stage && '-x' || '' }} \ + ) + echo "name=$name" >> $GITHUB_OUTPUT + id: package + + - name: Run `build.sh` + run: | + bash packaging_scripts/build.sh \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -n ${{ steps.min_package.outputs.name }} + + - name: Run `assemble.sh` + run: | + bash packaging_scripts/assemble.sh \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} + + - name: Test RPM package + if: ${{ matrix.distribution == 'rpm' }} + uses: addnab/docker-run-action@v3 + with: + image: redhat/ubi9:latest + options: -v ${{ github.workspace }}/artifacts/dist:/artifacts/dist + run: | + yum localinstall "/artifacts/dist/${{ steps.package.outputs.name }}" -y + + - name: Test DEB package + if: ${{ matrix.distribution == 'deb' }} + run: | + sudo dpkg -i "artifacts/dist/${{ steps.package.outputs.name }}" + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.package.outputs.name }} + path: artifacts/dist/${{ steps.package.outputs.name }} + if-no-files-found: error + + - name: Set up AWS CLI + if: ${{ inputs.upload }} + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} + aws-region: us-east-1 + + - name: Upload package to S3 + if: ${{ inputs.upload }} + run: | + src="artifacts/dist/${{ steps.package.outputs.name }}" + dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" + aws s3 cp "$src" "$dest" + + - name: Upload checksum to S3 + if: ${{ inputs.upload && inputs.checksum }} + run: | + src="artifacts/dist/${{ steps.package.outputs.name }}.sha512" + dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" + aws s3 cp "$src" "$dest" diff --git a/.github/workflows/build_on_push.yml b/.github/workflows/build_on_push.yml new file mode 100644 index 0000000000000..bc278450d3069 --- /dev/null +++ b/.github/workflows/build_on_push.yml @@ -0,0 +1,14 @@ +name: Build packages (on push) + +# This workflow runs when any of the following occur: +# - On push to branches named after ci/* +on: + push: + # Sequence of patterns matched against refs/heads + branches: + - "ci/*" + +jobs: + call-build-workflow: + uses: ./.github/workflows/build.yml + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/build_single.yml b/.github/workflows/build_single.yml new file mode 100644 index 0000000000000..669a987bdfd27 --- /dev/null +++ b/.github/workflows/build_single.yml @@ -0,0 +1,46 @@ +name: Build packages (single) + +# This workflow runs when any of the following occur: +# - Run manually +on: + workflow_dispatch: + inputs: + revision: + description: "Revision" + type: string + default: "0" + checksum: + description: "Checksum ?" + type: boolean + default: false + is_stage: + description: "Is stage ?" + type: boolean + default: false + system: + description: "Package OS" + type: choice + options: + - rpm + - deb + default: deb + architecture: + description: "Package architecture" + type: choice + options: + - amd64 + - x86_64 + default: amd64 + +jobs: + call-build-workflow: + uses: ./.github/workflows/build.yml + with: + revision: ${{ inputs.revision }} + checksum: ${{ inputs.checksum }} + is_stage: ${{ inputs.is_stage }} + distribution: '[ "${{ inputs.system }}" ]' + upload: true + # Architecture is always 'x64', which is the default value in ./build.yml + # It is an input just for convenience and standardisation. + secrets: inherit diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml deleted file mode 100644 index 11009429298e0..0000000000000 --- a/.github/workflows/r_assemble.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Assemble (reusable) - -env: - TEST: true - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - distribution: - description: "One of [ 'tar', 'rpm', 'deb' ]" - default: "rpm" - required: true - type: string - architecture: - description: "One of [ 'x64', 'arm64' ]" - default: "x64" - required: true - type: string - min: - description: "The name of the package to download." - required: true - type: string - -jobs: - r_assemble: - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v4 - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.min }} - path: artifacts/dist - - - name: Provision - if: ${{ inputs.distribution == 'deb' }} - run: | - sudo bash scripts/provision.sh - - - name: Run `assemble.sh` - run: | - bash scripts/assemble.sh -v ${{ vars.OPENSEARCH_VERSION }} -p linux -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} - - # The package's name is stored in artifacts/artifact_name.txt. - - name: Set package name - id: get_name - run: | - echo "name=$(cat artifacts/artifact_name.txt)" >> $GITHUB_OUTPUT - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.get_name.outputs.name }} - path: artifacts/dist/${{ steps.get_name.outputs.name }} - if-no-files-found: error diff --git a/.github/workflows/r_build.yml b/.github/workflows/r_build.yml deleted file mode 100644 index 4c430ce113ad6..0000000000000 --- a/.github/workflows/r_build.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Build (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - distribution: - description: "One of [ 'tar', 'rpm', 'deb' ]" - default: "rpm" - required: true - type: string - architecture: - description: "One of [ 'x64', 'arm64' ]" - default: "x64" - required: true - type: string - revision: - description: "Any string or number used to extend the package's identifier." - type: string - required: true - default: "1" - name: - description: "The name of the package to upload." - required: true - type: string - -jobs: - r_build: - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - - - name: Run `build.sh` - run: | - bash scripts/build.sh -v ${{ vars.OPENSEARCH_VERSION }} -s false -p linux -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} - - # The package's name is stored in artifacts/artifact_min_name.txt. - - name: Set package name - id: get_name - run: | - echo "name=$(cat artifacts/artifact_min_name.txt)" >> $GITHUB_OUTPUT - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.name }} - path: artifacts/dist/${{ steps.get_name.outputs.name }} - if-no-files-found: error diff --git a/.github/workflows/r_commit_sha.yml b/.github/workflows/r_commit_sha.yml deleted file mode 100644 index 44860cb3e7363..0000000000000 --- a/.github/workflows/r_commit_sha.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Get commit's short SHA (reusable)" - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - outputs: - commit_sha: - description: "Returns the short SHA of the latest commit" - value: ${{ jobs.r_commit_sha.outputs.commit_sha }} - -jobs: - r_commit_sha: - runs-on: ubuntu-latest - outputs: - commit_sha: ${{ steps.get_commit_sha.outputs.commit_sha }} - steps: - - uses: actions/checkout@v4 - - name: Get git commit SHA - id: get_commit_sha - run: | - echo "commit_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/r_test.yml b/.github/workflows/r_test.yml deleted file mode 100644 index fa9cd819ac8fe..0000000000000 --- a/.github/workflows/r_test.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Test (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - package: - description: "The name of the package to download." - required: true - type: string - -jobs: - r_test_rpm: - if: ${{ endsWith(inputs.package, 'rpm') }} - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: read - contents: read - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package }} - path: artifacts/dist - - - name: Install package - uses: addnab/docker-run-action@v3 - with: - image: redhat/ubi9:latest - options: -v ${{ github.workspace }}/artifacts/dist:/artifacts/dist - run: | - yum localinstall "/artifacts/dist/${{ inputs.package }}" -y - - r_test_deb: - if: ${{ endsWith(inputs.package, 'deb') }} - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: read - contents: read - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package }} - path: artifacts/dist - - - name: Install package - run: | - sudo dpkg -i "artifacts/dist/${{ inputs.package }}" - - - uses: actions/checkout@v4 - - name: Generate and deploy certificates - uses: addnab/docker-run-action@v3 - with: - image: wazuh/wazuh-certs-generator:0.0.1 - options: -v ${{ github.workspace }}/integrations/docker/config/certs.yml:/config/certs.yml -v /etc/wazuh-indexer/certs:/certs - shell: sh - run: | - mkdir -p /certificates /certs - /entrypoint.sh - chown -R 1000:999 /certificates - chmod 740 /certificates - chmod 440 /certificates/* - - mv /certificates/wazuh.indexer-key.pem /certs/indexer-key.pem - mv /certificates/wazuh.indexer.pem /certs/indexer.pem - mv /certificates/root-ca.pem /certs/root-ca.pem - ls /certs - - - run: sudo systemctl daemon-reload - - run: | - if ! sudo systemctl enable wazuh-indexer.service; then - sudo journalctl --no-pager -u wazuh-indexer.service - exit 1 - fi - - run: | - if ! sudo systemctl start wazuh-indexer; then - sudo journalctl --no-pager -u wazuh-indexer.service - exit 1 - fi - - run: | - if ! sudo systemctl status --no-pager wazuh-indexer -n 100; then - sudo journalctl --no-pager -u wazuh-indexer.service - exit 1 - fi diff --git a/.github/workflows/r_upload.yml b/.github/workflows/r_upload.yml deleted file mode 100644 index 05a08e40394c4..0000000000000 --- a/.github/workflows/r_upload.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Upload (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - package: - description: "The name of the package to upload to Wazuh's S3 development artifactory bucket." - required: true - type: string - secrets: - CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: - required: true - CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY: - required: true - -jobs: - r_upload: - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: read - contents: read - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package }} - path: artifacts/dist - - - name: Set up AWS CLI - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} - aws-region: us-east-1 - - - name: Upload package to S3 - run: | - aws s3 cp artifacts/dist/${{ inputs.package }} s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/ diff --git a/.github/workflows/r_version.yml b/.github/workflows/r_version.yml deleted file mode 100644 index d3c01135a6c35..0000000000000 --- a/.github/workflows/r_version.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Version (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - outputs: - version: - description: "Returns the version of Wazuh" - value: ${{ jobs.r_version.outputs.version }} - -jobs: - r_version: - runs-on: ubuntu-latest - outputs: - version: ${{ steps.get_version.outputs.version }} - steps: - - uses: actions/checkout@v4 - - name: Read 'VERSION' - id: get_version - run: | - echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT diff --git a/distribution/packages/src/rpm/wazuh-indexer.rpm.spec b/distribution/packages/src/rpm/wazuh-indexer.rpm.spec index b81d6a91ecb97..c7516f539a6e2 100644 --- a/distribution/packages/src/rpm/wazuh-indexer.rpm.spec +++ b/distribution/packages/src/rpm/wazuh-indexer.rpm.spec @@ -32,7 +32,7 @@ Name: wazuh-indexer Version: %{_version} -Release: 1 +Release: %{_release} License: Apache-2.0 Summary: An open source distributed and RESTful search engine URL: https://www.wazuh.com/ diff --git a/packaging_scripts/README.md b/packaging_scripts/README.md new file mode 100644 index 0000000000000..d81ce569be8e4 --- /dev/null +++ b/packaging_scripts/README.md @@ -0,0 +1,264 @@ +# `wazuh-indexer` packages generation guide + +The packages' generation process consists on 2 steps: + +- **Build**: compiles the Java application and bundles it into a package. +- **Assembly**: uses the package from the previous step and inflates it with plugins and + configuration files, ready for production deployment. + +We usually generate the packages using GitHub Actions, however, the process is designed to +be independent enough for maximum portability. GitHub Actions provides infrastructure, while +the building process is self-contained in the application code. + +Each section includes instructions to generate packages locally, using Act or Docker. + +- [Install Act](https://github.com/nektos/act) + +The names of the packages are managed by the `baptizer.sh` script. + +## Build + +For local package generation, use the `build.sh` script. Take a look at the `build.yml` +workflow file for an example of usage. + +```bash +bash packaging_scripts/build.sh -a x64 -d tar -n $(bash packaging_scripts/baptizer.sh -a x64 -d tar -m) +``` + +#### Act (GitHub Workflow locally) + +```console +act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts + +[Build slim packages/build] 🏁 Job succeeded +``` + +#### Running in Docker + +Using the [Docker environment](../docker): + +```console +docker exec -it wi-build_$( By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. + > The script takes care of creating the required folder structure, copying also the min package + > and the Makefile. + + Current folder loadout at this stage: + + ``` + artifacts/ + |-- dist + | |-- wazuh-indexer-min_4.9.0_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + `-- wazuh-indexer-min_4.9.0_amd64.deb + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. + `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + +2. Install the plugins using the `opensearch-plugin` CLI tool. +3. Set up configuration files. + + > Included in `min-package`. Default files are overwritten. + +4. Bundle a DEB file with `debmake` and the `Makefile`. + + > `debmake` and other dependencies can be installed using the `provision.sh` script. + > The script is invoked by the GitHub Workflow. + + Current folder loadout at this stage: + + ``` + artifacts/ + |-- artifact_name.txt + |-- dist + | |-- wazuh-indexer-min_4.9.0_amd64.deb + | `-- wazuh-indexer_4.9.0_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + |-- wazuh-indexer-min_4.9.0_amd64.deb + `-- debian/ + | -- control + | -- copyright + | -- rules + | -- preinst + | -- prerm + | -- postinst + ``` + +#### Running in Act + +```console +act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 + +[Build slim packages/build] 🏁 Job succeeded +``` + +#### Running in Docker + +Pre-requisites: + +- Current directory: `wazuh-indexer/` +- Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. +- Using the [Docker environment](../docker): + +```console +docker exec -it wi-assemble_$( By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`.The script takes care of creating the required folder structure, copying also the min package and the SPEC file. + + Current folder loadout at this stage: + + ``` + /rpm/$ARCH + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. + `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + +2. Install the plugins using the `opensearch-plugin` CLI tool. +3. Set up configuration files. + + > Included in `min-package`. Default files are overwritten. + +4. Bundle an RPM file with `rpmbuild` and the SPEC file `wazuh-indexer.rpm.spec`. + + > `rpmbuild` is part of the `rpm` OS package. + + > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. + + Current folder loadout at this stage: + + ``` + /rpm/$ARCH + /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` + +#### Running in Act + +```console +act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:rpm --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 + +[Build slim packages/build] 🏁 Job succeeded +``` + +#### Running in Docker + +Pre-requisites: + +- Current directory: `wazuh-indexer/` +- Existing rpm package in `wazuh-indexer/artifacts/dist/rpm`, as a result of the _Build_ stage. +- Using the [Docker environment](../docker): + +```console +docker exec -it wi-assemble_$("${OUTPUT}/artifact_name.txt" } # ==== @@ -305,11 +298,12 @@ function assemble_rpm() { --define "_topdir ${topdir}" \ --define "_version ${version}" \ --define "_architecture ${SUFFIX}" \ + --define "_release ${REVISION}" \ ${spec_file} # Move to the root folder, copy the package and clean. cd ../../.. - package_name="wazuh-indexer-${version}-1.${SUFFIX}.${EXT}" + package_name="wazuh-indexer-${version}-${REVISION}.${SUFFIX}.${EXT}" cp "${TMP_DIR}/RPMS/${SUFFIX}/${package_name}" "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME" clean @@ -359,7 +353,7 @@ function assemble_deb() { --invoke debuild \ --package wazuh-indexer \ --native \ - --revision 1 \ + --revision "${REVISION}" \ --upstreamversion "${version}" # Move to the root folder, copy the package and clean. @@ -379,9 +373,9 @@ function main() { echo "Assembling wazuh-indexer for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" - ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min_.*$SUFFIX.*\.$EXT") - - ARTIFACT_PACKAGE_NAME=${ARTIFACT_BUILD_NAME/min_/} + VERSION=$(bash packaging_scripts/upstream_version.sh) + ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min.*$SUFFIX.*\.$EXT") + ARTIFACT_PACKAGE_NAME=${ARTIFACT_BUILD_NAME/-min/} # Create temporal directory and copy the min package there for extraction TMP_DIR="${OUTPUT}/tmp/${TARGET}" @@ -399,6 +393,9 @@ function main() { assemble_deb ;; esac + + # Create checksum + sha512sum "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME" > "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME".sha512 } main "${@}" diff --git a/packaging_scripts/baptizer.sh b/packaging_scripts/baptizer.sh new file mode 100644 index 0000000000000..f17e43fa64a6b --- /dev/null +++ b/packaging_scripts/baptizer.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +set -e + +function usage() { + echo "Usage: $0 [args]" + echo "" + echo "Arguments:" + echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'." + echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." + echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'." + echo -e "-r REVISION\t[Optional] Package revision, default is '0'." + echo -e "-m MIN\t[Optional] Use naming convention for minimal packages, default is 'false'." + echo -e "-x RELEASE\t[Optional] Use release naming convention, default is 'false'." + echo -e "-h help" +} + +# ==== +# Parse arguments +# ==== +function parse_args() { + + while getopts ":h:p:a:d:r:mx" arg; do + case $arg in + h) + usage + exit 1 + ;; + p) + PLATFORM=$OPTARG + ;; + a) + ARCHITECTURE=$OPTARG + ;; + d) + DISTRIBUTION=$OPTARG + ;; + r) + REVISION=$OPTARG + ;; + m) + IS_MIN=true + ;; + x) + IS_RELEASE=true + ;; + :) + echo "Error: -${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${arg}" + exit 1 + ;; + esac + done + + [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') + [ -z "$ARCHITECTURE" ] && ARCHITECTURE=$(uname -m) + [ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar" + [ -z "$REVISION" ] && REVISION="0" + [ -z "$IS_MIN" ] && IS_MIN=false + [ -z "$IS_RELEASE" ] && IS_RELEASE=false + + case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in + linux-tar-x64 | darwin-tar-x64) + EXT="tar.gz" + SUFFIX="$PLATFORM-x64" + ;; + linux-tar-arm64 | darwin-tar-arm64) + EXT="tar.gz" + SUFFIX="$PLATFORM-arm64" + ;; + linux-deb-x64) + EXT="deb" + SUFFIX="amd64" + ;; + linux-deb-arm64) + EXT="deb" + SUFFIX="arm64" + ;; + linux-rpm-x64) + EXT="rpm" + SUFFIX="x86_64" + ;; + linux-rpm-arm64) + EXT="rpm" + SUFFIX="aarch64" + ;; + windows-zip-x64) + EXT="zip" + SUFFIX="$PLATFORM-x64" + ;; + windows-zip-arm64) + EXT="zip" + SUFFIX="$PLATFORM-arm64" + ;; + *) + echo "Unsupported platform-distribution-architecture combination: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" + exit 1 + ;; + esac + +} + +# ==== +# Naming convention for release packages +# ==== +function get_release_name() { + if [ "$EXT" = "rpm" ]; then + PACKAGE_NAME=wazuh-indexer-"$VERSION"-"$REVISION"."$SUFFIX"."$EXT" + else + PACKAGE_NAME=wazuh-indexer_"$VERSION"-"$REVISION"_"$SUFFIX"."$EXT" + fi + if "$IS_MIN"; then + PACKAGE_NAME=${PACKAGE_NAME/wazuh-indexer/wazuh-indexer-min} + fi +} + +# ==== +# Naming convention for pre-release packages +# ==== +function get_devel_name() { + if "$IS_MIN"; then + PACKAGE_NAME=wazuh-indexer-min_"$VERSION"-"$REVISION"_"$SUFFIX"_"$GIT_COMMIT"."$EXT" + else + PACKAGE_NAME=wazuh-indexer_"$VERSION"-"$REVISION"_"$SUFFIX"_"$GIT_COMMIT"."$EXT" + fi +} + +# ==== +# Naming convention control function +# ==== +function get_package_name() { + if "$IS_RELEASE"; then + get_release_name + else + get_devel_name + fi +} + +# ==== +# Main function +# ==== +function main() { + parse_args "${@}" + + get_package_name + echo "$PACKAGE_NAME" +} + +GIT_COMMIT=$(git rev-parse --short HEAD) +VERSION=$( - -### DEB - -The script will: - -- Extract the deb package using `ar` and `tar` tools. - - > By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. The script takes care of creating the required folder structure, copying also the min package and the Makefile. - - Current folder loadout at this stage: - - ``` - artifacts/ - |-- dist - | |-- wazuh-indexer-min_4.9.0_amd64.deb - `-- tmp - `-- deb - |-- Makefile - |-- data.tar.gz - |-- debmake_install.sh - |-- etc - |-- usr - |-- var - `-- wazuh-indexer-min_4.9.0_amd64.deb - ``` - - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. - `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. - -- Install the plugins using the `opensearch-plugin` CLI tool. -- Set up configuration files. - - > Included in `min-package`. Default files are overwritten. - -- Bundle a DEB file with `debmake` and the `Makefile`. - - > `debmake` and other dependencies can be installed using the provision.sh script. The - > script is invoked by the GitHub Workflow. - - Current folder loadout at this stage: - - ``` - artifacts/ - |-- artifact_name.txt - |-- dist - | |-- wazuh-indexer-min_4.9.0_amd64.deb - | `-- wazuh-indexer_4.9.0_amd64.deb - `-- tmp - `-- deb - |-- Makefile - |-- data.tar.gz - |-- debmake_install.sh - |-- etc - |-- usr - |-- var - |-- wazuh-indexer-min_4.9.0_amd64.deb - `-- debian/ - | -- control - | -- copyright - | -- rules - | -- preinst - | -- prerm - | -- postinst - ``` - -### Running in Act - -```console -act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 - -[Build slim packages/build] 🏁 Job succeeded -``` - -#### Running in Docker - -Pre-requisites: - -- Current directory: `wazuh-indexer/` -- Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. -- Using the [Docker environment](../docker): - -```console -docker exec -it wi-assemble_$( By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`. The script takes care of creating the required folder structure, copying also the min package and the SPEC file. - - Current folder loadout at this stage: - - ``` - /rpm/$ARCH - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` - - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. - `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. - -- Install the plugins using the `opensearch-plugin` CLI tool. -- Set up configuration files. - - > Included in `min-package`. Default files are overwritten. - -- Bundle an RPM file with `rpmbuild` and the SPEC file `wazuh-indexer.rpm.spec`. - - - `rpmbuild` is part of the `rpm` OS package. - - > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. - - Current folder loadout at this stage: - - ``` - /rpm/$ARCH - /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` - -### Running in Act - -```console -act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:rpm --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 - -[Build slim packages/build] 🏁 Job succeeded -``` - -#### Running in Docker - -Pre-requisites: - -- Current directory: `wazuh-indexer/` -- Existing rpm package in `wazuh-indexer/artifacts/dist/rpm`, as a result of the _Build_ stage. -- Using the [Docker environment](../docker): - -```console -docker exec -it wi-assemble_$("$OUTPUT/artifact_min_name.txt" - -mkdir -p "${OUTPUT}/dist" -cp "distribution/$TYPE/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME" "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME"