From 1ba23516568a582e12c5a6df783ac030a14c9efc Mon Sep 17 00:00:00 2001 From: Federico Gustavo Galland <99492720+f-galland@users.noreply.github.com> Date: Fri, 19 Jan 2024 08:32:37 -0300 Subject: [PATCH] Get Wazuh version from VERSION file (#122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add function to look for VERSION in the correct path * Update assemble.sh Adds wget as dependency * Download files using curl instead of wget * Update assemble.sh Revert assembly with minimal plugins for testing Signed-off-by: Álex Ruiz * Add Dockerfile and docker-compose for the package assembly stage * Assemble packages with minimal plugin set when "test" variable is set to "true" * Update README with assemble.sh docker image * Fixing env variable naming convention and removing wget dependency * Improve Docker environments Adds environments to build packages * Fix small typos * More fixes * Add documentation * Adding -p flag to mkdir so it doesnt fail when the folder is already present * Format files --------- Signed-off-by: Álex Ruiz Co-authored-by: Álex Ruiz --- docker/README.md | 17 +++- docker/ci/ci.sh | 62 +++++++++++++ docker/ci/ci.yml | 29 ++++++ docker/{ => ci}/images/.dockerignore | 0 docker/ci/images/Dockerfile | 17 ++++ docker/dev.yml | 26 ------ docker/{ => dev}/dev.sh | 12 ++- docker/dev/dev.yml | 19 ++++ docker/dev/images/.dockerignore | 68 ++++++++++++++ .../images/Dockerfile} | 10 +- scripts/README.md | 51 +++-------- scripts/assemble.sh | 91 +++++++++++-------- scripts/provision.sh | 2 +- 13 files changed, 283 insertions(+), 121 deletions(-) create mode 100755 docker/ci/ci.sh create mode 100644 docker/ci/ci.yml rename docker/{ => ci}/images/.dockerignore (100%) create mode 100644 docker/ci/images/Dockerfile delete mode 100644 docker/dev.yml rename docker/{ => dev}/dev.sh (82%) create mode 100644 docker/dev/dev.yml create mode 100644 docker/dev/images/.dockerignore rename docker/{images/wi-dev.Dockerfile => dev/images/Dockerfile} (67%) diff --git a/docker/README.md b/docker/README.md index 4b41a42e0e9c9..e012e8c42bed1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -51,19 +51,28 @@ Docker Desktop will change to its context automatically at start, so be sure that any existing Docker container using the default context is **stopped** before starting Docker Desktop and any of the environments in this folder. -## Starting up the environments +## Development environments -Use the sh script to up the environment. +Use the `dev/dev.sh` script to start a development environment. Example: ```bash -Usage: ./dev.sh {up|down|stop} [security] +Usage: ./dev.sh {up|down|stop} ``` -Once the `wazuh-indexer` container is up, attach a shell to it and run `./gradlew run` +Once the `wi-dev:x.y.z` container is up, attach a shell to it and run `./gradlew run` to start the application. +## Containers to generate packages + +Use the `ci/ci.sh` script to start provisioned containers to generate packages. + +```bash +Usage: ./ci.sh {up|down|stop} [ci] +``` + +Refer to [scripts/README.md](../scripts/README.md) for details about how to build packages. [docker-desktop]: https://docs.docker.com/get-docker [docker-variant]: https://docs.docker.com/desktop/install/linux-install/#differences-between-docker-desktop-for-linux-and-docker-engine diff --git a/docker/ci/ci.sh b/docker/ci/ci.sh new file mode 100755 index 0000000000000..13e84c0881b3f --- /dev/null +++ b/docker/ci/ci.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Start container with required tools to build packages +# Requires Docker +# Script usage: bash ./ci.sh + +set -e + +# ==== +# Checks that the script is run from the intended location +# ==== +function check_project_root_folder() { + current=$(basename "$(pwd)") + + if [[ "$0" != "./ci.sh" && "$0" != "ci.sh" ]]; then + echo "Run the script from its location" + usage + exit 1 + fi + # Change working directory to the root of the repository + cd ../.. +} + +# ==== +# Displays usage +# ==== +function usage() { + echo "Usage: ./ci.sh {up|down|stop}" +} + +# ==== +# Main function +# ==== +function main() { + check_project_root_folder "$@" + compose_file="docker/${current}/ci.yml" + compose_cmd="docker compose -f $compose_file" + REPO_PATH=$(pwd) + VERSION=$(cat VERSION) + export REPO_PATH + export VERSION + + case $1 in + up) + # Main folder created here to grant access to both containers + mkdir -p artifacts + $compose_cmd up -d + ;; + down) + $compose_cmd down + ;; + stop) + $compose_cmd stop + ;; + *) + usage + exit 1 + ;; + esac +} + +main "$@" diff --git a/docker/ci/ci.yml b/docker/ci/ci.yml new file mode 100644 index 0000000000000..a18ca915302f0 --- /dev/null +++ b/docker/ci/ci.yml @@ -0,0 +1,29 @@ +version: "3.9" + +services: + # Essentially wi-dev, but doesn't expose port 9200 + wi-build: + image: wi-build:${VERSION} + container_name: wi-build_${VERSION} + build: + context: ./../.. + dockerfile: ${REPO_PATH}/docker/dev/images/Dockerfile + volumes: + - ${REPO_PATH}:/home/wazuh-indexer + entrypoint: ["tail", "-f", "/dev/null"] + user: "1000:1000" + working_dir: /home/wazuh-indexer + + wi-assemble: + image: wi-assemble:${VERSION} + container_name: wi-assemble_${VERSION} + build: + context: ./../.. + dockerfile: ${REPO_PATH}/docker/ci/images/Dockerfile + volumes: + - ${REPO_PATH}/scripts:/home/wazuh-indexer/scripts + - ${REPO_PATH}/artifacts:/home/wazuh-indexer/artifacts + - ${REPO_PATH}/distribution/packages/src:/home/wazuh-indexer/distribution/packages/src + entrypoint: ["tail", "-f", "/dev/null"] + user: "1000:1000" + working_dir: /home/wazuh-indexer diff --git a/docker/images/.dockerignore b/docker/ci/images/.dockerignore similarity index 100% rename from docker/images/.dockerignore rename to docker/ci/images/.dockerignore diff --git a/docker/ci/images/Dockerfile b/docker/ci/images/Dockerfile new file mode 100644 index 0000000000000..c84791adcb3e5 --- /dev/null +++ b/docker/ci/images/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:jammy +RUN mkdir /home/wazuh-indexer && \ + apt-get update -y && \ + apt-get install curl gnupg2 -y && \ + curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && \ + echo "deb http://repo.aptly.info/ squeeze main" | tee -a /etc/apt/sources.list.d/aptly.list && \ + apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y aptly build-essential cpio debhelper-compat debmake freeglut3 libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-dev libcairo2 libcairo2-dev libcups2 libdrm2 libgbm-dev libgconf-2-4 libnspr4 libnspr4-dev libnss3 libpangocairo-1.0-0 libxcomposite-dev libxdamage1 libxfixes-dev libxfixes3 libxi6 libxkbcommon-x11-0 libxrandr2 libxrender1 libxtst6 rpm rpm2cpio && \ + apt-get clean -y && \ + dpkg -r lintian && \ + addgroup --gid 1000 wazuh-indexer && \ + adduser --uid 1000 --ingroup wazuh-indexer --disabled-password --home /home/wazuh-indexer wazuh-indexer && \ + chmod 0775 /home/wazuh-indexer && \ + chown -R 1000:1000 /home/wazuh-indexer +USER wazuh-indexer +WORKDIR /home/wazuh-indexer diff --git a/docker/dev.yml b/docker/dev.yml deleted file mode 100644 index 7e3b6202c2ee9..0000000000000 --- a/docker/dev.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: "3.9" - -services: - - wazuh-indexer: - image: wazuh-indexer-dev:${VERSION} - container_name: wazuh-indexer-dev-${VERSION} - build: - context: ./.. - dockerfile: ${REPO_PATH}/docker/images/wi-dev.Dockerfile - ports: - # OpenSearch REST API - - 9200:9200 - # Cross-cluster search - # - 9250:9250 - # Node communication and transport - # - 9300:9300 - # Performance Analyzer - # - 9600:9600 - expose: - - 9200 - volumes: - - ${REPO_PATH}:/home/wazuh-indexer/app - entrypoint: ['tail', '-f', '/dev/null'] - user: "1000:1000" - working_dir: /home/wazuh-indexer/app diff --git a/docker/dev.sh b/docker/dev/dev.sh similarity index 82% rename from docker/dev.sh rename to docker/dev/dev.sh index 991ed42de29e7..f20b7359c9206 100755 --- a/docker/dev.sh +++ b/docker/dev/dev.sh @@ -9,14 +9,16 @@ set -e # ==== # Checks that the script is run from the intended location # ==== -function check_project_root_folder () { +function check_project_root_folder() { + current=$(basename "$(pwd)") + if [[ "$0" != "./dev.sh" && "$0" != "dev.sh" ]]; then - echo "Run the script from its location" + echo "Run the script from its location" usage exit 1 fi # Change working directory to the root of the repository - cd .. + cd ../.. } # ==== @@ -31,7 +33,7 @@ function usage() { # ==== function main() { check_project_root_folder "$@" - compose_file=docker/dev.yml + compose_file="docker/${current}/dev.yml" compose_cmd="docker compose -f $compose_file" REPO_PATH=$(pwd) VERSION=$(cat VERSION) @@ -55,4 +57,4 @@ function main() { esac } -main "$@" \ No newline at end of file +main "$@" diff --git a/docker/dev/dev.yml b/docker/dev/dev.yml new file mode 100644 index 0000000000000..7176b044df5ba --- /dev/null +++ b/docker/dev/dev.yml @@ -0,0 +1,19 @@ +version: "3.9" + +services: + wi-dev: + image: wi-dev:${VERSION} + container_name: wi-dev_${VERSION} + build: + context: ./../.. + dockerfile: ${REPO_PATH}/docker/dev/images/Dockerfile + ports: + # OpenSearch REST API + - 9200:9200 + expose: + - 9200 + volumes: + - ${REPO_PATH}:/home/wazuh-indexer + entrypoint: ["tail", "-f", "/dev/null"] + user: "1000:1000" + working_dir: /home/wazuh-indexer diff --git a/docker/dev/images/.dockerignore b/docker/dev/images/.dockerignore new file mode 100644 index 0000000000000..96d12ad527ea5 --- /dev/null +++ b/docker/dev/images/.dockerignore @@ -0,0 +1,68 @@ +artifacts/ +.git/ + +# intellij files +.idea/ +*.iml +*.ipr +*.iws +build-idea/ +out/ + +# include shared intellij config +!.idea/inspectionProfiles/Project_Default.xml +!.idea/runConfigurations/Debug_OpenSearch.xml +!.idea/vcs.xml + +# These files are generated in the main tree by annotation processors +benchmarks/src/main/generated/* +benchmarks/bin/* +benchmarks/build-eclipse-default/* +server/bin/* +server/build-eclipse-default/* +test/framework/build-eclipse-default/* + +# eclipse files +.project +.classpath +.settings +build-eclipse/ + +# netbeans files +nb-configuration.xml +nbactions.xml + +# gradle stuff +.gradle/ +build/ + +# vscode stuff +.vscode/ + +# testing stuff +**/.local* +.vagrant/ +/logs/ + +# osx stuff +.DS_Store + +# default folders in which the create_bwc_index.py expects to find old es versions in +/backwards +/dev-tools/backwards + +# needed in case docs build is run...maybe we can configure doc build to generate files under build? +html_docs + +# random old stuff that we should look at the necessity of... +/tmp/ +eclipse-build + +# projects using testfixtures +testfixtures_shared/ + +# These are generated from .ci/jobs.t +.ci/jobs/ + +# build files generated +doc-tools/missing-doclet/bin/ \ No newline at end of file diff --git a/docker/images/wi-dev.Dockerfile b/docker/dev/images/Dockerfile similarity index 67% rename from docker/images/wi-dev.Dockerfile rename to docker/dev/images/Dockerfile index 215b956d14a82..f49aa4b1aba6e 100644 --- a/docker/images/wi-dev.Dockerfile +++ b/docker/dev/images/Dockerfile @@ -1,7 +1,7 @@ FROM gradle:jdk17-alpine AS builder USER gradle -WORKDIR /home/wazuh-indexer/app -COPY --chown=gradle:gradle . /home/wazuh-indexer/app +WORKDIR /home/wazuh-indexer +COPY --chown=gradle:gradle . /home/wazuh-indexer RUN gradle clean @@ -12,7 +12,7 @@ RUN apk add git && \ chmod 0775 /home/wazuh-indexer && \ chown -R 1000:0 /home/wazuh-indexer USER wazuh-indexer -COPY --from=builder --chown=1000:0 /home/wazuh-indexer/app /home/wazuh-indexer/app -WORKDIR /home/wazuh-indexer/app -RUN git config --global --add safe.directory /home/wazuh-indexer/app +COPY --from=builder --chown=1000:0 /home/wazuh-indexer /home/wazuh-indexer +WORKDIR /home/wazuh-indexer +RUN git config --global --add safe.directory /home/wazuh-indexer EXPOSE 9200 9300 diff --git a/scripts/README.md b/scripts/README.md index 8c98f8206cab5..c83f20f3a3aac 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -29,16 +29,19 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts #### Running in Docker -Within the [Docker environment](../docker): +Using the [Docker environment](../docker): ```console -bash scripts/build.sh -v 2.11.0 -s false -p linux -a {x64|arm64} -d {rpm|deb|tar} +docker exec -it wi-build_$( @@ -104,7 +107,7 @@ The script will: ### Running in Act ```console -act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.0 +act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 [Build slim packages/build] 🏁 Job succeeded ``` @@ -115,33 +118,10 @@ Pre-requisites: - Current directory: `wazuh-indexer/` - Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. +- Using the [Docker environment](../docker): ```console -MIN_PKG_PATH="./artifacts" -docker run --rm \ - -v ./scripts/:/home/wazuh-indexer/scripts \ - -v $MIN_PKG_PATH:/home/wazuh-indexer/artifacts \ - -v ./distribution/packages/src:/home/wazuh-indexer/distribution/packages/src \ - -w /home/wazuh-indexer \ - -it ubuntu:jammy /bin/bash - -# https://github.com/opensearch-project/opensearch-build/blob/2.11.1/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile - -# Install necessary packages -apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && - apt-get install -y debmake debhelper-compat && - apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && - apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && - apt-get clean -y - -# Install aptly and required changes to debmake -# Remove lintian for now due to it takes nearly 20 minutes for OpenSearch as well as nearly an hour for OpenSearch-Dashboards during debmake -curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && - echo "deb http://repo.aptly.info/ squeeze main" | tee -a /etc/apt/sources.list.d/aptly.list && - apt-get update -y && apt-get install -y aptly && apt-get clean -y && - dpkg -r lintian - -bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d deb +docker exec -it wi-assemble_$(