Create main branch for development builds (#279)

- Unify server and enterprise Dockerfiles
- Allow custom DOWNLOAD_URL to build snapshots
- Support snapshot builds in integration test
- Fix hook scripts with snapshots

docker/datanode/Dockerfile

@@ -4,8 +4,8 @@ FROM ubuntu:22.04
 ARG GRAYLOG_VERSION
 ARG VCS_REF
 ARG BUILD_DATE
-ARG SNAPSHOT_URL_X64=https://downloads.graylog.org/releases/graylog-datanode/graylog-datanode-${GRAYLOG_VERSION}-linux-x64.tgz
-ARG SNAPSHOT_URL_AARCH64=https://downloads.graylog.org/releases/graylog-datanode/graylog-datanode-${GRAYLOG_VERSION}-linux-aarch64.tgz
+ARG DOWNLOAD_URL_X64=https://downloads.graylog.org/releases/graylog-datanode/graylog-datanode-${GRAYLOG_VERSION}-linux-x64.tgz
+ARG DOWNLOAD_URL_AARCH64=https://downloads.graylog.org/releases/graylog-datanode/graylog-datanode-${GRAYLOG_VERSION}-linux-aarch64.tgz
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TARGETPLATFORM
 
@@ -67,11 +67,11 @@ RUN if [ -f /tmp/datanode-local.tar.gz ] && [ -s /tmp/datanode-local.tar.gz ]; t
     fi; \
     if [ "${LOCAL_BUILD_TGZ}" = ".empty" ]; then \
       if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
-         export SNAPSHOT_URL="$SNAPSHOT_URL_AARCH64"; \
+         export DOWNLOAD_URL="$DOWNLOAD_URL_AARCH64"; \
        else \
-         export SNAPSHOT_URL="$SNAPSHOT_URL_X64"; \
+         export DOWNLOAD_URL="$DOWNLOAD_URL_X64"; \
       fi; \
-      curl -fsSL --retry 3 "$SNAPSHOT_URL" -o /tmp/datanode.tar.gz; \
+      curl -fsSL --retry 3 "$DOWNLOAD_URL" -o /tmp/datanode.tar.gz; \
     fi; \
     tar -C "$GDN_APP_ROOT" --strip-components=1 -xzf /tmp/datanode.tar.gz \
     && rm -rf /tmp/datanode-local.tar.gz /tmp/datanode.tar.gz \

docker/datanode/hooks/build

@@ -8,9 +8,23 @@ cd ../..
 apt-get install -y python3-pip
 pip3 install pyyaml
 
+graylog_version="$(./release.py --get-graylog-version)"
+build_args=()
+
+if [[ "$graylog_version" =~ SNAPSHOT ]]; then
+	download_url_x64="$(curl -fsSL -G -d artifact=graylog-datanode-linux-x64 -d limit=1 https://downloads.graylog.org/nightly-builds | jq -r '.artifacts[0].url')"
+	download_url_aarch64="$(sed -e 's,linux-x64,linux-aarch64,' <<< "$download_url_x64")"
+	download_url="$(sed -e 's,-linux-x64,,' <<< "$download_url_x64")"
+
+	build_args+=(--build-arg DOWNLOAD_URL="$download_url")
+	build_args+=(--build-arg DOWNLOAD_URL_X64="$download_url_x64")
+	build_args+=(--build-arg DOWNLOAD_URL_AARCH64="$download_url_aarch64")
+fi
+
 # Build Graylog
 docker build --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
-             --build-arg GRAYLOG_VERSION="$(./release.py --get-graylog-version)" \
+             --build-arg GRAYLOG_VERSION="$graylog_version" \
              --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+	     ${build_args[*]} \
              --file docker/datanode/Dockerfile \
              --tag $IMAGE_NAME .

docker/enterprise/Dockerfile

@@ -4,6 +4,7 @@ FROM ubuntu:jammy as graylog-downloader
 ARG VCS_REF
 ARG BUILD_DATE
 ARG GRAYLOG_VERSION
+ARG GRAYLOG_PRODUCT=graylog-enterprise
 ARG GRAYLOG_HOME=/usr/share/graylog
 ARG GRAYLOG_UID=1100
 ARG GRAYLOG_GID=1100
@@ -17,6 +18,9 @@ ARG TARGETPLATFORM
 # all files from the build context.
 ARG LOCAL_BUILD_TGZ=.empty
 
+# Allows building a custom artifact. (e.g., snapshot builds)
+ARG DOWNLOAD_URL=none
+
 WORKDIR /tmp
 
 # hadolint ignore=DL3008,DL3015
@@ -28,13 +32,17 @@ RUN \
     curl > /dev/null
 
 COPY build/fetch-and-extract.sh /bin/fetch-and-extract
-RUN if [ "${LOCAL_BUILD_TGZ}" = ".empty" ]; then \
+
+RUN if [ "${LOCAL_BUILD_TGZ}" = ".empty" ] && [ "${DOWNLOAD_URL}" = "none" ]; then \
       chmod +x /bin/fetch-and-extract \
       && fetch-and-extract \
-         "https://packages.graylog2.org/releases/graylog-enterprise/graylog-enterprise-${GRAYLOG_VERSION}.tgz" \
-         "https://packages.graylog2.org/releases/graylog-enterprise/graylog-enterprise-${GRAYLOG_VERSION}.tgz.sha256.txt"; \
+         "https://downloads.graylog.org/releases/${GRAYLOG_PRODUCT}/${GRAYLOG_PRODUCT}-${GRAYLOG_VERSION}.tgz" \
+         "https://downloads.graylog.org/releases/${GRAYLOG_PRODUCT}/${GRAYLOG_PRODUCT}-${GRAYLOG_VERSION}.tgz.sha256.txt"; \
     fi
 
+RUN if [ "${DOWNLOAD_URL}" != "none" ]; then \
+      chmod +x /bin/fetch-and-extract && fetch-and-extract "${DOWNLOAD_URL}"; \
+    fi
 
 COPY "${LOCAL_BUILD_TGZ}" "/tmp/graylog.tgz"
 
@@ -150,7 +158,6 @@ RUN \
     /usr/share/X11 \
     /usr/share/doc/* 2> /dev/null
 
-
 COPY docker-entrypoint.sh /
 COPY health_check.sh /
 
@@ -170,8 +177,8 @@ HEALTHCHECK \
 # -------------------------------------------------------------------------------------------------
 
 LABEL maintainer="Graylog, Inc. <[email protected]>" \
-      org.label-schema.name="Graylog Docker Image" \
-      org.label-schema.description="Official Graylog Docker image - with Enterprise Plugins" \
+      org.label-schema.name="Graylog Enterprise Docker Image" \
+      org.label-schema.description="Official Graylog Enterprise Docker Image" \
       org.label-schema.url="https://www.graylog.org/" \
       org.label-schema.vcs-ref=${VCS_REF} \
       org.label-schema.vcs-url="https://github.com/Graylog2/graylog-docker" \

docker/enterprise/hooks/build

@@ -10,9 +10,23 @@ patch config/graylog.conf patches/graylog-server.conf.patch
 apt-get install -y python3-pip
 pip3 install pyyaml
 
+graylog_version="$(./release.py --get-graylog-version)"
+build_args=()
+
+if [[ "$graylog_version" =~ SNAPSHOT ]]; then
+	download_url_x64="$(curl -fsSL -G -d artifact=graylog-enterprise-linux-x64 -d limit=1 https://downloads.graylog.org/nightly-builds | jq -r '.artifacts[0].url')"
+	download_url_aarch64="$(sed -e 's,linux-x64,linux-aarch64,' <<< "$download_url_x64")"
+	download_url="$(sed -e 's,-linux-x64,,' <<< "$download_url_x64")"
+
+	build_args+=(--build-arg DOWNLOAD_URL="$download_url")
+	build_args+=(--build-arg DOWNLOAD_URL_X64="$download_url_x64")
+	build_args+=(--build-arg DOWNLOAD_URL_AARCH64="$download_url_aarch64")
+fi
+
 # Build Graylog Enterprise & Integration Included
 docker build --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
-             --build-arg GRAYLOG_VERSION="$(./release.py --get-graylog-version)" \
+             --build-arg GRAYLOG_VERSION="$graylog_version" \
              --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+	     ${build_args[*]} \
              --file docker/enterprise/Dockerfile \
              --tag $IMAGE_NAME .

docker/oss/Dockerfile

@@ -2,14 +2,28 @@
 FROM ubuntu:jammy as graylog-downloader
 
 ARG VCS_REF
+ARG BUILD_DATE
 ARG GRAYLOG_VERSION
+ARG GRAYLOG_PRODUCT=graylog
 ARG GRAYLOG_HOME=/usr/share/graylog
 ARG GRAYLOG_UID=1100
 ARG GRAYLOG_GID=1100
+ARG TARGETPLATFORM
+
+# We default to an empty file instead of leaving LOCAL_BUILD_TGZ blank
+# because Docker would execute the following COPY command with a blank
+# value:
+#   COPY "" "/tmp/graylog.tgz"
+# That creates a /tmp/graylog.tgz *directory* in the container with
+# all files from the build context.
+ARG LOCAL_BUILD_TGZ=.empty
+
+# Allows building a custom artifact. (e.g., snapshot builds)
+ARG DOWNLOAD_URL=none
 
 WORKDIR /tmp
 
-# hadolint ignore=DL3008,DL3015,DL3059
+# hadolint ignore=DL3008,DL3015
 RUN \
   apt-get update  > /dev/null && \
   apt-get upgrade -y > /dev/null && \
@@ -18,10 +32,27 @@ RUN \
     curl > /dev/null
 
 COPY build/fetch-and-extract.sh /bin/fetch-and-extract
-RUN chmod +x /bin/fetch-and-extract && fetch-and-extract \
-    "https://packages.graylog2.org/releases/graylog/graylog-${GRAYLOG_VERSION}.tgz" \
-    "https://packages.graylog2.org/releases/graylog/graylog-${GRAYLOG_VERSION}.tgz.sha256.txt"
 
+RUN if [ "${LOCAL_BUILD_TGZ}" = ".empty" ] && [ "${DOWNLOAD_URL}" = "none" ]; then \
+      chmod +x /bin/fetch-and-extract \
+      && fetch-and-extract \
+         "https://downloads.graylog.org/releases/${GRAYLOG_PRODUCT}/${GRAYLOG_PRODUCT}-${GRAYLOG_VERSION}.tgz" \
+         "https://downloads.graylog.org/releases/${GRAYLOG_PRODUCT}/${GRAYLOG_PRODUCT}-${GRAYLOG_VERSION}.tgz.sha256.txt"; \
+    fi
+
+RUN if [ "${DOWNLOAD_URL}" != "none" ]; then \
+      chmod +x /bin/fetch-and-extract && fetch-and-extract "${DOWNLOAD_URL}"; \
+    fi
+
+COPY "${LOCAL_BUILD_TGZ}" "/tmp/graylog.tgz"
+
+# An empty /tmp/graylog.tgz file indicates that we don't use a
+# custom LOCAL_BUILD_TGZ file.
+RUN if [ -f "/tmp/graylog.tgz" ] && [ -s "/tmp/graylog.tgz" ]; then \
+        mkdir /opt/graylog && \
+        tar --extract --gzip --file "/tmp/graylog.tgz" --strip-components=1 --directory /opt/graylog; \
+        rm -rf /tmp/graylog.tgz; \
+    fi
 
 RUN \
   install \
@@ -134,7 +165,7 @@ HEALTHCHECK \
 
 LABEL maintainer="Graylog, Inc. <[email protected]>" \
       org.label-schema.name="Graylog Docker Image" \
-      org.label-schema.description="Official Graylog Docker image" \
+      org.label-schema.description="Official Graylog Docker Image" \
       org.label-schema.url="https://www.graylog.org/" \
       org.label-schema.vcs-ref=${VCS_REF} \
       org.label-schema.vcs-url="https://github.com/Graylog2/graylog-docker" \

docker/oss/hooks/build

@@ -8,9 +8,23 @@ cd ../..
 apt-get install -y python3-pip
 pip3 install pyyaml
 
+graylog_version="$(./release.py --get-graylog-version)"
+build_args=()
+
+if [[ "$graylog_version" =~ SNAPSHOT ]]; then
+	download_url_x64="$(curl -fsSL -G -d artifact=graylog-linux-x64 -d limit=1 https://downloads.graylog.org/nightly-builds | jq -r '.artifacts[0].url')"
+	download_url_aarch64="$(sed -e 's,linux-x64,linux-aarch64,' <<< "$download_url_x64")"
+	download_url="$(sed -e 's,-linux-x64,,' <<< "$download_url_x64")"
+
+	build_args+=(--build-arg DOWNLOAD_URL="$download_url")
+	build_args+=(--build-arg DOWNLOAD_URL_X64="$download_url_x64")
+	build_args+=(--build-arg DOWNLOAD_URL_AARCH64="$download_url_aarch64")
+fi
+
 # Build Graylog
 docker build --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
-             --build-arg GRAYLOG_VERSION="$(./release.py --get-graylog-version)" \
+             --build-arg GRAYLOG_VERSION="$graylog_version" \
              --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+	     ${build_args[*]} \
              --file docker/oss/Dockerfile \
              --tag $IMAGE_NAME .

test/docker-compose.tpl

@@ -28,6 +28,7 @@ services:
       args:
         - VCS_REF
         - GRAYLOG_VERSION
+        - DOWNLOAD_URL
     entrypoint: /usr/bin/tini -- wait-for-it opensearch:9200 --  /docker-entrypoint.sh
     environment:
       # CHANGE ME!

test/integration_test.sh

@@ -53,14 +53,26 @@ then
   NC_OPTS=
 fi
 
+get_snapshot_url() {
+    curl -fsSL -G -d artifact=graylog -d limit=1 https://downloads.graylog.org/nightly-builds | \
+	    jq -r '.artifacts[0].url'
+}
 
 compose_up() {
+  local graylog_version="$(cd .. && ./release.py --get-graylog-version)"
 
+  echo "Using Graylog version: $graylog_version"
   cat << EOF > .env
 VCS_REF=$(git rev-parse --short HEAD)
-GRAYLOG_VERSION=$(cd .. && ./release.py --get-graylog-version)
+GRAYLOG_VERSION=$graylog_version
 EOF
 
+  if [[ "$graylog_version" =~ SNAPSHOT ]]; then
+    local snapshot_url="$(get_snapshot_url)"
+    echo "Using snapshot: $snapshot_url"
+    echo "DOWNLOAD_URL=$snapshot_url" >> .env
+  fi
+
   docker-compose --file docker-compose.tpl config  > ./docker-compose.yml
   docker-compose down -v
   docker-compose build --pull

version.yml

@@ -2,8 +2,8 @@
 # For GA releases:  patch_version=0
 graylog:
   major_version: '6'
-  minor_version: '1'
-  patch_version: '0'
+  minor_version: '2'
+  patch_version: 0-SNAPSHOT
   release: 1
 forwarder:
   version: '6.1'