Experimental arm64 support for Cassandra 3.11.6 (#36)

* Update netty for epoll arm64 support * Disable metrics monitor for arm64 since we have no binary for it * Update build for testing multiarch * Fix workflow typo * Update branch for testing * Try only adding aarch64 epoll binary instead of replace all netty * Revert "Try only adding aarch64 epoll binary instead of replace all netty" This reverts commit 8039c7c. * Fix release workflow for merging * Update ci to build 3.11 same as for release * Revert "Update ci to build 3.11 same as for release" This reverts commit fd5139b. * Revert "Revert "Update ci to build 3.11 same as for release"" This reverts commit 1a40ff1. * Update ci to build for amd64 and arm64 * Add use of buildx in integration test * Temporarily only test 3.11 * Revert "Temporarily only test 3.11" This reverts commit e64005d. * Update to use 3.11.7 explicitly
k8ssandra · Aug 26, 2020 · 5256425 · 5256425
1 parent 5b98bcc
commit 5256425
Show file tree

Hide file tree

Showing 7 changed files with 212 additions and 19 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -18,6 +18,11 @@ jobs:
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
+      - name: Setup Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v3
+        with:
+          version: latest
       - name: Build Management API in docker
         run: |
           cat <<EOF > ~/.m2/settings.xml
@@ -39,7 +44,11 @@ jobs:
           cp ~/.m2/settings.xml settings.xml
           docker build -t management-api-for-dse-builder -f ./Dockerfile-build-dse ./
           docker tag management-api-for-dse-builder management-api-for-apache-cassandra-builder
-          docker build -t mgmtapi-3_11 -f Dockerfile-3_11 .
+          docker buildx build \
+            --tag mgmtapi-3_11 \
+            --file Dockerfile-oss \
+            --target oss311 \
+            --platform linux/amd64,linux/arm64 .
           docker build -t mgmtapi-4_0 -f Dockerfile-4_0 .
           docker build -t mgmtapi-dse-68 -f Dockerfile-dse-68 .
       - name: Build with Maven

diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml
@@ -6,7 +6,7 @@ on:
       - 'v*.*.*'
 
 jobs:
-  build:
+  build-dse:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
@@ -32,14 +32,14 @@ jobs:
           cp ~/.m2/settings.xml settings.xml
           docker build -t management-api-for-dse-builder -f ./Dockerfile-build-dse ./
           docker tag management-api-for-dse-builder management-api-for-apache-cassandra-builder
-      - name: Publish 3.11 to Registry
+      - name: Publish DSE 6.8 to Registry
         uses: elgohr/Publish-Docker-Github-Action@master
         with:
-          name: datastax/cassandra-mgmtapi-3_11_7
+          name: datastax/dse-mgmtapi-6_8
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           tag_names: true
-          dockerfile: Dockerfile-3_11
+          dockerfile: Dockerfile-dse-68
       - name: Publish 4.0 to Registry
         uses: elgohr/Publish-Docker-Github-Action@master
         with:
@@ -48,11 +48,22 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
           tag_names: true
           dockerfile: Dockerfile-4_0
-      - name: Publish DSE 6.8 to Registry
-        uses: elgohr/Publish-Docker-Github-Action@master
+  build-oss:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Setup Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v3
         with:
-          name: datastax/dse-mgmtapi-6_8
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          tag_names: true
-          dockerfile: Dockerfile-dse-68
+          version: latest
+      - name: Login to Docker Hub
+        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+      - name: Publish 3.11 to Registry
+        run: |
+          RELEASE_VERSION="${GITHUB_REF##*/}"
+          docker buildx build --push \
+            --tag datastax/cassandra-mgmtapi-3_11_7:$RELEASE_VERSION \
+            --file Dockerfile-oss \
+            --target oss311 \
+            --platform linux/amd64,linux/arm64 .
diff --git a/Dockerfile-oss b/Dockerfile-oss
@@ -0,0 +1,115 @@
+FROM --platform=$BUILDPLATFORM maven:3.6.3-jdk-8-slim as builder
+
+WORKDIR /build
+
+COPY pom.xml ./
+COPY management-api-agent/pom.xml ./management-api-agent/pom.xml
+COPY management-api-common/pom.xml ./management-api-common/pom.xml
+COPY management-api-server/pom.xml ./management-api-server/pom.xml
+COPY management-api-shim-3.x/pom.xml ./management-api-shim-3.x/pom.xml
+COPY management-api-shim-4.x/pom.xml ./management-api-shim-4.x/pom.xml
+# this duplicates work done in the next steps, but this should provide
+# a solid cache layer that only gets reset on pom.xml changes
+RUN mvn -q -ff -T 1C install && rm -rf target
+
+COPY management-api-agent ./management-api-agent
+COPY management-api-common ./management-api-common
+COPY management-api-server ./management-api-server
+COPY management-api-shim-3.x ./management-api-shim-3.x
+COPY management-api-shim-4.x ./management-api-shim-4.x
+RUN mvn -q -ff package -DskipTests
+
+FROM --platform=$BUILDPLATFORM maven:3.6.3-jdk-8-slim as netty4150
+RUN mvn dependency:get -DgroupId=io.netty -DartifactId=netty-all -Dversion=4.1.50.Final -Dtransitive=false
+
+FROM --platform=linux/amd64 cassandra:3.11.7 as oss311-amd64
+
+FROM --platform=linux/arm64 cassandra:3.11.7 as oss311-arm64
+# Netty arm64 epoll support was not added until 4.1.50 (https://github.com/netty/netty/pull/9804)
+# Only replace this dependency for arm64 to avoid regressions
+RUN rm /opt/cassandra/lib/netty-all-*.jar
+COPY --from=netty4150 /root/.m2/repository/io/netty/netty-all/4.1.50.Final/netty-all-4.1.50.Final.jar /opt/cassandra/lib/netty-all-4.1.50.Final.jar
+
+FROM oss311-${TARGETARCH} as oss311
+
+ARG TARGETARCH
+
+COPY --from=builder /build/management-api-common/target/datastax-mgmtapi-common-0.1.0-SNAPSHOT.jar /etc/cassandra/
+COPY --from=builder /build/management-api-agent/target/datastax-mgmtapi-agent-0.1.0-SNAPSHOT.jar /etc/cassandra/
+COPY --from=builder /build/management-api-server/target/datastax-mgmtapi-server-0.1.0-SNAPSHOT.jar /opt/mgmtapi/
+COPY --from=builder /build/management-api-shim-3.x/target/datastax-mgmtapi-shim-3.x-0.1.0-SNAPSHOT.jar /opt/mgmtapi/
+COPY --from=builder /build/management-api-shim-4.x/target/datastax-mgmtapi-shim-4.x-0.1.0-SNAPSHOT.jar /opt/mgmtapi/
+
+ENV TINI_VERSION v0.18.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TARGETARCH} /tini
+RUN chmod +x /tini
+
+RUN set -eux; \
+  rm -fr /etc/apt/sources.list.d/*; \
+  rm -rf /var/lib/apt/lists/*; \
+  apt-get update; \
+  apt-get install -y --no-install-recommends wget iproute2; \
+  rm -rf /var/lib/apt/lists/*
+
+ENV MCAC_VERSION 0.1.7
+ADD https://github.com/datastax/metric-collector-for-apache-cassandra/releases/download/v${MCAC_VERSION}/datastax-mcac-agent-${MCAC_VERSION}.tar.gz /opt/mcac-agent.tar.gz
+RUN mkdir /opt/mcac-agent && tar zxvf /opt/mcac-agent.tar.gz -C /opt/mcac-agent --strip-components 1 && rm /opt/mcac-agent.tar.gz
+
+# backwards compat with upstream ENTRYPOINT
+COPY scripts/docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \
+  ln -sf /usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh
+
+EXPOSE 9103
+EXPOSE 8080
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["mgmtapi"]
+
+
+# NOTE: Presently, OSS doesn't have official Cassandra 4.0 builds on dockerhub
+# and our build at datastax/cassandra:4.0 is not a multiarch image like the
+# official ones. Once one of those issues is fixed, the following targets can
+# be used to build Cassandra 4.0 with Management API.
+
+# FROM --platform=linux/amd64 cassandra:4.0 as oss40-amd64
+
+# FROM --platform=linux/arm64 cassandra:4.0 as oss40-arm64
+# # Netty arm64 epoll support was not added until 4.1.50 (https://github.com/netty/netty/pull/9804)
+# # Only replace this dependency for arm64 to avoid regressions
+# RUN rm /opt/cassandra/lib/netty-all-*.jar
+# COPY --from=netty4150 /root/.m2/repository/io/netty/netty-all/4.1.50.Final/netty-all-4.1.50.Final.jar /opt/cassandra/lib/netty-all-4.1.50.Final.jar
+
+# FROM oss40-${TARGETARCH} as oss40
+
+# ARG TARGETARCH
+
+# COPY --from=builder /build/management-api-common/target/datastax-mgmtapi-common-0.1.0-SNAPSHOT.jar /etc/cassandra/
+# COPY --from=builder /build/management-api-agent/target/datastax-mgmtapi-agent-0.1.0-SNAPSHOT.jar /etc/cassandra/
+# COPY --from=builder /build/management-api-server/target/datastax-mgmtapi-server-0.1.0-SNAPSHOT.jar /opt/mgmtapi/
+# COPY --from=builder /build/management-api-shim-3.x/target/datastax-mgmtapi-shim-3.x-0.1.0-SNAPSHOT.jar /opt/mgmtapi/
+# COPY --from=builder /build/management-api-shim-4.x/target/datastax-mgmtapi-shim-4.x-0.1.0-SNAPSHOT.jar /opt/mgmtapi/
+
+# ENV TINI_VERSION v0.18.0
+# ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TARGETARCH} /tini
+# RUN chmod +x /tini
+
+# RUN set -eux; \
+#   apt-get update; \
+#   apt-get install -y --no-install-recommends wget iproute2; \
+#   rm -rf /var/lib/apt/lists/*
+
+# ENV MCAC_VERSION 0.1.7
+# ADD https://github.com/datastax/metric-collector-for-apache-cassandra/releases/download/v${MCAC_VERSION}/datastax-mcac-agent-${MCAC_VERSION}.tar.gz /opt/mcac-agent.tar.gz
+# RUN mkdir /opt/mcac-agent && tar zxvf /opt/mcac-agent.tar.gz -C /opt/mcac-agent --strip-components 1 && rm /opt/mcac-agent.tar.gz
+
+# # backwards compat with upstream ENTRYPOINT
+# COPY scripts/docker-entrypoint.sh /usr/local/bin/
+# RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \
+#   ln -sf /usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh
+
+# EXPOSE 9103
+# EXPOSE 8080
+
+# ENTRYPOINT ["/docker-entrypoint.sh"]
+# CMD ["mgmtapi"]
diff --git a/management-api-common/pom.xml b/management-api-common/pom.xml
@@ -14,7 +14,7 @@
     <properties>
         <slf4j.version>1.7.25</slf4j.version>
         <logback.version>1.2.3</logback.version>
-        <netty.version>4.1.45.Final</netty.version>
+        <netty.version>4.1.50.Final</netty.version>
         <cassandra.version>3.11.5</cassandra.version>
     </properties>
 

diff --git a/management-api-server/pom.xml b/management-api-server/pom.xml
@@ -27,7 +27,7 @@
         <airline.version>2.7.0</airline.version>
         <jaxrs.version>2.0.8</jaxrs.version>
         <resteasy.version>4.0.0.Final</resteasy.version>
-        <netty.version>4.1.45.Final</netty.version>
+        <netty.version>4.1.50.Final</netty.version>
         <driver.version>4.4.0</driver.version>
         <docker.java.version>3.1.2</docker.java.version>
         <cassandra.version>3.11.5</cassandra.version>

diff --git a/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java b/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java
@@ -66,7 +66,23 @@ public String getIpAddressOfContainer()
     public void startManagementAPI(String version, List<String> envVars)
     {
         File baseDir = new File(System.getProperty("dockerFileRoot","."));
-        File dockerFile = Paths.get(baseDir.getPath(), "Dockerfile-" + version).toFile();
+        File dockerFile;
+        String target;
+        boolean useBuildx;
+
+        if ("3_11".equals(version))
+        {
+            dockerFile = Paths.get(baseDir.getPath(), "Dockerfile-oss").toFile();
+            target = "oss311";
+            useBuildx = true;
+        }
+        else
+        {
+            dockerFile = Paths.get(baseDir.getPath(), "Dockerfile-" + version).toFile();
+            target = null;
+            useBuildx = false;
+        }
+
         if (!dockerFile.exists())
             throw new RuntimeException("Missing " + dockerFile.getAbsolutePath());
 
@@ -79,7 +95,7 @@ public void startManagementAPI(String version, List<String> envVars)
         if (envVars != null)
             envList.addAll(envVars);
 
-        this.container = startDocker(dockerFile, baseDir, name, ports, volumeDescList, envList, cmdList);
+        this.container = startDocker(dockerFile, baseDir, target, name, ports, volumeDescList, envList, cmdList, useBuildx);
 
         waitForPort("localhost",8080, Duration.ofMillis(50000), logger, false);
     }
@@ -159,7 +175,29 @@ public boolean started()
         return container != null;
     }
 
-    private String startDocker(File dockerFile, File baseDir, String name, List<Integer> ports, List<String> volumeDescList, List<String> envList, List<String> cmdList)
+    private void buildImageWithBuildx(File dockerFile, File baseDir, String target, String name) throws Exception {
+        ProcessBuilder pb = new ProcessBuilder("docker", "buildx", "build",
+            "--load",
+            "--tag", name,
+            "--file", dockerFile.getPath(),
+            "--target", target,
+            "--platform", "linux/amd64",
+            baseDir.getPath());
+
+        Process p = pb.inheritIO().start();
+        int exitCode = p.waitFor();
+
+        if (exitCode != 0)
+        {
+            throw new Exception("Command '" + String.join(" ", pb.command() + "' return error code: " + exitCode));
+        }
+    }
+    private String startDocker(File dockerFile, File baseDir, String target, String name, List<Integer> ports, List<String> volumeDescList, List<String> envList, List<String> cmdList)
+    {
+        return startDocker(dockerFile, baseDir, target, name, ports, volumeDescList, envList, cmdList, false);
+    }
+
+    private String startDocker(File dockerFile, File baseDir, String target, String name, List<Integer> ports, List<String> volumeDescList, List<String> envList, List<String> cmdList, boolean useBuildx)
     {
         ListContainersCmd listContainersCmd = dockerClient.listContainersCmd();
         listContainersCmd.getFilters().put("name", Arrays.asList(name));
@@ -200,12 +238,27 @@ public void onNext(BuildResponseItem item)
         };
 
         logger.info("Building container: " + name + " from " + dockerFile);
-        dockerClient.buildImageCmd()
+        if (useBuildx)
+        {
+            try
+            {
+                buildImageWithBuildx(dockerFile, baseDir, target, name);
+            }
+            catch (Exception e)
+            {
+                e.printStackTrace();
+                logger.error("Unable to build image");
+            }
+        }
+        else
+        {
+            dockerClient.buildImageCmd()
                 .withBaseDirectory(baseDir)
                 .withDockerfile(dockerFile)
                 .withTags(Sets.newHashSet(name))
                 .exec(callback)
                 .awaitImageId();
+        }
 
         List<ExposedPort> tcpPorts = new ArrayList<>();
         List<PortBinding> portBindings = new ArrayList<>();

diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh
@@ -40,6 +40,11 @@ _sed-in-place() {
 	rm "$tempFile"
 }
 
+_metrics_collector_supported() {
+	# currently, metrics collector does not work on arm64
+	[ "$(uname -m)" != "aarch64" ]
+}
+
 if [ "$1" = 'mgmtapi' ]; then
 	echo "Starting Management API"
 
@@ -55,7 +60,7 @@ if [ "$1" = 'mgmtapi' ]; then
 	# 2. We don't wan't operator or configbuilder to care so much about the version number or
 	#    the fact this jar even exists.
 
-	if ! grep -qxF "JVM_OPTS=\"\$JVM_OPTS -javaagent:/opt/mcac-agent/lib/datastax-mcac-agent.jar\"" < /etc/cassandra/cassandra-env.sh ; then
+	if _metrics_collector_supported && ! grep -qxF "JVM_OPTS=\"\$JVM_OPTS -javaagent:/opt/mcac-agent/lib/datastax-mcac-agent.jar\"" < /etc/cassandra/cassandra-env.sh ; then
     # ensure newline at end of file
 		echo "" >> /etc/cassandra/cassandra-env.sh
     echo "JVM_OPTS=\"\$JVM_OPTS -javaagent:/opt/mcac-agent/lib/datastax-mcac-agent.jar\"" >> /etc/cassandra/cassandra-env.sh