From 52564256ff3f692d24a494715a0815ac0004c5a3 Mon Sep 17 00:00:00 2001 From: John Trimble Date: Wed, 26 Aug 2020 11:28:30 -0700 Subject: [PATCH] Experimental arm64 support for Cassandra 3.11.6 (#36) * Update netty for epoll arm64 support * Disable metrics monitor for arm64 since we have no binary for it * Update build for testing multiarch * Fix workflow typo * Update branch for testing * Try only adding aarch64 epoll binary instead of replace all netty * Revert "Try only adding aarch64 epoll binary instead of replace all netty" This reverts commit 8039c7c12ce3c8a1019dc017a6728c601c0237c3. * Fix release workflow for merging * Update ci to build 3.11 same as for release * Revert "Update ci to build 3.11 same as for release" This reverts commit fd5139beefdb99e328619e51aa71849384390520. * Revert "Revert "Update ci to build 3.11 same as for release"" This reverts commit 1a40ff11324c77890d68d75413ef08aa1a2e72df. * Update ci to build for amd64 and arm64 * Add use of buildx in integration test * Temporarily only test 3.11 * Revert "Temporarily only test 3.11" This reverts commit e64005dff167f83f12e9474e88dd3435bb2dc945. * Update to use 3.11.7 explicitly --- .github/workflows/ci.yaml | 11 +- .github/workflows/docker-release.yaml | 33 +++-- Dockerfile-oss | 115 ++++++++++++++++++ management-api-common/pom.xml | 2 +- management-api-server/pom.xml | 2 +- .../mgmtapi/helpers/DockerHelper.java | 61 +++++++++- scripts/docker-entrypoint.sh | 7 +- 7 files changed, 212 insertions(+), 19 deletions(-) create mode 100644 Dockerfile-oss diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d3bf81a1..16467be5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,6 +18,11 @@ jobs: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 + - name: Setup Buildx + id: buildx + uses: crazy-max/ghaction-docker-buildx@v3 + with: + version: latest - name: Build Management API in docker run: | cat < ~/.m2/settings.xml @@ -39,7 +44,11 @@ jobs: cp ~/.m2/settings.xml settings.xml docker build -t management-api-for-dse-builder -f ./Dockerfile-build-dse ./ docker tag management-api-for-dse-builder management-api-for-apache-cassandra-builder - docker build -t mgmtapi-3_11 -f Dockerfile-3_11 . + docker buildx build \ + --tag mgmtapi-3_11 \ + --file Dockerfile-oss \ + --target oss311 \ + --platform linux/amd64,linux/arm64 . docker build -t mgmtapi-4_0 -f Dockerfile-4_0 . docker build -t mgmtapi-dse-68 -f Dockerfile-dse-68 . - name: Build with Maven diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml index 322457b4..2762c296 100644 --- a/.github/workflows/docker-release.yaml +++ b/.github/workflows/docker-release.yaml @@ -6,7 +6,7 @@ on: - 'v*.*.*' jobs: - build: + build-dse: runs-on: ubuntu-latest steps: - uses: actions/checkout@master @@ -32,14 +32,14 @@ jobs: cp ~/.m2/settings.xml settings.xml docker build -t management-api-for-dse-builder -f ./Dockerfile-build-dse ./ docker tag management-api-for-dse-builder management-api-for-apache-cassandra-builder - - name: Publish 3.11 to Registry + - name: Publish DSE 6.8 to Registry uses: elgohr/Publish-Docker-Github-Action@master with: - name: datastax/cassandra-mgmtapi-3_11_7 + name: datastax/dse-mgmtapi-6_8 username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} tag_names: true - dockerfile: Dockerfile-3_11 + dockerfile: Dockerfile-dse-68 - name: Publish 4.0 to Registry uses: elgohr/Publish-Docker-Github-Action@master with: @@ -48,11 +48,22 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} tag_names: true dockerfile: Dockerfile-4_0 - - name: Publish DSE 6.8 to Registry - uses: elgohr/Publish-Docker-Github-Action@master + build-oss: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Setup Buildx + id: buildx + uses: crazy-max/ghaction-docker-buildx@v3 with: - name: datastax/dse-mgmtapi-6_8 - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - tag_names: true - dockerfile: Dockerfile-dse-68 + version: latest + - name: Login to Docker Hub + run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + - name: Publish 3.11 to Registry + run: | + RELEASE_VERSION="${GITHUB_REF##*/}" + docker buildx build --push \ + --tag datastax/cassandra-mgmtapi-3_11_7:$RELEASE_VERSION \ + --file Dockerfile-oss \ + --target oss311 \ + --platform linux/amd64,linux/arm64 . diff --git a/Dockerfile-oss b/Dockerfile-oss new file mode 100644 index 00000000..d9589873 --- /dev/null +++ b/Dockerfile-oss @@ -0,0 +1,115 @@ +FROM --platform=$BUILDPLATFORM maven:3.6.3-jdk-8-slim as builder + +WORKDIR /build + +COPY pom.xml ./ +COPY management-api-agent/pom.xml ./management-api-agent/pom.xml +COPY management-api-common/pom.xml ./management-api-common/pom.xml +COPY management-api-server/pom.xml ./management-api-server/pom.xml +COPY management-api-shim-3.x/pom.xml ./management-api-shim-3.x/pom.xml +COPY management-api-shim-4.x/pom.xml ./management-api-shim-4.x/pom.xml +# this duplicates work done in the next steps, but this should provide +# a solid cache layer that only gets reset on pom.xml changes +RUN mvn -q -ff -T 1C install && rm -rf target + +COPY management-api-agent ./management-api-agent +COPY management-api-common ./management-api-common +COPY management-api-server ./management-api-server +COPY management-api-shim-3.x ./management-api-shim-3.x +COPY management-api-shim-4.x ./management-api-shim-4.x +RUN mvn -q -ff package -DskipTests + +FROM --platform=$BUILDPLATFORM maven:3.6.3-jdk-8-slim as netty4150 +RUN mvn dependency:get -DgroupId=io.netty -DartifactId=netty-all -Dversion=4.1.50.Final -Dtransitive=false + +FROM --platform=linux/amd64 cassandra:3.11.7 as oss311-amd64 + +FROM --platform=linux/arm64 cassandra:3.11.7 as oss311-arm64 +# Netty arm64 epoll support was not added until 4.1.50 (https://github.com/netty/netty/pull/9804) +# Only replace this dependency for arm64 to avoid regressions +RUN rm /opt/cassandra/lib/netty-all-*.jar +COPY --from=netty4150 /root/.m2/repository/io/netty/netty-all/4.1.50.Final/netty-all-4.1.50.Final.jar /opt/cassandra/lib/netty-all-4.1.50.Final.jar + +FROM oss311-${TARGETARCH} as oss311 + +ARG TARGETARCH + +COPY --from=builder /build/management-api-common/target/datastax-mgmtapi-common-0.1.0-SNAPSHOT.jar /etc/cassandra/ +COPY --from=builder /build/management-api-agent/target/datastax-mgmtapi-agent-0.1.0-SNAPSHOT.jar /etc/cassandra/ +COPY --from=builder /build/management-api-server/target/datastax-mgmtapi-server-0.1.0-SNAPSHOT.jar /opt/mgmtapi/ +COPY --from=builder /build/management-api-shim-3.x/target/datastax-mgmtapi-shim-3.x-0.1.0-SNAPSHOT.jar /opt/mgmtapi/ +COPY --from=builder /build/management-api-shim-4.x/target/datastax-mgmtapi-shim-4.x-0.1.0-SNAPSHOT.jar /opt/mgmtapi/ + +ENV TINI_VERSION v0.18.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TARGETARCH} /tini +RUN chmod +x /tini + +RUN set -eux; \ + rm -fr /etc/apt/sources.list.d/*; \ + rm -rf /var/lib/apt/lists/*; \ + apt-get update; \ + apt-get install -y --no-install-recommends wget iproute2; \ + rm -rf /var/lib/apt/lists/* + +ENV MCAC_VERSION 0.1.7 +ADD https://github.com/datastax/metric-collector-for-apache-cassandra/releases/download/v${MCAC_VERSION}/datastax-mcac-agent-${MCAC_VERSION}.tar.gz /opt/mcac-agent.tar.gz +RUN mkdir /opt/mcac-agent && tar zxvf /opt/mcac-agent.tar.gz -C /opt/mcac-agent --strip-components 1 && rm /opt/mcac-agent.tar.gz + +# backwards compat with upstream ENTRYPOINT +COPY scripts/docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ + ln -sf /usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh + +EXPOSE 9103 +EXPOSE 8080 + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["mgmtapi"] + + +# NOTE: Presently, OSS doesn't have official Cassandra 4.0 builds on dockerhub +# and our build at datastax/cassandra:4.0 is not a multiarch image like the +# official ones. Once one of those issues is fixed, the following targets can +# be used to build Cassandra 4.0 with Management API. + +# FROM --platform=linux/amd64 cassandra:4.0 as oss40-amd64 + +# FROM --platform=linux/arm64 cassandra:4.0 as oss40-arm64 +# # Netty arm64 epoll support was not added until 4.1.50 (https://github.com/netty/netty/pull/9804) +# # Only replace this dependency for arm64 to avoid regressions +# RUN rm /opt/cassandra/lib/netty-all-*.jar +# COPY --from=netty4150 /root/.m2/repository/io/netty/netty-all/4.1.50.Final/netty-all-4.1.50.Final.jar /opt/cassandra/lib/netty-all-4.1.50.Final.jar + +# FROM oss40-${TARGETARCH} as oss40 + +# ARG TARGETARCH + +# COPY --from=builder /build/management-api-common/target/datastax-mgmtapi-common-0.1.0-SNAPSHOT.jar /etc/cassandra/ +# COPY --from=builder /build/management-api-agent/target/datastax-mgmtapi-agent-0.1.0-SNAPSHOT.jar /etc/cassandra/ +# COPY --from=builder /build/management-api-server/target/datastax-mgmtapi-server-0.1.0-SNAPSHOT.jar /opt/mgmtapi/ +# COPY --from=builder /build/management-api-shim-3.x/target/datastax-mgmtapi-shim-3.x-0.1.0-SNAPSHOT.jar /opt/mgmtapi/ +# COPY --from=builder /build/management-api-shim-4.x/target/datastax-mgmtapi-shim-4.x-0.1.0-SNAPSHOT.jar /opt/mgmtapi/ + +# ENV TINI_VERSION v0.18.0 +# ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TARGETARCH} /tini +# RUN chmod +x /tini + +# RUN set -eux; \ +# apt-get update; \ +# apt-get install -y --no-install-recommends wget iproute2; \ +# rm -rf /var/lib/apt/lists/* + +# ENV MCAC_VERSION 0.1.7 +# ADD https://github.com/datastax/metric-collector-for-apache-cassandra/releases/download/v${MCAC_VERSION}/datastax-mcac-agent-${MCAC_VERSION}.tar.gz /opt/mcac-agent.tar.gz +# RUN mkdir /opt/mcac-agent && tar zxvf /opt/mcac-agent.tar.gz -C /opt/mcac-agent --strip-components 1 && rm /opt/mcac-agent.tar.gz + +# # backwards compat with upstream ENTRYPOINT +# COPY scripts/docker-entrypoint.sh /usr/local/bin/ +# RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ +# ln -sf /usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh + +# EXPOSE 9103 +# EXPOSE 8080 + +# ENTRYPOINT ["/docker-entrypoint.sh"] +# CMD ["mgmtapi"] diff --git a/management-api-common/pom.xml b/management-api-common/pom.xml index 66eccaa6..7e4636e6 100644 --- a/management-api-common/pom.xml +++ b/management-api-common/pom.xml @@ -14,7 +14,7 @@ 1.7.25 1.2.3 - 4.1.45.Final + 4.1.50.Final 3.11.5 diff --git a/management-api-server/pom.xml b/management-api-server/pom.xml index 237fb704..e49f88c2 100644 --- a/management-api-server/pom.xml +++ b/management-api-server/pom.xml @@ -27,7 +27,7 @@ 2.7.0 2.0.8 4.0.0.Final - 4.1.45.Final + 4.1.50.Final 4.4.0 3.1.2 3.11.5 diff --git a/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java b/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java index 8de6d2de..c12ac414 100644 --- a/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java +++ b/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java @@ -66,7 +66,23 @@ public String getIpAddressOfContainer() public void startManagementAPI(String version, List envVars) { File baseDir = new File(System.getProperty("dockerFileRoot",".")); - File dockerFile = Paths.get(baseDir.getPath(), "Dockerfile-" + version).toFile(); + File dockerFile; + String target; + boolean useBuildx; + + if ("3_11".equals(version)) + { + dockerFile = Paths.get(baseDir.getPath(), "Dockerfile-oss").toFile(); + target = "oss311"; + useBuildx = true; + } + else + { + dockerFile = Paths.get(baseDir.getPath(), "Dockerfile-" + version).toFile(); + target = null; + useBuildx = false; + } + if (!dockerFile.exists()) throw new RuntimeException("Missing " + dockerFile.getAbsolutePath()); @@ -79,7 +95,7 @@ public void startManagementAPI(String version, List envVars) if (envVars != null) envList.addAll(envVars); - this.container = startDocker(dockerFile, baseDir, name, ports, volumeDescList, envList, cmdList); + this.container = startDocker(dockerFile, baseDir, target, name, ports, volumeDescList, envList, cmdList, useBuildx); waitForPort("localhost",8080, Duration.ofMillis(50000), logger, false); } @@ -159,7 +175,29 @@ public boolean started() return container != null; } - private String startDocker(File dockerFile, File baseDir, String name, List ports, List volumeDescList, List envList, List cmdList) + private void buildImageWithBuildx(File dockerFile, File baseDir, String target, String name) throws Exception { + ProcessBuilder pb = new ProcessBuilder("docker", "buildx", "build", + "--load", + "--tag", name, + "--file", dockerFile.getPath(), + "--target", target, + "--platform", "linux/amd64", + baseDir.getPath()); + + Process p = pb.inheritIO().start(); + int exitCode = p.waitFor(); + + if (exitCode != 0) + { + throw new Exception("Command '" + String.join(" ", pb.command() + "' return error code: " + exitCode)); + } + } + private String startDocker(File dockerFile, File baseDir, String target, String name, List ports, List volumeDescList, List envList, List cmdList) + { + return startDocker(dockerFile, baseDir, target, name, ports, volumeDescList, envList, cmdList, false); + } + + private String startDocker(File dockerFile, File baseDir, String target, String name, List ports, List volumeDescList, List envList, List cmdList, boolean useBuildx) { ListContainersCmd listContainersCmd = dockerClient.listContainersCmd(); listContainersCmd.getFilters().put("name", Arrays.asList(name)); @@ -200,12 +238,27 @@ public void onNext(BuildResponseItem item) }; logger.info("Building container: " + name + " from " + dockerFile); - dockerClient.buildImageCmd() + if (useBuildx) + { + try + { + buildImageWithBuildx(dockerFile, baseDir, target, name); + } + catch (Exception e) + { + e.printStackTrace(); + logger.error("Unable to build image"); + } + } + else + { + dockerClient.buildImageCmd() .withBaseDirectory(baseDir) .withDockerfile(dockerFile) .withTags(Sets.newHashSet(name)) .exec(callback) .awaitImageId(); + } List tcpPorts = new ArrayList<>(); List portBindings = new ArrayList<>(); diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 7627a34f..505613ce 100755 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -40,6 +40,11 @@ _sed-in-place() { rm "$tempFile" } +_metrics_collector_supported() { + # currently, metrics collector does not work on arm64 + [ "$(uname -m)" != "aarch64" ] +} + if [ "$1" = 'mgmtapi' ]; then echo "Starting Management API" @@ -55,7 +60,7 @@ if [ "$1" = 'mgmtapi' ]; then # 2. We don't wan't operator or configbuilder to care so much about the version number or # the fact this jar even exists. - if ! grep -qxF "JVM_OPTS=\"\$JVM_OPTS -javaagent:/opt/mcac-agent/lib/datastax-mcac-agent.jar\"" < /etc/cassandra/cassandra-env.sh ; then + if _metrics_collector_supported && ! grep -qxF "JVM_OPTS=\"\$JVM_OPTS -javaagent:/opt/mcac-agent/lib/datastax-mcac-agent.jar\"" < /etc/cassandra/cassandra-env.sh ; then # ensure newline at end of file echo "" >> /etc/cassandra/cassandra-env.sh echo "JVM_OPTS=\"\$JVM_OPTS -javaagent:/opt/mcac-agent/lib/datastax-mcac-agent.jar\"" >> /etc/cassandra/cassandra-env.sh