Mixed version testing in bazel (rabbitmq#3200)

Unlike with gnu make, mixed version testing with bazel uses a package-generic-unix for the secondary umbrella rather than the source. This brings the benefit of being able to mixed version test releases built with older erlang versions (even though all nodes will run under the single version given to bazel) This introduces new test labels, adding a `-mixed` suffix for every existing test. They can be skipped if necessary with `--test_tag_filters` (see the github actions workflow for an example) As part of the change, it is now possible to run an old release of rabbit with rabbitmq_run rule, such as: `bazel run @rabbitmq-server-generic-unix-3.8.17//:rabbitmq-run run-broker`
Anwma · Jul 19, 2021 · d6399bb · d6399bb
1 parent 0f4cf27
commit d6399bb
Show file tree

Hide file tree

Showing 16 changed files with 316 additions and 77 deletions.
diff --git a/.github/workflows/test-mixed-versions.yaml b/.github/workflows/test-mixed-versions.yaml
@@ -0,0 +1,82 @@
+name: Test Mixed Version Clusters
+on: push
+jobs:
+  test-mixed-versions:
+    name: Test (Mixed Version Cluster)
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        erlang_version:
+        - "23"
+        - "24"
+    timeout-minutes: 120
+    steps:
+    - name: CHECKOUT REPOSITORY
+      uses: actions/[email protected]
+    - name: CONFIGURE BAZEL
+      run: |
+        echo "${{ secrets.BUILDBUDDY_CERT }}" > buildbuddy-cert.pem
+        echo "${{ secrets.BUILDBUDDY_KEY }}" > buildbuddy-key.pem
+        cat << EOF >> user.bazelrc
+          build:buildbuddy --tls_client_certificate=buildbuddy-cert.pem
+          build:buildbuddy --tls_client_key=buildbuddy-key.pem
+
+          build:buildbuddy --build_metadata=ROLE=CI
+          build:buildbuddy --build_metadata=VISIBILITY=PUBLIC
+          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-${{ matrix.erlang_version }}
+        EOF
+    #! - name: Setup tmate session
+    #!   uses: mxschmitt/action-tmate@v3
+    - name: RUN TESTS
+      run: |
+        bazelisk test //... \
+          --config=rbe-${{ matrix.erlang_version }} \
+          --test_tag_filters=mixed-version-cluster,-exclusive,-aws \
+          --verbose_failures
+  test-exclusive-mixed-versions:
+    name: Test (Exclusive Tests with Mixed Version Cluster)
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        erlang_version:
+        - "23"
+        - "24"
+    timeout-minutes: 60
+    steps:
+    - name: CHECKOUT REPOSITORY
+      uses: actions/[email protected]
+    - name: CONFIGURE OTP & ELIXIR
+      uses: erlef/[email protected]
+      with:
+        otp-version: ${{ matrix.erlang_version }}
+        elixir-version: 1.11.4
+    - name: CONFIGURE BAZEL
+      run: |
+        ERLANG_HOME="$(dirname $(dirname $(which erl)))"
+        ELIXIR_HOME="$(dirname $(dirname $(which iex)))"
+        echo "${{ secrets.BUILDBUDDY_CERT }}" > buildbuddy-cert.pem
+        echo "${{ secrets.BUILDBUDDY_KEY }}" > buildbuddy-key.pem
+        cat << EOF >> user.bazelrc
+          build:buildbuddy --tls_client_certificate=buildbuddy-cert.pem
+          build:buildbuddy --tls_client_key=buildbuddy-key.pem
+
+          build:buildbuddy --build_metadata=ROLE=CI
+          build:buildbuddy --build_metadata=VISIBILITY=PRIVATE
+          build:buildbuddy --remote_instance_name=buildbuddy-io/buildbuddy/ci-exclusive-${{ matrix.erlang_version }}
+
+          build --@bazel-erlang//:erlang_version=${{ matrix.erlang_version }}
+          build --@bazel-erlang//:erlang_home=${ERLANG_HOME}
+          build --//:elixir_home=${ELIXIR_HOME}
+        EOF
+    #! - name: Setup tmate session
+    #!   uses: mxschmitt/action-tmate@v3
+    - name: RUN EXCLUSIVE TESTS
+      run: |
+        MIXED_EXCLUSIVE_TESTS=$(bazel query 'attr(tags, "mixed-version-cluster", attr(tags, "exclusive", tests(//...)))')
+        bazelisk test $MIXED_EXCLUSIVE_TESTS \
+          --config=buildbuddy \
+          --test_tag_filters=-aws \
+          --build_tests_only \
+          --test_env RABBITMQ_CT_HELPERS_DELETE_UNUSED_NODES=true \
+          --verbose_failures
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -32,7 +32,7 @@ jobs:
       run: |
         bazelisk test //... \
           --config=rbe-${{ matrix.erlang_version }} \
-          --test_tag_filters=-exclusive,-aws \
+          --test_tag_filters=-exclusive,-aws,-mixed-version-cluster \
           --verbose_failures
   test-exclusive:
     name: Test (Exclusive Tests)
@@ -75,7 +75,7 @@ jobs:
       run: |
         bazelisk test //... \
           --config=buildbuddy \
-          --test_tag_filters=exclusive,-aws \
+          --test_tag_filters=exclusive,-aws,-mixed-version-cluster \
           --build_tests_only \
           --test_env RABBITMQ_CT_HELPERS_DELETE_UNUSED_NODES=true \
           --verbose_failures
diff --git a/BUILD.package_generic_unix b/BUILD.package_generic_unix
@@ -0,0 +1,18 @@
+load("@//:rabbitmq_package_generic_unix.bzl", "rabbitmq_package_generic_unix")
+load("@//:rabbitmq_run.bzl", "rabbitmq_run")
+
+rabbitmq_package_generic_unix(
+    name = "broker-home",
+    sbin = glob(["sbin/*"]),
+    escript = glob(["escript/*"]),
+    plugins = [
+        "//plugins:standard_plugins",
+        "//plugins:inet_tcp_proxy_ez",
+    ],
+)
+
+rabbitmq_run(
+    name = "rabbitmq-run",
+    home = ":broker-home",
+    visibility = ["//visibility:public"],
+)
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
@@ -16,6 +16,19 @@ load("@io_buildbuddy_buildbuddy_toolchain//:rules.bzl", "buildbuddy")
 
 buildbuddy(name = "buildbuddy_toolchain")
 
+http_archive(
+    name = "rules_pkg",
+    sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()
+
 http_archive(
     name = "bazel-erlang",
     strip_prefix = "bazel-erlang-main",
@@ -51,3 +64,34 @@ git_repository(
 load("//deps/amqp10_client:activemq.bzl", "activemq_archive")
 
 activemq_archive()
+
+ADD_PLUGINS_DIR_BUILD_FILE = """set -euo pipefail
+
+cat << EOF > plugins/BUILD.bazel
+load("@rules_pkg//:pkg.bzl", "pkg_zip")
+
+pkg_zip(
+    name = "inet_tcp_proxy_ez",
+    package_dir = "inet_tcp_proxy/ebin",
+    srcs = [
+        "@inet_tcp_proxy//:bazel_erlang_lib",
+    ],
+    package_file_name = "inet_tcp_proxy.ez",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "standard_plugins",
+    srcs = glob(["*.ez"]),
+    visibility = ["//visibility:public"],
+)
+EOF
+"""
+
+http_archive(
+    name = "rabbitmq-server-generic-unix-3.8.18",
+    build_file = "@//:BUILD.package_generic_unix",
+    patch_cmds = [ADD_PLUGINS_DIR_BUILD_FILE],
+    strip_prefix = "rabbitmq_server-3.8.18",
+    urls = ["https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.8.18/rabbitmq-server-generic-unix-3.8.18.tar.xz"],
+)
diff --git a/deps/rabbit/test/maintenance_mode_SUITE.erl b/deps/rabbit/test/maintenance_mode_SUITE.erl
@@ -10,6 +10,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("amqp_client/include/amqp_client.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl").
 
 -compile(export_all).
 
@@ -247,15 +248,14 @@ quorum_queue_leadership_transfer(Config) ->
                    Config, Nodenames),
     case AllTheSame of
         true ->
-            rabbit_ct_helpers:await_condition(
-              fun () ->
-                      LocalLeaders = rabbit_ct_broker_helpers:rpc(
-                                       Config, A,
-                                       rabbit_amqqueue,
-                                       list_local_leaders,
-                                       []),
-                      length(LocalLeaders) =:= 0
-              end, 20000);
+            ?awaitMatch(
+               LocalLeaders when length(LocalLeaders) == 0,
+                                 rabbit_ct_broker_helpers:rpc(
+                                   Config, A,
+                                   rabbit_amqqueue,
+                                   list_local_leaders,
+                                   []),
+                                 20000);
         false ->
             ct:pal(
               ?LOW_IMPORTANCE,

diff --git a/deps/rabbit/test/rabbit_stream_queue_SUITE.erl b/deps/rabbit/test/rabbit_stream_queue_SUITE.erl
@@ -118,7 +118,17 @@ init_per_suite(Config0) ->
 end_per_suite(Config) ->
     rabbit_ct_helpers:run_teardown_steps(Config).
 
+init_per_group(cluster_size_3_parallel = Group, Config) ->
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "not mixed versions compatible"};
+        _ ->
+            init_per_group1(Group, Config)
+    end;
 init_per_group(Group, Config) ->
+    init_per_group1(Group, Config).
+
+init_per_group1(Group, Config) ->
     ClusterSize = case Group of
                       single_node -> 1;
                       single_node_parallel -> 1;

diff --git a/deps/rabbitmq_federation/test/queue_SUITE.erl b/deps/rabbitmq_federation/test/queue_SUITE.erl
@@ -117,10 +117,15 @@ init_per_group(cluster_size_1 = Group, Config) ->
       ]),
     init_per_group1(Group, Config1);
 init_per_group(cluster_size_2 = Group, Config) ->
-    Config1 = rabbit_ct_helpers:set_config(Config, [
-        {rmq_nodes_count, 2}
-      ]),
-    init_per_group1(Group, Config1).
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "not mixed versions compatible"};
+        _ ->
+            Config1 = rabbit_ct_helpers:set_config(Config, [
+                {rmq_nodes_count, 2}
+            ]),
+            init_per_group1(Group, Config1)
+    end.
 
 init_per_group1(Group, Config) ->
     SetupFederation = case Group of

diff --git a/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl b/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl
@@ -91,6 +91,10 @@ end_per_group(_, Config) ->
     Steps = Teardown0 ++ Teardown1,
     rabbit_ct_helpers:run_teardown_steps(Config, Steps).
 
+init_per_testcase(Testcase, Config)
+        when Testcase == is_quorum_critical_test
+            orelse Testcase == is_mirror_sync_critical_test ->
+    {skip, "not mixed versions compatible"};
 init_per_testcase(Testcase, Config) ->
     rabbit_ct_helpers:testcase_started(Config, Testcase).
 

diff --git a/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl b/deps/rabbitmq_management_agent/test/rabbit_mgmt_gc_SUITE.erl
@@ -78,13 +78,20 @@ end_per_group(_, Config) ->
     Config.
 
 init_per_testcase(quorum_queue_stats = Testcase, Config) ->
-    case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of
-        ok ->
-            rabbit_ct_helpers:testcase_started(Config, Testcase),
-            rabbit_ct_helpers:run_steps(
-              Config, rabbit_ct_client_helpers:setup_steps());
-        Skip ->
-            Skip
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            {skip, "not mixed versions compatible"};
+        _ ->
+            case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of
+                ok ->
+                    rabbit_ct_helpers:testcase_started(Config, Testcase),
+                    rabbit_ct_helpers:run_steps(
+                    Config, rabbit_ct_client_helpers:setup_steps());
+                {skip, _} = Skip ->
+                    Skip;
+                Other ->
+                    {skip, Other}
+            end
     end;
 init_per_testcase(Testcase, Config) ->
     rabbit_ct_helpers:testcase_started(Config, Testcase),

diff --git a/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl b/deps/rabbitmq_peer_discovery_aws/test/integration_SUITE.erl
@@ -42,30 +42,37 @@ groups() ->
     ].
 
 init_per_suite(Config) ->
-    inets:start(),
-    rabbit_ct_helpers:log_environment(),
-    Config1 = rabbit_ct_helpers:set_config(
-                Config, [
-                         {ecs_region, "eu-west-1"},
-                         {ecs_cluster_name, os:getenv("AWS_ECS_CLUSTER_NAME", "rabbitmq-peer-discovery-aws")},
-                         {ecs_profile_name, "rabbitmq-peer-discovery-aws-profile"},
-                         {ecs_instance_role, "ecs-peer-discovery-aws"},
-                         {ecs_cluster_size, ?CLUSTER_SIZE},
-                         {rabbitmq_default_user, "test"},
-                         {rabbitmq_default_pass, rabbit_ct_helpers:random_term_checksum()},
-                         {rabbitmq_erlang_cookie, rabbit_ct_helpers:random_term_checksum()}
-                        ]),
-    Config2 = rabbit_ct_helpers:register_teardown_step(Config1, fun aws_ecs_util:destroy_ecs_cluster/1),
-    rabbit_ct_helpers:run_steps(
-      Config2, [
-                fun rabbit_ct_helpers:init_skip_as_error_flag/1,
-                fun rabbit_ct_helpers:start_long_running_testsuite_monitor/1,
-                fun aws_ecs_util:ensure_aws_cli/1,
-                fun aws_ecs_util:ensure_ecs_cli/1,
-                fun aws_ecs_util:init_aws_credentials/1,
-                fun aws_ecs_util:ensure_rabbitmq_image/1,
-                fun aws_ecs_util:start_ecs_cluster/1
-               ]).
+    case rabbit_ct_helpers:is_mixed_versions() of
+        true ->
+            %% These test would like passed in mixed versions, but they won't
+            %% actually honor mixed versions as currently specified via env var
+            {skip, "not mixed versions compatible"};
+        _ ->
+            inets:start(),
+            rabbit_ct_helpers:log_environment(),
+            Config1 = rabbit_ct_helpers:set_config(
+                        Config, [
+                                {ecs_region, "eu-west-1"},
+                                {ecs_cluster_name, os:getenv("AWS_ECS_CLUSTER_NAME", "rabbitmq-peer-discovery-aws")},
+                                {ecs_profile_name, "rabbitmq-peer-discovery-aws-profile"},
+                                {ecs_instance_role, "ecs-peer-discovery-aws"},
+                                {ecs_cluster_size, ?CLUSTER_SIZE},
+                                {rabbitmq_default_user, "test"},
+                                {rabbitmq_default_pass, rabbit_ct_helpers:random_term_checksum()},
+                                {rabbitmq_erlang_cookie, rabbit_ct_helpers:random_term_checksum()}
+                                ]),
+            Config2 = rabbit_ct_helpers:register_teardown_step(Config1, fun aws_ecs_util:destroy_ecs_cluster/1),
+            rabbit_ct_helpers:run_steps(
+            Config2, [
+                        fun rabbit_ct_helpers:init_skip_as_error_flag/1,
+                        fun rabbit_ct_helpers:start_long_running_testsuite_monitor/1,
+                        fun aws_ecs_util:ensure_aws_cli/1,
+                        fun aws_ecs_util:ensure_ecs_cli/1,
+                        fun aws_ecs_util:init_aws_credentials/1,
+                        fun aws_ecs_util:ensure_rabbitmq_image/1,
+                        fun aws_ecs_util:start_ecs_cluster/1
+                    ])
+    end.
 
 end_per_suite(Config) ->
     rabbit_ct_helpers:run_teardown_steps(Config).