Merge pull request #242 from keynslug/fix/autocluster-eager-stop

fix(autocluster): avoid stopping autocluster too eagerly
emqx · Dec 20, 2024 · 1281e1a · 1281e1a
2 parents f11855f + b7ef3f0
commit 1281e1a
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 14 deletions.
diff --git a/src/ekka.appup.src b/src/ekka.appup.src
@@ -1,46 +1,59 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"0.8.1.13", []},
+  [{"0.8.1.14",
+    [{load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]}]},
+   {"0.8.1.13",
+    [{load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]}]},
    {"0.8.1.12",
-    [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
+    [{load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
+     {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]}]},
    {"0.8.1.11",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]}]},
    {"0.8.1.10",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]}]},
    {"0.8.1.9",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]},
    {"0.8.1.8",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]},
    {"0.8.1.7",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
      {load_module,ekka_locker,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]},
    {<<"0\\.8\\.1\\.[3-6]">>,
     [{load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
@@ -50,6 +63,7 @@
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]},
    {"0.8.1.2",
     [{load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_ring,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_dns,brutal_purge,soft_purge,[]},
      {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
@@ -62,6 +76,7 @@
    {"0.8.1.1",
     [{load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_dns,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_ring,brutal_purge,soft_purge,[]},
      {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_httpc,brutal_purge,soft_purge,[]},
@@ -73,6 +88,7 @@
    {"0.8.1",
     [{load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_dns,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_ring,brutal_purge,soft_purge,[]},
      {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
@@ -81,46 +97,59 @@
      {load_module,ekka_httpc,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]}],
-  [{"0.8.1.13", []},
+  [{"0.8.1.14",
+    [{load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]}]},
+   {"0.8.1.13",
+    [{load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]}]},
    {"0.8.1.12",
-    [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
+    [{load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
+     {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]}]},
    {"0.8.1.11",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]}]},
    {"0.8.1.10",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]}]},
    {"0.8.1.9",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]},
    {"0.8.1.8",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]},
    {"0.8.1.7",
     [{load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
      {load_module,ekka_locker,brutal_purge,soft_purge,[]},
      {load_module,ekka_mnesia,brutal_purge,soft_purge,[]},
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]},
    {<<"0\\.8\\.1\\.[3-6]">>,
     [{load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},
      {load_module,ekka_autoheal,brutal_purge,soft_purge,[]},
@@ -130,6 +159,7 @@
      {load_module,ekka_dist,brutal_purge,soft_purge,[]}]},
    {"0.8.1.2",
     [{load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_ring,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_dns,brutal_purge,soft_purge,[]},
      {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
@@ -142,6 +172,7 @@
    {"0.8.1.1",
     [{load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_dns,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_ring,brutal_purge,soft_purge,[]},
      {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_httpc,brutal_purge,soft_purge,[]},
@@ -153,6 +184,7 @@
    {"0.8.1",
     [{load_module,ekka_cluster_strategy,brutal_purge,soft_purge,[]},
      {load_module,ekka_cluster_dns,brutal_purge,soft_purge,[]},
+     {load_module,ekka_cluster_static,brutal_purge,soft_purge,[]},
      {load_module,ekka_ring,brutal_purge,soft_purge,[]},
      {load_module,ekka_node_monitor,brutal_purge,soft_purge,[]},
      {load_module,ekka_autocluster,brutal_purge,soft_purge,[]},

diff --git a/src/ekka_autocluster.erl b/src/ekka_autocluster.erl
@@ -173,6 +173,9 @@ strategy_module(Strategy) ->
 discover_and_join(Mod, Options) ->
     ?tp(ekka_autocluster_discover_and_join, #{mod => Mod}),
     try ekka_cluster_strategy:discover(Mod, Options) of
+        {ok, []} ->
+            ?LOG(info, "Discovery result is empty", []),
+            error;
         {ok, Nodes} ->
             {AliveNodes, DeadNodes} = lists:partition(
                                         fun ekka_node:is_aliving/1,
@@ -202,8 +205,6 @@ discover_and_join(Mod, Options) ->
     end.
 
 -spec maybe_join([node()]) -> ignore | ok | {error, _}.
-maybe_join([]) ->
-    ignore;
 maybe_join(Nodes0) ->
     Nodes = lists:usort(Nodes0),
     KnownNodes = lists:usort(ekka_mnesia:cluster_nodes(all)),
@@ -212,16 +213,13 @@ maybe_join(Nodes0) ->
             ?LOG(info, "all discovered nodes already in cluster; ignoring", []),
             ignore;
         false ->
-            OldestNode = find_oldest_node(Nodes),
-            ?LOG(info, "joining with ~p", [OldestNode]),
-            join_with(OldestNode)
+            join_with(find_oldest_node(Nodes -- [node()]))
     end.
 
 join_with(false) ->
-    ignore;
-join_with(Node) when Node =:= node() ->
-    ignore;
+    {error, no_member};
 join_with(Node) ->
+    ?LOG(info, "joining with ~p", [Node]),
     ekka_cluster:join(Node).
 
 find_oldest_node([Node]) ->
@@ -241,4 +239,5 @@ find_oldest_node(Nodes) ->
 
 log_error(Format, {error, Reason}) ->
     ?LOG(error, Format ++ " error: ~p", [Reason]);
-log_error(_Format, _Ok) -> ok.
+log_error(_Format, _Ok) ->
+    ok.
diff --git a/src/ekka_cluster_static.erl b/src/ekka_cluster_static.erl
@@ -26,7 +26,7 @@
         ]).
 
 discover(Options) ->
-    {ok, proplists:get_value(seeds, Options, [])}.
+    {ok, proplists:get_value(seeds, Options, [node()])}.
 
 lock(_Options) ->
     ignore.

diff --git a/test/ekka_autocluster_SUITE.erl b/test/ekka_autocluster_SUITE.erl
@@ -147,6 +147,26 @@ t_autocluster_via_dns(_Config) ->
         ok = ekka_ct:stop_slave(N1)
     end.
 
+t_autocluster_via_dns_laggy(_Config) ->
+    N1 = ekka_ct:start_slave(ekka, n1),
+    ?check_trace(
+      #{timetrap => 15000},
+      try
+          ok = ekka_ct:wait_running(N1),
+          ok = set_app_env(N1, {dns, [{name, "thishostnameshouldnotexist"},
+                                      {app, "ct"}
+                                     ]}),
+          {ok, {ok, #{}}} = ?wait_async_action(
+            rpc:call(N1, ekka, autocluster, []),
+            #{?snk_kind := ekka_maybe_run_app_again}),
+          ok = set_app_env(N1, {dns, ?DNS_OPTIONS}),
+          ok = wait_for_node(N1),
+          ok = ekka:force_leave(N1)
+      after
+          ok = ekka_ct:stop_slave(N1)
+      end,
+      []).
+
 %%--------------------------------------------------------------------
 %% Autocluster via 'etcd' strategy