From 4c7a99bb97828402fbd095df7fe7c11b4ab9e66a Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 29 Mar 2024 16:06:33 +0900 Subject: [PATCH 1/4] Exclude non_voter from quorum calculation --- src/ra_server.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ra_server.erl b/src/ra_server.erl index 8f96d52f..dd10ea28 100644 --- a/src/ra_server.erl +++ b/src/ra_server.erl @@ -2827,6 +2827,8 @@ query_indexes(#{cfg := #cfg{id = Id}, query_index := QueryIndex}) -> maps:fold(fun (PeerId, _, Acc) when PeerId == Id -> Acc; + (_K, #{voter_status := #{membership := non_voter}}, Acc) -> + Acc; (_K, #{voter_status := #{membership := promotable}}, Acc) -> Acc; (_K, #{query_index := Idx}, Acc) -> @@ -2839,6 +2841,8 @@ match_indexes(#{cfg := #cfg{id = Id}, {LWIdx, _} = ra_log:last_written(Log), maps:fold(fun (PeerId, _, Acc) when PeerId == Id -> Acc; + (_K, #{voter_status := #{membership := non_voter}}, Acc) -> + Acc; (_K, #{voter_status := #{membership := promotable}}, Acc) -> Acc; (_K, #{match_index := Idx}, Acc) -> @@ -3149,7 +3153,9 @@ required_quorum(Cluster) -> count_voters(Cluster) -> maps:fold( - fun (_, #{voter_status := #{membership := promotable}}, Count) -> + fun (_, #{voter_status := #{membership := non_voter}}, Count) -> + Count; + (_, #{voter_status := #{membership := promotable}}, Count) -> Count; (_, _, Count) -> Count + 1 From a6edc0568aa67dae7a4e02c266dcc3cb1f874f1b Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 29 Mar 2024 16:34:47 +0900 Subject: [PATCH 2/4] Add non_voter test --- test/ra_2_SUITE.erl | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/ra_2_SUITE.erl b/test/ra_2_SUITE.erl index 817dd292..a1d71a18 100644 --- a/test/ra_2_SUITE.erl +++ b/test/ra_2_SUITE.erl @@ -86,6 +86,7 @@ init_per_testcase(TestCase, Config) -> ServerName2 = list_to_atom(atom_to_list(TestCase) ++ "2"), ServerName3 = list_to_atom(atom_to_list(TestCase) ++ "3"), ServerName4 = list_to_atom(atom_to_list(TestCase) ++ "4"), + ServerName5 = list_to_atom(atom_to_list(TestCase) ++ "5"), [{test_case, TestCase}, {modname, TestCase}, {cluster_name, TestCase}, @@ -96,7 +97,9 @@ init_per_testcase(TestCase, Config) -> {uid3, atom_to_binary(ServerName3, utf8)}, {server_id3, {ServerName3, node()}}, {uid4, atom_to_binary(ServerName4, utf8)}, - {server_id4, {ServerName4, node()}} + {server_id4, {ServerName4, node()}}, + {uid5, atom_to_binary(ServerName5, utf8)}, + {server_id5, {ServerName5, node()}} | Config]. enqueue(Server, Msg) -> @@ -714,15 +717,28 @@ force_start_follower_as_single_member_nonvoter(Config) -> ServerId4 = ?config(server_id4, Config), UId4 = ?config(uid4, Config), Conf4 = conf(ClusterName, UId4, ServerId4, PrivDir, [ServerId3]), - {ok, _, _} = ra:add_member(ServerId3, #{id => ServerId4, membership => promotable, uid => <<"test">>}), + {ok, _, _} = ra:add_member(ServerId3, #{id => ServerId4, membership => promotable, uid => <<"test4">>}), %% the membership has changed but member not running yet %% it is nonvoter and does not affect quorum size {ok, _, _} = ra:process_command(ServerId3, {enq, banana}), %% start new member - ok = ra:start_server(?SYS, Conf4#{membership => promotable, uid => <<"test">>}), + ok = ra:start_server(?SYS, Conf4#{membership => promotable, uid => <<"test4">>}), {ok, _, ServerId3} = ra:members(ServerId4), ok = enqueue(ServerId3, msg3), + %% add a non-voter member + ServerId5 = ?config(server_id5, Config), + UId5 = ?config(uid5, Config), + Conf5 = conf(ClusterName, UId5, ServerId5, PrivDir, [ServerId3]), + {ok, _, _} = ra:add_member(ServerId3, #{id => ServerId5, membership => non_voter, uid => <<"test5">>}), + %% the membership has changed but member not running yet + %% it is nonvoter and does not affect quorum size + {ok, _, _} = ra:process_command(ServerId3, {enq, banana}), + %% start new member + ok = ra:start_server(?SYS, Conf5#{membership => non_voter, uid => <<"test5">>}), + {ok, _, ServerId3} = ra:members(ServerId5), + ok = enqueue(ServerId3, msg4), + ok. initial_members_query(Config) -> From 164669593757d465bb5c7300bc93b6183721db2b Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 29 Mar 2024 16:47:41 +0900 Subject: [PATCH 3/4] Update test comment --- test/ra_2_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ra_2_SUITE.erl b/test/ra_2_SUITE.erl index a1d71a18..99572ce5 100644 --- a/test/ra_2_SUITE.erl +++ b/test/ra_2_SUITE.erl @@ -713,7 +713,7 @@ force_start_follower_as_single_member_nonvoter(Config) -> {ok, [_], ServerId3} = ra:members(ServerId3), ok = enqueue(ServerId3, msg2), - %% add a member + %% add a promotable member ServerId4 = ?config(server_id4, Config), UId4 = ?config(uid4, Config), Conf4 = conf(ClusterName, UId4, ServerId4, PrivDir, [ServerId3]), From 924c024577f09963a60a1e9e75a332ee19841611 Mon Sep 17 00:00:00 2001 From: Takeru Ohta Date: Fri, 26 Apr 2024 15:54:32 +0900 Subject: [PATCH 4/4] Update condition to decide non-voter --- src/ra_server.erl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/ra_server.erl b/src/ra_server.erl index dd10ea28..8470c135 100644 --- a/src/ra_server.erl +++ b/src/ra_server.erl @@ -2827,9 +2827,7 @@ query_indexes(#{cfg := #cfg{id = Id}, query_index := QueryIndex}) -> maps:fold(fun (PeerId, _, Acc) when PeerId == Id -> Acc; - (_K, #{voter_status := #{membership := non_voter}}, Acc) -> - Acc; - (_K, #{voter_status := #{membership := promotable}}, Acc) -> + (_K, #{voter_status := #{membership := Membership}}, Acc) when Membership =/= voter -> Acc; (_K, #{query_index := Idx}, Acc) -> [Idx | Acc] @@ -2841,9 +2839,7 @@ match_indexes(#{cfg := #cfg{id = Id}, {LWIdx, _} = ra_log:last_written(Log), maps:fold(fun (PeerId, _, Acc) when PeerId == Id -> Acc; - (_K, #{voter_status := #{membership := non_voter}}, Acc) -> - Acc; - (_K, #{voter_status := #{membership := promotable}}, Acc) -> + (_K, #{voter_status := #{membership := Membership}}, Acc) when Membership =/= voter -> Acc; (_K, #{match_index := Idx}, Acc) -> [Idx | Acc] @@ -3153,9 +3149,7 @@ required_quorum(Cluster) -> count_voters(Cluster) -> maps:fold( - fun (_, #{voter_status := #{membership := non_voter}}, Count) -> - Count; - (_, #{voter_status := #{membership := promotable}}, Count) -> + fun (_, #{voter_status := #{membership := Membership}}, Count) when Membership =/= voter -> Count; (_, _, Count) -> Count + 1