From 4a89c184688401ed8c5129f341491f9a321ff292 Mon Sep 17 00:00:00 2001
From: Jie Yao <jyao3@ebay.com>
Date: Tue, 10 Sep 2024 04:41:51 -0700
Subject: [PATCH] fix get leader bug

---
 src/include/homestore/replication/repl_dev.h   |  7 +++++++
 src/lib/replication/repl_dev/raft_repl_dev.cpp |  2 ++
 src/lib/replication/repl_dev/raft_repl_dev.h   |  1 +
 src/lib/replication/repl_dev/solo_repl_dev.h   |  1 +
 src/tests/test_raft_repl_dev.cpp               | 12 ++++++------
 5 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/include/homestore/replication/repl_dev.h b/src/include/homestore/replication/repl_dev.h
index 9965ada5d..bbdee98f6 100644
--- a/src/include/homestore/replication/repl_dev.h
+++ b/src/include/homestore/replication/repl_dev.h
@@ -451,6 +451,13 @@ class ReplDev {
         }
     }
 
+    /**
+     * Check if there is live leader in the current cluster.
+     *
+     * @return `true` if live leader exists.
+     */
+    virtual bool is_leader_alive() const = 0;
+
 protected:
     shared< ReplDevListener > m_listener;
 };
diff --git a/src/lib/replication/repl_dev/raft_repl_dev.cpp b/src/lib/replication/repl_dev/raft_repl_dev.cpp
index 4c358aa4e..5a014c47a 100644
--- a/src/lib/replication/repl_dev/raft_repl_dev.cpp
+++ b/src/lib/replication/repl_dev/raft_repl_dev.cpp
@@ -870,6 +870,8 @@ AsyncReplResult<> RaftReplDev::become_leader() {
 
 bool RaftReplDev::is_leader() const { return m_repl_svc_ctx->is_raft_leader(); }
 
+bool RaftReplDev::is_leader_alive() const { return m_repl_svc_ctx->_server->is_leader_alive(); }
+
 replica_id_t RaftReplDev::get_leader_id() const {
     static replica_id_t empty_uuid = boost::uuids::nil_uuid();
     auto leader = m_repl_svc_ctx->raft_leader_id();
diff --git a/src/lib/replication/repl_dev/raft_repl_dev.h b/src/lib/replication/repl_dev/raft_repl_dev.h
index 41594b528..a46ed21a2 100644
--- a/src/lib/replication/repl_dev/raft_repl_dev.h
+++ b/src/lib/replication/repl_dev/raft_repl_dev.h
@@ -171,6 +171,7 @@ class RaftReplDev : public ReplDev,
     void set_last_commit_lsn(repl_lsn_t lsn) { m_commit_upto_lsn.store(lsn); }
     bool is_destroy_pending() const;
     bool is_destroyed() const;
+    bool is_leader_alive() const override;
     Clock::time_point destroyed_time() const { return m_destroyed_time; }
 
     //////////////// Accessor/shortcut methods ///////////////////////
diff --git a/src/lib/replication/repl_dev/solo_repl_dev.h b/src/lib/replication/repl_dev/solo_repl_dev.h
index cddb94856..eaf7cb0dd 100644
--- a/src/lib/replication/repl_dev/solo_repl_dev.h
+++ b/src/lib/replication/repl_dev/solo_repl_dev.h
@@ -49,6 +49,7 @@ class SoloReplDev : public ReplDev {
 
     AsyncReplResult<> become_leader() override { return make_async_error(ReplServiceError::OK); }
     bool is_leader() const override { return true; }
+    bool is_leader_alive() const override { return true; }
     replica_id_t get_leader_id() const override { return m_group_id; }
     std::vector< peer_info > get_replication_status() const override {
         return std::vector< peer_info >{peer_info{.id_ = m_group_id, .replication_idx_ = 0, .last_succ_resp_us_ = 0}};
diff --git a/src/tests/test_raft_repl_dev.cpp b/src/tests/test_raft_repl_dev.cpp
index 88aa0ef5e..49f2b3863 100644
--- a/src/tests/test_raft_repl_dev.cpp
+++ b/src/tests/test_raft_repl_dev.cpp
@@ -522,12 +522,12 @@ class RaftReplDevTest : public testing::Test {
 
     void write_on_leader(uint32_t num_entries, bool wait_for_commit = true, shared< TestReplicatedDB > db = nullptr) {
         do {
-            auto leader_uuid = dbs_[0]->repl_dev()->get_leader_id();
-
-            if (leader_uuid.is_nil()) {
+            // we can not use dbs_[0]->repl_dev()->get_leader_id() since this will only get leader from the config file.
+            // if leader changes , this will return an wrong leader;
+            if (!dbs_[0]->repl_dev()->is_leader_alive()) {
                 LOGINFO("Waiting for leader to be elected");
                 std::this_thread::sleep_for(std::chrono::milliseconds{500});
-            } else if (leader_uuid == g_helper->my_replica_id()) {
+            } else if (dbs_[0]->repl_dev()->is_leader()) {
                 LOGINFO("Writing {} entries since I am the leader my_uuid={}", num_entries,
                         boost::uuids::to_string(g_helper->my_replica_id()));
                 auto const block_size = SISL_OPTIONS["block_size"].as< uint32_t >();
@@ -541,8 +541,8 @@ class RaftReplDevTest : public testing::Test {
                 if (wait_for_commit) { g_helper->runner().execute().get(); }
                 break;
             } else {
-                LOGINFO("{} entries were written on the leader_uuid={} my_uuid={}", num_entries,
-                        boost::uuids::to_string(leader_uuid), boost::uuids::to_string(g_helper->my_replica_id()));
+                LOGINFO("{} entries were written on the  my_uuid={}", num_entries,
+                        boost::uuids::to_string(g_helper->my_replica_id()));
                 break;
             }
         } while (true);