diff --git a/internal/locate/replica_selector.go b/internal/locate/replica_selector.go
index bc811903a..901592ca3 100644
--- a/internal/locate/replica_selector.go
+++ b/internal/locate/replica_selector.go
@@ -543,6 +543,9 @@ func (s *replicaSelector) onServerIsBusy(
 	backoffErr := errors.Errorf("server is busy, ctx: %v", ctx)
 	if s.canFastRetry() {
 		s.addPendingBackoff(store, retry.BoTiKVServerBusy, backoffErr)
+		if s.target != nil {
+			s.target.addFlag(serverIsBusyFlag)
+		}
 		return true, nil
 	}
 	err = bo.Backoff(retry.BoTiKVServerBusy, backoffErr)
diff --git a/internal/locate/replica_selector_test.go b/internal/locate/replica_selector_test.go
index 2b9793e3d..d928b0a02 100644
--- a/internal/locate/replica_selector_test.go
+++ b/internal/locate/replica_selector_test.go
@@ -362,7 +362,7 @@ func TestReplicaReadAccessPathByCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store2, replica-read: false, stale-read: true}",
 				"{addr: store1, replica-read: false, stale-read: false}",
-				"{addr: store2, replica-read: true, stale-read: false}"},
+				"{addr: store2, replica-read: false, stale-read: true}"},
 			respErr:         "",
 			respRegionError: nil,
 			backoffCnt:      0,
@@ -404,7 +404,7 @@ func TestReplicaReadAccessPathByCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store2, replica-read: false, stale-read: true}",
 				"{addr: store1, replica-read: false, stale-read: false}",
-				"{addr: store3, replica-read: true, stale-read: false}"}, // store2 has DeadLineExceededErr, so don't retry store2 even it is new leader.
+				"{addr: store3, replica-read: false, stale-read: true}"}, // store2 has DeadLineExceededErr, so don't retry store2 even it is new leader.
 			respErr:         "",
 			respRegionError: nil,
 			backoffCnt:      0,
@@ -504,8 +504,8 @@ func TestReplicaReadAccessPathByCase(t *testing.T) {
 		expect: &accessPathResult{
 			accessPath: []string{
 				"{addr: store1, replica-read: false, stale-read: true}",
-				"{addr: store2, replica-read: true, stale-read: false}",
-				"{addr: store3, replica-read: true, stale-read: false}"},
+				"{addr: store2, replica-read: false, stale-read: true}",
+				"{addr: store3, replica-read: false, stale-read: true}"},
 			respErr:         "",
 			respRegionError: fakeEpochNotMatch,
 			backoffCnt:      1,
@@ -548,7 +548,7 @@ func TestReplicaReadAccessPathByCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store3, replica-read: false, stale-read: true}",
 				"{addr: store1, replica-read: false, stale-read: false}",
-				"{addr: store2, replica-read: true, stale-read: false}"},
+				"{addr: store2, replica-read: false, stale-read: true}"},
 			respErr:         "",
 			respRegionError: nil,
 			backoffCnt:      0,
@@ -568,8 +568,8 @@ func TestReplicaReadAccessPathByCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store2, replica-read: false, stale-read: true}",
 				"{addr: store1, replica-read: false, stale-read: false}", // try leader with leader read.
-				"{addr: store2, replica-read: true, stale-read: false}",
-				"{addr: store3, replica-read: true, stale-read: false}",
+				"{addr: store2, replica-read: false, stale-read: true}",
+				"{addr: store3, replica-read: false, stale-read: true}",
 			},
 			respErr:         "",
 			respRegionError: fakeEpochNotMatch,
@@ -590,8 +590,8 @@ func TestReplicaReadAccessPathByCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store1, replica-read: false, stale-read: true}",
 				"{addr: store2, replica-read: false, stale-read: false}", // try leader with leader read.
-				"{addr: store3, replica-read: true, stale-read: false}",
-				"{addr: store1, replica-read: true, stale-read: false}",
+				"{addr: store3, replica-read: false, stale-read: true}",
+				"{addr: store1, replica-read: false, stale-read: true}",
 			},
 			respErr:         "",
 			respRegionError: nil,
@@ -611,8 +611,8 @@ func TestReplicaReadAccessPathByCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store1, replica-read: false, stale-read: true}",
 				"{addr: store2, replica-read: false, stale-read: false}", // try leader with leader read.
-				"{addr: store3, replica-read: true, stale-read: false}",
-				"{addr: store1, replica-read: true, stale-read: false}",
+				"{addr: store3, replica-read: false, stale-read: true}",
+				"{addr: store1, replica-read: false, stale-read: true}",
 			},
 			respErr:         "",
 			respRegionError: fakeEpochNotMatch,
@@ -654,8 +654,8 @@ func TestReplicaReadAccessPathByCase(t *testing.T) {
 		expect: &accessPathResult{
 			accessPath: []string{
 				"{addr: store1, replica-read: false, stale-read: true}",
-				"{addr: store2, replica-read: true, stale-read: false}",
-				"{addr: store3, replica-read: true, stale-read: false}",
+				"{addr: store2, replica-read: false, stale-read: true}",
+				"{addr: store3, replica-read: false, stale-read: true}",
 			},
 			respErr:         "",
 			respRegionError: nil,
@@ -1052,9 +1052,16 @@ func TestReplicaReadAccessPathByBasicCase(t *testing.T) {
 							backoff = []string{}
 						}
 						if staleRead {
-							accessPath = []string{
-								"{addr: store1, replica-read: false, stale-read: true}",
-								"{addr: store2, replica-read: true, stale-read: false}",
+							if tp == ServerIsBusyErr || tp == ServerIsBusyWithEstimatedWaitMsErr {
+								accessPath = []string{
+									"{addr: store1, replica-read: false, stale-read: true}",
+									"{addr: store2, replica-read: false, stale-read: true}",
+								}
+							} else {
+								accessPath = []string{
+									"{addr: store1, replica-read: false, stale-read: true}",
+									"{addr: store2, replica-read: true, stale-read: false}",
+								}
 							}
 						}
 					default:
@@ -1918,8 +1925,8 @@ func TestReplicaReadAccessPathByStaleReadCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store2, replica-read: false, stale-read: true}",
 				"{addr: store1, replica-read: false, stale-read: false}", // try leader with leader read.
-				"{addr: store2, replica-read: true, stale-read: false}",
-				"{addr: store3, replica-read: true, stale-read: false}",
+				"{addr: store2, replica-read: false, stale-read: true}",
+				"{addr: store3, replica-read: false, stale-read: true}",
 			},
 			respErr:         "",
 			respRegionError: fakeEpochNotMatch,
@@ -1940,8 +1947,8 @@ func TestReplicaReadAccessPathByStaleReadCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store1, replica-read: false, stale-read: true}",
 				"{addr: store2, replica-read: false, stale-read: false}", // try leader with leader read.
-				"{addr: store3, replica-read: true, stale-read: false}",
-				"{addr: store1, replica-read: true, stale-read: false}",
+				"{addr: store3, replica-read: false, stale-read: true}",
+				"{addr: store1, replica-read: false, stale-read: true}",
 			},
 			respErr:         "",
 			respRegionError: fakeEpochNotMatch,
@@ -2009,7 +2016,7 @@ func TestReplicaReadAccessPathByStaleReadCase(t *testing.T) {
 			accessPath: []string{
 				"{addr: store2, replica-read: false, stale-read: true}",
 				"{addr: store1, replica-read: false, stale-read: false}",
-				"{addr: store2, replica-read: false, stale-read: true}",
+				"{addr: store2, replica-read: true, stale-read: false}",
 				"{addr: store3, replica-read: true, stale-read: false}",
 			},
 			respErr:         "",
@@ -2118,7 +2125,7 @@ func TestReplicaReadAccessPathByStaleReadCase(t *testing.T) {
 				accessPath: []string{
 					"{addr: store2, replica-read: false, stale-read: true}",
 					"{addr: store1, replica-read: false, stale-read: false}",
-					"{addr: store3, replica-read: false, stale-read: true}",
+					"{addr: store3, replica-read: true, stale-read: false}",
 				},
 				respErr:         "",
 				respRegionError: nil,
@@ -2137,7 +2144,7 @@ func TestReplicaReadAccessPathByStaleReadCase(t *testing.T) {
 			beforeRun: func() { /* don't resetStoreState */ },
 			expect: &accessPathResult{
 				accessPath: []string{
-					"{addr: store3, replica-read: false, stale-read: true}",
+					"{addr: store3, replica-read: true, stale-read: false}",
 				},
 				respErr:         "",
 				respRegionError: fakeEpochNotMatch,
@@ -2666,7 +2673,11 @@ func TestReplicaReadAvoidSlowStore(t *testing.T) {
 			if expectedFirstStore == 3 {
 				// Retry on store 2 which is a follower.
 				// Stale-read mode falls back to replica-read mode.
-				expectedSecondPath = "{addr: store2, replica-read: true, stale-read: false}"
+				if staleRead {
+					expectedSecondPath = "{addr: store2, replica-read: false, stale-read: true}"
+				} else {
+					expectedSecondPath = "{addr: store2, replica-read: true, stale-read: false}"
+				}
 			} else {
 				if staleRead {
 					// Retry in leader read mode