diff --git a/exe/ruby-lsp b/exe/ruby-lsp
index 59b0270ef..431a9024b 100755
--- a/exe/ruby-lsp
+++ b/exe/ruby-lsp
@@ -84,7 +84,7 @@ if ENV["BUNDLE_GEMFILE"].nil?
     "bundle"
   end
 
-  exit exec(env, "#{base_bundle} exec ruby-lsp #{original_args.join(" ")}")
+  exit exec(env, "#{base_bundle} exec ruby-lsp #{original_args.join(" ")}".strip)
 end
 
 $LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
diff --git a/lib/ruby_lsp/setup_bundler.rb b/lib/ruby_lsp/setup_bundler.rb
index eb049c95d..3f29c842c 100644
--- a/lib/ruby_lsp/setup_bundler.rb
+++ b/lib/ruby_lsp/setup_bundler.rb
@@ -186,9 +186,22 @@ def run_bundle_install(bundle_gemfile = @gemfile)
         env["BUNDLE_PATH"] = File.expand_path(env["BUNDLE_PATH"], @project_path)
       end
 
+      path_parts = if Gem.win_platform?
+        ENV["Path"] || ENV["PATH"] || ENV["path"] || ""
+      else
+        ENV["PATH"] || ""
+      end.split(File::PATH_SEPARATOR)
+
+      bin_directory = File.expand_path("bin", @project_path)
+      bundle_binstub = File.join(@project_path, "bin", "bundle")
+
       # If there's a Bundler version locked, then we need to use that one to run bundle commands, so that the composed
       # lockfile is also locked to the same version. This avoids Bundler restarts on version mismatches
-      base_bundle = if @bundler_version
+      base_bundle = if File.exist?(bundle_binstub) && path_parts.any? do |path|
+        File.expand_path(path, @project_path) == bin_directory
+      end
+        bundle_binstub
+      elsif @bundler_version
         env["BUNDLER_VERSION"] = @bundler_version.to_s
         install_bundler_if_needed
         "bundle _#{@bundler_version}_"
diff --git a/project-words b/project-words
index 52fa38f32..452631a20 100644
--- a/project-words
+++ b/project-words
@@ -8,6 +8,7 @@ binread
 Bizt
 Bizw
 bufnr
+binstub
 byteslice
 codepoint
 codepoints
diff --git a/test/integration_test.rb b/test/integration_test.rb
index 4ae7ae003..664754daa 100644
--- a/test/integration_test.rb
+++ b/test/integration_test.rb
@@ -4,11 +4,9 @@
 require "test_helper"
 
 class IntegrationTest < Minitest::Test
-  def setup
+  def test_ruby_lsp_doctor_works
     skip("CI only") unless ENV["CI"]
-  end
 
-  def test_ruby_lsp_doctor_works
     in_isolation do
       system("bundle exec ruby-lsp --doctor")
       assert_equal(0, $CHILD_STATUS)
@@ -16,12 +14,74 @@ def test_ruby_lsp_doctor_works
   end
 
   def test_ruby_lsp_check_works
+    skip("CI only") unless ENV["CI"]
+
     in_isolation do
       system("bundle exec ruby-lsp-check")
       assert_equal(0, $CHILD_STATUS)
     end
   end
 
+  def test_adds_bundler_version_as_part_of_exec_command
+    Dir.mktmpdir do |dir|
+      Dir.chdir(dir) do
+        File.write(File.join(dir, "Gemfile"), <<~GEMFILE)
+          source "https://rubygems.org"
+          gem "ruby-lsp", path: "#{Bundler.root}"
+        GEMFILE
+
+        Bundler.with_unbundled_env do
+          capture_subprocess_io do
+            system("bundle install")
+
+            Object.any_instance.expects(:exec).with do |env, command|
+              env.key?("BUNDLE_GEMFILE") &&
+                env.key?("BUNDLER_VERSION") &&
+                /bundle _[\d\.]+_ exec ruby-lsp/.match?(command)
+            end.once.raises(StandardError.new("stop"))
+
+            # We raise intentionally to avoid continuing running the executable
+            assert_raises(StandardError) do
+              load(Gem.bin_path("ruby-lsp", "ruby-lsp"))
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def test_avoids_bundler_version_if_local_bin_is_in_path
+    Dir.mktmpdir do |dir|
+      Dir.chdir(dir) do
+        File.write(File.join(dir, "Gemfile"), <<~GEMFILE)
+          source "https://rubygems.org"
+          gem "ruby-lsp", path: "#{Bundler.root}"
+        GEMFILE
+
+        FileUtils.mkdir(File.join(dir, "bin"))
+        FileUtils.touch(File.join(dir, "bin", "bundle"))
+
+        Bundler.with_unbundled_env do
+          capture_subprocess_io do
+            system("bundle install")
+
+            Object.any_instance.expects(:exec).with do |env, command|
+              env.key?("BUNDLE_GEMFILE") &&
+                !env.key?("BUNDLER_VERSION") &&
+                "bundle exec ruby-lsp" == command
+            end.once.raises(StandardError.new("stop"))
+
+            ENV["PATH"] = "./bin#{File::PATH_SEPARATOR}#{ENV["PATH"]}"
+            # We raise intentionally to avoid continuing running the executable
+            assert_raises(StandardError) do
+              load(Gem.bin_path("ruby-lsp", "ruby-lsp"))
+            end
+          end
+        end
+      end
+    end
+  end
+
   private
 
   def in_isolation(&block)