diff --git a/exe/ruby-lsp b/exe/ruby-lsp
index b5c7bbb59..07c93302d 100755
--- a/exe/ruby-lsp
+++ b/exe/ruby-lsp
@@ -63,13 +63,27 @@ if ENV["BUNDLE_GEMFILE"].nil?
     exit(78)
   end
 
-  base_bundle = if env["BUNDLER_VERSION"]
+  path_parts = if Gem.win_platform?
+    ENV["Path"] || ENV["PATH"] || ENV["path"] || ""
+  else
+    ENV["PATH"] || ""
+  end.split(File::PATH_SEPARATOR)
+
+  bin_directory = File.expand_path("bin")
+  bundle_binstub = File.join("bin", "bundle")
+
+  # If there's a `bin/bundle` binstub and the local `bin` directory is added to the PATH, then we cannot concatenate the
+  # version as part of the bundle command because the binstub doesn't support it. In all other cases, if there's a
+  # locked Bundler version we want to use that to launch the LSP
+  base_bundle = if File.exist?(bundle_binstub) && path_parts.any? { |path| File.expand_path(path) == bin_directory }
+    bundle_binstub
+  elsif env["BUNDLER_VERSION"]
     "bundle _#{env["BUNDLER_VERSION"]}_"
   else
     "bundle"
   end
 
-  exit exec(env, "#{base_bundle} exec ruby-lsp #{original_args.join(" ")}")
+  exec(env, "#{base_bundle} exec ruby-lsp #{original_args.join(" ")}".strip)
 end
 
 require "ruby_lsp/load_sorbet"
diff --git a/test/integration_test.rb b/test/integration_test.rb
index 4ae7ae003..9859e40e4 100644
--- a/test/integration_test.rb
+++ b/test/integration_test.rb
@@ -4,11 +4,9 @@
 require "test_helper"
 
 class IntegrationTest < Minitest::Test
-  def setup
+  def test_ruby_lsp_doctor_works
     skip("CI only") unless ENV["CI"]
-  end
 
-  def test_ruby_lsp_doctor_works
     in_isolation do
       system("bundle exec ruby-lsp --doctor")
       assert_equal(0, $CHILD_STATUS)
@@ -16,12 +14,74 @@ def test_ruby_lsp_doctor_works
   end
 
   def test_ruby_lsp_check_works
+    skip("CI only") unless ENV["CI"]
+
     in_isolation do
       system("bundle exec ruby-lsp-check")
       assert_equal(0, $CHILD_STATUS)
     end
   end
 
+  def test_adds_bundler_version_as_part_of_exec_command
+    Dir.mktmpdir do |dir|
+      Dir.chdir(dir) do
+        File.write(File.join(dir, "Gemfile"), <<~GEMFILE)
+          source "https://rubygems.org"
+          gem "ruby-lsp", path: "#{Bundler.root}"
+        GEMFILE
+
+        Bundler.with_unbundled_env do
+          capture_subprocess_io do
+            system("bundle install")
+
+            Object.any_instance.expects(:exec).with do |env, command|
+              assert(env.key?("BUNDLE_GEMFILE"))
+              assert(env.key?("BUNDLER_VERSION"))
+              assert_match(/bundle _[\d\.]+_ exec ruby-lsp/, command)
+            end.once.raises(StandardError.new("stop"))
+
+            # We raise intentionally to avoid continuing running the executable
+            assert_raises(StandardError) do
+              load(Gem.bin_path("ruby-lsp", "ruby-lsp"))
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def test_avoids_bundler_version_if_local_bin_is_in_path
+    Dir.mktmpdir do |dir|
+      Dir.chdir(dir) do
+        File.write(File.join(dir, "Gemfile"), <<~GEMFILE)
+          source "https://rubygems.org"
+          gem "ruby-lsp", path: "#{Bundler.root}"
+        GEMFILE
+
+        FileUtils.mkdir(File.join(dir, "bin"))
+        FileUtils.touch(File.join(dir, "bin", "bundle"))
+
+        Bundler.with_unbundled_env do
+          capture_subprocess_io do
+            system("bundle install")
+
+            Object.any_instance.expects(:exec).with do |env, command|
+              assert(env.key?("BUNDLE_GEMFILE"))
+              assert(env.key?("BUNDLER_VERSION"))
+              assert_equal("#{File.join("bin", "bundle")} exec ruby-lsp", command)
+            end.once.raises(StandardError.new("stop"))
+
+            ENV["PATH"] = "./bin#{File::PATH_SEPARATOR}#{ENV["PATH"]}"
+            # We raise intentionally to avoid continuing running the executable
+            assert_raises(StandardError) do
+              load(Gem.bin_path("ruby-lsp", "ruby-lsp"))
+            end
+          end
+        end
+      end
+    end
+  end
+
   private
 
   def in_isolation(&block)