Use Bundler CLI directly and send errors to telemetry

exe/ruby-lsp-launcher

@@ -22,11 +22,15 @@ locked_bundler_version_file = File.join(".ruby-lsp", "locked_bundler_version")
 # Compose the Ruby LSP bundle in a forked process so that we can require gems without polluting the main process
 # `$LOAD_PATH` and `Gem.loaded_specs`. Windows doesn't support forking, so we need a separate path to support it
 pid = if Gem.win_platform?
-  spawn(Gem.ruby, File.expand_path("../lib/ruby_lsp/scripts/compose_bundle_windows.rb", __dir__), raw_initialize)
+  Process.spawn(
+    ENV.to_hash,
+    Gem.ruby,
+    File.expand_path("../lib/ruby_lsp/scripts/compose_bundle_windows.rb", __dir__),
+    raw_initialize,
+  )
 else
   fork do
-    $LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
-    require "ruby_lsp/scripts/compose_bundle"
+    require_relative "../lib/ruby_lsp/scripts/compose_bundle"
     compose(raw_initialize)
   end
 end
@@ -64,10 +68,15 @@ rescue StandardError => e
   Gem::Specification.find_by_name("ruby-lsp").activate
 end
 
-# Now that the bundle is set up, we can begin actually launching the server
+error_path = File.join(".ruby-lsp", "install_error")
 
-$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+install_error = if File.exist?(error_path)
+  Marshal.load(File.read(error_path))
+end
 
+# Now that the bundle is set up, we can begin actually launching the server. Note that `Bundler.setup` will have already
+# configured the load path using the version of the Ruby LSP present in the composed bundle. Do not push any Ruby LSP
+# paths into the load path manually or we may end up requiring the wrong version of the gem
 require "ruby_lsp/load_sorbet"
 require "ruby_lsp/internal"
 
@@ -91,7 +100,17 @@ $> = $stderr
 
 initialize_request = JSON.parse(raw_initialize, symbolize_names: true) if raw_initialize
 
-RubyLsp::Server.new(
-  setup_error: setup_error,
-  initialize_request: initialize_request,
-).start
+begin
+  RubyLsp::Server.new(
+    install_error: install_error,
+    setup_error: setup_error,
+    initialize_request: initialize_request,
+  ).start
+rescue ArgumentError
+  # If the launcher is booting an outdated version of the server, then the initializer doesn't accept a keyword splat
+  # and we already read the initialize request from the stdin pipe. In this case, we need to process the initialize
+  # request manually and then start the main loop
+  server = RubyLsp::Server.new
+  server.process_message(initialize_request)
+  server.start
+end

lib/ruby_lsp/base_server.rb

@@ -12,6 +12,7 @@ class BaseServer
     def initialize(**options)
       @test_mode = T.let(options[:test_mode], T.nilable(T::Boolean))
       @setup_error = T.let(options[:setup_error], T.nilable(StandardError))
+      @install_error = T.let(options[:install_error], T.nilable(StandardError))
       @writer = T.let(Transport::Stdio::Writer.new, Transport::Stdio::Writer)
       @reader = T.let(Transport::Stdio::Reader.new, Transport::Stdio::Reader)
       @incoming_queue = T.let(Thread::Queue.new, Thread::Queue)

lib/ruby_lsp/scripts/compose_bundle.rb

@@ -2,10 +2,10 @@
 # frozen_string_literal: true
 
 def compose(raw_initialize)
-  require "ruby_lsp/setup_bundler"
+  require_relative "../setup_bundler"
   require "json"
   require "uri"
-  require "core_ext/uri"
+  require_relative "../../core_ext/uri"
 
   initialize_request = JSON.parse(raw_initialize, symbolize_names: true)
   workspace_uri = initialize_request.dig(:params, :workspaceFolders, 0, :uri)

lib/ruby_lsp/server.rb

@@ -292,15 +292,21 @@ def run_initialize(message)
       global_state_notifications.each { |notification| send_message(notification) }
 
       if @setup_error
-        message = <<~MESSAGE
-          An error occurred while setting up Bundler. This may be due to a failure when installing dependencies.
-          The Ruby LSP will continue to run, but features related to the missing dependencies will be limited.
-
-          Error:
-          #{@setup_error.full_message}
-        MESSAGE
+        send_message(Notification.telemetry(
+          type: "error",
+          errorMessage: @setup_error.message,
+          errorClass: @setup_error.class,
+          stack: @setup_error.backtrace&.join("\n"),
+        ))
+      end
 
-        send_message(Notification.window_log_message(message, type: Constant::MessageType::ERROR))
+      if @install_error
+        send_message(Notification.telemetry(
+          type: "error",
+          errorMessage: @install_error.message,
+          errorClass: @install_error.class,
+          stack: @install_error.backtrace&.join("\n"),
+        ))
       end
     end
 

lib/ruby_lsp/setup_bundler.rb

@@ -3,6 +3,9 @@
 
 require "sorbet-runtime"
 require "bundler"
+require "bundler/cli"
+require "bundler/cli/install"
+require "bundler/cli/update"
 require "fileutils"
 require "pathname"
 require "digest"
@@ -48,6 +51,7 @@ def initialize(project_path, **options)
       @custom_lockfile = T.let(@custom_dir + (@lockfile&.basename || "Gemfile.lock"), Pathname)
       @lockfile_hash_path = T.let(@custom_dir + "main_lockfile_hash", Pathname)
       @last_updated_path = T.let(@custom_dir + "last_updated", Pathname)
+      @error_path = T.let(@custom_dir + "install_error", Pathname)
 
       dependencies, bundler_version = load_dependencies
       @dependencies = T.let(dependencies, T::Hash[String, T.untyped])
@@ -107,17 +111,19 @@ def setup!
     def custom_bundle_dependencies
       @custom_bundle_dependencies ||= T.let(
         begin
+          original_bundle_gemfile = ENV["BUNDLE_GEMFILE"]
+
           if @custom_lockfile.exist?
             ENV["BUNDLE_GEMFILE"] = @custom_gemfile.to_s
             Bundler::LockfileParser.new(@custom_lockfile.read).dependencies
           else
             {}
           end
+        ensure
+          ENV["BUNDLE_GEMFILE"] = original_bundle_gemfile
         end,
         T.nilable(T::Hash[String, T.untyped]),
       )
-    ensure
-      ENV.delete("BUNDLE_GEMFILE")
     end
 
     sig { void }
@@ -187,6 +193,52 @@ def run_bundle_install(bundle_gemfile = @gemfile)
         env["BUNDLE_PATH"] = File.expand_path(env["BUNDLE_PATH"], @project_path)
       end
 
+      return run_bundle_install_through_command(env) unless @launcher
+
+      # This same check happens conditionally when running through the command. For invoking the CLI directly, it's
+      # important that we ensure the Bundler version is set to avoid restarts
+      if @bundler_version
+        env["BUNDLER_VERSION"] = @bundler_version.to_s
+        install_bundler_if_needed
+      end
+
+      begin
+        run_bundle_install_directly(env)
+        # If no error occurred, then clear previous errors
+        @error_path.delete if @error_path.exist?
+        $stderr.puts("Ruby LSP> Composed bundle setup complete")
+      rescue => e
+        # Write the error object to a file so that we can read it from the parent process
+        @error_path.write(Marshal.dump(e))
+      end
+
+      env
+    end
+
+    sig { params(env: T::Hash[String, String]).returns(T::Hash[String, String]) }
+    def run_bundle_install_directly(env)
+      RubyVM::YJIT.enable if defined?(RubyVM::YJIT.enable)
+      T.unsafe(ENV).merge!(env)
+
+      unless should_bundle_update?
+        Bundler::CLI::Install.new({}).run
+        return env
+      end
+
+      # If any of `ruby-lsp`, `ruby-lsp-rails` or `debug` are not in the Gemfile, try to update them to the latest
+      # version
+      gems = []
+      gems << "ruby-lsp" unless @dependencies["ruby-lsp"]
+      gems << "debug" unless @dependencies["debug"]
+      gems << "ruby-lsp-rails" if @rails_app && !@dependencies["ruby-lsp-rails"]
+
+      Bundler::CLI::Update.new({}, gems).run
+      @last_updated_path.write(Time.now.iso8601)
+      env
+    end
+
+    sig { params(env: T::Hash[String, String]).returns(T::Hash[String, String]) }
+    def run_bundle_install_through_command(env)
       base_bundle = base_bundle_command(env)
 
       # If `ruby-lsp` and `debug` (and potentially `ruby-lsp-rails`) are already in the Gemfile, then we shouldn't try

lib/ruby_lsp/utils.rb

@@ -79,6 +79,14 @@ def window_log_message(message, type: Constant::MessageType::LOG)
           params: Interface::LogMessageParams.new(type: type, message: message),
         )
       end
+
+      sig { params(data: T::Hash[Symbol, T.untyped]).returns(Notification) }
+      def telemetry(data)
+        new(
+          method: "telemetry/event",
+          params: data,
+        )
+      end
     end
 
     extend T::Sig

sorbet/rbi/shims/bundler.rbi

@@ -1,6 +1,26 @@
 # typed: true
 
-class Bundler::Settings
-  sig { params(name: String).returns(String) }
-  def self.key_for(name); end
+module Bundler
+  class Settings
+    sig { params(name: String).returns(String) }
+    def self.key_for(name); end
+  end
+
+  module CLI
+    class Install
+      sig { params(options: T::Hash[String, T.untyped]).void }
+      def initialize(options); end
+
+      sig { void }
+      def run; end
+    end
+
+    class Update
+      sig { params(options: T::Hash[String, T.untyped], gems: T::Array[String]).void }
+      def initialize(options, gems); end
+
+      sig { void }
+      def run; end
+    end
+  end
 end

test/integration_test.rb

@@ -2,8 +2,13 @@
 # frozen_string_literal: true
 
 require "test_helper"
+require "open3"
 
 class IntegrationTest < Minitest::Test
+  def setup
+    @root = Bundler.root
+  end
+
   def test_ruby_lsp_doctor_works
     skip("CI only") unless ENV["CI"]
 
@@ -79,8 +84,6 @@ def test_avoids_bundler_version_if_local_bin_is_in_path
   end
 
   def test_launch_mode_with_no_gemfile
-    skip("CI only") unless ENV["CI"]
-
     in_temp_dir do |dir|
       Bundler.with_unbundled_env do
         launch(dir)
@@ -89,8 +92,6 @@ def test_launch_mode_with_no_gemfile
   end
 
   def test_launch_mode_with_missing_lockfile
-    skip("CI only") unless ENV["CI"]
-
     in_temp_dir do |dir|
       File.write(File.join(dir, "Gemfile"), <<~RUBY)
         source "https://rubygems.org"
@@ -104,8 +105,6 @@ def test_launch_mode_with_missing_lockfile
   end
 
   def test_launch_mode_with_full_bundle
-    skip("CI only") unless ENV["CI"]
-
     in_temp_dir do |dir|
       File.write(File.join(dir, "Gemfile"), <<~RUBY)
         source "https://rubygems.org"
@@ -137,13 +136,13 @@ def test_launch_mode_with_full_bundle
   end
 
   def test_launch_mode_with_no_gemfile_and_bundle_path
-    skip("CI only") unless ENV["CI"]
-
     in_temp_dir do |dir|
       Bundler.with_unbundled_env do
         system("bundle config --local path #{File.join("vendor", "bundle")}")
         assert_path_exists(File.join(dir, ".bundle", "config"))
+      end
 
+      Bundler.with_unbundled_env do
         launch(dir)
       end
     end
@@ -152,42 +151,66 @@ def test_launch_mode_with_no_gemfile_and_bundle_path
   private
 
   def launch(workspace_path)
-    initialize_request = {
+    load_path = $-I.map do |p|
+      "-I#{File.expand_path(p)}"
+    end.join(" ")
+
+    stdin, stdout, stderr, wait_thr = T.unsafe(Open3).popen3(
+      ENV.to_hash,
+      Gem.ruby,
+      load_path,
+      File.join(@root, "exe", "ruby-lsp-launcher"),
+    )
+    stdin.sync = true
+    stdin.binmode
+    stdout.sync = true
+    stdout.binmode
+    stderr.sync = true
+    stderr.binmode
+
+    send_message(stdin, {
       id: 1,
       method: "initialize",
       params: {
         initializationOptions: {},
         capabilities: { general: { positionEncodings: ["utf-8"] } },
         workspaceFolders: [{ uri: URI::Generic.from_path(path: workspace_path).to_s }],
       },
-    }.to_json
-
-    $stdin.expects(:gets).with("\r\n\r\n").once.returns("Content-Length: #{initialize_request.bytesize}")
-    $stdin.expects(:read).with(initialize_request.bytesize).once.returns(initialize_request)
-
-    # Make `new` return a mock that raises so that we don't print to stdout and stop immediately after boot
-    server_object = mock("server")
-    server_object.expects(:start).once.raises(StandardError.new("stop"))
-    RubyLsp::Server.expects(:new).returns(server_object)
-
-    # We load the launcher binary in the same process as the tests are running. We cannot try to re-activate a different
-    # Bundler version, because that throws an error
-    if File.exist?(File.join(workspace_path, "Gemfile.lock"))
-      spec_mock = mock("specification")
-      spec_mock.expects(:activate).once
-      Gem::Specification.expects(:find_by_name).with do |name, version|
-        name == "bundler" && !version.empty?
-      end.returns(spec_mock)
-    end
+    })
 
-    # Verify that we are setting up the bundle, but there's no actual need to do it
-    Bundler.expects(:setup).once
+    # Block until we receive a response back from the server, which indicates it finished booting
+    response = read_response(stdout)
+    response = read_response(stdout) until response[:method] == "$/progress"
 
-    assert_raises(StandardError) do
-      load(File.expand_path("../exe/ruby-lsp-launcher", __dir__))
+    assert_path_exists(File.join(workspace_path, ".ruby-lsp", "bundle_gemfile"))
+    assert_path_exists(File.join(workspace_path, ".ruby-lsp", "Gemfile"))
+    assert_path_exists(File.join(workspace_path, ".ruby-lsp", "Gemfile.lock"))
+    refute_path_exists(File.join(workspace_path, ".ruby-lsp", "install_error"))
+
+    send_message(stdin, { id: 2, method: "shutdown" })
+    read_response(stdout)
+    send_message(stdin, { method: "exit" })
+
+    begin
+      Process.wait(wait_thr.pid)
+    rescue Errno::ECHILD
+      nil
     end
 
-    assert_path_exists(File.join(workspace_path, ".ruby-lsp", "bundle_gemfile"))
+    wait_thr.join
+    refute_predicate(wait_thr, :alive?)
+  end
+
+  def read_response(stdout)
+    headers = stdout.gets("\r\n\r\n")
+    content_length = headers[/Content-Length: (\d+)/i, 1].to_i
+    JSON.parse(stdout.read(content_length), symbolize_names: true)
+  end
+
+  def send_message(stdin, message)
+    json_message = message.to_json
+    stdin.write("Content-Length: #{json_message.bytesize}\r\n\r\n#{json_message}")
+    stdin.flush
   end
 
   def in_temp_dir