Start accepting any URI scheme in the index (#2934)

### Motivation

Another step towards #1908

After the recent changes, we can now make the index treat entries coming from any URIs appropriately.

### Implementation

The main aspect of change is changing the `@files_to_entries` variable into `@uris_to_entries`. Since entries can now be discovered in URIs not backed by a file on disk, we need to be able to add, remove and handle changes based on URIs.

Additionally, for synchronizing changes in the index, we need to accept the source now, which will be kept in memory for unsaved files.

**Note**: this is a breaking change because it means that the `entries_for` API now _needs_ to accept a URI instead of a file path.

### Automated Tests

Added a test that verifies all of the important functionality for entries discovered in unsaved URIs.

exe/ruby-lsp

@@ -140,8 +140,8 @@ if options[:doctor]
   puts "Globbing for indexable files"
 
   index.configuration.indexable_uris.each do |uri|
-    puts "indexing: #{uri.full_path}"
-    index.index_single(uri)
+    puts "indexing: #{uri}"
+    index.index_file(uri)
   end
   return
 end

exe/ruby-lsp-check

@@ -47,7 +47,7 @@ index = RubyIndexer::Index.new
 uris = index.configuration.indexable_uris
 
 uris.each_with_index do |uri, i|
-  index.index_single(uri)
+  index.index_file(uri)
 rescue => e
   errors[uri.full_path] = e
 ensure

lib/ruby_indexer/lib/ruby_indexer/entry.rb

@@ -60,19 +60,24 @@ def private?
 
     sig { returns(String) }
     def file_name
-      File.basename(file_path)
+      if @uri.scheme == "untitled"
+        T.must(@uri.opaque)
+      else
+        File.basename(T.must(file_path))
+      end
     end
 
-    sig { returns(String) }
+    sig { returns(T.nilable(String)) }
     def file_path
-      T.must(@uri.full_path)
+      @uri.full_path
     end
 
     sig { returns(String) }
     def comments
       @comments ||= begin
         # Parse only the comments based on the file path, which is much faster than parsing the entire file
-        parsed_comments = Prism.parse_file_comments(file_path)
+        path = file_path
+        parsed_comments = path ? Prism.parse_file_comments(path) : []
 
         # Group comments based on whether they belong to a single block of comments
         grouped = parsed_comments.slice_when do |left, right|

lib/ruby_indexer/lib/ruby_indexer/index.rb

@@ -29,10 +29,11 @@ def initialize
 
       # Holds references to where entries where discovered so that we can easily delete them
       # {
-      #  "/my/project/foo.rb" => [#<Entry::Class>, #<Entry::Class>],
-      #  "/my/project/bar.rb" => [#<Entry::Class>],
+      #  "file:///my/project/foo.rb" => [#<Entry::Class>, #<Entry::Class>],
+      #  "file:///my/project/bar.rb" => [#<Entry::Class>],
+      #  "untitled:Untitled-1" => [#<Entry::Class>],
       # }
-      @files_to_entries = T.let({}, T::Hash[String, T::Array[Entry]])
+      @uris_to_entries = T.let({}, T::Hash[String, T::Array[Entry]])
 
       # Holds all require paths for every indexed item so that we can provide autocomplete for requires
       @require_paths_tree = T.let(PrefixTree[URI::Generic].new, PrefixTree[URI::Generic])
@@ -57,12 +58,10 @@ def register_included_hook(module_name, &hook)
 
     sig { params(uri: URI::Generic).void }
     def delete(uri)
-      full_path = uri.full_path
-      return unless full_path
-
+      key = uri.to_s
       # For each constant discovered in `path`, delete the associated entry from the index. If there are no entries
       # left, delete the constant from the index.
-      @files_to_entries[full_path]&.each do |entry|
+      @uris_to_entries[key]&.each do |entry|
         name = entry.name
         entries = @entries[name]
         next unless entries
@@ -80,7 +79,7 @@ def delete(uri)
         end
       end
 
-      @files_to_entries.delete(full_path)
+      @uris_to_entries.delete(key)
 
       require_path = uri.require_path
       @require_paths_tree.delete(require_path) if require_path
@@ -91,7 +90,7 @@ def add(entry, skip_prefix_tree: false)
       name = entry.name
 
       (@entries[name] ||= []) << entry
-      (@files_to_entries[entry.file_path] ||= []) << entry
+      (@uris_to_entries[entry.uri.to_s] ||= []) << entry
       @entries_tree.insert(name, T.must(@entries[name])) unless skip_prefix_tree
     end
 
@@ -374,49 +373,40 @@ def index_all(uris: @configuration.indexable_uris, &block)
           break unless block.call(progress)
         end
 
-        index_single(uri, collect_comments: false)
+        index_file(uri, collect_comments: false)
       end
     end
 
-    sig { params(uri: URI::Generic, source: T.nilable(String), collect_comments: T::Boolean).void }
-    def index_single(uri, source = nil, collect_comments: true)
-      full_path = uri.full_path
-      return unless full_path
-
-      content = source || File.read(full_path)
+    sig { params(uri: URI::Generic, source: String, collect_comments: T::Boolean).void }
+    def index_single(uri, source, collect_comments: true)
       dispatcher = Prism::Dispatcher.new
 
-      result = Prism.parse(content)
-      listener = DeclarationListener.new(
-        self,
-        dispatcher,
-        result,
-        uri,
-        collect_comments: collect_comments,
-      )
+      result = Prism.parse(source)
+      listener = DeclarationListener.new(self, dispatcher, result, uri, collect_comments: collect_comments)
       dispatcher.dispatch(result.value)
 
-      indexing_errors = listener.indexing_errors.uniq
-
       require_path = uri.require_path
       @require_paths_tree.insert(require_path, uri) if require_path
 
-      if indexing_errors.any?
-        indexing_errors.each do |error|
-          $stderr.puts error
-        end
-      end
-    rescue Errno::EISDIR, Errno::ENOENT
-      # If `path` is a directory, just ignore it and continue indexing. If the file doesn't exist, then we also ignore
-      # it
+      indexing_errors = listener.indexing_errors.uniq
+      indexing_errors.each { |error| $stderr.puts(error) } if indexing_errors.any?
     rescue SystemStackError => e
       if e.backtrace&.first&.include?("prism")
-        $stderr.puts "Prism error indexing #{T.must(full_path)}: #{e.message}"
+        $stderr.puts "Prism error indexing #{uri}: #{e.message}"
       else
         raise
       end
     end
 
+    # Indexes a File URI by reading the contents from disk
+    sig { params(uri: URI::Generic, collect_comments: T::Boolean).void }
+    def index_file(uri, collect_comments: true)
+      index_single(uri, File.read(T.must(uri.full_path)), collect_comments: collect_comments)
+    rescue Errno::EISDIR, Errno::ENOENT
+      # If `path` is a directory, just ignore it and continue indexing. If the file doesn't exist, then we also ignore
+      # it
+    end
+
     # Follows aliases in a namespace. The algorithm keeps checking if the name is an alias and then recursively follows
     # it. The idea is that we test the name in parts starting from the complete name to the first namespace. For
     # `Foo::Bar::Baz`, we would test:
@@ -608,17 +598,15 @@ def instance_variable_completion_candidates(name, owner_name)
 
     # Synchronizes a change made to the given URI. This method will ensure that new declarations are indexed, removed
     # declarations removed and that the ancestor linearization cache is cleared if necessary
-    sig { params(uri: URI::Generic).void }
-    def handle_change(uri)
-      full_path = uri.full_path
-      return unless full_path
-
-      original_entries = @files_to_entries[full_path]
+    sig { params(uri: URI::Generic, source: String).void }
+    def handle_change(uri, source)
+      key = uri.to_s
+      original_entries = @uris_to_entries[key]
 
       delete(uri)
-      index_single(uri)
+      index_single(uri, source)
 
-      updated_entries = @files_to_entries[full_path]
+      updated_entries = @uris_to_entries[key]
 
       return unless original_entries && updated_entries
 
@@ -684,12 +672,12 @@ def existing_or_new_singleton_class(name)
 
     sig do
       type_parameters(:T).params(
-        path: String,
+        uri: String,
         type: T.nilable(T::Class[T.all(T.type_parameter(:T), Entry)]),
       ).returns(T.nilable(T.any(T::Array[Entry], T::Array[T.type_parameter(:T)])))
     end
-    def entries_for(path, type = nil)
-      entries = @files_to_entries[path]
+    def entries_for(uri, type = nil)
+      entries = @uris_to_entries[uri.to_s]
       return entries unless type
 
       entries&.grep(type)

lib/ruby_indexer/test/classes_and_modules_test.rb

@@ -623,7 +623,7 @@ def test_lazy_comment_fetching_uses_correct_line_breaks_for_rendering
         path: "#{Dir.pwd}/lib/ruby_lsp/node_context.rb",
       )
 
-      @index.index_single(uri, collect_comments: false)
+      @index.index_file(uri, collect_comments: false)
 
       entry = @index["RubyLsp::NodeContext"].first
 

lib/ruby_indexer/test/index_test.rb

@@ -124,7 +124,7 @@ def test_index_single_ignores_directories
       FileUtils.mkdir(path)
 
       begin
-        @index.index_single(URI::Generic.from_path(path: path))
+        @index.index_file(URI::Generic.from_path(path: path))
       ensure
         FileUtils.rm_r(path)
       end
@@ -351,14 +351,14 @@ def test_indexing_prism_fixtures_succeeds
 
       fixtures.each do |fixture|
         uri = URI::Generic.from_path(path: fixture)
-        @index.index_single(uri)
+        @index.index_file(uri)
       end
 
       refute_empty(@index)
     end
 
     def test_index_single_does_not_fail_for_non_existing_file
-      @index.index_single(URI::Generic.from_path(path: "/fake/path/foo.rb"))
+      @index.index_file(URI::Generic.from_path(path: "/fake/path/foo.rb"))
       entries_after_indexing = @index.names
       assert_equal(@default_indexed_entries.keys, entries_after_indexing)
     end
@@ -787,7 +787,7 @@ class Bar
           RUBY
 
           uri = URI::Generic.from_path(path: File.join(dir, "foo.rb"))
-          @index.index_single(uri)
+          @index.index_file(uri)
 
           assert_equal(["Bar", "Foo", "Object", "Kernel", "BasicObject"], @index.linearized_ancestors_of("Bar"))
 
@@ -800,7 +800,7 @@ class Bar
             end
           RUBY
 
-          @index.handle_change(uri)
+          @index.handle_change(uri, File.read(T.must(uri.full_path)))
           assert_empty(@index.instance_variable_get(:@ancestors))
           assert_equal(["Bar", "Object", "Kernel", "BasicObject"], @index.linearized_ancestors_of("Bar"))
         end
@@ -821,7 +821,7 @@ class Bar
           RUBY
 
           uri = URI::Generic.from_path(path: File.join(dir, "foo.rb"))
-          @index.index_single(uri)
+          @index.index_file(uri)
 
           assert_equal(["Bar", "Foo", "Object", "Kernel", "BasicObject"], @index.linearized_ancestors_of("Bar"))
 
@@ -837,7 +837,7 @@ def baz; end
             end
           RUBY
 
-          @index.handle_change(uri)
+          @index.handle_change(uri, File.read(T.must(uri.full_path)))
           refute_empty(@index.instance_variable_get(:@ancestors))
           assert_equal(["Bar", "Foo", "Object", "Kernel", "BasicObject"], @index.linearized_ancestors_of("Bar"))
         end
@@ -857,7 +857,7 @@ class Bar < Foo
           RUBY
 
           uri = URI::Generic.from_path(path: File.join(dir, "foo.rb"))
-          @index.index_single(uri)
+          @index.index_file(uri)
 
           assert_equal(["Bar", "Foo", "Object", "Kernel", "BasicObject"], @index.linearized_ancestors_of("Bar"))
 
@@ -870,7 +870,7 @@ class Bar
             end
           RUBY
 
-          @index.handle_change(uri)
+          @index.handle_change(uri, File.read(T.must(uri.full_path)))
           assert_empty(@index.instance_variable_get(:@ancestors))
           assert_equal(["Bar", "Object", "Kernel", "BasicObject"], @index.linearized_ancestors_of("Bar"))
         end
@@ -1889,13 +1889,13 @@ def self.my_singleton_def; end
         end
       RUBY
 
-      entries = @index.entries_for("/fake/path/foo.rb", Entry)
+      entries = @index.entries_for("file:///fake/path/foo.rb", Entry)
       assert_equal(["Foo", "Bar", "my_def", "Bar::<Class:Bar>", "my_singleton_def"], entries.map(&:name))
 
-      entries = @index.entries_for("/fake/path/foo.rb", RubyIndexer::Entry::Namespace)
+      entries = @index.entries_for("file:///fake/path/foo.rb", RubyIndexer::Entry::Namespace)
       assert_equal(["Foo", "Bar", "Bar::<Class:Bar>"], entries.map(&:name))
 
-      entries = @index.entries_for("/fake/path/foo.rb")
+      entries = @index.entries_for("file:///fake/path/foo.rb")
       assert_equal(["Foo", "Bar", "my_def", "Bar::<Class:Bar>", "my_singleton_def"], entries.map(&:name))
     end
 
@@ -2066,5 +2066,30 @@ def test_prevents_multiple_calls_to_index_all
         @index.index_all
       end
     end
+
+    def test_index_can_handle_entries_from_untitled_scheme
+      uri = URI("untitled:Untitled-1")
+
+      index(<<~RUBY, uri: uri)
+        class Foo
+        end
+      RUBY
+
+      entry = @index["Foo"]&.first
+      refute_nil(entry, "Expected indexer to be able to handle unsaved URIs")
+      assert_equal("untitled:Untitled-1", entry.uri.to_s)
+      assert_equal("Untitled-1", entry.file_name)
+      assert_nil(entry.file_path)
+
+      @index.handle_change(uri, <<~RUBY)
+        # I added this comment!
+        class Foo
+        end
+      RUBY
+
+      entry = @index["Foo"]&.first
+      refute_nil(entry, "Expected indexer to be able to handle unsaved URIs")
+      assert_equal("I added this comment!", entry.comments)
+    end
   end
 end

lib/ruby_indexer/test/test_case.rb

@@ -13,8 +13,8 @@ def setup
 
     private
 
-    def index(source)
-      @index.index_single(URI::Generic.from_path(path: "/fake/path/foo.rb"), source)
+    def index(source, uri: URI::Generic.from_path(path: "/fake/path/foo.rb"))
+      @index.index_single(uri, source)
     end
 
     def assert_entry(expected_name, type, expected_location, visibility: nil)

lib/ruby_lsp/server.rb

@@ -1004,11 +1004,13 @@ def workspace_did_change_watched_files(message)
         load_path_entry = $LOAD_PATH.find { |load_path| file_path.start_with?(load_path) }
         uri.add_require_path_from_load_entry(load_path_entry) if load_path_entry
 
+        content = File.read(file_path)
+
         case change[:type]
         when Constant::FileChangeType::CREATED
-          index.index_single(uri)
+          index.index_single(uri, content)
         when Constant::FileChangeType::CHANGED
-          index.handle_change(uri)
+          index.handle_change(uri, content)
         when Constant::FileChangeType::DELETED
           index.delete(uri)
         end

lib/ruby_lsp/test_helper.rb

@@ -36,20 +36,21 @@ def with_server(source = nil, uri = Kernel.URI("file:///fake.rb"), stub_no_typec
             },
           },
         })
+
+        server.global_state.index.index_single(uri, source)
       end
 
-      server.global_state.index.index_single(
-        URI::Generic.from_path(path: T.must(uri.to_standardized_path)),
-        source,
-      )
       server.load_addons(include_project_addons: false) if load_addons
-      block.call(server, uri)
-    ensure
-      if load_addons
-        RubyLsp::Addon.addons.each(&:deactivate)
-        RubyLsp::Addon.addons.clear
+
+      begin
+        block.call(server, uri)
+      ensure
+        if load_addons
+          RubyLsp::Addon.addons.each(&:deactivate)
+          RubyLsp::Addon.addons.clear
+        end
+        server.run_shutdown
       end
-      T.must(server).run_shutdown
     end
   end
 end