Merge pull request ruby#2101 from nixme/rbs-typechecking

Type check RBS and Ruby files

.github/workflows/main.yml

@@ -51,6 +51,7 @@ jobs:
       run: |
         bundle exec rake compile
         bundle exec rake check_annotations
+        bundle exec rake typecheck:steep
 
   build:
     strategy:

.gitignore

@@ -53,6 +53,11 @@ out.svg
 /src/token_type.c
 /src/**/*.o
 /sig/prism.rbs
+/sig/prism/dsl.rbs
+/sig/prism/mutation_compiler.rbs
+/sig/prism/node.rbs
+/sig/prism/visitor.rbs
+/sig/prism/_private/dot_visitor.rbs
 /rbi/prism.rbi
 
 compile_commands.json

Gemfile

@@ -12,11 +12,15 @@ gem "test-unit"
 platforms = %i[mri mswin mingw x64_mingw]
 
 gem "ffi", platform: platforms
+gem "parser", platform: platforms
+gem "ruby_parser", platform: platforms
+gem "benchmark-ips"
+
 group :memcheck do
   gem "ruby_memcheck", platform: platforms
 end
 
-gem "rbs", platform: platforms
-gem "parser", platform: platforms
-gem "ruby_parser", platform: platforms
-gem "benchmark-ips"
+group :types do
+  gem "rbs", platform: platforms
+  gem "steep", ">= 1.7.0.dev.1", platform: platforms
+end

Gemfile.lock

@@ -7,10 +7,38 @@ GEM
   remote: https://rubygems.org/
   specs:
     abbrev (0.1.2)
+    activesupport (7.1.3.2)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      mutex_m
+      tzinfo (~> 2.0)
     ast (2.4.2)
+    base64 (0.2.0)
     benchmark-ips (2.13.0)
+    bigdecimal (3.1.6)
+    concurrent-ruby (1.2.3)
+    connection_pool (2.4.1)
+    csv (3.2.8)
+    drb (2.2.0)
+      ruby2_keywords
     ffi (1.16.3)
+    fileutils (1.7.2)
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
+    json (2.7.1)
+    language_server-protocol (3.17.0.3)
+    listen (3.8.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    logger (1.6.0)
     mini_portile2 (2.8.5)
+    minitest (5.22.2)
+    mutex_m (0.2.0)
     nokogiri (1.16.2)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
@@ -19,19 +47,46 @@ GEM
       racc
     power_assert (2.0.3)
     racc (1.7.3)
+    rainbow (3.1.1)
     rake (13.1.0)
-    rake-compiler (1.2.6)
+    rake-compiler (1.2.7)
       rake
-    rbs (3.4.3)
+    rb-fsevent (0.11.2)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
+    rbs (3.4.4)
       abbrev
+    ruby2_keywords (0.0.5)
     ruby_memcheck (2.3.0)
       nokogiri
     ruby_parser (3.21.0)
       racc (~> 1.5)
       sexp_processor (~> 4.16)
+    securerandom (0.3.1)
     sexp_processor (4.17.1)
-    test-unit (3.6.1)
+    steep (1.7.0.dev.2)
+      activesupport (>= 5.1)
+      concurrent-ruby (>= 1.1.10)
+      csv (>= 3.0.9)
+      fileutils (>= 1.1.0)
+      json (>= 2.1.0)
+      language_server-protocol (>= 3.15, < 4.0)
+      listen (~> 3.0)
+      logger (>= 1.3.0)
+      parser (>= 3.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rbs (>= 3.1.0)
+      securerandom (>= 0.1)
+      strscan (>= 1.0.0)
+      terminal-table (>= 2, < 4)
+    strscan (3.1.0)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    test-unit (3.6.2)
       power_assert
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   arm64-darwin-23
@@ -47,6 +102,7 @@ DEPENDENCIES
   rbs
   ruby_memcheck
   ruby_parser
+  steep (>= 1.7.0.dev.1)
   test-unit
 
 BUNDLED WITH

Steepfile

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+target :lib do
+  signature "sig"
+
+  library "cgi" # in lib/prism/dot_visitor.rb (Prism::DotVisitor)
+
+  check "lib"
+
+  # TODO: Type-checking these files is still WIP
+  ignore "lib/prism/debug.rb"
+  ignore "lib/prism/desugar_compiler.rb"
+  ignore "lib/prism/lex_compat.rb"
+  ignore "lib/prism/serialize.rb"
+  ignore "lib/prism/ffi.rb"
+  ignore "lib/prism/translation"
+end

config.yml

@@ -727,6 +727,7 @@ nodes:
         kind: ParametersNode
       - name: locals
         type: node[]
+        kind: BlockLocalVariableNode
       - name: opening_loc
         type: location?
       - name: closing_loc
@@ -1134,6 +1135,9 @@ nodes:
         type: node?
       - name: child
         type: node
+        kind:
+          - ConstantReadNode
+          - MissingNode
       - name: delimiter_loc
         type: location
     comment: |
@@ -1177,6 +1181,9 @@ nodes:
         type: node?
       - name: child
         type: node
+        kind:
+          - ConstantReadNode
+          - MissingNode
       - name: delimiter_loc
         type: location
     comment: |
@@ -1575,8 +1582,12 @@ nodes:
         type: node?
       - name: elements
         type: node[]
+        kind: AssocNode
       - name: rest
         type: node?
+        kind:
+          - AssocSplatNode
+          - NoKeywordsParameterNode
       - name: opening_loc
         type: location?
       - name: closing_loc
@@ -1617,6 +1628,10 @@ nodes:
     fields:
       - name: numeric
         type: node
+        kind:
+          - FloatNode
+          - IntegerNode
+          - RationalNode
     comment: |
       Represents an imaginary number literal.
 
@@ -2370,16 +2385,36 @@ nodes:
     fields:
       - name: requireds
         type: node[]
+        kind:
+          - RequiredParameterNode
+          - MultiTargetNode
       - name: optionals
         type: node[]
+        kind: OptionalParameterNode
       - name: rest
         type: node?
+        kind:
+          - RestParameterNode
+          - ImplicitRestNode # Only in block parameters
       - name: posts
         type: node[]
+        kind:
+          - RequiredParameterNode
+          - MultiTargetNode
+          # On parsing error of `f(**kwargs, ...)` or `f(**nil, ...)`, the keyword_rest value is moved here:
+          - KeywordRestParameterNode
+          - NoKeywordsParameterNode
       - name: keywords
         type: node[]
+        kind:
+          - RequiredKeywordParameterNode
+          - OptionalKeywordParameterNode
       - name: keyword_rest
         type: node?
+        kind:
+          - KeywordRestParameterNode
+          - ForwardingParameterNode
+          - NoKeywordsParameterNode
       - name: block
         type: node?
         kind: BlockParameterNode

lib/prism.rb

@@ -44,7 +44,7 @@ module Prism
   #
   # For supported options, see Prism::parse.
   def self.lex_compat(source, **options)
-    LexCompat.new(source, **options).result
+    LexCompat.new(source, **options).result # steep:ignore
   end
 
   # :call-seq:
@@ -54,7 +54,7 @@ def self.lex_compat(source, **options)
   # returns the same tokens. Raises SyntaxError if the syntax in source is
   # invalid.
   def self.lex_ripper(source)
-    LexRipper.new(source).result
+    LexRipper.new(source).result # steep:ignore
   end
 
   # :call-seq:

lib/prism/debug.rb

@@ -55,7 +55,7 @@ def self.cruby_locals(source)
       verbose, $VERBOSE = $VERBOSE, nil
 
       begin
-        locals = []
+        locals = [] #: Array[Array[Symbol | Integer]]
         stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
 
         while (iseq = stack.pop)
@@ -96,8 +96,8 @@ def self.cruby_locals(source)
     # For the given source, parses with prism and returns a list of all of the
     # sets of local variables that were encountered.
     def self.prism_locals(source)
-      locals = []
-      stack = [Prism.parse(source).value]
+      locals = [] #: Array[Array[Symbol | Integer]]
+      stack = [Prism.parse(source).value] #: Array[Prism::node]
 
       while (node = stack.pop)
         case node

lib/prism/desugar_compiler.rb

@@ -82,7 +82,7 @@ def compile
           0,
           read_class.new(source, *arguments, node.name_loc),
           nil,
-          node.operator_loc.slice.chomp("="),
+          node.operator_loc.slice.chomp("=").to_sym,
           node.operator_loc.copy(length: node.operator_loc.length - 1),
           nil,
           ArgumentsNode.new(source, 0, [node.value], node.value.location),

lib/prism/lex_compat.rb

@@ -185,6 +185,8 @@ class LexCompat # :nodoc:
     # However, we add a couple of convenience methods onto them to make them a
     # little easier to work with. We delegate all other methods to the array.
     class Token < SimpleDelegator
+      # @dynamic initialize, each, []
+
       # The location of the token in the source.
       def location
         self[0]
@@ -241,10 +243,10 @@ class IgnoredNewlineToken < Token
       def ==(other) # :nodoc:
         return false unless self[0...-1] == other[0...-1]
 
-        if self[4] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
-          other[4] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED > 0
+        if self[3] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
+          other[3] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED != 0
         else
-          self[4] == other[4]
+          self[3] == other[3]
         end
       end
     end
@@ -308,7 +310,7 @@ def <<(token)
         def to_a
           embexpr_balance = 0
 
-          tokens.each_with_object([]) do |token, results|
+          tokens.each_with_object([]) do |token, results| #$ Array[Token]
             case token.event
             when :on_embexpr_beg
               embexpr_balance += 1
@@ -409,7 +411,7 @@ def to_a
           # If every line in the heredoc is blank, we still need to split up the
           # string content token into multiple tokens.
           if dedent.nil?
-            results = []
+            results = [] #: Array[Token]
             embexpr_balance = 0
 
             tokens.each do |token|
@@ -444,7 +446,7 @@ def to_a
           # If the minimum common whitespace is 0, then we need to concatenate
           # string nodes together that are immediately adjacent.
           if dedent == 0
-            results = []
+            results = [] #: Array[Token]
             embexpr_balance = 0
 
             index = 0
@@ -477,7 +479,7 @@ def to_a
           # insert on_ignored_sp tokens for the amount of dedent that we need to
           # perform. We also need to remove the dedent from the beginning of
           # each line of plain string content tokens.
-          results = []
+          results = [] #: Array[Token]
           dedent_next = true
           embexpr_balance = 0
 
@@ -526,7 +528,7 @@ def to_a
                   # dedent from the beginning of the line.
                   if (dedent > 0) && (dedent_next || index > 0)
                     deleting = 0
-                    deleted_chars = []
+                    deleted_chars = [] #: Array[String]
 
                     # Gather up all of the characters that we're going to
                     # delete, stopping when you hit a character that would put
@@ -603,15 +605,15 @@ def initialize(source, **options)
     end
 
     def result
-      tokens = []
+      tokens = [] #: Array[LexCompat::Token]
 
       state = :default
-      heredoc_stack = [[]]
+      heredoc_stack = [[]] #: Array[Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]]
 
       result = Prism.lex(source, **options)
       result_value = result.value
-      previous_state = nil
-      last_heredoc_end = nil
+      previous_state = nil #: Ripper::Lexer::State?
+      last_heredoc_end = nil #: Integer?
 
       # In previous versions of Ruby, Ripper wouldn't flush the bom before the
       # first token, so we had to have a hack in place to account for that. This
@@ -842,7 +844,7 @@ def result
       # We sort by location to compare against Ripper's output
       tokens.sort_by!(&:location)
 
-      ParseResult.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, [])
+      ParseResult.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.new(source))
     end
   end
 
@@ -858,8 +860,8 @@ def initialize(source)
     end
 
     def result
-      previous = []
-      results = []
+      previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
+      results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
 
       lex(source).each do |token|
         case token[1]