Use Rubydex to document symbols instead of YARD

Gemfile

@@ -29,6 +29,7 @@ gem "ruby-lsp-rails", ">= 0.4"
 
 group :deployment, :development do
   gem "rake"
+  gem "yard"
 end
 
 group :development, :test do

Gemfile.lock

@@ -21,11 +21,11 @@ PATH
       parallel (>= 1.21.0)
       rbi (>= 0.3.7)
       require-hooks (>= 0.2.2)
+      rubydex (>= 0.1.0.beta8)
       sorbet-static-and-runtime (>= 0.5.11087)
       spoom (>= 1.7.9)
       thor (>= 1.2.0)
       tsort
-      yard-sorbet
 
 GEM
   remote: https://rubygems.org/
@@ -356,6 +356,9 @@ GEM
     ruby-lsp-rails (0.4.8)
       ruby-lsp (>= 0.26.0, < 0.27.0)
     ruby-progressbar (1.13.0)
+    rubydex (0.1.0.beta8)
+    rubydex (0.1.0.beta8-arm64-darwin)
+    rubydex (0.1.0.beta8-x86_64-linux)
     securerandom (0.4.1)
     shopify-money (4.0.0)
       bigdecimal (>= 3.0)
@@ -415,10 +418,7 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    yard (0.9.37)
-    yard-sorbet (0.9.0)
-      sorbet-runtime
-      yard
+    yard (0.9.38)
     zeitwerk (2.7.4)
     zlib (3.2.2)
 
@@ -473,6 +473,7 @@ DEPENDENCIES
   tapioca!
   webmock
   xpath
+  yard
 
 BUNDLED WITH
   4.0.7

README.md

@@ -161,7 +161,7 @@ All operations performed in working directory.
 Please review changes and commit them.
 ```
 
-This will load your application, find all the gems required by it and generate an RBI file for each gem under the `sorbet/rbi/gems` directory for each of those gems. This process will also import signatures that can be found inside each gem sources, and, optionally, any YARD documentation inside the gem.
+This will load your application, find all the gems required by it and generate an RBI file for each gem under the `sorbet/rbi/gems` directory for each of those gems. This process will also import signatures that can be found inside each gem sources, and, optionally, any documentation inside the gem.
 
 <!-- START_HELP_COMMAND_GEM -->
 ```shell
@@ -187,7 +187,7 @@ Options:
                                                                                                      # Default: {"activesupport" => "false"}
                [--verify], [--no-verify], [--skip-verify]                                            # Verify RBIs are up-to-date
                                                                                                      # Default: false
-               [--doc], [--no-doc], [--skip-doc]                                                     # Include YARD documentation from sources when generating RBIs. Warning: this might be slow
+               [--doc], [--no-doc], [--skip-doc]                                                     # Include documentation from sources when generating RBIs
                                                                                                      # Default: true
                [--loc], [--no-loc], [--skip-loc]                                                     # Include comments with source location when generating RBIs
                                                                                                      # Default: true

lib/tapioca/cli.rb

@@ -225,7 +225,7 @@ def dsl(*constant_or_paths)
       default: false
     option :doc,
       type: :boolean,
-      desc: "Include YARD documentation from sources when generating RBIs. Warning: this might be slow",
+      desc: "Include documentation from sources when generating RBIs",
       default: true
     option :loc,
       type: :boolean,

lib/tapioca/gem/listeners.rb

@@ -14,5 +14,5 @@
 require "tapioca/gem/listeners/sorbet_type_variables"
 require "tapioca/gem/listeners/subconstants"
 require "tapioca/gem/listeners/foreign_constants"
-require "tapioca/gem/listeners/yard_doc"
+require "tapioca/gem/listeners/documentation"
 require "tapioca/gem/listeners/source_location"

lib/tapioca/gem/listeners/documentation.rb

@@ -0,0 +1,94 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Tapioca
+  module Gem
+    module Listeners
+      class Documentation < Base
+        IGNORED_COMMENTS = [
+          ":doc:",
+          ":nodoc:",
+          "typed:",
+          "frozen_string_literal:",
+          "encoding:",
+          "warn_indent:",
+          "shareable_constant_value:",
+          "rubocop:",
+          "@requires_ancestor:",
+        ] #: Array[String]
+
+        #: (Pipeline pipeline, Rubydex::Graph gem_graph) -> void
+        def initialize(pipeline, gem_graph)
+          super(pipeline)
+
+          @gem_graph = gem_graph
+        end
+
+        private
+
+        #: (String line) -> bool
+        def rbs_comment?(line)
+          line.start_with?(": ", "| ")
+        end
+
+        # @override
+        #: (ConstNodeAdded event) -> void
+        def on_const(event)
+          event.node.comments = documentation_comments(event.symbol)
+        end
+
+        # @override
+        #: (ScopeNodeAdded event) -> void
+        def on_scope(event)
+          event.node.comments = documentation_comments(event.symbol)
+        end
+
+        # @override
+        #: (MethodNodeAdded event) -> void
+        def on_method(event)
+          name = if event.constant.singleton_class?
+            "#{event.symbol}::<#{event.symbol.split("::").last}>##{event.node.name}()"
+          else
+            "#{event.symbol}##{event.node.name}()"
+          end
+          event.node.comments = documentation_comments(name, sigs: event.node.sigs)
+        end
+
+        #: (String name, ?sigs: Array[RBI::Sig]) -> Array[RBI::Comment]
+        def documentation_comments(name, sigs: [])
+          declaration = @gem_graph[name]
+          # For attr_writer methods (name ending in =), fall back to reader docs
+          if declaration.nil? && name.end_with?("=()")
+            declaration = @gem_graph[name.delete_suffix("=()") + "()"]
+          end
+          # For singleton methods (Class::<Class>#method()), fall back to instance method docs.
+          # This handles module_function and extend self methods which Rubydex indexes
+          # only under the instance method name.
+          if declaration.nil? && name.include?("::<")
+            declaration = @gem_graph[name.sub(/::<[^>]+>#/, "#")]
+          end
+          return [] unless declaration
+
+          comments = declaration.definitions.flat_map(&:comments)
+          comments.uniq!
+          return [] if comments.empty?
+
+          lines = comments
+            .map { |comment| comment.string.gsub(/^#+ ?/, "") }
+            .reject { |line| IGNORED_COMMENTS.any? { |comment| line.include?(comment) } || rbs_comment?(line) }
+
+          # Strip leading and trailing blank lines, matching YARD's behavior
+          lines = lines.drop_while(&:empty?).reverse.drop_while(&:empty?).reverse
+
+          lines.map! { |line| RBI::Comment.new(line) }
+        end
+
+        # @override
+        #: (NodeAdded event) -> bool
+        def ignore?(event)
+          event.is_a?(Tapioca::Gem::ForeignScopeNodeAdded)
+        end
+      end
+    end
+  end
+end

lib/tapioca/gem/pipeline.rb

@@ -32,6 +32,7 @@ def initialize(
 
         @payload_symbols = Static::SymbolLoader.payload_symbols #: Set[String]
         @bootstrap_symbols = load_bootstrap_symbols(@gem) #: Set[String]
+        gem_graph = Static::SymbolLoader.graph_from_paths(@gem.files) if include_doc
 
         @bootstrap_symbols.each { |symbol| push_symbol(symbol) }
 
@@ -46,7 +47,7 @@ def initialize(
         @node_listeners << Gem::Listeners::SorbetRequiredAncestors.new(self)
         @node_listeners << Gem::Listeners::SorbetSignatures.new(self)
         @node_listeners << Gem::Listeners::Subconstants.new(self)
-        @node_listeners << Gem::Listeners::YardDoc.new(self) if include_doc
+        @node_listeners << Gem::Listeners::Documentation.new(self, gem_graph) if include_doc
         @node_listeners << Gem::Listeners::ForeignConstants.new(self)
         @node_listeners << Gem::Listeners::SourceLocation.new(self) if include_loc
         @node_listeners << Gem::Listeners::RemoveEmptyPayloadScopes.new(self)

lib/tapioca/gemfile.rb

@@ -177,22 +177,6 @@ def contains_path?(path)
         end
       end
 
-      #: -> void
-      def parse_yard_docs
-        files.each do |path|
-          YARD.parse(path.to_s, [], Logger::Severity::FATAL)
-        rescue RangeError
-          # In some circumstances, YARD will raise an error when parsing a file
-          # that is actually valid Ruby. We don't want tapioca to halt in these
-          # cases, so we'll rescue the error, pretend like there was no
-          # documentation, and move on.
-          #
-          # This can be removed when https://github.com/lsegal/yard/issues/1536
-          # is resolved and released.
-          []
-        end
-      end
-
       #: -> Array[String]
       def exported_rbi_files
         @exported_rbi_files ||= Dir.glob("#{full_gem_path}/rbi/**/*.rbi").sort

lib/tapioca/internal.rb

@@ -41,7 +41,7 @@
 require "tempfile"
 require "thor"
 require "yaml"
-require "yard-sorbet"
+require "rubydex"
 require "prism"
 
 require "tapioca/helpers/gem_helper"

lib/tapioca/static/symbol_loader.rb

@@ -18,6 +18,19 @@ def payload_symbols
           T.must(@payload_symbols)
         end
 
+        #: (Array[Pathname] paths) -> Rubydex::Graph
+        def graph_from_paths(paths)
+          graph = Rubydex::Graph.new
+          graph.index_all(paths.map(&:to_s))
+          graph.resolve
+          graph
+        end
+
+        #: (Gemfile::GemSpec gem) -> Set[String]
+        def gem_symbols(gem)
+          symbols_from_paths(gem.files)
+        end
+
         #: (Gemfile::GemSpec gem) -> Set[String]
         def engine_symbols(gem)
           gem_engine = engines.find do |engine|
@@ -43,11 +56,6 @@ def engine_symbols(gem)
           Set.new
         end
 
-        #: (Gemfile::GemSpec gem) -> Set[String]
-        def gem_symbols(gem)
-          symbols_from_paths(gem.files)
-        end
-
         #: (Array[Pathname] paths) -> Set[String]
         def symbols_from_paths(paths)
           return Set.new if paths.empty?

sorbet/config

@@ -4,3 +4,4 @@
 --enable-experimental-requires-ancestor
 --enable-experimental-rbs-comments
 --suppress-payload-superclass-redefinition-for=Net::IMAP::CommandData
+--suppress-payload-superclass-redefinition-for=RDoc::Markup::Heading