Reload DSL compilers in the workspace automatically (#2178)

### Motivation

If the user edits custom DSL compilers that exist in the codebase, we never reload them and thus may end up generating incorrect RBI files.

This is not a _super_ common scenario as most people aren't working on custom compilers on a regular basis, but we can still perform automatic reloading for them.

### Implementation

We do not need to care about compilers defined in gems or in Tapioca itself. If a gem is bumped, the entire LSP is restarted and so we will pick up the changes.

So I started paying attention to changes to compilers inside the workspace and then firing a reload notification.

The implementation simply lists all descendants of `Tapioca::Dsl::Compiler` that are defined inside of the codebase. Then we proceed to remove the constant, delete the file entry from `$LOADED_FEATURES` and re-require the file to load the current state.

**Note**

I did not figure out a way to reload DSL extensions. Those are usually used to monkeypatch classes of the application itself or even create constants that don't exist elsewhere, so I'm not sure what we could do that would be guaranteed to work on every case.

### Tests

Added a test.

lib/ruby_lsp/tapioca/addon.rb

@@ -85,12 +85,18 @@ def workspace_did_change_watched_files(changes)
 
         has_route_change = T.let(false, T::Boolean)
         has_fixtures_change = T.let(false, T::Boolean)
+        needs_compiler_reload = T.let(false, T::Boolean)
 
         constants = changes.flat_map do |change|
           path = URI(change[:uri]).to_standardized_path
           next if path.end_with?("_test.rb", "_spec.rb")
           next unless file_updated?(change, path)
 
+          if File.fnmatch?("**/tapioca/**/compilers/**/*.rb", path, File::FNM_PATHNAME)
+            needs_compiler_reload = true
+            next
+          end
+
           if File.basename(path) == "routes.rb" || File.fnmatch?("**/routes/**/*.rb", path, File::FNM_PATHNAME)
             has_route_change = true
             next
@@ -110,10 +116,18 @@ def workspace_did_change_watched_files(changes)
           end
         end.compact
 
-        return if constants.empty? && !has_route_change && !has_fixtures_change
+        return if constants.empty? && !has_route_change && !has_fixtures_change && !needs_compiler_reload
 
         @rails_runner_client.trigger_reload
 
+        if needs_compiler_reload
+          @rails_runner_client.delegate_notification(
+            server_addon_name: "Tapioca",
+            request_name: "reload_workspace_compilers",
+            workspace_path: T.must(@global_state).workspace_path,
+          )
+        end
+
         if has_route_change
           @rails_runner_client.delegate_notification(server_addon_name: "Tapioca", request_name: "route_dsl")
         end

lib/ruby_lsp/tapioca/server_addon.rb

@@ -14,15 +14,20 @@ def name
 
       def execute(request, params)
         case request
+        when "reload_workspace_compilers"
+          with_notification_wrapper("reload_workspace_compilers", "Reloading DSL compilers") do
+            @loader&.reload_custom_compilers
+          end
         when "load_compilers_and_extensions"
           # Load DSL extensions and compilers ahead of time, so that we don't have to pay the price of invoking
           # `Gem.find_files` on every execution, which is quite expensive
-          ::Tapioca::Loaders::Dsl.new(
+          @loader = ::Tapioca::Loaders::Dsl.new(
             tapioca_path: ::Tapioca::TAPIOCA_DIR,
             eager_load: false,
             app_root: params[:workspace_path],
             halt_upon_load_error: false,
-          ).load_dsl_extensions_and_compilers
+          )
+          @loader.load_dsl_extensions_and_compilers
         when "dsl"
           fork do
             with_notification_wrapper("dsl", "Generating DSL RBIs") do

lib/tapioca/loaders/dsl.rb

@@ -45,6 +45,27 @@ def load_dsl_extensions_and_compilers
         load_dsl_compilers
       end
 
+      sig { void }
+      def reload_custom_compilers
+        # Remove all loaded custom compilers
+        ::Tapioca::Dsl::Compiler.descendants.each do |compiler|
+          name = compiler.name
+          next unless name && @custom_compiler_paths.include?(Module.const_source_location(name)&.first)
+
+          *parts, unqualified_name = name.split("::")
+
+          if parts.empty?
+            Object.send(:remove_const, unqualified_name)
+          else
+            parts.join("::").safe_constantize.send(:remove_const, unqualified_name)
+          end
+        end
+
+        # Remove from $LOADED_FEATURES each workspace compiler file and then re-load
+        @custom_compiler_paths.each { |path| $LOADED_FEATURES.delete(path) }
+        load_custom_dsl_compilers
+      end
+
       protected
 
       sig do
@@ -57,6 +78,7 @@ def initialize(tapioca_path:, eager_load: true, app_root: ".", halt_upon_load_er
         @eager_load = eager_load
         @app_root = app_root
         @halt_upon_load_error = halt_upon_load_error
+        @custom_compiler_paths = T.let([], T::Array[String])
       end
 
       sig { void }
@@ -89,12 +111,7 @@ def load_dsl_compilers
         end
 
         # Load all custom compilers from the project
-        Dir.glob([
-          "#{@tapioca_path}/generators/**/*.rb", # TODO: Here for backcompat, remove later
-          "#{@tapioca_path}/compilers/**/*.rb",
-        ]).each do |compiler|
-          require File.expand_path(compiler)
-        end
+        load_custom_dsl_compilers
 
         say("Done", :green)
       end
@@ -112,6 +129,17 @@ def load_application
 
         say("Done", :green)
       end
+
+      private
+
+      sig { void }
+      def load_custom_dsl_compilers
+        @custom_compiler_paths = Dir.glob([
+          "#{@tapioca_path}/generators/**/*.rb", # TODO: Here for backcompat, remove later
+          "#{@tapioca_path}/compilers/**/*.rb",
+        ])
+        @custom_compiler_paths.each { |compiler| require File.expand_path(compiler) }
+      end
     end
   end
 end

spec/tapioca/addon_spec.rb

@@ -105,6 +105,49 @@ class AddonSpec < Minitest::HooksSpec
         FileUtils.rm("spec/dummy/test/fixtures/users.yml") if File.exist?("spec/dummy/test/fixtures/users.yml")
       end
 
+      it "reloads compilers automatically" do
+        create_client
+
+        FileUtils.mkdir_p("spec/dummy/sorbet/tapioca/compilers")
+        File.write("spec/dummy/sorbet/tapioca/compilers/test_compiler.rb", <<~RUBY)
+          class TestCompiler < ::Tapioca::Dsl::Compiler
+            def self.gather_constants
+              descendants_of(::ActiveJob::Base)
+            end
+
+            def decorate
+              root.create_path(constant) do |job|
+                job.create_method(
+                  "hello_from_spec",
+                  parameters: [],
+                  return_type: "T.untyped",
+                  class_method: true,
+                )
+              end
+            end
+          end
+        RUBY
+
+        @client.delegate_notification(
+          server_addon_name: "Tapioca",
+          request_name: "reload_workspace_compilers",
+          workspace_path: File.expand_path("spec/dummy"),
+        )
+
+        @client.delegate_notification(
+          server_addon_name: "Tapioca",
+          request_name: "dsl",
+          constants: ["NotifyUserJob"],
+        )
+        wait_until_exists("spec/dummy/sorbet/rbi/dsl/notify_user_job.rbi")
+        shutdown_client
+
+        assert_match("hello_from_spec", File.read("spec/dummy/sorbet/rbi/dsl/notify_user_job.rbi"))
+      ensure
+        FileUtils.rm_rf("spec/dummy/sorbet/rbi")
+        FileUtils.rm_rf("spec/dummy/sorbet/tapioca")
+      end
+
       private
 
       # Starts a new client