diff --git a/lua/lazy/core/loader.lua b/lua/lazy/core/loader.lua
index edf9ba72..7cbd6f89 100644
--- a/lua/lazy/core/loader.lua
+++ b/lua/lazy/core/loader.lua
@@ -16,6 +16,7 @@ M.init_done = false
 M.disabled_rtp_plugins = { packer_compiled = true }
 ---@type table<string,string>
 M.did_ftdetect = {}
+M.did_handlers = false
 
 function M.disable_rtp_plugin(plugin)
   M.disabled_rtp_plugins[plugin] = true
@@ -56,6 +57,7 @@ function M.setup()
   -- setup handlers
   Util.track("handlers")
   Handler.setup()
+  M.did_handlers = true
   Util.track()
 end
 
@@ -498,8 +500,11 @@ function M.auto_load(modname, modpath)
   local plugin = Plugin.find(modpath)
   if plugin and modpath:find(plugin.dir, 1, true) == 1 then
     plugin._.rtp_loaded = true
-    -- don't load if we're loading specs or if the plugin is already loaded
-    if not (Plugin.loading or plugin._.loaded) then
+    -- don't load if:
+    -- * handlers haven't been setup yet
+    -- * we're loading specs
+    -- * the plugin is already loaded
+    if M.did_handlers and not (Plugin.loading or plugin._.loaded) then
       if plugin.module == false then
         error("Plugin " .. plugin.name .. " is not loaded and is configured with module=false")
       end
@@ -508,9 +513,7 @@ function M.auto_load(modname, modpath)
         error("You're trying to load `" .. plugin.name .. "` for which `cond==false`")
       end
     end
-    return true
   end
-  return false
 end
 
 ---@param modname string