diff --git a/docs/plugins/installing-plugins.md b/docs/plugins/installing-plugins.md
index ca595d58..ab8ae30e 100644
--- a/docs/plugins/installing-plugins.md
+++ b/docs/plugins/installing-plugins.md
@@ -69,3 +69,25 @@ llm plugins
   }
 ]
 ```
+
+(llm-load-plugins)=
+## Running with a subset of plugins
+
+By default, LLM will load all plugins that are installed in the same virtual environment as LLM itself.
+
+You can control the set of plugins that is loaded using the `LLM_LOAD_PLUGINS` environment variable.
+
+Set that to the empty string to disable all plugins:
+
+```bash
+LLM_LOAD_PLUGINS='' llm ...
+```
+Or to a comma-separated list of plugin names to load only those plugins:
+
+```bash
+LLM_LOAD_PLUGINS='llm-gpt4all,llm-cluster' llm ...
+```
+You can use the `llm plugins` command to check that it is working correctly:
+```
+LLM_LOAD_PLUGINS='' llm plugins
+```
diff --git a/docs/plugins/tutorial-model-plugin.md b/docs/plugins/tutorial-model-plugin.md
index 766347ae..d54a5a50 100644
--- a/docs/plugins/tutorial-model-plugin.md
+++ b/docs/plugins/tutorial-model-plugin.md
@@ -559,3 +559,23 @@ It adds `llm` as a dependency, ensuring it will be installed if someone tries to
 It adds some links to useful pages (you can drop the `project.urls` section if those links are not useful for your project).
 
 You should drop a `LICENSE` file into the GitHub repository for your package as well. I like to use the Apache 2 license [like this](https://github.com/simonw/llm/blob/main/LICENSE).
+
+## What to do if it breaks
+
+Sometimes you may make a change to your plugin that causes it to break, preventing `llm` from starting. For example you may see an error like this one:
+
+```
+$ llm 'hi'
+Traceback (most recent call last):
+  ...
+  File llm-markov/llm_markov.py", line 10
+    register(Markov()):
+                      ^
+SyntaxError: invalid syntax
+```
+You may find that you are unable to uninstall the plugin using `llm uninstall llm-markov` because the command itself fails with the same error.
+
+Should this happen, you can uninstall the plugin after first disabling it using the {ref}`LLM_LOAD_PLUGINS <llm-load-plugins>` environment variable like this:
+```bash
+LLM_LOAD_PLUGINS='' llm uninstall llm-markov
+```
\ No newline at end of file
diff --git a/llm/__init__.py b/llm/__init__.py
index 2c0560c7..de9e4e24 100644
--- a/llm/__init__.py
+++ b/llm/__init__.py
@@ -44,6 +44,8 @@ def get_plugins():
     plugins = []
     plugin_to_distinfo = dict(pm.list_plugin_distinfo())
     for plugin in pm.get_plugins():
+        if plugin.__name__.startswith("llm.default_plugins."):
+            continue
         plugin_info = {
             "name": plugin.__name__,
             "hooks": [h.name for h in pm.get_hookcallers(plugin)],
diff --git a/llm/plugins.py b/llm/plugins.py
index 230b41c3..544cc008 100644
--- a/llm/plugins.py
+++ b/llm/plugins.py
@@ -1,4 +1,6 @@
 import importlib
+import os
+import pkg_resources
 import pluggy
 import sys
 from . import hookspecs
@@ -8,10 +10,29 @@
 pm = pluggy.PluginManager("llm")
 pm.add_hookspecs(hookspecs)
 
-if not hasattr(sys, "_called_from_test"):
+LLM_LOAD_PLUGINS = os.environ.get("LLM_LOAD_PLUGINS", None)
+
+if not hasattr(sys, "_called_from_test") and LLM_LOAD_PLUGINS is None:
     # Only load plugins if not running tests
     pm.load_setuptools_entrypoints("llm")
 
+
+# Load any plugins specified in LLM_LOAD_PLUGINS")
+if LLM_LOAD_PLUGINS is not None:
+    for package_name in [name for name in LLM_LOAD_PLUGINS.split(",") if name.strip()]:
+        try:
+            distribution = pkg_resources.get_distribution(package_name)
+            entry_map = distribution.get_entry_map()
+            if "llm" in entry_map:
+                for plugin_name, entry_point in entry_map["llm"].items():
+                    mod = entry_point.load()
+                    pm.register(mod, name=entry_point.name)
+                    # Ensure name can be found in plugin_to_distinfo later:
+                    pm._plugin_distinfo.append(mod, distribution)  # type: ignore
+        except pkg_resources.DistributionNotFound:
+            sys.stderr.write("Plugin {} could not be found\n".format(package_name))
+
+
 for plugin in DEFAULT_PLUGINS:
     mod = importlib.import_module(plugin)
     pm.register(mod, plugin)
diff --git a/setup.py b/setup.py
index 80815714..457f8381 100644
--- a/setup.py
+++ b/setup.py
@@ -59,6 +59,7 @@ def get_long_description():
             "types-click",
             "types-PyYAML",
             "types-requests",
+            "types-setuptools",
         ]
     },
     python_requires=">=3.7",