diff --git a/README.md b/README.md
index b9cf9fb..63fb171 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,7 @@
-[![CircleCI](https://circleci.com/gh/HumanCompatibleAI/seals.svg?style=svg)](https://circleci.com/gh/HumanCompatibleAI/seals) [![codecov](https://codecov.io/gh/HumanCompatibleAI/seals/branch/master/graph/badge.svg)](https://codecov.io/gh/HumanCompatibleAI/seals)
+[![CircleCI](https://circleci.com/gh/HumanCompatibleAI/seals.svg?style=svg)](https://circleci.com/gh/HumanCompatibleAI/seals)
+[![Documentation Status](https://readthedocs.org/projects/chai-seals/badge/?version=latest)](https://chai-seals.readthedocs.io/en/latest/?badge=latest)
+[![codecov](https://codecov.io/gh/HumanCompatibleAI/seals/branch/master/graph/badge.svg)](https://codecov.io/gh/HumanCompatibleAI/seals)
+[![PyPI version](https://badge.fury.io/py/seals.svg)](https://badge.fury.io/py/seals)
 
 **Status**: alpha, pre-release.
 
diff --git a/ci/build_venv.sh b/ci/build_venv.sh
index 84e40ff..7669de7 100755
--- a/ci/build_venv.sh
+++ b/ci/build_venv.sh
@@ -9,4 +9,4 @@ fi
 
 virtualenv -p python3.7 ${venv}
 source ${venv}/bin/activate
-pip install .[cpu,test]
+pip install .[cpu,docs,mujoco,test]
diff --git a/readthedocs.yml b/readthedocs.yml
new file mode 100644
index 0000000..dee2f3e
--- /dev/null
+++ b/readthedocs.yml
@@ -0,0 +1,14 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+formats: all
+
+python:
+  version: 3.7
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs
diff --git a/setup.py b/setup.py
index f0963c6..5ad58de 100644
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,6 @@ def get_readme() -> str:
         return f.read()
 
 
-TF_VERSION = ">=1.15.0,<2.0"
 TESTS_REQUIRE = [
     # remove pin once https://github.com/nedbat/coveragepy/issues/881 fixed
     "black",
@@ -47,11 +46,13 @@ def get_readme() -> str:
     "pytest-shard",
     "pytest-xdist",
     "pytype",
-    "sphinx",
-    "sphinxcontrib-napoleon",
     "stable-baselines>=2.8.0",
     "tensorflow>=1.8.0,<2.0.0",
 ]
+DOCS_REQUIRE = [
+    "sphinx",
+    "sphinxcontrib-napoleon",
+]
 
 setup(
     name="seals",
@@ -64,12 +65,16 @@ def get_readme() -> str:
     packages=find_packages("src"),
     package_dir={"": "src"},
     package_data={"seals": ["py.typed"]},
-    install_requires=["gym[mujoco]"],
+    install_requires=["gym"],
     tests_require=TESTS_REQUIRE,
     extras_require={
         # recommended packages for development
-        "dev": ["ipdb", "jupyter", *TESTS_REQUIRE],
+        "dev": ["ipdb", "jupyter", *TESTS_REQUIRE, *DOCS_REQUIRE],
+        "docs": DOCS_REQUIRE,
         "test": TESTS_REQUIRE,
+        # We'd like to specify `gym[mujoco]`, but this is a no-op when Gym is already
+        # installed. See https://github.com/pypa/pip/issues/4957 for issue.
+        "mujoco": ["mujoco_py>=1.50, <2.0", "imageio"],
     },
     url="https://github.com/HumanCompatibleAI/benchmark-environments",
     license="MIT",