Hosting: manual integrations via build contract

dockerfiles/nginx/proxito.conf.template

@@ -88,6 +88,18 @@ server {
         proxy_hide_header Content-Security-Policy;
         set $content_security_policy $upstream_http_content_security_policy;
         add_header Content-Security-Policy $content_security_policy always;
+
+        # This header allows us to decide whether or not inject the script at CloudFlare level
+        # Now, I'm injecting it in all the NGINX responses because `sub_filter` is not allowed inside an `if` statement.
+        set $rtd_hosting_integrations $upstream_http_x_rtd_hosting_integrations;
+        add_header X-RTD-Hosting-Integrations $rtd_hosting_integrations always;
+
+        # Inject our own script dynamically
+        # TODO: decide where is the best place to do this
+        sub_filter '</head>' '<script language="javascript" src="http://devthedocs.org/static/core/js/readthedocs-client.js"></script>\n</head>';
+        # sub_filter_types text/html;
+        sub_filter_last_modified on;
+        sub_filter_once on;
     }
 
     # Serve 404 pages here

readthedocs/builds/migrations/0048_add_build_data.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.18 on 2023-03-13 15:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("builds", "0047_build_default_triggered"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="version",
+            name="build_data",
+            field=models.JSONField(
+                default=None,
+                null=True,
+                verbose_name="Data generated at build time by the doctool (`readthedocs-build.yaml`).",
+            ),
+        ),
+    ]

readthedocs/builds/models.py

@@ -186,6 +186,12 @@ class Version(TimeStampedModel):
         ),
     )
 
+    build_data = models.JSONField(
+        _("Data generated at build time by the doctool (`readthedocs-build.yaml`)."),
+        default=None,
+        null=True,
+    )
+
     objects = VersionManager.from_queryset(VersionQuerySet)()
     # Only include BRANCH, TAG, UNKNOWN type Versions.
     internal = InternalVersionManager.from_queryset(partial(VersionQuerySet, internal_only=True))()

readthedocs/doc_builder/director.py

@@ -2,6 +2,7 @@
 import tarfile
 
 import structlog
+import yaml
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 
@@ -187,6 +188,7 @@ def build(self):
         self.build_epub()
 
         self.run_build_job("post_build")
+        self.store_readthedocs_build_yaml()
 
         after_build.send(
             sender=self.data.version,
@@ -392,6 +394,7 @@ def run_build_commands(self):
         # Update the `Version.documentation_type` to match the doctype defined
         # by the config file. When using `build.commands` it will be `GENERIC`
         self.data.version.documentation_type = self.data.config.doctype
+        self.store_readthedocs_build_yaml()
 
     def install_build_tools(self):
         """
@@ -625,3 +628,33 @@ def get_build_env_vars(self):
     def is_type_sphinx(self):
         """Is documentation type Sphinx."""
         return "sphinx" in self.data.config.doctype
+
+    def store_readthedocs_build_yaml(self):
+        # load YAML from user
+        yaml_path = os.path.join(
+            self.data.project.artifact_path(
+                version=self.data.version.slug, type_="html"
+            ),
+            "readthedocs-build.yaml",
+        )
+
+        try:
+            with open(yaml_path, "r") as f:
+                data = yaml.safe_load(f)
+        except Exception:
+            # NOTE: skip this work for now until we decide whether or not this
+            # YAML file is required.
+            #
+            # NOTE: decide whether or not we want this
+            # file to be mandatory and raise an exception here.
+            return
+
+        log.info("readthedocs-build.yaml loaded.", path=yaml_path)
+
+        # TODO: validate the YAML generated by the user
+        # self._validate_readthedocs_build_yaml(data)
+
+        # Copy the YAML data into `Version.build_data`.
+        # It will be saved when the API is hit.
+        # This data will be used by the `/_/readthedocs-config.json` API endpoint.
+        self.data.version.build_data = data

readthedocs/doc_builder/environments.py

@@ -14,7 +14,7 @@
 from docker.errors import APIError as DockerAPIError
 from docker.errors import DockerException
 from docker.errors import NotFound as DockerNotFoundError
-from requests.exceptions import ConnectionError, ReadTimeout
+from requests.exceptions import ConnectionError, ReadTimeout  # noqa
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 from readthedocs.api.v2.client import api as api_v2
@@ -73,7 +73,7 @@ def __init__(
         bin_path=None,
         record_as_success=False,
         demux=False,
-        **kwargs,
+        **kwargs,  # pylint: disable=unused-argument
     ):
         self.command = command
         self.shell = shell
@@ -252,8 +252,8 @@ def save(self):
                 {key: str(value) for key, value in data.items()}
             )
             resource = api_v2.command
-            resp = resource._store['session'].post(
-                resource._store['base_url'] + '/',
+            resp = resource._store["session"].post(  # pylint: disable=protected-access
+                resource._store["base_url"] + "/",  # pylint: disable=protected-access
                 data=encoder,
                 headers={
                     'Content-Type': encoder.content_type,
@@ -301,11 +301,35 @@ def run(self):
 
         self.start_time = datetime.utcnow()
         client = self.build_env.get_client()
+
+        # Create a copy of the environment to update PATH variable
+        environment = self._environment.copy()
+        # Default PATH variable
+        environment[
+            "PATH"
+        ] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+        # Add asdf extra paths
+        environment["PATH"] += (
+            ":/home/{settings.RTD_DOCKER_USER}/.asdf/shims"
+            ":/home/{settings.RTD_DOCKER_USER}/.asdf/bin"
+        )
+
+        if settings.RTD_DOCKER_COMPOSE:
+            environment["PATH"] += ":/root/.asdf/shims:/root/.asdf/bin"
+
+        # Prepend the BIN_PATH if it's defined
+        if self.bin_path:
+            path = environment.get("PATH")
+            bin_path = self._escape_command(self.bin_path)
+            environment["PATH"] = bin_path
+            if path:
+                environment["PATH"] += f":{path}"
+
         try:
             exec_cmd = client.exec_create(
                 container=self.build_env.container_id,
                 cmd=self.get_wrapped_command(),
-                environment=self._environment,
+                environment=environment,
                 user=self.user,
                 workdir=self.cwd,
                 stdout=True,
@@ -357,31 +381,18 @@ def get_wrapped_command(self):
         """
         Wrap command in a shell and optionally escape special bash characters.
 
-        In order to set the current working path inside a docker container, we
-        need to wrap the command in a shell call manually.
-
         Some characters will be interpreted as shell characters without
         escaping, such as: ``pip install requests<0.8``. When passing
         ``escape_command=True`` in the init method this escapes a good majority
         of those characters.
         """
-        prefix = ''
-        if self.bin_path:
-            bin_path = self._escape_command(self.bin_path)
-            prefix += f'PATH={bin_path}:$PATH '
-
         command = (
             ' '.join(
                 self._escape_command(part) if self.escape_command else part
                 for part in self.command
             )
         )
-        return (
-            "/bin/sh -c '{prefix}{cmd}'".format(
-                prefix=prefix,
-                cmd=command,
-            )
-        )
+        return f"/bin/bash -c '{command}'"
 
     def _escape_command(self, cmd):
         r"""Escape the command by prefixing suspicious chars with `\`."""
@@ -524,14 +535,14 @@ class BuildEnvironment(BaseEnvironment):
     """
 
     def __init__(
-            self,
-            project=None,
-            version=None,
-            build=None,
-            config=None,
-            environment=None,
-            record=True,
-            **kwargs,
+        self,
+        project=None,
+        version=None,
+        build=None,
+        config=None,
+        environment=None,
+        record=True,
+        **kwargs,  # pylint: disable=unused-argument
     ):
         super().__init__(project, environment)
         self.version = version
@@ -557,7 +568,7 @@ def run(self, *cmd, **kwargs):
         })
         return super().run(*cmd, **kwargs)
 
-    def run_command_class(self, *cmd, **kwargs):  # pylint: disable=arguments-differ
+    def run_command_class(self, *cmd, **kwargs):  # pylint: disable=signature-differs
         kwargs.update({
             'build_env': self,
         })

readthedocs/projects/models.py

@@ -1884,6 +1884,7 @@ def add_features(sender, **kwargs):
     CANCEL_OLD_BUILDS = "cancel_old_builds"
     DONT_CREATE_INDEX = "dont_create_index"
     USE_RCLONE = "use_rclone"
+    HOSTING_INTEGRATIONS = "hosting_integrations"
 
     FEATURES = (
         (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')),
@@ -2058,6 +2059,10 @@ def add_features(sender, **kwargs):
             USE_RCLONE,
             _("Use rclone for syncing files to the media storage."),
         ),
+        (
+            HOSTING_INTEGRATIONS,
+            _("Inject 'readthedocs-client.js' as <script> HTML tag in responses."),
+        ),
     )
 
     projects = models.ManyToManyField(

readthedocs/projects/tasks/builds.py

@@ -598,6 +598,7 @@ def on_success(self, retval, task_id, args, kwargs):
                         "has_pdf": "pdf" in valid_artifacts,
                         "has_epub": "epub" in valid_artifacts,
                         "has_htmlzip": "htmlzip" in valid_artifacts,
+                        "build_data": self.data.version.build_data,
                     }
                 )
             except HttpClientError:

readthedocs/proxito/middleware.py

@@ -26,6 +26,7 @@
     unresolver,
 )
 from readthedocs.core.utils import get_cache_tag
+from readthedocs.projects.models import Feature, Project
 
 log = structlog.get_logger(__name__)
 
@@ -278,9 +279,19 @@ def process_request(self, request):  # noqa
 
         return None
 
+    def add_hosting_integrations_headers(self, request, response):
+        project_slug = getattr(request, "path_project_slug", "")
+        if project_slug:
+            project = Project.objects.get(slug=project_slug)
+            if project.has_feature(Feature.HOSTING_INTEGRATIONS):
+                response["X-RTD-Hosting-Integrations"] = "true"
+            else:
+                response["X-RTD-Hosting-Integrations"] = "false"
+
     def process_response(self, request, response):  # noqa
         self.add_proxito_headers(request, response)
         self.add_cache_headers(request, response)
         self.add_hsts_headers(request, response)
         self.add_user_headers(request, response)
+        self.add_hosting_integrations_headers(request, response)
         return response

readthedocs/proxito/urls.py

@@ -40,6 +40,7 @@
 from readthedocs.constants import pattern_opts
 from readthedocs.core.views import HealthCheckView
 from readthedocs.projects.views.public import ProjectDownloadMedia
+from readthedocs.proxito.views.hosting import ReadTheDocsConfigJson
 from readthedocs.proxito.views.serve import (
     ServeDocs,
     ServeError404,
@@ -114,6 +115,12 @@
         ServeStaticFiles.as_view(),
         name="proxito_static_files",
     ),
+    # readthedocs-config.js
+    path(
+        f"{DOC_PATH_PREFIX}readthedocs-config.json",
+        ReadTheDocsConfigJson.as_view(),
+        name="proxito_readthedocs_config_json",
+    ),
 ]
 
 core_urls = [