wip

tutor/commands/jobs.py

@@ -43,6 +43,15 @@ def _add_core_init_tasks() -> None:
             ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh"))
         )
     with hooks.Contexts.app("lms").enter():
+        hooks.Filters.CLI_DO_INIT_TASKS.add_item(
+            (
+                "lms",
+                env.read_core_template_file("jobs", "init", "mounted-directories.sh"),
+            ),
+            # If edx-platform is mounted, then we may need to perform some setup
+            # before other initialization scripts can be run.
+            priority=priorities.HIGH,
+        )
         hooks.Filters.CLI_DO_INIT_TASKS.add_item(
             ("lms", env.read_core_template_file("jobs", "init", "lms.sh"))
         )

tutor/templates/build/openedx/Dockerfile

@@ -228,16 +228,16 @@ COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=python-requirements /opened
 COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=python-requirements /mnt /mnt
 COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs /openedx/nodeenv /openedx/nodeenv
 
-# Merge JS requirements into /openedx/artifacts
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/common/static/common/js/vendor /openedx/artifacts/common/static/common/js/vendor
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/common/static/common/css/vendor /openedx/artifacts/common/static/common/css/vendor
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/artifacts/node_modules
+# Merge JS requirements into /openedx/assets
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/common/static/common/js/vendor /openedx/assets/common/static/common/js/vendor
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/common/static/common/css/vendor /openedx/assets/common/static/common/css/vendor
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/assets/node_modules
 
 WORKDIR /openedx/edx-platform
 
 # We install edx-platform here because it creates an egg-info folder in the current
 # repo. We need both the source code and the virtualenv to run this command.
-RUN pip install -e . && mv Open_edX.egg-info /openedx/artifacts
+RUN pip install -e .
 
 # Create folder that will store lms/cms.env.yml files, as well as
 # the tutor-specific settings files.
@@ -254,8 +254,8 @@ COPY --chown=app:app ./bin /openedx/bin
 RUN chmod a+x /openedx/bin/*
 ENV PATH /openedx/bin:${PATH}
 
-# Create symlinks from /openedx/edx-platform to /openedx/artifacts. See script definition for details.
-RUN create-artifact-links
+# Create symlinks from /openedx/edx-platform to /openedx/assets. See script definition for details.
+RUN ln-assets /opendx/assets /openedx/edx-platform
 
 # Create a data directory, which might be used (or not)
 RUN mkdir /openedx/data
@@ -304,12 +304,12 @@ ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.development
 
 CMD ./manage.py $SERVICE_VARIANT runserver 0.0.0.0:8000
 
-# Merge dev assets into dev image, placing edx-platform assets in /openedx/artifacts
+# Merge dev assets into dev image, placing edx-platform assets in /openedx/assets
 COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-development /openedx/staticfiles /openedx/staticfiles
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-development /openedx/edx-platform/common/static/bundles /openedx/artifacts/common/static/bundles
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/cms/static/css /openedx/artifacts/cms/static/css
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/css /openedx/artifacts/lms/static/css
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/certificates/css /openedx/artifacts/lms/static/certificates/css
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-development /openedx/edx-platform/common/static/bundles /openedx/assets/common/static/bundles
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/cms/static/css /openedx/assets/cms/static/css
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/css /openedx/assets/lms/static/css
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/certificates/css /openedx/assets/lms/static/certificates/css
 COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/themes /openedx/themes
 
 ###### Intermediate image with translations pulled from atlas
@@ -336,12 +336,12 @@ COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=mnt-{{ name }} / /mnt/{{ na
 RUN pip install -e "/mnt/{{ name }}"
 {% endfor %}
 
-# Merge prod assets into prod image, placing edx-platform assets in /openedx/artifacts
+# Merge prod assets into prod image, placing edx-platform assets in /openedx/assets
 COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/staticfiles /openedx/staticfiles
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/edx-platform/common/static/bundles /openedx/artifacts/common/static/bundles
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/cms/static/css /openedx/artifacts/cms/static/css
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/css /openedx/artifacts/lms/static/css
-COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/certificates/css /openedx/artifacts/lms/static/certificates/css
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/edx-platform/common/static/bundles /openedx/assets/common/static/bundles
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/cms/static/css /openedx/assets/cms/static/css
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/css /openedx/assets/lms/static/css
+COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/certificates/css /openedx/assets/lms/static/certificates/css
 COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/themes /openedx/themes
 
 # Merge translations into prod image

tutor/templates/build/openedx/bin/create-artifact-links → tutor/templates/build/openedx/bin/ln-assets

@@ -1,15 +1,21 @@
 #!/bin/sh
 #
-# Several important build artifacts need to be generated in the edx-platform repo.
+# Frontend assets need to be generated in the edx-platform repository.
 # However, when a developer bind-mounts edx-platform, it completely overwrites the repo.
-# So, the Dockerfile generates the artifacts outside of edx-platform (at /openedx/artifacts)
+
+# So, the Dockerfile instead generates the assets *outside* of edx-platform
 # where they will not be overwritten by a bind-mount. This script ensures that edx-platform
-# contains symlinks into edx-platfor. This script is run both in the Dockerfile and in lms's
+# contains symlinks to those assets. This script is run both in the Dockerfile and in lms's
 # init job; that way, the symlinks exist regardless of whether edx-platform is bind-mounted.
 #
-# ARTIFACT DIRECTORY                | PURPOSE
+# USAGE:
+# 
+#     ln-assets ASSETS_DIR EDX_PLATFORM_DIR
+#
+# where EDX_PLATFORM_DIR is to contain symlinks to assets in ASSETS_DIR.
+#
+# ASSET DIRECTORY                   | PURPOSE
 # ----------------------------------+---------------------------------------------
-# Open_edX.egg-info                 | edx-platform metadata generated by setup.py
 # node_modules                      | npm packages
 # common/static/common/js/vendor    | npm JS copies, for use by RequireJS
 # common/static/common/css/vendor   | npm CSS copies, for use by RequireJS
@@ -18,13 +24,18 @@
 # lms/static/css                    | LMS CSS, compiled from Sass
 # lms/static/certificates/css       | Certificate CSS, compiled from Sass
 
-echo "Symlinking build artifacts in /openedx/edx-platform to /openedx/artifacts..."
+set -euo pipefail
+
+assets="$(realpath "$1")"
+edx_platform="$(realpath "$2")"
+
+echo "Create static asset symlinks in "$edx_platform" towards $assets..."
 set -x
 
-mkdir -p /openedx/artifacts
+mkdir -p "$assets"
+cd "$edx_platform" || { echo "could not cd to $edx_platform" && exit 1 }
 
 for dir in \
-		Open_edX.egg-info \
 		node_modules \
 		common/static/common/js/vendor \
 		common/static/common/css/vendor \
@@ -34,7 +45,7 @@ for dir in \
 		lms/static/certificates/css ; do
 
 	# If there isn't a symlink or there's one to the wrong place, then fix it
-	if test "$(readlink -f $dir)" != /openedx/artifacts/$dir ; then
+	if test "$(readlink -f $dir)" != "$assets/$dir" ; then
 
 		# If there's an existing symlink (to the wrong place), delete it
 		if [ -L $dir ] ; then
@@ -46,9 +57,9 @@ for dir in \
 		fi
 
 		# Create the correct symlink
-		ln -s /openedx/artifacts/$dir $dir
+		ln -s "$assets/$dir" $dir
 	fi
 done
 
 set -x
-echo "Done symlinking build artifacts."
+echo "Done symlinking static assets."

tutor/templates/jobs/init/mounted-directories.sh

@@ -0,0 +1,29 @@
+# The initialization job contains various re-install operations needed to be done
+# on mounted directories (edx-platform, /mnt/*xblock, /mnt/<edx-ora, search, enterprise>)
+
+echo "Performing additional setup for bind-mounted directories."
+set -x # Echo out executed lines
+
+cd /openedx/edx-platform || exit 1
+
+# Whenever edx-platform or installable packages (e.g., xblocks) are mounted,
+# during the image build, they are copied over to container and installed. This
+# results in egg_info generation for the mounted directories. However, the
+# egg_info is not carried over to host. When the containers are launched, the
+# host directories without egg_info are mounted on runtime and disappear from
+# pip list. To fix this, we `pip install` edx-platform (".") and every mounted
+# package ("./mnt/*") again, re-generating the egg-infos.
+for mounted_dir in . /mnt/*; do
+    if [ -f $mounted_dir/setup.py ] && ! ls $mounted_dir/*.egg-info >/dev/null 2>&1 ; then
+        echo "Unable to locate egg-info in $mounted_dir -- generating now."
+        pip install -e $mounted_dir
+    fi
+done
+
+# The same problem exists for edx-platform's compiled frontend assets, but recompiling
+# them is very slow. So, instead of re-compiling, we create symlinks to an alternate
+# directory in which we expect them to have been cached (/openedx/assets).
+ln-assets /openedx/assets /openedx/edx-platform
+
+set -x
+echo "Done setting up bind-mounted directories."