diff --git a/.github/workflows/master_merge.yml b/.github/workflows/master_merge.yml index 2b53c90..b6edae0 100644 --- a/.github/workflows/master_merge.yml +++ b/.github/workflows/master_merge.yml @@ -33,6 +33,7 @@ jobs: path: | /tmp/.buildx-cache-alpine-2-11 /tmp/.buildx-cache-ubuntu-2-11 + /tmp/.buildx-cache-alpine-2-11-xloader key: ${{ runner.os }}-buildx-2-11-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-2-11 @@ -54,6 +55,23 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-11 cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-11 + - name: Get docker tag for alpine xloader image + id: alpine-xloader + run: | + echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile.xloader)" >> $GITHUB_OUTPUT + + - name: Build and push CKAN 2.11 alpine xloader + uses: docker/build-push-action@v5 + with: + context: ./images/ckan/2.11 + file: ./images/ckan/2.11/Dockerfile.xloader + push: true + tags: | + keitaro/ckan:${{ steps.alpine-xloader.outputs.IMAGE_TAG }} + ghcr.io/keitaroinc/ckan:${{ steps.alpine-xloader.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-11-xloader + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-11-xloader + - name: Get docker tag for Ubuntu image id: ubuntu run: | diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 350e4bb..300fb30 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -20,6 +20,7 @@ jobs: path: | /tmp/.buildx-cache-alpine-2-11 /tmp/.buildx-cache-ubuntu-2-11 + /tmp/.buildx-cache-alpine-2-11-xloader key: ${{ runner.os }}-buildx-2-11-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-2-11 @@ -39,6 +40,22 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-11 cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-11 + - name: Get docker tag for alpine xloader image + id: alpine-xloader + run: | + echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.11/Dockerfile.xloader)" >> $GITHUB_OUTPUT + + + - name: Build CKAN 2.11-xloader alpine + uses: docker/build-push-action@v5 + with: + context: ./images/ckan/2.11 + file: ./images/ckan/2.11/Dockerfile.xloader + push: false + tags: keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-11-xloader + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-11-xloader + - name: Get docker tag for Ubuntu image id: ubuntu run: | diff --git a/images/ckan/2.11/Dockerfile.xloader b/images/ckan/2.11/Dockerfile.xloader new file mode 100644 index 0000000..07e18d4 --- /dev/null +++ b/images/ckan/2.11/Dockerfile.xloader @@ -0,0 +1,213 @@ +################## +### Build CKAN ### +################## +FROM alpine:3.17.2 AS ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.11.0-xloader + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.11.0 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Packages to build CKAN requirements and plugins +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl \ + postgresql-dev \ + linux-headers \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + pcre-dev \ + pcre \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +# RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 + + +########################### +### Default-Extensions #### +########################### +FROM alpine:3.17.2 AS extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.2 + +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM alpine:3.17.2 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +# Xloader version control + +ENV XLOADER_URL=https://github.com/ckan/ckanext-xloader.git +ENV XLOADER_VERSION=1.2.0 + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore xloader + +# Install necessary packages to run CKAN +RUN apk add --no-cache \ + supervisor \ + python3 \ + bash \ + git \ + gettext \ + curl \ + postgresql-client \ + libmagic \ + pcre \ + libxslt \ + libxml2 \ + tzdata \ + apache2-utils && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} + + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +### XLoader ### +RUN pip3 install -e git+${XLOADER_URL}@${XLOADER_VERSION}#egg=ckanext-xloader && \ + pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt && \ + pip3 install -U requests[security] + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2 + +# Create a local user and group to run the app +RUN addgroup -g 92 -S ckan && \ + adduser -u 92 -h /srv/app -H -D -S -G ckan ckan + +WORKDIR ${CKAN_DIR} + +#Removes bugged package +RUN rm -rf /usr/lib/python3.10/site-packages/packaging* +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Set timezone + echo "UTC" > /etc/timezone && \ + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +RUN mkdir docker-afterinit.d + +RUN pip install supervisor +RUN mkdir /var/supervisor +RUN mkdir /var/log/ckan +RUN mkdir /etc/supervisor/conf.d/ -p +COPY supervisorvalidation.conf /etc/supervisor/conf.d/supervisor-ckan-worker.conf +COPY supervisord.conf /etc/supervisord.conf +COPY supervisorstart.sh /srv/app/docker-afterinit.d + +# Change ownership to app user +RUN chown -R ckan:ckan ${APP_DIR} /var/supervisor /var/log/ckan /etc/supervisor/conf.d/ + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] \ No newline at end of file diff --git a/images/ckan/2.11/supervisord.conf b/images/ckan/2.11/supervisord.conf new file mode 100644 index 0000000..82c3d19 --- /dev/null +++ b/images/ckan/2.11/supervisord.conf @@ -0,0 +1,23 @@ +; supervisor config file +[unix_http_server] +file=/var/supervisor/supervisor.sock ; (the path to the socket file) +chmod=0700 ; sockef file mode (default 0700) +[supervisord] +logfile=/var/log/ckan/supervisord.log ; (main log file;default $CWD/supervisord.log) +;pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +;childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +user=ckan +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface +[supervisorctl] +serverurl=unix:///var/supervisor/supervisor.sock ; use a unix:// URL for a unix socket +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. +[include] +files = /etc/supervisor/conf.d/*.conf \ No newline at end of file diff --git a/images/ckan/2.11/supervisorstart.sh b/images/ckan/2.11/supervisorstart.sh new file mode 100644 index 0000000..7c4858c --- /dev/null +++ b/images/ckan/2.11/supervisorstart.sh @@ -0,0 +1,2 @@ +#!/bin/sh +supervisord \ No newline at end of file diff --git a/images/ckan/2.11/supervisorvalidation.conf b/images/ckan/2.11/supervisorvalidation.conf new file mode 100644 index 0000000..d6082cf --- /dev/null +++ b/images/ckan/2.11/supervisorvalidation.conf @@ -0,0 +1,54 @@ +; ======================================================= +; Supervisor configuration for CKAN background job worker +; ======================================================= +; 1. Copy this file to /etc/supervisor/conf.d +; 2. Make sure the paths below match your setup +[program:ckan-worker-default] +command=ckan -c /srv/app/production.ini jobs worker default +; Log files. +redirect_stderr=true +stdout_logfile=/var/log/ckan/ckan-worker.log +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 + +[program:ckan-worker-bulk] +command=ckan -c /srv/app/production.ini jobs worker bulk +; Log files. +redirect_stderr=true +stdout_logfile=/var/log/ckan/ckan-worker.log +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 + +[program:ckan-worker-priority] +command=ckan -c /srv/app/production.ini jobs worker priority +numprocs=2 +process_name=%(program_name)s-%(process_num)02d +; Log files. +redirect_stderr=true +stdout_logfile=/var/log/ckan/ckan-worker.log +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 \ No newline at end of file