From 4994ff9e6f1dd477a90a6d00da88259566fc2110 Mon Sep 17 00:00:00 2001 From: Aleksey Karpov <86011874+alekseyolg@users.noreply.github.com> Date: Fri, 23 Jun 2023 22:43:25 +0300 Subject: [PATCH] build(docker): simplification of assembly to facilitate further support --- .github/workflows/docker.yml | 2 +- .github/workflows/docker_build_push.sh | 8 ++-- Dockerfile | 64 ++++++++++---------------- docker/docker-bootstrap.sh | 38 +++++++++------ 4 files changed, 52 insertions(+), 60 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7c5b50baf430f..8a081a303c27e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -50,7 +50,7 @@ jobs: mkdir -p ./build echo ${{ github.sha }} > ./build/SHA echo ${{ github.event.pull_request.number }} > ./build/PR-NUM - docker build --target ci -t ${{ github.sha }} -t "pr-${{ github.event.pull_request.number }}" . + DOCKER_BUILDKIT=1 docker build --target ci -t ${{ github.sha }} -t "pr-${{ github.event.pull_request.number }}" . docker save ${{ github.sha }} | gzip > ./build/${{ github.sha }}.tar.gz - name: Upload build artifacts diff --git a/.github/workflows/docker_build_push.sh b/.github/workflows/docker_build_push.sh index 55ccf51767a1d..3ee615bc014c7 100755 --- a/.github/workflows/docker_build_push.sh +++ b/.github/workflows/docker_build_push.sh @@ -46,7 +46,7 @@ EOF # # Build the "lean" image # -docker build --target lean \ +DOCKER_BUILDKIT=1 docker build --target lean \ -t "${REPO_NAME}:${SHA}" \ -t "${REPO_NAME}:${REFSPEC}" \ -t "${REPO_NAME}:${LATEST_TAG}" \ @@ -59,7 +59,7 @@ docker build --target lean \ # # Build the "lean310" image # -docker build --target lean \ +DOCKER_BUILDKIT=1 docker build --target lean \ -t "${REPO_NAME}:${SHA}-py310" \ -t "${REPO_NAME}:${REFSPEC}-py310" \ -t "${REPO_NAME}:${LATEST_TAG}-py310" \ @@ -73,7 +73,7 @@ docker build --target lean \ # # Build the "websocket" image # -docker build \ +DOCKER_BUILDKIT=1 docker build \ -t "${REPO_NAME}:${SHA}-websocket" \ -t "${REPO_NAME}:${REFSPEC}-websocket" \ -t "${REPO_NAME}:${LATEST_TAG}-websocket" \ @@ -86,7 +86,7 @@ docker build \ # # Build the dev image # -docker build --target dev \ +DOCKER_BUILDKIT=1 docker build --target dev \ -t "${REPO_NAME}:${SHA}-dev" \ -t "${REPO_NAME}:${REFSPEC}-dev" \ -t "${REPO_NAME}:${LATEST_TAG}-dev" \ diff --git a/Dockerfile b/Dockerfile index 494bf90bd19cb..a5ee4e0a7168d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,17 +29,17 @@ ENV BUILD_CMD=${NPM_BUILD_CMD} ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true # NPM ci first, as to NOT invalidate previous steps except for when package.json changes -RUN mkdir -p /app/superset-frontend +WORKDIR /app/superset-frontend COPY ./docker/frontend-mem-nag.sh / -RUN /frontend-mem-nag.sh -WORKDIR /app/superset-frontend/ +RUN /frontend-mem-nag.sh COPY superset-frontend/package*.json ./ + RUN npm ci -COPY ./superset-frontend . +COPY ./superset-frontend ./ # This seems to be the most expensive step RUN npm run ${BUILD_CMD} @@ -49,6 +49,7 @@ RUN npm run ${BUILD_CMD} ###################################################################### FROM python:${PY_VER} AS lean +WORKDIR /app ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ FLASK_ENV=production \ @@ -59,8 +60,8 @@ ENV LANG=C.UTF-8 \ RUN mkdir -p ${PYTHONPATH} \ && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ - && apt-get update -y \ - && apt-get install -y --no-install-recommends \ + && apt-get update -q \ + && apt-get install -yq --no-install-recommends \ build-essential \ curl \ default-libmysqlclient-dev \ @@ -71,11 +72,8 @@ RUN mkdir -p ${PYTHONPATH} \ libldap2-dev \ && rm -rf /var/lib/apt/lists/* -WORKDIR /app - COPY --chown=superset:superset ./requirements/*.txt requirements/ COPY --chown=superset:superset setup.py MANIFEST.in README.md ./ - # setup.py uses the version information in package.json COPY --chown=superset:superset superset-frontend/package.json superset-frontend/ @@ -84,16 +82,14 @@ RUN mkdir -p superset/static \ && pip install --no-cache-dir -r requirements/local.txt COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets - ## Lastly, let's install superset itself COPY --chown=superset:superset superset superset + RUN chown -R superset:superset ./* \ && pip install --no-cache-dir -e . \ && flask fab babel-compile --target superset/translations -COPY ./docker/run-server.sh /usr/bin/ -RUN chmod a+x /usr/bin/run-server.sh - +COPY --chmod=755 ./docker/run-server.sh /usr/bin/ USER superset HEALTHCHECK CMD curl -f "http://localhost:$SUPERSET_PORT/health" @@ -109,47 +105,35 @@ FROM lean AS dev ARG GECKODRIVER_VERSION=v0.32.0 ARG FIREFOX_VERSION=106.0.3 -COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/ - USER root -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends \ +RUN apt-get update -q \ + && apt-get install -yq --no-install-recommends \ libnss3 \ libdbus-glib-1-2 \ libgtk-3-0 \ libx11-xcb1 \ libasound2 \ libxtst6 \ - wget - -# Install GeckoDriver WebDriver -RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O /tmp/geckodriver.tar.gz && \ - tar xvfz /tmp/geckodriver.tar.gz -C /tmp && \ - mv /tmp/geckodriver /usr/local/bin/geckodriver && \ - rm /tmp/geckodriver.tar.gz - -# Install Firefox -RUN wget https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O /opt/firefox.tar.bz2 && \ - tar xvf /opt/firefox.tar.bz2 -C /opt && \ - ln -s /opt/firefox/firefox /usr/local/bin/firefox + wget \ + # Install GeckoDriver WebDriver + && wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \ + # Install Firefox + && wget https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \ + && ln -s /opt/firefox/firefox /usr/local/bin/firefox \ + && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/* && apt-get clean +COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/ # Cache everything for dev purposes... -RUN cd /app \ - && pip install --no-cache -r requirements/docker.txt \ - && pip install --no-cache -r requirements/requirements-local.txt || true -USER superset - +RUN pip install --no-cache-dir -r /app/requirements/docker.txt \ + && pip install --no-cache-dir -r /app/requirements/requirements-local.txt || true +USER superset ###################################################################### # CI image... ###################################################################### FROM lean AS ci -COPY --chown=superset ./docker/docker-bootstrap.sh /app/docker/ -COPY --chown=superset ./docker/docker-init.sh /app/docker/ -COPY --chown=superset ./docker/docker-ci.sh /app/docker/ - -RUN chmod a+x /app/docker/*.sh +COPY --chown=superset --chmod=755 ./docker/*.sh /app/docker/ -CMD /app/docker/docker-ci.sh +CMD ["/app/docker/docker-ci.sh"] diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh index 6b92e970028b4..ad6ee57616fd6 100755 --- a/docker/docker-bootstrap.sh +++ b/docker/docker-bootstrap.sh @@ -30,22 +30,30 @@ fi # if [ -f "${REQUIREMENTS_LOCAL}" ]; then echo "Installing local overrides at ${REQUIREMENTS_LOCAL}" - pip install -r "${REQUIREMENTS_LOCAL}" + pip install --no-cache-dir -r "${REQUIREMENTS_LOCAL}" else echo "Skipping local overrides" fi -if [[ "${1}" == "worker" ]]; then - echo "Starting Celery worker..." - celery --app=superset.tasks.celery_app:app worker -O fair -l INFO -elif [[ "${1}" == "beat" ]]; then - echo "Starting Celery beat..." - rm -f /tmp/celerybeat.pid - celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule -elif [[ "${1}" == "app" ]]; then - echo "Starting web app..." - flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0 -elif [[ "${1}" == "app-gunicorn" ]]; then - echo "Starting web app..." - /usr/bin/run-server.sh -fi +case "${1}" in + worker) + echo "Starting Celery worker..." + celery --app=superset.tasks.celery_app:app worker -O fair -l INFO + ;; + beat) + echo "Starting Celery beat..." + rm -f /tmp/celerybeat.pid + celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule + ;; + app) + echo "Starting web app..." + flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0 + ;; + app-gunicorn) + echo "Starting web app..." + /usr/bin/run-server.sh + ;; + *) + echo "Unknown Operation!!!" + ;; +esac