multiarch build

getredash · Nov 7, 2023 · 0b64109 · 0b64109
1 parent 63cef66
commit 0b64109
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 66 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,5 +1,4 @@
 client/.tmp/
-client/dist/
 node_modules/
 viz-lib/node_modules/
 .tmp/

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,15 +2,16 @@ name: Tests
 on:
   push:
     branches:
-      - master
+      - multiarch-build
   pull_request:
 env:
   NODE_VERSION: 16.20.1
+  REGISTRY_IMAGE: achubatiuk/redash
 jobs:
   backend-lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - uses: actions/setup-python@v4
@@ -29,13 +30,16 @@ jobs:
       COMPOSE_DOCKER_CLI_BUILD: 1
       DOCKER_BUILDKIT: 1
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: Build Docker Images
         run: |
           set -x
-          docker compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true
+          mkdir client/dist
+          touch client/dist/multi_org.html
+          touch client/dist/index.html
+          docker compose build --build-arg install_groups="main,all_ds,dev"
           docker compose up -d
           sleep 10
       - name: Create Test Database
@@ -65,10 +69,10 @@ jobs:
   frontend-lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
@@ -88,17 +92,18 @@ jobs:
     runs-on: ubuntu-22.04
     needs: frontend-lint
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
       - name: Install Dependencies
         run: |
           npm install --global --force yarn@1.22.19
-          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
+          yarn cache clean
+          yarn --frozen-lockfile --network-concurrency 1
       - name: Run App Tests
         run: yarn test
       - name: Run Visualizations Tests
@@ -117,10 +122,10 @@ jobs:
       CYPRESS_INSTALL_BINARY: 0
       PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
@@ -150,6 +155,11 @@ jobs:
         with:
           name: coverage
           path: coverage
+      - uses: actions/upload-artifact@v3
+        with:
+          name: frontend
+          path: client/dist
+          retention-days: 1
 
   build-skip-check:
     runs-on: ubuntu-22.04
@@ -159,13 +169,13 @@ jobs:
       - name: Skip?
         id: skip-check
         run: |
-          if [[ "${{ vars.DOCKER_USER }}" == '' ]]; then
+          if [[ "${{ secrets.DOCKER_USER }}" == '' ]]; then
             echo 'Docker user is empty. Skipping build+push'
             echo skip=true >> "$GITHUB_OUTPUT"
           elif [[ "${{ secrets.DOCKER_PASS }}" == '' ]]; then
             echo 'Docker password is empty. Skipping build+push'
             echo skip=true >> "$GITHUB_OUTPUT"
-          elif [[ "${{ github.ref_name }}" != 'master' ]]; then
+          elif [[ "${{ github.ref_name }}" != 'multiarch-build' ]]; then
             echo 'Ref name is not `master`. Skipping build+push'
             echo skip=true >> "$GITHUB_OUTPUT"
           else
@@ -182,30 +192,32 @@ jobs:
       - frontend-e2e-tests
       - build-skip-check
     if: needs.build-skip-check.outputs.skip == 'false'
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/setup-node@v3
+
+      - uses: actions/download-artifact@v3
         with:
-          node-version: ${{ env.NODE_VERSION }}
-          cache: 'yarn'
-      - name: Install Dependencies
-        run: |
-          npm install --global --force yarn@1.22.19
-          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
+          name: frontend
 
       - name: Set up QEMU
         timeout-minutes: 1
-        uses: docker/setup-qemu-action@v2.2.0
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
-          username: ${{ vars.DOCKER_USER }}
+          username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
 
       - name: Bump version
@@ -218,21 +230,73 @@ jobs:
           echo "VERSION_TAG=$VERSION_TAG" >> "$GITHUB_OUTPUT"
 
       - name: Build and push preview image to Docker Hub
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
+        id: build
         with:
           push: true
-          tags: |
-            redash/redash:preview
-            redash/preview:${{ steps.version.outputs.VERSION_TAG }}
           context: .
           build-args: |
             test_all_deps=true
           cache-from: type=ghq
           cache-to: type=gha,mode=max
-          platforms: linux/amd64
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
         env:
           DOCKER_CONTENT_TRUST: true
 
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
       - name: "Failure: output container logs to console"
         if: failure()
         run: docker compose logs
+
+  publish-docker-manifest:
+    runs-on: ubuntu-22.04
+    needs:
+      - build-skip-check
+      - build-docker-image
+    if: needs.build-skip-check.outputs.skip == 'false'
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          tags: preview
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+          $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+      - name: Push image to redash/preview
+        run: |
+          docker buildx imagetools create \
+            --tag achubatiuk/preview:${{ steps.version.outputs.VERSION_TAG }} \
+            ${{ env.REGISTRY_IMAGE }}:preview
diff --git a/Dockerfile b/Dockerfile
@@ -1,30 +1,3 @@
-FROM node:16.20.1-bookworm as frontend-builder
-
-RUN npm install --global --force yarn@1.22.19
-
-# Controls whether to build the frontend assets
-ARG skip_frontend_build
-
-ENV CYPRESS_INSTALL_BINARY=0
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
-
-RUN useradd -m -d /frontend redash
-USER redash
-
-WORKDIR /frontend
-COPY --chown=redash package.json yarn.lock .yarnrc /frontend/
-COPY --chown=redash viz-lib /frontend/viz-lib
-
-# Controls whether to instrument code for coverage information
-ARG code_coverage
-ENV BABEL_ENV=${code_coverage:+test}
-
-RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn --frozen-lockfile --network-concurrency 1; fi
-
-COPY --chown=redash client /frontend/client
-COPY --chown=redash webpack.config.js /frontend/
-RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn build; else mkdir -p /frontend/client/dist && touch /frontend/client/dist/multi_org.html && touch /frontend/client/dist/index.html; fi
-
 FROM python:3.8-slim-bookworm
 
 EXPOSE 5000
@@ -62,21 +35,21 @@ RUN apt-get update && \
 
 
 ARG TARGETPLATFORM
-ARG databricks_odbc_driver_url=https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.6.26/SimbaSparkODBC-2.6.26.1045-Debian-64bit.zip
-RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
+ARG DATABRICKS_ODBC_DRIVER_URL=https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.5/SimbaSparkODBC-2.7.5.1012-Debian-64bit.zip
+RUN \
   curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \
   && curl https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list \
   && apt-get update \
-  && ACCEPT_EULA=Y apt-get install  -y --no-install-recommends msodbcsql17 \
+  && ACCEPT_EULA=Y apt-get install  -y --no-install-recommends msodbcsql18 \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
-  && curl "$databricks_odbc_driver_url" --location --output /tmp/simba_odbc.zip \
+  && curl "$DATABRICKS_ODBC_DRIVER_URL" --location --output /tmp/simba_odbc.zip \
   && chmod 600 /tmp/simba_odbc.zip \
   && unzip /tmp/simba_odbc.zip -d /tmp/simba \
   && dpkg -i /tmp/simba/*.deb \
   && printf "[Simba]\nDriver = /opt/simba/spark/lib/64/libsparkodbc_sb64.so" >> /etc/odbcinst.ini \
   && rm /tmp/simba_odbc.zip \
-  && rm -rf /tmp/simba; fi
+  && rm -rf /tmp/simba
 
 WORKDIR /app
 
@@ -94,7 +67,6 @@ ARG install_groups="main,all_ds,dev"
 RUN /etc/poetry/bin/poetry install --only $install_groups $POETRY_OPTIONS
 
 COPY --chown=redash . /app
-COPY --from=frontend-builder --chown=redash /frontend/client/dist /app/client/dist
 RUN chown redash /app
 USER redash
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,8 +4,6 @@ version: "2.2"
 x-redash-service: &redash-service
   build:
     context: .
-    args:
-      skip_frontend_build: "true"  # set to empty string to build
   volumes:
     - .:/app
   env_file: