Merge pull request #658 from BoostryJP/feature/image-build

Multi-stage builds and slimmed down image sizes

.dockerignore

@@ -1,3 +1,10 @@
 .github/
 .python-version
 /node_modules/
+**/__pycache__/
+**/*.py[cod]
+.cache/
+.pytest_cache/
+.git/
+.idea/
+.DS_Store

.github/actions/setup-test-image/action.yml

@@ -0,0 +1,22 @@
+name: 'Setup Docker Build'
+description: 'Setup docker builder for using github actions cache'
+runs:
+  using: "composite"
+  steps:
+    - uses: docker/setup-buildx-action@v3
+    - name: Set metadata
+      id: metadata
+      uses: docker/metadata-action@v5
+      with:
+        images: ibet-prime
+        tags: |
+          type=sha,format=short
+    - uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./tests/Dockerfile_unittest
+        tags: ${{ steps.metadata.outputs.tags }}
+        push: false
+        load: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=min

.github/workflows/pr.yml

@@ -2,6 +2,12 @@ name: Unit Test
 on: [pull_request]
 
 jobs:
+  build:
+    name: 'Build docker image'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-test-image
   lint-black:
     name: 'Lint check (black)'
     runs-on: ubuntu-latest
@@ -11,14 +17,18 @@ jobs:
   unit-test-postgres:
     name: 'Unit tests (PostgreSQL)'
     runs-on: ubuntu-latest
+    needs: build
     steps:
       - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-test-image
       - name: run unit test using postgres
         run: docker compose run ibet-prime-postgres
   migration-test-postgres:
     name: 'Migration tests (PostgreSQL)'
     runs-on: ubuntu-latest
+    needs: build
     steps:
       - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-test-image
       - name: run unit test using postgres
         run: docker compose run ibet-prime-postgres bash --login -c "cd /app/ibet-Prime && pytest -vv --test-alembic -m 'alembic'"

Dockerfile

@@ -1,7 +1,10 @@
-FROM ubuntu:22.04
+FROM ubuntu:22.04 AS builder
+
+ENV PYTHON_VERSION=3.12.2
+ENV POETRY_VERSION=1.8.2
 
 # make application directory
-RUN mkdir -p /app/ibet-Prime/
+RUN mkdir -p /app
 
 # add apl user/group
 RUN groupadd -g 1000 apl \
@@ -34,37 +37,34 @@ RUN apt-get update -q \
  git \
  libyaml-cpp-dev \
  libc-bin \
- liblzma-dev
-
-# remove unnessesory package files
-RUN apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+ liblzma-dev \
+ && apt clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # install pyenv
 RUN git clone https://github.com/pyenv/pyenv.git /home/apl/.pyenv
 RUN chown -R apl:apl /home/apl
-
 USER apl
 RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~apl/.bash_profile \
  && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~apl/.bash_profile \
+ && echo 'export POETRY_CACHE_DIR=/tmp/poetry_cache' >> ~apl/.bash_profile \
  && echo 'eval "$(pyenv init --path)"' >> ~apl/.bash_profile \
  && echo 'export LANG=ja_JP.utf8' >> ~apl/.bash_profile
 
 # install python
-USER apl
 RUN . ~/.bash_profile \
- && pyenv install 3.12.2 \
- && pyenv global 3.12.2 \
- && pip install --upgrade pip setuptools
+ && pyenv install $PYTHON_VERSION \
+ && pyenv global $PYTHON_VERSION \
+ && pip install --upgrade --no-cache-dir pip setuptools
 
 # install poetry
 RUN . ~/.bash_profile \
- && python -m pip install poetry==1.8.2
-RUN . ~/.bash_profile \
+ && python -m pip install poetry==$POETRY_VERSION \
+ && . ~/.bash_profile \
  && poetry config virtualenvs.create false \
  && poetry config installer.max-workers 1
 
 # install python packages
-USER root
 COPY --chown=apl:apl LICENSE /app/ibet-Prime/
 RUN mkdir -p /app/ibet-Prime/bin/
 COPY --chown=apl:apl bin/ /app/ibet-Prime/bin/
@@ -84,20 +84,50 @@ COPY --chown=apl:apl app/ /app/ibet-Prime/app/
 RUN find /app/ibet-Prime/ -type d -name __pycache__ | xargs rm -fr \
  && chmod -R 755 /app/ibet-Prime/
 
-USER apl
 COPY pyproject.toml /app/ibet-Prime/pyproject.toml
 COPY poetry.lock /app/ibet-Prime/poetry.lock
 RUN . ~/.bash_profile \
  && cd /app/ibet-Prime \
  && poetry install --only main --no-root --all-extras \
  && rm -f /app/ibet-Prime/pyproject.toml \
  && rm -f /app/ibet-Prime/poetry.lock
-ENV PYTHONPATH /app/ibet-Prime:/app/ibet-Prime/cmd
 
-# command deploy
+FROM ubuntu:22.04 AS runner
+
+# make application directory
+RUN mkdir -p /app/ibet-Prime/
+
+# add apl user/group
+RUN groupadd -g 1000 apl \
+ && useradd -g apl -s /bin/bash -u 1000 -p apl apl \
+ && echo 'apl ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
+ && chown -R apl:apl /app
+
+# install packages
+RUN apt-get update -q \
+  && apt-get upgrade -qy \
+  && apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  libssl-dev \
+  libpq-dev \
+  language-pack-ja-base \
+  language-pack-ja \
+  jq \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# copy python and dependencies from builder stage
 USER apl
+COPY --from=builder --chown=apl:apl /home/apl/ /home/apl/
+COPY --from=builder --chown=apl:apl /app/ibet-Prime/ /app/ibet-Prime/
+RUN . ~/.bash_profile
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH /app/ibet-Prime:/app/ibet-Prime/cmd
+
 COPY run.sh healthcheck.sh /app/
 
 EXPOSE 5000
-CMD /app/run.sh
-HEALTHCHECK --interval=10s CMD /app/healthcheck.sh
+CMD ["/app/run.sh"]
+HEALTHCHECK --interval=10s CMD ["/app/healthcheck.sh"]

tests/Dockerfile_unittest

@@ -1,7 +1,10 @@
-FROM ubuntu:22.04
+FROM ubuntu:22.04 AS builder
+
+ENV PYTHON_VERSION=3.12.2
+ENV POETRY_VERSION=1.8.2
 
 # make application directory
-RUN mkdir -p /app/ibet-Prime/
+RUN mkdir -p /app
 
 # add apl user/group
 RUN groupadd -g 1000 apl \
@@ -34,37 +37,34 @@ RUN apt-get update -q \
  git \
  libyaml-cpp-dev \
  libc-bin \
- liblzma-dev
-
-# remove unnessesory package files
-RUN apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+ liblzma-dev \
+ && apt clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # install pyenv
 RUN git clone https://github.com/pyenv/pyenv.git /home/apl/.pyenv
 RUN chown -R apl:apl /home/apl
-
 USER apl
 RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~apl/.bash_profile \
  && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~apl/.bash_profile \
+ && echo 'export POETRY_CACHE_DIR=/tmp/poetry_cache' >> ~apl/.bash_profile \
  && echo 'eval "$(pyenv init --path)"' >> ~apl/.bash_profile \
  && echo 'export LANG=ja_JP.utf8' >> ~apl/.bash_profile
 
 # install python
-USER apl
 RUN . ~/.bash_profile \
- && pyenv install 3.12.2 \
- && pyenv global 3.12.2 \
- && pip install --upgrade pip setuptools
+ && pyenv install $PYTHON_VERSION \
+ && pyenv global $PYTHON_VERSION \
+ && pip install --upgrade --no-cache-dir pip setuptools
 
 # install poetry
 RUN . ~/.bash_profile \
- && python -m pip install poetry==1.8.2
-RUN . ~/.bash_profile \
+ && python -m pip install poetry==$POETRY_VERSION \
+ && . ~/.bash_profile \
  && poetry config virtualenvs.create false \
  && poetry config installer.max-workers 1
 
 # install python packages
-USER root
 COPY --chown=apl:apl LICENSE /app/ibet-Prime/
 RUN mkdir -p /app/ibet-Prime/bin/
 COPY --chown=apl:apl bin/ /app/ibet-Prime/bin/
@@ -84,7 +84,6 @@ COPY --chown=apl:apl app/ /app/ibet-Prime/app/
 RUN find /app/ibet-Prime/ -type d -name __pycache__ | xargs rm -fr \
  && chmod -R 755 /app/ibet-Prime/
 
-USER apl
 COPY pyproject.toml /app/ibet-Prime/pyproject.toml
 COPY poetry.lock /app/ibet-Prime/poetry.lock
 RUN . ~/.bash_profile \
@@ -93,14 +92,46 @@ RUN . ~/.bash_profile \
  && rm -f /app/ibet-Prime/pyproject.toml \
  && rm -f /app/ibet-Prime/poetry.lock
 
-# test build layer
-USER root
+FROM ubuntu:22.04 AS runner
+
+# make application directory
+RUN mkdir -p /app/ibet-Prime/
+
+# add apl user/group
+RUN groupadd -g 1000 apl \
+ && useradd -g apl -s /bin/bash -u 1000 -p apl apl \
+ && echo 'apl ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
+ && chown -R apl:apl /app
+
+# install packages
+RUN apt-get update -q \
+  && apt-get upgrade -qy \
+  && apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  libssl-dev \
+  libpq-dev \
+  language-pack-ja-base \
+  language-pack-ja \
+  jq \
+  libsqlite3-dev \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# copy test cases
+USER apl
 RUN mkdir -p /app/ibet-Prime/tests/
+RUN mkdir -p /app/ibet-Prime/cov/
 COPY --chown=apl:apl tests/ /app/ibet-Prime/tests/
 RUN chmod -R 755 /app/ibet-Prime/tests/
-ENV PYTHONPATH /app/ibet-Prime:/app/ibet-Prime/cmd
 
+# copy python and dependencies from builder stage
 USER apl
-RUN mkdir -p /app/ibet-Prime/cov/
+COPY --from=builder --chown=apl:apl /home/apl/ /home/apl/
+COPY --from=builder --chown=apl:apl /app/ibet-Prime/ /app/ibet-Prime/
+RUN . ~/.bash_profile
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH /app/ibet-Prime:/app/ibet-Prime/cmd
 
-CMD /app/ibet-Prime/tests/qa.sh
+CMD ["/app/ibet-Prime/tests/qa.sh"]

tests/qa.sh

@@ -24,7 +24,7 @@ cd /app/ibet-Prime
 sleep 10
 
 # test
-pytest tests/ -v --cov=app/routers/ --cov-report=xml --cov-branch -vv
+pytest tests/ -v --cov=app/routers/ --cov-report=xml --cov-branch
 
 status_code=$?