From 277dfa3381ae67174d25d2f6b68d0fb4e5af1031 Mon Sep 17 00:00:00 2001 From: YoshihitoAso Date: Fri, 5 Jul 2024 16:13:34 +0900 Subject: [PATCH 1/2] Multi-stage builds and slimmed down image sizes --- .dockerignore | 7 +++ .github/actions/setup-test-image/action.yml | 22 +++++++ .github/workflows/pr.yml | 10 +++ Dockerfile | 68 ++++++++++++++------ tests/Dockerfile_unittest | 69 +++++++++++++++------ 5 files changed, 138 insertions(+), 38 deletions(-) create mode 100644 .github/actions/setup-test-image/action.yml diff --git a/.dockerignore b/.dockerignore index 2fc1cbae..56541493 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,10 @@ .github/ .python-version /node_modules/ +**/__pycache__/ +**/*.py[cod] +.cache/ +.pytest_cache/ +.git/ +.idea/ +.DS_Store diff --git a/.github/actions/setup-test-image/action.yml b/.github/actions/setup-test-image/action.yml new file mode 100644 index 00000000..502c7366 --- /dev/null +++ b/.github/actions/setup-test-image/action.yml @@ -0,0 +1,22 @@ +name: 'Setup Docker Build' +description: 'Setup docker builder for using github actions cache' +runs: + using: "composite" + steps: + - uses: docker/setup-buildx-action@v3 + - name: Set metadata + id: metadata + uses: docker/metadata-action@v5 + with: + images: ibet-prime + tags: | + type=sha,format=short + - uses: docker/build-push-action@v6 + with: + context: . + file: ./tests/Dockerfile_unittest + tags: ${{ steps.metadata.outputs.tags }} + push: false + load: true + cache-from: type=gha + cache-to: type=gha,mode=min \ No newline at end of file diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8da10e9f..0c6f5f89 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -2,6 +2,12 @@ name: Unit Test on: [pull_request] jobs: + build: + name: 'Build docker image' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-test-image lint-black: name: 'Lint check (black)' runs-on: ubuntu-latest @@ -11,14 +17,18 @@ jobs: unit-test-postgres: name: 'Unit tests (PostgreSQL)' runs-on: ubuntu-latest + needs: build steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-test-image - name: run unit test using postgres run: docker compose run ibet-prime-postgres migration-test-postgres: name: 'Migration tests (PostgreSQL)' runs-on: ubuntu-latest + needs: build steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-test-image - name: run unit test using postgres run: docker compose run ibet-prime-postgres bash --login -c "cd /app/ibet-Prime && pytest -vv --test-alembic -m 'alembic'" diff --git a/Dockerfile b/Dockerfile index 11cc55d0..6ddfb92a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ -FROM ubuntu:22.04 +FROM ubuntu:22.04 AS builder + +ENV PYTHON_VERSION=3.12.2 +ENV POETRY_VERSION=1.8.2 # make application directory -RUN mkdir -p /app/ibet-Prime/ +RUN mkdir -p /app # add apl user/group RUN groupadd -g 1000 apl \ @@ -34,37 +37,34 @@ RUN apt-get update -q \ git \ libyaml-cpp-dev \ libc-bin \ - liblzma-dev - -# remove unnessesory package files -RUN apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + liblzma-dev \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # install pyenv RUN git clone https://github.com/pyenv/pyenv.git /home/apl/.pyenv RUN chown -R apl:apl /home/apl - USER apl RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~apl/.bash_profile \ && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~apl/.bash_profile \ + && echo 'export POETRY_CACHE_DIR=/tmp/poetry_cache' >> ~apl/.bash_profile \ && echo 'eval "$(pyenv init --path)"' >> ~apl/.bash_profile \ && echo 'export LANG=ja_JP.utf8' >> ~apl/.bash_profile # install python -USER apl RUN . ~/.bash_profile \ - && pyenv install 3.12.2 \ - && pyenv global 3.12.2 \ - && pip install --upgrade pip setuptools + && pyenv install $PYTHON_VERSION \ + && pyenv global $PYTHON_VERSION \ + && pip install --upgrade --no-cache-dir pip setuptools # install poetry RUN . ~/.bash_profile \ - && python -m pip install poetry==1.8.2 -RUN . ~/.bash_profile \ + && python -m pip install poetry==$POETRY_VERSION \ + && . ~/.bash_profile \ && poetry config virtualenvs.create false \ && poetry config installer.max-workers 1 # install python packages -USER root COPY --chown=apl:apl LICENSE /app/ibet-Prime/ RUN mkdir -p /app/ibet-Prime/bin/ COPY --chown=apl:apl bin/ /app/ibet-Prime/bin/ @@ -84,7 +84,6 @@ COPY --chown=apl:apl app/ /app/ibet-Prime/app/ RUN find /app/ibet-Prime/ -type d -name __pycache__ | xargs rm -fr \ && chmod -R 755 /app/ibet-Prime/ -USER apl COPY pyproject.toml /app/ibet-Prime/pyproject.toml COPY poetry.lock /app/ibet-Prime/poetry.lock RUN . ~/.bash_profile \ @@ -92,12 +91,43 @@ RUN . ~/.bash_profile \ && poetry install --only main --no-root --all-extras \ && rm -f /app/ibet-Prime/pyproject.toml \ && rm -f /app/ibet-Prime/poetry.lock -ENV PYTHONPATH /app/ibet-Prime:/app/ibet-Prime/cmd -# command deploy +FROM ubuntu:22.04 AS runner + +# make application directory +RUN mkdir -p /app + +# add apl user/group +RUN groupadd -g 1000 apl \ + && useradd -g apl -s /bin/bash -u 1000 -p apl apl \ + && echo 'apl ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \ + && chown -R apl:apl /app + +# install packages +RUN apt-get update -q \ + && apt-get upgrade -qy \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + libssl-dev \ + libpq-dev \ + language-pack-ja-base \ + language-pack-ja \ + jq \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# copy python and dependencies from builder stage USER apl +COPY --from=builder --chown=apl:apl /home/apl/ /home/apl/ +COPY --from=builder --chown=apl:apl /app/ibet-Prime/ /app/ibet-Prime/ +RUN . ~/.bash_profile + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH /app/ibet-Prime:/app/ibet-Prime/cmd + COPY run.sh healthcheck.sh /app/ EXPOSE 5000 -CMD /app/run.sh -HEALTHCHECK --interval=10s CMD /app/healthcheck.sh \ No newline at end of file +CMD ["/app/run.sh"] +HEALTHCHECK --interval=10s CMD ["/app/healthcheck.sh"] \ No newline at end of file diff --git a/tests/Dockerfile_unittest b/tests/Dockerfile_unittest index 224711ae..fa20e9f4 100644 --- a/tests/Dockerfile_unittest +++ b/tests/Dockerfile_unittest @@ -1,7 +1,10 @@ -FROM ubuntu:22.04 +FROM ubuntu:22.04 AS builder + +ENV PYTHON_VERSION=3.12.2 +ENV POETRY_VERSION=1.8.2 # make application directory -RUN mkdir -p /app/ibet-Prime/ +RUN mkdir -p /app # add apl user/group RUN groupadd -g 1000 apl \ @@ -34,37 +37,34 @@ RUN apt-get update -q \ git \ libyaml-cpp-dev \ libc-bin \ - liblzma-dev - -# remove unnessesory package files -RUN apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + liblzma-dev \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # install pyenv RUN git clone https://github.com/pyenv/pyenv.git /home/apl/.pyenv RUN chown -R apl:apl /home/apl - USER apl RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~apl/.bash_profile \ && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~apl/.bash_profile \ + && echo 'export POETRY_CACHE_DIR=/tmp/poetry_cache' >> ~apl/.bash_profile \ && echo 'eval "$(pyenv init --path)"' >> ~apl/.bash_profile \ && echo 'export LANG=ja_JP.utf8' >> ~apl/.bash_profile # install python -USER apl RUN . ~/.bash_profile \ - && pyenv install 3.12.2 \ - && pyenv global 3.12.2 \ - && pip install --upgrade pip setuptools + && pyenv install $PYTHON_VERSION \ + && pyenv global $PYTHON_VERSION \ + && pip install --upgrade --no-cache-dir pip setuptools # install poetry RUN . ~/.bash_profile \ - && python -m pip install poetry==1.8.2 -RUN . ~/.bash_profile \ + && python -m pip install poetry==$POETRY_VERSION \ + && . ~/.bash_profile \ && poetry config virtualenvs.create false \ && poetry config installer.max-workers 1 # install python packages -USER root COPY --chown=apl:apl LICENSE /app/ibet-Prime/ RUN mkdir -p /app/ibet-Prime/bin/ COPY --chown=apl:apl bin/ /app/ibet-Prime/bin/ @@ -84,7 +84,6 @@ COPY --chown=apl:apl app/ /app/ibet-Prime/app/ RUN find /app/ibet-Prime/ -type d -name __pycache__ | xargs rm -fr \ && chmod -R 755 /app/ibet-Prime/ -USER apl COPY pyproject.toml /app/ibet-Prime/pyproject.toml COPY poetry.lock /app/ibet-Prime/poetry.lock RUN . ~/.bash_profile \ @@ -93,14 +92,46 @@ RUN . ~/.bash_profile \ && rm -f /app/ibet-Prime/pyproject.toml \ && rm -f /app/ibet-Prime/poetry.lock -# test build layer +FROM ubuntu:22.04 AS runner + +# make application directory +RUN mkdir -p /app + +# add apl user/group +RUN groupadd -g 1000 apl \ + && useradd -g apl -s /bin/bash -u 1000 -p apl apl \ + && echo 'apl ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \ + && chown -R apl:apl /app + +# install packages +RUN apt-get update -q \ + && apt-get upgrade -qy \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + libssl-dev \ + libpq-dev \ + language-pack-ja-base \ + language-pack-ja \ + jq \ + libsqlite3-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# copy test cases USER root RUN mkdir -p /app/ibet-Prime/tests/ +RUN mkdir -p /app/ibet-Prime/cov/ COPY --chown=apl:apl tests/ /app/ibet-Prime/tests/ RUN chmod -R 755 /app/ibet-Prime/tests/ -ENV PYTHONPATH /app/ibet-Prime:/app/ibet-Prime/cmd +# copy python and dependencies from builder stage USER apl -RUN mkdir -p /app/ibet-Prime/cov/ +COPY --from=builder --chown=apl:apl /home/apl/ /home/apl/ +COPY --from=builder --chown=apl:apl /app/ibet-Prime/ /app/ibet-Prime/ +RUN . ~/.bash_profile + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH /app/ibet-Prime:/app/ibet-Prime/cmd -CMD /app/ibet-Prime/tests/qa.sh \ No newline at end of file +CMD ["/app/ibet-Prime/tests/qa.sh"] \ No newline at end of file From 174061173fcfa4c6bdf097987a38465b24ba321b Mon Sep 17 00:00:00 2001 From: YoshihitoAso Date: Fri, 5 Jul 2024 17:58:26 +0900 Subject: [PATCH 2/2] fix --- Dockerfile | 2 +- tests/Dockerfile_unittest | 4 ++-- tests/qa.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6ddfb92a..6ca93562 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,7 +95,7 @@ RUN . ~/.bash_profile \ FROM ubuntu:22.04 AS runner # make application directory -RUN mkdir -p /app +RUN mkdir -p /app/ibet-Prime/ # add apl user/group RUN groupadd -g 1000 apl \ diff --git a/tests/Dockerfile_unittest b/tests/Dockerfile_unittest index fa20e9f4..bce814d2 100644 --- a/tests/Dockerfile_unittest +++ b/tests/Dockerfile_unittest @@ -95,7 +95,7 @@ RUN . ~/.bash_profile \ FROM ubuntu:22.04 AS runner # make application directory -RUN mkdir -p /app +RUN mkdir -p /app/ibet-Prime/ # add apl user/group RUN groupadd -g 1000 apl \ @@ -119,7 +119,7 @@ RUN apt-get update -q \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # copy test cases -USER root +USER apl RUN mkdir -p /app/ibet-Prime/tests/ RUN mkdir -p /app/ibet-Prime/cov/ COPY --chown=apl:apl tests/ /app/ibet-Prime/tests/ diff --git a/tests/qa.sh b/tests/qa.sh index ca887a38..e78ce30a 100755 --- a/tests/qa.sh +++ b/tests/qa.sh @@ -24,7 +24,7 @@ cd /app/ibet-Prime sleep 10 # test -pytest tests/ -v --cov=app/routers/ --cov-report=xml --cov-branch -vv +pytest tests/ -v --cov=app/routers/ --cov-report=xml --cov-branch status_code=$?