From 1f7b964d9b72f95ae37276475903a4b222381547 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 14:38:40 -0400 Subject: [PATCH 01/10] Podman support --- .dockerignore | 3 +++ .github/workflows/ci.yml | 11 ++++++++++ .gitignore | 3 +++ README.md | 24 +++++++++++++++++++++- docker-compose_just.yml | 4 ++-- justfile | 43 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 84 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index 142895ab..e687fd57 100755 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,6 @@ CHRIS_REMOTE_FS swarm/prod/secrets kubernetes/prod/base/secrets dc.out + +# created by `just prefer` +.preference diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b00143a..3350196b 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,9 +11,20 @@ on: jobs: test: runs-on: ubuntu-24.04 + strategy: + matrix: + engine: [ 'docker', 'podman' ] steps: - uses: actions/checkout@v4 - uses: taiki-e/install-action@just + - name: Start podman daemon + if: matrix.engine == 'podman' + run: | + set -ex + systemctl --user start podman.service + just get-socket + - name: Set preference to use ${{ matrix.engine }} + run: just prefer ${{ matrix.engine }} - name: Build development image run: just build - name: Pull other images diff --git a/.gitignore b/.gitignore index 8bd8d4bd..f4bc8066 100755 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ venv # created by dev container bc UBI sets HOME to the project src dir chris_backend/.config chris_backend/.bash_history + +# created by `just prefer` +.preference diff --git a/README.md b/README.md index 4721cad4..47ffd674 100755 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ The HTTP API primarily supports the [collection+json](http://amundsen.com/media- Development is mainly supported on Linux. MacOS and WSL on Windows also work (because Docker Desktop is a Linux VM). You will need at least 8GM RAM, 20GB disk space, and a good internet connection. -Install Docker (version 27 or above), Docker Compose, and [just](https://github.com/casey/just?tab=readme-ov-file#installation). +Install Docker (version 27 or above) or Podman (version 5.2 or above), Docker Compose, and [just](https://github.com/casey/just?tab=readme-ov-file#installation).
@@ -54,6 +54,28 @@ Docker Installation Instructions
+
+ +Podman Setup Instructions + + +Rootless Podman is supported. You must install and configure Podman to use `docker-compose`, _not_ `podman-compose`. `podman-compose` is missing features, see issues [#575](https://github.com/containers/podman-compose/issues/575) and [#866](https://github.com/containers/podman-compose/issues/866). + +A Podman daemon must be running, because _ChRIS_ runs containers of its own. To start the Podman daemon on Linux, run + +```shell +systemctl --user start podman.service +``` + +If both Podman and Docker are installed, Podman will be used by default. A preference to use either Podman or Docker can be set by running + +```shell +just prefer podman # or +just prefer docker +``` + +
+ ### Just Commands Development is handled by [`just`](https://just.systems). diff --git a/docker-compose_just.yml b/docker-compose_just.yml index f6aea634..0c4c0150 100644 --- a/docker-compose_just.yml +++ b/docker-compose_just.yml @@ -8,7 +8,7 @@ services: - tools volumes: - "./chrisomatic:/etc/chrisomatic:ro" - - "/var/run/docker.sock:/var/run/docker.sock" + - "${DOCKER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock" working_dir: /etc/chrisomatic userns_mode: host depends_on: @@ -139,7 +139,7 @@ services: SECRET_KEY: secret REMOVE_JOBS: "yes" volumes: - - /var/run/docker.sock:/var/run/docker.sock + - "${DOCKER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock" depends_on: - pfcon ports: diff --git a/justfile b/justfile index 987c636b..426ef814 100644 --- a/justfile +++ b/justfile @@ -78,4 +78,45 @@ run +command: # docker-compose ... helper function. docker-compose +command: - env UID=$(id -u) GID=$(id -g) docker compose -f '{{ compose_file }}' {{command}} + env UID=$(id -u) GID=$(id -g) DOCKER_SOCK=$(just get-socket) $(just get-engine) compose -f '{{ compose_file }}' {{command}} + +# Get the container engine to use (docker or podman) +get-engine: + @if [ -f '.preference' ]; then \ + cat .preference && exit 0; \ + elif type podman > /dev/null 2>&1; then \ + echo podman; \ + else \ + echo docker; \ + fi \ + +# Get the docker daemon socket +get-socket: + @if [ "$(just get-engine)" = 'podman' ]; then \ + just get-podman-socket; \ + else \ + echo '/var/lib/docker.sock'; \ + fi + +get-podman-socket: check-podman-socket + @podman info --format '{{{{ .Host.RemoteSocket.Path }}' + +# Ensure that the podman daemon is running. +check-podman-socket: + @if [ "$(podman info --format '{{{{ .Host.RemoteSocket.Exists }}')" != 'true' ]; then \ + echo 'Podman daemon not running. Please run `systemctl --user start podman.service`'; \ + exit 1; \ + fi + +# Set a preference for using either Docker or Podman. +prefer docker_or_podman: + @[ '{{docker_or_podman}}' = 'docker' ] || [ '{{docker_or_podman}}' = 'podman' ] \ + || ( \ + >&2 echo 'argument must be either "docker" or "podman"'; \ + exit 1 \ + ) + echo '{{docker_or_podman}}' > .preference + +# Remove your preference for Docker or Podman. +unset-preference: + rm -f .preference From 99255b6688ccfdd725a5d2eb7764eec96c0b5a01 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 15:34:01 -0400 Subject: [PATCH 02/10] Add GitHub Actions scripts --- action.yml | 23 +++++++++++++++++++++++ githubActions/cleanup.js | 6 ++++++ githubActions/main.js | 17 +++++++++++++++++ githubActions/setup.js | 7 +++++++ justfile | 4 ++++ 5 files changed, 57 insertions(+) create mode 100644 action.yml create mode 100644 githubActions/cleanup.js create mode 100644 githubActions/main.js create mode 100644 githubActions/setup.js diff --git a/action.yml b/action.yml new file mode 100644 index 00000000..d8bbe996 --- /dev/null +++ b/action.yml @@ -0,0 +1,23 @@ +name: "ChRIS Backend Integration Tests" +description: "Run ChRIS_ultron_backEnd integration tests" +author: "FNNDSC" + +inputs: + engine: + description: "Container engine to use (docker or podman)" + required: false + default: "docker" + command: + description: "Just command (e.g. test-all, test-integration, test-unit)" + required: false + default: "test-integration" + +runs: + using: node20 + pre: githubActions/pre.js + main: githubActions/main.js + post: githubActions/cleanup.js + +branding: + color: "blue" + icon: "box" diff --git a/githubActions/cleanup.js b/githubActions/cleanup.js new file mode 100644 index 00000000..67c2bcf4 --- /dev/null +++ b/githubActions/cleanup.js @@ -0,0 +1,6 @@ +const child_process = require('child_process'); + +child_process.execFileSync( + 'just', ['nuke'] + { stdio: 'inherit' } +); diff --git a/githubActions/main.js b/githubActions/main.js new file mode 100644 index 00000000..89d7a996 --- /dev/null +++ b/githubActions/main.js @@ -0,0 +1,17 @@ +const child_process = require('child_process'); + +const CONTAINER_ENGINE = process.env.INPUT_ENGINE; +const JUST_COMMAND = process.env.INPUT_COMMAND; + +const script = ` +set -x +just prefer ${CONTAINER_ENGINE} +rc=$(just ${JUST_COMMAND}) +if [ "$rc" != '0' ]; then + just logs +fi +exit $rc +`; + +child_process.execFileSync('bash', [script], { stdio: 'inherit' }); + diff --git a/githubActions/setup.js b/githubActions/setup.js new file mode 100644 index 00000000..2c91aaf5 --- /dev/null +++ b/githubActions/setup.js @@ -0,0 +1,7 @@ +const child_process = require('child_process'); + +child_process.execFileSync( + 'sudo', + ['apt-get', 'install', '-y', 'just'], + { stdio: 'inherit' } +); diff --git a/justfile b/justfile index 987c636b..5130a811 100644 --- a/justfile +++ b/justfile @@ -72,6 +72,10 @@ build: (docker-compose '--profile=cube build') # Pull container images. pull: (docker-compose 'pull') +# Get container logs. +logs *args: + @just docker-compose --profile=cube logs {{args}} + # docker-compose ... run helper function. run +command: @just docker-compose --profile=cube run --rm chris {{command}} From b9809f09ae4cd5e14b0c505e8f5bed61bd5f8183 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 15:38:10 -0400 Subject: [PATCH 03/10] Test using itself --- .github/workflows/ci.yml | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3350196b..bee02947 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,27 +14,17 @@ jobs: strategy: matrix: engine: [ 'docker', 'podman' ] + test-kind: [ 'unit', 'integration' ] steps: - - uses: actions/checkout@v4 - - uses: taiki-e/install-action@just - name: Start podman daemon if: matrix.engine == 'podman' - run: | - set -ex - systemctl --user start podman.service - just get-socket - - name: Set preference to use ${{ matrix.engine }} - run: just prefer ${{ matrix.engine }} - - name: Build development image - run: just build - - name: Pull other images - run: just pull - - name: Start ancillary services - run: just start-ancillary - - name: Run unit tests - run: just test-unit - - name: Run integration tests - run: just test-integration + run: systemctl --user start podman.service + - uses: actions/checkout@v4 + - name: Run ${{ matrix.test-kind }} tests on ${{ matrix.engine }} + uses: ./ + with: + engine: ${{ matrix.engine }} + command: test-${{ matrix.test-kind }} build: needs: [ test ] From 0cc686578d9d48a80b6cd5af752b37c7c5eec1b2 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 15:47:54 -0400 Subject: [PATCH 04/10] Add documentation on GitHub Actions usage --- README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/README.md b/README.md index 47ffd674..9ba5275b 100755 --- a/README.md +++ b/README.md @@ -166,6 +166,41 @@ Old development scripts usage is described in [OLD_DEVELOP.md](./OLD_DEVELOP.md) See https://chrisproject.org/docs/run/helm +## GitHub Actions + +This repository can also be used as a GitHub Actions step for running _CUBE_ integration tests, e.g. + +```yaml +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-24.04 + steps: + - name: Build something else + uses: docker/build-push-action@v6 + with: + tags: localhost/fnndsc/pman:dev + load: true + - name: Run ChRIS backend integration tests + uses: FNNDSC/ChRIS_ultron_backEnd@master + # all inputs are optional + with: + engine: docker # or podman + command: test-integration # or test-unit, ... + # optionally change image used for pman, pfcon, or cube + env: + CUBE_IMAGE: localhost/fnndsc/cube:dev + PFCON_IMAGE: localhost/fnndsc/pfcon:dev + PMAN_IMAGE: localhost/fnndsc/pman:dev +``` + ## Documentation > [!CAUTION] From 09dc8c8c392d559d2148ced50511068018440a89 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 16:00:26 -0400 Subject: [PATCH 05/10] Fix small mistakes --- githubActions/cleanup.js | 2 +- githubActions/main.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/githubActions/cleanup.js b/githubActions/cleanup.js index 67c2bcf4..bbf3ed2f 100644 --- a/githubActions/cleanup.js +++ b/githubActions/cleanup.js @@ -1,6 +1,6 @@ const child_process = require('child_process'); child_process.execFileSync( - 'just', ['nuke'] + 'just', ['nuke'], { stdio: 'inherit' } ); diff --git a/githubActions/main.js b/githubActions/main.js index 89d7a996..7bd71d64 100644 --- a/githubActions/main.js +++ b/githubActions/main.js @@ -13,5 +13,5 @@ fi exit $rc `; -child_process.execFileSync('bash', [script], { stdio: 'inherit' }); +child_process.execFileSync('bash', ['-c', script], { stdio: 'inherit' }); From 1f2ff573646f3067fdc743f37e9bf3dc788b0756 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 16:01:27 -0400 Subject: [PATCH 06/10] Use taiki-e/install-action@just --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bee02947..1c2dc4c3 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ jobs: - name: Start podman daemon if: matrix.engine == 'podman' run: systemctl --user start podman.service + - uses: taiki-e/install-action@just - uses: actions/checkout@v4 - name: Run ${{ matrix.test-kind }} tests on ${{ matrix.engine }} uses: ./ From 70363391cc62ca048b40c3aaee1fd407541e4f96 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 16:07:43 -0400 Subject: [PATCH 07/10] Fix another rookie mistake --- githubActions/main.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/githubActions/main.js b/githubActions/main.js index 7bd71d64..ca7d97be 100644 --- a/githubActions/main.js +++ b/githubActions/main.js @@ -6,7 +6,8 @@ const JUST_COMMAND = process.env.INPUT_COMMAND; const script = ` set -x just prefer ${CONTAINER_ENGINE} -rc=$(just ${JUST_COMMAND}) +just ${JUST_COMMAND} +rc=$? if [ "$rc" != '0' ]; then just logs fi From 9ed9771d88d12553051598e2249b8a3bd23557a0 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 16:28:01 -0400 Subject: [PATCH 08/10] Fix docker socket location --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index d0ea6a4d..326216dd 100644 --- a/justfile +++ b/justfile @@ -99,7 +99,7 @@ get-socket: @if [ "$(just get-engine)" = 'podman' ]; then \ just get-podman-socket; \ else \ - echo '/var/lib/docker.sock'; \ + echo '/var/run/docker.sock'; \ fi get-podman-socket: check-podman-socket From c5ff6fa9c3bac4aa926c27a653a283bdc2c8c055 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 16:59:49 -0400 Subject: [PATCH 09/10] Disable fail-fast --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c2dc4c3..4db9ff7a 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ jobs: test: runs-on: ubuntu-24.04 strategy: + fail-fast: false matrix: engine: [ 'docker', 'podman' ] test-kind: [ 'unit', 'integration' ] From 1b46878d7430dbb4e942500cd9e3fc29328245c2 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 29 Aug 2024 17:10:55 -0400 Subject: [PATCH 10/10] Add test-unit dependency on start-ancillary --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 326216dd..266fd81e 100644 --- a/justfile +++ b/justfile @@ -36,7 +36,7 @@ test *args: test-all: test-unit test-integration # Run unit tests. -test-unit: (run 'python manage.py test --force-color --exclude-tag integration') +test-unit: start-ancillary (run 'python manage.py test --force-color --exclude-tag integration') # Run integration tests. test-integration: start-ancillary (run 'python manage.py test --force-color --tag integration')