From b2634e53409cd8f19760f62fd03c669419bdf8b3 Mon Sep 17 00:00:00 2001 From: Vladimir Tarbaev <93940171+tarbaev-vl@users.noreply.github.com> Date: Sun, 13 Oct 2024 00:36:18 +0300 Subject: [PATCH] update CI part: refactoring --- .github/workflows/build.yml | 135 +++++++++++++ .github/workflows/ci.yml | 264 -------------------------- .github/workflows/docs.yml | 47 +++++ .github/workflows/release.yml | 29 +++ .github/workflows/tests.yml | 42 ++++ .github/workflows/update_dev_docs.yml | 16 ++ 6 files changed, 269 insertions(+), 264 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/tests.yml create mode 100644 .github/workflows/update_dev_docs.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..a09ca772 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,135 @@ +name: Build binary and Docker images + +on: + workflow_call: + +env: + go-version: '1.22.6' + docker-io-registry: greenmask + docker-ghcr-registry: greenmaskio + app-name: greenmask + +jobs: + build-binary: + runs-on: ubuntu-22.04 + strategy: + matrix: + platforms: + - 'windows/arm64' + - 'windows/amd64' + - 'darwin/amd64' + - 'darwin/arm64' + - 'linux/amd64' + - 'linux/arm64' + - 'linux/arm/v6' + - 'linux/arm/v7' + - 'linux/ppc64le' + - 'linux/riscv64' + - 'linux/s390x' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.go-version }} + + - name: Build with different arch + run: | + export GOOS=$(echo ${{ matrix.platforms }} | cut -d '/' -f 1) + export GOARCH=$(echo ${{ matrix.platforms }} | cut -d '/' -f 2) + export GOARM=$(echo ${{ matrix.platforms }} | cut -d '/' -f 3 | cut -d 'v' -f 2) + if [[ "$GOOS" == "windows" ]]; then + make build CMD_NAME="builds/${{ env.app-name }}.exe" + else + make build CMD_NAME="builds/${{ env.app-name }}" + fi + + - name: Create checksum + if: startsWith(github.ref, 'refs/tags/v') + working-directory: builds + run: | + find . -type f -exec shasum -a 256 -b {} + | sed 's# \*\./# *#' | while read sum file; do echo "$sum $file" > "${file#\*}".sha256; done + + - name: Create archive + if: startsWith(github.ref, 'refs/tags/v') + run: | + export GOOS=$(echo ${{ matrix.platforms }} | cut -d '/' -f 1) + export GOARCH=$(echo ${{ matrix.platforms }} | cut -d '/' -f 2) + export GOARM=$(echo ${{ matrix.platforms }} | cut -d '/' -f 3 | cut -d 'v' -f 2) + export ARCHIVE_NAME=$(echo "${{ env.app-name }}-${GOOS}-${GOARCH}$(if [ -n "${GOARM}" ]; then echo v${GOARM}; fi).$(if [ "${GOOS}" = "windows" ]; then echo "zip"; else echo "tar.gz"; fi)") + cp LICENSE builds/ + cd builds + if [[ "$GOOS" == "windows" ]]; then + zip "${ARCHIVE_NAME}" * + else + tar -czvf "${ARCHIVE_NAME}" * + fi + find . -maxdepth 1 -type f ! -name ${ARCHIVE_NAME} -exec rm -f {} + + + - name: GitHub Release + if: startsWith(github.ref, 'refs/tags/v') + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + draft: true + files: builds/* + + build-docker-images-and-push: + runs-on: ubuntu-22.04 + if: startsWith(github.ref, 'refs/tags/v') + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get Git tag + run: echo "TAG=$(git tag --points-at HEAD)" >> $GITHUB_ENV + + - name: Build docker image and push + uses: docker/build-push-action@v5 + with: + file: docker/greenmask/Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.docker-io-registry }}/${{ env.app-name }}:${{ env.TAG }} + ghcr.io/${{ env.docker-ghcr-registry }}/${{ env.app-name }}:${{ env.TAG }} + + - name: Build docker imaget with latest tag and push + uses: docker/build-push-action@v5 + if: | + ! contains(github.ref, 'rc') && + ! contains(github.ref, 'dev') && + ! contains(github.ref, 'pre') && + ! contains(github.ref, 'beta') && + ! contains(github.ref, 'b') + with: + file: docker/greenmask/Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.docker-io-registry }}/${{ env.app-name }}:latest + ghcr.io/${{ env.docker-ghcr-registry }}/${{ env.app-name }}:latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 37dc4530..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,264 +0,0 @@ -name: ci - -on: - workflow_dispatch: - push: - branches: - - 'main' - tags: - - 'v*' - pull_request: - paths-ignore: - - 'README.md' - - 'docs/**' - - 'LICENSE' - - 'getting_started.md' - -env: - go-version: '1.22.6' - python-version: 'pypy3.10' - cmd-name: 'greenmask' - docker-io-registry: greenmask - docker-ghcr-registry: greenmaskio - docker-image-name: greenmask - -jobs: - unit-tests: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ${{ env.go-version }} - - - name: Echo Go version - run: go version - - - name: Run tests - run: make tests - - integration-tests: - runs-on: ubuntu-22.04 - needs: - - unit-tests - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Run integration tests - run: | - docker compose -f docker-compose-integration.yml -p greenmask up \ - --renew-anon-volumes --force-recreate --build --exit-code-from greenmask \ - --abort-on-container-exit greenmask - - build-binaries: - runs-on: ubuntu-22.04 - needs: - - unit-tests - - integration-tests - strategy: - matrix: - platforms: - - 'windows/arm64' - - 'windows/amd64' - - 'darwin/amd64' - - 'darwin/arm64' - - 'linux/amd64' - - 'linux/arm64' - - 'linux/arm/v6' - - 'linux/arm/v7' - - 'linux/ppc64le' - - 'linux/riscv64' - - 'linux/s390x' - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ${{ env.go-version }} - - - name: Build with different arch - run: | - export GOOS=$(echo ${{ matrix.platforms }} | cut -d '/' -f 1) - export GOARCH=$(echo ${{ matrix.platforms }} | cut -d '/' -f 2) - export GOARM=$(echo ${{ matrix.platforms }} | cut -d '/' -f 3 | cut -d 'v' -f 2) - if [[ "$GOOS" == "windows" ]]; then - make build CMD_NAME="builds/${{ env.cmd-name }}.exe" - else - make build CMD_NAME="builds/${{ env.cmd-name }}" - fi - - - name: Create checksum - if: startsWith(github.ref, 'refs/tags/v') - working-directory: builds - run: | - find . -type f -exec shasum -a 256 -b {} + | sed 's# \*\./# *#' | while read sum file; do echo "$sum $file" > "${file#\*}".sha256; done - - - name: Create archive - if: startsWith(github.ref, 'refs/tags/v') - run: | - export GOOS=$(echo ${{ matrix.platforms }} | cut -d '/' -f 1) - export GOARCH=$(echo ${{ matrix.platforms }} | cut -d '/' -f 2) - export GOARM=$(echo ${{ matrix.platforms }} | cut -d '/' -f 3 | cut -d 'v' -f 2) - export ARCHIVE_NAME=$(echo "${{ env.cmd-name }}-${GOOS}-${GOARCH}$(if [ -n "${GOARM}" ]; then echo v${GOARM}; fi).$(if [ "${GOOS}" = "windows" ]; then echo "zip"; else echo "tar.gz"; fi)") - cp LICENSE builds/ - cd builds - if [[ "$GOOS" == "windows" ]]; then - zip "${ARCHIVE_NAME}" * - else - tar -czvf "${ARCHIVE_NAME}" * - fi - find . -maxdepth 1 -type f ! -name ${ARCHIVE_NAME} -exec rm -f {} + - - - name: GitHub Release - if: startsWith(github.ref, 'refs/tags/v') - uses: softprops/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: true - files: builds/* - - build-docker-images-and-push: - runs-on: ubuntu-22.04 - needs: - - build-binaries - if: startsWith(github.ref, 'refs/tags/v') - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Git tag - id: git_tag - run: echo "TAG=$(git tag --points-at HEAD)" >> $GITHUB_ENV - - - name: Build docker image and push - uses: docker/build-push-action@v5 - with: - file: docker/greenmask/Dockerfile - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ${{ env.docker-io-registry }}/${{ env.docker-image-name }}:${{ env.TAG }} - ghcr.io/${{ env.docker-ghcr-registry }}/${{ env.docker-image-name }}:${{ env.TAG }} - - - name: Build docker imaget with latest tag and push - uses: docker/build-push-action@v5 - if: | - ! contains(github.ref, 'rc') && - ! contains(github.ref, 'dev') && - ! contains(github.ref, 'pre') && - ! contains(github.ref, 'beta') && - ! contains(github.ref, 'b') - with: - file: docker/greenmask/Dockerfile - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ${{ env.docker-io-registry }}/${{ env.docker-image-name }}:latest - ghcr.io/${{ env.docker-ghcr-registry }}/${{ env.docker-image-name }}:latest - - deploy-docs: - runs-on: self-hosted - needs: - - build-binaries - - build-docker-images-and-push - if: startsWith(github.ref, 'refs/tags/v') - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.python-version }} - - - name: Install dependicies - run: pip install -r requirements.txt - - - name: Get all tags - uses: octokit/request-action@v2.3.1 - id: get_tags - with: - route: GET /repos/${{ github.repository }}/git/matching-refs/tags - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup docs deploy - run: | - git config --global user.name "Docs Deployer" - git config --global user.email docs@greenmask.io - echo "Get tags from JSON output" - export TAGS=$(echo '${{ steps.get_tags.outputs.data }}' | jq ".[].ref") - echo "Remove substring from tag name" - export TAGS=($(echo "${TAGS[@]}" | sed 's/"//g' | cut -d '/' -f 3)) - echo "Find index of start tag" - export TAG_INDEX=$(echo "${TAGS[@]/v0.1.13//}" | cut -d/ -f1 | wc -w | tr -d ' ') - echo "Get last index of array" - export LAST_INDEX=$((${#TAGS[@]} - 1)) - echo "Export sorted tags to GITHUB_ENV variable" - echo "TAGS="${TAGS[@]:${TAG_INDEX}:${LAST_INDEX}}"" >> $GITHUB_ENV - - - name: Build docs - run: | - export TAGS="${{ env.TAGS }}" - git fetch --prune --unshallow --tags --force - for tag in ${{ env.TAGS }}; do - echo "### CHECKOUT TO ${tag} ###" - git checkout ${tag} - if [[ "$tag" == *"rc"* || "$tag" == *"dev"* || "$tag" == *"pre"* || "$tag" == *"beta"* || "$tag" == *"b"* ]]; then - mike deploy $tag - else - mike deploy --update-aliases $tag latest - fi - done - - - name: Set latest version as default - run: mike set-default latest - - - name: Change branch to get html files - run: git checkout gh-pages - - - name: Remove old folder - run: sudo rm -rf ${{ secrets.DOCS_DEPLOY_DIR }}/html - - - name: Create docs directory - run: sudo mkdir -p ${{ secrets.DOCS_DEPLOY_DIR }}/html - - - name: Move html files to docs directory - run: sudo cp -r * ${{ secrets.DOCS_DEPLOY_DIR }}/html - - - name: Restart web service - run: sudo systemctl restart nginx diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..a1363b1d --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +name: Deploy documentation + +on: + workflow_call: + +env: + python-version: 'pypy3.10' + +jobs: + deploy-docs: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.python-version }} + + - name: Install dependicies + run: pip install -r requirements.txt + + - name: Setup docs deploy + run: | + git config --global user.name "Greenmask CI" + git config --global user.email ci@greenmask.io + + - name: Build dev docs + if: github.ref == 'refs/heads/main' + run: mike deploy --push dev + + - name: Get Git tag + if: startsWith(github.ref, 'refs/tags/v') + run: echo "TAG=$(git tag --points-at HEAD)" >> $GITHUB_ENV + + - name: Build release docs + if: startsWith(github.ref, 'refs/tags/v') + run: | + if [[ "${{ env.TAG }}" == *"rc"* || "${{ env.TAG }}" == *"dev"* || "${{ env.TAG }}" == *"pre"* || "${{ env.TAG }}" == *"beta"* || "${{ env.TAG }}" == *"b"* ]]; then + mike deploy --push $tag + else + mike deploy --push --update-aliases $tag latest + mike set-default --push latest + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..a7417cd8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: release + +on: + workflow_dispatch: + push: + tags: + - 'v*' + pull_request: + paths-ignore: + - 'README.md' + - 'docs/**' + - 'LICENSE' + - 'getting_started.md' + +jobs: + tests: + uses: ./.github/workflows/tests.yml + + build: + uses: ./.github/workflows/build.yml + needs: + - tests + + docs: + uses: ./.github/workflows/docs.yml + if: startsWith(github.ref, 'refs/tags/v') + needs: + - build + \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..7368cb4c --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,42 @@ +name: Run unit and integration tests + +on: + workflow_call: + +env: + go-version: '1.22.6' + +jobs: + unit-tests: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.go-version }} + + - name: Run tests + run: make tests + + integration-tests: + runs-on: ubuntu-22.04 + needs: + - unit-tests + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run integration tests + run: | + docker compose -f docker-compose-integration.yml -p greenmask up \ + --renew-anon-volumes --force-recreate --build --exit-code-from greenmask \ + --abort-on-container-exit greenmask diff --git a/.github/workflows/update_dev_docs.yml b/.github/workflows/update_dev_docs.yml new file mode 100644 index 00000000..0540f4e3 --- /dev/null +++ b/.github/workflows/update_dev_docs.yml @@ -0,0 +1,16 @@ +name: Deploy development documentation (from main branch) + +on: + workflow_dispatch: + push: + branches: + - 'main' + paths: + - 'docs/**' + +env: + python-version: 'pypy3.10' + +jobs: + docs: + uses: ./.github/workflows/docs.yml