ci: updated base-image #599
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Docker Image CI: crawlab" | |
| on: | |
| push: | |
| branches: [ develop, test, main ] | |
| release: | |
| types: [ published ] | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: [ docker-crawlab ] | |
| env: | |
| IMAGE_PATH_CRAWLAB_BACKEND: backend | |
| IMAGE_PATH_CRAWLAB_FRONTEND: frontend | |
| GH_PKG_NAME_CRAWLAB_BASE: ${{ github.repository_owner }}/crawlab-base | |
| GH_PKG_NAME_CRAWLAB_BACKEND: ${{ github.repository_owner }}/crawlab-backend | |
| GH_PKG_NAME_CRAWLAB_FRONTEND: ${{ github.repository_owner }}/crawlab-frontend | |
| GH_PKG_NAME_CRAWLAB: ${{ github.repository_owner }}/crawlab | |
| IMAGE_NAME_CRAWLAB_BASE: ghcr.io/${{ github.repository_owner }}/crawlab-base | |
| IMAGE_NAME_CRAWLAB_BACKEND: ghcr.io/${{ github.repository_owner }}/crawlab-backend | |
| IMAGE_NAME_CRAWLAB_FRONTEND: ghcr.io/${{ github.repository_owner }}/crawlab-frontend | |
| IMAGE_NAME_CRAWLAB: ghcr.io/${{ github.repository_owner }}/crawlab | |
| IMAGE_NAME_CRAWLAB_DOCKERHUB: crawlabteam/crawlab | |
| IMAGE_NAME_CRAWLAB_TENCENT: ccr.ccs.tencentyun.com/crawlab/crawlab | |
| E2E_TESTS_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/e2e-tests | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| backend_changed: ${{ steps.check_changed_files.outputs.backend_changed }} | |
| frontend_changed: ${{ steps.check_changed_files.outputs.frontend_changed }} | |
| dockerfile_changed: ${{ steps.check_changed_files.outputs.dockerfile_changed }} | |
| workflow_changed: ${{ steps.check_changed_files.outputs.workflow_changed }} | |
| base_image_changed: ${{ steps.check_changed_files.outputs.base_image_changed }} | |
| version: ${{ steps.version.outputs.version }} | |
| test_script: ${{ steps.test_config.outputs.test_script }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get changed files | |
| id: changed_files | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| files_yaml: | | |
| backend: | |
| - 'backend/**' | |
| - 'core/**' | |
| - 'db/**' | |
| - 'fs/**' | |
| - 'grpc/**' | |
| - 'vcs/**' | |
| - 'trace/**' | |
| frontend: | |
| - '${{ env.IMAGE_PATH_CRAWLAB_FRONTEND }}/**' | |
| dockerfile: | |
| - 'Dockerfile' | |
| workflow: | |
| - '.github/workflows/docker-crawlab.yml' | |
| base_image: | |
| - 'docker/base-image/**' | |
| - id: check_changed_files | |
| name: Check changed files | |
| run: | | |
| # set outputs | |
| if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then | |
| echo "backend_changed=true" >> $GITHUB_OUTPUT | |
| echo "frontend_changed=true" >> $GITHUB_OUTPUT | |
| echo "dockerfile_changed=true" >> $GITHUB_OUTPUT | |
| echo "workflow_changed=true" >> $GITHUB_OUTPUT | |
| echo "base_image_changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "backend_changed=${{ steps.changed_files.outputs.backend_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "frontend_changed=${{ steps.changed_files.outputs.frontend_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "dockerfile_changed=${{ steps.changed_files.outputs.dockerfile_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "workflow_changed=${{ steps.changed_files.outputs.workflow_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "base_image_changed=${{ steps.changed_files.outputs.base_image_any_changed }}" >> $GITHUB_OUTPUT | |
| fi | |
| # Display change status for each component | |
| echo "Backend changed: ${{ steps.changed_files.outputs.backend_any_changed }}" | |
| echo "Frontend changed: ${{ steps.changed_files.outputs.frontend_any_changed }}" | |
| echo "Dockerfile changed: ${{ steps.changed_files.outputs.dockerfile_any_changed }}" | |
| echo "Workflow changed: ${{ steps.changed_files.outputs.workflow_any_changed }}" | |
| echo "Base image changed: ${{ steps.changed_files.outputs.base_image_any_changed }}" | |
| - id: version | |
| name: Get version | |
| run: | | |
| # Strip git ref prefix from version | |
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | |
| # Strip "v" prefix from tag name | |
| [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') | |
| # Use Docker `latest` tag convention | |
| [ "$VERSION" == "main" ] && VERSION=latest | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| - id: test_config | |
| name: Set test configuration | |
| run: | | |
| if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then | |
| echo "test_script=test:full" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.ref }}" == "refs/heads/test" ]]; then | |
| echo "test_script=test:extended" >> $GITHUB_OUTPUT | |
| else | |
| echo "test_script=test:normal" >> $GITHUB_OUTPUT | |
| fi | |
| build_base_image: | |
| needs: [ setup ] | |
| if: needs.setup.outputs.base_image_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| failed: ${{ steps.set_output.outputs.failed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/base-image | |
| push: true | |
| tags: ${{ env.IMAGE_NAME_CRAWLAB_BASE }}:${{ needs.setup.outputs.version }} | |
| - name: Set output | |
| id: set_output | |
| if: failure() | |
| run: echo "failed=true" >> $GITHUB_OUTPUT | |
| test_backend: | |
| needs: [ setup ] | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| failed: ${{ steps.set_output.outputs.failed }} | |
| services: | |
| mongo: | |
| image: mongo:5 | |
| ports: | |
| - 27017:27017 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'core/go.mod' | |
| cache-dependency-path: 'core/go.sum' | |
| - name: Run tests | |
| working-directory: core | |
| run: | | |
| # Find all directories containing *_test.go files | |
| test_dirs=$(find . -name "*_test.go" -exec dirname {} \; | sort -u) | |
| # Run go test on each directory | |
| for dir in $test_dirs | |
| do | |
| echo "Running tests in $dir" | |
| go test ./$dir | |
| done | |
| - name: Set output | |
| id: set_output | |
| if: failure() | |
| run: echo "failed=true" >> $GITHUB_OUTPUT | |
| build_backend: | |
| needs: [ setup, test_backend ] | |
| if: needs.test_backend.result == 'success' || needs.setup.outputs.workflow_changed == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| failed: ${{ steps.set_output.outputs.failed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ${{ env.IMAGE_PATH_CRAWLAB_BACKEND }} | |
| push: true | |
| tags: ${{ env.IMAGE_NAME_CRAWLAB_BACKEND }}:${{ needs.setup.outputs.version }} | |
| - name: Set output | |
| id: set_output | |
| if: failure() | |
| run: echo "failed=true" >> $GITHUB_OUTPUT | |
| build_frontend: | |
| needs: [ setup ] | |
| if: needs.setup.outputs.frontend_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| failed: ${{ steps.set_output.outputs.failed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ${{ env.IMAGE_PATH_CRAWLAB_FRONTEND }} | |
| push: true | |
| tags: ${{ env.IMAGE_NAME_CRAWLAB_FRONTEND }}:${{ needs.setup.outputs.version }} | |
| - name: Set output | |
| id: set_output | |
| if: failure() | |
| run: echo "failed=true" >> $GITHUB_OUTPUT | |
| build_crawlab: | |
| needs: [setup, build_base_image, test_backend, build_backend, build_frontend] | |
| if: | | |
| always() && | |
| ( | |
| needs.test_backend.outputs.failed != 'true' && | |
| needs.build_backend.outputs.failed != 'true' && | |
| needs.build_frontend.outputs.failed != 'true' && | |
| needs.build_base_image.outputs.failed != 'true' | |
| ) && | |
| ( | |
| needs.setup.outputs.backend_changed == 'true' || | |
| needs.setup.outputs.frontend_changed == 'true' || | |
| needs.setup.outputs.dockerfile_changed == 'true' || | |
| needs.setup.outputs.base_image_changed == 'true' || | |
| needs.setup.outputs.workflow_changed == 'true' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Update Dockerfile | |
| run: | | |
| IMAGE_NAMES=( | |
| "crawlab-base" | |
| "crawlab-backend" | |
| "crawlab-frontend" | |
| ) | |
| for name in "${IMAGE_NAMES[@]}"; do | |
| IMAGE_NAME="ghcr.io/${{ github.repository_owner }}/$name" | |
| OLD_IMAGE="crawlabteam/${name}:latest" | |
| NEW_IMAGE="${IMAGE_NAME}:${{ needs.setup.outputs.version }}" | |
| sed -i "s|${OLD_IMAGE}|${NEW_IMAGE}|" Dockerfile | |
| done | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./Dockerfile | |
| build-args: | | |
| CRAWLAB_TAG=${{ needs.setup.outputs.version }} | |
| push: true | |
| tags: | | |
| ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }} | |
| test_crawlab: | |
| needs: [setup, build_crawlab] | |
| if: ${{ always() && needs.build_crawlab.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| services: | |
| mongo: | |
| image: mongo:5 | |
| options: >- | |
| --health-cmd "mongosh --eval 'db.adminCommand(\"ping\")' || exit 1" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| master: | |
| image: ghcr.io/${{ github.repository_owner }}/crawlab:${{ needs.setup.outputs.version }} | |
| env: | |
| CRAWLAB_NODE_MASTER: Y | |
| CRAWLAB_MONGO_HOST: mongo | |
| CRAWLAB_MONGO_PORT: 27017 | |
| ports: | |
| - 8080:8080 | |
| options: >- | |
| --health-cmd "curl -f http://localhost:8080/api/system-info || exit 1" | |
| --health-interval 30s | |
| --health-timeout 10s | |
| --health-retries 5 | |
| worker: | |
| image: ghcr.io/${{ github.repository_owner }}/crawlab:${{ needs.setup.outputs.version }} | |
| env: | |
| CRAWLAB_NODE_MASTER: N | |
| CRAWLAB_MASTER_HOST: master | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull e2e test image | |
| run: | | |
| docker pull ${{ env.E2E_TESTS_IMAGE_NAME }}:latest | |
| - name: Run e2e test image | |
| run: | | |
| docker run --network host \ | |
| -e BASE_URL=http://localhost:8080 \ | |
| -e TEST_SCRIPT=${{ needs.setup.outputs.test_script }} \ | |
| -v ${{ github.workspace }}/playwright-report:/app/playwright-report \ | |
| ${{ env.E2E_TESTS_IMAGE_NAME }}:latest | |
| - name: Deploy to GitHub Pages | |
| if: always() | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ${{ github.workspace }}/playwright-report | |
| destination_dir: playwright-report/${{ needs.setup.outputs.version }} | |
| push_images: | |
| if: ${{ always() && needs.test_crawlab.result == 'success' }} | |
| needs: [setup, test_crawlab] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| registry: [dockerhub, tencent] | |
| steps: | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull Docker image from GitHub Container Registry | |
| run: docker pull ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }} | |
| - name: Login to DockerHub or Tencent Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ (matrix.registry == 'dockerhub' && 'https://index.docker.io/v1/') || (matrix.registry == 'tencent' && 'ccr.ccs.tencentyun.com') }} | |
| username: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_USERNAME) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_USERNAME) }} | |
| password: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_PASSWORD) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_PASSWORD) }} | |
| - name: Tag and push image | |
| if: ${{ matrix.registry == 'dockerhub' || (matrix.registry == 'tencent' && github.ref != 'refs/heads/develop') }} | |
| run: | | |
| docker tag ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }} ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB_DOCKERHUB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }} | |
| docker push ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB_DOCKERHUB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }} |