add TimeoutSeconds config option for HttpDatasetSource #626
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docker Image CI | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| branches: [master] | |
| release: | |
| types: | |
| - published | |
| env: | |
| DOCKERHUB_REPOSITORY: mimiro/datahub | |
| jobs: | |
| Docker_AMD64: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_tag: ${{ steps.image_tag.outputs.image_tag }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Docker Buildx (enable caching) | |
| uses: docker/setup-buildx-action@v1 | |
| - name: install semver tool | |
| run: pip3 install semver | |
| - name: strip v from tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Strip v from tag" | |
| LAST_PART=$(echo $GITHUB_REF | cut -d / -f 3) | |
| STRIPPED_VAR="${LAST_PART:1}" | |
| echo "VERSION_TAG=$STRIPPED_VAR" >> $GITHUB_ENV | |
| fi | |
| - name: validate release tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Validate that release tag is valid semver" | |
| pysemver check $VERSION_TAG | |
| fi | |
| - name: get next version | |
| id: semver-tag | |
| run: | | |
| git fetch --all --tags | |
| DESCRIBE=$( git describe --always --tags --long --first-parent ) | |
| VERSION=$(echo $DESCRIBE | cut -d "-" -f 1) | |
| if [[ $VERSION == v* ]]; then | |
| VERSION=${VERSION:1} | |
| fi | |
| NEXT=$( pysemver bump patch $VERSION ) | |
| echo "tag=$NEXT" >> $GITHUB_OUTPUT | |
| - name: "Calculated unstable tag" | |
| run: echo "base tag next unstable version... ${{ steps.semver-tag.outputs.tag }}" | |
| - name: Set Image Tag | |
| id: image_tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Setting Stable Image Tag" | |
| echo "image_tag=${{ env.DOCKERHUB_REPOSITORY }}:$VERSION_TAG-$(uname -m)" >> $GITHUB_OUTPUT | |
| else | |
| echo "Setting Unstable Image" | |
| echo "image_tag=${{ env.DOCKERHUB_REPOSITORY }}:${{ steps.semver-tag.outputs.tag }}-$GITHUB_RUN_NUMBER-unstable-$(uname -m)" >> $GITHUB_OUTPUT | |
| fi | |
| # Login to Docker registry except on PR | |
| - name: Login to DockerHub | |
| id: docker_login | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build Docker Image | |
| uses: docker/build-push-action@v2 | |
| id: docker_build | |
| with: | |
| push: false | |
| load: true | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| tags: ${{ steps.image_tag.outputs.image_tag }} | |
| - name: Trivy vulnerability scan | |
| uses: aquasecurity/trivy-action@master | |
| with: | |
| image-ref: "${{ steps.image_tag.outputs.image_tag }}" | |
| format: "table" | |
| exit-code: "1" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| severity: "CRITICAL,HIGH" | |
| - name: Push Image to DockerHub | |
| id: dockerhub_push | |
| if: | | |
| ( | |
| ( github.ref == 'refs/heads/master' && github.event_name == 'push' ) | |
| || | |
| ( github.event_name == 'release') | |
| ) | |
| && | |
| ( github.event_name != 'pull_request') | |
| run: | | |
| docker image push ${{ steps.image_tag.outputs.image_tag }} | |
| Docker_ARM64: | |
| runs-on: ARM64 | |
| if: false # skip this build temporarily, until we have a compatible runner | |
| outputs: | |
| image_tag: ${{ steps.image_tag.outputs.image_tag }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Docker Buildx (enable caching) | |
| uses: docker/setup-buildx-action@v1 | |
| - name: install semver tool | |
| run: pip3 install semver | |
| - name: strip v from tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Strip v from tag" | |
| LAST_PART=$(echo $GITHUB_REF | cut -d / -f 3) | |
| STRIPPED_VAR="${LAST_PART:1}" | |
| echo "VERSION_TAG=$STRIPPED_VAR" >> $GITHUB_ENV | |
| fi | |
| - name: validate release tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Validate that release tag is valid semver" | |
| /home/ec2-user/.local/bin/pysemver check $VERSION_TAG | |
| fi | |
| - name: get next version | |
| id: semver-tag | |
| run: | | |
| git fetch --all --tags | |
| DESCRIBE=$( git describe --always --tags --long --first-parent ) | |
| VERSION=$(echo $DESCRIBE | cut -d "-" -f 1) | |
| if [[ $VERSION == v* ]]; then | |
| VERSION=${VERSION:1} | |
| fi | |
| NEXT=$( /home/ec2-user/.local/bin/pysemver bump patch $VERSION ) | |
| echo "tag=$NEXT" >> $GITHUB_OUTPUT | |
| - name: "Calculated unstable tag" | |
| run: echo "base tag next unstable version... ${{ steps.semver-tag.outputs.tag }}" | |
| - name: Set Image Tag | |
| id: image_tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Setting Stable Image Tag" | |
| echo "image_tag=${{ env.DOCKERHUB_REPOSITORY }}:$VERSION_TAG-$(uname -m)" >> $GITHUB_OUTPUT | |
| else | |
| echo "Setting Unstable Image" | |
| echo "image_tag=${{ env.DOCKERHUB_REPOSITORY }}:${{ steps.semver-tag.outputs.tag }}-$GITHUB_RUN_NUMBER-unstable-$(uname -m)" >> $GITHUB_OUTPUT | |
| fi | |
| # Login to Docker registry except on PR | |
| - name: Login to DockerHub | |
| id: docker_login | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build Docker Image | |
| uses: docker/build-push-action@v2 | |
| id: docker_build | |
| with: | |
| push: false | |
| load: true | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| tags: ${{ steps.image_tag.outputs.image_tag }} | |
| - name: Trivy vulnerability scan | |
| uses: aquasecurity/trivy-action@master | |
| with: | |
| image-ref: "${{ steps.image_tag.outputs.image_tag }}" | |
| format: "table" | |
| exit-code: "1" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| severity: "CRITICAL,HIGH" | |
| - name: Push Image to DockerHub | |
| id: dockerhub_push | |
| if: | | |
| ( | |
| ( github.ref == 'refs/heads/master' && github.event_name == 'push' ) | |
| || | |
| ( github.event_name == 'release') | |
| ) | |
| && | |
| ( github.event_name != 'pull_request') | |
| run: | | |
| docker image push ${{ steps.image_tag.outputs.image_tag }} | |
| Docker_Manifest: | |
| runs-on: ubuntu-latest | |
| #needs: [Docker_AMD64, Docker_ARM64] # do not require ARM64 while disabled | |
| needs: [Docker_AMD64] | |
| if: | | |
| ( | |
| ( github.ref == 'refs/heads/master' && github.event_name == 'push' ) | |
| || | |
| ( github.event_name == 'release') | |
| ) | |
| && | |
| ( github.event_name != 'pull_request') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: install semver tool | |
| run: pip3 install semver | |
| - name: strip v from tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Strip v from tag" | |
| LAST_PART=$(echo $GITHUB_REF | cut -d / -f 3) | |
| STRIPPED_VAR="${LAST_PART:1}" | |
| echo "VERSION_TAG=$STRIPPED_VAR" >> $GITHUB_ENV | |
| fi | |
| - name: validate release tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Validate that release tag is valid semver" | |
| pysemver check $VERSION_TAG | |
| fi | |
| - name: get next version | |
| id: semver-tag | |
| run: | | |
| git fetch --all --tags | |
| DESCRIBE=$( git describe --always --tags --long --first-parent ) | |
| VERSION=$(echo $DESCRIBE | cut -d "-" -f 1) | |
| if [[ $VERSION == v* ]]; then | |
| VERSION=${VERSION:1} | |
| fi | |
| NEXT=$( pysemver bump patch $VERSION ) | |
| echo "tag=$NEXT" >> $GITHUB_OUTPUT | |
| - name: "Calculated unstable tag" | |
| run: echo "base tag next unstable version... ${{ steps.semver-tag.outputs.tag }}" | |
| - name: Login to DockerHub | |
| id: docker_login | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set Image Tag | |
| id: image_tag | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| echo "Setting Stable Image Tag" | |
| echo "image_tag=${{ env.DOCKERHUB_REPOSITORY }}:$VERSION_TAG" >> $GITHUB_OUTPUT | |
| else | |
| echo "Setting Unstable Image" | |
| echo "image_tag=${{ env.DOCKERHUB_REPOSITORY }}:${{ steps.semver-tag.outputs.tag }}-$GITHUB_RUN_NUMBER-unstable" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Show Tag | |
| run: echo ${{ steps.image_tag.outputs.image_tag }} | |
| - name: Docker Manifest | |
| id: docker_manifest | |
| run: | | |
| if [ ${{ github.event_name }} == 'release' ] | |
| then | |
| #tag:release version | |
| docker manifest create ${{ steps.image_tag.outputs.image_tag }} \ | |
| ${{needs.Docker_AMD64.outputs.image_tag}} \ | |
| ${{needs.Docker_ARM64.outputs.image_tag}} | |
| docker manifest push ${{ steps.image_tag.outputs.image_tag }} | |
| #tag:latest | |
| docker manifest create ${{ env.DOCKERHUB_REPOSITORY }}:latest \ | |
| ${{needs.Docker_AMD64.outputs.image_tag}} \ | |
| ${{needs.Docker_ARM64.outputs.image_tag}} | |
| docker manifest push ${{ env.DOCKERHUB_REPOSITORY }}:latest | |
| else | |
| docker manifest create ${{ steps.image_tag.outputs.image_tag }} \ | |
| ${{needs.Docker_AMD64.outputs.image_tag}} \ | |
| ${{needs.Docker_ARM64.outputs.image_tag}} | |
| docker manifest push ${{ steps.image_tag.outputs.image_tag }} | |
| fi |