Skip to content
Back to pull request #5110

feat: snowpipe streaming #18096

Sign in to view logs

feat: snowpipe streaming

feat: snowpipe streaming #18096

Workflow file for this run

.github/workflows/builds.yml at db5364b
name: builds
on:
release:
types: [ created ]
push:
branches:
- master
- main
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
cancel-in-progress: true
env:
arch_amd64: amd64
arch_arm64: arm64
docker_oss_images: |
name=rudderlabs/develop-rudder-server
name=rudderlabs/rudder-server,enable=${{ github.ref == format('refs/heads/{0}', 'master') || github.event_name == 'release' }}
docker_oss_tags: |
type=ref,event=branch
type=raw,value=1-alpine,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }}
type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }}
type=raw,value=${{ github.head_ref }},enable=${{ github.event_name == 'pull_request' }}
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
docker_ent_images: |
name=rudderstack/develop-rudder-server-enterprise
name=rudderstack/rudder-server-enterprise,enable=${{ github.ref == format('refs/heads/{0}', 'master') || github.event_name == 'release' }}
docker_ent_tags: |
type=ref,event=branch
type=raw,value=latest,enable=${{ github.event_name == 'release' }}
type=raw,value=${{ github.head_ref }},enable=${{ github.event_name == 'pull_request' }}
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
docker_sbsvc_images: |
name=rudderstack/develop-suppression-backup-service
name=rudderstack/suppression-backup-service,enable=${{ github.ref == format('refs/heads/{0}', 'master') || github.event_name == 'release' }}
docker_sbsvc_tags: |
type=ref,event=branch
type=raw,value=latest,enable=${{ github.event_name == 'release' }}
type=raw,value=${{ github.head_ref }},enable=${{ github.event_name == 'pull_request' }}
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
jobs:
docker-oss-meta:
runs-on: ubuntu-latest
outputs:
labels: ${{ steps.meta.outputs.labels }}
build-date: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
version: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
revision: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
tags: ${{ steps.meta.outputs.tags }}
arm64_tags: ${{ steps.arm64_meta.outputs.tags }}
arm64_labels: ${{ steps.arm64_meta.outputs.labels }}
amd64_tags: ${{ steps.amd64_meta.outputs.tags }}
amd64_labels: ${{ steps.amd64_meta.outputs.labels }}
steps:
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_oss_images}}
tags: ${{env.docker_oss_tags}}
- name: Docker arm64 meta
id: arm64_meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_oss_images}}
tags: ${{env.docker_oss_tags}}
flavor: |
suffix=-${{env.arch_arm64}},onlatest=true
- name: Docker amd64 meta
id: amd64_meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_oss_images}}
tags: ${{env.docker_oss_tags}}
flavor: |
suffix=-${{env.arch_amd64}},onlatest=true
docker-oss:
needs:
- docker-oss-meta
strategy:
matrix:
build-config:
- os: [ self-hosted, Linux, ARM64 ]
tags: ${{needs.docker-oss-meta.outputs.arm64_tags}}
labels: ${{needs.docker-oss-meta.outputs.arm64_labels}}
platform: linux/arm64
- os: ubuntu-latest
tags: ${{needs.docker-oss-meta.outputs.amd64_tags}}
labels: ${{needs.docker-oss-meta.outputs.amd64_labels}}
platform: linux/amd64
runs-on: ${{matrix.build-config.os}}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Login to DockerHub
uses: docker/login-action@v3.3.0
with:
username: rudderlabs
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build image (no-push)
uses: docker/build-push-action@v6
with:
context: .
platforms: ${{ matrix.build-config.platform }}
push: false
tags: ${{ matrix.build-config.tags }}
labels: ${{ matrix.build-config.labels }}
build-args: |
BUILD_DATE=${{ needs.docker-oss-meta.outputs.build-date }}
VERSION=${{ needs.docker-oss-meta.outputs.version }}
COMMIT_HASH=${{ github.sha }}
REVISION=${{ needs.docker-oss-meta.outputs.revision }}
outputs: type=docker,dest=${{ runner.temp }}/local-rudder-server.tar
cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp
- name: Run TruffleHog
run: |
docker pull trufflesecurity/trufflehog:latest
docker run --rm -v ${{ runner.temp }}:/tmp \
trufflesecurity/trufflehog:latest \
docker --image file:///tmp/local-rudder-server.tar \
--github-actions \
--no-verification \
--fail
- name: Build and push
if: success()
uses: docker/build-push-action@v6
with:
context: .
platforms: ${{ matrix.build-config.platform }}
push: true
tags: ${{ matrix.build-config.tags }}
labels: ${{ matrix.build-config.labels }}
build-args: |
BUILD_DATE=${{ needs.docker-oss-meta.outputs.build-date }}
VERSION=${{ needs.docker-oss-meta.outputs.version }}
COMMIT_HASH=${{ github.sha }}
REVISION=${{ needs.docker-oss-meta.outputs.revision }}
cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp
docker-ent-meta:
runs-on: ubuntu-latest
outputs:
labels: ${{ steps.meta.outputs.labels }}
build-date: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
version: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
revision: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
tags: ${{ steps.meta.outputs.tags }}
arm64_tags: ${{ steps.arm64_meta.outputs.tags }}
arm64_labels: ${{ steps.arm64_meta.outputs.labels }}
amd64_tags: ${{ steps.amd64_meta.outputs.tags }}
amd64_labels: ${{ steps.amd64_meta.outputs.labels }}
steps:
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_ent_images}}
tags: ${{env.docker_ent_tags}}
- name: Docker arm64 meta
id: arm64_meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_ent_images}}
tags: ${{env.docker_ent_tags}}
flavor: |
suffix=-${{env.arch_arm64}},onlatest=true
- name: Docker amd64 meta
id: amd64_meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_ent_images}}
tags: ${{env.docker_ent_tags}}
flavor: |
suffix=-${{env.arch_amd64}},onlatest=true
docker-ent:
needs:
- docker-ent-meta
strategy:
matrix:
build-config:
- os: [ self-hosted, Linux, ARM64 ]
tags: ${{needs.docker-ent-meta.outputs.arm64_tags}}
labels: ${{needs.docker-ent-meta.outputs.arm64_labels}}
platform: linux/arm64
- os: ubuntu-latest
tags: ${{needs.docker-ent-meta.outputs.amd64_tags}}
labels: ${{needs.docker-ent-meta.outputs.amd64_labels}}
platforms: linux/amd64
runs-on: ${{matrix.build-config.os}}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3.3.0
with:
username: rudderlabs
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build image (no-push)
uses: docker/build-push-action@v6
with:
context: .
platforms: ${{ matrix.build-config.platform }}
push: false
tags: ${{ matrix.build-config.tags }}
labels: ${{ matrix.build-config.labels }}
build-args: |
BUILD_DATE=${{ needs.docker-ent-meta.outputs.build-date }}
VERSION=${{ needs.docker-ent-meta.outputs.version }}
COMMIT_HASH=${{ github.sha }}
REVISION=${{ needs.docker-ent-meta.outputs.revision }}
ENTERPRISE_TOKEN=${{ secrets.ENTERPRISE_TOKEN }}
outputs: type=docker,dest=${{ runner.temp }}/local-rudder-server.tar
cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp
- name: Run TruffleHog
run: |
docker pull trufflesecurity/trufflehog:latest
docker run --rm -v ${{ runner.temp }}:/tmp \
trufflesecurity/trufflehog:latest \
docker --image file:///tmp/local-rudder-server.tar \
--github-actions \
--no-verification \
--fail
- name: Build and push
if: success()
uses: docker/build-push-action@v6
with:
context: .
platforms: ${{ matrix.build-config.platform }}
push: true
tags: ${{ matrix.build-config.tags }}
labels: ${{ matrix.build-config.labels }}
build-args: |
BUILD_DATE=${{ needs.docker-ent-meta.outputs.build-date }}
VERSION=${{ needs.docker-ent-meta.outputs.version }}
COMMIT_HASH=${{ github.sha }}
REVISION=${{ needs.docker-ent-meta.outputs.revision }}
ENTERPRISE_TOKEN=${{ secrets.ENTERPRISE_TOKEN }}
cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp
docker-sbsvc-meta:
runs-on: ubuntu-latest
outputs:
labels: ${{ steps.meta.outputs.labels }}
build-date: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
version: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
revision: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
tags: ${{ steps.meta.outputs.tags }}
arm64_tags: ${{ steps.arm64_meta.outputs.tags }}
arm64_labels: ${{ steps.arm64_meta.outputs.labels }}
amd64_tags: ${{ steps.amd64_meta.outputs.tags }}
amd64_labels: ${{ steps.amd64_meta.outputs.labels }}
steps:
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_sbsvc_images}}
tags: ${{env.docker_sbsvc_tags}}
- name: Docker arm64 meta
id: arm64_meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_sbsvc_images}}
tags: ${{env.docker_sbsvc_tags}}
flavor: |
suffix=-${{env.arch_arm64}},onlatest=true
- name: Docker amd64 meta
id: amd64_meta
uses: docker/metadata-action@v5
with:
images: ${{env.docker_sbsvc_images}}
tags: ${{env.docker_sbsvc_tags}}
flavor: |
suffix=-${{env.arch_amd64}},onlatest=true
docker-suppression-backup-service:
needs:
- docker-sbsvc-meta
strategy:
matrix:
build-config:
- os: [ self-hosted, Linux, ARM64 ]
tags: ${{needs.docker-sbsvc-meta.outputs.arm64_tags}}
labels: ${{needs.docker-sbsvc-meta.outputs.arm64_labels}}
platform: linux/arm64
- os: ubuntu-latest
tags: ${{needs.docker-sbsvc-meta.outputs.amd64_tags}}
labels: ${{needs.docker-sbsvc-meta.outputs.amd64_labels}}
platform: linux/amd64
runs-on: ${{matrix.build-config.os}}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3.3.0
with:
username: rudderlabs
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build image (no-push)
uses: docker/build-push-action@v6
with:
context: .
platforms: ${{ matrix.build-config.platform }}
file: ./suppression-backup-service/Dockerfile
push: false
tags: ${{ matrix.build-config.tags }}
labels: ${{ matrix.build-config.labels }}
build-args: |
BUILD_DATE=${{ needs.docker-sbsvc-meta.outputs.build-date }}
VERSION=${{ needs.docker-sbsvc-meta.outputs.version }}
COMMIT_HASH=${{ github.sha }}
REVISION=${{ needs.docker-sbsvc-meta.outputs.revision }}
outputs: type=docker,dest=${{ runner.temp }}/suppression-backup-service.tar
cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp
- name: Run TruffleHog
run: |
docker pull trufflesecurity/trufflehog:latest
docker run --rm -v ${{ runner.temp }}:/tmp \
trufflesecurity/trufflehog:latest \
docker --image file:///tmp/suppression-backup-service.tar \
--github-actions \
--no-verification \
--fail
- name: Build and push
if: success()
uses: docker/build-push-action@v6
with:
context: .
platforms: ${{ matrix.build-config.platform }}
file: ./suppression-backup-service/Dockerfile
push: true
tags: ${{ matrix.build-config.tags }}
labels: ${{ matrix.build-config.labels }}
build-args: |
BUILD_DATE=${{ needs.docker-sbsvc-meta.outputs.build-date }}
VERSION=${{ needs.docker-sbsvc-meta.outputs.version }}
COMMIT_HASH=${{ github.sha }}
REVISION=${{ needs.docker-sbsvc-meta.outputs.revision }}
cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp
create-manifest-docker-oss:
runs-on: ubuntu-latest
needs: [ docker-oss, docker-oss-meta ]
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3.3.0
with:
username: rudderlabs
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Create multi-arch manifest for Docker OSS
run: |
while read -r tag; do
echo "$tag"
arm_tag=$(echo "${{ needs.docker-oss-meta.outputs.arm64_tags }}" | grep "$tag")
amd_tag=$(echo "${{ needs.docker-oss-meta.outputs.amd64_tags }}" | grep "$tag")
docker buildx imagetools create -t $tag $arm_tag $amd_tag
done <<< "${{ needs.docker-oss-meta.outputs.tags }}"
create-manifest-docker-ent:
runs-on: ubuntu-latest
needs: [ docker-ent, docker-ent-meta ]
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3.3.0
with:
username: rudderlabs
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Create multi-arch manifest for Docker ENT
run: |
while read -r tag; do
echo "$tag"
arm_tag=$(echo "${{ needs.docker-ent-meta.outputs.arm64_tags }}" | grep "$tag")
amd_tag=$(echo "${{ needs.docker-ent-meta.outputs.amd64_tags }}" | grep "$tag")
docker buildx imagetools create -t $tag $arm_tag $amd_tag
done <<< "${{ needs.docker-ent-meta.outputs.tags }}"
create-manifest-docker-sbsvc:
runs-on: ubuntu-latest
needs: [ docker-suppression-backup-service, docker-sbsvc-meta ]
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3.3.0
with:
username: rudderlabs
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Create multi-arch manifest for Docker Suppression Backup Service
run: |
while read -r tag; do
echo "$tag"
arm_tag=$(echo "${{ needs.docker-sbsvc-meta.outputs.arm64_tags }}" | grep "$tag")
amd_tag=$(echo "${{ needs.docker-sbsvc-meta.outputs.amd64_tags }}" | grep "$tag")
docker buildx imagetools create -t $tag $arm_tag $amd_tag
done <<< "${{ needs.docker-sbsvc-meta.outputs.tags }}"