Skip to content
Build

chore(testing-drivers): arrange tables creation/removal in incrementalSchemaLoading test #15081

Sign in to view logs

chore(testing-drivers): arrange tables creation/removal in incrementalSchemaLoading test

chore(testing-drivers): arrange tables creation/removal in incrementalSchemaLoading test #15081

Workflow file for this run

.github/workflows/push.yml at 84b7968
name: Build
on:
push:
paths:
- '.github/actions/smoke.sh'
- '.github/workflows/push.yml'
- '.github/workflows/master.yml'
- 'packages/**'
- 'rust/cubestore/js-wrapper/**'
- 'rust/cubestore/tsconfig.json'
- 'rust/cubestore/package.json'
- 'rust/cubesql/**'
- '.eslintrc.js'
- '.prettierrc'
- 'package.json'
- 'lerna.json'
- 'rollup.config.js'
- 'yarn.lock'
branches:
- 'master'
pull_request:
paths:
- '.github/workflows/push.yml'
- '.github/workflows/master.yml'
- 'packages/**'
- 'rust/cubestore/js-wrapper/**'
- 'rust/cubestore/tsconfig.json'
- 'rust/cubestore/package.json'
- 'rust/cubesql/**'
- '.eslintrc.js'
- '.prettierrc'
- 'package.json'
- 'lerna.json'
- 'rollup.config.js'
- 'yarn.lock'
jobs:
unit:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: latest-tag-sha
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
strategy:
matrix:
node-version: [14.x, 16.x, 18.x]
fail-fast: false
steps:
- id: get-tag-out
run: echo "$OUT"
env:
OUT: ${{ needs['latest-tag-sha'].outputs.sha }}
- name: Checkout
uses: actions/checkout@v4
with:
# pulls all commits (needed for codecov)
fetch-depth: 2
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2022-03-22
override: true
components: rustfmt
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Restore yarn cache
uses: actions/cache@v3
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.19
- name: Yarn install
uses: nick-invision/retry@v2
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Lerna tsc
run: yarn tsc
- name: Build client
run: yarn build
- name: Lerna test
run: yarn lerna run --concurrency 1 --stream --no-prefix unit
- uses: codecov/codecov-action@v1
if: (matrix.node-version == '16.x')
with:
files: ./packages/*/coverage/clover.xml
flags: cube-backend
verbose: true # optional (default = false)
lint:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: latest-tag-sha
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2022-03-22
override: true
components: rustfmt
- name: Install Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Restore yarn cache
uses: actions/cache@v3
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.19
- name: Yarn install
uses: nick-invision/retry@v2
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: NPM lint
run: yarn lint:npm
- name: Lerna lint
run: yarn lerna run --concurrency 1 lint
build:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: latest-tag-sha
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2022-03-22
override: true
components: rustfmt
- name: Install Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Restore yarn cache
uses: actions/cache@v3
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.19
- name: Yarn install
uses: nick-invision/retry@v2
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Check Yarn lock wasn't modified
run: if [ "$(git status | grep nothing)x" = "x" ]; then echo "Non empty changeset after lerna bootstrap"; git status; exit 1; else echo "Nothing to commit. Proceeding"; fi;
- name: Build Core Client libraries
run: yarn build
- name: Build other packages
run: yarn lerna run --concurrency 1 build
env:
NODE_OPTIONS: --max_old_space_size=4096
integration-redis:
needs: [unit, lint, latest-tag-sha]
runs-on: ubuntu-20.04
timeout-minutes: 60
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
services:
redis:
image: redis
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
matrix:
node-version: [16.x]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2022-03-22
override: true
components: rustfmt
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Restore yarn cache
uses: actions/cache@v3
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.19
- name: Yarn install
uses: nick-invision/retry@v2
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Lerna tsc
run: yarn tsc
- name: Run Redis Integration with Redis Driver
timeout-minutes: 10
run: |
yarn lerna run --concurrency 1 --stream --no-prefix integration:redis
env:
CUBEJS_REDIS_USE_IOREDIS: false
- name: Run Redis Integration with IORedis Driver
timeout-minutes: 10
run: |
yarn lerna run --concurrency 1 --stream --no-prefix integration:redis
env:
CUBEJS_REDIS_USE_IOREDIS: true
- name: Setup Redis Cluster
uses: vishnudxb/redis-cluster@1.0.5
with:
master1-port: 5000
master2-port: 5001
master3-port: 5002
slave1-port: 5003
slave2-port: 5004
slave3-port: 5005
- name: Run Redis Integration with Cluster
timeout-minutes: 10
run: |
yarn lerna run --concurrency 1 --stream --no-prefix integration:redis
env:
CUBEJS_REDIS_USE_IOREDIS: true
CUBEJS_REDIS_SENTINEL: "redis+sentinel://localhost:5000,localhost:5001,localhost:5002/mymaster/0"
integration-cubestore:
needs: [latest-tag-sha]
runs-on: ubuntu-20.04
timeout-minutes: 60
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
strategy:
matrix:
node-version: [16.x]
fail-fast: false
steps:
- name: Maximize build space (disk space limitations)
run: |
echo "Before"
df -h
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^mongodb-.*'
sudo apt-get remove -y '^mysql-.*'
sudo apt-get autoremove -y
sudo apt-get clean
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
echo "After"
df -h
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2022-06-22
override: true
components: rustfmt
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Restore yarn cache
uses: actions/cache@v3
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.19
- name: Yarn install
uses: nick-invision/retry@v2
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Lerna tsc
run: yarn tsc
- uses: Swatinem/rust-cache@v2
with:
workspaces: ./rust/cubestore -> target
shared-key: cubestore
key: ubuntu-20.04
- name: Build cubestore
uses: actions-rs/cargo@v1
with:
command: build
args: --manifest-path rust/cubestore/Cargo.toml -j 4
- name: Run Cube Store in background
run: RUNNER_TRACKING_ID="" && ./rust/cubestore/target/debug/cubestored &
- name: Run Cubestore Integration
timeout-minutes: 10
run: |
yarn lerna run --concurrency 1 --stream --no-prefix integration:cubestore
integration:
needs: [unit, lint, latest-tag-sha]
runs-on: ubuntu-20.04
timeout-minutes: 60
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
strategy:
matrix:
node-version: [16.x]
db: [
'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
'mysql-aurora-serverless', 'crate', 'mongobi'
]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2022-03-22
override: true
components: rustfmt
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Restore yarn cache
uses: actions/cache@v3
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.19
- name: Yarn install
uses: nick-invision/retry@v2
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Lerna tsc
run: yarn tsc
- name: Run Integration tests for ${{ matrix.db }} matrix
timeout-minutes: 30
run: ./.github/actions/integration/${{ matrix.db }}.sh
integration-smoke:
needs: [ latest-tag-sha ]
runs-on: ubuntu-20.04
timeout-minutes: 60
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
strategy:
matrix:
node-version: [ 16.x ]
fail-fast: false
steps:
- name: Maximize build space (disk space limitations)
run: |
echo "Before"
df -h
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^mongodb-.*'
sudo apt-get remove -y '^mysql-.*'
sudo apt-get autoremove -y
sudo apt-get clean
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
echo "After"
df -h
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2022-06-22
override: true
components: rustfmt
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Restore yarn cache
uses: actions/cache@v3
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.19
- name: Yarn install
uses: nick-invision/retry@v2
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Install instant client for Oracle
uses: GoodManWEN/oracle-client-action@main
- name: Build client
run: yarn build
- name: Lerna tsc
run: yarn tsc
- uses: Swatinem/rust-cache@v2
with:
workspaces: ./rust/cubestore -> target
shared-key: cubestore
key: ubuntu-20.04
- name: Build cubestore
uses: actions-rs/cargo@v1
with:
command: build
args: --manifest-path rust/cubestore/Cargo.toml -j 4 --release
- name: Copy Cube Store
run: |
mkdir -p rust/cubestore/downloaded/latest/bin
cp rust/cubestore/target/release/cubestored rust/cubestore/downloaded/latest/bin/cubestored
- name: Run Integration smoke tests
timeout-minutes: 30
run: ./.github/actions/smoke.sh
docker-image-latest-set-tag:
# At least git should be completed pushed up until this moment
needs: [lint, latest-tag-sha]
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
runs-on: ubuntu-20.04
outputs:
tag: ${{ steps.get-tag.outputs.tag }}
steps:
- name: Checkout
uses: actions/checkout@v4
- id: get-tag
run: echo "::set-output name=tag::$(git tag --contains $GITHUB_SHA)"
env:
GITHUB_SHA: ${{ github.sha }}
latest-tag-sha:
runs-on: ubuntu-20.04
outputs:
sha: ${{ steps.get-tag.outputs.sha }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- id: git-log
run: git log HEAD~30..HEAD
- id: get-tag-test
run: echo "$SHA $(git rev-list -n 1 $(git tag --contains $SHA))"
env:
SHA: ${{ github.sha }}
- id: get-tag
run: echo "::set-output name=sha::$(git rev-list -n 1 $(git tag --contains $SHA))"
env:
SHA: ${{ github.sha }}
- id: get-tag-out
run: echo "$OUT"
env:
OUT: ${{ steps.get-tag.outputs.sha }}
docker-dev:
needs: [latest-tag-sha]
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
name: Build & Test :dev for ${{ matrix.name }} without pushing
runs-on: ubuntu-22.04
timeout-minutes: 60
services:
registry:
image: registry:2
ports:
- 5000:5000
strategy:
matrix:
dockerfile:
- dev.Dockerfile
include:
- dockerfile: dev.Dockerfile
name: Debian
tag: tmp-dev
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
version: v0.9.1
driver-opts: network=host
- name: Build image
uses: docker/build-push-action@v3
timeout-minutes: 30
with:
context: .
file: ./packages/cubejs-docker/${{ matrix.dockerfile }}
platforms: linux/amd64
push: true
tags: localhost:5000/cubejs/cube:${{ matrix.tag }}
- name: Use Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Restore yarn cache
uses: actions/cache@v3
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.19
- name: Yarn install
uses: nick-invision/retry@v2
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Build client
run: yarn build
- name: Lerna tsc
run: yarn tsc
- name: Testing CubeJS (container mode) via BirdBox
run: |
cd packages/cubejs-testing/
export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
export DEBUG=testcontainers
yarn run dataset:minimal
yarn run birdbox:postgresql
yarn run birdbox:postgresql-pre-aggregations
# - name: Testing Athena driver (container mode) via BirdBox
# env:
# CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
# CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
# CUBEJS_AWS_REGION: us-east-1
# CUBEJS_AWS_S3_OUTPUT_LOCATION: s3://cubejs-opensource/testing/output
# CUBEJS_DB_EXPORT_BUCKET: s3://cubejs-opensource/testing/export
# run: |
# cd packages/cubejs-testing/
# export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
# export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
# export DEBUG=testcontainers
# yarn run driver:athena --log=ignore --mode=docker
# - name: Testing BigQuery driver (container mode) via BirdBox
# env:
# CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}
# CUBEJS_DB_BQ_PROJECT_ID: cube-open-source
# CUBEJS_DB_EXPORT_BUCKET: cube-open-source-export-bucket
# run: |
# cd packages/cubejs-testing/
# export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
# export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
# export DEBUG=testcontainers
# yarn run driver:bigquery --log=ignore --mode=docker
- name: Testing PostgreSQL driver (container mode) via BirdBox
env:
CUBEJS_DB_TYPE: postgres
CUBEJS_DB_USER: postgres
CUBEJS_DB_PASS: postgres
run: |
cd packages/cubejs-testing/
export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
export DEBUG=testcontainers
yarn run driver:postgres --log=ignore --mode=docker
- name: Testing Docker image via Cypress (Chrome)
env:
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
BIRDBOX_CYPRESS_UPDATE_SCREENSHOTS: ${{ contains(github.event.head_commit.message, '[update screenshots]') }}
run: |
cd packages/cubejs-testing/
export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
export BIRDBOX_CYPRESS_BROWSER=chrome
export BIRDBOX_CYPRESS_TARGET=postgresql
export DEBUG=testcontainers
yarn run cypress:install
yarn run cypress:birdbox
- name: Upload screenshots on failure
uses: actions/upload-artifact@v2
if: failure()
with:
name: cypress-screenshots-docker-dev-${{ matrix.name }}
path: packages/cubejs-testing/cypress/screenshots