feat(flow): flow runtime render reduce operator #15154

Workflow file for this run

.github/workflows/develop.yml at f377499

	on:
	merge_group:
	pull_request:
	types: [ opened, synchronize, reopened, ready_for_review ]
	paths-ignore:
	- 'docs/**'
	- 'config/**'
	- '**.md'
	- '.dockerignore'
	- 'docker/**'
	- '.gitignore'
	- 'grafana/**'
	push:
	branches:
	- main
	paths-ignore:
	- 'docs/**'
	- 'config/**'
	- '**.md'
	- '.dockerignore'
	- 'docker/**'
	- '.gitignore'
	- 'grafana/**'
	workflow_dispatch:

	name: CI

	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	env:
	RUST_TOOLCHAIN: nightly-2023-12-19

	jobs:
	check-typos-and-docs:
	name: Check typos and docs
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v4
	- uses: crate-ci/typos@v1.13.10
	- name: Check the config docs
	run: \|
	make config-docs && \
	git diff --name-only --exit-code ./config/config.md \
	\|\| (echo "'config/config.md' is not up-to-date, please run 'make config-docs'." && exit 1)

	check:
	name: Check
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ windows-latest, ubuntu-20.04 ]
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	# Shares with `Clippy` job
	shared-key: "check-lint"
	- name: Run cargo check
	run: cargo check --locked --workspace --all-targets

	toml:
	name: Toml Check
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: stable
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "check-toml"
	- name: Install taplo
	run: cargo +stable install taplo-cli --version ^0.9 --locked
	- name: Run taplo
	run: taplo format --check

	build:
	name: Build GreptimeDB binaries
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ ubuntu-20.04 ]
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "build-binaries"
	- name: Build greptime binaries
	shell: bash
	run: cargo build --bin greptime --bin sqlness-runner
	- name: Pack greptime binaries
	shell: bash
	run: \|
	mkdir bins && \
	mv ./target/debug/greptime bins && \
	mv ./target/debug/sqlness-runner bins
	- name: Print greptime binaries info
	run: ls -lh bins
	- name: Upload artifacts
	uses: ./.github/actions/upload-artifacts
	with:
	artifacts-dir: bins
	version: current

	fuzztest:
	name: Fuzz Test
	needs: build
	runs-on: ubuntu-latest
	strategy:
	matrix:
	target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database" ]
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "fuzz-test-targets"
	- name: Set Rust Fuzz
	shell: bash
	run: \|
	sudo apt update && sudo apt install -y libfuzzer-14-dev
	cargo install cargo-fuzz
	- name: Download pre-built binaries
	uses: actions/download-artifact@v4
	with:
	name: bins
	path: .
	- name: Unzip binaries
	run: tar -xvf ./bins.tar.gz
	- name: Run GreptimeDB
	run: \|
	./bins/greptime standalone start&
	- name: Fuzz Test
	uses: ./.github/actions/fuzz-test
	env:
	CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
	with:
	target: ${{ matrix.target }}

	sqlness:
	name: Sqlness Test
	needs: build
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ ubuntu-20.04 ]
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- name: Download pre-built binaries
	uses: actions/download-artifact@v4
	with:
	name: bins
	path: .
	- name: Unzip binaries
	run: tar -xvf ./bins.tar.gz
	- name: Run sqlness
	run: RUST_BACKTRACE=1 ./bins/sqlness-runner -c ./tests/cases --bins-dir ./bins
	- name: Upload sqlness logs
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: sqlness-logs
	path: /tmp/greptime-*.log
	retention-days: 3

	sqlness-kafka-wal:
	name: Sqlness Test with Kafka Wal
	needs: build
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ ubuntu-20.04 ]
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- name: Download pre-built binaries
	uses: actions/download-artifact@v4
	with:
	name: bins
	path: .
	- name: Unzip binaries
	run: tar -xvf ./bins.tar.gz
	- name: Setup kafka server
	working-directory: tests-integration/fixtures/kafka
	run: docker compose -f docker-compose-standalone.yml up -d --wait
	- name: Run sqlness
	run: RUST_BACKTRACE=1 ./bins/sqlness-runner -w kafka -k 127.0.0.1:9092 -c ./tests/cases --bins-dir ./bins
	- name: Upload sqlness logs
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: sqlness-logs-with-kafka-wal
	path: /tmp/greptime-*.log
	retention-days: 3

	fmt:
	name: Rustfmt
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	components: rustfmt
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "check-rust-fmt"
	- name: Run cargo fmt
	run: cargo fmt --all -- --check

	clippy:
	name: Clippy
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	components: clippy
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	# Shares with `Check` job
	shared-key: "check-lint"
	- name: Run cargo clippy
	run: cargo clippy --workspace --all-targets -- -D warnings

	coverage:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-20.04-8-cores
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: KyleMayes/install-llvm-action@v1
	with:
	version: "14.0"
	- name: Install toolchain
	uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	components: llvm-tools-preview
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares cross multiple jobs
	shared-key: "coverage-test"
	- name: Docker Cache
	uses: ScribeMD/docker-cache@0.3.7
	with:
	key: docker-${{ runner.os }}-coverage
	- name: Install latest nextest release
	uses: taiki-e/install-action@nextest
	- name: Install cargo-llvm-cov
	uses: taiki-e/install-action@cargo-llvm-cov
	- name: Install Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.10'
	- name: Install PyArrow Package
	run: pip install pyarrow
	- name: Setup etcd server
	working-directory: tests-integration/fixtures/etcd
	run: docker compose -f docker-compose-standalone.yml up -d --wait
	- name: Setup kafka server
	working-directory: tests-integration/fixtures/kafka
	run: docker compose -f docker-compose-standalone.yml up -d --wait
	- name: Run nextest cases
	run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
	env:
	CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
	RUST_BACKTRACE: 1
	CARGO_INCREMENTAL: 0
	GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
	GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
	GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
	GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
	GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
	GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
	UNITTEST_LOG_DIR: "__unittest_logs"
	- name: Codecov upload
	uses: codecov/codecov-action@v4
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	files: ./lcov.info
	flags: rust
	fail_ci_if_error: false
	verbose: true

	compat:
	name: Compatibility Test
	needs: build
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- name: Download pre-built binaries
	uses: actions/download-artifact@v4
	with:
	name: bins
	path: .
	- name: Unzip binaries
	run: \|
	mkdir -p ./bins/current
	tar -xvf ./bins.tar.gz --strip-components=1 -C ./bins/current
	- run: ./tests/compat/test-compat.sh 0.6.0

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(flow): flow runtime render reduce operator #15154

Workflow file

feat(flow): flow runtime render reduce operator #15154

Jobs

Run details

Workflow file for this run