Pull out message cache from layout reader (#1773) #6051

Workflow file for this run

	name: CI

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}

	on:
	push:
	branches: ["develop"]
	pull_request: {}
	workflow_dispatch: {}

	permissions:
	actions: read
	contents: read
	checks: write # audit-check creates checks
	issues: write # audit-check creates issues

	env:
	CARGO_TERM_COLOR: always
	RUST_BACKTRACE: 1

	jobs:
	python-lint:
	name: "Python (lint)"
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: spiraldb/actions/.github/actions/setup-uv@0.2.0

	- name: Python Lint - Format
	run: uv run ruff format --check .
	- name: Python Lint - Ruff
	run: uv run ruff check .

	python-test:
	name: "Python (test)"
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	- uses: spiraldb/actions/.github/actions/setup-uv@0.2.0

	- name: Pytest - PyVortex
	run: \|
	uv run pytest --benchmark-disable test/
	working-directory: pyvortex/

	- name: Doctest - PyVortex
	run: \|
	uv run make clean && uv run make doctest
	working-directory: docs/

	- name: Ensure docs build - PyVortex
	run: \|
	uv run make html
	working-directory: docs/

	- name: Ensure wheel and sdist can be built on Linux - PyVortex
	run: \|
	# force linux compatibility to ensure there is exactly one wheel ergo no ambiguity in installation
	uv run maturin build --interpreter python3.10 --compatibility linux
	working-directory: pyvortex/

	- name: Ensure wheel can run tests successfully
	run: \|
	set -ex

	echo creating a fresh venv which does not have the normal dependencies to ensure the wheel dependencies are correct

	rm -rf .venv
	uv sync --only-dev

	uv run pip install target/wheels/vortex_array-*.whl \|\| { uv run pip debug --verbose ; exit 1 ; }
	uv run pip install pytest
	cd pyvortex/test
	uv run pytest

	rust-docs:
	name: "Rust (docs)"
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: ./.github/actions/setup-rust
	- name: Docs
	run: cargo doc --no-deps

	build-rust:
	name: "Rust build (${{matrix.config.name}})"
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	config:
	- name: "all-features"
	command: "build"
	args: "--all-features --all-targets"
	- name: "default features"
	command: "build"
	args: "--all-targets"
	- name: "with tokio dispatcher"
	command: "build"
	args: "--no-default-features --features tokio --all-targets --ignore-unknown-features"
	- name: "with compio dispatcher"
	command: "build"
	args: "--no-default-features --features compio --all-targets --ignore-unknown-features"
	- name: "with tokio+compio dispatcher"
	command: "build"
	args: "--no-default-features --features tokio,compio --all-targets --ignore-unknown-features"
	- name: "wasm32 with default features"
	command: "build"
	target: wasm32-unknown-unknown
	args: "--target wasm32-unknown-unknown --exclude vortex-roaring --exclude vortex-datafusion "
	steps:
	- uses: actions/checkout@v4
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	with:
	targets: ${{matrix.config.target \|\| ''}}
	- name: Install cargo-hack
	uses: taiki-e/install-action@cargo-hack
	- name: Rust Build (${{matrix.config.name}})
	run: cargo hack ${{matrix.config.command}} ${{matrix.config.args}} --ignore-private

	rust-lint:
	name: "Rust (lint)"
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	- name: Rust Lint - Format
	run: cargo fmt --all --check
	- name: Rust Lint - Clippy All Features
	run: cargo clippy --all-features --all-targets
	- name: Rust Lint - Clippy Default Features
	run: cargo clippy --all-targets
	- name: Rust Lint - Clippy No Default Features
	run: \|
	set -ex

	# https://spiraldb.slack.com/archives/C07BV3GKAJ2/p1732736281946729
	for package in $(cargo check -p 2>&1 \| grep '^ ')
	do
	echo ---- $package ----
	cargo clippy --package $package --no-default-features
	done

	rust-test:
	name: "Rust (tests)"
	runs-on: ubuntu-latest
	env:
	# Disable incremental compilation to get accurate coverage
	CARGO_INCREMENTAL: "0"
	RUSTFLAGS: "-Cinstrument-coverage"
	LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw"
	GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov"
	steps:
	- uses: actions/checkout@v4
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	- name: Install grcov
	run: cargo install grcov
	- name: Rust Test
	run: cargo test --workspace --all-features
	- name: Generate coverage report
	run: \|
	grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
	--ignore '../' --ignore '/' --ignore 'bench-vortex/' --ignore 'fuzz/' \
	--ignore 'home/' --ignore 'xtask/' --ignore 'target/*'\
	-o ${{ env.GRCOV_OUTPUT_FILE }}
	- name: Coveralls
	uses: coverallsapp/github-action@v2
	with:
	file: ${{ env.GRCOV_OUTPUT_FILE }}

	license-check-and-audit-check:
	name: License Check and Audit Check
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	- uses: actions/checkout@v4
	- name: License Check
	run: cargo install --locked cargo-deny && cargo deny check
	- uses: rustsec/audit-check@v2.0.0
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	ignore: "RUSTSEC-2023-0384"

	miri:
	name: "miri"
	runs-on: ubuntu-latest
	env:
	MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-backtrace=full -Zmiri-disable-isolation
	steps:
	- uses: rui314/setup-mold@v1
	- uses: actions/checkout@v4
	- uses: ./.github/actions/setup-rust
	- name: Run tests with Miri
	run: cargo miri test

	bench-test:
	name: "bench test"
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	# Required to run benchmarks
	- name: Install DuckDB
	uses: opt-nc/setup-duckdb-action@v1.0.10
	with:
	version: v1.0.0
	- name: Rust TPC-H and Random access Bench as test
	run: cargo bench --bench tpch --bench random_access --profile benchtest -- --test

	generated-files:
	name: "Check generated proto/fbs files are up to date"
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	- uses: ./.github/actions/setup-flatc
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	- name: "regenerate all .fbs/.proto Rust code"
	run: \|
	cargo xtask generate-fbs
	cargo xtask generate-proto
	- name: "Make sure no files changed after regenerating"
	run: \|
	test -z "$(git status --porcelain)"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Pull out message cache from layout reader (#1773) #6051

Workflow file

Pull out message cache from layout reader (#1773) #6051

Jobs

Run details

Workflow file for this run