moved blogs from old site (#843) #855

Workflow file for this run

.github/workflows/ci_python.yml at 229bb87

	# This file was initially generated by maturin v1.2.0, using:
	#
	# maturin generate-ci github --pytest -o ../.github/workflows/ci_python.yml
	#
	# TODO:
	# - `safety`
	# - `typeguard`
	# - `xdoctest`
	#
	# Currently, `mypy` and `pytest` are running as part of building the rules.
	# We could (in theory) use a matrix to do these in parallel fetching the
	# built wheels, but it doesn't seem likely to provide a significant speedup.
	name: Python CI

	# Only one job per-ref
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	env:
	# See https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Workflow.
	# Speeds up CI builds, which won't have previous results anyway.
	CARGO_INCREMENTAL: 0

	on:
	push:
	branches:
	- main
	tags:
	- '*'
	pull_request:
	branches:
	- main
	merge_group:
	branches:
	- main

	defaults:
	run:
	shell: bash
	working-directory: python

	permissions:
	actions: write
	# Used by clippy action to report lint errors.
	checks: write
	contents: read

	jobs:
	# Run lint separately from the build -- it doesn't need the built Rust code.
	#
	# Also, only do it on one machine.
	python-lint:
	if: github.event_name == 'pull_request' \|\| github.event_name == 'merge_group'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	- name: Install poetry
	run: \|
	pipx install poetry
	poetry config virtualenvs.create true --local
	poetry config virtualenvs.in-project true --local
	- uses: actions/setup-python@v4
	with:
	python-version: 3.11
	cache: poetry
	- name: install deps
	run: \|
	poetry install --only=main --only=lint
	- name: black
	run: \|
	poetry run black --diff pysrc pytests docs
	- name: flake8
	run: \|
	poetry run flake8 pysrc pytests docs
	- name: isort
	run: \|
	poetry run isort --filter-files --diff pysrc pytests docs
	- name: pydocstyle
	run: \|
	poetry run pydocstyle pysrc

	rust-build-test:
	if: \|
	github.event_name == 'pull_request' \|\|
	github.event_name == 'merge_group' \|\|
	(github.event_name == 'push' && github.ref == 'refs/heads/main')
	runs-on:
	group: linux-rust-builds
	steps:
	- uses: actions/checkout@v3
	- name: Install stable Rust toolchain
	id: toolchain
	uses: dtolnay/rust-toolchain@master
	with:
	toolchain: stable
	components: clippy
	- uses: awalsh128/cache-apt-pkgs-action@latest
	with:
	packages: protobuf-compiler lld
	version: 1.0
	# For caching, we use a special `rust-cache` action, which ensures we
	# cache based on the architecture, and also deletes stuff that shouldn't
	# be put in the cache (such as the toolchain).
	#
	# See
	# https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
	#
	# This currently saves the cache on each branch. This may exceed GitHub's
	# cache size limit. If we start hitting that, we could choose to only
	# cache the `main` branch, which would provide caching for cases where the
	# dependencies of crates haven't changed. See
	# https://github.com/Swatinem/rust-cache/issues/95.
	- name: Cache Rust Dependencies
	uses: Swatinem/rust-cache@v2
	with:
	workspaces:
	.
	python
	# Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
	cache-directories: "~/.cargo/registry/src/*/librocksdb-sys-"
	- name: Rust Tests
	uses: actions-rs/cargo@v1
	with:
	command: test
	args: --all-features -- --nocapture --quiet
	# Clippy needs to build the code, so we run it as part of the main CI since
	# we've already spent the time building it.
	- name: Cargo Clippy
	uses: actions-rs/clippy-check@v1
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	# The `lib.rs` or `bin.rs` files for each create should specify the same
	# flags to ensure vscode, etc. reports warnings. But, specifying them in the
	# CI protects against cases where we forgot to do that.
	args: >
	--all-features
	-- -Dwarnings -Drust-2018-idioms -Dnonstandard-style -Dfuture-incompatible
	-Dclippy::mod_module_files
	-Dclippy::print-stdout
	-Dclippy::print-stderr
	-Dclippy::undocumented-unsafe-blocks

	python-build-test:
	if: \|
	github.event_name == 'pull_request' \|\|
	github.event_name == 'merge_group' \|\|
	(github.event_name == 'push' && github.ref == 'refs/heads/main')
	runs-on:
	group: linux-rust-builds
	steps:
	- uses: actions/checkout@v3
	- name: Install poetry
	run: \|
	pipx install poetry
	poetry config virtualenvs.create true --local
	poetry config virtualenvs.in-project true --local
	- uses: actions/setup-python@v4
	with:
	python-version: \|
	3.9
	3.10
	3.11
	cache: poetry
	- name: Install stable Rust toolchain
	id: toolchain
	uses: dtolnay/rust-toolchain@master
	with:
	toolchain: stable
	components: clippy
	- uses: awalsh128/cache-apt-pkgs-action@latest
	with:
	packages: protobuf-compiler lld
	version: 1.0
	# For caching, we use a special `rust-cache` action, which ensures we
	# cache based on the architecture, and also deletes stuff that shouldn't
	# be put in the cache (such as the toolchain).
	#
	# See
	# https://matklad.github.io/2021/09/04/fast-rust-builds.html#CI-Caching.
	#
	# This currently saves the cache on each branch. This may exceed GitHub's
	# cache size limit. If we start hitting that, we could choose to only
	# cache the `main` branch, which would provide caching for cases where the
	# dependencies of crates haven't changed. See
	# https://github.com/Swatinem/rust-cache/issues/95.
	- name: Cache Rust Dependencies
	uses: Swatinem/rust-cache@v2
	with:
	workspaces:
	.
	python
	# Cache the rocksdb crate. https://github.com/MystenLabs/sui/pull/1164
	cache-directories: "~/.cargo/registry/src/*/librocksdb-sys-"
	- name: Build debug wheels
	uses: messense/maturin-action@v1
	with:
	args: --out dist --profile dev
	# Build on host instead of a manylinux container.
	container: off
	working-directory: python
	- name: Determine Version
	# The wheel suffix is determined by seeing what `maturin build` printed.
	run: \|
	pip install tomlkit packaging
	VERSION=$(python ../scripts/version.py get --normalize pyproject.toml project.version)
	WHEEL=dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl
	echo "WHEEL: ${WHEEL}"
	echo "WHEEL=${WHEEL}" >> $GITHUB_ENV
	- name: pytest and mypy
	# This installs the kaskada package using the wheel.
	# This ensures that we don't accidentally install the version from pypi.
	run: \|
	for V in 3.9 3.10 3.11; do
	echo "::group::Install for Python $V"
	poetry env use $V
	poetry env info
	source $(poetry env info --path)/bin/activate
	poetry install --only=test --only=typecheck
	pip install ${WHEEL} --force-reinstall
	echo "::endgroup::"
	echo "::group::Test Python $V"
	poetry run pytest pytests -S minio
	echo "::endgroup::"
	echo "::group::MyPy Python $V"
	poetry run mypy -- --install-types --non-interactive pysrc pytests
	echo "::endgroup::"
	deactivate
	done
	- name: Lint reference docs
	run: \|
	poetry env use 3.11
	source $(poetry env info --path)/bin/activate
	poetry install --with=docs
	cd docs
	python _scripts/lint_reference.py
	deactivate
	- name: Set up Quarto
	uses: quarto-dev/quarto-actions/setup@v2
	- name: Build docs
	run: \|
	poetry env use 3.11
	source $(poetry env info --path)/bin/activate
	cd docs
	python _scripts/gen_reference.py
	python -m quartodoc interlinks
	quarto render --output-dir _site
	deactivate
	- name: Upload docs
	uses: actions/upload-pages-artifact@v2
	with:
	# Automatically uploads an artifact from the './_site' directory by default
	path: ${{ github.workspace }}/python/docs/_site

	rust-format:
	runs-on: ubuntu-20.04
	steps:
	- name: Checkout sources
	uses: actions/checkout@v3

	- name: Install stable toolchain
	id: toolchain
	uses: dtolnay/rust-toolchain@master
	with:
	toolchain: stable
	components: rustfmt

	# `cargo fmt` doesn't need any dependencies, so no need to cache.
	# Restoring the cache takes 2m. Running rustfmt takes 10s.

	- name: Run cargo fmt (main Rust code)
	uses: actions-rs/cargo@v1
	with:
	command: fmt
	args: --all -- --check

	- name: Run cargo fmt (Python Rust code)
	uses: actions-rs/cargo@v1
	with:
	command: fmt
	args: --all --manifest-path python/Cargo.toml -- --check

	rust-audit:
	runs-on: ubuntu-20.04

	# Audits don't need a toolchain or any of the dependencies, so we don't cache.
	# Restoring the cache takes 2m. Running `cargo deny` takes 10s.

	strategy:
	matrix:
	checks:
	- advisories
	- bans licenses sources

	# Prevent sudden announcement of a new advisory from failing ci:
	continue-on-error: ${{ matrix.checks == 'advisories' }}

	steps:
	- uses: actions/checkout@v3
	- uses: EmbarkStudios/cargo-deny-action@v1
	with:
	command: check ${{ matrix.checks }}

	rust-unused-deps:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v3
	# This doesn't use the `cargo-machete` action since that doesn't cache
	# the machete binary.env:
	- name: Install cargo-machete from crates.io
	uses: baptiste0928/cargo-install@v2
	with:
	crate: cargo-machete
	- name: Machete (Sparrow)
	working-directory:
	run:
	cargo machete
	- name: Machete (Python)
	run:
	cargo machete

	docs-deploy:
	# Deploy docs on push to main.
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	# Deployment job
	environment:
	name: github-pages
	url: ${{ steps.deployment.outputs.page_url }}

	# Grant GITHUB_TOKEN the permissions required to make a Pages deployment
	permissions:
	pages: write # to deploy to Pages
	id-token: write # to verify the deployment originates from an appropriate source

	runs-on: ubuntu-latest
	needs: [python-build-test]
	steps:
	- name: Deploy to GitHub Pages
	id: deployment
	uses: actions/deploy-pages@v2

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

moved blogs from old site (#843) #855

Workflow file

moved blogs from old site (#843) #855

Jobs

Run details

Workflow file for this run