Allow longer timeout for starting pageserver, safe keeper and storage controller in test cases to make test cases less flaky #22768

Workflow file for this run

.github/workflows/build_and_test.yml at cfd8645

	name: Build and Test

	on:
	push:
	branches:
	- main
	- release
	- release-proxy
	pull_request:

	defaults:
	run:
	shell: bash -euxo pipefail {0}

	concurrency:
	# Allow only one workflow per any non-`main` branch.
	group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha \|\| 'anysha' }}
	cancel-in-progress: true

	env:
	RUST_BACKTRACE: 1
	COPT: '-Werror'
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}
	# A concurrency group that we use for e2e-tests runs, matches `concurrency.group` above with `github.repository` as a prefix
	E2E_CONCURRENCY_GROUP: ${{ github.repository }}-e2e-tests-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha \|\| 'anysha' }}

	jobs:
	check-permissions:
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }}
	uses: ./.github/workflows/check-permissions.yml
	with:
	github-event-name: ${{ github.event_name}}

	cancel-previous-e2e-tests:
	needs: [ check-permissions ]
	if: github.event_name == 'pull_request'
	runs-on: ubuntu-22.04

	steps:
	- name: Cancel previous e2e-tests runs for this PR
	env:
	GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
	run: \|
	gh workflow --repo neondatabase/cloud \
	run cancel-previous-in-concurrency-group.yml \
	--field concurrency_group="${{ env.E2E_CONCURRENCY_GROUP }}"

	tag:
	needs: [ check-permissions ]
	runs-on: [ self-hosted, gen3, small ]
	container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
	outputs:
	build-tag: ${{steps.build-tag.outputs.tag}}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Get build tag
	run: \|
	echo run:$GITHUB_RUN_ID
	echo ref:$GITHUB_REF_NAME
	echo rev:$(git rev-list --count HEAD)
	if [[ "$GITHUB_REF_NAME" == "main" ]]; then
	echo "tag=$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
	elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
	echo "tag=release-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
	elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then
	echo "tag=release-proxy-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
	else
	echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
	echo "tag=$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
	fi
	shell: bash
	id: build-tag

	check-build-tools-image:
	needs: [ check-permissions ]
	uses: ./.github/workflows/check-build-tools-image.yml

	build-build-tools-image:
	needs: [ check-build-tools-image ]
	uses: ./.github/workflows/build-build-tools-image.yml
	with:
	image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }}
	secrets: inherit

	check-codestyle-python:
	needs: [ check-permissions, build-build-tools-image ]
	runs-on: [ self-hosted, gen3, small ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	options: --init

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: false
	fetch-depth: 1

	- name: Cache poetry deps
	uses: actions/cache@v4
	with:
	path: ~/.cache/pypoetry/virtualenvs
	key: v2-${{ runner.os }}-python-deps-${{ hashFiles('poetry.lock') }}

	- name: Install Python deps
	run: ./scripts/pysync

	- name: Run `ruff check` to ensure code format
	run: poetry run ruff check .

	- name: Run `ruff format` to ensure code format
	run: poetry run ruff format --check .

	- name: Run mypy to check types
	run: poetry run mypy .

	check-codestyle-rust:
	needs: [ check-permissions, build-build-tools-image ]
	runs-on: [ self-hosted, gen3, small ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	options: --init

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 1

	# Disabled for now
	# - name: Restore cargo deps cache
	# id: cache_cargo
	# uses: actions/cache@v4
	# with:
	# path: \|
	# !~/.cargo/registry/src
	# ~/.cargo/git/
	# target/
	# key: v1-${{ runner.os }}-cargo-clippy-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }}

	# Some of our rust modules use FFI and need those to be checked
	- name: Get postgres headers
	run: make postgres-headers -j$(nproc)

	# cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations.
	# This will catch compiler & clippy warnings in all feature combinations.
	# TODO: use cargo hack for build and test as well, but, that's quite expensive.
	# NB: keep clippy args in sync with ./run_clippy.sh
	- run: \|
	CLIPPY_COMMON_ARGS="$( source .neon_clippy_args; echo "$CLIPPY_COMMON_ARGS")"
	if [ "$CLIPPY_COMMON_ARGS" = "" ]; then
	echo "No clippy args found in .neon_clippy_args"
	exit 1
	fi
	echo "CLIPPY_COMMON_ARGS=${CLIPPY_COMMON_ARGS}" >> $GITHUB_ENV
	- name: Run cargo clippy (debug)
	run: cargo hack --feature-powerset clippy $CLIPPY_COMMON_ARGS
	- name: Run cargo clippy (release)
	run: cargo hack --feature-powerset clippy --release $CLIPPY_COMMON_ARGS

	- name: Check documentation generation
	run: cargo doc --workspace --no-deps --document-private-items
	env:
	RUSTDOCFLAGS: "-Dwarnings -Arustdoc::private_intra_doc_links"

	# Use `${{ !cancelled() }}` to run quck tests after the longer clippy run
	- name: Check formatting
	if: ${{ !cancelled() }}
	run: cargo fmt --all -- --check

	# https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci
	- name: Check rust dependencies
	if: ${{ !cancelled() }}
	run: \|
	cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date
	cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack

	# https://github.com/EmbarkStudios/cargo-deny
	- name: Check rust licenses/bans/advisories/sources
	if: ${{ !cancelled() }}
	run: cargo deny check --hide-inclusion-graph

	build-neon:
	needs: [ check-permissions, tag, build-build-tools-image ]
	runs-on: [ self-hosted, gen3, large ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	# Raise locked memory limit for tokio-epoll-uring.
	# On 5.10 LTS kernels < 5.10.162 (and generally mainline kernels < 5.12),
	# io_uring will account the memory of the CQ and SQ as locked.
	# More details: https://github.com/neondatabase/neon/issues/6373#issuecomment-1905814391
	options: --init --shm-size=512mb --ulimit memlock=67108864:67108864
	strategy:
	fail-fast: false
	matrix:
	build_type: [ debug, release ]
	env:
	BUILD_TYPE: ${{ matrix.build_type }}
	GIT_VERSION: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	BUILD_TAG: ${{ needs.tag.outputs.build-tag }}

	steps:
	- name: Fix git ownership
	run: \|
	# Workaround for `fatal: detected dubious ownership in repository at ...`
	#
	# Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers
	# Ref https://github.com/actions/checkout/issues/785
	#
	git config --global --add safe.directory ${{ github.workspace }}
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	for r in 14 15 16; do
	git config --global --add safe.directory "${{ github.workspace }}/vendor/postgres-v$r"
	git config --global --add safe.directory "${GITHUB_WORKSPACE}/vendor/postgres-v$r"
	done

	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 1

	- name: Set pg 14 revision for caching
	id: pg_v14_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT

	- name: Set pg 15 revision for caching
	id: pg_v15_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT

	- name: Set pg 16 revision for caching
	id: pg_v16_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) >> $GITHUB_OUTPUT

	# Set some environment variables used by all the steps.
	#
	# CARGO_FLAGS is extra options to pass to "cargo build", "cargo test" etc.
	# It also includes --features, if any
	#
	# CARGO_FEATURES is passed to "cargo metadata". It is separate from CARGO_FLAGS,
	# because "cargo metadata" doesn't accept --release or --debug options
	#
	# We run tests with addtional features, that are turned off by default (e.g. in release builds), see
	# corresponding Cargo.toml files for their descriptions.
	- name: Set env variables
	run: \|
	CARGO_FEATURES="--features testing"
	if [[ $BUILD_TYPE == "debug" ]]; then
	cov_prefix="scripts/coverage --profraw-prefix=$GITHUB_JOB --dir=/tmp/coverage run"
	CARGO_FLAGS="--locked"
	elif [[ $BUILD_TYPE == "release" ]]; then
	cov_prefix=""
	CARGO_FLAGS="--locked --release"
	fi
	{
	echo "cov_prefix=${cov_prefix}"
	echo "CARGO_FEATURES=${CARGO_FEATURES}"
	echo "CARGO_FLAGS=${CARGO_FLAGS}"
	echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo"
	} >> $GITHUB_ENV

	# Disabled for now
	# Don't include the ~/.cargo/registry/src directory. It contains just
	# uncompressed versions of the crates in ~/.cargo/registry/cache
	# directory, and it's faster to let 'cargo' to rebuild it from the
	# compressed crates.
	# - name: Cache cargo deps
	# id: cache_cargo
	# uses: actions/cache@v4
	# with:
	# path: \|
	# ~/.cargo/registry/
	# !~/.cargo/registry/src
	# ~/.cargo/git/
	# target/
	# # Fall back to older versions of the key, if no cache for current Cargo.lock was found
	# key: \|
	# v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }}
	# v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-

	- name: Cache postgres v14 build
	id: cache_pg_14
	uses: actions/cache@v4
	with:
	path: pg_install/v14
	key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }}

	- name: Cache postgres v15 build
	id: cache_pg_15
	uses: actions/cache@v4
	with:
	path: pg_install/v15
	key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }}

	- name: Cache postgres v16 build
	id: cache_pg_16
	uses: actions/cache@v4
	with:
	path: pg_install/v16
	key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }}

	- name: Build postgres v14
	if: steps.cache_pg_14.outputs.cache-hit != 'true'
	run: mold -run make postgres-v14 -j$(nproc)

	- name: Build postgres v15
	if: steps.cache_pg_15.outputs.cache-hit != 'true'
	run: mold -run make postgres-v15 -j$(nproc)

	- name: Build postgres v16
	if: steps.cache_pg_16.outputs.cache-hit != 'true'
	run: mold -run make postgres-v16 -j$(nproc)

	- name: Build neon extensions
	run: mold -run make neon-pg-ext -j$(nproc)

	- name: Build walproposer-lib
	run: mold -run make walproposer-lib -j$(nproc)

	- name: Run cargo build
	run: \|
	${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests

	# Do install before running rust tests because they might recompile the
	# binaries with different features/flags.
	- name: Install rust binaries
	run: \|
	# Install target binaries
	mkdir -p /tmp/neon/bin/
	binaries=$(
	${cov_prefix} cargo metadata $CARGO_FEATURES --format-version=1 --no-deps \|
	jq -r '.packages[].targets[] \| select(.kind \| index("bin")) \| .name'
	)
	for bin in $binaries; do
	SRC=target/$BUILD_TYPE/$bin
	DST=/tmp/neon/bin/$bin
	cp "$SRC" "$DST"
	done

	# Install test executables and write list of all binaries (for code coverage)
	if [[ $BUILD_TYPE == "debug" ]]; then
	# Keep bloated coverage data files away from the rest of the artifact
	mkdir -p /tmp/coverage/

	mkdir -p /tmp/neon/test_bin/

	test_exe_paths=$(
	${cov_prefix} cargo test $CARGO_FLAGS $CARGO_FEATURES --message-format=json --no-run \|
	jq -r '.executable \| select(. != null)'
	)
	for bin in $test_exe_paths; do
	SRC=$bin
	DST=/tmp/neon/test_bin/$(basename $bin)

	# We don't need debug symbols for code coverage, so strip them out to make
	# the artifact smaller.
	strip "$SRC" -o "$DST"
	echo "$DST" >> /tmp/coverage/binaries.list
	done

	for bin in $binaries; do
	echo "/tmp/neon/bin/$bin" >> /tmp/coverage/binaries.list
	done
	fi

	- name: Run rust tests
	env:
	NEXTEST_RETRIES: 3
	run: \|
	#nextest does not yet support running doctests
	cargo test --doc $CARGO_FLAGS $CARGO_FEATURES

	for io_engine in std-fs tokio-epoll-uring ; do
	NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IOENGINE=$io_engine ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES
	done

	# Run separate tests for real S3
	export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
	export REMOTE_STORAGE_S3_BUCKET=neon-github-ci-tests
	export REMOTE_STORAGE_S3_REGION=eu-central-1
	${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(remote_storage)' -E 'test(test_real_s3)'

	# Run separate tests for real Azure Blob Storage
	# XXX: replace region with `eu-central-1`-like region
	export ENABLE_REAL_AZURE_REMOTE_STORAGE=y
	export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}"
	export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}"
	export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}"
	export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}"
	${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(remote_storage)' -E 'test(test_real_azure)'

	- name: Install postgres binaries
	run: cp -a pg_install /tmp/neon/pg_install

	- name: Upload Neon artifact
	uses: ./.github/actions/upload
	with:
	name: neon-${{ runner.os }}-${{ matrix.build_type }}-artifact
	path: /tmp/neon

	# XXX: keep this after the binaries.list is formed, so the coverage can properly work later
	- name: Merge and upload coverage data
	if: matrix.build_type == 'debug'
	uses: ./.github/actions/save-coverage-data

	regress-tests:
	needs: [ check-permissions, build-neon, build-build-tools-image, tag ]
	runs-on: [ self-hosted, gen3, large ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	# for changed limits, see comments on `options:` earlier in this file
	options: --init --shm-size=512mb --ulimit memlock=67108864:67108864
	strategy:
	fail-fast: false
	matrix:
	build_type: [ debug, release ]
	pg_version: [ v14, v15, v16 ]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 1

	- name: Pytest regression tests
	uses: ./.github/actions/run-python-test-set
	timeout-minutes: 60
	with:
	build_type: ${{ matrix.build_type }}
	test_selection: regress
	needs_postgres_source: true
	run_with_real_s3: true
	real_s3_bucket: neon-github-ci-tests
	real_s3_region: eu-central-1
	rerun_flaky: true
	pg_version: ${{ matrix.pg_version }}
	env:
	TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
	CHECK_ONDISK_DATA_COMPATIBILITY: nonempty
	BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
	PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring
	PAGESERVER_GET_VECTORED_IMPL: vectored
	PAGESERVER_GET_IMPL: vectored
	PAGESERVER_VALIDATE_VEC_GET: true

	# Temporary disable this step until we figure out why it's so flaky
	# Ref https://github.com/neondatabase/neon/issues/4540
	- name: Merge and upload coverage data
	if: \|
	false &&
	matrix.build_type == 'debug' && matrix.pg_version == 'v14'
	uses: ./.github/actions/save-coverage-data

	get-benchmarks-durations:
	outputs:
	json: ${{ steps.get-benchmark-durations.outputs.json }}
	needs: [ check-permissions, build-build-tools-image ]
	runs-on: [ self-hosted, gen3, small ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	options: --init
	if: github.ref_name == 'main' \|\| contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Cache poetry deps
	uses: actions/cache@v4
	with:
	path: ~/.cache/pypoetry/virtualenvs
	key: v1-${{ runner.os }}-python-deps-${{ hashFiles('poetry.lock') }}

	- name: Install Python deps
	run: ./scripts/pysync

	- name: get benchmark durations
	id: get-benchmark-durations
	env:
	TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
	run: \|
	poetry run ./scripts/benchmark_durations.py "${TEST_RESULT_CONNSTR}" \
	--days 10 \
	--output /tmp/benchmark_durations.json
	echo "json=$(jq --compact-output '.' /tmp/benchmark_durations.json)" >> $GITHUB_OUTPUT

	benchmarks:
	needs: [ check-permissions, build-neon, build-build-tools-image, get-benchmarks-durations ]
	runs-on: [ self-hosted, gen3, small ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	# for changed limits, see comments on `options:` earlier in this file
	options: --init --shm-size=512mb --ulimit memlock=67108864:67108864
	if: github.ref_name == 'main' \|\| contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
	strategy:
	fail-fast: false
	matrix:
	# the amount of groups (N) should be reflected in `extra_params: --splits N ...`
	pytest_split_group: [ 1, 2, 3, 4, 5 ]
	build_type: [ release ]
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Pytest benchmarks
	uses: ./.github/actions/run-python-test-set
	with:
	build_type: ${{ matrix.build_type }}
	test_selection: performance
	run_in_parallel: false
	save_perf_report: ${{ github.ref_name == 'main' }}
	extra_params: --splits 5 --group ${{ matrix.pytest_split_group }}
	benchmark_durations: ${{ needs.get-benchmarks-durations.outputs.json }}
	env:
	VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
	PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
	TEST_RESULT_CONNSTR: "${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}"
	PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring
	PAGESERVER_GET_VECTORED_IMPL: vectored
	PAGESERVER_GET_IMPL: vectored
	PAGESERVER_VALIDATE_VEC_GET: false
	# XXX: no coverage data handling here, since benchmarks are run on release builds,
	# while coverage is currently collected for the debug ones

	report-benchmarks-failures:
	needs: [ benchmarks, create-test-report ]
	if: github.ref_name == 'main' && failure() && needs.benchmarks.result == 'failure'
	runs-on: ubuntu-22.04

	steps:
	- uses: slackapi/slack-github-action@v1
	with:
	channel-id: C060CNA47S9 # on-call-staging-storage-stream
	slack-message: \|
	Benchmarks failed on main: ${{ github.event.head_commit.url }}

	Allure report: ${{ needs.create-test-report.outputs.report-url }}
	env:
	SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

	create-test-report:
	needs: [ check-permissions, regress-tests, coverage-report, benchmarks, build-build-tools-image ]
	if: ${{ !cancelled() && contains(fromJSON('["skipped", "success"]'), needs.check-permissions.result) }}
	outputs:
	report-url: ${{ steps.create-allure-report.outputs.report-url }}

	runs-on: [ self-hosted, gen3, small ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	options: --init

	steps:
	- uses: actions/checkout@v4

	- name: Create Allure report
	if: ${{ !cancelled() }}
	id: create-allure-report
	uses: ./.github/actions/allure-report-generate
	with:
	store-test-results-into-db: true
	env:
	REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}

	- uses: actions/github-script@v7
	if: ${{ !cancelled() }}
	with:
	# Retry script for 5XX server errors: https://github.com/actions/github-script#retries
	retries: 5
	script: \|
	const report = {
	reportUrl: "${{ steps.create-allure-report.outputs.report-url }}",
	reportJsonUrl: "${{ steps.create-allure-report.outputs.report-json-url }}",
	}

	const coverage = {
	coverageUrl: "${{ needs.coverage-report.outputs.coverage-html }}",
	summaryJsonUrl: "${{ needs.coverage-report.outputs.coverage-json }}",
	}

	const script = require("./scripts/comment-test-report.js")
	await script({
	github,
	context,
	fetch,
	report,
	coverage,
	})

	coverage-report:
	needs: [ check-permissions, regress-tests, build-build-tools-image ]
	runs-on: [ self-hosted, gen3, small ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	options: --init
	strategy:
	fail-fast: false
	matrix:
	build_type: [ debug ]
	outputs:
	coverage-html: ${{ steps.upload-coverage-report-new.outputs.report-url }}
	coverage-json: ${{ steps.upload-coverage-report-new.outputs.summary-json }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 0

	- name: Get Neon artifact
	uses: ./.github/actions/download
	with:
	name: neon-${{ runner.os }}-${{ matrix.build_type }}-artifact
	path: /tmp/neon

	- name: Get coverage artifact
	uses: ./.github/actions/download
	with:
	name: coverage-data-artifact
	path: /tmp/coverage

	- name: Merge coverage data
	run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge

	- name: Build coverage report
	env:
	COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.event.pull_request.head.sha \|\| github.sha }}
	run: \|
	scripts/coverage --dir=/tmp/coverage \
	report \
	--input-objects=/tmp/coverage/binaries.list \
	--commit-url=${COMMIT_URL} \
	--format=github

	scripts/coverage --dir=/tmp/coverage \
	report \
	--input-objects=/tmp/coverage/binaries.list \
	--format=lcov

	- name: Build coverage report NEW
	id: upload-coverage-report-new
	env:
	BUCKET: neon-github-public-dev
	# A differential coverage report is available only for PRs.
	# (i.e. for pushes into main/release branches we have a regular coverage report)
	COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	BASE_SHA: ${{ github.event.pull_request.base.sha \|\| github.sha }}
	run: \|
	CURRENT="${COMMIT_SHA}"
	BASELINE="$(git merge-base $BASE_SHA $CURRENT)"

	cp /tmp/coverage/report/lcov.info ./${CURRENT}.info

	GENHTML_ARGS="--ignore-errors path,unmapped,empty --synthesize-missing --demangle-cpp rustfilt --output-directory lcov-html ${CURRENT}.info"

	# Use differential coverage if the baseline coverage exists.
	# It can be missing if the coverage repoer wasn't uploaded yet or tests has failed on BASELINE commit.
	if aws s3 cp --only-show-errors s3://${BUCKET}/code-coverage/${BASELINE}/lcov.info ./${BASELINE}.info; then
	git diff ${BASELINE} ${CURRENT} -- '*.rs' > baseline-current.diff

	GENHTML_ARGS="--baseline-file ${BASELINE}.info --diff-file baseline-current.diff ${GENHTML_ARGS}"
	fi

	genhtml ${GENHTML_ARGS}

	aws s3 cp --only-show-errors --recursive ./lcov-html/ s3://${BUCKET}/code-coverage/${COMMIT_SHA}/lcov

	REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/lcov/index.html
	echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT

	REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/lcov/summary.json
	echo "summary-json=${REPORT_URL}" >> $GITHUB_OUTPUT

	- uses: actions/github-script@v7
	env:
	REPORT_URL_NEW: ${{ steps.upload-coverage-report-new.outputs.report-url }}
	COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	with:
	script: \|
	const { REPORT_URL_NEW, COMMIT_SHA } = process.env

	await github.rest.repos.createCommitStatus({
	owner: context.repo.owner,
	repo: context.repo.repo,
	sha: `${COMMIT_SHA}`,
	state: 'success',
	target_url: `${REPORT_URL_NEW}`,
	context: 'Code coverage report NEW',
	})

	trigger-e2e-tests:
	if: ${{ !github.event.pull_request.draft \|\| contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') \|\| github.ref_name == 'main' \|\| github.ref_name == 'release' \|\| github.ref_name == 'release-proxy' }}
	needs: [ check-permissions, promote-images, tag ]
	uses: ./.github/workflows/trigger-e2e-tests.yml
	secrets: inherit

	neon-image-arch:
	needs: [ check-permissions, build-build-tools-image, tag ]
	strategy:
	matrix:
	arch: [ x64, arm64 ]

	runs-on: ${{ fromJson(format('["self-hosted", "gen3", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' \|\| 'large')) }}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 0

	# Use custom DOCKER_CONFIG directory to avoid conflicts with default settings
	# The default value is ~/.docker
	- name: Set custom docker config directory
	run: \|
	mkdir -p .docker-custom
	echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV
	- uses: docker/setup-buildx-action@v2

	- uses: docker/login-action@v3
	with:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

	- uses: docker/build-push-action@v5
	with:
	context: .
	build-args: \|
	GIT_VERSION=${{ github.event.pull_request.head.sha \|\| github.sha }}
	BUILD_TAG=${{ needs.tag.outputs.build-tag }}
	TAG=${{ needs.build-build-tools-image.outputs.image-tag }}
	provenance: false
	push: true
	pull: true
	file: Dockerfile
	cache-from: type=registry,ref=neondatabase/neon:cache-${{ matrix.arch }}
	cache-to: type=registry,ref=neondatabase/neon:cache-${{ matrix.arch }},mode=max
	tags: \|
	neondatabase/neon:${{ needs.tag.outputs.build-tag }}-${{ matrix.arch }}

	- name: Remove custom docker config directory
	if: always()
	run: \|
	rm -rf .docker-custom

	neon-image:
	needs: [ neon-image-arch, tag ]
	runs-on: ubuntu-22.04

	steps:
	- uses: docker/login-action@v3
	with:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

	- name: Create multi-arch image
	run: \|
	docker buildx imagetools create -t neondatabase/neon:${{ needs.tag.outputs.build-tag }} \
	neondatabase/neon:${{ needs.tag.outputs.build-tag }}-x64 \
	neondatabase/neon:${{ needs.tag.outputs.build-tag }}-arm64

	- uses: docker/login-action@v3
	with:
	registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com
	username: ${{ secrets.AWS_ACCESS_KEY_DEV }}
	password: ${{ secrets.AWS_SECRET_KEY_DEV }}

	- name: Push multi-arch image to ECR
	run: \|
	docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{ needs.tag.outputs.build-tag }} \
	neondatabase/neon:${{ needs.tag.outputs.build-tag }}

	compute-node-image-arch:
	needs: [ check-permissions, build-build-tools-image, tag ]
	strategy:
	fail-fast: false
	matrix:
	version: [ v14, v15, v16 ]
	arch: [ x64, arm64 ]

	runs-on: ${{ fromJson(format('["self-hosted", "gen3", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' \|\| 'large')) }}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 0

	# Use custom DOCKER_CONFIG directory to avoid conflicts with default settings
	# The default value is ~/.docker
	- name: Set custom docker config directory
	run: \|
	mkdir -p .docker-custom
	echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV
	- uses: docker/setup-buildx-action@v2
	with:
	# Disable parallelism for docker buildkit.
	# As we already build everything with `make -j$(nproc)`, running it in additional level of parallelisam blows up the Runner.
	config-inline: \|
	[worker.oci]
	max-parallelism = 1

	- uses: docker/login-action@v3
	with:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

	- uses: docker/login-action@v3
	with:
	registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com
	username: ${{ secrets.AWS_ACCESS_KEY_DEV }}
	password: ${{ secrets.AWS_SECRET_KEY_DEV }}

	- name: Build compute-node image
	uses: docker/build-push-action@v5
	with:
	context: .
	build-args: \|
	GIT_VERSION=${{ github.event.pull_request.head.sha \|\| github.sha }}
	PG_VERSION=${{ matrix.version }}
	BUILD_TAG=${{ needs.tag.outputs.build-tag }}
	TAG=${{ needs.build-build-tools-image.outputs.image-tag }}
	provenance: false
	push: true
	pull: true
	file: Dockerfile.compute-node
	cache-from: type=registry,ref=neondatabase/compute-node-${{ matrix.version }}:cache-${{ matrix.arch }}
	cache-to: type=registry,ref=neondatabase/compute-node-${{ matrix.version }}:cache-${{ matrix.arch }},mode=max
	tags: \|
	neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.arch }}

	- name: Build neon extensions test image
	if: matrix.version == 'v16'
	uses: docker/build-push-action@v5
	with:
	context: .
	build-args: \|
	GIT_VERSION=${{ github.event.pull_request.head.sha \|\| github.sha }}
	PG_VERSION=${{ matrix.version }}
	BUILD_TAG=${{ needs.tag.outputs.build-tag }}
	TAG=${{ needs.build-build-tools-image.outputs.image-tag }}
	provenance: false
	push: true
	pull: true
	file: Dockerfile.compute-node
	target: neon-pg-ext-test
	cache-from: type=registry,ref=neondatabase/neon-test-extensions-${{ matrix.version }}:cache-${{ matrix.arch }}
	cache-to: type=registry,ref=neondatabase/neon-test-extensions-${{ matrix.version }}:cache-${{ matrix.arch }},mode=max
	tags: \|
	neondatabase/neon-test-extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}-${{ matrix.arch }}

	- name: Build compute-tools image
	# compute-tools are Postgres independent, so build it only once
	if: matrix.version == 'v16'
	uses: docker/build-push-action@v5
	with:
	target: compute-tools-image
	context: .
	build-args: \|
	GIT_VERSION=${{ github.event.pull_request.head.sha \|\| github.sha }}
	BUILD_TAG=${{ needs.tag.outputs.build-tag }}
	TAG=${{ needs.build-build-tools-image.outputs.image-tag }}
	provenance: false
	push: true
	pull: true
	file: Dockerfile.compute-node
	tags: \|
	neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.arch }}

	- name: Remove custom docker config directory
	if: always()
	run: \|
	rm -rf .docker-custom

	compute-node-image:
	needs: [ compute-node-image-arch, tag ]
	runs-on: ubuntu-22.04

	strategy:
	matrix:
	version: [ v14, v15, v16 ]

	steps:
	- uses: docker/login-action@v3
	with:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

	- name: Create multi-arch compute-node image
	run: \|
	docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} \
	neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-x64 \
	neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-arm64

	- name: Create multi-arch neon-test-extensions image
	if: matrix.version == 'v16'
	run: \|
	docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} \
	neondatabase/neon-test-extensions-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-x64 \
	neondatabase/neon-test-extensions-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-arm64

	- name: Create multi-arch compute-tools image
	if: matrix.version == 'v16'
	run: \|
	docker buildx imagetools create -t neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }} \
	neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-x64 \
	neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-arm64

	- uses: docker/login-action@v3
	with:
	registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com
	username: ${{ secrets.AWS_ACCESS_KEY_DEV }}
	password: ${{ secrets.AWS_SECRET_KEY_DEV }}

	- name: Push multi-arch compute-node-${{ matrix.version }} image to ECR
	run: \|
	docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} \
	neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}

	- name: Push multi-arch compute-tools image to ECR
	if: matrix.version == 'v16'
	run: \|
	docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{ needs.tag.outputs.build-tag }} \
	neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}

	vm-compute-node-image:
	needs: [ check-permissions, tag, compute-node-image ]
	runs-on: [ self-hosted, gen3, large ]
	strategy:
	fail-fast: false
	matrix:
	version: [ v14, v15, v16 ]
	env:
	VM_BUILDER_VERSION: v0.29.3

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Downloading vm-builder
	run: \|
	curl -fL https://github.com/neondatabase/autoscaling/releases/download/$VM_BUILDER_VERSION/vm-builder -o vm-builder
	chmod +x vm-builder

	# Use custom DOCKER_CONFIG directory to avoid conflicts with default settings
	# The default value is ~/.docker
	- name: Set custom docker config directory
	run: \|
	mkdir -p .docker-custom
	echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV

	- uses: docker/login-action@v3
	with:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

	# Note: we need a separate pull step here because otherwise vm-builder will try to pull, and
	# it won't have the proper authentication (written at v0.6.0)
	- name: Pulling compute-node image
	run: \|
	docker pull neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}

	- name: Build vm image
	run: \|
	./vm-builder \
	-spec=vm-image-spec.yaml \
	-src=neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} \
	-dst=neondatabase/vm-compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}

	- name: Pushing vm-compute-node image
	run: \|
	docker push neondatabase/vm-compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}

	- name: Remove custom docker config directory
	if: always()
	run: \|
	rm -rf .docker-custom

	test-images:
	needs: [ check-permissions, tag, neon-image, compute-node-image ]
	strategy:
	fail-fast: false
	matrix:
	arch: [ x64, arm64 ]

	runs-on: ${{ fromJson(format('["self-hosted", "gen3", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' \|\| 'small')) }}

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	# `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
	# Pick pageserver as currently the only binary with extra "version" features printed in the string to verify.
	# Regular pageserver version string looks like
	# Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c failpoints: true, features: []
	# Bad versions might loop like:
	# Neon page server git-env:local failpoints: true, features: ["testing"]
	# Ensure that we don't have bad versions.
	- name: Verify image versions
	shell: bash # ensure no set -e for better error messages
	run: \|
	pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.tag.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")

	echo "Pageserver version string: $pageserver_version"

	if ! echo "$pageserver_version" \| grep -qv 'git-env:local' ; then
	echo "Pageserver version should not be the default Dockerfile one"
	exit 1
	fi

	if ! echo "$pageserver_version" \| grep -qv '"testing"' ; then
	echo "Pageserver version should have no testing feature enabled"
	exit 1
	fi

	- name: Verify docker-compose example and test extensions
	timeout-minutes: 20
	run: env TAG=${{needs.tag.outputs.build-tag}} ./docker-compose/docker_compose_test.sh

	- name: Print logs and clean up
	if: always()
	run: \|
	docker compose -f ./docker-compose/docker-compose.yml logs \|\| 0
	docker compose -f ./docker-compose/docker-compose.yml down

	promote-images:
	needs: [ check-permissions, tag, test-images, vm-compute-node-image ]
	runs-on: ubuntu-22.04

	env:
	VERSIONS: v14 v15 v16

	steps:
	- uses: docker/login-action@v3
	with:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

	- name: Login to dev ECR
	uses: docker/login-action@v3
	with:
	registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com
	username: ${{ secrets.AWS_ACCESS_KEY_DEV }}
	password: ${{ secrets.AWS_SECRET_KEY_DEV }}

	- name: Copy vm-compute-node images to ECR
	run: \|
	for version in ${VERSIONS}; do
	docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} \
	neondatabase/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }}
	done

	- name: Add latest tag to images
	if: github.ref_name == 'main'
	run: \|
	for repo in neondatabase 369495373322.dkr.ecr.eu-central-1.amazonaws.com; do
	docker buildx imagetools create -t $repo/neon:latest \
	$repo/neon:${{ needs.tag.outputs.build-tag }}

	docker buildx imagetools create -t $repo/compute-tools:latest \
	$repo/compute-tools:${{ needs.tag.outputs.build-tag }}

	for version in ${VERSIONS}; do
	docker buildx imagetools create -t $repo/compute-node-${version}:latest \
	$repo/compute-node-${version}:${{ needs.tag.outputs.build-tag }}

	docker buildx imagetools create -t $repo/vm-compute-node-${version}:latest \
	$repo/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }}
	done
	done
	docker buildx imagetools create -t neondatabase/neon-test-extensions-v16:latest \
	neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }}

	- name: Login to prod ECR
	uses: docker/login-action@v3
	if: github.ref_name == 'release'\|\| github.ref_name == 'release-proxy'
	with:
	registry: 093970136003.dkr.ecr.eu-central-1.amazonaws.com
	username: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_ACCESS_KEY_ID }}
	password: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_SECRET_ACCESS_KEY }}

	- name: Copy all images to prod ECR
	if: github.ref_name == 'release'\|\| github.ref_name == 'release-proxy'
	run: \|
	for image in neon compute-tools {vm-,}compute-node-{v14,v15,v16}; do
	docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} \
	369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }}
	done

	trigger-custom-extensions-build-and-wait:
	needs: [ check-permissions, tag ]
	runs-on: ubuntu-22.04
	steps:
	- name: Set PR's status to pending and request a remote CI test
	run: \|
	COMMIT_SHA=${{ github.event.pull_request.head.sha \|\| github.sha }}
	REMOTE_REPO="${{ github.repository_owner }}/build-custom-extensions"

	curl -f -X POST \
	https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \
	-H "Accept: application/vnd.github.v3+json" \
	--user "${{ secrets.CI_ACCESS_TOKEN }}" \
	--data \
	"{
	\"state\": \"pending\",
	\"context\": \"build-and-upload-extensions\",
	\"description\": \"[$REMOTE_REPO] Remote CI job is about to start\"
	}"

	curl -f -X POST \
	https://api.github.com/repos/$REMOTE_REPO/actions/workflows/build_and_upload_extensions.yml/dispatches \
	-H "Accept: application/vnd.github.v3+json" \
	--user "${{ secrets.CI_ACCESS_TOKEN }}" \
	--data \
	"{
	\"ref\": \"main\",
	\"inputs\": {
	\"ci_job_name\": \"build-and-upload-extensions\",
	\"commit_hash\": \"$COMMIT_SHA\",
	\"remote_repo\": \"${{ github.repository }}\",
	\"compute_image_tag\": \"${{ needs.tag.outputs.build-tag }}\",
	\"remote_branch_name\": \"${{ github.ref_name }}\"
	}
	}"

	- name: Wait for extension build to finish
	env:
	GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
	run: \|
	TIMEOUT=1800 # 30 minutes, usually it takes ~2-3 minutes, but if runners are busy, it might take longer
	INTERVAL=15 # try each N seconds

	last_status="" # a variable to carry the last status of the "build-and-upload-extensions" context

	for ((i=0; i <= TIMEOUT; i+=INTERVAL)); do
	sleep $INTERVAL

	# Get statuses for the latest commit in the PR / branch
	gh api \
	-H "Accept: application/vnd.github+json" \
	-H "X-GitHub-Api-Version: 2022-11-28" \
	"/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha \|\| github.sha }}" > statuses.json

	# Get the latest status for the "build-and-upload-extensions" context
	last_status=$(jq --raw-output '[.[] \| select(.context == "build-and-upload-extensions")] \| sort_by(.created_at)[-1].state' statuses.json)
	if [ "${last_status}" = "pending" ]; then
	# Extension build is still in progress.
	continue
	elif [ "${last_status}" = "success" ]; then
	# Extension build is successful.
	exit 0
	else
	# Status is neither "pending" nor "success", exit the loop and fail the job.
	break
	fi
	done

	# Extension build failed, print `statuses.json` for debugging and fail the job.
	jq '.' statuses.json

	echo >&2 "Status of extension build is '${last_status}' != 'success'"
	exit 1

	deploy:
	needs: [ check-permissions, promote-images, tag, regress-tests, trigger-custom-extensions-build-and-wait ]
	if: github.ref_name == 'main' \|\| github.ref_name == 'release'\|\| github.ref_name == 'release-proxy'

	runs-on: [ self-hosted, gen3, small ]
	container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
	steps:
	- name: Fix git ownership
	run: \|
	# Workaround for `fatal: detected dubious ownership in repository at ...`
	#
	# Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers
	# Ref https://github.com/actions/checkout/issues/785
	#
	git config --global --add safe.directory ${{ github.workspace }}
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	for r in 14 15 16; do
	git config --global --add safe.directory "${{ github.workspace }}/vendor/postgres-v$r"
	git config --global --add safe.directory "${GITHUB_WORKSPACE}/vendor/postgres-v$r"
	done

	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: false
	fetch-depth: 0

	- name: Trigger deploy workflow
	env:
	GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
	run: \|
	if [[ "$GITHUB_REF_NAME" == "main" ]]; then
	gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main -f branch=main -f dockerTag=${{needs.tag.outputs.build-tag}} -f deployPreprodRegion=false
	elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
	gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main \
	-f deployPgSniRouter=false \
	-f deployProxy=false \
	-f deployStorage=true \
	-f deployStorageBroker=true \
	-f deployStorageController=true \
	-f branch=main \
	-f dockerTag=${{needs.tag.outputs.build-tag}} \
	-f deployPreprodRegion=true

	gh workflow --repo neondatabase/aws run deploy-prod.yml --ref main \
	-f deployStorage=true \
	-f deployStorageBroker=true \
	-f deployStorageController=true \
	-f branch=main \
	-f dockerTag=${{needs.tag.outputs.build-tag}}
	elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then
	gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main \
	-f deployPgSniRouter=true \
	-f deployProxy=true \
	-f deployStorage=false \
	-f deployStorageBroker=false \
	-f deployStorageController=false \
	-f branch=main \
	-f dockerTag=${{needs.tag.outputs.build-tag}} \
	-f deployPreprodRegion=true

	gh workflow --repo neondatabase/aws run deploy-proxy-prod.yml --ref main \
	-f deployPgSniRouter=true \
	-f deployProxy=true \
	-f branch=main \
	-f dockerTag=${{needs.tag.outputs.build-tag}}
	else
	echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
	exit 1
	fi

	- name: Create git tag
	if: github.ref_name == 'release' \|\| github.ref_name == 'release-proxy'
	uses: actions/github-script@v7
	with:
	# Retry script for 5XX server errors: https://github.com/actions/github-script#retries
	retries: 5
	script: \|
	await github.rest.git.createRef({
	owner: context.repo.owner,
	repo: context.repo.repo,
	ref: "refs/tags/${{ needs.tag.outputs.build-tag }}",
	sha: context.sha,
	})

	# TODO: check how GitHub releases looks for proxy releases and enable it if it's ok
	- name: Create GitHub release
	if: github.ref_name == 'release'
	uses: actions/github-script@v7
	with:
	# Retry script for 5XX server errors: https://github.com/actions/github-script#retries
	retries: 5
	script: \|
	await github.rest.repos.createRelease({
	owner: context.repo.owner,
	repo: context.repo.repo,
	tag_name: "${{ needs.tag.outputs.build-tag }}",
	generate_release_notes: true,
	})

	promote-compatibility-data:
	needs: [ check-permissions, promote-images, tag, regress-tests ]
	if: github.ref_name == 'release'

	runs-on: [ self-hosted, gen3, small ]
	container:
	image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
	options: --init
	steps:
	- name: Promote compatibility snapshot for the release
	env:
	BUCKET: neon-github-public-dev
	PREFIX: artifacts/latest
	run: \|
	# Update compatibility snapshot for the release
	for pg_version in v14 v15 v16; do
	for build_type in debug release; do
	OLD_FILENAME=compatibility-snapshot-${build_type}-pg${pg_version}-${GITHUB_RUN_ID}.tar.zst
	NEW_FILENAME=compatibility-snapshot-${build_type}-pg${pg_version}.tar.zst

	time aws s3 mv --only-show-errors s3://${BUCKET}/${PREFIX}/${OLD_FILENAME} s3://${BUCKET}/${PREFIX}/${NEW_FILENAME}
	done
	done

	# Update Neon artifact for the release (reuse already uploaded artifact)
	for build_type in debug release; do
	OLD_PREFIX=artifacts/${GITHUB_RUN_ID}
	FILENAME=neon-${{ runner.os }}-${build_type}-artifact.tar.zst

	S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${OLD_PREFIX} \| jq -r '.Contents[]?.Key' \| grep ${FILENAME} \| sort --version-sort \| tail -1 \|\| true)
	if [ -z "${S3_KEY}" ]; then
	echo >&2 "Neither s3://${BUCKET}/${OLD_PREFIX}/${FILENAME} nor its version from previous attempts exist"
	exit 1
	fi

	time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/${PREFIX}/${FILENAME}
	done

	pin-build-tools-image:
	needs: [ build-build-tools-image, promote-images, regress-tests ]
	if: github.ref_name == 'main'
	uses: ./.github/workflows/pin-build-tools-image.yml
	with:
	from-tag: ${{ needs.build-build-tools-image.outputs.image-tag }}
	secrets: inherit

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Allow longer timeout for starting pageserver, safe keeper and storage controller in test cases to make test cases less flaky #22768

Workflow file

Allow longer timeout for starting pageserver, safe keeper and storage controller in test cases to make test cases less flaky #22768

Jobs

Run details

Workflow file for this run