From cfb141fc7bcee52e5fbcc85768510069e9124995 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Thu, 20 Apr 2023 20:07:02 +0200 Subject: [PATCH 001/106] New reusable workflow for linting --- .github/workflows/reusable_lint.yml | 239 ++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 .github/workflows/reusable_lint.yml diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml new file mode 100644 index 000000000000..39048eb3b848 --- /dev/null +++ b/.github/workflows/reusable_lint.yml @@ -0,0 +1,239 @@ +name: Reusable Lint + +on: + workflow_call: + secrets: + CALLER_GITHUB_TOKEN: + required: true + inputs: + SAVE_CACHE: + required: false + type: boolean + default: false +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + +jobs: + py-lints: + name: Python lints (black, mypy, flake8) + runs-on: ubuntu-latest + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + - uses: actions/checkout@v3 + - uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.CALLER_GITHUB_TOKEN }} + with: + just-version: 1.5 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "rerun_py/requirements-lint.txt" + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install -r rerun_py/requirements-lint.txt + + - name: Lint Python + run: | + just py-lint + + - name: Check requirements + run: | + just py-requirements + + rs-lints: + name: Rust lints (fmt, check, cranky, tests, doc) + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.6 + env: + RUSTFLAGS: ${{env.RUSTFLAGS}} + RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Don't do a shallow clone since we need to push gh-pages + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # See: https://github.com/rerun-io/rerun/pull/497 + save-if: ${{ inputs.SAVE_CACHE }} + + # First do our check with --locked to make sure `Cargo.lock` is up to date + - name: Check all features + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked --all-features + + - name: Rustfmt + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + - name: Cranky + uses: actions-rs/cargo@v1 + with: + command: cranky + args: --all-targets --all-features -- --deny warnings + + # -------------------------------------------------------------------------------- + # Check a few important permutations of the feature flags for our `rerun` library: + - name: Check rerun with `--no-default-features`` + uses: actions-rs/cargo@v1 + with: + command: cranky + args: --locked -p rerun --no-default-features + + - name: Check rerun with `--features sdk` + uses: actions-rs/cargo@v1 + with: + command: cranky + args: --locked -p rerun --no-default-features --features sdk + # -------------------------------------------------------------------------------- + + - name: Test doc-tests + uses: actions-rs/cargo@v1 + with: + command: test + args: --doc --all-features + + - name: cargo doc + uses: actions-rs/cargo@v1 + with: + command: doc + args: --no-deps --all-features + + - name: cargo doc --document-private-items + uses: actions-rs/cargo@v1 + with: + command: doc + args: --document-private-items --no-deps --all-features + + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-targets --all-features + + rs-cargo-deny: + name: Cargo Deny ${{ matrix.platform }} + runs-on: ubuntu-latest + container: + image: rerunio/ci_docker:0.6 + env: + RUSTFLAGS: ${{env.RUSTFLAGS}} + RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} + + # TODO(emilk): remove this matrix when https://github.com/EmbarkStudios/cargo-deny/issues/324 is resolved + strategy: + fail-fast: false + matrix: + platform: + - i686-pc-windows-gnu + - i686-pc-windows-msvc + - i686-unknown-linux-gnu + - wasm32-unknown-unknown + - x86_64-apple-darwin + - x86_64-pc-windows-gnu + - x86_64-pc-windows-msvc + - x86_64-unknown-linux-gnu + - x86_64-unknown-redox + + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + - uses: actions/checkout@v3 + - uses: EmbarkStudios/cargo-deny-action@v1 + with: + command: check + log-level: error + arguments: --all-features --target ${{ matrix.platform }} + + toml-lints: + name: Lint TOML files + runs-on: ubuntu-latest + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.67.1 + override: true + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + # See: https://github.com/rerun-io/rerun/pull/497 + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: false + + - name: Install taplo-cli + uses: baptiste0928/cargo-install@v1 + with: + crate: taplo-cli + + - name: Taplo check + run: | + taplo fmt --check From 817419c42df1b1d51db69aaadd158f0d968be822 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Thu, 20 Apr 2023 22:08:48 +0200 Subject: [PATCH 002/106] Reusable workflow to build/test wheels --- .github/workflows/reusable_build_and_test.yml | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 .github/workflows/reusable_build_and_test.yml diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml new file mode 100644 index 000000000000..8f1c240ac04a --- /dev/null +++ b/.github/workflows/reusable_build_and_test.yml @@ -0,0 +1,231 @@ +name: Reusable Build and Test + +on: + workflow_call: + secrets: + CALLER_GITHUB_TOKEN: + required: true + inputs: + SAVE_CACHE: + required: false + type: boolean + default: false + PLATFORM: + required: true + type: string + PRERELEASE: + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + +jobs: + set_config: + name: Set Config + runs-on: ubuntu-latest + outputs: + RUNNER: ${{ steps.set_config.outputs.runner }} + TARGET: ${{ steps.set_config.outputs.target }} + RUN_TESTS: ${{ steps.set_config.outputs.run_tests }} + CONTAINER: ${{ steps.set_config.outputs.container }} + steps: + - name: Set runner and target based on platform + id: set_config + run: | + case "${{ inputs.platform }}" in + linux) + runner="ubuntu-latest" + target="x86_64-unknown-linux-gnu" + run_tests="true" + container="{'image': 'rerunio/ci_docker:0.6'}" + ;; + windows) + runner="windows-latest" + target="x86_64-pc-windows-msvc" + run_tests="true" + container="null" + ;; + macos-arm) + runner="macos-latest" + target="aarch64-apple-darwin" + run_tests="false" + container="null" + ;; + macos-intel) + runner="macos-latest" + target="x86_64-apple-darwin" + run_tests="false" + container="null" + ;; + *) echo "Invalid platform" && exit 1 ;; + esac + echo "::set-output name=runner::$runner" + echo "::set-output name=target::$target" + echo "::set-output name=run_tests::$run_tests" + echo "::set-output name=container::$container" + + build_wheels: + name: Build Wheels + + needs: [set_config] + + runs-on: ${{ needs.set_config.outputs.RUNNER }} + container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} + + outputs: + expected_version: ${{ steps.expected_version.outputs.expected_version }} + + steps: + - uses: actions/checkout@v3 + + # The pip-cache setup logic doesn't work in the ubuntu docker container + # That's probably fine since we bake these deps into the container already + - name: Setup python + if: ${{ inputs.PLATFORM != 'linux' }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" + + # These should already be in the docker container, but run for good measure. A no-op install + # should be fast, and this way things don't break if we add new packages without rebuilding + # docker + - run: pip install -r rerun_py/requirements-build.txt + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: ${{ inputs.SAVE_CACHE }} + + # We have a nice script for that: ./scripts/setup_web.sh + # Unfortunately, we can't run that on Windows, because Windows doesn't come with + # a package manager like grown-up OSes do (at least not the CI version of Windows). + # Also we can't run it on linux because the 20.04 Docker container will install + # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag + # So we only run the script on macos, and then on Windows we do the parts of the script manually. + # On ubuntu, the correct packages are pre-installed in our docker container. + + - name: Install prerequisites for building the web-viewer Wasm (non-Windows) + if: (inputs.PLATFORM == 'macos-intel') || (inputs.PLATFORM == 'macos-arm') + shell: bash + run: ./scripts/setup_web.sh + + # The first steps of setup_web.sh, for Windows: + - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows + if: inputs.platform == 'windows' + shell: bash + run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84 + + # The last step of setup_web.sh, for Windows. + # Since 'winget' is not available within the GitHub runner, we download the package directly: + # See: https://github.com/marketplace/actions/engineerd-configurator + - name: Install binaryen for building the web-viewer Wasm on windows + if: inputs.platform == 'windows' + uses: engineerd/configurator@v0.0.9 + with: + name: "wasm-opt.exe" + url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz" + pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" + + - name: Patch Cargo.toml for pre-release + if: inputs.PRERELEASE + # After patching the pre-release version, run cargo update. + # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing + run: | + python3 scripts/version_util.py --patch_prerelease + cargo update -w + + # TODO(jleibs): Figure out what this check should be doing now + - name: Version check for tagged-release + if: false + #if: startsWith(github.ref, 'refs/tags/v') + # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z + run: | + python3 scripts/version_util.py --check_version + + - name: Store the expected version + # Find the current cargo version and store it in the GITHUB_ENV var: `expected_version` + shell: bash + id: expected_version + run: | + echo "::set-output expected_version=$(python3 scripts/version_util.py --bare_cargo_version)" + + - name: Build Wheel + uses: PyO3/maturin-action@v1 + with: + maturin-version: "0.14.10" + manylinux: manylinux_2_31 + container: off + command: build + args: | + --manifest-path rerun_py/Cargo.toml + --release + --target ${{ needs.set_config.outputs.TARGET }} + --no-default-features + --features pypi + --out pre-dist + + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: ${{inputs.PLATFORM}}-wheels + path: dist + + run_tests: + name: Test Wheels + + needs: [set_config, build_wheels] + + runs-on: ${{ needs.set_config.outputs.RUNNER }} + container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} + + # Note string instead of boolean since it's a job output + if: needs.set_config.outputs.RUN_TESTS == 'true' + + steps: + - name: Install wheel dependencies + # First we install the dependencies manually so we can use `--no-index` when installing the wheel. + # This needs to be a separate step for some reason or the following step fails + # TODO(jleibs): pull these deps from pyproject.toml + # TODO(jleibs): understand why deps can't be installed in the same step as the wheel + shell: bash + run: | + pip install deprecated numpy>=1.23 pyarrow==10.0.1 + + - name: Install built wheel + # Now install the wheel using a specific version and --no-index to guarantee we get the version from + # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't + # find the dependencies to reinstall them. + shell: bash + run: | + pip uninstall rerun-sdk + pip install rerun-sdk==${{ needs.build_wheels.outputs.expected_version }} --no-index --find-links pre-dist + + - name: Verify built wheel version + shell: bash + run: | + python3 -m rerun --version + which rerun + rerun --version + + - name: Run unit tests + shell: bash + run: cd rerun_py/tests && pytest + + - name: Run e2e test + shell: bash + run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed From e0a65d3818a6fd3bf09cd60b1156ae7eb377d3f2 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 00:12:48 +0200 Subject: [PATCH 003/106] Get rid of windows error with an extra gate --- crates/re_log_types/src/component_types/arrow_convert_shims.rs | 1 + crates/re_log_types/src/component_types/mod.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/re_log_types/src/component_types/arrow_convert_shims.rs b/crates/re_log_types/src/component_types/arrow_convert_shims.rs index 8d196842169d..b78ff5f4c065 100644 --- a/crates/re_log_types/src/component_types/arrow_convert_shims.rs +++ b/crates/re_log_types/src/component_types/arrow_convert_shims.rs @@ -79,6 +79,7 @@ impl<'a> Iterator for BufferBinaryArrayIter<'a> { /// Internal `ArrowArray` helper to iterate over a `BinaryArray` while exposing Buffer slices pub struct BufferBinaryArray; +#[cfg(not(target_os = "windows"))] extern "C" { fn do_not_call_into_iter(); // we never define this function, so the linker will fail } diff --git a/crates/re_log_types/src/component_types/mod.rs b/crates/re_log_types/src/component_types/mod.rs index 61066b8c8f58..a65650364660 100644 --- a/crates/re_log_types/src/component_types/mod.rs +++ b/crates/re_log_types/src/component_types/mod.rs @@ -210,6 +210,7 @@ where pub struct FastFixedSizeListArray(std::marker::PhantomData); +#[cfg(not(target_os = "windows"))] extern "C" { fn do_not_call_into_iter(); // we never define this function, so the linker will fail } From 9a9a3fb3c85fc198d4fc5dc0cd18ac0f548cce5b Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 00:15:12 +0200 Subject: [PATCH 004/106] Upload/download artifact --- .github/workflows/reusable_build_and_test.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 8f1c240ac04a..b0b8bec3b248 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -177,7 +177,7 @@ jobs: --target ${{ needs.set_config.outputs.TARGET }} --no-default-features --features pypi - --out pre-dist + --out dist - name: Upload wheels uses: actions/upload-artifact@v3 @@ -197,6 +197,12 @@ jobs: if: needs.set_config.outputs.RUN_TESTS == 'true' steps: + - name: Download Artifact + uses: actions/download-artifact@v3 + with: + name: ${{inputs.PLATFORM}}-wheels + path: dist + - name: Install wheel dependencies # First we install the dependencies manually so we can use `--no-index` when installing the wheel. # This needs to be a separate step for some reason or the following step fails @@ -213,7 +219,7 @@ jobs: shell: bash run: | pip uninstall rerun-sdk - pip install rerun-sdk==${{ needs.build_wheels.outputs.expected_version }} --no-index --find-links pre-dist + pip install rerun-sdk==${{ needs.build_wheels.outputs.expected_version }} --no-index --find-links dist - name: Verify built wheel version shell: bash From f5e9a007e891653b63832672faa921061662022c Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 00:20:39 +0200 Subject: [PATCH 005/106] Remove deprecated ::set-output --- .github/workflows/reusable_build_and_test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index b0b8bec3b248..3d00b5dccf9a 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -68,10 +68,10 @@ jobs: ;; *) echo "Invalid platform" && exit 1 ;; esac - echo "::set-output name=runner::$runner" - echo "::set-output name=target::$target" - echo "::set-output name=run_tests::$run_tests" - echo "::set-output name=container::$container" + echo "name=runner::$runner" >> $GITHUB_OUTPUT + echo "name=target::$target" >> $GITHUB_OUTPUT + echo "name=run_tests::$run_tests" >> $GITHUB_OUTPUT + echo "name=container::$container" >> $GITHUB_OUTPUT build_wheels: name: Build Wheels @@ -158,11 +158,11 @@ jobs: python3 scripts/version_util.py --check_version - name: Store the expected version - # Find the current cargo version and store it in the GITHUB_ENV var: `expected_version` + # Find the current cargo version and store it in the output: `expected_version` shell: bash id: expected_version run: | - echo "::set-output expected_version=$(python3 scripts/version_util.py --bare_cargo_version)" + echo "expected_version=$(python3 scripts/version_util.py --bare_cargo_version)" >> $GITHUB_OUTPUT - name: Build Wheel uses: PyO3/maturin-action@v1 From 35f269bc99f7e200fc73a5e6d8fa694288808c90 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 00:42:02 +0200 Subject: [PATCH 006/106] more linting --- .github/workflows/reusable_lint.yml | 137 +++++++++++++++------------- 1 file changed, 73 insertions(+), 64 deletions(-) diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index 39048eb3b848..a065b50ec91e 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -21,21 +21,13 @@ env: RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs jobs: + +################################################################################ + py-lints: name: Python lints (black, mypy, flake8) runs-on: ubuntu-latest steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - uses: actions/checkout@v3 - uses: extractions/setup-just@v1 env: @@ -63,6 +55,8 @@ jobs: run: | just py-requirements +################################################################################ + rs-lints: name: Rust lints (fmt, check, cranky, tests, doc) runs-on: ubuntu-latest-16-cores @@ -152,64 +146,12 @@ jobs: command: test args: --all-targets --all-features - rs-cargo-deny: - name: Cargo Deny ${{ matrix.platform }} - runs-on: ubuntu-latest - container: - image: rerunio/ci_docker:0.6 - env: - RUSTFLAGS: ${{env.RUSTFLAGS}} - RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} - - # TODO(emilk): remove this matrix when https://github.com/EmbarkStudios/cargo-deny/issues/324 is resolved - strategy: - fail-fast: false - matrix: - platform: - - i686-pc-windows-gnu - - i686-pc-windows-msvc - - i686-unknown-linux-gnu - - wasm32-unknown-unknown - - x86_64-apple-darwin - - x86_64-pc-windows-gnu - - x86_64-pc-windows-msvc - - x86_64-unknown-linux-gnu - - x86_64-unknown-redox - - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - - uses: actions/checkout@v3 - - uses: EmbarkStudios/cargo-deny-action@v1 - with: - command: check - log-level: error - arguments: --all-features --target ${{ matrix.platform }} +################################################################################ toml-lints: name: Lint TOML files runs-on: ubuntu-latest steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - uses: actions/checkout@v3 @@ -237,3 +179,70 @@ jobs: - name: Taplo check run: | taplo fmt --check + +################################################################################ + + misc-rerun-lints: + name: Rerun lints + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - name: Rerun lints + run: | + ./scripts/lint.py + + - name: Check for too large files + run: | + ./scripts/check_large_files.sh + +################################################################################ + + spell-check: + name: Spell Check + runs-on: ubuntu-latest + steps: + - name: Checkout Actions Repository + uses: actions/checkout@v3 + + - name: Check spelling of entire workspace + uses: crate-ci/typos@master + +################################################################################ + + rs-cargo-deny: + name: Cargo Deny ${{ matrix.platform }} + runs-on: ubuntu-latest + container: + image: rerunio/ci_docker:0.6 + env: + RUSTFLAGS: ${{env.RUSTFLAGS}} + RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} + + # TODO(emilk): remove this matrix when https://github.com/EmbarkStudios/cargo-deny/issues/324 is resolved + strategy: + fail-fast: false + matrix: + platform: + - i686-pc-windows-gnu + - i686-pc-windows-msvc + - i686-unknown-linux-gnu + - wasm32-unknown-unknown + - x86_64-apple-darwin + - x86_64-pc-windows-gnu + - x86_64-pc-windows-msvc + - x86_64-unknown-linux-gnu + - x86_64-unknown-redox + + steps: + - uses: actions/checkout@v3 + - uses: EmbarkStudios/cargo-deny-action@v1 + with: + command: check + log-level: error + arguments: --all-features --target ${{ matrix.platform }} From 8bca6ed4614b4493e83e3c04adad951104b26614 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 00:49:12 +0200 Subject: [PATCH 007/106] Trying to debug EXPECTED_VERSION --- .github/workflows/reusable_build_and_test.yml | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 3d00b5dccf9a..870f49af2140 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -29,6 +29,9 @@ env: RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs jobs: + +################################################################################ + set_config: name: Set Config runs-on: ubuntu-latest @@ -73,6 +76,8 @@ jobs: echo "name=run_tests::$run_tests" >> $GITHUB_OUTPUT echo "name=container::$container" >> $GITHUB_OUTPUT +################################################################################ + build_wheels: name: Build Wheels @@ -82,7 +87,7 @@ jobs: container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} outputs: - expected_version: ${{ steps.expected_version.outputs.expected_version }} + EXPECTED_VERSION: ${{ steps.expected_version.outputs.expected_version }} steps: - uses: actions/checkout@v3 @@ -185,6 +190,8 @@ jobs: name: ${{inputs.PLATFORM}}-wheels path: dist +################################################################################ + run_tests: name: Test Wheels @@ -197,6 +204,20 @@ jobs: if: needs.set_config.outputs.RUN_TESTS == 'true' steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + echo "EXPECTED_VERSION": $EXPECTED_VERSION + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + EXPECTED_VERSION: ${{ needs.build_wheels.outputs.EXPECTED_VERSION }} + - name: Download Artifact uses: actions/download-artifact@v3 with: @@ -219,7 +240,7 @@ jobs: shell: bash run: | pip uninstall rerun-sdk - pip install rerun-sdk==${{ needs.build_wheels.outputs.expected_version }} --no-index --find-links dist + pip install rerun-sdk==${{ needs.build_wheels.outputs.EXPECTED_VERSION }} --no-index --find-links dist - name: Verify built wheel version shell: bash From 686833127a44d3379910f50bae5d5efe5a492532 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 01:00:10 +0200 Subject: [PATCH 008/106] Fixing github output --- .github/workflows/reusable_build_and_test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 870f49af2140..b4d1a380b26d 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -71,10 +71,10 @@ jobs: ;; *) echo "Invalid platform" && exit 1 ;; esac - echo "name=runner::$runner" >> $GITHUB_OUTPUT - echo "name=target::$target" >> $GITHUB_OUTPUT - echo "name=run_tests::$run_tests" >> $GITHUB_OUTPUT - echo "name=container::$container" >> $GITHUB_OUTPUT + echo "runner=$runner" >> "$GITHUB_OUTPUT" + echo "target=$target" >> "$GITHUB_OUTPUT" + echo "run_tests=$run_tests" >> "$GITHUB_OUTPUT" + echo "container=$container" >> "$GITHUB_OUTPUT" ################################################################################ @@ -167,7 +167,7 @@ jobs: shell: bash id: expected_version run: | - echo "expected_version=$(python3 scripts/version_util.py --bare_cargo_version)" >> $GITHUB_OUTPUT + echo "expected_version=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" - name: Build Wheel uses: PyO3/maturin-action@v1 From 55acf6da2794d8027f0a217540a69ef8134e7c56 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 01:09:22 +0200 Subject: [PATCH 009/106] Replace cargo-deny matrix with a script --- .github/workflows/reusable_lint.yml | 25 +++++-------------------- scripts/cargo_deny.sh | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 20 deletions(-) create mode 100755 scripts/cargo_deny.sh diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index a065b50ec91e..526da7ca814a 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -224,25 +224,10 @@ jobs: RUSTFLAGS: ${{env.RUSTFLAGS}} RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} - # TODO(emilk): remove this matrix when https://github.com/EmbarkStudios/cargo-deny/issues/324 is resolved - strategy: - fail-fast: false - matrix: - platform: - - i686-pc-windows-gnu - - i686-pc-windows-msvc - - i686-unknown-linux-gnu - - wasm32-unknown-unknown - - x86_64-apple-darwin - - x86_64-pc-windows-gnu - - x86_64-pc-windows-msvc - - x86_64-unknown-linux-gnu - - x86_64-unknown-redox - steps: - uses: actions/checkout@v3 - - uses: EmbarkStudios/cargo-deny-action@v1 - with: - command: check - log-level: error - arguments: --all-features --target ${{ matrix.platform }} + + - name: Cargo Deny + shell: bash + id: expected_version + run: ./scripts/cargo_deny.sh diff --git a/scripts/cargo_deny.sh b/scripts/cargo_deny.sh new file mode 100755 index 000000000000..41b65fcda881 --- /dev/null +++ b/scripts/cargo_deny.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -eu +script_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) +cd "$script_path/.." +set -x + +# cargo install cargo-deny +cargo deny --all-features --log-level error --target aarch64-apple-darwin check +cargo deny --all-features --log-level error --target i686-pc-windows-gnu check +cargo deny --all-features --log-level error --target i686-pc-windows-msvc check +cargo deny --all-features --log-level error --target i686-unknown-linux-gnu check +cargo deny --all-features --log-level error --target wasm32-unknown-unknown check +cargo deny --all-features --log-level error --target x86_64-apple-darwin check +cargo deny --all-features --log-level error --target x86_64-pc-windows-gnu check +cargo deny --all-features --log-level error --target x86_64-pc-windows-msvc check +cargo deny --all-features --log-level error --target x86_64-unknown-linux-gnu check +cargo deny --all-features --log-level error --target x86_64-unknown-linux-musl check +cargo deny --all-features --log-level error --target x86_64-unknown-redox check From 83cf5db819b2601a51f6ccfe264cb122e36c0169 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 01:36:52 +0200 Subject: [PATCH 010/106] Share cache key between build-and-test / linting --- .github/workflows/reusable_build_and_test.yml | 1 + .github/workflows/reusable_lint.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index b4d1a380b26d..830e3576b635 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -110,6 +110,7 @@ jobs: - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: + shared-key: "build-${{ inputs.PLATFORM }}" env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY # Don't update the cache -- it will be updated by the lint job # TODO(jleibs): this job will likely run before rust.yml updates diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index 526da7ca814a..ed2473c7fdd5 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -84,6 +84,7 @@ jobs: - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: + shared-key: "build-linux" env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY # See: https://github.com/rerun-io/rerun/pull/497 save-if: ${{ inputs.SAVE_CACHE }} From bbae14b9fb07fd0bba071038b8fbe51a3c1ebef8 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 01:39:00 +0200 Subject: [PATCH 011/106] Actually checkout code when running tests --- .github/workflows/reusable_build_and_test.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 830e3576b635..9c69a5221884 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -90,6 +90,19 @@ jobs: EXPECTED_VERSION: ${{ steps.expected_version.outputs.expected_version }} steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + echo "EXPECTED_VERSION": $EXPECTED_VERSION + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + - uses: actions/checkout@v3 # The pip-cache setup logic doesn't work in the ubuntu docker container @@ -219,6 +232,8 @@ jobs: INPUTS_CONTEXT: ${{ toJson(inputs) }} EXPECTED_VERSION: ${{ needs.build_wheels.outputs.EXPECTED_VERSION }} + - uses: actions/checkout@v3 + - name: Download Artifact uses: actions/download-artifact@v3 with: From 3a00fee82ed8b1ecacbd7daa33b87c15b7039e62 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 02:39:45 +0200 Subject: [PATCH 012/106] Switch to sccache --- .github/workflows/reusable_build_and_test.yml | 32 +++++++++++++------ .github/workflows/reusable_lint.yml | 27 ++++++++++++---- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 9c69a5221884..6951b9bd8a12 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -28,6 +28,10 @@ env: # See https://github.com/ericseppanen/cargo-cranky/issues/8 RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + jobs: ################################################################################ @@ -105,6 +109,23 @@ jobs: - uses: actions/checkout@v3 + # Rust-cache will cache our dependencies, which is a large chunk of the build + # See: https://github.com/Swatinem/rust-cache + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-${{ inputs.PLATFORM }}" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: ${{ inputs.SAVE_CACHE }} + + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + # The pip-cache setup logic doesn't work in the ubuntu docker container # That's probably fine since we bake these deps into the container already - name: Setup python @@ -120,16 +141,6 @@ jobs: # docker - run: pip install -r rerun_py/requirements-build.txt - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - shared-key: "build-${{ inputs.PLATFORM }}" - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # Don't update the cache -- it will be updated by the lint job - # TODO(jleibs): this job will likely run before rust.yml updates - # the cache. Better cross-job sequencing would be nice here - save-if: ${{ inputs.SAVE_CACHE }} - # We have a nice script for that: ./scripts/setup_web.sh # Unfortunately, we can't run that on Windows, because Windows doesn't come with # a package manager like grown-up OSes do (at least not the CI version of Windows). @@ -190,6 +201,7 @@ jobs: manylinux: manylinux_2_31 container: off command: build + sccache: 'true' args: | --manifest-path rerun_py/Cargo.toml --release diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index ed2473c7fdd5..dd188ff3a8ca 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -20,6 +20,10 @@ env: # See https://github.com/ericseppanen/cargo-cranky/issues/8 RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + jobs: ################################################################################ @@ -62,9 +66,6 @@ jobs: runs-on: ubuntu-latest-16-cores container: image: rerunio/ci_docker:0.6 - env: - RUSTFLAGS: ${{env.RUSTFLAGS}} - RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} steps: - name: Show context run: | @@ -77,18 +78,29 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} JOB_CONTEXT: ${{ toJson(job) }} INPUTS_CONTEXT: ${{ toJson(inputs) }} + - uses: actions/checkout@v3 with: fetch-depth: 0 # Don't do a shallow clone since we need to push gh-pages + # Rust-cache will cache our dependencies, which is a large chunk of the build + # See: https://github.com/Swatinem/rust-cache - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: shared-key: "build-linux" env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # See: https://github.com/rerun-io/rerun/pull/497 + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here save-if: ${{ inputs.SAVE_CACHE }} + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + # First do our check with --locked to make sure `Cargo.lock` is up to date - name: Check all features uses: actions-rs/cargo@v1 @@ -221,13 +233,14 @@ jobs: runs-on: ubuntu-latest container: image: rerunio/ci_docker:0.6 - env: - RUSTFLAGS: ${{env.RUSTFLAGS}} - RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} steps: - uses: actions/checkout@v3 + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + - name: Cargo Deny shell: bash id: expected_version From 1ab9d3c2a57b8bdab7f35b3d0b12a73126a3ea51 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 04:06:04 +0200 Subject: [PATCH 013/106] Remove --deny warnings for rustflags --- .github/workflows/reusable_build_and_test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 6951b9bd8a12..789f7a5fa60c 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -23,7 +23,10 @@ env: # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # TODO(jleibs) --deny warnings causes installation of wasm-bindgen to fail on mac + # RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + RUSTFLAGS: --cfg=web_sys_unstable_apis # See https://github.com/ericseppanen/cargo-cranky/issues/8 RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs From 601ae970244c439c399a731d85bdbf4bd9553599 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 04:59:16 +0200 Subject: [PATCH 014/106] New workflow for packaging and uploading --- .github/workflows/reusable_build_and_test.yml | 38 +++++- .github/workflows/reusable_lint.yml | 1 + .../workflows/reusable_package_and_upload.yml | 127 ++++++++++++++++++ 3 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/reusable_package_and_upload.yml diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 789f7a5fa60c..b58f705c8360 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -17,6 +17,10 @@ on: required: false type: boolean default: false + outputs: + EXPECTED_VERSION: + description: "The expected version of the package" + value: ${{ jobs.build_wheels.outputs.EXPECTED_VERSION }} env: PYTHON_VERSION: "3.8" @@ -94,7 +98,7 @@ jobs: container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} outputs: - EXPECTED_VERSION: ${{ steps.expected_version.outputs.expected_version }} + EXPECTED_VERSION: ${{ steps.expected_version.outputs.EXPECTED_VERSION }} steps: - name: Show context @@ -195,7 +199,7 @@ jobs: shell: bash id: expected_version run: | - echo "expected_version=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" + echo "EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" - name: Build Wheel uses: PyO3/maturin-action@v1 @@ -216,7 +220,7 @@ jobs: - name: Upload wheels uses: actions/upload-artifact@v3 with: - name: ${{inputs.PLATFORM}}-wheels + name: ${{inputs.PLATFORM}}-wheel path: dist ################################################################################ @@ -252,7 +256,7 @@ jobs: - name: Download Artifact uses: actions/download-artifact@v3 with: - name: ${{inputs.PLATFORM}}-wheels + name: ${{inputs.PLATFORM}}-wheel path: dist - name: Install wheel dependencies @@ -287,3 +291,29 @@ jobs: - name: Run e2e test shell: bash run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed + + - name: Cache RRD dataset + if: inputs.PLATFORM == 'linux' + id: dataset + uses: actions/cache@v3 + with: + path: examples/python/colmap/dataset/ + # TODO(jleibs): Derive this key from the invocation below + key: colmap-dataset-colmap-fiat-v0 + + - name: Generate Embedded RRD file + if: inputs.PLATFORM == 'linux' + shell: bash + # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name + run: | + mkdir rrd + pip install -r examples/python/colmap/requirements.txt + python3 examples/python/colmap/main.py --dataset colmap_fiat --resize 800x600 --save rrd/colmap_fiat.rrd + + # All platforms are currently creating the same rrd file, upload one of them + - name: Save RRD artifact + if: inputs.PLATFORM == 'linux' + uses: actions/upload-artifact@v3 + with: + name: rrd + path: rrd diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index dd188ff3a8ca..a6d9fa6cc960 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -10,6 +10,7 @@ on: required: false type: boolean default: false + env: PYTHON_VERSION: "3.8" # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml new file mode 100644 index 000000000000..9b9d56a0cfa1 --- /dev/null +++ b/.github/workflows/reusable_package_and_upload.yml @@ -0,0 +1,127 @@ +name: Reusable Package Wheels + +on: + workflow_call: + secrets: + CALLER_GITHUB_TOKEN: + required: true + inputs: + PLATFORM: + required: true + type: string + PRERELEASE: + required: false + type: boolean + default: false + EXPECTED_VERSION: + required: true + type: string + BUILD_LINUX: + description: 'Build/Test Linux' + type: boolean + required: true + default: true + BUILD_MACOS_INTEL: + description: 'Build/Test Mac (Intel)' + type: boolean + required: true + default: false + BUILD_MACOS_ARM: + description: 'Build/Test Mac (Arm)' + type: boolean + required: true + default: false + BUILD_WINDOWS: + description: 'Build/Test Win' + type: boolean + required: true + default: false + +jobs: + package_wheels: + name: Package Wheels + + runs-on: ubuntu-latest + container: + image: rerunio/ci_docker:0.6 + + strategy: + fail-fast: false + matrix: + platform: + - linux + - macos-arm + - macos-intel + - windows + + if: | + ((inputs.PLATFORM == 'linux' && inputs.BUILD_LINUX) || + (inputs.PLATFORM == 'macos-intel' && inputs.BUILD_MACOS_INTEL) || + (inputs.PLATFORM == 'macos-arm' && inputs.BUILD_MACOS_ARM) || + (inputs.PLATFORM == 'windows' && inputs.BUILD_WINDOWS)) + + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + echo "MATRIX_CONTEXT": $MATRIX_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + MATRIX_CONTEXT: ${{ toJson(matrix) }} + + - uses: actions/checkout@v3 + + - name: Download RRD + uses: actions/download-artifact@v3 + with: + name: rrd + path: rrd + + - name: Download Wheel + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.platform }}-wheel + path: pre-dist + + - name: Unpack the wheel + shell: bash + run: | + mkdir unpack-dist + wheel unpack pre-dist/*.whl --dest unpack-dist + + - name: Get the folder name + shell: bash + id: get_folder_name + run: | + echo "PKG_FOLDER=$(ls unpack-dist)" >> "$GITHUB_OUTPUT" + + - name: Download RRD + uses: actions/download-artifact@v3 + with: + name: rrd + path: rrd + + - name: Insert the rrd + shell: bash + # If you change the line below you should almost definitely change the `key:` line + # in 'Cache RRD dataset'reusable_build_and_test.yml + run: | + cp rrd/colmap_fiat.rrd unpack-dist/${{ steps.get_folder_name.outputs.PKG_FOLDER }}/rerun_sdk/rerun_demo/colmap_fiat.rrd + + - name: Repack the wheel + shell: bash + run: | + mkdir dist + wheel pack unpack-dist/${{ steps.get_folder_name.outputs.PKG_FOLDER }} --dest dist/ + + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist From ce5c50fd95b86698bd4e3b1499297fdbf41f6e86 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 05:16:49 +0200 Subject: [PATCH 015/106] Update manual dispatch to pass EXPECTED_VERSION --- .github/workflows/manual_dispatch.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index edae126ebea4..b3897cdfd78b 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -91,10 +91,12 @@ jobs: secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - package_and_uplaod: + package_and_upload: name: Package and Upload needs: [build_linux, build_windows] if: ${{ github.event.inputs.PACKAGE == 'true' }} uses: ./.github/workflows/reusable_package_and_upload.yml + with: + EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 56eeac0e3431137c08a82c57d392b9c063a1bdf7 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 05:21:28 +0200 Subject: [PATCH 016/106] Remove PLATFORM from package since it's part of matrix --- .github/workflows/reusable_package_and_upload.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml index 9b9d56a0cfa1..216394f1bced 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_and_upload.yml @@ -6,9 +6,6 @@ on: CALLER_GITHUB_TOKEN: required: true inputs: - PLATFORM: - required: true - type: string PRERELEASE: required: false type: boolean From b757365e8cb2cd5e43b470d35f91a5b4a1a0c46b Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 05:24:30 +0200 Subject: [PATCH 017/106] Plumb through BUILD flags --- .github/workflows/manual_dispatch.yml | 4 ++++ .github/workflows/reusable_package_and_upload.yml | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index b3897cdfd78b..db1050a19bc2 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -97,6 +97,10 @@ jobs: if: ${{ github.event.inputs.PACKAGE == 'true' }} uses: ./.github/workflows/reusable_package_and_upload.yml with: + BUILD_LINUX: ${{ github.event.inputs.BUILD_LINUX == 'true' }} + BUILD_MACOS_ARM: ${{ github.event.inputs.BUILD_MACOS_ARM == 'true' }} + BUILD_MACOS_INTEL: ${{ github.event.inputs.BUILD_MACOS_INTEL == 'true' }} + BUILD_WINDOWS: ${{ github.event.inputs.BUILD_WINDOWS == 'true' }} EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml index 216394f1bced..06feafb4d81f 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_and_upload.yml @@ -52,10 +52,10 @@ jobs: - windows if: | - ((inputs.PLATFORM == 'linux' && inputs.BUILD_LINUX) || - (inputs.PLATFORM == 'macos-intel' && inputs.BUILD_MACOS_INTEL) || - (inputs.PLATFORM == 'macos-arm' && inputs.BUILD_MACOS_ARM) || - (inputs.PLATFORM == 'windows' && inputs.BUILD_WINDOWS)) + ((matrix.platform == 'linux' && inputs.BUILD_LINUX) || + (matrix.platform == 'macos-intel' && inputs.BUILD_MACOS_INTEL) || + (matrix.platform == 'macos-arm' && inputs.BUILD_MACOS_ARM) || + (matrix.platform == 'windows' && inputs.BUILD_WINDOWS)) steps: - name: Show context From 77e6a601533237d5a2fd799ab636360bd73fd0f8 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 05:33:46 +0200 Subject: [PATCH 018/106] two-stage matrix --- .../workflows/reusable_package_and_upload.yml | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml index 06feafb4d81f..14fa8ac094ee 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_and_upload.yml @@ -35,27 +35,49 @@ on: default: false jobs: + + matrix-setup: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Set matrix + id: set-matrix + run: | + matrix_data='[' + + if [ "${{ inputs.BUILD_LINUX }}" = "true" ]; then + matrix_data+='{"platform": "linux"},' + fi + + if [ "${{ inputs.BUILD_MACOS_INTEL }}" = "true" ]; then + matrix_data+='{"platform": "macos-intel"},' + fi + + if [ "${{ inputs.BUILD_MACOS_ARM }}" = "true" ]; then + matrix_data+='{"platform": "macos-arm"},' + fi + + if [ "${{ inputs.BUILD_WINDOWS }}" = "true" ]; then + matrix_data+='{"platform": "windows"},' + fi + + matrix_data="${matrix_data%,}]" + echo "Matrix data: $matrix_data" + echo "matrix=$matrix_data" >> "$GITHUB_OUTPUT" + package_wheels: name: Package Wheels + needs: matrix-setup + runs-on: ubuntu-latest container: image: rerunio/ci_docker:0.6 strategy: fail-fast: false - matrix: - platform: - - linux - - macos-arm - - macos-intel - - windows - - if: | - ((matrix.platform == 'linux' && inputs.BUILD_LINUX) || - (matrix.platform == 'macos-intel' && inputs.BUILD_MACOS_INTEL) || - (matrix.platform == 'macos-arm' && inputs.BUILD_MACOS_ARM) || - (matrix.platform == 'windows' && inputs.BUILD_WINDOWS)) + matrix: ${{ fromJson(needs.matrix-setup.outputs.matrix) }} steps: - name: Show context From f4f2fafdf12d1189d97f42d8481c71acf764bd7e Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 06:05:31 +0200 Subject: [PATCH 019/106] Was missing pytest from non-linux builds --- .github/workflows/reusable_build_and_test.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index b58f705c8360..538855288ee8 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -253,11 +253,13 @@ jobs: - uses: actions/checkout@v3 - - name: Download Artifact - uses: actions/download-artifact@v3 + - name: Setup python + if: ${{ inputs.PLATFORM != 'linux' }} + uses: actions/setup-python@v4 with: - name: ${{inputs.PLATFORM}}-wheel - path: dist + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" - name: Install wheel dependencies # First we install the dependencies manually so we can use `--no-index` when installing the wheel. @@ -266,7 +268,13 @@ jobs: # TODO(jleibs): understand why deps can't be installed in the same step as the wheel shell: bash run: | - pip install deprecated numpy>=1.23 pyarrow==10.0.1 + pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 + + - name: Download Artifact + uses: actions/download-artifact@v3 + with: + name: ${{inputs.PLATFORM}}-wheel + path: dist - name: Install built wheel # Now install the wheel using a specific version and --no-index to guarantee we get the version from From bae44903cabc6522d0d87d04b1dedef6e0663e23 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 16:16:08 +0200 Subject: [PATCH 020/106] Refactor matrix as several build jobs again --- .github/workflows/manual_dispatch.yml | 30 ++++++--- .../workflows/reusable_package_and_upload.yml | 61 ++----------------- 2 files changed, 27 insertions(+), 64 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index db1050a19bc2..026752c2f2dd 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -53,7 +53,15 @@ jobs: build_linux: name: Build/Test Linux - if: ${{ github.event.inputs.BUILD_LINUX == 'true' }} + + # If we are packaging any build target, we also need the linux job to run since it produces the RRD + if: | + ${{ (github.event.inputs.BUILD_LINUX == 'true') || + ((github.event.inputs.PACKAGE == 'true') && ( + (github.event.inputs.BUILD_MACOS_INTEL == 'true') || + (github.event.inputs.BUILD_MACOS_ARM == 'true') || + (github.event.inputs.BUILD_WINDOWS == 'true'))) }} + uses: ./.github/workflows/reusable_build_and_test.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} @@ -91,16 +99,24 @@ jobs: secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - package_and_upload: + package_and_upload_linux: + name: Package and Upload + needs: [build_linux] + if: ${{ (github.event.inputs.BUILD_LINUX) == 'true' && (github.event.inputs.PACKAGE == 'true') }} + uses: ./.github/workflows/reusable_package_and_upload.yml + with: + PLATFORM: linux + EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} + secrets: + CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + package_and_upload_windows: name: Package and Upload needs: [build_linux, build_windows] - if: ${{ github.event.inputs.PACKAGE == 'true' }} + if: ${{ (github.event.inputs.BUILD_WINDOWS) == 'true' && (github.event.inputs.PACKAGE == 'true') }} uses: ./.github/workflows/reusable_package_and_upload.yml with: - BUILD_LINUX: ${{ github.event.inputs.BUILD_LINUX == 'true' }} - BUILD_MACOS_ARM: ${{ github.event.inputs.BUILD_MACOS_ARM == 'true' }} - BUILD_MACOS_INTEL: ${{ github.event.inputs.BUILD_MACOS_INTEL == 'true' }} - BUILD_WINDOWS: ${{ github.event.inputs.BUILD_WINDOWS == 'true' }} + PLATFORM: linux EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml index 14fa8ac094ee..82a9100815ca 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_and_upload.yml @@ -6,6 +6,9 @@ on: CALLER_GITHUB_TOKEN: required: true inputs: + PLATFORM: + required: true + type: string PRERELEASE: required: false type: boolean @@ -13,72 +16,17 @@ on: EXPECTED_VERSION: required: true type: string - BUILD_LINUX: - description: 'Build/Test Linux' - type: boolean - required: true - default: true - BUILD_MACOS_INTEL: - description: 'Build/Test Mac (Intel)' - type: boolean - required: true - default: false - BUILD_MACOS_ARM: - description: 'Build/Test Mac (Arm)' - type: boolean - required: true - default: false - BUILD_WINDOWS: - description: 'Build/Test Win' - type: boolean - required: true - default: false jobs: - matrix-setup: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Set matrix - id: set-matrix - run: | - matrix_data='[' - - if [ "${{ inputs.BUILD_LINUX }}" = "true" ]; then - matrix_data+='{"platform": "linux"},' - fi - - if [ "${{ inputs.BUILD_MACOS_INTEL }}" = "true" ]; then - matrix_data+='{"platform": "macos-intel"},' - fi - - if [ "${{ inputs.BUILD_MACOS_ARM }}" = "true" ]; then - matrix_data+='{"platform": "macos-arm"},' - fi - - if [ "${{ inputs.BUILD_WINDOWS }}" = "true" ]; then - matrix_data+='{"platform": "windows"},' - fi - - matrix_data="${matrix_data%,}]" - echo "Matrix data: $matrix_data" - echo "matrix=$matrix_data" >> "$GITHUB_OUTPUT" package_wheels: name: Package Wheels - needs: matrix-setup - runs-on: ubuntu-latest container: image: rerunio/ci_docker:0.6 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.matrix-setup.outputs.matrix) }} - steps: - name: Show context run: | @@ -92,7 +40,6 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} JOB_CONTEXT: ${{ toJson(job) }} INPUTS_CONTEXT: ${{ toJson(inputs) }} - MATRIX_CONTEXT: ${{ toJson(matrix) }} - uses: actions/checkout@v3 @@ -105,7 +52,7 @@ jobs: - name: Download Wheel uses: actions/download-artifact@v3 with: - name: ${{ matrix.platform }}-wheel + name: ${{inputs.PLATFORM}}-wheel path: pre-dist - name: Unpack the wheel From a784b8a3df6eec2422deadb4f0e145a10ce1c257 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 16:54:00 +0200 Subject: [PATCH 021/106] Pass through platform on packaging jobs --- .github/workflows/manual_dispatch.yml | 38 +++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 026752c2f2dd..d3ee1f2e866c 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -52,7 +52,7 @@ jobs: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build_linux: - name: Build/Test Linux + name: 'Linux: Build/Test' # If we are packaging any build target, we also need the linux job to run since it produces the RRD if: | @@ -70,7 +70,7 @@ jobs: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build_windows: - name: Build/Test Windows + name: 'Windows: Build/Test' if: ${{ github.event.inputs.BUILD_WINDOWS == 'true'}} uses: ./.github/workflows/reusable_build_and_test.yml with: @@ -80,7 +80,7 @@ jobs: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build_macos_arm: - name: Build/Test Mac (Arm) + name: 'Macos-Arm: Build/Test' if: ${{ github.event.inputs.BUILD_MACOS_ARM == 'true' }} uses: ./.github/workflows/reusable_build_and_test.yml with: @@ -90,7 +90,7 @@ jobs: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build_macos_intel: - name: Build/Test Mac (Intel) + name: 'Macos-Intel: Build/Test' if: ${{ github.event.inputs.BUILD_MACOS_INTEL == 'true' }} uses: ./.github/workflows/reusable_build_and_test.yml with: @@ -100,7 +100,7 @@ jobs: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} package_and_upload_linux: - name: Package and Upload + name: 'Linux: Package and Upload' needs: [build_linux] if: ${{ (github.event.inputs.BUILD_LINUX) == 'true' && (github.event.inputs.PACKAGE == 'true') }} uses: ./.github/workflows/reusable_package_and_upload.yml @@ -111,12 +111,34 @@ jobs: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} package_and_upload_windows: - name: Package and Upload + name: 'Windows: Package and Upload' needs: [build_linux, build_windows] if: ${{ (github.event.inputs.BUILD_WINDOWS) == 'true' && (github.event.inputs.PACKAGE == 'true') }} uses: ./.github/workflows/reusable_package_and_upload.yml with: - PLATFORM: linux - EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} + PLATFORM: windows + EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} + secrets: + CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + package_and_upload_macos_arm: + name: 'Macos-Arm: Package and Upload' + needs: [build_linux, build_macos_arm] + if: ${{ (github.event.inputs.BUILD_MACOS_ARM) == 'true' && (github.event.inputs.PACKAGE == 'true') }} + uses: ./.github/workflows/reusable_package_and_upload.yml + with: + PLATFORM: macos-arm + EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} + secrets: + CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + package_and_upload_macos_intel: + name: 'Macos-Intel: Package and Upload' + needs: [build_linux, build_macos_intel] + if: ${{ (github.event.inputs.BUILD_MACOS_INTEL) == 'true' && (github.event.inputs.PACKAGE == 'true') }} + uses: ./.github/workflows/reusable_package_and_upload.yml + with: + PLATFORM: macos-intel + EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From ac7dcbeb33f739599496629ed78f7d9cdfe5e45b Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 17:28:41 +0200 Subject: [PATCH 022/106] Always patch for prerelease if it's not a release --- .github/workflows/manual_dispatch.yml | 27 +++++++++++-------- .github/workflows/reusable_build_and_test.yml | 12 ++++----- .../workflows/reusable_package_and_upload.yml | 8 +++--- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index d3ee1f2e866c..35aed2a85d11 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -8,37 +8,42 @@ on: LINTS: description: 'Run Lints' type: boolean - required: true + required: false default: true BUILD_LINUX: description: 'Build/Test Linux' type: boolean - required: true + required: false default: true BUILD_MACOS_INTEL: description: 'Build/Test Mac (Intel)' type: boolean - required: true + required: false default: false BUILD_MACOS_ARM: description: 'Build/Test Mac (Arm)' type: boolean - required: true + required: false default: false BUILD_WINDOWS: description: 'Build/Test Win' type: boolean - required: true - default: false - SAVE_CACHE: - description: 'Save Cache' - type: boolean - required: true + required: false default: false PACKAGE: description: 'Package' type: boolean - required: true + required: false + default: false + RELEASE: + description: 'Release' + type: boolean + required: false + default: false + SAVE_CACHE: + description: 'Save Cache' + type: boolean + required: false default: false jobs: diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 538855288ee8..60765a18de89 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -6,14 +6,14 @@ on: CALLER_GITHUB_TOKEN: required: true inputs: - SAVE_CACHE: - required: false - type: boolean - default: false PLATFORM: required: true type: string - PRERELEASE: + RELEASE: + required: false + type: boolean + default: false + SAVE_CACHE: required: false type: boolean default: false @@ -179,7 +179,7 @@ jobs: pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" - name: Patch Cargo.toml for pre-release - if: inputs.PRERELEASE + if: inputs.RELEASE != 'true' # After patching the pre-release version, run cargo update. # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing run: | diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml index 82a9100815ca..b3c85ca76258 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_and_upload.yml @@ -6,16 +6,16 @@ on: CALLER_GITHUB_TOKEN: required: true inputs: + EXPECTED_VERSION: + required: true + type: string PLATFORM: required: true type: string - PRERELEASE: + RELEASE: required: false type: boolean default: false - EXPECTED_VERSION: - required: true - type: string jobs: From 40e4a4588ad156e46ba91da9ea1ab712031a6881 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 18:48:43 +0200 Subject: [PATCH 023/106] Push wheels to gcloud --- .github/workflows/manual_dispatch.yml | 8 +++++++ .../workflows/reusable_package_and_upload.yml | 22 +++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 35aed2a85d11..bdbfaa83d205 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -114,6 +114,8 @@ jobs: EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} package_and_upload_windows: name: 'Windows: Package and Upload' @@ -125,6 +127,8 @@ jobs: EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} package_and_upload_macos_arm: name: 'Macos-Arm: Package and Upload' @@ -136,6 +140,8 @@ jobs: EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} package_and_upload_macos_intel: name: 'Macos-Intel: Package and Upload' @@ -147,3 +153,5 @@ jobs: EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml index b3c85ca76258..21927e890413 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_and_upload.yml @@ -5,6 +5,10 @@ on: secrets: CALLER_GITHUB_TOKEN: required: true + GOOGLE_WORKLOAD_IDENTITY_PROVIDER: + required: true + GOOGLE_SERVICE_ACCOUNT: + required: true inputs: EXPECTED_VERSION: required: true @@ -86,8 +90,18 @@ jobs: mkdir dist wheel pack unpack-dist/${{ steps.get_folder_name.outputs.PKG_FOLDER }} --dest dist/ - - name: Upload wheels - uses: actions/upload-artifact@v3 + - id: "auth" + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + - name: Add SHORT_SHA env property with commit short sha + run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV + + - name: "Upload Wheel" + uses: google-github-actions/upload-cloud-storage@v1 with: - name: wheels - path: dist + path: "dist" + destination: "rerun-packages/commit/${{env.SHORT_SHA}}/wheels" + parent: false From fa4d2487d536a423b35f30d15150d30934cf785f Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 19:08:19 +0200 Subject: [PATCH 024/106] Add job for running benchmarks --- .github/workflows/manual_dispatch.yml | 12 ++++ .github/workflows/reusable_bench.yml | 89 +++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 .github/workflows/reusable_bench.yml diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index bdbfaa83d205..57d27d0c3906 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -10,6 +10,11 @@ on: type: boolean required: false default: true + BENCHES: + description: 'Run Benches' + type: boolean + required: false + default: false BUILD_LINUX: description: 'Build/Test Linux' type: boolean @@ -56,6 +61,13 @@ jobs: secrets: CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + benches: + name: Benchmarks + if: ${{ github.event.inputs.BENCHES == 'true' }} + uses: ./.github/workflows/reusable_bench.yml + secrets: + CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + build_linux: name: 'Linux: Build/Test' diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml new file mode 100644 index 000000000000..4f62c05d5efe --- /dev/null +++ b/.github/workflows/reusable_bench.yml @@ -0,0 +1,89 @@ +name: Reusable Bench + +on: + workflow_call: + secrets: + CALLER_GITHUB_TOKEN: + required: true + inputs: + SAVE_BENCHES: + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + +################################################################################ + + rs-benchmarks: + name: Rust Criterion benchmarks + + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.6 + steps: + - uses: actions/checkout@v3 + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-linux" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: false + + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Run benchmark + # Use bash shell so we get pipefail behavior with tee + shell: bash + run: | + cargo bench \ + --all-features \ + -p re_arrow_store \ + -p re_data_store \ + -p re_log_encoding \ + -p re_query \ + -p re_tuid \ + -- --output-format=bencher | tee output.txt + + - name: Store benchmark result + # https://github.com/benchmark-action/github-action-benchmark + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Rust Benchmark + tool: "cargo" + output-file-path: output.txt + github-token: ${{ secrets.CALLER_GITHUB_TOKEN }} + + # Show alert with commit comment on detecting possible performance regression + comment-on-alert: true + alert-threshold: "125%" + fail-on-alert: true + comment-always: false # Generates too much GitHub notification spam + + # Save, results and push to GitHub only on main + save-data-file: ${{ inputs.SAVE_BENCHES }} + auto-push: ${{ inputs.SAVE_BENCHES }} + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench + max-items-in-chart: 30 From f818f09583570a5d87633751d30a7a4a09af0ae3 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 19:14:03 +0200 Subject: [PATCH 025/106] Inherit secrets --- .github/workflows/manual_dispatch.yml | 38 +++++-------------- .github/workflows/reusable_bench.yml | 5 +-- .github/workflows/reusable_build_and_test.yml | 3 -- .github/workflows/reusable_lint.yml | 5 +-- .../workflows/reusable_package_and_upload.yml | 7 ---- 5 files changed, 12 insertions(+), 46 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 57d27d0c3906..405c63f88bff 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -58,15 +58,13 @@ jobs: uses: ./.github/workflows/reusable_lint.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + secrets: inherit benches: name: Benchmarks if: ${{ github.event.inputs.BENCHES == 'true' }} uses: ./.github/workflows/reusable_bench.yml - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + secrets: inherit build_linux: name: 'Linux: Build/Test' @@ -83,8 +81,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + secrets: inherit build_windows: name: 'Windows: Build/Test' @@ -93,8 +90,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: windows - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + secrets: inherit build_macos_arm: name: 'Macos-Arm: Build/Test' @@ -103,8 +99,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-arm - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + secrets: inherit build_macos_intel: name: 'Macos-Intel: Build/Test' @@ -113,8 +108,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-intel - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + secrets: inherit package_and_upload_linux: name: 'Linux: Package and Upload' @@ -124,10 +118,7 @@ jobs: with: PLATFORM: linux EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + secrets: inherit package_and_upload_windows: name: 'Windows: Package and Upload' @@ -137,10 +128,7 @@ jobs: with: PLATFORM: windows EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + secrets: inherit package_and_upload_macos_arm: name: 'Macos-Arm: Package and Upload' @@ -150,10 +138,7 @@ jobs: with: PLATFORM: macos-arm EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + secrets: inherit package_and_upload_macos_intel: name: 'Macos-Intel: Package and Upload' @@ -163,7 +148,4 @@ jobs: with: PLATFORM: macos-intel EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + secrets: inherit diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 4f62c05d5efe..76b5cfd0c057 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -2,9 +2,6 @@ name: Reusable Bench on: workflow_call: - secrets: - CALLER_GITHUB_TOKEN: - required: true inputs: SAVE_BENCHES: required: false @@ -73,7 +70,7 @@ jobs: name: Rust Benchmark tool: "cargo" output-file-path: output.txt - github-token: ${{ secrets.CALLER_GITHUB_TOKEN }} + github-token: ${{ secrets.GITHUB_TOKEN }} # Show alert with commit comment on detecting possible performance regression comment-on-alert: true diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 60765a18de89..a382475d7748 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -2,9 +2,6 @@ name: Reusable Build and Test on: workflow_call: - secrets: - CALLER_GITHUB_TOKEN: - required: true inputs: PLATFORM: required: true diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index a6d9fa6cc960..0d1b8ff910f2 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -2,9 +2,6 @@ name: Reusable Lint on: workflow_call: - secrets: - CALLER_GITHUB_TOKEN: - required: true inputs: SAVE_CACHE: required: false @@ -36,7 +33,7 @@ jobs: - uses: actions/checkout@v3 - uses: extractions/setup-just@v1 env: - GITHUB_TOKEN: ${{ secrets.CALLER_GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: just-version: 1.5 diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml index 21927e890413..085ae704eba3 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_and_upload.yml @@ -2,13 +2,6 @@ name: Reusable Package Wheels on: workflow_call: - secrets: - CALLER_GITHUB_TOKEN: - required: true - GOOGLE_WORKLOAD_IDENTITY_PROVIDER: - required: true - GOOGLE_SERVICE_ACCOUNT: - required: true inputs: EXPECTED_VERSION: required: true From a1aad0722012d1c3a94f43198eb7e20aff0bd233 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 19:22:48 +0200 Subject: [PATCH 026/106] Add missing permissions --- .github/workflows/reusable_package_and_upload.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_and_upload.yml index 085ae704eba3..a824d394a47a 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_and_upload.yml @@ -16,14 +16,18 @@ on: jobs: - package_wheels: name: Package Wheels runs-on: ubuntu-latest + container: image: rerunio/ci_docker:0.6 + permissions: + contents: "read" + id-token: "write" + steps: - name: Show context run: | From 470ec1707a9c9bea4edfda7c189a00dcc4a40f17 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 20:28:45 +0200 Subject: [PATCH 027/106] Include steps to upload docs --- .github/workflows/manual_dispatch.yml | 6 ++ .github/workflows/reusable_lint.yml | 113 +++++++++++++++++++++++++- 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 405c63f88bff..e7fe90bb11e4 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -50,6 +50,11 @@ on: type: boolean required: false default: false + SAVE_DOCS_AS: + description: 'Save Docs' + type: string + required: false + default: '' jobs: lint_all: @@ -58,6 +63,7 @@ jobs: uses: ./.github/workflows/reusable_lint.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} + SAVE_DOCS_AS: ${{ github.event.inputs.SAVE_DOCS_AS }} secrets: inherit benches: diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index 0d1b8ff910f2..24c1688e5789 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -1,4 +1,4 @@ -name: Reusable Lint +name: Reusable Lint & Docs on: workflow_call: @@ -7,6 +7,10 @@ on: required: false type: boolean default: false + SAVE_DOCS_AS: + required: false + type: string + default: "" env: PYTHON_VERSION: "3.8" @@ -57,6 +61,85 @@ jobs: run: | just py-requirements + py-test-docs: + name: Test Python Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + cache: "pip" + cache-dependency-path: "rerun_py/requirements-doc.txt" + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install -r rerun_py/requirements-doc.txt + + - name: Build via mkdocs + run: | + mkdocs build -f rerun_py/mkdocs.yml + + py-docs: + needs: [py-test-docs] + name: Build and deploy docs + if: ${{ inputs.SAVE_DOCS_AS != '' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Don't do a shallow clone + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + cache: "pip" + cache-dependency-path: "rerun_py/requirements-doc.txt" + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install -r rerun_py/requirements-doc.txt + + - name: Set up git author + run: | + remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Mike will incrementally update the existing gh-pages branch + # We then check it out, and reset it to a new orphaned branch, which we force-push to origin + # to make sure we don't accumulate unnecessary history in gh-pages branch + - name: Deploy via mike # https://github.com/jimporter/mike + if: ${{ inputs.SAVE_DOCS_AS != 'HEAD' }} + run: | + git fetch + mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.SAVE_DOCS_AS}} latest + git checkout gh-pages + git checkout --orphan gh-pages-orphan + git commit -m "Update docs for ${GITHUB_SHA}" + git push origin gh-pages-orphan:gh-pages -f + + # Mike will incrementally update the existing gh-pages branch + # We then check it out, and reset it to a new orphaned branch, which we force-push to origin + # to make sure we don't accumulate unnecessary history in gh-pages branch + - name: Deploy tag via mike # https://github.com/jimporter/mike + if: ${{ inputs.SAVE_DOCS_AS == 'HEAD' }} + run: | + git fetch + mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python HEAD + git checkout gh-pages + git checkout --orphan gh-pages-orphan + git commit -m "Update docs for ${GITHUB_SHA}" + git push origin gh-pages-orphan:gh-pages -f + + ################################################################################ rs-lints: @@ -157,6 +240,34 @@ jobs: command: test args: --all-targets --all-features + # TODO(jleibs): Maybe we should run this on a separate job? + - name: Set up git author + if: ${{ inputs.SAVE_DOCS_AS != '' }} + run: | + remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up ghp-import + if: ${{ inputs.SAVE_DOCS_AS != '' }} + run: pip install ghp-import + + - name: Patch in a redirect page + if: ${{ inputs.SAVE_DOCS_AS != '' }} + run: echo "" > target/doc/index.html + env: + REDIRECT_CRATE: rerun + + # See: https://github.com/c-w/ghp-import + - name: Deploy the docs + if: ${{ inputs.SAVE_DOCS_AS != '' }} + run: | + git fetch + python3 -m ghp_import -n -p -x docs/rust/head target/doc/ -m "Update the rust docs" + + ################################################################################ toml-lints: From a0db1392608dc9c0e522a5cde47559ebe77aa316 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 20:33:05 +0200 Subject: [PATCH 028/106] Lighter bar for separation --- .github/workflows/reusable_bench.yml | 2 +- .github/workflows/reusable_build_and_test.yml | 6 +++--- .github/workflows/reusable_lint.yml | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 76b5cfd0c057..7a48dda8c141 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -24,7 +24,7 @@ env: jobs: -################################################################################ +# --------------------------------------------------------------------------- rs-benchmarks: name: Rust Criterion benchmarks diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index a382475d7748..21630bc7d7ca 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -38,7 +38,7 @@ env: jobs: -################################################################################ +# --------------------------------------------------------------------------- set_config: name: Set Config @@ -84,7 +84,7 @@ jobs: echo "run_tests=$run_tests" >> "$GITHUB_OUTPUT" echo "container=$container" >> "$GITHUB_OUTPUT" -################################################################################ +# --------------------------------------------------------------------------- build_wheels: name: Build Wheels @@ -220,7 +220,7 @@ jobs: name: ${{inputs.PLATFORM}}-wheel path: dist -################################################################################ +# --------------------------------------------------------------------------- run_tests: name: Test Wheels diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index 24c1688e5789..49410451660f 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -28,7 +28,7 @@ env: jobs: -################################################################################ +# --------------------------------------------------------------------------- py-lints: name: Python lints (black, mypy, flake8) @@ -140,7 +140,7 @@ jobs: git push origin gh-pages-orphan:gh-pages -f -################################################################################ +# --------------------------------------------------------------------------- rs-lints: name: Rust lints (fmt, check, cranky, tests, doc) @@ -268,7 +268,7 @@ jobs: python3 -m ghp_import -n -p -x docs/rust/head target/doc/ -m "Update the rust docs" -################################################################################ +# --------------------------------------------------------------------------- toml-lints: name: Lint TOML files @@ -302,7 +302,7 @@ jobs: run: | taplo fmt --check -################################################################################ +# --------------------------------------------------------------------------- misc-rerun-lints: name: Rerun lints @@ -323,7 +323,7 @@ jobs: run: | ./scripts/check_large_files.sh -################################################################################ +# --------------------------------------------------------------------------- spell-check: name: Spell Check @@ -335,7 +335,7 @@ jobs: - name: Check spelling of entire workspace uses: crate-ci/typos@master -################################################################################ +# --------------------------------------------------------------------------- rs-cargo-deny: name: Cargo Deny ${{ matrix.platform }} From ef40c45669b7f1015617bf327cada74ea9579beb Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 20:36:28 +0200 Subject: [PATCH 029/106] Remove multi-line if --- .github/workflows/manual_dispatch.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index e7fe90bb11e4..d9a96da29561 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -76,12 +76,8 @@ jobs: name: 'Linux: Build/Test' # If we are packaging any build target, we also need the linux job to run since it produces the RRD - if: | - ${{ (github.event.inputs.BUILD_LINUX == 'true') || - ((github.event.inputs.PACKAGE == 'true') && ( - (github.event.inputs.BUILD_MACOS_INTEL == 'true') || - (github.event.inputs.BUILD_MACOS_ARM == 'true') || - (github.event.inputs.BUILD_WINDOWS == 'true'))) }} + # TODO(jleibs): Debug why multi-line if statements don't work here + if: ${{ (github.event.inputs.BUILD_LINUX == 'true') || ((github.event.inputs.PACKAGE == 'true') && ( (github.event.inputs.BUILD_MACOS_INTEL == 'true') || (github.event.inputs.BUILD_MACOS_ARM == 'true') || (github.event.inputs.BUILD_WINDOWS == 'true'))) }} uses: ./.github/workflows/reusable_build_and_test.yml with: From ef5b989160e98da19445672d539f23ac36908523 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 21:38:02 +0200 Subject: [PATCH 030/106] jobs for building and uploading to web --- .github/workflows/manual_dispatch.yml | 50 ++++++--- .github/workflows/reusable_build_and_test.yml | 6 +- .github/workflows/reusable_build_web.yml | 68 ++++++++++++ .github/workflows/reusable_lint.yml | 50 ++++++++- ...upload.yml => reusable_package_wheels.yml} | 4 +- .github/workflows/reusable_upload_web.yml | 101 ++++++++++++++++++ 6 files changed, 255 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/reusable_build_web.yml rename .github/workflows/{reusable_package_and_upload.yml => reusable_package_wheels.yml} (98%) create mode 100644 .github/workflows/reusable_upload_web.yml diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index d9a96da29561..6f96cece3630 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -35,6 +35,11 @@ on: type: boolean required: false default: false + BUILD_WEB: + description: 'Build Web' + type: boolean + required: false + default: false PACKAGE: description: 'Package' type: boolean @@ -42,19 +47,14 @@ on: default: false RELEASE: description: 'Release' - type: boolean + type: string required: false - default: false + default: '' SAVE_CACHE: description: 'Save Cache' type: boolean required: false default: false - SAVE_DOCS_AS: - description: 'Save Docs' - type: string - required: false - default: '' jobs: lint_all: @@ -113,41 +113,61 @@ jobs: secrets: inherit package_and_upload_linux: - name: 'Linux: Package and Upload' + name: 'Linux: Package Wheels' needs: [build_linux] if: ${{ (github.event.inputs.BUILD_LINUX) == 'true' && (github.event.inputs.PACKAGE == 'true') }} - uses: ./.github/workflows/reusable_package_and_upload.yml + uses: ./.github/workflows/reusable_package_wheels.yml with: PLATFORM: linux EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} secrets: inherit package_and_upload_windows: - name: 'Windows: Package and Upload' + name: 'Windows: Package Wheels' needs: [build_linux, build_windows] if: ${{ (github.event.inputs.BUILD_WINDOWS) == 'true' && (github.event.inputs.PACKAGE == 'true') }} - uses: ./.github/workflows/reusable_package_and_upload.yml + uses: ./.github/workflows/reusable_package_wheels.yml with: PLATFORM: windows EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} secrets: inherit package_and_upload_macos_arm: - name: 'Macos-Arm: Package and Upload' + name: 'Macos-Arm: Package Wheels' needs: [build_linux, build_macos_arm] if: ${{ (github.event.inputs.BUILD_MACOS_ARM) == 'true' && (github.event.inputs.PACKAGE == 'true') }} - uses: ./.github/workflows/reusable_package_and_upload.yml + uses: ./.github/workflows/reusable_package_wheels.yml with: PLATFORM: macos-arm EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} secrets: inherit package_and_upload_macos_intel: - name: 'Macos-Intel: Package and Upload' + name: 'Macos-Intel: Package Wheels' needs: [build_linux, build_macos_intel] if: ${{ (github.event.inputs.BUILD_MACOS_INTEL) == 'true' && (github.event.inputs.PACKAGE == 'true') }} - uses: ./.github/workflows/reusable_package_and_upload.yml + uses: ./.github/workflows/reusable_package_wheels.yml with: PLATFORM: macos-intel EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} secrets: inherit + + build_web: + name: 'Build Web' + if: ${{ github.event.inputs.BUILD_WEB == 'true'}} + uses: ./.github/workflows/reusable_build_web.yml + with: + RELEASE: ${{ github.event.inputs.RELEASE }} + secrets: inherit + + upload_web: + name: 'Upload Web' + # Uses Assets: + # build_linux: rrd + # build_web: web_viewer + needs: [build_linux, build_web] + if: ${{ github.event.inputs.BUILD_WEB == 'true'}} + uses: ./.github/workflows/reusable_build_web.yml + with: + RELEASE: ${{ github.event.inputs.RELEASE }} + secrets: inherit diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 21630bc7d7ca..0f42ef35dc41 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -8,8 +8,8 @@ on: type: string RELEASE: required: false - type: boolean - default: false + type: string + default: '' SAVE_CACHE: required: false type: boolean @@ -176,7 +176,7 @@ jobs: pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" - name: Patch Cargo.toml for pre-release - if: inputs.RELEASE != 'true' + if: ${{ (inputs.RELEASE == '') || (inputs.RELEASE == 'prerelease') }} # After patching the pre-release version, run cargo update. # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing run: | diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml new file mode 100644 index 000000000000..57c8ba38038f --- /dev/null +++ b/.github/workflows/reusable_build_web.yml @@ -0,0 +1,68 @@ +name: Reusable Build Web + +on: + workflow_call: + inputs: + RELEASE: + required: false + type: boolean + default: false + +env: + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + +jobs: + + rs-build-web-viewer: + name: Build Web (wasm32 + wasm-bindgen) + permissions: + contents: "read" + id-token: "write" + + runs-on: ubuntu-latest-16-cores + + container: + image: rerunio/ci_docker:0.6 + + steps: + - uses: actions/checkout@v2 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.67.0 + target: wasm32-unknown-unknown + override: true + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # See: https://github.com/rerun-io/rerun/pull/497 + save-if: ${{ github.event_name == 'push'}} + + - name: Patch Cargo.toml for pre-release + if: ${{ (inputs.RELEASE == '') || (inputs.RELEASE == 'prerelease') }} + # After patching the pre-release version, run cargo update. + # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing + run: | + python3 scripts/version_util.py --patch_prerelease + cargo update -w + + - name: Build web-viewer (release) + uses: actions-rs/cargo@v1 + with: + command: run + args: --locked -p re_build_web_viewer -- --release + + - name: Upload web assets + uses: actions/upload-artifact@v3 + with: + name: web_viewer + path: web_viewer diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_lint.yml index 49410451660f..2694cd478946 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_lint.yml @@ -61,6 +61,8 @@ jobs: run: | just py-requirements +# --------------------------------------------------------------------------- + py-test-docs: name: Test Python Docs runs-on: ubuntu-latest @@ -83,9 +85,12 @@ jobs: run: | mkdocs build -f rerun_py/mkdocs.yml - py-docs: + +# --------------------------------------------------------------------------- + + py-deploy-docs: needs: [py-test-docs] - name: Build and deploy docs + name: Deploy Python Docs if: ${{ inputs.SAVE_DOCS_AS != '' }} runs-on: ubuntu-latest steps: @@ -201,7 +206,6 @@ jobs: command: cranky args: --all-targets --all-features -- --deny warnings - # -------------------------------------------------------------------------------- # Check a few important permutations of the feature flags for our `rerun` library: - name: Check rerun with `--no-default-features`` uses: actions-rs/cargo@v1 @@ -214,7 +218,6 @@ jobs: with: command: cranky args: --locked -p rerun --no-default-features --features sdk - # -------------------------------------------------------------------------------- - name: Test doc-tests uses: actions-rs/cargo@v1 @@ -267,6 +270,45 @@ jobs: git fetch python3 -m ghp_import -n -p -x docs/rust/head target/doc/ -m "Update the rust docs" +# --------------------------------------------------------------------------- + + rs-check-wasm: + name: Check Rust web build (wasm32 + wasm-bindgen) + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.6 + steps: + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: 1.67.1 + target: wasm32-unknown-unknown + override: true + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-wasm" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # See: https://github.com/rerun-io/rerun/pull/497 + save-if: ${{ inputs.SAVE_CACHE }} + + - name: clippy check re_viewer wasm32 + run: ./scripts/clippy_wasm.sh + + - name: Check re_renderer examples wasm32 + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked --target wasm32-unknown-unknown --target-dir target_wasm -p re_renderer --examples + + - name: Build web-viewer (debug) + uses: actions-rs/cargo@v1 + with: + command: run + args: --locked -p re_build_web_viewer -- --debug # --------------------------------------------------------------------------- diff --git a/.github/workflows/reusable_package_and_upload.yml b/.github/workflows/reusable_package_wheels.yml similarity index 98% rename from .github/workflows/reusable_package_and_upload.yml rename to .github/workflows/reusable_package_wheels.yml index a824d394a47a..91577a3d01a2 100644 --- a/.github/workflows/reusable_package_and_upload.yml +++ b/.github/workflows/reusable_package_wheels.yml @@ -11,8 +11,8 @@ on: type: string RELEASE: required: false - type: boolean - default: false + type: string + default: '' jobs: diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml new file mode 100644 index 000000000000..861f4b99e5dc --- /dev/null +++ b/.github/workflows/reusable_upload_web.yml @@ -0,0 +1,101 @@ +name: Reusable Upload Web + +on: + workflow_call: + inputs: + RELEASE: + required: false + type: string + default: '' + + +jobs: + + package_web: + name: Upload web build to google cloud (wasm32 + wasm-bindgen) + permissions: + contents: "read" + id-token: "write" + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Download RRD + uses: actions/download-artifact@v3 + with: + name: rrd + path: rrd + + - name: Download Web Viewer + uses: actions/download-artifact@v3 + with: + name: web_viewer + path: web_viewer + + + # Upload the wasm, html etc to a Google cloud bucket: + - id: "auth" + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + - name: Add SHORT_SHA env property with commit short sha + run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV + + - name: "Upload web-viewer (commit)" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "web_viewer" + destination: "rerun-web-viewer/commit/${{env.SHORT_SHA}}" + parent: false + + - name: "Upload RRD (commit)" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "rrd" + destination: "rerun-example-rrd/commit/${{env.SHORT_SHA}}" + parent: false + + - name: "Upload web-viewer (prerelease)" + if: inputs.RELEASE == 'prerelease' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "web_viewer" + destination: "rerun-web-viewer/prerelease" + parent: false + + + - name: "Upload RRD (prerelease)" + if: inputs.RELEASE == 'prerelease' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "rrd" + destination: "rerun-example-rrd/prerelease" + parent: false + + - name: "Upload web-viewer (tagged)" + if: ${{ (inputs.RELEASE != '') && (inputs.RELEASE != 'prerelease') }} + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "web_viewer" + destination: "rerun-web-viewer/version/${{github.ref_name}}" + parent: false + + - name: "Upload RRD (tagged)" + if: ${{ (inputs.RELEASE != '') && (inputs.RELEASE != 'prerelease') }} + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "rrd" + destination: "rerun-example-rrd/version/${{github.ref_name}}" + parent: false + + - name: "Upload RRD (latest release)" + if: github.ref == 'latest' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "rrd" + destination: "rerun-example-rrd/latest" + parent: false From bc88d6965132e76822cd6c40fba7ae9e3a4b4f92 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 21:46:21 +0200 Subject: [PATCH 031/106] Lints is doing a lot more than linting --- .github/workflows/manual_dispatch.yml | 4 ++-- .../workflows/{reusable_lint.yml => reusable_checks.yml} | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) rename .github/workflows/{reusable_lint.yml => reusable_checks.yml} (98%) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 6f96cece3630..1e18b89805cf 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -57,10 +57,10 @@ on: default: false jobs: - lint_all: + checks: name: Lint All if: ${{ github.event.inputs.LINTS == 'true' }} - uses: ./.github/workflows/reusable_lint.yml + uses: ./.github/workflows/reusable_checks.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} SAVE_DOCS_AS: ${{ github.event.inputs.SAVE_DOCS_AS }} diff --git a/.github/workflows/reusable_lint.yml b/.github/workflows/reusable_checks.yml similarity index 98% rename from .github/workflows/reusable_lint.yml rename to .github/workflows/reusable_checks.yml index 2694cd478946..6dae62366f96 100644 --- a/.github/workflows/reusable_lint.yml +++ b/.github/workflows/reusable_checks.yml @@ -1,4 +1,4 @@ -name: Reusable Lint & Docs +name: All lints, checks, and tests on: workflow_call: @@ -295,6 +295,10 @@ jobs: # See: https://github.com/rerun-io/rerun/pull/497 save-if: ${{ inputs.SAVE_CACHE }} + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + - name: clippy check re_viewer wasm32 run: ./scripts/clippy_wasm.sh From cac23dfb255c91e0fa8bf5ea0782ead72a74a15e Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 21:47:10 +0200 Subject: [PATCH 032/106] Use the right scripts --- .github/workflows/manual_dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 1e18b89805cf..6cacbca4b7dd 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -167,7 +167,7 @@ jobs: # build_web: web_viewer needs: [build_linux, build_web] if: ${{ github.event.inputs.BUILD_WEB == 'true'}} - uses: ./.github/workflows/reusable_build_web.yml + uses: ./.github/workflows/reusable_upload_web.yml with: RELEASE: ${{ github.event.inputs.RELEASE }} secrets: inherit From 2efe3472d6f24546586c2d12b5bcf4aa3dbe8d7f Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 21:51:42 +0200 Subject: [PATCH 033/106] Fix input param for build web --- .github/workflows/reusable_build_web.yml | 4 ++-- .github/workflows/reusable_upload_web.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 57c8ba38038f..93d77671e1eb 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -5,8 +5,8 @@ on: inputs: RELEASE: required: false - type: boolean - default: false + type: string + default: '' env: # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index 861f4b99e5dc..ef41f729c848 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -8,7 +8,6 @@ on: type: string default: '' - jobs: package_web: From 18915676ec0f3554d5db7856e38d25c748a7b36b Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 21 Apr 2023 21:54:08 +0200 Subject: [PATCH 034/106] More renaming lints -> checks --- .github/workflows/manual_dispatch.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 6cacbca4b7dd..8eec2a261638 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -5,16 +5,16 @@ on: # NOTE: boolean inputs are still actually strings # See: https://github.com/actions/runner/issues/1483 inputs: - LINTS: - description: 'Run Lints' + CHECKS: + description: 'Run All Checks' type: boolean required: false default: true BENCHES: - description: 'Run Benches' - type: boolean - required: false - default: false + description: 'Run Benches' + type: boolean + required: false + default: false BUILD_LINUX: description: 'Build/Test Linux' type: boolean @@ -58,8 +58,8 @@ on: jobs: checks: - name: Lint All - if: ${{ github.event.inputs.LINTS == 'true' }} + name: Run All Checks + if: ${{ github.event.inputs.CHECKS == 'true' }} uses: ./.github/workflows/reusable_checks.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} From 6114558c98675e087dd1c342a049bb28f110a60b Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 00:14:53 +0200 Subject: [PATCH 035/106] More naming --- .github/workflows/manual_dispatch.yml | 32 +++++++++---------- ..._wheels.yml => reusable_upload_wheels.yml} | 6 ++-- 2 files changed, 19 insertions(+), 19 deletions(-) rename .github/workflows/{reusable_package_wheels.yml => reusable_upload_wheels.yml} (96%) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 8eec2a261638..c0c4fc8b44eb 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -16,22 +16,22 @@ on: required: false default: false BUILD_LINUX: - description: 'Build/Test Linux' + description: 'Build/Test Wheels Linux' type: boolean required: false default: true BUILD_MACOS_INTEL: - description: 'Build/Test Mac (Intel)' + description: 'Build/Test Wheels Mac (Intel)' type: boolean required: false default: false BUILD_MACOS_ARM: - description: 'Build/Test Mac (Arm)' + description: 'Build/Test Wheels Mac (Arm)' type: boolean required: false default: false BUILD_WINDOWS: - description: 'Build/Test Win' + description: 'Build/Test Wheels Win' type: boolean required: false default: false @@ -73,7 +73,7 @@ jobs: secrets: inherit build_linux: - name: 'Linux: Build/Test' + name: 'Linux: Build/Test Wheels' # If we are packaging any build target, we also need the linux job to run since it produces the RRD # TODO(jleibs): Debug why multi-line if statements don't work here @@ -86,7 +86,7 @@ jobs: secrets: inherit build_windows: - name: 'Windows: Build/Test' + name: 'Windows: Build/Test Wheels' if: ${{ github.event.inputs.BUILD_WINDOWS == 'true'}} uses: ./.github/workflows/reusable_build_and_test.yml with: @@ -95,7 +95,7 @@ jobs: secrets: inherit build_macos_arm: - name: 'Macos-Arm: Build/Test' + name: 'Macos-Arm: Build/Test Wheels' if: ${{ github.event.inputs.BUILD_MACOS_ARM == 'true' }} uses: ./.github/workflows/reusable_build_and_test.yml with: @@ -104,7 +104,7 @@ jobs: secrets: inherit build_macos_intel: - name: 'Macos-Intel: Build/Test' + name: 'Macos-Intel: Build/Test Wheels' if: ${{ github.event.inputs.BUILD_MACOS_INTEL == 'true' }} uses: ./.github/workflows/reusable_build_and_test.yml with: @@ -113,40 +113,40 @@ jobs: secrets: inherit package_and_upload_linux: - name: 'Linux: Package Wheels' + name: 'Linux: Upload Wheels' needs: [build_linux] if: ${{ (github.event.inputs.BUILD_LINUX) == 'true' && (github.event.inputs.PACKAGE == 'true') }} - uses: ./.github/workflows/reusable_package_wheels.yml + uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: linux EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} secrets: inherit package_and_upload_windows: - name: 'Windows: Package Wheels' + name: 'Windows: Upload Wheels' needs: [build_linux, build_windows] if: ${{ (github.event.inputs.BUILD_WINDOWS) == 'true' && (github.event.inputs.PACKAGE == 'true') }} - uses: ./.github/workflows/reusable_package_wheels.yml + uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: windows EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} secrets: inherit package_and_upload_macos_arm: - name: 'Macos-Arm: Package Wheels' + name: 'Macos-Arm: Upload Wheels' needs: [build_linux, build_macos_arm] if: ${{ (github.event.inputs.BUILD_MACOS_ARM) == 'true' && (github.event.inputs.PACKAGE == 'true') }} - uses: ./.github/workflows/reusable_package_wheels.yml + uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: macos-arm EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} secrets: inherit package_and_upload_macos_intel: - name: 'Macos-Intel: Package Wheels' + name: 'Macos-Intel: Upload Wheels' needs: [build_linux, build_macos_intel] if: ${{ (github.event.inputs.BUILD_MACOS_INTEL) == 'true' && (github.event.inputs.PACKAGE == 'true') }} - uses: ./.github/workflows/reusable_package_wheels.yml + uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: macos-intel EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} diff --git a/.github/workflows/reusable_package_wheels.yml b/.github/workflows/reusable_upload_wheels.yml similarity index 96% rename from .github/workflows/reusable_package_wheels.yml rename to .github/workflows/reusable_upload_wheels.yml index 91577a3d01a2..7a5097bf72e7 100644 --- a/.github/workflows/reusable_package_wheels.yml +++ b/.github/workflows/reusable_upload_wheels.yml @@ -1,4 +1,4 @@ -name: Reusable Package Wheels +name: Reusable Upload Wheels on: workflow_call: @@ -16,8 +16,8 @@ on: jobs: - package_wheels: - name: Package Wheels + upload_wheels: + name: Upload Wheels to google cloud runs-on: ubuntu-latest From 03b98537927778dc946eb8997cad9e697c673bb3 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 00:24:03 +0200 Subject: [PATCH 036/106] More name cleanup --- .github/workflows/manual_dispatch.yml | 34 +++++++++---------- ...yml => reusable_build_and_test_wheels.yml} | 2 +- .github/workflows/reusable_checks.yml | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) rename .github/workflows/{reusable_build_and_test.yml => reusable_build_and_test_wheels.yml} (99%) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index c0c4fc8b44eb..36204ecf3740 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -40,8 +40,8 @@ on: type: boolean required: false default: false - PACKAGE: - description: 'Package' + UPLOAD_GCLOUD: + description: 'Upload to gcloud' type: boolean required: false default: false @@ -75,11 +75,11 @@ jobs: build_linux: name: 'Linux: Build/Test Wheels' - # If we are packaging any build target, we also need the linux job to run since it produces the RRD + # If we are uploading any build target, we also need the linux job to run since it produces the RRD # TODO(jleibs): Debug why multi-line if statements don't work here - if: ${{ (github.event.inputs.BUILD_LINUX == 'true') || ((github.event.inputs.PACKAGE == 'true') && ( (github.event.inputs.BUILD_MACOS_INTEL == 'true') || (github.event.inputs.BUILD_MACOS_ARM == 'true') || (github.event.inputs.BUILD_WINDOWS == 'true'))) }} + if: ${{ (github.event.inputs.BUILD_LINUX == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( (github.event.inputs.BUILD_MACOS_INTEL == 'true') || (github.event.inputs.BUILD_MACOS_ARM == 'true') || (github.event.inputs.BUILD_WINDOWS == 'true') || (github.event.inputs.BUILD_WEB == 'true'))) }} - uses: ./.github/workflows/reusable_build_and_test.yml + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux @@ -88,7 +88,7 @@ jobs: build_windows: name: 'Windows: Build/Test Wheels' if: ${{ github.event.inputs.BUILD_WINDOWS == 'true'}} - uses: ./.github/workflows/reusable_build_and_test.yml + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: windows @@ -97,7 +97,7 @@ jobs: build_macos_arm: name: 'Macos-Arm: Build/Test Wheels' if: ${{ github.event.inputs.BUILD_MACOS_ARM == 'true' }} - uses: ./.github/workflows/reusable_build_and_test.yml + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-arm @@ -106,46 +106,46 @@ jobs: build_macos_intel: name: 'Macos-Intel: Build/Test Wheels' if: ${{ github.event.inputs.BUILD_MACOS_INTEL == 'true' }} - uses: ./.github/workflows/reusable_build_and_test.yml + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-intel secrets: inherit - package_and_upload_linux: + upload_wheels_linux: name: 'Linux: Upload Wheels' needs: [build_linux] - if: ${{ (github.event.inputs.BUILD_LINUX) == 'true' && (github.event.inputs.PACKAGE == 'true') }} + if: ${{ (github.event.inputs.BUILD_LINUX) == 'true' && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: linux EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} secrets: inherit - package_and_upload_windows: + upload_wheels_windows: name: 'Windows: Upload Wheels' needs: [build_linux, build_windows] - if: ${{ (github.event.inputs.BUILD_WINDOWS) == 'true' && (github.event.inputs.PACKAGE == 'true') }} + if: ${{ (github.event.inputs.BUILD_WINDOWS) == 'true' && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: windows EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} secrets: inherit - package_and_upload_macos_arm: + upload_wheels_macos_arm: name: 'Macos-Arm: Upload Wheels' needs: [build_linux, build_macos_arm] - if: ${{ (github.event.inputs.BUILD_MACOS_ARM) == 'true' && (github.event.inputs.PACKAGE == 'true') }} + if: ${{ (github.event.inputs.BUILD_MACOS_ARM) == 'true' && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: macos-arm EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} secrets: inherit - package_and_upload_macos_intel: + upload_wheels_macos_intel: name: 'Macos-Intel: Upload Wheels' needs: [build_linux, build_macos_intel] - if: ${{ (github.event.inputs.BUILD_MACOS_INTEL) == 'true' && (github.event.inputs.PACKAGE == 'true') }} + if: ${{ (github.event.inputs.BUILD_MACOS_INTEL) == 'true' && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: macos-intel @@ -166,7 +166,7 @@ jobs: # build_linux: rrd # build_web: web_viewer needs: [build_linux, build_web] - if: ${{ github.event.inputs.BUILD_WEB == 'true'}} + if: ${{ (github.event.inputs.BUILD_WEB == 'true') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_web.yml with: RELEASE: ${{ github.event.inputs.RELEASE }} diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test_wheels.yml similarity index 99% rename from .github/workflows/reusable_build_and_test.yml rename to .github/workflows/reusable_build_and_test_wheels.yml index 0f42ef35dc41..e4656bbabda0 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -1,4 +1,4 @@ -name: Reusable Build and Test +name: Reusable Build and Test Wheels on: workflow_call: diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index 6dae62366f96..781a32ce3dfb 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -1,4 +1,4 @@ -name: All lints, checks, and tests +name: 'Checks: Lints, Tests, Docs' on: workflow_call: From 4213c8fc94935524723a0ea2239f37027c83ca8b Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 15:10:43 +0200 Subject: [PATCH 037/106] Introduce new faster job for e2e tests to use in CI --- .github/workflows/manual_dispatch.yml | 20 +- .../reusable_build_and_test_wheels.yml | 5 +- .github/workflows/reusable_min_test_wheel.yml | 238 ++++++++++++++++++ .github/workflows/reusable_upload_web.yml | 6 +- .github/workflows/reusable_upload_wheels.yml | 6 +- 5 files changed, 268 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/reusable_min_test_wheel.yml diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 36204ecf3740..ff31c0934325 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -10,6 +10,11 @@ on: type: boolean required: false default: true + MIN_TEST_WHEEL: + description: 'Test Minimum Wheel' + type: boolean + required: false + default: true BENCHES: description: 'Run Benches' type: boolean @@ -66,6 +71,14 @@ jobs: SAVE_DOCS_AS: ${{ github.event.inputs.SAVE_DOCS_AS }} secrets: inherit + min_test_wheel: + if: ${{ github.event.inputs.CHECKS == 'true' }} + uses: ./.github/workflows/reusable_min_test_wheel.yml + with: + SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} + PLATFORM: linux + secrets: inherit + benches: name: Benchmarks if: ${{ github.event.inputs.BENCHES == 'true' }} @@ -120,6 +133,7 @@ jobs: with: PLATFORM: linux EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} + RRD_ARTIFACT_NAME: "rrd-wheel-linux" secrets: inherit upload_wheels_windows: @@ -130,6 +144,7 @@ jobs: with: PLATFORM: windows EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} + RRD_ARTIFACT_NAME: "rrd-wheel-linux" secrets: inherit upload_wheels_macos_arm: @@ -140,6 +155,7 @@ jobs: with: PLATFORM: macos-arm EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} + RRD_ARTIFACT_NAME: "rrd-wheel-linux" secrets: inherit upload_wheels_macos_intel: @@ -150,6 +166,7 @@ jobs: with: PLATFORM: macos-intel EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} + RRD_ARTIFACT_NAME: "rrd-wheel-linux" secrets: inherit build_web: @@ -165,9 +182,10 @@ jobs: # Uses Assets: # build_linux: rrd # build_web: web_viewer - needs: [build_linux, build_web] + needs: [min_test_wheel, build_web] if: ${{ (github.event.inputs.BUILD_WEB == 'true') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_web.yml with: RELEASE: ${{ github.event.inputs.RELEASE }} + RRD_ARTIFACT_NAME: "rrd-fast-linux" secrets: inherit diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index e4656bbabda0..c51e0b2b25a9 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -298,7 +298,6 @@ jobs: run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - name: Cache RRD dataset - if: inputs.PLATFORM == 'linux' id: dataset uses: actions/cache@v3 with: @@ -307,7 +306,6 @@ jobs: key: colmap-dataset-colmap-fiat-v0 - name: Generate Embedded RRD file - if: inputs.PLATFORM == 'linux' shell: bash # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name run: | @@ -317,8 +315,7 @@ jobs: # All platforms are currently creating the same rrd file, upload one of them - name: Save RRD artifact - if: inputs.PLATFORM == 'linux' uses: actions/upload-artifact@v3 with: - name: rrd + name: rrd-wheels-${{ inputs.PLATFORM }} path: rrd diff --git a/.github/workflows/reusable_min_test_wheel.yml b/.github/workflows/reusable_min_test_wheel.yml new file mode 100644 index 000000000000..83f7c7cdb57a --- /dev/null +++ b/.github/workflows/reusable_min_test_wheel.yml @@ -0,0 +1,238 @@ +name: Build and Test a Minimum Wheel + +on: + workflow_call: + inputs: + PLATFORM: + required: true + type: string + RELEASE: + required: false + type: string + default: '' + SAVE_CACHE: + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + + # TODO(jleibs) --deny warnings causes installation of wasm-bindgen to fail on mac + # RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + RUSTFLAGS: --cfg=web_sys_unstable_apis + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + +# --------------------------------------------------------------------------- + + set_config: + name: Set Config + runs-on: ubuntu-latest + outputs: + RUNNER: ${{ steps.set_config.outputs.runner }} + TARGET: ${{ steps.set_config.outputs.target }} + RUN_TESTS: ${{ steps.set_config.outputs.run_tests }} + CONTAINER: ${{ steps.set_config.outputs.container }} + steps: + - name: Set runner and target based on platform + id: set_config + run: | + case "${{ inputs.platform }}" in + linux) + runner="ubuntu-latest" + target="x86_64-unknown-linux-gnu" + run_tests="true" + container="{'image': 'rerunio/ci_docker:0.6'}" + ;; + windows) + runner="windows-latest" + target="x86_64-pc-windows-msvc" + run_tests="true" + container="null" + ;; + macos-arm) + runner="macos-latest" + target="aarch64-apple-darwin" + run_tests="false" + container="null" + ;; + macos-intel) + runner="macos-latest" + target="x86_64-apple-darwin" + run_tests="false" + container="null" + ;; + *) echo "Invalid platform" && exit 1 ;; + esac + echo "runner=$runner" >> "$GITHUB_OUTPUT" + echo "target=$target" >> "$GITHUB_OUTPUT" + echo "run_tests=$run_tests" >> "$GITHUB_OUTPUT" + echo "container=$container" >> "$GITHUB_OUTPUT" + +# --------------------------------------------------------------------------- + + build_and_test: + name: Build and Test Minimum Wheel + + needs: [set_config] + + runs-on: ${{ needs.set_config.outputs.RUNNER }} + container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} + + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + + - uses: actions/checkout@v3 + + # Rust-cache will cache our dependencies, which is a large chunk of the build + # See: https://github.com/Swatinem/rust-cache + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-${{ inputs.PLATFORM }}" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: ${{ inputs.SAVE_CACHE }} + + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + # The pip-cache setup logic doesn't work in the ubuntu docker container + # That's probably fine since we bake these deps into the container already + - name: Setup python + if: ${{ inputs.PLATFORM != 'linux' }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" + + # These should already be in the docker container, but run for good measure. A no-op install + # should be fast, and this way things don't break if we add new packages without rebuilding + # docker + - run: pip install -r rerun_py/requirements-build.txt + + # We have a nice script for that: ./scripts/setup_web.sh + # Unfortunately, we can't run that on Windows, because Windows doesn't come with + # a package manager like grown-up OSes do (at least not the CI version of Windows). + # Also we can't run it on linux because the 20.04 Docker container will install + # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag + # So we only run the script on macos, and then on Windows we do the parts of the script manually. + # On ubuntu, the correct packages are pre-installed in our docker container. + + - name: Install prerequisites for building the web-viewer Wasm (non-Windows) + if: (inputs.PLATFORM == 'macos-intel') || (inputs.PLATFORM == 'macos-arm') + shell: bash + run: ./scripts/setup_web.sh + + # The first steps of setup_web.sh, for Windows: + - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows + if: inputs.platform == 'windows' + shell: bash + run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84 + + # The last step of setup_web.sh, for Windows. + # Since 'winget' is not available within the GitHub runner, we download the package directly: + # See: https://github.com/marketplace/actions/engineerd-configurator + - name: Install binaryen for building the web-viewer Wasm on windows + if: inputs.platform == 'windows' + uses: engineerd/configurator@v0.0.9 + with: + name: "wasm-opt.exe" + url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz" + pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" + + - name: Patch Cargo.toml for pre-release + if: ${{ (inputs.RELEASE == '') || (inputs.RELEASE == 'prerelease') }} + # After patching the pre-release version, run cargo update. + # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing + run: | + python3 scripts/version_util.py --patch_prerelease + cargo update -w + + # TODO(jleibs): Figure out what this check should be doing now + - name: Version check for tagged-release + if: false + #if: startsWith(github.ref, 'refs/tags/v') + # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z + run: | + python3 scripts/version_util.py --check_version + + - name: Store the expected version + # Find the current cargo version and store it in the output: `expected_version` + shell: bash + id: expected_version + run: | + echo "EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" + + # Build and install a stripped down wheel -- no native or web viewer + - name: Build and Install Wheel + uses: PyO3/maturin-action@v1 + with: + maturin-version: "0.14.10" + manylinux: manylinux_2_31 + container: off + command: develop + sccache: 'true' + args: | + --manifest-path rerun_py/Cargo.toml + --release + --target ${{ needs.set_config.outputs.TARGET }} + --no-default-features + + - name: Run unit tests + shell: bash + run: cd rerun_py/tests && pytest + + - name: Run e2e test + shell: bash + run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed + + - name: Cache RRD dataset + if: inputs.PLATFORM == 'linux' + id: dataset + uses: actions/cache@v3 + with: + path: examples/python/colmap/dataset/ + # TODO(jleibs): Derive this key from the invocation below + key: colmap-dataset-colmap-fiat-v0 + + - name: Generate the RRD file + shell: bash + # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name + run: | + mkdir rrd + pip install -r examples/python/colmap/requirements.txt + python3 examples/python/colmap/main.py --dataset colmap_fiat --resize 800x600 --save rrd/colmap_fiat.rrd + + # All platforms are currently creating the same rrd file, upload one of them + - name: Save RRD artifact + uses: actions/upload-artifact@v3 + with: + name: rrd-fast-${{ inputs.PLATFORM }} + path: rrd diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index ef41f729c848..c4633d47f335 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -7,6 +7,10 @@ on: required: false type: string default: '' + RRD_ARTIFACT_NAME: + required: false + type: string + default: '' jobs: @@ -24,7 +28,7 @@ jobs: - name: Download RRD uses: actions/download-artifact@v3 with: - name: rrd + name: ${{ inputs.RRD_ARTIFACT_NAME }} path: rrd - name: Download Web Viewer diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml index 7a5097bf72e7..ba54341688b4 100644 --- a/.github/workflows/reusable_upload_wheels.yml +++ b/.github/workflows/reusable_upload_wheels.yml @@ -13,6 +13,10 @@ on: required: false type: string default: '' + RRD_ARTIFACT_NAME: + required: false + type: string + default: '' jobs: @@ -71,7 +75,7 @@ jobs: - name: Download RRD uses: actions/download-artifact@v3 with: - name: rrd + name: ${{inputs.RRD_ARTIFACT_NAME}} path: rrd - name: Insert the rrd From 4f8e4b76c4ddcffb80625c26125fe186908bf874 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 15:29:22 +0200 Subject: [PATCH 038/106] Refactor args to stay under 10 --- .github/workflows/manual_dispatch.yml | 51 ++++++++++++--------------- .github/workflows/reusable_checks.yml | 26 +++++++++----- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index ff31c0934325..43f9f22fa807 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -20,26 +20,11 @@ on: type: boolean required: false default: false - BUILD_LINUX: - description: 'Build/Test Wheels Linux' - type: boolean - required: false - default: true - BUILD_MACOS_INTEL: - description: 'Build/Test Wheels Mac (Intel)' - type: boolean - required: false - default: false - BUILD_MACOS_ARM: - description: 'Build/Test Wheels Mac (Arm)' - type: boolean - required: false - default: false - BUILD_WINDOWS: - description: 'Build/Test Wheels Win' - type: boolean + WHEEL_PLATFORMS: + description: 'Platforms to build wheels for' + type: string required: false - default: false + default: 'linux' BUILD_WEB: description: 'Build Web' type: boolean @@ -60,6 +45,11 @@ on: type: boolean required: false default: false + SAVE_DOCS: + description: 'Save Docs to refs.rerun.io' + type: boolean + required: false + default: false jobs: checks: @@ -68,11 +58,14 @@ jobs: uses: ./.github/workflows/reusable_checks.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} - SAVE_DOCS_AS: ${{ github.event.inputs.SAVE_DOCS_AS }} + SAVE_PY_DOCS: ${{ github.event.inputs.SAVE_DOCS == 'true' }} + SAVE_RUST_DOCS: ${{ github.event.inputs.SAVE_DOCS == 'true' }} + SAVE_PY_DOCS_AS: ${{ github.event.inputs.RELEASE }} secrets: inherit min_test_wheel: - if: ${{ github.event.inputs.CHECKS == 'true' }} + # The upload-web job uses the min-test-wheel to get the RRD in manual mode + if: ${{ (github.event.inputs.MIN_TEST_WHEEL == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.BUILD_WEB == 'true') ) }} uses: ./.github/workflows/reusable_min_test_wheel.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} @@ -90,7 +83,7 @@ jobs: # If we are uploading any build target, we also need the linux job to run since it produces the RRD # TODO(jleibs): Debug why multi-line if statements don't work here - if: ${{ (github.event.inputs.BUILD_LINUX == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( (github.event.inputs.BUILD_MACOS_INTEL == 'true') || (github.event.inputs.BUILD_MACOS_ARM == 'true') || (github.event.inputs.BUILD_WINDOWS == 'true') || (github.event.inputs.BUILD_WEB == 'true'))) }} + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.WHEEL_PLATFORMS != '') ) }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: @@ -100,7 +93,7 @@ jobs: build_windows: name: 'Windows: Build/Test Wheels' - if: ${{ github.event.inputs.BUILD_WINDOWS == 'true'}} + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} @@ -109,7 +102,7 @@ jobs: build_macos_arm: name: 'Macos-Arm: Build/Test Wheels' - if: ${{ github.event.inputs.BUILD_MACOS_ARM == 'true' }} + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} @@ -118,7 +111,7 @@ jobs: build_macos_intel: name: 'Macos-Intel: Build/Test Wheels' - if: ${{ github.event.inputs.BUILD_MACOS_INTEL == 'true' }} + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} @@ -128,7 +121,7 @@ jobs: upload_wheels_linux: name: 'Linux: Upload Wheels' needs: [build_linux] - if: ${{ (github.event.inputs.BUILD_LINUX) == 'true' && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: linux @@ -139,7 +132,7 @@ jobs: upload_wheels_windows: name: 'Windows: Upload Wheels' needs: [build_linux, build_windows] - if: ${{ (github.event.inputs.BUILD_WINDOWS) == 'true' && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: windows @@ -150,7 +143,7 @@ jobs: upload_wheels_macos_arm: name: 'Macos-Arm: Upload Wheels' needs: [build_linux, build_macos_arm] - if: ${{ (github.event.inputs.BUILD_MACOS_ARM) == 'true' && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: macos-arm @@ -161,7 +154,7 @@ jobs: upload_wheels_macos_intel: name: 'Macos-Intel: Upload Wheels' needs: [build_linux, build_macos_intel] - if: ${{ (github.event.inputs.BUILD_MACOS_INTEL) == 'true' && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: PLATFORM: macos-intel diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index 781a32ce3dfb..d1cb1e8858c8 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -7,10 +7,18 @@ on: required: false type: boolean default: false - SAVE_DOCS_AS: + SAVE_PY_DOCS: + required: false + type: boolean + default: false + SAVE_PY_DOCS_AS: required: false type: string default: "" + SAVE_RUST_DOCS: + required: false + type: boolean + default: false env: PYTHON_VERSION: "3.8" @@ -91,7 +99,7 @@ jobs: py-deploy-docs: needs: [py-test-docs] name: Deploy Python Docs - if: ${{ inputs.SAVE_DOCS_AS != '' }} + if: ${{ inputs.SAVE_PY_DOCS }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -122,10 +130,10 @@ jobs: # We then check it out, and reset it to a new orphaned branch, which we force-push to origin # to make sure we don't accumulate unnecessary history in gh-pages branch - name: Deploy via mike # https://github.com/jimporter/mike - if: ${{ inputs.SAVE_DOCS_AS != 'HEAD' }} + if: ${{ inputs.SAVE_PY_DOCS_AS != 'HEAD' }} run: | git fetch - mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.SAVE_DOCS_AS}} latest + mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.SAVE_PY_DOCS_AS}} latest git checkout gh-pages git checkout --orphan gh-pages-orphan git commit -m "Update docs for ${GITHUB_SHA}" @@ -135,7 +143,7 @@ jobs: # We then check it out, and reset it to a new orphaned branch, which we force-push to origin # to make sure we don't accumulate unnecessary history in gh-pages branch - name: Deploy tag via mike # https://github.com/jimporter/mike - if: ${{ inputs.SAVE_DOCS_AS == 'HEAD' }} + if: ${{ inputs.SAVE_PY_DOCS_AS == 'HEAD' }} run: | git fetch mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python HEAD @@ -245,7 +253,7 @@ jobs: # TODO(jleibs): Maybe we should run this on a separate job? - name: Set up git author - if: ${{ inputs.SAVE_DOCS_AS != '' }} + if: ${{ inputs.SAVE_RUST_DOCS }} run: | remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" git config --global user.name "${GITHUB_ACTOR}" @@ -254,18 +262,18 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Set up ghp-import - if: ${{ inputs.SAVE_DOCS_AS != '' }} + if: ${{ inputs.SAVE_RUST_DOCS }} run: pip install ghp-import - name: Patch in a redirect page - if: ${{ inputs.SAVE_DOCS_AS != '' }} + if: ${{ inputs.SAVE_RUST_DOCS }} run: echo "" > target/doc/index.html env: REDIRECT_CRATE: rerun # See: https://github.com/c-w/ghp-import - name: Deploy the docs - if: ${{ inputs.SAVE_DOCS_AS != '' }} + if: ${{ inputs.SAVE_RUST_DOCS }} run: | git fetch python3 -m ghp_import -n -p -x docs/rust/head target/doc/ -m "Update the rust docs" From 934ea005de2ab134474e9d85adb6ac4cabc8fb74 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 15:37:14 +0200 Subject: [PATCH 039/106] Can't use develop in our environment so use build / pip install --- .github/workflows/reusable_min_test_wheel.yml | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_min_test_wheel.yml b/.github/workflows/reusable_min_test_wheel.yml index 83f7c7cdb57a..320e59a7325e 100644 --- a/.github/workflows/reusable_min_test_wheel.yml +++ b/.github/workflows/reusable_min_test_wheel.yml @@ -197,13 +197,31 @@ jobs: maturin-version: "0.14.10" manylinux: manylinux_2_31 container: off - command: develop + command: build sccache: 'true' args: | --manifest-path rerun_py/Cargo.toml - --release --target ${{ needs.set_config.outputs.TARGET }} --no-default-features + --out dist + + - name: Install wheel dependencies + # First we install the dependencies manually so we can use `--no-index` when installing the wheel. + # This needs to be a separate step for some reason or the following step fails + # TODO(jleibs): pull these deps from pyproject.toml + # TODO(jleibs): understand why deps can't be installed in the same step as the wheel + shell: bash + run: | + pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 + + - name: Install built wheel + # Now install the wheel using a specific version and --no-index to guarantee we get the version from + # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't + # find the dependencies to reinstall them. + shell: bash + run: | + pip uninstall rerun-sdk + pip install rerun-sdk==${{ steps.expected_version.outputs.EXPECTED_VERSION }} --no-index --find-links dist - name: Run unit tests shell: bash From 6f2b0bb9c401e3f513b55fc4b05c167bf806f6d3 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 15:53:07 +0200 Subject: [PATCH 040/106] Use the same build-and-test-wheels for the fast job --- .github/workflows/manual_dispatch.yml | 16 +- .../reusable_build_and_test_wheels.yml | 74 ++--- .github/workflows/reusable_min_test_wheel.yml | 256 ------------------ 3 files changed, 32 insertions(+), 314 deletions(-) delete mode 100644 .github/workflows/reusable_min_test_wheel.yml diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 43f9f22fa807..0002778df5f3 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -66,10 +66,12 @@ jobs: min_test_wheel: # The upload-web job uses the min-test-wheel to get the RRD in manual mode if: ${{ (github.event.inputs.MIN_TEST_WHEEL == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.BUILD_WEB == 'true') ) }} - uses: ./.github/workflows/reusable_min_test_wheel.yml + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux + MATURIN_BUILD_ARGS: '--no-default-features' + RRD_ARTIFACT_NAME: "linux-rrd-fast" secrets: inherit benches: @@ -89,6 +91,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux + RRD_ARTIFACT_NAME: "linux-rrd" secrets: inherit build_windows: @@ -98,6 +101,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: windows + RRD_ARTIFACT_NAME: '' # All packages use the linux-rrd secrets: inherit build_macos_arm: @@ -107,6 +111,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-arm + RRD_ARTIFACT_NAME: '' # All packages use the linux-rrd secrets: inherit build_macos_intel: @@ -116,6 +121,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-intel + RRD_ARTIFACT_NAME: '' # All packages use the linux-rrd secrets: inherit upload_wheels_linux: @@ -126,7 +132,7 @@ jobs: with: PLATFORM: linux EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} - RRD_ARTIFACT_NAME: "rrd-wheel-linux" + RRD_ARTIFACT_NAME: '' # All packages use the linux-rrd secrets: inherit upload_wheels_windows: @@ -137,7 +143,7 @@ jobs: with: PLATFORM: windows EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} - RRD_ARTIFACT_NAME: "rrd-wheel-linux" + RRD_ARTIFACT_NAME: "linux-rrd" secrets: inherit upload_wheels_macos_arm: @@ -148,7 +154,7 @@ jobs: with: PLATFORM: macos-arm EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} - RRD_ARTIFACT_NAME: "rrd-wheel-linux" + RRD_ARTIFACT_NAME: "linux-rrd" secrets: inherit upload_wheels_macos_intel: @@ -159,7 +165,7 @@ jobs: with: PLATFORM: macos-intel EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} - RRD_ARTIFACT_NAME: "rrd-wheel-linux" + RRD_ARTIFACT_NAME: "linux-rrd" secrets: inherit build_web: diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index c51e0b2b25a9..eb515aa2a3a2 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -3,6 +3,10 @@ name: Reusable Build and Test Wheels on: workflow_call: inputs: + MATURIN_BUILD_ARGS: + required: false + type: string + default: '--release --no-default-features --features pypi' PLATFORM: required: true type: string @@ -10,6 +14,10 @@ on: required: false type: string default: '' + RRD_ARTIFACT_NAME: + required: false + type: string + default: '' SAVE_CACHE: required: false type: boolean @@ -210,55 +218,14 @@ jobs: --manifest-path rerun_py/Cargo.toml --release --target ${{ needs.set_config.outputs.TARGET }} - --no-default-features - --features pypi + ${{ inputs.MATURIN_BUILD_ARGS }} --out dist - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: ${{inputs.PLATFORM}}-wheel - path: dist - -# --------------------------------------------------------------------------- - - run_tests: - name: Test Wheels - - needs: [set_config, build_wheels] - - runs-on: ${{ needs.set_config.outputs.RUNNER }} - container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} - - # Note string instead of boolean since it's a job output - if: needs.set_config.outputs.RUN_TESTS == 'true' - - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - echo "EXPECTED_VERSION": $EXPECTED_VERSION - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - EXPECTED_VERSION: ${{ needs.build_wheels.outputs.EXPECTED_VERSION }} - - - uses: actions/checkout@v3 - - - name: Setup python - if: ${{ inputs.PLATFORM != 'linux' }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "rerun_py/requirements-build.txt" + # --------------------------------------------------------------------------- + # Test the wheel - name: Install wheel dependencies + if: needs.set_config.outputs.RUN_TESTS == 'true' # First we install the dependencies manually so we can use `--no-index` when installing the wheel. # This needs to be a separate step for some reason or the following step fails # TODO(jleibs): pull these deps from pyproject.toml @@ -267,12 +234,6 @@ jobs: run: | pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 - - name: Download Artifact - uses: actions/download-artifact@v3 - with: - name: ${{inputs.PLATFORM}}-wheel - path: dist - - name: Install built wheel # Now install the wheel using a specific version and --no-index to guarantee we get the version from # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't @@ -280,7 +241,7 @@ jobs: shell: bash run: | pip uninstall rerun-sdk - pip install rerun-sdk==${{ needs.build_wheels.outputs.EXPECTED_VERSION }} --no-index --find-links dist + pip install rerun-sdk==${{ steps.expected_version.outputs.EXPECTED_VERSION }} --no-index --find-links dist - name: Verify built wheel version shell: bash @@ -315,7 +276,14 @@ jobs: # All platforms are currently creating the same rrd file, upload one of them - name: Save RRD artifact + if: ${{ inputs.RRD_ARTIFACT_NAME != '' }} uses: actions/upload-artifact@v3 with: - name: rrd-wheels-${{ inputs.PLATFORM }} + name: ${{ inputs.RRD_ARTIFACT_NAME }} path: rrd + + - name: Save wheel artifact + uses: actions/upload-artifact@v3 + with: + name: ${{inputs.PLATFORM}}-wheel + path: dist diff --git a/.github/workflows/reusable_min_test_wheel.yml b/.github/workflows/reusable_min_test_wheel.yml deleted file mode 100644 index 320e59a7325e..000000000000 --- a/.github/workflows/reusable_min_test_wheel.yml +++ /dev/null @@ -1,256 +0,0 @@ -name: Build and Test a Minimum Wheel - -on: - workflow_call: - inputs: - PLATFORM: - required: true - type: string - RELEASE: - required: false - type: string - default: '' - SAVE_CACHE: - required: false - type: boolean - default: false - -env: - PYTHON_VERSION: "3.8" - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - - # TODO(jleibs) --deny warnings causes installation of wasm-bindgen to fail on mac - # RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - RUSTFLAGS: --cfg=web_sys_unstable_apis - - # See https://github.com/ericseppanen/cargo-cranky/issues/8 - RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs - - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "true" - RUSTC_WRAPPER: "sccache" - -jobs: - -# --------------------------------------------------------------------------- - - set_config: - name: Set Config - runs-on: ubuntu-latest - outputs: - RUNNER: ${{ steps.set_config.outputs.runner }} - TARGET: ${{ steps.set_config.outputs.target }} - RUN_TESTS: ${{ steps.set_config.outputs.run_tests }} - CONTAINER: ${{ steps.set_config.outputs.container }} - steps: - - name: Set runner and target based on platform - id: set_config - run: | - case "${{ inputs.platform }}" in - linux) - runner="ubuntu-latest" - target="x86_64-unknown-linux-gnu" - run_tests="true" - container="{'image': 'rerunio/ci_docker:0.6'}" - ;; - windows) - runner="windows-latest" - target="x86_64-pc-windows-msvc" - run_tests="true" - container="null" - ;; - macos-arm) - runner="macos-latest" - target="aarch64-apple-darwin" - run_tests="false" - container="null" - ;; - macos-intel) - runner="macos-latest" - target="x86_64-apple-darwin" - run_tests="false" - container="null" - ;; - *) echo "Invalid platform" && exit 1 ;; - esac - echo "runner=$runner" >> "$GITHUB_OUTPUT" - echo "target=$target" >> "$GITHUB_OUTPUT" - echo "run_tests=$run_tests" >> "$GITHUB_OUTPUT" - echo "container=$container" >> "$GITHUB_OUTPUT" - -# --------------------------------------------------------------------------- - - build_and_test: - name: Build and Test Minimum Wheel - - needs: [set_config] - - runs-on: ${{ needs.set_config.outputs.RUNNER }} - container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} - - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - - - uses: actions/checkout@v3 - - # Rust-cache will cache our dependencies, which is a large chunk of the build - # See: https://github.com/Swatinem/rust-cache - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - shared-key: "build-${{ inputs.PLATFORM }}" - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # Don't update the cache -- it will be updated by the lint job - # TODO(jleibs): this job will likely run before rust.yml updates - # the cache. Better cross-job sequencing would be nice here - save-if: ${{ inputs.SAVE_CACHE }} - - # Sccache will cache everything else - # See: https://github.com/marketplace/actions/sccache-action - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 - - # The pip-cache setup logic doesn't work in the ubuntu docker container - # That's probably fine since we bake these deps into the container already - - name: Setup python - if: ${{ inputs.PLATFORM != 'linux' }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "rerun_py/requirements-build.txt" - - # These should already be in the docker container, but run for good measure. A no-op install - # should be fast, and this way things don't break if we add new packages without rebuilding - # docker - - run: pip install -r rerun_py/requirements-build.txt - - # We have a nice script for that: ./scripts/setup_web.sh - # Unfortunately, we can't run that on Windows, because Windows doesn't come with - # a package manager like grown-up OSes do (at least not the CI version of Windows). - # Also we can't run it on linux because the 20.04 Docker container will install - # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag - # So we only run the script on macos, and then on Windows we do the parts of the script manually. - # On ubuntu, the correct packages are pre-installed in our docker container. - - - name: Install prerequisites for building the web-viewer Wasm (non-Windows) - if: (inputs.PLATFORM == 'macos-intel') || (inputs.PLATFORM == 'macos-arm') - shell: bash - run: ./scripts/setup_web.sh - - # The first steps of setup_web.sh, for Windows: - - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows - if: inputs.platform == 'windows' - shell: bash - run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84 - - # The last step of setup_web.sh, for Windows. - # Since 'winget' is not available within the GitHub runner, we download the package directly: - # See: https://github.com/marketplace/actions/engineerd-configurator - - name: Install binaryen for building the web-viewer Wasm on windows - if: inputs.platform == 'windows' - uses: engineerd/configurator@v0.0.9 - with: - name: "wasm-opt.exe" - url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz" - pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" - - - name: Patch Cargo.toml for pre-release - if: ${{ (inputs.RELEASE == '') || (inputs.RELEASE == 'prerelease') }} - # After patching the pre-release version, run cargo update. - # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing - run: | - python3 scripts/version_util.py --patch_prerelease - cargo update -w - - # TODO(jleibs): Figure out what this check should be doing now - - name: Version check for tagged-release - if: false - #if: startsWith(github.ref, 'refs/tags/v') - # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z - run: | - python3 scripts/version_util.py --check_version - - - name: Store the expected version - # Find the current cargo version and store it in the output: `expected_version` - shell: bash - id: expected_version - run: | - echo "EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" - - # Build and install a stripped down wheel -- no native or web viewer - - name: Build and Install Wheel - uses: PyO3/maturin-action@v1 - with: - maturin-version: "0.14.10" - manylinux: manylinux_2_31 - container: off - command: build - sccache: 'true' - args: | - --manifest-path rerun_py/Cargo.toml - --target ${{ needs.set_config.outputs.TARGET }} - --no-default-features - --out dist - - - name: Install wheel dependencies - # First we install the dependencies manually so we can use `--no-index` when installing the wheel. - # This needs to be a separate step for some reason or the following step fails - # TODO(jleibs): pull these deps from pyproject.toml - # TODO(jleibs): understand why deps can't be installed in the same step as the wheel - shell: bash - run: | - pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 - - - name: Install built wheel - # Now install the wheel using a specific version and --no-index to guarantee we get the version from - # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't - # find the dependencies to reinstall them. - shell: bash - run: | - pip uninstall rerun-sdk - pip install rerun-sdk==${{ steps.expected_version.outputs.EXPECTED_VERSION }} --no-index --find-links dist - - - name: Run unit tests - shell: bash - run: cd rerun_py/tests && pytest - - - name: Run e2e test - shell: bash - run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - - - name: Cache RRD dataset - if: inputs.PLATFORM == 'linux' - id: dataset - uses: actions/cache@v3 - with: - path: examples/python/colmap/dataset/ - # TODO(jleibs): Derive this key from the invocation below - key: colmap-dataset-colmap-fiat-v0 - - - name: Generate the RRD file - shell: bash - # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name - run: | - mkdir rrd - pip install -r examples/python/colmap/requirements.txt - python3 examples/python/colmap/main.py --dataset colmap_fiat --resize 800x600 --save rrd/colmap_fiat.rrd - - # All platforms are currently creating the same rrd file, upload one of them - - name: Save RRD artifact - uses: actions/upload-artifact@v3 - with: - name: rrd-fast-${{ inputs.PLATFORM }} - path: rrd From 8fdf39f40da379bbcd739b22a000e8912fb52dc1 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 15:59:00 +0200 Subject: [PATCH 041/106] More parameterization --- .github/workflows/manual_dispatch.yml | 33 +++++++++++-------- .../reusable_build_and_test_wheels.yml | 7 +++- .github/workflows/reusable_upload_wheels.yml | 9 ++--- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 0002778df5f3..fb173f1882af 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -71,7 +71,8 @@ jobs: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux MATURIN_BUILD_ARGS: '--no-default-features' - RRD_ARTIFACT_NAME: "linux-rrd-fast" + WHEEL_ARTIFACT_NAME: '' + RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit benches: @@ -91,7 +92,8 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux - RRD_ARTIFACT_NAME: "linux-rrd" + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd secrets: inherit build_windows: @@ -101,7 +103,8 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: windows - RRD_ARTIFACT_NAME: '' # All packages use the linux-rrd + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: '' secrets: inherit build_macos_arm: @@ -111,7 +114,8 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-arm - RRD_ARTIFACT_NAME: '' # All packages use the linux-rrd + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: '' secrets: inherit build_macos_intel: @@ -121,7 +125,8 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-intel - RRD_ARTIFACT_NAME: '' # All packages use the linux-rrd + WHEEL_ARTIFACT_NAME: 'macos-intel-wheel' + RRD_ARTIFACT_NAME: '' secrets: inherit upload_wheels_linux: @@ -130,9 +135,9 @@ jobs: if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: - PLATFORM: linux EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} - RRD_ARTIFACT_NAME: '' # All packages use the linux-rrd + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd secrets: inherit upload_wheels_windows: @@ -141,9 +146,9 @@ jobs: if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: - PLATFORM: windows EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} - RRD_ARTIFACT_NAME: "linux-rrd" + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: linux-rrd secrets: inherit upload_wheels_macos_arm: @@ -152,9 +157,9 @@ jobs: if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: - PLATFORM: macos-arm EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} - RRD_ARTIFACT_NAME: "linux-rrd" + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: linux-rrd secrets: inherit upload_wheels_macos_intel: @@ -163,9 +168,9 @@ jobs: if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: - PLATFORM: macos-intel EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} - RRD_ARTIFACT_NAME: "linux-rrd" + WHEEL_ARTIFACT_NAME: macos-intel-wheel + RRD_ARTIFACT_NAME: linux-rrd secrets: inherit build_web: @@ -186,5 +191,5 @@ jobs: uses: ./.github/workflows/reusable_upload_web.yml with: RELEASE: ${{ github.event.inputs.RELEASE }} - RRD_ARTIFACT_NAME: "rrd-fast-linux" + RRD_ARTIFACT_NAME: rlinux-rrd-fast secrets: inherit diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index eb515aa2a3a2..fc48716a4af7 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -18,6 +18,10 @@ on: required: false type: string default: '' + WHEEL_ARTIFACT_NAME: + required: false + type: string + default: '' SAVE_CACHE: required: false type: boolean @@ -283,7 +287,8 @@ jobs: path: rrd - name: Save wheel artifact + if: ${{ inputs.WHEEL_ARTIFACT_NAME != '' }} uses: actions/upload-artifact@v3 with: - name: ${{inputs.PLATFORM}}-wheel + name: ${{inputs.WHEEL_ARTIFACT_NAME}} path: dist diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml index ba54341688b4..9bb090018e17 100644 --- a/.github/workflows/reusable_upload_wheels.yml +++ b/.github/workflows/reusable_upload_wheels.yml @@ -6,9 +6,6 @@ on: EXPECTED_VERSION: required: true type: string - PLATFORM: - required: true - type: string RELEASE: required: false type: string @@ -17,6 +14,10 @@ on: required: false type: string default: '' + WHEEL_ARTIFACT_NAME: + required: false + type: string + default: '' jobs: @@ -57,7 +58,7 @@ jobs: - name: Download Wheel uses: actions/download-artifact@v3 with: - name: ${{inputs.PLATFORM}}-wheel + name: ${{inputs.WHEEL_ARTIFACT_NAME}} path: pre-dist - name: Unpack the wheel From e1f17cc12753d57122f0c3e2556b9c5304c2d571 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 16:05:30 +0200 Subject: [PATCH 042/106] Rename maturin arg --- .github/workflows/manual_dispatch.yml | 2 +- .github/workflows/reusable_build_and_test_wheels.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index fb173f1882af..563a1434c661 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -70,7 +70,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux - MATURIN_BUILD_ARGS: '--no-default-features' + MATURIN_FEATURE_FLAGS: '--no-default-features' WHEEL_ARTIFACT_NAME: '' RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index fc48716a4af7..8633b70cd5ea 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -3,10 +3,10 @@ name: Reusable Build and Test Wheels on: workflow_call: inputs: - MATURIN_BUILD_ARGS: + MATURIN_FEATURE_FLAGS: required: false type: string - default: '--release --no-default-features --features pypi' + default: '--no-default-features --features pypi' PLATFORM: required: true type: string @@ -222,7 +222,7 @@ jobs: --manifest-path rerun_py/Cargo.toml --release --target ${{ needs.set_config.outputs.TARGET }} - ${{ inputs.MATURIN_BUILD_ARGS }} + ${{ inputs.MATURIN_FEATURE_FLAGS }} --out dist # --------------------------------------------------------------------------- From 0c5f9fd3c1d2f8f84dd7c083b07cbb25ade866d7 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 16:08:13 +0200 Subject: [PATCH 043/106] Still enable extension-module feature --- .github/workflows/manual_dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 563a1434c661..2cf054aab837 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -70,7 +70,7 @@ jobs: with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux - MATURIN_FEATURE_FLAGS: '--no-default-features' + MATURIN_FEATURE_FLAGS: '--no-default-features --features extension-module' WHEEL_ARTIFACT_NAME: '' RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit From 72ef83386a15b46aa506880dfb749d3c37b436d1 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 16:10:19 +0200 Subject: [PATCH 044/106] Default wheel platforms to none --- .github/workflows/manual_dispatch.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 2cf054aab837..e0e1f6aa538f 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -24,7 +24,7 @@ on: description: 'Platforms to build wheels for' type: string required: false - default: 'linux' + default: '' BUILD_WEB: description: 'Build Web' type: boolean @@ -64,6 +64,7 @@ jobs: secrets: inherit min_test_wheel: + name: 'Minimum Test Wheel' # The upload-web job uses the min-test-wheel to get the RRD in manual mode if: ${{ (github.event.inputs.MIN_TEST_WHEEL == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.BUILD_WEB == 'true') ) }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml From bb283273e376f287c346b3a864103f16e23240af Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 17:17:36 +0200 Subject: [PATCH 045/106] Fix typo in web artifact --- .github/workflows/manual_dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index e0e1f6aa538f..5d04d9d2c4a2 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -192,5 +192,5 @@ jobs: uses: ./.github/workflows/reusable_upload_web.yml with: RELEASE: ${{ github.event.inputs.RELEASE }} - RRD_ARTIFACT_NAME: rlinux-rrd-fast + RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit From a9b69d96093b5f6056601d2441e5f9ac341724be Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 17:26:33 +0200 Subject: [PATCH 046/106] Rename packages -> builds --- .github/workflows/reusable_upload_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml index 9bb090018e17..dd3c4fa30ea5 100644 --- a/.github/workflows/reusable_upload_wheels.yml +++ b/.github/workflows/reusable_upload_wheels.yml @@ -105,5 +105,5 @@ jobs: uses: google-github-actions/upload-cloud-storage@v1 with: path: "dist" - destination: "rerun-packages/commit/${{env.SHORT_SHA}}/wheels" + destination: "rerun-builds/commit/${{env.SHORT_SHA}}/wheels" parent: false From 90421a648d35a68194486fcc4882dc9f019f17f9 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 17:51:31 +0200 Subject: [PATCH 047/106] New plumbing for PR summary page --- .github/workflows/manual_dispatch.yml | 50 ++++++++++------ .github/workflows/reusable_pr_summary.yml | 69 +++++++++++++++++++++++ .github/workflows/reusable_upload_web.yml | 14 ++++- templates/pr_results_summary.html | 24 ++++++++ 4 files changed, 138 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/reusable_pr_summary.yml create mode 100644 templates/pr_results_summary.html diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 5d04d9d2c4a2..05f0920e670a 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -5,36 +5,25 @@ on: # NOTE: boolean inputs are still actually strings # See: https://github.com/actions/runner/issues/1483 inputs: - CHECKS: - description: 'Run All Checks' - type: boolean - required: false - default: true - MIN_TEST_WHEEL: - description: 'Test Minimum Wheel' - type: boolean - required: false - default: true BENCHES: description: 'Run Benches' type: boolean required: false default: false - WHEEL_PLATFORMS: - description: 'Platforms to build wheels for' - type: string - required: false - default: '' BUILD_WEB: description: 'Build Web' type: boolean required: false default: false - UPLOAD_GCLOUD: - description: 'Upload to gcloud' + CHECKS: + description: 'Run All Checks' + type: boolean + required: false + default: true + MIN_TEST_WHEEL: + description: 'Test Minimum Wheel' type: boolean required: false - default: false RELEASE: description: 'Release' type: string @@ -50,6 +39,21 @@ on: type: boolean required: false default: false + SAVE_PR: + description: 'Save PR Summary' + type: string + required: false + default: '' + UPLOAD_GCLOUD: + description: 'Upload to gcloud' + type: boolean + required: false + default: false + WHEEL_PLATFORMS: + description: 'Platforms to build wheels for' + type: string + required: false + default: '' jobs: checks: @@ -194,3 +198,13 @@ jobs: RELEASE: ${{ github.event.inputs.RELEASE }} RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit + + save_pr_summary: + name: 'Save PR Summary' + needs: [upload_web] + if: ${{ github.event.inputs.SAVE_PR != '' }} + uses: ./.github/workflows/reusable_pr_summary.yml + with: + PR_NUMBER: ${{ github.event.inputs.SAVE_PR }} + HOSTED_APP_URL: ${{ needs.upload_web.outputs.HOSTED_APP_URL }} + secrets: inherit diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml new file mode 100644 index 000000000000..ebcb1f938b07 --- /dev/null +++ b/.github/workflows/reusable_pr_summary.yml @@ -0,0 +1,69 @@ +name: Reusable PR Summary + +on: + workflow_call: + inputs: + PR_NUMBER: + required: true + type: string + HOSTED_APP_URL: + required: false + type: string + default: '' + LINUX_WHEEL_URL: + required: false + type: string + default: '' + WINDOWS_WHEEL_URL: + required: false + type: string + default: '' + MACOS_ARM_WHEEL_URL: + required: false + type: string + default: '' + MACOS_INTEL_WHEEL_URL: + required: false + type: string + default: '' + +jobs: + create_html_summary: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install jinja-cli + run: pip install jinja2-cli + + - name: Render HTML template + run: | + jinja2 \ + --format=json \ + --var="pr_number=${{ inputs.PR_NUMBER }}" \ + --var="commit=${{ github.sha }}" \ + --var="hosted_app_url=${{ inputs.HOSTED_APP_URL }}" \ + --var="linux_wheel_url=${{ inputs.LINUX_WHEEL_URL }}" \ + --var="windows_wheel_url=${{ inputs.WINDOWS_WHEEL_URL }}" \ + --var="macos_arm_wheel_url=${{ inputs.MACOS_ARM_WHEEL_URL }}" \ + --var="macos_intel_wheel_url=${{ inputs.MACOS_INTEL_WHEEL_URL }}" \ + html_template.html \ + > build_summary.html + + # Replace the following with your actual artifact upload implementation. + - name: Upload HTML summary + run: | + echo "Uploading build_summary.html to rerun-builds/commit/${{ github.sha }}/wheels" + # Implement the upload logic here + + - name: "Upload Wheel" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "build_summary.html" + destination: "rerun-builds/pull_request/${{inputs.PR_NUMBER}}/" diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index c4633d47f335..5c49e5e1c393 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -11,10 +11,14 @@ on: required: false type: string default: '' + outputs: + HOSTED_APP_URL: + description: "The URL where the app will be hosted" + value: ${{ jobs.upload_web.outputs.HOSTED_APP_URL }} jobs: - package_web: + upload_web: name: Upload web build to google cloud (wasm32 + wasm-bindgen) permissions: contents: "read" @@ -22,6 +26,9 @@ jobs: runs-on: ubuntu-latest + outputs: + HOSTED_APP_URL: ${{ steps.expected_url.outputs.HOSTED_APP_URL }} + steps: - uses: actions/checkout@v3 @@ -62,6 +69,11 @@ jobs: destination: "rerun-example-rrd/commit/${{env.SHORT_SHA}}" parent: false + - name: "Expected URL" + id: expected_url + run: | + echo "HOSTED_APP_URL=https://storage.googleapis.com/rerun-web-viewer/commit/${{env.SHORT_SHA}}/index.html" >> "$GITHUB_OUTPUT" + - name: "Upload web-viewer (prerelease)" if: inputs.RELEASE == 'prerelease' uses: google-github-actions/upload-cloud-storage@v1 diff --git a/templates/pr_results_summary.html b/templates/pr_results_summary.html new file mode 100644 index 000000000000..6a3826356ea2 --- /dev/null +++ b/templates/pr_results_summary.html @@ -0,0 +1,24 @@ + + + + + + + Build Summary + + + +

Build Summary for PR #{{ pr_number }}

+ {% if hosted_app_url %} +

Hosted App: {{ hosted_app_url }}

+ {% endif %} +

Wheels:

+
    + {% if linux_wheel_url %}
  • Linux
    • {% endif %} + {% if windows_wheel_url %}
  • Windows
    • {% endif %} + {% if macos_arm_wheel_url %}
  • macOS (ARM)
    • {% endif %} + {% if macos_intel_wheel_url %}
  • macOS (Intel)
    • {% endif %} +
+ + + From 833277aac8171775da90313187e23a14066b10d6 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 18:11:56 +0200 Subject: [PATCH 048/106] Fix PR template --- .github/workflows/reusable_pr_summary.yml | 19 +++++++++---------- .github/workflows/reusable_upload_web.yml | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index ebcb1f938b07..6ed55b561a86 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -45,16 +45,15 @@ jobs: - name: Render HTML template run: | jinja2 \ - --format=json \ - --var="pr_number=${{ inputs.PR_NUMBER }}" \ - --var="commit=${{ github.sha }}" \ - --var="hosted_app_url=${{ inputs.HOSTED_APP_URL }}" \ - --var="linux_wheel_url=${{ inputs.LINUX_WHEEL_URL }}" \ - --var="windows_wheel_url=${{ inputs.WINDOWS_WHEEL_URL }}" \ - --var="macos_arm_wheel_url=${{ inputs.MACOS_ARM_WHEEL_URL }}" \ - --var="macos_intel_wheel_url=${{ inputs.MACOS_INTEL_WHEEL_URL }}" \ - html_template.html \ - > build_summary.html + -D "pr_number=${{ inputs.PR_NUMBER }}" \ + -D "commit=${{ github.sha }}" \ + -D "hosted_app_url=${{ inputs.HOSTED_APP_URL }}" \ + -D "linux_wheel_url=${{ inputs.LINUX_WHEEL_URL }}" \ + -D "windows_wheel_url=${{ inputs.WINDOWS_WHEEL_URL }}" \ + -D "macos_arm_wheel_url=${{ inputs.MACOS_ARM_WHEEL_URL }}" \ + -D "macos_intel_wheel_url=${{ inputs.MACOS_INTEL_WHEEL_URL }}" \ + templates/pr_results_summary.html \ + -o build_summary.html # Replace the following with your actual artifact upload implementation. - name: Upload HTML summary diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index 5c49e5e1c393..fb3fa4f3e258 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -72,7 +72,7 @@ jobs: - name: "Expected URL" id: expected_url run: | - echo "HOSTED_APP_URL=https://storage.googleapis.com/rerun-web-viewer/commit/${{env.SHORT_SHA}}/index.html" >> "$GITHUB_OUTPUT" + echo "HOSTED_APP_URL=https://app.rerun.io/commit/${{env.SHORT_SHA}}/index.html" >> "$GITHUB_OUTPUT" - name: "Upload web-viewer (prerelease)" if: inputs.RELEASE == 'prerelease' From a20b4083556aec792004fb036ef540bd31d004b2 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 18:29:12 +0200 Subject: [PATCH 049/106] Add uploaded wheel to PR summary --- .github/workflows/manual_dispatch.yml | 3 ++- .github/workflows/reusable_pr_summary.yml | 5 ----- .github/workflows/reusable_upload_wheels.yml | 22 ++++++++++++++++++-- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 05f0920e670a..720a01698304 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -201,10 +201,11 @@ jobs: save_pr_summary: name: 'Save PR Summary' - needs: [upload_web] + needs: [upload_web, upload_wheels_linux] if: ${{ github.event.inputs.SAVE_PR != '' }} uses: ./.github/workflows/reusable_pr_summary.yml with: PR_NUMBER: ${{ github.event.inputs.SAVE_PR }} HOSTED_APP_URL: ${{ needs.upload_web.outputs.HOSTED_APP_URL }} + LINUX_WHEEL_URL: ${{ needs.upload_wheels_linux.outputs.WHEEL_URL }} secrets: inherit diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index 6ed55b561a86..f1ddf9e78684 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -57,11 +57,6 @@ jobs: # Replace the following with your actual artifact upload implementation. - name: Upload HTML summary - run: | - echo "Uploading build_summary.html to rerun-builds/commit/${{ github.sha }}/wheels" - # Implement the upload logic here - - - name: "Upload Wheel" uses: google-github-actions/upload-cloud-storage@v1 with: path: "build_summary.html" diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml index dd3c4fa30ea5..6a7875a091a8 100644 --- a/.github/workflows/reusable_upload_wheels.yml +++ b/.github/workflows/reusable_upload_wheels.yml @@ -18,11 +18,15 @@ on: required: false type: string default: '' + outputs: + WHEEL_URL: + description: "The gcloud URL where the wheel will be hosted" + value: ${{ jobs.upload_wheel.outputs.HOSTED_APP_URL }} jobs: - upload_wheels: - name: Upload Wheels to google cloud + upload_wheel: + name: Upload Wheel to google cloud runs-on: ubuntu-latest @@ -33,6 +37,9 @@ jobs: contents: "read" id-token: "write" + outputs: + WHEEL_URL: ${{ steps.expected_url.outputs.WHEEL_URL }} + steps: - name: Show context run: | @@ -101,9 +108,20 @@ jobs: - name: Add SHORT_SHA env property with commit short sha run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV + - name: Get the file name + shell: bash + id: get_file_name + run: | + echo "WHL_FILE=$(ls dist)" >> "$GITHUB_OUTPUT" + - name: "Upload Wheel" uses: google-github-actions/upload-cloud-storage@v1 with: path: "dist" destination: "rerun-builds/commit/${{env.SHORT_SHA}}/wheels" parent: false + + - name: "Expected URL" + id: expected_url + run: | + echo "WHEEL_URL=https://storage.googleapis.com/rerun-builds/commit/${{env.SHORT_SHA}}/wheels/${{steps.get_folder_name.outputs.PKG_FOLDER}} " >> "$GITHUB_OUTPUT" From 8389afd7b21a07aa7d1df032c011fc1051ca8f2e Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 18:32:18 +0200 Subject: [PATCH 050/106] PR job needs permissions --- .github/workflows/reusable_pr_summary.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index f1ddf9e78684..f7d6ac66cbf8 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -29,7 +29,14 @@ on: jobs: create_html_summary: + name: Create HTML summary for PR + + permissions: + contents: "read" + id-token: "write" + runs-on: ubuntu-latest + steps: - name: Checkout repository uses: actions/checkout@v2 From ec4e5bbae9faba97cc1c782ba6cdaa567c860d7d Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 19:31:05 +0200 Subject: [PATCH 051/106] Move the jinja inocation to a script that calls j2 --- .github/workflows/manual_dispatch.yml | 2 -- .github/workflows/reusable_pr_summary.yml | 19 ++++--------- .github/workflows/reusable_upload_web.yml | 12 -------- .github/workflows/reusable_upload_wheels.yml | 20 +------------- scripts/generate_pr_summary.sh | 29 ++++++++++++++++++++ templates/pr_results_summary.html | 17 ++++++------ 6 files changed, 45 insertions(+), 54 deletions(-) create mode 100755 scripts/generate_pr_summary.sh diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 720a01698304..297cb944d449 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -206,6 +206,4 @@ jobs: uses: ./.github/workflows/reusable_pr_summary.yml with: PR_NUMBER: ${{ github.event.inputs.SAVE_PR }} - HOSTED_APP_URL: ${{ needs.upload_web.outputs.HOSTED_APP_URL }} - LINUX_WHEEL_URL: ${{ needs.upload_wheels_linux.outputs.WHEEL_URL }} secrets: inherit diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index f7d6ac66cbf8..9f8c42164fc9 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -46,21 +46,14 @@ jobs: with: python-version: 3.x - - name: Install jinja-cli - run: pip install jinja2-cli + - name: Install j2cli + run: pip install j2cli - name: Render HTML template - run: | - jinja2 \ - -D "pr_number=${{ inputs.PR_NUMBER }}" \ - -D "commit=${{ github.sha }}" \ - -D "hosted_app_url=${{ inputs.HOSTED_APP_URL }}" \ - -D "linux_wheel_url=${{ inputs.LINUX_WHEEL_URL }}" \ - -D "windows_wheel_url=${{ inputs.WINDOWS_WHEEL_URL }}" \ - -D "macos_arm_wheel_url=${{ inputs.MACOS_ARM_WHEEL_URL }}" \ - -D "macos_intel_wheel_url=${{ inputs.MACOS_INTEL_WHEEL_URL }}" \ - templates/pr_results_summary.html \ - -o build_summary.html + run: scripts/generate_pr_summary.sh + env: + GIT_SHA: ${{ github.sha }} + PR_NUMBER: ${{ inputs.PR_NUMBER }} # Replace the following with your actual artifact upload implementation. - name: Upload HTML summary diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index fb3fa4f3e258..056cc5c44b9e 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -11,10 +11,6 @@ on: required: false type: string default: '' - outputs: - HOSTED_APP_URL: - description: "The URL where the app will be hosted" - value: ${{ jobs.upload_web.outputs.HOSTED_APP_URL }} jobs: @@ -26,9 +22,6 @@ jobs: runs-on: ubuntu-latest - outputs: - HOSTED_APP_URL: ${{ steps.expected_url.outputs.HOSTED_APP_URL }} - steps: - uses: actions/checkout@v3 @@ -69,11 +62,6 @@ jobs: destination: "rerun-example-rrd/commit/${{env.SHORT_SHA}}" parent: false - - name: "Expected URL" - id: expected_url - run: | - echo "HOSTED_APP_URL=https://app.rerun.io/commit/${{env.SHORT_SHA}}/index.html" >> "$GITHUB_OUTPUT" - - name: "Upload web-viewer (prerelease)" if: inputs.RELEASE == 'prerelease' uses: google-github-actions/upload-cloud-storage@v1 diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml index 6a7875a091a8..b2efe7bf1217 100644 --- a/.github/workflows/reusable_upload_wheels.yml +++ b/.github/workflows/reusable_upload_wheels.yml @@ -18,10 +18,6 @@ on: required: false type: string default: '' - outputs: - WHEEL_URL: - description: "The gcloud URL where the wheel will be hosted" - value: ${{ jobs.upload_wheel.outputs.HOSTED_APP_URL }} jobs: @@ -37,9 +33,6 @@ jobs: contents: "read" id-token: "write" - outputs: - WHEEL_URL: ${{ steps.expected_url.outputs.WHEEL_URL }} - steps: - name: Show context run: | @@ -59,7 +52,7 @@ jobs: - name: Download RRD uses: actions/download-artifact@v3 with: - name: rrd + name: ${{inputs.RRD_ARTIFACT_NAME}} path: rrd - name: Download Wheel @@ -80,12 +73,6 @@ jobs: run: | echo "PKG_FOLDER=$(ls unpack-dist)" >> "$GITHUB_OUTPUT" - - name: Download RRD - uses: actions/download-artifact@v3 - with: - name: ${{inputs.RRD_ARTIFACT_NAME}} - path: rrd - - name: Insert the rrd shell: bash # If you change the line below you should almost definitely change the `key:` line @@ -120,8 +107,3 @@ jobs: path: "dist" destination: "rerun-builds/commit/${{env.SHORT_SHA}}/wheels" parent: false - - - name: "Expected URL" - id: expected_url - run: | - echo "WHEEL_URL=https://storage.googleapis.com/rerun-builds/commit/${{env.SHORT_SHA}}/wheels/${{steps.get_folder_name.outputs.PKG_FOLDER}} " >> "$GITHUB_OUTPUT" diff --git a/scripts/generate_pr_summary.sh b/scripts/generate_pr_summary.sh new file mode 100755 index 000000000000..3f9da875fa11 --- /dev/null +++ b/scripts/generate_pr_summary.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Exit if GITHUB_SHA is not defined or empty +if [ -z "${GITHUB_SHA}" ]; then + echo "Error: GITHUB_SHA is not defined or empty." + exit 1 +fi + +if [ -z "${PR_NUMBER}" ]; then + echo "Error: PR_NUMBER is not defined or empty." + exit 1 +fi + +GITHUB_SHORT=$(echo $GITHUB_SHA | cut -c1-7) + +# Check if the bucket resource exists +gcloud storage ls gs://rerun-web-viewer/commit/${GITHUB_SHORT} > /dev/null 2>&1 + +if [ $? -eq 0 ]; then + export HOSTED_APP_URL="https://app.rerun.io/${GITHUB_SHORT}" +else + export HOSTED_APP_URL="" +fi + +# Get the list of wheel files +export WHEELS=$(gcloud storage ls gs://rerun-build/commit/${GITHUB_SHORT}/wheels | grep -oP '(?<=gs://rerun-build/commit/)[^/]+/wheels/wheel\d+.whl') + +# Render Jinja template with the HOSTED_APP_URL and WHEELS variables +j2 templates/pr_results_summary.html > build_summary.html diff --git a/templates/pr_results_summary.html b/templates/pr_results_summary.html index 6a3826356ea2..90f04b523e72 100644 --- a/templates/pr_results_summary.html +++ b/templates/pr_results_summary.html @@ -8,17 +8,18 @@ -

Build Summary for PR #{{ pr_number }}

- {% if hosted_app_url %} -

Hosted App: {{ hosted_app_url }}

+

Build Summary for PR #{{ PR_NUMBER }}

+ {% if HOSTED_APP_URL %} +

Hosted App: {{ HOSTED_APP_URL }}

{% endif %} -

Wheels:

+ {% if WHEELS %} +

List of Wheels

    - {% if linux_wheel_url %}
  • Linux
    • {% endif %} - {% if windows_wheel_url %}
  • Windows
    • {% endif %} - {% if macos_arm_wheel_url %}
  • macOS (ARM)
    • {% endif %} - {% if macos_intel_wheel_url %}
  • macOS (Intel)
    • {% endif %} + {% for wheel in WHEELS.split() %} +
  • {{ wheel }}
    • + {% endfor %}
+ {% endif %} From 021012121ef62a7ec556913a1b333d372b27932d Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 19:34:49 +0200 Subject: [PATCH 052/106] Set up gcloud APIs --- .github/workflows/reusable_pr_summary.yml | 25 +++++------------------ 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index 9f8c42164fc9..222e1c36c979 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -6,26 +6,6 @@ on: PR_NUMBER: required: true type: string - HOSTED_APP_URL: - required: false - type: string - default: '' - LINUX_WHEEL_URL: - required: false - type: string - default: '' - WINDOWS_WHEEL_URL: - required: false - type: string - default: '' - MACOS_ARM_WHEEL_URL: - required: false - type: string - default: '' - MACOS_INTEL_WHEEL_URL: - required: false - type: string - default: '' jobs: create_html_summary: @@ -46,6 +26,11 @@ jobs: with: python-version: 3.x + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + with: + version: '>= 363.0.0' + - name: Install j2cli run: pip install j2cli From d1e5b71f7f1287c4675fe1c46ecefcf55b2e0caa Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 19:53:59 +0200 Subject: [PATCH 053/106] Forgot to auth so we can upload the results --- .github/workflows/reusable_pr_summary.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index 222e1c36c979..7faab58e4510 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -26,6 +26,12 @@ jobs: with: python-version: 3.x + - id: "auth" + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + - name: 'Set up Cloud SDK' uses: 'google-github-actions/setup-gcloud@v1' with: @@ -40,7 +46,6 @@ jobs: GIT_SHA: ${{ github.sha }} PR_NUMBER: ${{ inputs.PR_NUMBER }} - # Replace the following with your actual artifact upload implementation. - name: Upload HTML summary uses: google-github-actions/upload-cloud-storage@v1 with: From 295c34e75035ea49623885f25deff3dfdf7d5c27 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 20:10:29 +0200 Subject: [PATCH 054/106] Fix the PR summary script --- scripts/generate_pr_summary.sh | 7 ++++--- templates/pr_results_summary.html | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/generate_pr_summary.sh b/scripts/generate_pr_summary.sh index 3f9da875fa11..00c0197533c3 100755 --- a/scripts/generate_pr_summary.sh +++ b/scripts/generate_pr_summary.sh @@ -11,19 +11,20 @@ if [ -z "${PR_NUMBER}" ]; then exit 1 fi -GITHUB_SHORT=$(echo $GITHUB_SHA | cut -c1-7) +export GITHUB_SHORT=$(echo $GITHUB_SHA | cut -c1-7) # Check if the bucket resource exists gcloud storage ls gs://rerun-web-viewer/commit/${GITHUB_SHORT} > /dev/null 2>&1 if [ $? -eq 0 ]; then - export HOSTED_APP_URL="https://app.rerun.io/${GITHUB_SHORT}" + export HOSTED_APP_URL="https://app.rerun.io/commit/${GITHUB_SHORT}" else export HOSTED_APP_URL="" fi # Get the list of wheel files -export WHEELS=$(gcloud storage ls gs://rerun-build/commit/${GITHUB_SHORT}/wheels | grep -oP '(?<=gs://rerun-build/commit/)[^/]+/wheels/wheel\d+.whl') +export WHEELS=$(gcloud storage ls gs://rerun-builds/commit/${GITHUB_SHORT}/wheels | awk -F/ '{print $NF}' | grep -E '.whl$') +export WHEEL_BASE_URL="https://storage.googleapis.com/rerun-builds/commit/${GITHUB_SHORT}/wheels" # Render Jinja template with the HOSTED_APP_URL and WHEELS variables j2 templates/pr_results_summary.html > build_summary.html diff --git a/templates/pr_results_summary.html b/templates/pr_results_summary.html index 90f04b523e72..f5aaeb855690 100644 --- a/templates/pr_results_summary.html +++ b/templates/pr_results_summary.html @@ -16,7 +16,8 @@

Build Summary for PR #{{ PR_NUMBER }}

List of Wheels

{% endif %} From 688add58120a4cbcb947c73c6a1fd3a45780ad79 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 20:10:54 +0200 Subject: [PATCH 055/106] new job to just build the wheels for a PR --- .github/workflows/build_wheels_for_pr.yml | 117 ++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 .github/workflows/build_wheels_for_pr.yml diff --git a/.github/workflows/build_wheels_for_pr.yml b/.github/workflows/build_wheels_for_pr.yml new file mode 100644 index 000000000000..ff0749a6073f --- /dev/null +++ b/.github/workflows/build_wheels_for_pr.yml @@ -0,0 +1,117 @@ +name: Build and Upload Wheels for PR + +on: + workflow_dispatch: + +jobs: + + check_for_pr: + runs-on: ubuntu-latest + outputs: + PR_NUMBER: ${{ steps.get_pr.outputs.PR_NUMBER }} + steps: + - name: Check if commit belongs to a PR + id: get_pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pr_number=$(curl --silent --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --url "https://api.github.com/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls" \ + | jq '.[] | .number') + + if [ -z "$pr_number" ]; then + echo "No PR associated with this commit" + exit 1 + else + echo "Commit is associated with PR: $pr_number" + echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUTS" + fi + + build_linux: + needs: [check_for_pr] + name: 'Linux: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: linux + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + build_windows: + needs: [check_for_pr] + name: 'Windows: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: windows + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: '' + secrets: inherit + + build_macos_arm: + needs: [check_for_pr] + name: 'Macos-Arm: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: macos-arm + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: '' + secrets: inherit + + build_macos_intel: + needs: [check_for_pr] + name: 'Macos-Intel: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: macos-intel + WHEEL_ARTIFACT_NAME: 'macos-intel-wheel' + RRD_ARTIFACT_NAME: '' + secrets: inherit + + upload_wheels_linux: + name: 'Linux: Upload Wheels' + needs: [build_linux] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload_wheels_windows: + name: 'Windows: Upload Wheels' + needs: [build_linux, build_windows] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload_wheels_macos_arm: + name: 'Macos-Arm: Upload Wheels' + needs: [build_linux, build_macos_arm] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload_wheels_macos_intel: + name: 'Macos-Intel: Upload Wheels' + needs: [build_linux, build_macos_intel] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} + WHEEL_ARTIFACT_NAME: macos-intel-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + + save_pr_summary: + name: 'Save PR Summary' + needs: [check_for_pr, upload_wheels_linux, upload_wheels_windows, upload_wheels_macos_arm, upload_wheels_macos_intel] + uses: ./.github/workflows/reusable_pr_summary.yml + with: + PR_NUMBER: ${{ needs.check_for_pr.outputs.PR_NUMBER}} + secrets: inherit From 10b16631cfe40a12fc03b9ffd60e5094b47425f6 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 20:20:21 +0200 Subject: [PATCH 056/106] Use correct output path --- .github/workflows/build_wheels_for_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels_for_pr.yml b/.github/workflows/build_wheels_for_pr.yml index ff0749a6073f..533621e29db5 100644 --- a/.github/workflows/build_wheels_for_pr.yml +++ b/.github/workflows/build_wheels_for_pr.yml @@ -24,7 +24,7 @@ jobs: exit 1 else echo "Commit is associated with PR: $pr_number" - echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUTS" + echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT" fi build_linux: From 2295e3db1057b2da7bced0ed5e61ba07dcea98d6 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 21:04:15 +0200 Subject: [PATCH 057/106] Fix RUN_TESTS check now that the build and test happen in one step --- .github/workflows/reusable_build_and_test_wheels.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index 8633b70cd5ea..544b558623eb 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -239,6 +239,7 @@ jobs: pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 - name: Install built wheel + if: needs.set_config.outputs.RUN_TESTS == 'true' # Now install the wheel using a specific version and --no-index to guarantee we get the version from # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't # find the dependencies to reinstall them. @@ -248,6 +249,7 @@ jobs: pip install rerun-sdk==${{ steps.expected_version.outputs.EXPECTED_VERSION }} --no-index --find-links dist - name: Verify built wheel version + if: needs.set_config.outputs.RUN_TESTS == 'true' shell: bash run: | python3 -m rerun --version @@ -255,14 +257,17 @@ jobs: rerun --version - name: Run unit tests + if: needs.set_config.outputs.RUN_TESTS == 'true' shell: bash run: cd rerun_py/tests && pytest - name: Run e2e test + if: needs.set_config.outputs.RUN_TESTS == 'true' shell: bash run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - name: Cache RRD dataset + if: needs.set_config.outputs.RUN_TESTS == 'true' id: dataset uses: actions/cache@v3 with: @@ -271,6 +276,7 @@ jobs: key: colmap-dataset-colmap-fiat-v0 - name: Generate Embedded RRD file + if: needs.set_config.outputs.RUN_TESTS == 'true' shell: bash # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name run: | @@ -280,14 +286,14 @@ jobs: # All platforms are currently creating the same rrd file, upload one of them - name: Save RRD artifact - if: ${{ inputs.RRD_ARTIFACT_NAME != '' }} + if: ${{ (needs.set_config.outputs.RUN_TESTS == 'true') && (inputs.RRD_ARTIFACT_NAME != '') }} uses: actions/upload-artifact@v3 with: name: ${{ inputs.RRD_ARTIFACT_NAME }} path: rrd - name: Save wheel artifact - if: ${{ inputs.WHEEL_ARTIFACT_NAME != '' }} + if: ${{ (needs.set_config.outputs.RUN_TESTS == 'true') && (inputs.WHEEL_ARTIFACT_NAME != '') }} uses: actions/upload-artifact@v3 with: name: ${{inputs.WHEEL_ARTIFACT_NAME}} From 579f8c2dc6853ef1637e2a1f2125760d19252bd3 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 21:49:03 +0200 Subject: [PATCH 058/106] Migrate the summary script to python --- .github/workflows/reusable_pr_summary.yml | 7 ++- scripts/generate_pr_summary.py | 66 +++++++++++++++++++++++ scripts/generate_pr_summary.sh | 30 ----------- templates/pr_results_summary.html | 52 ++++++++++++------ 4 files changed, 106 insertions(+), 49 deletions(-) create mode 100644 scripts/generate_pr_summary.py delete mode 100755 scripts/generate_pr_summary.sh diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index 7faab58e4510..52e384873d42 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -37,13 +37,12 @@ jobs: with: version: '>= 363.0.0' - - name: Install j2cli - run: pip install j2cli + - name: Install deps + run: pip install google-cloud-storage Jinja2 PyGithub - name: Render HTML template - run: scripts/generate_pr_summary.sh + run: python scripts/generate_pr_summary.py env: - GIT_SHA: ${{ github.sha }} PR_NUMBER: ${{ inputs.PR_NUMBER }} - name: Upload HTML summary diff --git a/scripts/generate_pr_summary.py b/scripts/generate_pr_summary.py new file mode 100644 index 000000000000..eb3e59f7dccf --- /dev/null +++ b/scripts/generate_pr_summary.py @@ -0,0 +1,66 @@ +""" +Script to generate a PR summary page. + +This script combines the github and google cloud storage APIs +to find and link to the builds associated with a given PR. + +This is expected to be run by the `reusable_pr_summary.yml` github workflow. + +Requires the following packages: + pip install google-cloud-storage Jinja2 PyGithub +""" + +import os +from github import Github +from google.cloud import storage +from jinja2 import Template + +GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] +GITHUB_REPOSITORY = os.environ["GITHUB_REPOSITORY"] +PR_NUMBER = int(os.environ["PR_NUMBER"]) + + +# Initialize the Github and GCS clients +gh = Github(GITHUB_TOKEN) +gcs_client = storage.Client() + +# Get the list of commits associated with the PR +repo = gh.get_repo(GITHUB_REPOSITORY) +pull = repo.get_pull(PR_NUMBER) +all_commits = [commit.sha for commit in pull.get_commits()] +all_commits.reverse() + +# Prepare the found_builds list +found_builds = [] +viewer_bucket = gcs_client.bucket("rerun-web-viewer") +wheels_bucket = gcs_client.bucket("rerun-builds") + +for commit in all_commits: + commit_short = commit[:7] + print("Checking commit: {}...".format(commit_short)) + + found = {} + + # Check if there is a hosted app for the current commit + commit_blob = viewer_bucket.blob(f"commit/{commit_short}/index.html") + if commit_blob.exists(): + print("Found web assets commit: {}".format(commit_short)) + found["hosted_app"] = f"https://app.rerun.io/commit/{commit_short}" + + # Get the wheel files for the commit + wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) + wheels = [f"https://storage.googleapis.com/{blob.bucket.name}/{blob.name}" for blob in wheel_blobs] + if wheels: + print("Found wheels for commit: {}".format(commit_short)) + found["wheels"] = wheels + + if found: + found["commit"] = commit_short + found_builds.append(found) + +# Render the Jinja template with the found_builds variable +with open("templates/pr_results_summary.html") as f: + template = Template(f.read()) + +with open("build_summary.html", "w") as f: + f.write(template.render(found_builds=found_builds, pr_number=PR_NUMBER)) diff --git a/scripts/generate_pr_summary.sh b/scripts/generate_pr_summary.sh deleted file mode 100755 index 00c0197533c3..000000000000 --- a/scripts/generate_pr_summary.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Exit if GITHUB_SHA is not defined or empty -if [ -z "${GITHUB_SHA}" ]; then - echo "Error: GITHUB_SHA is not defined or empty." - exit 1 -fi - -if [ -z "${PR_NUMBER}" ]; then - echo "Error: PR_NUMBER is not defined or empty." - exit 1 -fi - -export GITHUB_SHORT=$(echo $GITHUB_SHA | cut -c1-7) - -# Check if the bucket resource exists -gcloud storage ls gs://rerun-web-viewer/commit/${GITHUB_SHORT} > /dev/null 2>&1 - -if [ $? -eq 0 ]; then - export HOSTED_APP_URL="https://app.rerun.io/commit/${GITHUB_SHORT}" -else - export HOSTED_APP_URL="" -fi - -# Get the list of wheel files -export WHEELS=$(gcloud storage ls gs://rerun-builds/commit/${GITHUB_SHORT}/wheels | awk -F/ '{print $NF}' | grep -E '.whl$') -export WHEEL_BASE_URL="https://storage.googleapis.com/rerun-builds/commit/${GITHUB_SHORT}/wheels" - -# Render Jinja template with the HOSTED_APP_URL and WHEELS variables -j2 templates/pr_results_summary.html > build_summary.html diff --git a/templates/pr_results_summary.html b/templates/pr_results_summary.html index f5aaeb855690..69b5bcabc51c 100644 --- a/templates/pr_results_summary.html +++ b/templates/pr_results_summary.html @@ -2,25 +2,47 @@ - - Build Summary + -

Build Summary for PR #{{ PR_NUMBER }}

- {% if HOSTED_APP_URL %} -

Hosted App: {{ HOSTED_APP_URL }}

- {% endif %} - {% if WHEELS %} -

List of Wheels

-
    - {% for wheel in WHEELS.split() %} -
  • {{ wheel }} -
    • - {% endfor %} -
- {% endif %} +

Build Summary for #{{pr_number}}

+ + {% for build in found_builds %} +
+

Commit: {{ build.commit }}

+ {% if build.hosted_app %} +
+

Hosted App:

+ {{ build.hosted_app }} +
+ {% endif %} + {% if build.wheels %} +
+

Wheels:

+ +
+ {% endif %} +
+ {% endfor %} From 4675c9fe18688c07c9ddb26f86742bded4564edc Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 22:04:45 +0200 Subject: [PATCH 059/106] Lints --- scripts/generate_pr_summary.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/generate_pr_summary.py b/scripts/generate_pr_summary.py index eb3e59f7dccf..e5606cb8be1f 100644 --- a/scripts/generate_pr_summary.py +++ b/scripts/generate_pr_summary.py @@ -1,17 +1,19 @@ """ Script to generate a PR summary page. -This script combines the github and google cloud storage APIs +This script combines the GitHub and google cloud storage APIs to find and link to the builds associated with a given PR. -This is expected to be run by the `reusable_pr_summary.yml` github workflow. +This is expected to be run by the `reusable_pr_summary.yml` GitHub workflow. Requires the following packages: - pip install google-cloud-storage Jinja2 PyGithub + pip install google-cloud-storage Jinja2 PyGithub # NOLINT """ import os -from github import Github +from typing import Any, Dict + +from github import Github # NOLINT from google.cloud import storage from jinja2 import Template @@ -20,8 +22,8 @@ PR_NUMBER = int(os.environ["PR_NUMBER"]) -# Initialize the Github and GCS clients -gh = Github(GITHUB_TOKEN) +# Initialize the GitHub and GCS clients +gh = Github(GITHUB_TOKEN) # NOLINT gcs_client = storage.Client() # Get the list of commits associated with the PR @@ -39,7 +41,7 @@ commit_short = commit[:7] print("Checking commit: {}...".format(commit_short)) - found = {} + found: Dict[str, Any] = {} # Check if there is a hosted app for the current commit commit_blob = viewer_bucket.blob(f"commit/{commit_short}/index.html") From 118d86108b597d3827ec9113707771c080047766 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 22:12:58 +0200 Subject: [PATCH 060/106] Python script needs the github token --- .github/workflows/reusable_pr_summary.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index 52e384873d42..a27786131cc1 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -43,6 +43,7 @@ jobs: - name: Render HTML template run: python scripts/generate_pr_summary.py env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ inputs.PR_NUMBER }} - name: Upload HTML summary From 5915a9ac314dd060776821b6a2ef9f720d371725 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 22:15:49 +0200 Subject: [PATCH 061/106] More NOLINT for PyGithub --- .github/workflows/reusable_pr_summary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index a27786131cc1..56f3a2dd4ca9 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -38,7 +38,7 @@ jobs: version: '>= 363.0.0' - name: Install deps - run: pip install google-cloud-storage Jinja2 PyGithub + run: pip install google-cloud-storage Jinja2 PyGithub # NOLINT - name: Render HTML template run: python scripts/generate_pr_summary.py From a4baa724ac2cd55d1b6179994de6a5ea25c2e227 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 23:08:33 +0200 Subject: [PATCH 062/106] Add new mechanism to override the upload commit --- .github/workflows/reusable_build_web.yml | 2 +- .github/workflows/reusable_pr_summary.yml | 2 +- .github/workflows/reusable_upload_web.yml | 26 +++++++++++++---------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 93d77671e1eb..40e6fd87ae0e 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -31,7 +31,7 @@ jobs: image: rerunio/ci_docker:0.6 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index 56f3a2dd4ca9..e0d4c3cf8dd2 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v2 diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index 056cc5c44b9e..7255dd23b0c3 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -11,6 +11,12 @@ on: required: false type: string default: '' + # We need this because PRs use a merged commit but we really want + # to track uploads based on the source commit. + UPLOAD_COMMIT_OVERRIDE: + required: false + type: string + default: '' jobs: @@ -46,7 +52,13 @@ jobs: service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - name: Add SHORT_SHA env property with commit short sha - run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV + run: | + if [ -z "${{ inputs.UPLOAD_COMMIT_OVERRIDE }}" ]; then + USED_SHA=${{ github.sha }} + else + USED_SHA=${{ inputs.UPLOAD_COMMIT_OVERRIDE }} + fi + echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_ENV - name: "Upload web-viewer (commit)" uses: google-github-actions/upload-cloud-storage@v1 @@ -84,7 +96,7 @@ jobs: uses: google-github-actions/upload-cloud-storage@v1 with: path: "web_viewer" - destination: "rerun-web-viewer/version/${{github.ref_name}}" + destination: "rerun-web-viewer/version/${{inputs.RELEASE}}" parent: false - name: "Upload RRD (tagged)" @@ -92,13 +104,5 @@ jobs: uses: google-github-actions/upload-cloud-storage@v1 with: path: "rrd" - destination: "rerun-example-rrd/version/${{github.ref_name}}" - parent: false - - - name: "Upload RRD (latest release)" - if: github.ref == 'latest' - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "rrd" - destination: "rerun-example-rrd/latest" + destination: "rerun-example-rrd/version/${{inputs.RELEASE}}" parent: false From c6ed4e2e39b406b9c34b8c5504b5e19e3fdbf9e4 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 00:54:20 +0200 Subject: [PATCH 063/106] Use new CI workflows on pull-request --- .github/workflows/misc.yml | 32 -- .github/workflows/on_pull_request.yml | 57 +++ .github/workflows/python.yml | 629 -------------------------- .github/workflows/rust.yml | 368 --------------- .github/workflows/toml.yml | 43 -- .github/workflows/typos.yml | 17 - 6 files changed, 57 insertions(+), 1089 deletions(-) delete mode 100644 .github/workflows/misc.yml create mode 100644 .github/workflows/on_pull_request.yml delete mode 100644 .github/workflows/python.yml delete mode 100644 .github/workflows/rust.yml delete mode 100644 .github/workflows/toml.yml delete mode 100644 .github/workflows/typos.yml diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml deleted file mode 100644 index acdcfff2cd7d..000000000000 --- a/.github/workflows/misc.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: CI (Misc) - -on: - workflow_dispatch: - pull_request: - push: - branches: - - "main" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} # Cancel previous CI jobs on the same branch - cancel-in-progress: true - -jobs: - misc-rerun-lints: - name: Rerun lints - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Rerun lints - run: | - ./scripts/lint.py - - - name: Check for too large files - run: | - ./scripts/check_large_files.sh diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml new file mode 100644 index 000000000000..ce04c594b06a --- /dev/null +++ b/.github/workflows/on_pull_request.yml @@ -0,0 +1,57 @@ +name: Run Checks on Pull-Request + +on: + pull_request: + types: + - opened + - synchronize + +jobs: + checks: + name: Run All Checks + uses: ./.github/workflows/reusable_checks.yml + secrets: inherit + + benches: + name: Benchmarks + uses: ./.github/workflows/reusable_bench.yml + secrets: inherit + + build_linux_fast: + name: 'Minimum Wheel Test' + + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + SAVE_CACHE: false + PLATFORM: linux + MATURIN_FEATURE_FLAGS: '--no-default-features --features extension-module' + WHEEL_ARTIFACT_NAME: '' + RRD_ARTIFACT_NAME: linux-rrd-fast + secrets: inherit + + build_web: + name: 'Build Web' + uses: ./.github/workflows/reusable_build_web.yml + with: + RELEASE: '' + secrets: inherit + + upload_web: + name: 'Upload Web' + # Uses Assets: + # build_linux: rrd + # build_web: web_viewer + needs: [build_linux_fast, build_web] + uses: ./.github/workflows/reusable_upload_web.yml + with: + RELEASE: '' + RRD_ARTIFACT_NAME: linux-rrd-fast + secrets: inherit + + save_pr_summary: + name: 'Save PR Summary' + needs: [upload_web] + uses: ./.github/workflows/reusable_pr_summary.yml + with: + PR_NUMBER: ${{ github.event.pull_request.number }} + secrets: inherit diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index 491c5353550a..000000000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,629 +0,0 @@ -name: CI (Python) - -on: - pull_request: - types: [labeled, synchronize, opened] - push: - branches: - - "main" - tags: - - "v*.*.*" # on release tag - workflow_dispatch: - inputs: - force_build_wheel: - description: "Build python wheels" - required: true - default: false - type: boolean - -env: - PYTHON_VERSION: "3.8" - PRE_RELEASE_INSTRUCTIONS: | - ## Installing the pre-release Python SDK - 1. Download the correct `.whl`. - 2. Run `pip install rerun_sdk<...>.whl` (replace `<...>` with the actual filename) - 3. Test it: `rerun --version` - UBUNTU_REQUIRED_PKGS: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} # Cancel previous CI jobs only on pull-requests - cancel-in-progress: true - -jobs: - lint: - name: Python lints (black, mypy, flake8) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - just-version: 1.5 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "rerun_py/requirements-lint.txt" - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-lint.txt - - - name: Lint Python - run: | - just py-lint - - - name: Check requirements - run: | - just py-requirements - - # --------------------------------------------------------------------------- - # We need one wheel-build to be special so the other builds (namely mac arm) can use its rrd - # This copy-paste is awful, but we'll refactor the build soon. - wheels-linux: - if: github.event_name == 'push' || github.event.inputs.force_build_wheel - name: Build Python Wheels (Linux) - runs-on: ubuntu-latest - container: - image: rerunio/ci_docker:0.6 - steps: - - uses: actions/checkout@v3 - - # These should already be in the docker container, but run for good measure. A no-op install - # should be fast, and this way things don't break if we add new packages without rebuilding - # docker - - name: Cache APT Packages - uses: awalsh128/cache-apt-pkgs-action@v1.2.2 - with: - packages: ${{ env.UBUNTU_REQUIRED_PKGS }} - version: 2.0 # Increment this to pull newer packages - execute_install_scripts: true - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # Don't update the cache -- it will be updated by the lint job - # TODO(jleibs): this job will likely run before rust.yml updates - # the cache. Better cross-job sequencing would be nice here - save-if: False - - # These should already be in the docker container, but run for good measure. A no-op install - # should be fast, and this way things don't break if we add new packages without rebuilding - # docker - - run: pip install -r rerun_py/requirements-build.txt - - # ---------------------------------------------------------------------------------- - - - name: Patch Cargo.toml for pre-release - if: github.ref == 'refs/heads/main' - # After patching the pre-release version, run cargo update. - # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing - run: | - python3 scripts/version_util.py --patch_prerelease - cargo update -w - - - name: Version check for tagged-release - if: startsWith(github.ref, 'refs/tags/v') - # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z - run: | - python3 scripts/version_util.py --check_version - - - name: Store the expected version - # Find the current cargo version and store it in the GITHUB_ENV var: `expected_version` - shell: bash - run: | - echo "expected_version=$(python3 scripts/version_util.py --bare_cargo_version)" >> $GITHUB_ENV - - - name: Build Wheel - uses: PyO3/maturin-action@v1 - with: - maturin-version: "0.14.10" - manylinux: manylinux_2_31 - container: off - command: build - args: | - --manifest-path rerun_py/Cargo.toml - --release - --target x86_64-unknown-linux-gnu - --no-default-features - --features pypi - --out pre-dist - - - name: Install wheel dependencies - # First we install the dependencies manually so we can use `--no-index` when installing the wheel. - # This needs to be a separate step for some reason or the following step fails - # TODO(jleibs): pull these deps from pyproject.toml - # TODO(jleibs): understand why deps can't be installed in the same step as the wheel - shell: bash - run: | - pip install deprecated numpy>=1.23 pyarrow==10.0.1 - - - name: Install built wheel - # Now install the wheel using a specific version and --no-index to guarantee we get the version from - # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't - # find the dependencies to reinstall them. - shell: bash - run: | - pip uninstall rerun-sdk - pip install rerun-sdk==${{ env.expected_version }} --no-index --find-links pre-dist - - - name: Verify built wheel version - shell: bash - run: | - python3 -m rerun --version - which rerun - rerun --version - - - name: Run unit tests - shell: bash - run: cd rerun_py/tests && pytest - - - name: Run e2e test - shell: bash - run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - - - name: Unpack the wheel - shell: bash - run: | - mkdir unpack-dist - wheel unpack pre-dist/*.whl --dest unpack-dist - - - name: Get the folder name - shell: bash - run: | - echo "pkg_folder=$(ls unpack-dist)" >> $GITHUB_ENV - - - name: Cache RRD dataset - id: dataset - uses: actions/cache@v3 - with: - path: examples/python/colmap/dataset/ - # TODO(jleibs): Derive this key from the invocation below - key: colmap-dataset-colmap-fiat-v0 - - - name: Generate Embedded RRD file - shell: bash - # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name - run: | - mkdir rrd - pip install -r examples/python/colmap/requirements.txt - python3 examples/python/colmap/main.py --dataset colmap_fiat --resize 800x600 --save rrd/colmap_fiat.rrd - cp rrd/colmap_fiat.rrd unpack-dist/${{ env.pkg_folder }}/rerun_sdk/rerun_demo/colmap_fiat.rrd - - - name: Repack the wheel - shell: bash - run: | - mkdir dist - wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/ - - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: dist - - # All platforms are currently creating the same rrd file, upload one of them - - name: Save RRD artifact - uses: actions/upload-artifact@v3 - with: - name: rrd - path: rrd - - # --------------------------------------------------------------------------- - matrix-setup: - # Building all the wheels is expensive, so we only run this job when we push (to main or release tags), - # or if the job was manually triggered with `force_build_wheel` set to true. - if: github.event_name == 'push' || github.event.inputs.force_build_wheel - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - - steps: - - name: Dump GitHub context - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: echo "$GITHUB_CONTEXT" - - name: Dump job context - env: - JOB_CONTEXT: ${{ toJson(job) }} - run: echo "$JOB_CONTEXT" - - - id: set-matrix - shell: bash - # TODO(jleibs): figure out why tests are failing to complete on `x86_64-apple-darwin` - # See: https://github.com/rerun-io/rerun/pull/1853 - run: | - matrix=() - matrix+=('{"platform": "macos", "target": "x86_64-apple-darwin", "run_tests": false, "runs_on": "macos-latest" },') - matrix+=('{"platform": "macos", "target": "aarch64-apple-darwin", "run_tests": false, "runs_on": "macos-latest" },') # NOTE: we can't run tests on arm since our macos runner is x86_64 - matrix+=('{"platform": "windows", "target": "x86_64-pc-windows-msvc", "run_tests": true, "runs_on": "windows-latest-8-cores"},') - - echo "Matrix values: ${matrix[@]}" - - echo "matrix={\"include\":[${matrix[@]}]}" >> $GITHUB_OUTPUT - - wheels: - name: Build Remaining Python Wheels - needs: [lint, matrix-setup, wheels-linux] - - strategy: - matrix: ${{fromJson(needs.matrix-setup.outputs.matrix)}} - - runs-on: ${{ matrix.runs_on }} - - steps: - - uses: actions/checkout@v3 - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # Don't update the cache -- it will be updated by the lint job - # TODO(jleibs): this job will likely run before rust.yml updates - # the cache. Better cross-job sequencing would be nice here - save-if: False - - # The pip-cache setup logic doesn't work in the ubuntu docker container - # That's probably fine since we bake these deps into the container already - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "rerun_py/requirements-build.txt" - - # These should already be in the docker container, but run for good measure. A no-op install - # should be fast, and this way things don't break if we add new packages without rebuilding - # docker - - run: pip install -r rerun_py/requirements-build.txt - - # ---------------------------------------------------------------------------------- - # Install prerequisites for building the web-viewer Wasm - - # We have a nice script for that: ./scripts/setup_web.sh - # Unfortunately, we can't run that on Windows, because Windows doesn't come with - # a package manager like grown-up OSes do (at least not the CI version of Windows). - # Also we can't run it on linux because the 20.04 Docker container will install - # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag - # So we only run the script on macos, and then on Windows we do the parts of the script manually. - # On ubuntu, the correct packages are pre-installed in our docker container. - - - name: Install prerequisites for building the web-viewer Wasm (non-Windows) - if: matrix.platform == 'macos' - shell: bash - run: ./scripts/setup_web.sh - - # The first steps of setup_web.sh, for Windows: - - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows - if: matrix.platform == 'windows' - shell: bash - run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84 - - # The last step of setup_web.sh, for Windows. - # Since 'winget' is not available within the GitHub runner, we download the package directly: - # See: https://github.com/marketplace/actions/engineerd-configurator - - name: Install binaryen for building the web-viewer Wasm on windows - if: matrix.platform == 'windows' - uses: engineerd/configurator@v0.0.9 - with: - name: "wasm-opt.exe" - url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz" - pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" - - # ---------------------------------------------------------------------------------- - - - name: Patch Cargo.toml for pre-release - if: github.ref == 'refs/heads/main' - # After patching the pre-release version, run cargo update. - # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing - run: | - python3 scripts/version_util.py --patch_prerelease - cargo update -w - - - name: Version check for tagged-release - if: startsWith(github.ref, 'refs/tags/v') - # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z - run: | - python3 scripts/version_util.py --check_version - - - name: Store the expected version - # Find the current cargo version and store it in the GITHUB_ENV var: `expected_version` - shell: bash - run: | - echo "expected_version=$(python3 scripts/version_util.py --bare_cargo_version)" >> $GITHUB_ENV - - - name: Build Wheel - uses: PyO3/maturin-action@v1 - with: - maturin-version: "0.14.10" - manylinux: manylinux_2_31 - container: off - command: build - args: | - --manifest-path rerun_py/Cargo.toml - --release - --target ${{ matrix.target }} - --no-default-features - --features pypi - --out pre-dist - - - name: Install built wheel - if: ${{ matrix.run_tests }} - # First we install the dependencies manually so we can use `--no-index` when installing the wheel. - # Then install the wheel using a specific version and --no-index to guarantee we get the version from - # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't - # find the dependencies to reinstall them. - # TODO(jleibs): pull these deps from pyproject.toml - shell: bash - run: | - pip uninstall rerun-sdk - pip install deprecated numpy>=1.23 pyarrow==10.0.1 - pip install rerun-sdk==${{ env.expected_version }} --no-index --find-links pre-dist - - - name: Verify built wheel version - if: ${{ matrix.run_tests }} - shell: bash - run: | - python3 -m rerun --version - which rerun - rerun --version - - - name: Run unit tests - if: ${{ matrix.run_tests }} - shell: bash - run: cd rerun_py/tests && pytest - - - name: Run e2e test - if: ${{ matrix.run_tests }} - shell: bash - run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - - - name: Unpack the wheel - shell: bash - run: | - mkdir unpack-dist - wheel unpack pre-dist/*.whl --dest unpack-dist - - - name: Get the folder name - shell: bash - run: | - echo "pkg_folder=$(ls unpack-dist)" >> $GITHUB_ENV - - - name: Download RRD - uses: actions/download-artifact@v3 - with: - name: rrd - path: rrd - - - name: Insert the rrd - shell: bash - # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name - run: | - cp rrd/colmap_fiat.rrd unpack-dist/${{ env.pkg_folder }}/rerun_sdk/rerun_demo/colmap_fiat.rrd - - - name: Repack the wheel - shell: bash - run: | - mkdir dist - wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/ - - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: dist - - # --------------------------------------------------------------------------- - - upload_rrd: - name: Upload RRD to GCloud - permissions: - contents: "read" - id-token: "write" - needs: [wheels] - runs-on: "ubuntu-latest" - steps: - - name: Download Artifact - uses: actions/download-artifact@v3 - with: - name: rrd - path: rrd - - - id: "auth" - uses: google-github-actions/auth@v1 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: Add SHORT_SHA env property with commit short sha - run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV - - - name: "Upload RRD (commit)" - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "rrd" - destination: "rerun-example-rrd/commit/${{env.SHORT_SHA}}" - parent: false - - - name: "Upload RRD (prerelease)" - if: "!startsWith(github.ref , 'refs/tags/v')" - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "rrd" - destination: "rerun-example-rrd/prerelease" - parent: false - - - name: "Upload RRD (tagged)" - if: startsWith(github.ref, 'refs/tags/v') - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "rrd" - destination: "rerun-example-rrd/version/${{github.ref_name}}" - parent: false - - - name: "Upload RRD (latest release)" - if: github.ref == 'latest' - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "rrd" - destination: "rerun-example-rrd/latest" - parent: false - - # See https://github.com/ncipollo/release-action - pre-release: - name: Pre Release - needs: [wheels] - if: github.ref == 'refs/heads/main' - runs-on: "ubuntu-latest" - steps: - - name: Download Artifact - uses: actions/download-artifact@v3 - with: - name: wheels - path: dist - - # First delete the old prerelease. If we don't do this, we don't get things like - # proper source-archives and changelog info. - # https://github.com/dev-drprasad/delete-tag-and-release - - uses: dev-drprasad/delete-tag-and-release@v0.2.1 - with: - tag_name: prerelease - delete_release: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Create the actual prerelease - # https://github.com/ncipollo/release-action - - name: GitHub Release - uses: ncipollo/release-action@v1.12.0 - with: - body: ${{ env.PRE_RELEASE_INSTRUCTIONS }} - prerelease: true - artifacts: dist/* - name: "Development Build" - tag: "prerelease" - token: ${{ secrets.GITHUB_TOKEN }} - generateReleaseNotes: true - allowUpdates: true - removeArtifacts: true - replacesArtifacts: true - - # --------------------------------------------------------------------------- - - # This job is run on tags starting with "v", e.g., "v0.1.0" - tagged-release: - name: Release - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/v') - needs: [wheels] - steps: - - name: Download Artifact - uses: actions/download-artifact@v3 - with: - name: wheels - path: dist - - # https://github.com/ncipollo/release-action - - name: GitHub Release - uses: ncipollo/release-action@v1.12.0 - with: - prerelease: true - artifacts: dist/* - token: ${{ secrets.GITHUB_TOKEN }} - generateReleaseNotes: true - - - name: Publish to PyPI - uses: PyO3/maturin-action@v1 - env: - # These are both set in the GitHub project configuration - MATURIN_REPOSITORY: ${{ vars.PYPI_REPOSITORY }} - MATURIN_PYPI_TOKEN: ${{ secrets.MATURIN_PYPI_TOKEN }} - with: - command: upload - args: --skip-existing dist/* - - # --------------------------------------------------------------------------- - - py-test-docs: - name: Verify the docs build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - cache: "pip" - cache-dependency-path: "rerun_py/requirements-doc.txt" - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-doc.txt - - - name: Build via mkdocs - run: | - mkdocs build -f rerun_py/mkdocs.yml - - py-docs: - name: Build and deploy docs - if: ${{ github.event_name == 'push' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Don't do a shallow clone - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - cache: "pip" - cache-dependency-path: "rerun_py/requirements-doc.txt" - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-doc.txt - - - name: Set up git author - run: | - remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Mike will incrementally update the existing gh-pages branch - # We then check it out, and reset it to a new orphaned branch, which we force-push to origin - # to make sure we don't accumulate unnecessary history in gh-pages branch - - name: Deploy via mike # https://github.com/jimporter/mike - if: startsWith(github.ref, 'refs/tags/v') - run: | - git fetch - mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{github.ref_name}} latest - git checkout gh-pages - git checkout --orphan gh-pages-orphan - git commit -m "Update docs for ${GITHUB_SHA}" - git push origin gh-pages-orphan:gh-pages -f - - # Mike will incrementally update the existing gh-pages branch - # We then check it out, and reset it to a new orphaned branch, which we force-push to origin - # to make sure we don't accumulate unnecessary history in gh-pages branch - - name: Deploy tag via mike # https://github.com/jimporter/mike - if: github.ref == 'refs/heads/main' - run: | - git fetch - mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python HEAD - git checkout gh-pages - git checkout --orphan gh-pages-orphan - git commit -m "Update docs for ${GITHUB_SHA}" - git push origin gh-pages-orphan:gh-pages -f diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index f48703b53395..000000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,368 +0,0 @@ -name: CI (Rust) - -on: - push: - branches: - - "main" - pull_request: - workflow_dispatch: - inputs: - force_update_web_build: - description: "Upload web build to google cloud" - required: true - default: false - type: boolean - force_run_benchmarks: - description: "Run the rust benchmarks" - required: true - default: false - type: boolean - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} # Cancel previous CI jobs on the same branch - cancel-in-progress: true - -env: - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - # See https://github.com/ericseppanen/cargo-cranky/issues/8 - RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs - -permissions: - # deployments permission to deploy GitHub pages website - deployments: write - # contents permission to update benchmark contents in gh-pages branch - contents: write - -jobs: - # See the benchmarks at https://ref.rerun.io/dev/bench/ - rs-benchmarks: - name: Rust Criterion benchmarks - - # Only run the rs-benchmarks job when a commit gets pushed to main or we manually specify it - if: ${{ github.ref == 'refs/heads/main' || github.event.inputs.force_run_benchmarks }} - - runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.6 - env: - RUSTFLAGS: ${{env.RUSTFLAGS}} - RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} - steps: - - uses: actions/checkout@v3 - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # Don't update the cache -- it will be updated by the lint job - # TODO(jleibs): this job will likely run before rust.yml updates - # the cache. Better cross-job sequencing would be nice here - save-if: false - - - name: Run benchmark - # Use bash shell so we get pipefail behavior with tee - shell: bash - run: | - cargo bench \ - --all-features \ - -p re_arrow_store \ - -p re_data_store \ - -p re_log_encoding \ - -p re_query \ - -p re_tuid \ - -- --output-format=bencher | tee output.txt - - - name: Store benchmark result - # https://github.com/benchmark-action/github-action-benchmark - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Rust Benchmark - tool: "cargo" - output-file-path: output.txt - github-token: ${{ secrets.GITHUB_TOKEN }} - - # Show alert with commit comment on detecting possible performance regression - comment-on-alert: true - alert-threshold: "150%" - fail-on-alert: true - comment-always: false # Generates too much GitHub notification spam - - # Save, results and push to GitHub only on main - save-data-file: ${{ github.ref == 'refs/heads/main' }} - auto-push: ${{ github.ref == 'refs/heads/main' }} - gh-pages-branch: gh-pages - benchmark-data-dir-path: dev/bench - max-items-in-chart: 30 - - # --------------------------------------------------------------------------- - - rs-lints: - name: Rust lints (fmt, check, cranky, tests, doc) - runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.6 - env: - RUSTFLAGS: ${{env.RUSTFLAGS}} - RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Don't do a shallow clone since we need to push gh-pages - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # See: https://github.com/rerun-io/rerun/pull/497 - save-if: ${{ github.event_name == 'push'}} - - # First do our check with --locked to make sure `Cargo.lock` is up to date - - name: Check all features - uses: actions-rs/cargo@v1 - with: - command: check - args: --locked --all-features - - - name: Rustfmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check - - - name: Cranky - uses: actions-rs/cargo@v1 - with: - command: cranky - args: --all-targets --all-features -- --deny warnings - - # -------------------------------------------------------------------------------- - # Check a few important permutations of the feature flags for our `rerun` library: - - name: Check rerun with `--no-default-features`` - uses: actions-rs/cargo@v1 - with: - command: cranky - args: --locked -p rerun --no-default-features - - - name: Check rerun with `--features sdk` - uses: actions-rs/cargo@v1 - with: - command: cranky - args: --locked -p rerun --no-default-features --features sdk - # -------------------------------------------------------------------------------- - - - name: Test doc-tests - uses: actions-rs/cargo@v1 - with: - command: test - args: --doc --all-features - - - name: cargo doc - uses: actions-rs/cargo@v1 - with: - command: doc - args: --no-deps --all-features - - - name: cargo doc --document-private-items - uses: actions-rs/cargo@v1 - with: - command: doc - args: --document-private-items --no-deps --all-features - - - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: --all-targets --all-features - - - name: Set up git author - if: ${{ github.event_name == 'push' }} - run: | - remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up ghp-import - if: ${{ github.event_name == 'push' }} - run: pip install ghp-import - - - name: Patch in a redirect page - if: ${{ github.event_name == 'push' }} - run: echo "" > target/doc/index.html - env: - REDIRECT_CRATE: rerun - - # See: https://github.com/c-w/ghp-import - - name: Deploy the docs - if: ${{ github.event_name == 'push' }} - run: | - git fetch - python3 -m ghp_import -n -p -x docs/rust/head target/doc/ -m "Update the rust docs" - - # --------------------------------------------------------------------------- - - rs-check-wasm: - name: Check Rust web build (wasm32 + wasm-bindgen) - runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.6 - env: - RUSTFLAGS: ${{env.RUSTFLAGS}} - RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} - steps: - - uses: actions/checkout@v2 - - - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: 1.67.1 - target: wasm32-unknown-unknown - override: true - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # See: https://github.com/rerun-io/rerun/pull/497 - save-if: ${{ github.event_name == 'push'}} - - - name: clippy check re_viewer wasm32 - run: ./scripts/clippy_wasm.sh - - - name: Check re_renderer examples wasm32 - uses: actions-rs/cargo@v1 - with: - command: check - args: --locked --target wasm32-unknown-unknown --target-dir target_wasm -p re_renderer --examples - - - name: Build web-viewer (debug) - uses: actions-rs/cargo@v1 - with: - command: run - args: --locked -p re_build_web_viewer -- --debug - - # --------------------------------------------------------------------------- - - rs-build-web-viewer: - name: Upload web build to google cloud (wasm32 + wasm-bindgen) - permissions: - contents: "read" - id-token: "write" - - if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || github.event.inputs.force_update_web_build - runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.6 - env: - RUSTFLAGS: ${{env.RUSTFLAGS}} - RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} - steps: - - uses: actions/checkout@v2 - - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: 1.67.0 - target: wasm32-unknown-unknown - override: true - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # See: https://github.com/rerun-io/rerun/pull/497 - save-if: ${{ github.event_name == 'push'}} - - - name: Patch Cargo.toml for pre-release - if: "!startsWith(github.ref , 'refs/tags/v')" - # After patching the pre-release version, run cargo update. - # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing - run: | - python3 scripts/version_util.py --patch_prerelease - cargo update -w - - - name: Build web-viewer (release) - uses: actions-rs/cargo@v1 - with: - command: run - args: --locked -p re_build_web_viewer -- --release - - # Upload the wasm, html etc to a Google cloud bucket: - - id: "auth" - uses: google-github-actions/auth@v1 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: Add SHORT_SHA env property with commit short sha - run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV - - - name: "Upload web-viewer (commit)" - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "web_viewer" - destination: "rerun-web-viewer/commit/${{env.SHORT_SHA}}" - parent: false - - - name: "Upload web-viewer (prerelease)" - if: github.ref == 'refs/heads/main' - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "web_viewer" - destination: "rerun-web-viewer/prerelease" - parent: false - - - name: "Upload web-viewer (tagged)" - if: startsWith(github.ref, 'refs/tags/v') - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "web_viewer" - destination: "rerun-web-viewer/version/${{github.ref_name}}" - parent: false - - - name: "Upload web-viewer (latest release)" - if: github.ref == 'latest' - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "web_viewer" - destination: "rerun-web-viewer/latest" - parent: false - - # --------------------------------------------------------------------------- - - rs-cargo-deny: - name: Check Rust dependencies (cargo-deny) - runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.6 - env: - RUSTFLAGS: ${{env.RUSTFLAGS}} - RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}} - - # TODO(emilk): remove this matrix when https://github.com/EmbarkStudios/cargo-deny/issues/324 is resolved - strategy: - fail-fast: false - matrix: - platform: - - i686-pc-windows-gnu - - i686-pc-windows-msvc - - i686-unknown-linux-gnu - - wasm32-unknown-unknown - - x86_64-apple-darwin - - x86_64-pc-windows-gnu - - x86_64-pc-windows-msvc - - x86_64-unknown-linux-gnu - - x86_64-unknown-redox - - steps: - - uses: actions/checkout@v3 - - uses: EmbarkStudios/cargo-deny-action@v1 - with: - command: check - log-level: error - arguments: --all-features --target ${{ matrix.platform }} diff --git a/.github/workflows/toml.yml b/.github/workflows/toml.yml deleted file mode 100644 index f3a60ec74885..000000000000 --- a/.github/workflows/toml.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: CI (TOML) - -on: - pull_request: - push: - branches: - - "main" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} # Cancel previous CI jobs on the same branch - cancel-in-progress: true - -jobs: - toml-lints: - name: Lint TOML files - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: 1.67.1 - override: true - - - name: Set up cargo cache - uses: Swatinem/rust-cache@v2 - with: - # See: https://github.com/rerun-io/rerun/pull/497 - env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # Don't update the cache -- it will be updated by the lint job - # TODO(jleibs): this job will likely run before rust.yml updates - # the cache. Better cross-job sequencing would be nice here - save-if: false - - - name: Install taplo-cli - uses: baptiste0928/cargo-install@v1 - with: - crate: taplo-cli - - - name: Taplo check - run: | - taplo fmt --check diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml deleted file mode 100644 index c3bc84125f51..000000000000 --- a/.github/workflows/typos.yml +++ /dev/null @@ -1,17 +0,0 @@ -# https://github.com/crate-ci/typos -# Add exceptions to _typos.toml -# install and run locally: cargo install typos-cli && typos - -name: Spell Check -on: [pull_request] - -jobs: - run: - name: Spell Check - runs-on: ubuntu-latest - steps: - - name: Checkout Actions Repository - uses: actions/checkout@v2 - - - name: Check spelling of entire workspace - uses: crate-ci/typos@master From 4316b7b98e0817a80c150b5535d766128761b056 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 22:27:07 +0200 Subject: [PATCH 064/106] Enable concurrency check for on_pull_request --- .github/workflows/on_pull_request.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index ce04c594b06a..3ef7adb64a82 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -6,6 +6,10 @@ on: - opened - synchronize +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: checks: name: Run All Checks From 2ee0a03f459674160eedf1d6216051dc266605f3 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 23:11:35 +0200 Subject: [PATCH 065/106] Override the commit used for uploading web --- .github/workflows/on_pull_request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index 3ef7adb64a82..c976d2870588 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -50,6 +50,7 @@ jobs: with: RELEASE: '' RRD_ARTIFACT_NAME: linux-rrd-fast + UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }} secrets: inherit save_pr_summary: From c97b9af2b125f4f6e15b311571c19b090abfe922 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Sat, 22 Apr 2023 23:47:37 +0200 Subject: [PATCH 066/106] Don't gate wheel-upload on running the tests since we can't run tests on mac --- .../workflows/reusable_build_and_test_wheels.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index 544b558623eb..9e93f27dda00 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -225,6 +225,13 @@ jobs: ${{ inputs.MATURIN_FEATURE_FLAGS }} --out dist + - name: Save wheel artifact + if: ${{ inputs.WHEEL_ARTIFACT_NAME != '' }} + uses: actions/upload-artifact@v3 + with: + name: ${{inputs.WHEEL_ARTIFACT_NAME}} + path: dist + # --------------------------------------------------------------------------- # Test the wheel @@ -292,9 +299,3 @@ jobs: name: ${{ inputs.RRD_ARTIFACT_NAME }} path: rrd - - name: Save wheel artifact - if: ${{ (needs.set_config.outputs.RUN_TESTS == 'true') && (inputs.WHEEL_ARTIFACT_NAME != '') }} - uses: actions/upload-artifact@v3 - with: - name: ${{inputs.WHEEL_ARTIFACT_NAME}} - path: dist From 4bf9deddc73f6f80bcbc211d40fffd6b04b0c630 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 19:13:36 +0200 Subject: [PATCH 067/106] Split out the deploy docs workflow --- .github/workflows/manual_dispatch.yml | 22 +-- .github/workflows/reusable_checks.yml | 89 ------------ .github/workflows/reusable_deploy_docs.yml | 154 +++++++++++++++++++++ 3 files changed, 168 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/reusable_deploy_docs.yml diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 297cb944d449..86eaacfe886d 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -20,6 +20,11 @@ on: type: boolean required: false default: true + DEPLOY_DOCS: + description: 'Deploy the docs to refs.rerun.io' + type: boolean + required: false + default: false MIN_TEST_WHEEL: description: 'Test Minimum Wheel' type: boolean @@ -34,11 +39,6 @@ on: type: boolean required: false default: false - SAVE_DOCS: - description: 'Save Docs to refs.rerun.io' - type: boolean - required: false - default: false SAVE_PR: description: 'Save PR Summary' type: string @@ -62,9 +62,15 @@ jobs: uses: ./.github/workflows/reusable_checks.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} - SAVE_PY_DOCS: ${{ github.event.inputs.SAVE_DOCS == 'true' }} - SAVE_RUST_DOCS: ${{ github.event.inputs.SAVE_DOCS == 'true' }} - SAVE_PY_DOCS_AS: ${{ github.event.inputs.RELEASE }} + secrets: inherit + + deploy_docs: + name: Deploy Docs + if: ${{ github.event.inputs.DEPLOY_DOCS == 'true' }} + uses: ./.github/workflows/reusable_deploy_docs.yml + with: + PY_DOCS_VERSION_NAME: "test" + UPDATE_LATEST: false secrets: inherit min_test_wheel: diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index d1cb1e8858c8..e62e7d17419f 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -93,66 +93,6 @@ jobs: run: | mkdocs build -f rerun_py/mkdocs.yml - -# --------------------------------------------------------------------------- - - py-deploy-docs: - needs: [py-test-docs] - name: Deploy Python Docs - if: ${{ inputs.SAVE_PY_DOCS }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Don't do a shallow clone - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - cache: "pip" - cache-dependency-path: "rerun_py/requirements-doc.txt" - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-doc.txt - - - name: Set up git author - run: | - remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Mike will incrementally update the existing gh-pages branch - # We then check it out, and reset it to a new orphaned branch, which we force-push to origin - # to make sure we don't accumulate unnecessary history in gh-pages branch - - name: Deploy via mike # https://github.com/jimporter/mike - if: ${{ inputs.SAVE_PY_DOCS_AS != 'HEAD' }} - run: | - git fetch - mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.SAVE_PY_DOCS_AS}} latest - git checkout gh-pages - git checkout --orphan gh-pages-orphan - git commit -m "Update docs for ${GITHUB_SHA}" - git push origin gh-pages-orphan:gh-pages -f - - # Mike will incrementally update the existing gh-pages branch - # We then check it out, and reset it to a new orphaned branch, which we force-push to origin - # to make sure we don't accumulate unnecessary history in gh-pages branch - - name: Deploy tag via mike # https://github.com/jimporter/mike - if: ${{ inputs.SAVE_PY_DOCS_AS == 'HEAD' }} - run: | - git fetch - mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python HEAD - git checkout gh-pages - git checkout --orphan gh-pages-orphan - git commit -m "Update docs for ${GITHUB_SHA}" - git push origin gh-pages-orphan:gh-pages -f - - # --------------------------------------------------------------------------- rs-lints: @@ -174,8 +114,6 @@ jobs: INPUTS_CONTEXT: ${{ toJson(inputs) }} - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Don't do a shallow clone since we need to push gh-pages # Rust-cache will cache our dependencies, which is a large chunk of the build # See: https://github.com/Swatinem/rust-cache @@ -251,33 +189,6 @@ jobs: command: test args: --all-targets --all-features - # TODO(jleibs): Maybe we should run this on a separate job? - - name: Set up git author - if: ${{ inputs.SAVE_RUST_DOCS }} - run: | - remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up ghp-import - if: ${{ inputs.SAVE_RUST_DOCS }} - run: pip install ghp-import - - - name: Patch in a redirect page - if: ${{ inputs.SAVE_RUST_DOCS }} - run: echo "" > target/doc/index.html - env: - REDIRECT_CRATE: rerun - - # See: https://github.com/c-w/ghp-import - - name: Deploy the docs - if: ${{ inputs.SAVE_RUST_DOCS }} - run: | - git fetch - python3 -m ghp_import -n -p -x docs/rust/head target/doc/ -m "Update the rust docs" - # --------------------------------------------------------------------------- rs-check-wasm: diff --git a/.github/workflows/reusable_deploy_docs.yml b/.github/workflows/reusable_deploy_docs.yml new file mode 100644 index 000000000000..85629327663c --- /dev/null +++ b/.github/workflows/reusable_deploy_docs.yml @@ -0,0 +1,154 @@ +name: 'Reusable Deploy Docs' + +on: + workflow_call: + inputs: + PY_DOCS_VERSION_NAME: + required: true + type: string + UPDATE_LATEST: + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + +# --------------------------------------------------------------------------- + + deploy-docs-py: + name: Python + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Don't do a shallow clone + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + cache: "pip" + cache-dependency-path: "rerun_py/requirements-doc.txt" + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install -r rerun_py/requirements-doc.txt + + - name: Set up git author + run: | + remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Mike will incrementally update the existing gh-pages branch + # We then check it out, and reset it to a new orphaned branch, which we force-push to origin + # to make sure we don't accumulate unnecessary history in gh-pages branch + - name: Deploy via mike # https://github.com/jimporter/mike + if: ${{ inputs.UPDATE_LATEST }} + run: | + git fetch + mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.PY_DOCS_VERSION_NAME}} latest + git checkout gh-pages + git checkout --orphan gh-pages-orphan + git commit -m "Update docs for ${GITHUB_SHA}" + git push origin gh-pages-orphan:gh-pages -f + + # Mike will incrementally update the existing gh-pages branch + # We then check it out, and reset it to a new orphaned branch, which we force-push to origin + # to make sure we don't accumulate unnecessary history in gh-pages branch + - name: Deploy tag via mike # https://github.com/jimporter/mike + if: ${{ ! inputs.UPDATE_LATEST }} + run: | + git fetch + mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python ${{inputs.PY_DOCS_VERSION_NAME}} + git checkout gh-pages + git checkout --orphan gh-pages-orphan + git commit -m "Update docs for ${GITHUB_SHA}" + git push origin gh-pages-orphan:gh-pages -f + + +# --------------------------------------------------------------------------- + + deploy-docs-rs: + name: Rust + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.6 + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Don't do a shallow clone since we need to push gh-pages + + # Rust-cache will cache our dependencies, which is a large chunk of the build + # See: https://github.com/Swatinem/rust-cache + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-linux" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: false + + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: cargo doc --document-private-items + uses: actions-rs/cargo@v1 + with: + command: doc + args: --document-private-items --no-deps --all-features + + - name: Set up git author + run: | + remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up ghp-import + run: pip install ghp-import + + - name: Patch in a redirect page + run: echo "" > target/doc/index.html + env: + REDIRECT_CRATE: rerun + + # See: https://github.com/c-w/ghp-import + - name: Deploy the docs + run: | + git fetch + python3 -m ghp_import -n -p -x docs/rust/head target/doc/ -m "Update the rust docs" From 009482e0f38767d59bba864e8917dac10f8ee42d Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 19:24:48 +0200 Subject: [PATCH 068/106] Comment and dep cleanup --- .github/workflows/manual_dispatch.yml | 4 +++- .github/workflows/reusable_deploy_docs.yml | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 86eaacfe886d..0921184e91c1 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -65,6 +65,8 @@ jobs: secrets: inherit deploy_docs: + # Never deploy docs if checks haven't passed + needs: [checks] name: Deploy Docs if: ${{ github.event.inputs.DEPLOY_DOCS == 'true' }} uses: ./.github/workflows/reusable_deploy_docs.yml @@ -75,7 +77,7 @@ jobs: min_test_wheel: name: 'Minimum Test Wheel' - # The upload-web job uses the min-test-wheel to get the RRD in manual mode + # The upload-web job uses the min-test-wheel to get the RRD if: ${{ (github.event.inputs.MIN_TEST_WHEEL == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.BUILD_WEB == 'true') ) }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: diff --git a/.github/workflows/reusable_deploy_docs.yml b/.github/workflows/reusable_deploy_docs.yml index 85629327663c..86e5fb403693 100644 --- a/.github/workflows/reusable_deploy_docs.yml +++ b/.github/workflows/reusable_deploy_docs.yml @@ -115,9 +115,6 @@ jobs: with: shared-key: "build-linux" env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY - # Don't update the cache -- it will be updated by the lint job - # TODO(jleibs): this job will likely run before rust.yml updates - # the cache. Better cross-job sequencing would be nice here save-if: false # Sccache will cache everything else From 0404db9a2b2fb8fcc77704e16f734d6080c915e4 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 19:38:39 +0200 Subject: [PATCH 069/106] Clarify RELEASE_VERSION --- .github/workflows/manual_dispatch.yml | 14 ++++++--- .../reusable_build_and_test_wheels.yml | 27 ++++++++++------ .github/workflows/reusable_build_web.yml | 31 +++++++++++++++++-- .github/workflows/reusable_upload_web.yml | 24 +++++++++----- 4 files changed, 72 insertions(+), 24 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 0921184e91c1..9a9774c3d67a 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -29,11 +29,11 @@ on: description: 'Test Minimum Wheel' type: boolean required: false - RELEASE: + RELEASE_VERSION: description: 'Release' type: string required: false - default: '' + default: 'prerelease' SAVE_CACHE: description: 'Save Cache' type: boolean @@ -107,6 +107,7 @@ jobs: PLATFORM: linux WHEEL_ARTIFACT_NAME: linux-wheel RRD_ARTIFACT_NAME: linux-rrd + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} secrets: inherit build_windows: @@ -118,6 +119,7 @@ jobs: PLATFORM: windows WHEEL_ARTIFACT_NAME: windows-wheel RRD_ARTIFACT_NAME: '' + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} secrets: inherit build_macos_arm: @@ -129,6 +131,7 @@ jobs: PLATFORM: macos-arm WHEEL_ARTIFACT_NAME: macos-arm-wheel RRD_ARTIFACT_NAME: '' + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} secrets: inherit build_macos_intel: @@ -140,6 +143,8 @@ jobs: PLATFORM: macos-intel WHEEL_ARTIFACT_NAME: 'macos-intel-wheel' RRD_ARTIFACT_NAME: '' + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} + secrets: inherit upload_wheels_linux: @@ -191,7 +196,7 @@ jobs: if: ${{ github.event.inputs.BUILD_WEB == 'true'}} uses: ./.github/workflows/reusable_build_web.yml with: - RELEASE: ${{ github.event.inputs.RELEASE }} + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} secrets: inherit upload_web: @@ -203,7 +208,8 @@ jobs: if: ${{ (github.event.inputs.BUILD_WEB == 'true') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_web.yml with: - RELEASE: ${{ github.event.inputs.RELEASE }} + MARK_PRERELEASE_FOR_MAINLINE: false + MARK_TAGGED_VERSION: false RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index 9e93f27dda00..6be32798e427 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -10,22 +10,22 @@ on: PLATFORM: required: true type: string - RELEASE: + RELEASE_VERSION: required: false type: string - default: '' + default: 'prerelease' RRD_ARTIFACT_NAME: required: false type: string default: '' - WHEEL_ARTIFACT_NAME: - required: false - type: string - default: '' SAVE_CACHE: required: false type: boolean default: false + WHEEL_ARTIFACT_NAME: + required: false + type: string + default: '' outputs: EXPECTED_VERSION: description: "The expected version of the package" @@ -188,17 +188,14 @@ jobs: pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" - name: Patch Cargo.toml for pre-release - if: ${{ (inputs.RELEASE == '') || (inputs.RELEASE == 'prerelease') }} + if: ${{ inputs.RELEASE_VERSION == 'prerelease' }} # After patching the pre-release version, run cargo update. # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing run: | python3 scripts/version_util.py --patch_prerelease cargo update -w - # TODO(jleibs): Figure out what this check should be doing now - name: Version check for tagged-release - if: false - #if: startsWith(github.ref, 'refs/tags/v') # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z run: | python3 scripts/version_util.py --check_version @@ -210,6 +207,16 @@ jobs: run: | echo "EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" + - name: Check the expected version + if: ${{ inputs.RELEASE_VERSION != 'prerelease' }} + # After patching the pre-release version, run cargo update. + # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing + run: | + if [ "${{steps.expected_version.outputs.EXPECTED_VERSION}}" != "${{inputs.RELEASE_VERSION}}" ]; then + echo "Error: EXPECTED_VERSION (${{steps.expected_version.outputs.EXPECTED_VERSION}}) does not match RELEASE_VERSION (${{inputs.RELEASE_VERSION}})" + exit 1 + fi + - name: Build Wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 40e6fd87ae0e..243971979ca6 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -3,7 +3,7 @@ name: Reusable Build Web on: workflow_call: inputs: - RELEASE: + RELEASE_VERSION: required: false type: string default: '' @@ -48,13 +48,40 @@ jobs: save-if: ${{ github.event_name == 'push'}} - name: Patch Cargo.toml for pre-release - if: ${{ (inputs.RELEASE == '') || (inputs.RELEASE == 'prerelease') }} + if: ${{ inputs.RELEASE_VERSION == 'prerelease' }} # After patching the pre-release version, run cargo update. # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing run: | python3 scripts/version_util.py --patch_prerelease cargo update -w + - name: Version check for tagged-release + # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z + run: | + python3 scripts/version_util.py --check_version + + - name: Version check for tagged-release + # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z + run: | + python3 scripts/version_util.py --check_version + + - name: Store the expected version + # Find the current cargo version and store it in the output: `expected_version` + shell: bash + id: expected_version + run: | + echo "EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" + + - name: Check the expected version + if: ${{ inputs.RELEASE_VERSION != 'prerelease' }} + # After patching the pre-release version, run cargo update. + # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing + run: | + if [ "${{steps.expected_version.outputs.EXPECTED_VERSION}}" != "${{inputs.RELEASE_VERSION}}" ]; then + echo "Error: EXPECTED_VERSION (${{steps.expected_version.outputs.EXPECTED_VERSION}}) does not match RELEASE_VERSION (${{inputs.RELEASE_VERSION}})" + exit 1 + fi + - name: Build web-viewer (release) uses: actions-rs/cargo@v1 with: diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index 7255dd23b0c3..f43ca28b4616 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -3,10 +3,18 @@ name: Reusable Upload Web on: workflow_call: inputs: - RELEASE: + MARK_PRERELEASE_FOR_MAINLINE: + required: false + type: boolean + default: false + MARK_TAGGED_VERSION: + required: false + type: boolean + default: false + RELEASE_VERSION: required: false type: string - default: '' + default: 'prerelease' RRD_ARTIFACT_NAME: required: false type: string @@ -75,7 +83,7 @@ jobs: parent: false - name: "Upload web-viewer (prerelease)" - if: inputs.RELEASE == 'prerelease' + if: inputs.MARK_PRERELEASE_FOR_MAINLINE uses: google-github-actions/upload-cloud-storage@v1 with: path: "web_viewer" @@ -84,7 +92,7 @@ jobs: - name: "Upload RRD (prerelease)" - if: inputs.RELEASE == 'prerelease' + if: inputs.MARK_PRERELEASE_FOR_MAINLINE uses: google-github-actions/upload-cloud-storage@v1 with: path: "rrd" @@ -92,17 +100,17 @@ jobs: parent: false - name: "Upload web-viewer (tagged)" - if: ${{ (inputs.RELEASE != '') && (inputs.RELEASE != 'prerelease') }} + if: inputs.MARK_TAGGED_VERSION uses: google-github-actions/upload-cloud-storage@v1 with: path: "web_viewer" - destination: "rerun-web-viewer/version/${{inputs.RELEASE}}" + destination: "rerun-web-viewer/version/${{inputs.RELEASE_VERSION}}" parent: false - name: "Upload RRD (tagged)" - if: ${{ (inputs.RELEASE != '') && (inputs.RELEASE != 'prerelease') }} + if: inputs.MARK_TAGGED_VERSION uses: google-github-actions/upload-cloud-storage@v1 with: path: "rrd" - destination: "rerun-example-rrd/version/${{inputs.RELEASE}}" + destination: "rerun-example-rrd/version/${{inputs.RELEASE_VERSION}}" parent: false From 2b332b7c0d5f92b974c47bf3cbbe648a9e6ad7f2 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 20:11:26 +0200 Subject: [PATCH 070/106] The version check now happens between RELEASE_VERSION rather than GITHUB_REF --- .../reusable_build_and_test_wheels.yml | 5 --- .github/workflows/reusable_build_web.yml | 5 --- .github/workflows/reusable_upload_wheels.yml | 13 ------- scripts/version_util.py | 34 ++----------------- 4 files changed, 2 insertions(+), 55 deletions(-) diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index 6be32798e427..854ffa0c5a4c 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -195,11 +195,6 @@ jobs: python3 scripts/version_util.py --patch_prerelease cargo update -w - - name: Version check for tagged-release - # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z - run: | - python3 scripts/version_util.py --check_version - - name: Store the expected version # Find the current cargo version and store it in the output: `expected_version` shell: bash diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 243971979ca6..6c1bbb7b23a0 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -60,11 +60,6 @@ jobs: run: | python3 scripts/version_util.py --check_version - - name: Version check for tagged-release - # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z - run: | - python3 scripts/version_util.py --check_version - - name: Store the expected version # Find the current cargo version and store it in the output: `expected_version` shell: bash diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml index b2efe7bf1217..208f0fad2d5e 100644 --- a/.github/workflows/reusable_upload_wheels.yml +++ b/.github/workflows/reusable_upload_wheels.yml @@ -3,13 +3,6 @@ name: Reusable Upload Wheels on: workflow_call: inputs: - EXPECTED_VERSION: - required: true - type: string - RELEASE: - required: false - type: string - default: '' RRD_ARTIFACT_NAME: required: false type: string @@ -95,12 +88,6 @@ jobs: - name: Add SHORT_SHA env property with commit short sha run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV - - name: Get the file name - shell: bash - id: get_file_name - run: | - echo "WHL_FILE=$(ls dist)" >> "$GITHUB_OUTPUT" - - name: "Upload Wheel" uses: google-github-actions/upload-cloud-storage@v1 with: diff --git a/scripts/version_util.py b/scripts/version_util.py index 52f555966f23..e694f24c0b80 100755 --- a/scripts/version_util.py +++ b/scripts/version_util.py @@ -7,12 +7,10 @@ --patch_prerelease: This will patch the version in rerun/Cargo.toml with the current git sha. This is intended to create a prerelease version for continuous releases. - --check_version: This will check that the version in rerun/Cargo.toml matches the version in the tag name from - `GITHUB_REF_NAME`. This is intended to be used to check that the version number in Cargo.toml is correct before - creating a release on PyPI. If the versions don't match, an exception will be raised. + --bare_cargo_version Outputs the bare cargo version. This is helpful for setting an environment variable, such as: + EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version) """ -import os import re import subprocess import sys @@ -24,9 +22,6 @@ # A regex to match the version number in Cargo.toml as SemVer, e.g., 1.2.3-alpha.0 CARGO_VERSION_REGEX: Final = r"^version\s*=\s*\"(.+)\"$" -# A regex to match the version number in the tag name, e.g. v1.2.3 -VERSION_TAG_REGEX: Final = r"^v(.+)$" - def get_cargo_version(cargo_toml: str) -> semver.VersionInfo: """Using regex, parse the version number from Cargo.toml.""" @@ -44,24 +39,6 @@ def get_git_sha() -> str: return subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("utf-8").strip() -def get_ref_name_version() -> semver.VersionInfo: - """Return the parsed tag version from the GITHUB_REF_NAME environment variable.""" - - # This is the branch, or tag name that triggered the workflow. - ref_name = os.environ.get("GITHUB_REF_NAME") - - if ref_name is None: - raise Exception("GITHUB_REF_NAME environment variable not set") - - # Extract the version number from the tag name - match = re.search(VERSION_TAG_REGEX, ref_name) - - if match is None: - raise Exception("Could not find valid version number in GITHUB_REF_NAME") - - return semver.parse_version_info(match.groups()[0]) - - def patch_cargo_version(cargo_toml: str, new_version: str) -> str: """Patch the version number in Cargo.toml with `new_version`.""" @@ -99,13 +76,6 @@ def main() -> None: with open("Cargo.toml", "w") as f: f.write(new_cargo_toml) - elif sys.argv[1] == "--check_version": - ref_version = get_ref_name_version() - if cargo_version != ref_version: - raise Exception( - f"Version number in Cargo.toml ({cargo_version}) does not match tag version ({ref_version})" - ) - print(f"Version numbers match: {cargo_version} == {ref_version}") elif sys.argv[1] == "--bare_cargo_version": # Print the bare cargo version. NOTE: do not add additional formatting here. This output # is expected to be fed into an environment variable. From 492bf5d980ec47cfc46ca6087f3f41677d88b538 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 20:17:20 +0200 Subject: [PATCH 071/106] Upload wheels no longer needs the expected value --- .github/workflows/manual_dispatch.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 9a9774c3d67a..742f1806c2bd 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -153,7 +153,6 @@ jobs: if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: linux-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit @@ -164,7 +163,6 @@ jobs: if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: windows-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit @@ -175,7 +173,6 @@ jobs: if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: macos-arm-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit @@ -186,7 +183,6 @@ jobs: if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: macos-intel-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit From 52e60c5574b568ffc0acb7f765abc6a379bb5191 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 20:18:24 +0200 Subject: [PATCH 072/106] More sane defaults --- .github/workflows/manual_dispatch.yml | 4 ++-- .github/workflows/reusable_build_and_test_wheels.yml | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 742f1806c2bd..00591cec3b14 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -14,7 +14,7 @@ on: description: 'Build Web' type: boolean required: false - default: false + default: true CHECKS: description: 'Run All Checks' type: boolean @@ -53,7 +53,7 @@ on: description: 'Platforms to build wheels for' type: string required: false - default: '' + default: 'linux' jobs: checks: diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index 854ffa0c5a4c..28eb483985fb 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -26,10 +26,6 @@ on: required: false type: string default: '' - outputs: - EXPECTED_VERSION: - description: "The expected version of the package" - value: ${{ jobs.build_wheels.outputs.EXPECTED_VERSION }} env: PYTHON_VERSION: "3.8" @@ -106,9 +102,6 @@ jobs: runs-on: ${{ needs.set_config.outputs.RUNNER }} container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} - outputs: - EXPECTED_VERSION: ${{ steps.expected_version.outputs.EXPECTED_VERSION }} - steps: - name: Show context run: | @@ -116,7 +109,6 @@ jobs: echo "JOB_CONTEXT": $JOB_CONTEXT echo "INPUTS_CONTEXT": $INPUTS_CONTEXT echo "ENV_CONTEXT": $ENV_CONTEXT - echo "EXPECTED_VERSION": $EXPECTED_VERSION env: ENV_CONTEXT: ${{ toJson(env) }} GITHUB_CONTEXT: ${{ toJson(github) }} From 27dfcdfa79dbca0b0b138b0d0d445433b18b9f03 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 20:31:47 +0200 Subject: [PATCH 073/106] Fix bad workflow edit --- .github/workflows/reusable_build_web.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 6c1bbb7b23a0..1bb9f829077b 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -45,7 +45,7 @@ jobs: with: env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY # See: https://github.com/rerun-io/rerun/pull/497 - save-if: ${{ github.event_name == 'push'}} + save-if: false - name: Patch Cargo.toml for pre-release if: ${{ inputs.RELEASE_VERSION == 'prerelease' }} @@ -55,11 +55,6 @@ jobs: python3 scripts/version_util.py --patch_prerelease cargo update -w - - name: Version check for tagged-release - # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z - run: | - python3 scripts/version_util.py --check_version - - name: Store the expected version # Find the current cargo version and store it in the output: `expected_version` shell: bash From 52ae3a5ee5a09a2decc213198f426106a883113d Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 20:44:35 +0200 Subject: [PATCH 074/106] Add manual_ prefix to build_wheels_for_PR --- .../{build_wheels_for_pr.yml => manual_build_wheels_for_pr.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{build_wheels_for_pr.yml => manual_build_wheels_for_pr.yml} (100%) diff --git a/.github/workflows/build_wheels_for_pr.yml b/.github/workflows/manual_build_wheels_for_pr.yml similarity index 100% rename from .github/workflows/build_wheels_for_pr.yml rename to .github/workflows/manual_build_wheels_for_pr.yml From 4fe2265c88fd50bca6788889d7d50adcbacb65a5 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Mon, 24 Apr 2023 20:44:57 +0200 Subject: [PATCH 075/106] Send build_summary to index.html --- .github/workflows/reusable_pr_summary.yml | 2 +- scripts/generate_pr_summary.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index e0d4c3cf8dd2..f862e4210951 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -49,5 +49,5 @@ jobs: - name: Upload HTML summary uses: google-github-actions/upload-cloud-storage@v1 with: - path: "build_summary.html" + path: "index.html" destination: "rerun-builds/pull_request/${{inputs.PR_NUMBER}}/" diff --git a/scripts/generate_pr_summary.py b/scripts/generate_pr_summary.py index e5606cb8be1f..ad2e9020e485 100644 --- a/scripts/generate_pr_summary.py +++ b/scripts/generate_pr_summary.py @@ -64,5 +64,5 @@ with open("templates/pr_results_summary.html") as f: template = Template(f.read()) -with open("build_summary.html", "w") as f: +with open("index.html", "w") as f: f.write(template.render(found_builds=found_builds, pr_number=PR_NUMBER)) From 14d5c90e2372b571f4687d0ed8412d0669fada13 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 02:16:41 +0200 Subject: [PATCH 076/106] Standardize on more common - in github action names --- .../workflows/manual_build_wheels_for_pr.yml | 49 +++++++++---------- .github/workflows/manual_dispatch.yml | 40 +++++++-------- .../reusable_build_and_test_wheels.yml | 38 +++++++------- .github/workflows/reusable_deploy_docs.yml | 4 +- .github/workflows/reusable_pr_summary.yml | 2 +- .github/workflows/reusable_upload_web.yml | 2 +- .github/workflows/reusable_upload_wheels.yml | 2 +- 7 files changed, 68 insertions(+), 69 deletions(-) diff --git a/.github/workflows/manual_build_wheels_for_pr.yml b/.github/workflows/manual_build_wheels_for_pr.yml index 533621e29db5..49671babee82 100644 --- a/.github/workflows/manual_build_wheels_for_pr.yml +++ b/.github/workflows/manual_build_wheels_for_pr.yml @@ -5,7 +5,7 @@ on: jobs: - check_for_pr: + check-for-pr: runs-on: ubuntu-latest outputs: PR_NUMBER: ${{ steps.get_pr.outputs.PR_NUMBER }} @@ -27,8 +27,8 @@ jobs: echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT" fi - build_linux: - needs: [check_for_pr] + build-linux: + needs: [check-for-pr] name: 'Linux: Build/Test Wheels' uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: @@ -37,8 +37,8 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - build_windows: - needs: [check_for_pr] + build-windows: + needs: [check-for-pr] name: 'Windows: Build/Test Wheels' uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: @@ -47,8 +47,8 @@ jobs: RRD_ARTIFACT_NAME: '' secrets: inherit - build_macos_arm: - needs: [check_for_pr] + build-macos-arm: + needs: [check-for-pr] name: 'Macos-Arm: Build/Test Wheels' uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: @@ -57,8 +57,8 @@ jobs: RRD_ARTIFACT_NAME: '' secrets: inherit - build_macos_intel: - needs: [check_for_pr] + build-macos-intel: + needs: [check-for-pr] name: 'Macos-Intel: Build/Test Wheels' uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: @@ -67,51 +67,50 @@ jobs: RRD_ARTIFACT_NAME: '' secrets: inherit - upload_wheels_linux: + upload-wheels-linux: name: 'Linux: Upload Wheels' - needs: [build_linux] + needs: [build-linux] uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build_linux.outputs.EXPECTED_VERSION }} + EXPECTED_VERSION: ${{ needs.build-linux.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: linux-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - upload_wheels_windows: + upload-wheels-windows: name: 'Windows: Upload Wheels' - needs: [build_linux, build_windows] + needs: [build-linux, build-windows] uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build_windows.outputs.EXPECTED_VERSION }} + EXPECTED_VERSION: ${{ needs.build-windows.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: windows-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - upload_wheels_macos_arm: + upload-wheels-macos-arm: name: 'Macos-Arm: Upload Wheels' - needs: [build_linux, build_macos_arm] + needs: [build-linux, build-macos-arm] uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build_macos_arm.outputs.EXPECTED_VERSION }} + EXPECTED_VERSION: ${{ needs.build-macos-arm.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: macos-arm-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - upload_wheels_macos_intel: + upload-wheels-macos-intel: name: 'Macos-Intel: Upload Wheels' - needs: [build_linux, build_macos_intel] + needs: [build-linux, build-macos-intel] uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build_macos_intel.outputs.EXPECTED_VERSION }} + EXPECTED_VERSION: ${{ needs.build-macos-intel.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: macos-intel-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - - save_pr_summary: + save-pr-summary: name: 'Save PR Summary' - needs: [check_for_pr, upload_wheels_linux, upload_wheels_windows, upload_wheels_macos_arm, upload_wheels_macos_intel] + needs: [check-for-pr, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] uses: ./.github/workflows/reusable_pr_summary.yml with: - PR_NUMBER: ${{ needs.check_for_pr.outputs.PR_NUMBER}} + PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER}} secrets: inherit diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 00591cec3b14..8c57bd82b3ec 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -75,7 +75,7 @@ jobs: UPDATE_LATEST: false secrets: inherit - min_test_wheel: + min-test-wheel: name: 'Minimum Test Wheel' # The upload-web job uses the min-test-wheel to get the RRD if: ${{ (github.event.inputs.MIN_TEST_WHEEL == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.BUILD_WEB == 'true') ) }} @@ -94,7 +94,7 @@ jobs: uses: ./.github/workflows/reusable_bench.yml secrets: inherit - build_linux: + build-linux: name: 'Linux: Build/Test Wheels' # If we are uploading any build target, we also need the linux job to run since it produces the RRD @@ -110,7 +110,7 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} secrets: inherit - build_windows: + build-windows: name: 'Windows: Build/Test Wheels' if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml @@ -122,7 +122,7 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} secrets: inherit - build_macos_arm: + build-macos-arm: name: 'Macos-Arm: Build/Test Wheels' if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml @@ -134,7 +134,7 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} secrets: inherit - build_macos_intel: + build-macos-intel: name: 'Macos-Intel: Build/Test Wheels' if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') }} uses: ./.github/workflows/reusable_build_and_test_wheels.yml @@ -147,9 +147,9 @@ jobs: secrets: inherit - upload_wheels_linux: + upload-wheels-linux: name: 'Linux: Upload Wheels' - needs: [build_linux] + needs: [build-linux] if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: @@ -157,9 +157,9 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - upload_wheels_windows: + upload-wheels-windows: name: 'Windows: Upload Wheels' - needs: [build_linux, build_windows] + needs: [build-linux, build-windows] if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: @@ -167,9 +167,9 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - upload_wheels_macos_arm: + upload-wheels-macos-arm: name: 'Macos-Arm: Upload Wheels' - needs: [build_linux, build_macos_arm] + needs: [build-linux, build-macos-arm] if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: @@ -177,9 +177,9 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - upload_wheels_macos_intel: + upload-wheels-macos-intel: name: 'Macos-Intel: Upload Wheels' - needs: [build_linux, build_macos_intel] + needs: [build-linux, build-macos-intel] if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_wheels.yml with: @@ -187,7 +187,7 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - build_web: + build-web: name: 'Build Web' if: ${{ github.event.inputs.BUILD_WEB == 'true'}} uses: ./.github/workflows/reusable_build_web.yml @@ -195,12 +195,12 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} secrets: inherit - upload_web: + upload-web: name: 'Upload Web' # Uses Assets: - # build_linux: rrd - # build_web: web_viewer - needs: [min_test_wheel, build_web] + # build-linux: rrd + # build-web: web_viewer + needs: [min-test-wheel, build-web] if: ${{ (github.event.inputs.BUILD_WEB == 'true') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_web.yml with: @@ -209,9 +209,9 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit - save_pr_summary: + save-pr-summary: name: 'Save PR Summary' - needs: [upload_web, upload_wheels_linux] + needs: [upload-web, upload-wheels-linux] if: ${{ github.event.inputs.SAVE_PR != '' }} uses: ./.github/workflows/reusable_pr_summary.yml with: diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index 28eb483985fb..b2876c738eb0 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -48,17 +48,17 @@ jobs: # --------------------------------------------------------------------------- - set_config: + set-config: name: Set Config runs-on: ubuntu-latest outputs: - RUNNER: ${{ steps.set_config.outputs.runner }} - TARGET: ${{ steps.set_config.outputs.target }} - RUN_TESTS: ${{ steps.set_config.outputs.run_tests }} - CONTAINER: ${{ steps.set_config.outputs.container }} + RUNNER: ${{ steps.set-config.outputs.runner }} + TARGET: ${{ steps.set-config.outputs.target }} + RUN_TESTS: ${{ steps.set-config.outputs.run_tests }} + CONTAINER: ${{ steps.set-config.outputs.container }} steps: - name: Set runner and target based on platform - id: set_config + id: set-config run: | case "${{ inputs.platform }}" in linux) @@ -94,13 +94,13 @@ jobs: # --------------------------------------------------------------------------- - build_wheels: + build-wheels: name: Build Wheels - needs: [set_config] + needs: [set-config] - runs-on: ${{ needs.set_config.outputs.RUNNER }} - container: ${{ fromJson(needs.set_config.outputs.CONTAINER) }} + runs-on: ${{ needs.set-config.outputs.RUNNER }} + container: ${{ fromJson(needs.set-config.outputs.CONTAINER) }} steps: - name: Show context @@ -215,7 +215,7 @@ jobs: args: | --manifest-path rerun_py/Cargo.toml --release - --target ${{ needs.set_config.outputs.TARGET }} + --target ${{ needs.set-config.outputs.TARGET }} ${{ inputs.MATURIN_FEATURE_FLAGS }} --out dist @@ -230,7 +230,7 @@ jobs: # Test the wheel - name: Install wheel dependencies - if: needs.set_config.outputs.RUN_TESTS == 'true' + if: needs.set-config.outputs.RUN_TESTS == 'true' # First we install the dependencies manually so we can use `--no-index` when installing the wheel. # This needs to be a separate step for some reason or the following step fails # TODO(jleibs): pull these deps from pyproject.toml @@ -240,7 +240,7 @@ jobs: pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 - name: Install built wheel - if: needs.set_config.outputs.RUN_TESTS == 'true' + if: needs.set-config.outputs.RUN_TESTS == 'true' # Now install the wheel using a specific version and --no-index to guarantee we get the version from # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't # find the dependencies to reinstall them. @@ -250,7 +250,7 @@ jobs: pip install rerun-sdk==${{ steps.expected_version.outputs.EXPECTED_VERSION }} --no-index --find-links dist - name: Verify built wheel version - if: needs.set_config.outputs.RUN_TESTS == 'true' + if: needs.set-config.outputs.RUN_TESTS == 'true' shell: bash run: | python3 -m rerun --version @@ -258,17 +258,17 @@ jobs: rerun --version - name: Run unit tests - if: needs.set_config.outputs.RUN_TESTS == 'true' + if: needs.set-config.outputs.RUN_TESTS == 'true' shell: bash run: cd rerun_py/tests && pytest - name: Run e2e test - if: needs.set_config.outputs.RUN_TESTS == 'true' + if: needs.set-config.outputs.RUN_TESTS == 'true' shell: bash run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - name: Cache RRD dataset - if: needs.set_config.outputs.RUN_TESTS == 'true' + if: needs.set-config.outputs.RUN_TESTS == 'true' id: dataset uses: actions/cache@v3 with: @@ -277,7 +277,7 @@ jobs: key: colmap-dataset-colmap-fiat-v0 - name: Generate Embedded RRD file - if: needs.set_config.outputs.RUN_TESTS == 'true' + if: needs.set-config.outputs.RUN_TESTS == 'true' shell: bash # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name run: | @@ -287,7 +287,7 @@ jobs: # All platforms are currently creating the same rrd file, upload one of them - name: Save RRD artifact - if: ${{ (needs.set_config.outputs.RUN_TESTS == 'true') && (inputs.RRD_ARTIFACT_NAME != '') }} + if: ${{ (needs.set-config.outputs.RUN_TESTS == 'true') && (inputs.RRD_ARTIFACT_NAME != '') }} uses: actions/upload-artifact@v3 with: name: ${{ inputs.RRD_ARTIFACT_NAME }} diff --git a/.github/workflows/reusable_deploy_docs.yml b/.github/workflows/reusable_deploy_docs.yml index 86e5fb403693..f2bceec77ce3 100644 --- a/.github/workflows/reusable_deploy_docs.yml +++ b/.github/workflows/reusable_deploy_docs.yml @@ -29,7 +29,7 @@ jobs: # --------------------------------------------------------------------------- - deploy-docs-py: + py-deploy-docs: name: Python runs-on: ubuntu-latest steps: @@ -86,7 +86,7 @@ jobs: # --------------------------------------------------------------------------- - deploy-docs-rs: + rs-deploy-docs: name: Rust runs-on: ubuntu-latest-16-cores container: diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index f862e4210951..4e52efe2b80c 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -8,7 +8,7 @@ on: type: string jobs: - create_html_summary: + pr-summary: name: Create HTML summary for PR permissions: diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index f43ca28b4616..16142cfc654b 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -28,7 +28,7 @@ on: jobs: - upload_web: + upload-web: name: Upload web build to google cloud (wasm32 + wasm-bindgen) permissions: contents: "read" diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml index 208f0fad2d5e..575b2a3cffb1 100644 --- a/.github/workflows/reusable_upload_wheels.yml +++ b/.github/workflows/reusable_upload_wheels.yml @@ -14,7 +14,7 @@ on: jobs: - upload_wheel: + upload-wheel: name: Upload Wheel to google cloud runs-on: ubuntu-latest From fc193be4c218521f1e9ece877ef7e27955480afd Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 02:30:01 +0200 Subject: [PATCH 077/106] Clearer descriptions and README --- .github/workflows/README.md | 62 +++++++++++++++++++++++++++ .github/workflows/manual_dispatch.yml | 60 +++++++++++++++----------- 2 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000000..9634f0a610c0 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,62 @@ +# Overview + +Our CI workflows make heavy usage of [Reusable Workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows). These reusable workflows can then be tested manually via the `manual_dispatch.yml` workflow. +Or integrated into CI jobs such has `on_pull_request.yml` or `on_main.yml`. + +By convention: +- All reusable workflows start with the `reusable_` prefix. +- All workflows that are triggered via `workflow_dispatch` start with the `manual_` prefix. +- All workflows that are triggered via an event start with the `on_` prefix. + - `on_pull_request` is triggered on pull requests. + - `on_main` is triggered on pushes to the main branch. + +## Reusable Workflows +- [reusable_checks.yml](reusable_checks.yml) - These are all the checks that run to ensure the code is formatted, + linted, and tested. This job produces no artifacts other than a pass/fail criteria for the build. + - `SAVE_CACHE` - If true, the rust cache will be saved. Generally we only do this for builds on `main` +- [reusable_bench.yml](reusable_bench.yml) - This job runs the benchmarks to check for performance regressions. + - `SAVE_BENCH` - If true, then the benchmark results are saved to update https://ref.rerun.io/dev/bench/ +- [reusable_deploy_docs](reusable_deploy_docs.yml) - This job deploys the python and rust documentation to https://ref.rerun.io + - `PY_DOCS_VERSION_NAME` - The name to use for versioning the python docs. This should generally match the version in + `Cargo.toml`. + - `UPDATE_LATEST` - If true, then the docs will be deployed to `latest/` as well as the versioned directory. +- [reusable_build_and_test_wheels.yml](reusable_build_and_test_wheels.yml) - This job builds the wheels, runs the +end-to-end test, and produces a sample RRD. The artifacts are accessible via GitHub artifacts, but not otherwise +uploaded anywhere. + - `MATURIN_FEATURE_FLAGS` - The feature flags to pass to maturin. + - `PLATFORM` - Which platform to build for: `linux`, `macos-arm`, `macos-intel`, or `windows`. + - `RELEASE_VERSION` - If producing a release, the version number. This must match the version in `Cargo.toml`. + - `RRD_ARTIFACT_NAME` - Intermediate name of the GitHub rrd artifact for passing to `reusable_upload_wheels.yml` + - `SAVE_CACHE` - If true, the rust cache will be saved. Generally we only do this for builds on `main` + - `WHEEL_ARTIFACT_NAME` - Intermediate name of the GitHub wheel artifact for passing to `reusable_upload_wheels.yml` +- [reusable_upload_wheels.yml](reusable_upload_wheels.yml) - This job uploads the wheels to google cloud + - `RRD_ARTIFACT_NAME` - Intermediate name of the GitHub rrd artifact. This should match the name passed to + `reusable_build_and_test_wheels.yml` + - `WHEEL_ARTIFACT_NAME` - Intermediate name of the GitHub wheel artifact. This should match the name passed to + `reusable_build_and_test_wheels.yml` +- [reusable_build_web.yml](reusable_build_web.yml) - This job builds the wasm artifacts for the web. + - `RELEASE_VERSION` - If producing a release, the version number. This must match the version in `Cargo.toml`. +- [reusable_upload_web.yml](reusable_upload_web.yml) - This job uploads the web assets to google cloud. By default this + only uploads to: `app.rerun.io/commit//` + - `MARK_PRERELEASE_FOR_MAINLINE` - If true, then the web assets will go to `app.rerun.io/preleease/ + - `MARK_TAGGED_VERSION` - If true, then the web assets will go to `app.rerun.io/version/` + - `RELEASE_VERSION` - If producing a release, the version number. + - `RRD_ARTIFACT_NAME` - Intermediate name of the GitHub rrd artifact. This should match the name passed to + `reusable_build_and_test_wheels.yml` + - `UPLOAD_COMMIT_OVERRIDE` - If set, will replace the value of ``. This is necessary because we want pull + request builds associated with their originating commit, even if the web-build happens on an ephemeral merge-commit. +- [reusable_pr_summary.yml](reusable_pr_summary.yml) - This job updates the PR summary with the results of the CI run. + - This summary can be found at: + `https://storage.googleapis.com/rerun-builds/pull_request//index.html` + - `PR_NUMBER` - The PR number to update. This will generally be set by the `on_pull_request.yml` workflow using: + `${{github.event.pull_request.number}}` + +## Manual Workflows +- [manual_dispatch](manual_dispatch.yml) - This workflow is used to manually trigger the assorted reusable workflows for + testing. + - See the workflow file for the list of parameters. +- [manual_build_wheels_for_pr.yml](manual_build_wheels_for_pr.yml) - This workflow can be dispatched on a branch and + will build all of the wheels for the associated pull-request. Uses: + - [reusable_build_and_test_wheels.yml](reusable_build_and_test_wheels.yml) + - [reusable_upload_wheels.yml](reusable_upload_wheels.yml) + - [reusable_pr_summary.yml](reusable_pr_summary.yml) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 8c57bd82b3ec..36644b0cfc37 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -5,55 +5,65 @@ on: # NOTE: boolean inputs are still actually strings # See: https://github.com/actions/runner/issues/1483 inputs: + # Sorted based on job workflow logic rather than alphabetical + CHECKS: + description: 'Run reuseable_checks' + type: boolean + required: false + default: true + BENCHES: - description: 'Run Benches' + description: 'Run reusable_bench.yml' type: boolean required: false default: false - BUILD_WEB: - description: 'Build Web' + + MIN_TEST_WHEEL: + description: 'Run reuseable_build_and_test_wheels (Minimal Wheel Linux Only)' type: boolean required: false - default: true - CHECKS: - description: 'Run All Checks' + + WHEEL_PLATFORMS: + description: 'Run reusable_build_and_test_wheels (Full build: linux,windows,macos-arm,macos-intel)' + type: string + required: false + default: 'linux' + + BUILD_WEB: + description: 'Run reusable_build_web' type: boolean required: false default: true + DEPLOY_DOCS: - description: 'Deploy the docs to refs.rerun.io' + description: 'Run reusable_deploy_docs' type: boolean required: false default: false - MIN_TEST_WHEEL: - description: 'Test Minimum Wheel' - type: boolean - required: false + RELEASE_VERSION: - description: 'Release' + description: 'Release Version Number (Must match Cargo.toml)' type: string required: false default: 'prerelease' - SAVE_CACHE: - description: 'Save Cache' + + UPLOAD_GCLOUD: + description: 'Upload wheels and wasm to gcloud' type: boolean required: false default: false - SAVE_PR: - description: 'Save PR Summary' + + UPDATE_PR_SUMMARY: + description: 'Update the PR Summary for PR#' type: string required: false default: '' - UPLOAD_GCLOUD: - description: 'Upload to gcloud' + + SAVE_CACHE: + description: 'Save the rust-cache where relevant' type: boolean required: false default: false - WHEEL_PLATFORMS: - description: 'Platforms to build wheels for' - type: string - required: false - default: 'linux' jobs: checks: @@ -212,8 +222,8 @@ jobs: save-pr-summary: name: 'Save PR Summary' needs: [upload-web, upload-wheels-linux] - if: ${{ github.event.inputs.SAVE_PR != '' }} + if: ${{ github.event.inputs.UPDATE_PR_SUMMARY != '' }} uses: ./.github/workflows/reusable_pr_summary.yml with: - PR_NUMBER: ${{ github.event.inputs.SAVE_PR }} + PR_NUMBER: ${{ github.event.inputs.UPDATE_PR_SUMMARY }} secrets: inherit From 576f359f4f9e11fff7002d0545a7c51d22160505 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 02:37:53 +0200 Subject: [PATCH 078/106] More tweaks --- .github/workflows/manual_dispatch.yml | 32 ++++++++++++--------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 36644b0cfc37..6a681d1f170b 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -12,6 +12,12 @@ on: required: false default: true + DEPLOY_DOCS: + description: 'Run reusable_deploy_docs' + type: boolean + required: false + default: false + BENCHES: description: 'Run reusable_bench.yml' type: boolean @@ -35,12 +41,6 @@ on: required: false default: true - DEPLOY_DOCS: - description: 'Run reusable_deploy_docs' - type: boolean - required: false - default: false - RELEASE_VERSION: description: 'Release Version Number (Must match Cargo.toml)' type: string @@ -74,7 +74,7 @@ jobs: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} secrets: inherit - deploy_docs: + deploy-docs: # Never deploy docs if checks haven't passed needs: [checks] name: Deploy Docs @@ -85,6 +85,12 @@ jobs: UPDATE_LATEST: false secrets: inherit + benches: + name: Benchmarks + if: ${{ github.event.inputs.BENCHES == 'true' }} + uses: ./.github/workflows/reusable_bench.yml + secrets: inherit + min-test-wheel: name: 'Minimum Test Wheel' # The upload-web job uses the min-test-wheel to get the RRD @@ -98,16 +104,9 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit - benches: - name: Benchmarks - if: ${{ github.event.inputs.BENCHES == 'true' }} - uses: ./.github/workflows/reusable_bench.yml - secrets: inherit - build-linux: name: 'Linux: Build/Test Wheels' - - # If we are uploading any build target, we also need the linux job to run since it produces the RRD + # The upload-wheels jobs all use the linux build to get the RRD # TODO(jleibs): Debug why multi-line if statements don't work here if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.WHEEL_PLATFORMS != '') ) }} @@ -207,9 +206,6 @@ jobs: upload-web: name: 'Upload Web' - # Uses Assets: - # build-linux: rrd - # build-web: web_viewer needs: [min-test-wheel, build-web] if: ${{ (github.event.inputs.BUILD_WEB == 'true') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} uses: ./.github/workflows/reusable_upload_web.yml From 15e2da9d363ff84e42dcca2a96469926aa65f25c Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 02:41:31 +0200 Subject: [PATCH 079/106] Remove needs from Save PR --- .github/workflows/manual_dispatch.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 6a681d1f170b..bb63cb1de6cb 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -217,7 +217,11 @@ jobs: save-pr-summary: name: 'Save PR Summary' - needs: [upload-web, upload-wheels-linux] + # TODO(jleibs): There's no good way to have an optional needs. + # If we leave this here the job fails if we don't build all the wheels + web. + # Since this just just for testing, leave this out. We can manually run it to update + # a PR if we want to see changes. + #needs: [upload-web, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] if: ${{ github.event.inputs.UPDATE_PR_SUMMARY != '' }} uses: ./.github/workflows/reusable_pr_summary.yml with: From 140965ff873386cad133a97ab5b2c0d185fa431e Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 04:44:15 +0200 Subject: [PATCH 080/106] Some name cleanup --- .github/workflows/on_pull_request.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index c976d2870588..6b15f8bdaa00 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -1,4 +1,4 @@ -name: Run Checks on Pull-Request +name: Pull-Request on: pull_request: @@ -10,9 +10,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# These jobs use fairly short names as they are a prefix in the display hierarchy jobs: checks: - name: Run All Checks + name: Checks uses: ./.github/workflows/reusable_checks.yml secrets: inherit @@ -21,8 +22,8 @@ jobs: uses: ./.github/workflows/reusable_bench.yml secrets: inherit - build_linux_fast: - name: 'Minimum Wheel Test' + min-test-wheel: + name: 'Minimum Wheel' uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: @@ -33,19 +34,16 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit - build_web: + build-web: name: 'Build Web' uses: ./.github/workflows/reusable_build_web.yml with: RELEASE: '' secrets: inherit - upload_web: + upload-web: name: 'Upload Web' - # Uses Assets: - # build_linux: rrd - # build_web: web_viewer - needs: [build_linux_fast, build_web] + needs: [min-test-wheel, build-web] uses: ./.github/workflows/reusable_upload_web.yml with: RELEASE: '' @@ -53,9 +51,9 @@ jobs: UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }} secrets: inherit - save_pr_summary: + save-pr-summary: name: 'Save PR Summary' - needs: [upload_web] + needs: [upload-web] uses: ./.github/workflows/reusable_pr_summary.yml with: PR_NUMBER: ${{ github.event.pull_request.number }} From 66a59ab5fcc9f964d45601c9c7ecec051e2f492e Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 04:50:23 +0200 Subject: [PATCH 081/106] Clean up copy-pasted comments --- .github/workflows/reusable_build_and_test_wheels.yml | 2 -- .github/workflows/reusable_build_web.yml | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml index b2876c738eb0..61dd0f5d84d4 100644 --- a/.github/workflows/reusable_build_and_test_wheels.yml +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -196,8 +196,6 @@ jobs: - name: Check the expected version if: ${{ inputs.RELEASE_VERSION != 'prerelease' }} - # After patching the pre-release version, run cargo update. - # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing run: | if [ "${{steps.expected_version.outputs.EXPECTED_VERSION}}" != "${{inputs.RELEASE_VERSION}}" ]; then echo "Error: EXPECTED_VERSION (${{steps.expected_version.outputs.EXPECTED_VERSION}}) does not match RELEASE_VERSION (${{inputs.RELEASE_VERSION}})" diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 1bb9f829077b..a3dd17bbaf1c 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -6,7 +6,7 @@ on: RELEASE_VERSION: required: false type: string - default: '' + default: 'prerelease' env: # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses @@ -64,8 +64,6 @@ jobs: - name: Check the expected version if: ${{ inputs.RELEASE_VERSION != 'prerelease' }} - # After patching the pre-release version, run cargo update. - # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing run: | if [ "${{steps.expected_version.outputs.EXPECTED_VERSION}}" != "${{inputs.RELEASE_VERSION}}" ]; then echo "Error: EXPECTED_VERSION (${{steps.expected_version.outputs.EXPECTED_VERSION}}) does not match RELEASE_VERSION (${{inputs.RELEASE_VERSION}})" From bfce6715b3691873fddb270e4aa2918d193723a2 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 04:52:01 +0200 Subject: [PATCH 082/106] Fix some inputs --- .github/workflows/on_pull_request.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index 6b15f8bdaa00..d5c2bda9375d 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -24,21 +24,18 @@ jobs: min-test-wheel: name: 'Minimum Wheel' - uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: false PLATFORM: linux MATURIN_FEATURE_FLAGS: '--no-default-features --features extension-module' - WHEEL_ARTIFACT_NAME: '' + WHEEL_ARTIFACT_NAME: '' # the min-test wheel isn't used for anything RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit build-web: name: 'Build Web' uses: ./.github/workflows/reusable_build_web.yml - with: - RELEASE: '' secrets: inherit upload-web: @@ -46,7 +43,6 @@ jobs: needs: [min-test-wheel, build-web] uses: ./.github/workflows/reusable_upload_web.yml with: - RELEASE: '' RRD_ARTIFACT_NAME: linux-rrd-fast UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }} secrets: inherit From d32e3d3d7b1a0bd20ca91fe0ea09f4be8f2bc456 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 05:02:13 +0200 Subject: [PATCH 083/106] Cleanup manual_build_wheels_for_pr --- .../workflows/manual_build_wheels_for_pr.yml | 8 +--- .github/workflows/manual_dispatch.yml | 39 +++++++++++++++---- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/workflows/manual_build_wheels_for_pr.yml b/.github/workflows/manual_build_wheels_for_pr.yml index 49671babee82..162b41df1022 100644 --- a/.github/workflows/manual_build_wheels_for_pr.yml +++ b/.github/workflows/manual_build_wheels_for_pr.yml @@ -72,7 +72,6 @@ jobs: needs: [build-linux] uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build-linux.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: linux-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit @@ -82,7 +81,6 @@ jobs: needs: [build-linux, build-windows] uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build-windows.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: windows-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit @@ -92,7 +90,6 @@ jobs: needs: [build-linux, build-macos-arm] uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build-macos-arm.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: macos-arm-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit @@ -102,13 +99,12 @@ jobs: needs: [build-linux, build-macos-intel] uses: ./.github/workflows/reusable_upload_wheels.yml with: - EXPECTED_VERSION: ${{ needs.build-macos-intel.outputs.EXPECTED_VERSION }} WHEEL_ARTIFACT_NAME: macos-intel-wheel RRD_ARTIFACT_NAME: linux-rrd secrets: inherit - save-pr-summary: - name: 'Save PR Summary' + update-pr-summary: + name: 'Update PR Summary' needs: [check-for-pr, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] uses: ./.github/workflows/reusable_pr_summary.yml with: diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index bb63cb1de6cb..77c0d3199522 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -54,10 +54,10 @@ on: default: false UPDATE_PR_SUMMARY: - description: 'Update the PR Summary for PR#' - type: string + description: 'Update the PR Summary' + type: boolean required: false - default: '' + default: false SAVE_CACHE: description: 'Save the rust-cache where relevant' @@ -66,6 +66,28 @@ on: default: false jobs: + check-for-pr: + runs-on: ubuntu-latest + outputs: + PR_NUMBER: ${{ steps.get_pr.outputs.PR_NUMBER }} + steps: + - name: Check if commit belongs to a PR + id: get_pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pr_number=$(curl --silent --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --url "https://api.github.com/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls" \ + | jq '.[] | .number') + + if [ -z "$pr_number" ]; then + echo "No PR associated with this commit" + exit 1 + else + echo "Commit is associated with PR: $pr_number" + echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT" + fi + checks: name: Run All Checks if: ${{ github.event.inputs.CHECKS == 'true' }} @@ -215,15 +237,16 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit - save-pr-summary: - name: 'Save PR Summary' + update-pr-summary: + name: 'Update PR Summary' # TODO(jleibs): There's no good way to have an optional needs. # If we leave this here the job fails if we don't build all the wheels + web. # Since this just just for testing, leave this out. We can manually run it to update # a PR if we want to see changes. - #needs: [upload-web, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] - if: ${{ github.event.inputs.UPDATE_PR_SUMMARY != '' }} + #needs: [check-for-pr, upload-web, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] + needs: [check-for-pr] + if: ${{ github.event.inputs.UPDATE_PR_SUMMARY == 'true' }} uses: ./.github/workflows/reusable_pr_summary.yml with: - PR_NUMBER: ${{ github.event.inputs.UPDATE_PR_SUMMARY }} + PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER }} secrets: inherit From 901417ef723b21cf568d6fb0fc338e247889fa38 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 05:07:29 +0200 Subject: [PATCH 084/106] Detect PR rather than needing to enter it via form --- .github/workflows/manual_dispatch.yml | 44 ++++++++++++++------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index 77c0d3199522..6147c4013df1 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -66,27 +66,6 @@ on: default: false jobs: - check-for-pr: - runs-on: ubuntu-latest - outputs: - PR_NUMBER: ${{ steps.get_pr.outputs.PR_NUMBER }} - steps: - - name: Check if commit belongs to a PR - id: get_pr - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pr_number=$(curl --silent --header "Authorization: Bearer ${GITHUB_TOKEN}" \ - --url "https://api.github.com/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls" \ - | jq '.[] | .number') - - if [ -z "$pr_number" ]; then - echo "No PR associated with this commit" - exit 1 - else - echo "Commit is associated with PR: $pr_number" - echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT" - fi checks: name: Run All Checks @@ -237,6 +216,29 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd-fast secrets: inherit + check-for-pr: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.UPDATE_PR_SUMMARY == 'true' }} + outputs: + PR_NUMBER: ${{ steps.get_pr.outputs.PR_NUMBER }} + steps: + - name: Check if commit belongs to a PR + id: get_pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pr_number=$(curl --silent --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --url "https://api.github.com/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls" \ + | jq '.[] | .number') + + if [ -z "$pr_number" ]; then + echo "No PR associated with this commit" + exit 1 + else + echo "Commit is associated with PR: $pr_number" + echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT" + fi + update-pr-summary: name: 'Update PR Summary' # TODO(jleibs): There's no good way to have an optional needs. From 0f4c826d66559ff7bdbd33e1b64bbda57083fb35 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 06:32:23 +0200 Subject: [PATCH 085/106] Index helper for prerelease wheels --- .../workflows/manual_build_wheels_for_pr.yml | 6 ++ .github/workflows/reusable_pip_index.yml | 41 ++++++++++++++ scripts/generate_prerelease_pip_index.py | 55 +++++++++++++++++++ templates/pip_index.html | 42 ++++++++++++++ 4 files changed, 144 insertions(+) create mode 100644 .github/workflows/reusable_pip_index.yml create mode 100644 scripts/generate_prerelease_pip_index.py create mode 100644 templates/pip_index.html diff --git a/.github/workflows/manual_build_wheels_for_pr.yml b/.github/workflows/manual_build_wheels_for_pr.yml index 162b41df1022..6820362bf16f 100644 --- a/.github/workflows/manual_build_wheels_for_pr.yml +++ b/.github/workflows/manual_build_wheels_for_pr.yml @@ -103,6 +103,12 @@ jobs: RRD_ARTIFACT_NAME: linux-rrd secrets: inherit + generate-wheel-index: + name: 'Generate Pip Index' + needs: [check-for-pr, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] + uses: ./.github/workflows/reusable_pip_index.yml + secrets: inherit + update-pr-summary: name: 'Update PR Summary' needs: [check-for-pr, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] diff --git a/.github/workflows/reusable_pip_index.yml b/.github/workflows/reusable_pip_index.yml new file mode 100644 index 000000000000..cbcc0054788c --- /dev/null +++ b/.github/workflows/reusable_pip_index.yml @@ -0,0 +1,41 @@ +name: Reusable Pip Index + +on: + workflow_call: + +jobs: + pr-summary: + name: Create a Pip Index file + + permissions: + contents: "read" + id-token: "write" + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - id: "auth" + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + with: + version: '>= 363.0.0' + + - name: Install deps + run: pip install google-cloud-storage Jinja2 + + - name: Render pip index and upload to gcloud + run: python scripts/generate_preprelease_pip_index.py + diff --git a/scripts/generate_prerelease_pip_index.py b/scripts/generate_prerelease_pip_index.py new file mode 100644 index 000000000000..8655450d4f9b --- /dev/null +++ b/scripts/generate_prerelease_pip_index.py @@ -0,0 +1,55 @@ +""" +Script to generate a minimal pip index. + +This script use the google cloud storage APIs to find and link to the builds +associated with a given commit. + +This is expected to be run by the `reusable_pip_index.yml` GitHub workflow. + +Requires the following packages: + pip install google-cloud-storage Jinja2 PyGithub # NOLINT +""" + +import io +import os +from typing import Any, Dict + +from google.cloud import storage +from jinja2 import Template + +GITHUB_SHA = os.environ["GITHUB_SHA"] + +# Initialize the GCS clients +gcs_client = storage.Client() + +# Prepare the found_builds list +found_builds = [] +wheels_bucket = gcs_client.bucket("rerun-builds") + +commit = GITHUB_SHA[:7] + +commit_short = commit[:7] +print("Checking commit: {}...".format(commit_short)) + +found: Dict[str, Any] = {} + +# Get the wheel files for the commit +wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) +wheels = [blob.name.split("/")[-1] for blob in wheel_blobs if blob.name.endswith(".whl")] +if wheels: + print("Found wheels for commit: {}: {}".format(commit_short, wheels)) + found["wheels"] = wheels + +if found: + found["commit"] = commit_short + found_builds.append(found) + +# Render the Jinja template with the found_builds variable +with open("templates/pypi_index.html") as f: + template = Template(f.read()) + +buffer = io.BytesIO(template.render(found_builds=found_builds).encode("utf-8")) +buffer.seek(0) + +upload_blob = wheels_bucket.blob(f"commit/{commit}/wheels/index.html") +upload_blob.upload_from_file(buffer, content_type="text/html") diff --git a/templates/pip_index.html b/templates/pip_index.html new file mode 100644 index 000000000000..624c924601ae --- /dev/null +++ b/templates/pip_index.html @@ -0,0 +1,42 @@ + + + + + Pip Index + + + + +

Minimum Pip Index

+ + {% for build in found_builds %} +
+

Commit: {{ build.commit }}

+ {% if build.wheels %} +
+

Wheels:

+
    + {% for wheel in build.wheels %} +
  • {{ wheel }}
    • + {% endfor %} +
+
+ {% endif %} +
+ {% endfor %} + + + From 2d5e2409e2246ac7c20dfed8cebf388dc3ab26af Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 06:36:59 +0200 Subject: [PATCH 086/106] Add job for pushing to main --- .github/workflows/README.md | 2 +- .github/workflows/on_push_main.yml | 169 +++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/on_push_main.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 9634f0a610c0..209c8d464cdf 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,7 +8,7 @@ By convention: - All workflows that are triggered via `workflow_dispatch` start with the `manual_` prefix. - All workflows that are triggered via an event start with the `on_` prefix. - `on_pull_request` is triggered on pull requests. - - `on_main` is triggered on pushes to the main branch. + - `on_push_main` is triggered on pushes to the main branch. ## Reusable Workflows - [reusable_checks.yml](reusable_checks.yml) - These are all the checks that run to ensure the code is formatted, diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml new file mode 100644 index 000000000000..1314b1d32b30 --- /dev/null +++ b/.github/workflows/on_push_main.yml @@ -0,0 +1,169 @@ +name: Pull-Request + +on: + push: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + checks: + name: Checks + uses: ./.github/workflows/reusable_checks.yml + secrets: inherit + + benches: + name: Benchmarks + uses: ./.github/workflows/reusable_bench.yml + secrets: inherit + + deploy-docs: + needs: [checks, benches] + name: Deploy Docs + uses: ./.github/workflows/reusable_deploy_docs.yml + with: + PY_DOCS_VERSION_NAME: "HEAD" + UPDATE_LATEST: false + secrets: inherit + + build-web: + name: 'Build Web' + uses: ./.github/workflows/reusable_build_web.yml + secrets: inherit + + upload-web: + name: 'Upload Web' + needs: [build-linux, build-web] + uses: ./.github/workflows/reusable_upload_web.yml + with: + RRD_ARTIFACT_NAME: linux-rrd + MARK_PRERELEASE_FOR_MAINLINE: true + secrets: inherit + + build-linux: + needs: [checks] + name: 'Linux: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: linux + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + build-windows: + needs: [checks] + name: 'Windows: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: windows + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: '' + secrets: inherit + + build-macos-arm: + needs: [checks] + name: 'Macos-Arm: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: macos-arm + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: '' + secrets: inherit + + build-macos-intel: + needs: [checks] + name: 'Macos-Intel: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: macos-intel + WHEEL_ARTIFACT_NAME: 'macos-intel-wheel' + RRD_ARTIFACT_NAME: '' + secrets: inherit + + upload-wheels-linux: + name: 'Linux: Upload Wheels' + needs: [build-linux] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-windows: + name: 'Windows: Upload Wheels' + needs: [build-linux, build-windows] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-macos-arm: + name: 'Macos-Arm: Upload Wheels' + needs: [build-linux, build-macos-arm] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-macos-intel: + name: 'Macos-Intel: Upload Wheels' + needs: [build-linux, build-macos-intel] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: macos-intel-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + generate-pip-index: + name: 'Generate Pip Index' + needs: [upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] + uses: ./.github/workflows/reusable_pip_index.yml + secrets: inherit + + pre-release: + name: Pre Release + needs: [upload-web, generate-pip-index] + runs-on: "ubuntu-latest" + steps: + - name: Add SHORT_SHA env property with commit short sha + run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV + + # First delete the old prerelease. If we don't do this, we don't get things like + # proper source-archives and changelog info. + # https://github.com/dev-drprasad/delete-tag-and-release + - uses: dev-drprasad/delete-tag-and-release@v0.2.1 + with: + tag_name: prerelease + delete_release: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Create the actual prerelease + # https://github.com/ncipollo/release-action + - name: GitHub Release + uses: ncipollo/release-action@v1.12.0 + with: + body: | + This is a prerelease. It is not intended for production use. + Please report any issues you find. + + ## Example Hosted App + https://app.rerun.io/commit/${{ env.SHORT_SHA }} + + ## Wheels can be installed with: + ``` + pip install --pre -f https://build.rerun.io/commit/${{ env.SHORT_SHA}}/wheels --upgrade rerun-sdk + ``` + prerelease: true + name: "Development Build" + tag: "prerelease" + token: ${{ secrets.GITHUB_TOKEN }} + generateReleaseNotes: true + allowUpdates: true + removeArtifacts: true + replacesArtifacts: true From 1b026c5a058860ef477a7a4fb896265ed52d362e Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 14:45:09 +0200 Subject: [PATCH 087/106] Fix typo --- .github/workflows/reusable_pip_index.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_pip_index.yml b/.github/workflows/reusable_pip_index.yml index cbcc0054788c..a5808e648ada 100644 --- a/.github/workflows/reusable_pip_index.yml +++ b/.github/workflows/reusable_pip_index.yml @@ -37,5 +37,5 @@ jobs: run: pip install google-cloud-storage Jinja2 - name: Render pip index and upload to gcloud - run: python scripts/generate_preprelease_pip_index.py + run: python scripts/generate_prerelease_pip_index.py From 71beb3f7339885ee5169e9b312789bca8fbd33d3 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 15:40:29 +0200 Subject: [PATCH 088/106] Add new bench comparison functionality to reusable bench --- .github/workflows/reusable_bench.yml | 67 +++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 7a48dda8c141..9438dd78daa8 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -7,6 +7,14 @@ on: required: false type: boolean default: false + BENCH_NAME: + required: false + type: string + default: '' + COMPARE_TO: + required: false + type: string + default: '' env: PYTHON_VERSION: "3.8" @@ -29,9 +37,15 @@ jobs: rs-benchmarks: name: Rust Criterion benchmarks + permissions: + contents: "read" + id-token: "write" + runs-on: ubuntu-latest-16-cores + container: image: rerunio/ci_docker:0.6 + steps: - uses: actions/checkout@v3 @@ -50,6 +64,16 @@ jobs: - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 + - name: Add BENCH_NAME env property with commit short sha + run: | + if [ -z "${{ inputs.BENCH_NAME }}" ]; then + BENCH_NAME=${{ inputs.BENCH_NAME }} + else + BENCH_NAME=$(echo "${{ github.sha }}"" | cut -c1-7)" + fi + echo "BENCH_NAME=$(echo $BENCH_NAME)" >> $GITHUB_ENV + + - name: Run benchmark # Use bash shell so we get pipefail behavior with tee shell: bash @@ -61,7 +85,7 @@ jobs: -p re_log_encoding \ -p re_query \ -p re_tuid \ - -- --output-format=bencher | tee output.txt + -- --output-format=bencher | tee ${{ runner.temp }}/${{ env.BENCH_NAME }} - name: Store benchmark result # https://github.com/benchmark-action/github-action-benchmark @@ -69,7 +93,7 @@ jobs: with: name: Rust Benchmark tool: "cargo" - output-file-path: output.txt + output-file-path: ${{ runner.temp }}/${{ env.BENCH_NAME }} github-token: ${{ secrets.GITHUB_TOKEN }} # Show alert with commit comment on detecting possible performance regression @@ -84,3 +108,42 @@ jobs: gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench max-items-in-chart: 30 + + - id: "auth" + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + with: + version: '>= 363.0.0' + + - name: "Upload bench to GCS" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: ${{ runner.temp }}/${{ env.BENCH_NAME }} + destination: "rerun-builds/benches/" + + - name: Download file from GCS + if: ${{ inputs.COMPARE_TO != '' }} + run: | + gsutil cp gs://rerun-builds/benches/${{inputs.COMPARE_TO}} ${{ runner.temp }}/${{ inputs.COMPARE_TO }} + + - name: Install benchcmp + if: ${{ inputs.COMPARE_TO != '' }} + run: cargo install cargo-benchcmp + + - name: Install benchcmp + if: ${{ inputs.COMPARE_TO != '' }} + run: cargo run ${{ runner.temp }}/${{ inputs.COMPARE_TO }} ${{ runner.temp }}/${{ env.BENCH_NAME }} > ${{ runner.temp }}/bench_results.txt + + - name: Add SHORT_SHA env property with commit short sha + run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV + + - name: "Upload bench-results to GCS" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: ${{ runner.temp }}/bench_results.txt + destination: "rerun-builds/commit/${{env.SHORT_SHA}}/bench_results.txt" From e3de72790a7b95a3d9b868a1f30a1a3520c4ec41 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 16:00:36 +0200 Subject: [PATCH 089/106] Cleanup generate_pr_summary.py --- .github/workflows/reusable_pr_summary.yml | 13 ++- scripts/generate_pr_summary.py | 97 +++++++++++-------- .../templates}/pr_results_summary.html | 2 +- 3 files changed, 66 insertions(+), 46 deletions(-) rename {templates => scripts/templates}/pr_results_summary.html (90%) diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml index 4e52efe2b80c..c148bcf076f9 100644 --- a/.github/workflows/reusable_pr_summary.yml +++ b/.github/workflows/reusable_pr_summary.yml @@ -41,13 +41,12 @@ jobs: run: pip install google-cloud-storage Jinja2 PyGithub # NOLINT - name: Render HTML template - run: python scripts/generate_pr_summary.py + run: | + python scripts/generate_pr_summary.py \ + --github-token ${{secrets.GITHUB_TOKEN}} \ + --github-repository ${GITHUB_REPOSITORY} \ + --pr-number ${{ inputs.PR_NUMBER }} \ + --upload env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ inputs.PR_NUMBER }} - - - name: Upload HTML summary - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: "index.html" - destination: "rerun-builds/pull_request/${{inputs.PR_NUMBER}}/" diff --git a/scripts/generate_pr_summary.py b/scripts/generate_pr_summary.py index ad2e9020e485..a4ff04dabd71 100644 --- a/scripts/generate_pr_summary.py +++ b/scripts/generate_pr_summary.py @@ -10,6 +10,8 @@ pip install google-cloud-storage Jinja2 PyGithub # NOLINT """ +import argparse +import io import os from typing import Any, Dict @@ -17,52 +19,71 @@ from google.cloud import storage from jinja2 import Template -GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] -GITHUB_REPOSITORY = os.environ["GITHUB_REPOSITORY"] -PR_NUMBER = int(os.environ["PR_NUMBER"]) +def generate_pr_summary(github_token: str, github_repository: str, pr_number: int, upload: bool) -> None: + # Initialize the GitHub and GCS clients + gh = Github(github_token) # NOLINT + gcs_client = storage.Client() -# Initialize the GitHub and GCS clients -gh = Github(GITHUB_TOKEN) # NOLINT -gcs_client = storage.Client() + # Get the list of commits associated with the PR + repo = gh.get_repo(github_repository) + pull = repo.get_pull(pr_number) + all_commits = [commit.sha for commit in pull.get_commits()] + all_commits.reverse() -# Get the list of commits associated with the PR -repo = gh.get_repo(GITHUB_REPOSITORY) -pull = repo.get_pull(PR_NUMBER) -all_commits = [commit.sha for commit in pull.get_commits()] -all_commits.reverse() + # Prepare the found_builds list + found_builds = [] + viewer_bucket = gcs_client.bucket("rerun-web-viewer") + wheels_bucket = gcs_client.bucket("rerun-builds") -# Prepare the found_builds list -found_builds = [] -viewer_bucket = gcs_client.bucket("rerun-web-viewer") -wheels_bucket = gcs_client.bucket("rerun-builds") + for commit in all_commits: + commit_short = commit[:7] + print("Checking commit: {}...".format(commit_short)) -for commit in all_commits: - commit_short = commit[:7] - print("Checking commit: {}...".format(commit_short)) + found: Dict[str, Any] = {} - found: Dict[str, Any] = {} + # Check if there is a hosted app for the current commit + commit_blob = viewer_bucket.blob(f"commit/{commit_short}/index.html") + if commit_blob.exists(): + print("Found web assets commit: {}".format(commit_short)) + found["hosted_app"] = f"https://app.rerun.io/commit/{commit_short}" - # Check if there is a hosted app for the current commit - commit_blob = viewer_bucket.blob(f"commit/{commit_short}/index.html") - if commit_blob.exists(): - print("Found web assets commit: {}".format(commit_short)) - found["hosted_app"] = f"https://app.rerun.io/commit/{commit_short}" + # Get the wheel files for the commit + wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) + wheels = [f"https://storage.googleapis.com/{blob.bucket.name}/{blob.name}" for blob in wheel_blobs] + if wheels: + print("Found wheels for commit: {}".format(commit_short)) + found["wheels"] = wheels - # Get the wheel files for the commit - wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) - wheels = [f"https://storage.googleapis.com/{blob.bucket.name}/{blob.name}" for blob in wheel_blobs] - if wheels: - print("Found wheels for commit: {}".format(commit_short)) - found["wheels"] = wheels + if found: + found["commit"] = commit_short + found_builds.append(found) - if found: - found["commit"] = commit_short - found_builds.append(found) + template_path = os.path.join(os.path.dirname(os.path.relpath(__file__)), "templates/pr_results_summary.html") -# Render the Jinja template with the found_builds variable -with open("templates/pr_results_summary.html") as f: - template = Template(f.read()) + # Render the Jinja template with the found_builds variable + with open(template_path) as f: + template = Template(f.read()) -with open("index.html", "w") as f: - f.write(template.render(found_builds=found_builds, pr_number=PR_NUMBER)) + buffer = io.BytesIO(template.render(found_builds=found_builds, pr_number=pr_number).encode("utf-8")) + buffer.seek(0) + + if upload: + upload_blob = wheels_bucket.blob(f"pull_request/{pr_number}/index.html") + print("Uploading results to {}".format(upload_blob)) + upload_blob.upload_from_file(buffer, content_type="text/html") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate a PR summary page") + parser.add_argument("--github-token", required=True, help="GitHub token") + parser.add_argument("--github-repository", required=True, help="GitHub repository") + parser.add_argument("--pr-number", required=True, type=int, help="PR number") + parser.add_argument("--upload", action="store_true", help="Upload the summary page to GCS") + args = parser.parse_args() + + generate_pr_summary(args.github_token, args.github_repository, args.pr_number, args.upload) + + +if __name__ == "__main__": + main() diff --git a/templates/pr_results_summary.html b/scripts/templates/pr_results_summary.html similarity index 90% rename from templates/pr_results_summary.html rename to scripts/templates/pr_results_summary.html index 69b5bcabc51c..08667bd36e79 100644 --- a/templates/pr_results_summary.html +++ b/scripts/templates/pr_results_summary.html @@ -24,7 +24,7 @@

Build Summary for -

Commit: {{ build.commit }}

+

Commit: {{ build.commit }}

{% if build.hosted_app %}

Hosted App:

From b92bb86f52a819c67c3346d9223ecd2ddfc752d6 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 16:11:03 +0200 Subject: [PATCH 090/106] Refactor generate_prerelease_pip_index.py to take args --- .github/workflows/reusable_pip_index.yml | 5 +- scripts/generate_pr_summary.py | 2 +- scripts/generate_prerelease_pip_index.py | 68 +++++++++++++++--------- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/.github/workflows/reusable_pip_index.yml b/.github/workflows/reusable_pip_index.yml index a5808e648ada..e893f7bd0f7a 100644 --- a/.github/workflows/reusable_pip_index.yml +++ b/.github/workflows/reusable_pip_index.yml @@ -37,5 +37,8 @@ jobs: run: pip install google-cloud-storage Jinja2 - name: Render pip index and upload to gcloud - run: python scripts/generate_prerelease_pip_index.py + run: | + python scripts/generate_prerelease_pip_index.py \ + --commit ${{ github.sha }} \ + --upload diff --git a/scripts/generate_pr_summary.py b/scripts/generate_pr_summary.py index a4ff04dabd71..1d267d5fec78 100644 --- a/scripts/generate_pr_summary.py +++ b/scripts/generate_pr_summary.py @@ -70,7 +70,7 @@ def generate_pr_summary(github_token: str, github_repository: str, pr_number: in if upload: upload_blob = wheels_bucket.blob(f"pull_request/{pr_number}/index.html") - print("Uploading results to {}".format(upload_blob)) + print("Uploading results to {}".format(upload_blob.name)) upload_blob.upload_from_file(buffer, content_type="text/html") diff --git a/scripts/generate_prerelease_pip_index.py b/scripts/generate_prerelease_pip_index.py index 8655450d4f9b..0731798fe95a 100644 --- a/scripts/generate_prerelease_pip_index.py +++ b/scripts/generate_prerelease_pip_index.py @@ -10,6 +10,7 @@ pip install google-cloud-storage Jinja2 PyGithub # NOLINT """ +import argparse import io import os from typing import Any, Dict @@ -17,39 +18,54 @@ from google.cloud import storage from jinja2 import Template -GITHUB_SHA = os.environ["GITHUB_SHA"] -# Initialize the GCS clients -gcs_client = storage.Client() +def generate_pip_index(commit: str, upload: bool) -> None: + # Initialize the GCS clients + gcs_client = storage.Client() -# Prepare the found_builds list -found_builds = [] -wheels_bucket = gcs_client.bucket("rerun-builds") + # Prepare the found_builds list + found_builds = [] + wheels_bucket = gcs_client.bucket("rerun-builds") -commit = GITHUB_SHA[:7] + commit_short = commit[:7] + print("Checking commit: {}...".format(commit_short)) -commit_short = commit[:7] -print("Checking commit: {}...".format(commit_short)) + found: Dict[str, Any] = {} -found: Dict[str, Any] = {} + # Get the wheel files for the commit + wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) + wheels = [blob.name.split("/")[-1] for blob in wheel_blobs if blob.name.endswith(".whl")] + if wheels: + print("Found wheels for commit: {}: {}".format(commit_short, wheels)) + found["wheels"] = wheels -# Get the wheel files for the commit -wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) -wheels = [blob.name.split("/")[-1] for blob in wheel_blobs if blob.name.endswith(".whl")] -if wheels: - print("Found wheels for commit: {}: {}".format(commit_short, wheels)) - found["wheels"] = wheels + if found: + found["commit"] = commit_short + found_builds.append(found) -if found: - found["commit"] = commit_short - found_builds.append(found) + template_path = os.path.join(os.path.dirname(os.path.relpath(__file__)), "templates/pip_index.html") -# Render the Jinja template with the found_builds variable -with open("templates/pypi_index.html") as f: - template = Template(f.read()) + # Render the Jinja template with the found_builds variable + with open(template_path) as f: + template = Template(f.read()) -buffer = io.BytesIO(template.render(found_builds=found_builds).encode("utf-8")) -buffer.seek(0) + buffer = io.BytesIO(template.render(found_builds=found_builds).encode("utf-8")) + buffer.seek(0) -upload_blob = wheels_bucket.blob(f"commit/{commit}/wheels/index.html") -upload_blob.upload_from_file(buffer, content_type="text/html") + if upload: + upload_blob = wheels_bucket.blob(f"commit/{commit}/wheels/index.html") + print("Uploading results to {}".format(upload_blob.name)) + upload_blob.upload_from_file(buffer, content_type="text/html") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate a minimal pip index") + parser.add_argument("--commit", required=True, help="Commit SHA") + parser.add_argument("--upload", action="store_true", help="Upload the index to GCS") + args = parser.parse_args() + + generate_pip_index(args.commit, args.upload) + + +if __name__ == "__main__": + main() From a6759e955832cc7a4dff86b0fa02d36c3dd92048 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 16:14:35 +0200 Subject: [PATCH 091/106] Manual job for benchmarks --- .github/workflows/manual_run_bench.yml | 25 ++++++++++++++++++------- .github/workflows/on_pull_request.yml | 4 ---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manual_run_bench.yml b/.github/workflows/manual_run_bench.yml index f490bd34f88b..d32f32a25eaa 100644 --- a/.github/workflows/manual_run_bench.yml +++ b/.github/workflows/manual_run_bench.yml @@ -2,13 +2,24 @@ name: Manually run the benchmarks on: workflow_dispatch: + inputs: + BENCH_NAME: + description: 'Name of the benchmark to save' + type: string + required: false + default: '' + COMPARE_TO: + description: 'Saved Benchmark to compare to' + type: string + required: false + default: 'main' jobs: - run-benchmarks: - runs-on: ubuntu-latest - steps: - - name: Check if commit belongs to a PR - run: | - echo "TODO(jleibs): Run benches" - + run_benches: + name: Benchmarks + uses: ./.github/workflows/reusable_bench.yml + with: + BENCH_NAME: ${{ github.event.inputs.BENCH_NAME }} + COMPARE_TO: ${{ github.event.inputs.COMPARE_TO }} + secrets: inherit diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index d5c2bda9375d..b7951a3a0583 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -17,10 +17,6 @@ jobs: uses: ./.github/workflows/reusable_checks.yml secrets: inherit - benches: - name: Benchmarks - uses: ./.github/workflows/reusable_bench.yml - secrets: inherit min-test-wheel: name: 'Minimum Wheel' From 7aee304aded636c6ec84ec9eed0df15e158087d1 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 16:43:10 +0200 Subject: [PATCH 092/106] Fix typo in resuable bench --- .github/workflows/reusable_bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 9438dd78daa8..40e8b33c2e5b 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -69,7 +69,7 @@ jobs: if [ -z "${{ inputs.BENCH_NAME }}" ]; then BENCH_NAME=${{ inputs.BENCH_NAME }} else - BENCH_NAME=$(echo "${{ github.sha }}"" | cut -c1-7)" + BENCH_NAME=$(echo "${{ github.sha }}" | cut -c1-7)" fi echo "BENCH_NAME=$(echo $BENCH_NAME)" >> $GITHUB_ENV From d42a6e03d3b2363e521c4fc56b5d1bb2a63d13e7 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 16:49:38 +0200 Subject: [PATCH 093/106] Fix bench naming logic --- .github/workflows/reusable_bench.yml | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 40e8b33c2e5b..a014a0f898ae 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -64,15 +64,8 @@ jobs: - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 - - name: Add BENCH_NAME env property with commit short sha - run: | - if [ -z "${{ inputs.BENCH_NAME }}" ]; then - BENCH_NAME=${{ inputs.BENCH_NAME }} - else - BENCH_NAME=$(echo "${{ github.sha }}" | cut -c1-7)" - fi - echo "BENCH_NAME=$(echo $BENCH_NAME)" >> $GITHUB_ENV - + - name: Add SHORT_SHA env property with commit short sha + run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV - name: Run benchmark # Use bash shell so we get pipefail behavior with tee @@ -85,7 +78,7 @@ jobs: -p re_log_encoding \ -p re_query \ -p re_tuid \ - -- --output-format=bencher | tee ${{ runner.temp }}/${{ env.BENCH_NAME }} + -- --output-format=bencher | tee ${{ runner.temp }}/${{ env.SHORT_SHA }}.txt - name: Store benchmark result # https://github.com/benchmark-action/github-action-benchmark @@ -93,7 +86,7 @@ jobs: with: name: Rust Benchmark tool: "cargo" - output-file-path: ${{ runner.temp }}/${{ env.BENCH_NAME }} + output-file-path: ${{ runner.temp }}/${{ env.SHORT_SHA }} github-token: ${{ secrets.GITHUB_TOKEN }} # Show alert with commit comment on detecting possible performance regression @@ -121,6 +114,13 @@ jobs: version: '>= 363.0.0' - name: "Upload bench to GCS" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: ${{ runner.temp }}/${{ env.SHORT_SHA }} + destination: "rerun-builds/benches/${{ env.SHORT_SHA }}" + + - name: "Upload named bench to GCS" + if: ${{ inputs.BENCH_NAME != '' }} uses: google-github-actions/upload-cloud-storage@v1 with: path: ${{ runner.temp }}/${{ env.BENCH_NAME }} @@ -137,10 +137,7 @@ jobs: - name: Install benchcmp if: ${{ inputs.COMPARE_TO != '' }} - run: cargo run ${{ runner.temp }}/${{ inputs.COMPARE_TO }} ${{ runner.temp }}/${{ env.BENCH_NAME }} > ${{ runner.temp }}/bench_results.txt - - - name: Add SHORT_SHA env property with commit short sha - run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV + run: cargo run ${{ runner.temp }}/${{ inputs.COMPARE_TO }} ${{ runner.temp }}/${{ env.SHORT_SHA }} > ${{ runner.temp }}/bench_results.txt - name: "Upload bench-results to GCS" uses: google-github-actions/upload-cloud-storage@v1 From 5fcd31a589562609290a6be19227386db80bf731 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 17:19:48 +0200 Subject: [PATCH 094/106] Runner.tmp isn't available in the docker container --- .github/workflows/reusable_bench.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index a014a0f898ae..ce8c9bd93703 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -78,7 +78,7 @@ jobs: -p re_log_encoding \ -p re_query \ -p re_tuid \ - -- --output-format=bencher | tee ${{ runner.temp }}/${{ env.SHORT_SHA }}.txt + -- --output-format=bencher | tee /tmp/${{ env.SHORT_SHA }} - name: Store benchmark result # https://github.com/benchmark-action/github-action-benchmark @@ -86,7 +86,7 @@ jobs: with: name: Rust Benchmark tool: "cargo" - output-file-path: ${{ runner.temp }}/${{ env.SHORT_SHA }} + output-file-path: /tmp/${{ env.SHORT_SHA }} github-token: ${{ secrets.GITHUB_TOKEN }} # Show alert with commit comment on detecting possible performance regression @@ -116,20 +116,20 @@ jobs: - name: "Upload bench to GCS" uses: google-github-actions/upload-cloud-storage@v1 with: - path: ${{ runner.temp }}/${{ env.SHORT_SHA }} + path: /tmp/${{ env.SHORT_SHA }} destination: "rerun-builds/benches/${{ env.SHORT_SHA }}" - name: "Upload named bench to GCS" if: ${{ inputs.BENCH_NAME != '' }} uses: google-github-actions/upload-cloud-storage@v1 with: - path: ${{ runner.temp }}/${{ env.BENCH_NAME }} + path: /tmp/${{ env.BENCH_NAME }} destination: "rerun-builds/benches/" - name: Download file from GCS if: ${{ inputs.COMPARE_TO != '' }} run: | - gsutil cp gs://rerun-builds/benches/${{inputs.COMPARE_TO}} ${{ runner.temp }}/${{ inputs.COMPARE_TO }} + gsutil cp gs://rerun-builds/benches/${{inputs.COMPARE_TO}} /tmp/${{ inputs.COMPARE_TO }} - name: Install benchcmp if: ${{ inputs.COMPARE_TO != '' }} @@ -137,10 +137,10 @@ jobs: - name: Install benchcmp if: ${{ inputs.COMPARE_TO != '' }} - run: cargo run ${{ runner.temp }}/${{ inputs.COMPARE_TO }} ${{ runner.temp }}/${{ env.SHORT_SHA }} > ${{ runner.temp }}/bench_results.txt + run: cargo run /tmp/${{ inputs.COMPARE_TO }} /tmp/${{ env.SHORT_SHA }} > /tmp/bench_results.txt - name: "Upload bench-results to GCS" uses: google-github-actions/upload-cloud-storage@v1 with: - path: ${{ runner.temp }}/bench_results.txt + path: /tmp/bench_results.txt destination: "rerun-builds/commit/${{env.SHORT_SHA}}/bench_results.txt" From 92cba84880d13db7ba8bd3dff2d31d5cae397935 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 17:30:29 +0200 Subject: [PATCH 095/106] Add extension to recommendations and readme --- .github/workflows/README.md | 4 ++++ .vscode/extensions.json | 17 +++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 9634f0a610c0..2070b933fb2c 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -10,6 +10,10 @@ By convention: - `on_pull_request` is triggered on pull requests. - `on_main` is triggered on pushes to the main branch. +If you are going to be doing any editing of workflows, the +[VS Code extension](https://marketplace.visualstudio.com/items?itemName=cschleiden.vscode-github-actions) +for GitHub Actions is highly recommended. + ## Reusable Workflows - [reusable_checks.yml](reusable_checks.yml) - These are all the checks that run to ensure the code is formatted, linted, and tested. This job produces no artifacts other than a pass/fail criteria for the build. diff --git a/.vscode/extensions.json b/.vscode/extensions.json index aae305c13ea6..32731cdfadeb 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -2,17 +2,18 @@ // See https://go.microsoft.com/fwlink/?LinkId=827846 // for the documentation about the extensions.json format "recommendations": [ - "charliermarsh.ruff", // Ruff for linting + "charliermarsh.ruff", + "github.vscode-github-actions", "ms-python.python", - "ms-vsliveshare.vsliveshare", // Live Share + "ms-vsliveshare.vsliveshare", "polymeilex.wgsl", - "rust-lang.rust-analyzer", // Rust-analyzer + "rust-lang.rust-analyzer", "serayuzgur.crates", "streetsidesoftware.code-spell-checker", - "tamasfe.even-better-toml", // TOML LSP - "vadimcn.vscode-lldb", // CodeLLDB - "wayou.vscode-todo-highlight", // TODO Highlight - "webfreak.debug", // Native Debug - "zxh404.vscode-proto3", // vscode-proto3 + "tamasfe.even-better-toml", + "vadimcn.vscode-lldb", + "wayou.vscode-todo-highlight", + "webfreak.debug", + "zxh404.vscode-proto3", ] } From e2f7925efac79646150c9a86c243e60a40a755ab Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 17:53:07 +0200 Subject: [PATCH 096/106] Upload benches to the right path --- .github/workflows/reusable_bench.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index ce8c9bd93703..1bb94b134edc 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -117,13 +117,19 @@ jobs: uses: google-github-actions/upload-cloud-storage@v1 with: path: /tmp/${{ env.SHORT_SHA }} - destination: "rerun-builds/benches/${{ env.SHORT_SHA }}" + destination: "rerun-builds/benches/" + + # Gcloud upload action depends on local filename for destination filename + # TODO(jleibs) make this whole thing a python script + - name: "Upload named bench to GCS" + if: ${{ inputs.BENCH_NAME != '' }} + run: cp /tmp/${{ env.SHORT_SHA }} /tmp/${{ inputs.BENCH_NAME }} - name: "Upload named bench to GCS" if: ${{ inputs.BENCH_NAME != '' }} uses: google-github-actions/upload-cloud-storage@v1 with: - path: /tmp/${{ env.BENCH_NAME }} + path: /tmp/${{ inputs.BENCH_NAME }} destination: "rerun-builds/benches/" - name: Download file from GCS From 1243cfb7da1fc9fd44f9cd4d84bf675409f42146 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 17:57:31 +0200 Subject: [PATCH 097/106] Clean up some bench step naming --- .github/workflows/reusable_bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 1bb94b134edc..4c3f1aa41cab 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -121,7 +121,7 @@ jobs: # Gcloud upload action depends on local filename for destination filename # TODO(jleibs) make this whole thing a python script - - name: "Upload named bench to GCS" + - name: "Copy bench to named file" if: ${{ inputs.BENCH_NAME != '' }} run: cp /tmp/${{ env.SHORT_SHA }} /tmp/${{ inputs.BENCH_NAME }} @@ -132,7 +132,7 @@ jobs: path: /tmp/${{ inputs.BENCH_NAME }} destination: "rerun-builds/benches/" - - name: Download file from GCS + - name: Download comparison bench from GCS if: ${{ inputs.COMPARE_TO != '' }} run: | gsutil cp gs://rerun-builds/benches/${{inputs.COMPARE_TO}} /tmp/${{ inputs.COMPARE_TO }} From cf6b5cd5963eaeaacfc77a02aaec129517a04941 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 18:22:39 +0200 Subject: [PATCH 098/106] Add permissions and parameterize fail on alert --- .github/workflows/reusable_bench.yml | 53 ++++++++++++++++------------ 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 4c3f1aa41cab..e0d400eda2b1 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -15,6 +15,10 @@ on: required: false type: string default: '' + FAIL_ON_ALERT: + required: false + type: boolean + default: false env: PYTHON_VERSION: "3.8" @@ -38,8 +42,11 @@ jobs: name: Rust Criterion benchmarks permissions: - contents: "read" + # contents permission to update benchmark contents in gh-pages branch + contents: write id-token: "write" + # deployments permission to deploy GitHub pages website + deployments: write runs-on: ubuntu-latest-16-cores @@ -80,28 +87,6 @@ jobs: -p re_tuid \ -- --output-format=bencher | tee /tmp/${{ env.SHORT_SHA }} - - name: Store benchmark result - # https://github.com/benchmark-action/github-action-benchmark - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Rust Benchmark - tool: "cargo" - output-file-path: /tmp/${{ env.SHORT_SHA }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - # Show alert with commit comment on detecting possible performance regression - comment-on-alert: true - alert-threshold: "125%" - fail-on-alert: true - comment-always: false # Generates too much GitHub notification spam - - # Save, results and push to GitHub only on main - save-data-file: ${{ inputs.SAVE_BENCHES }} - auto-push: ${{ inputs.SAVE_BENCHES }} - gh-pages-branch: gh-pages - benchmark-data-dir-path: dev/bench - max-items-in-chart: 30 - - id: "auth" uses: google-github-actions/auth@v1 with: @@ -150,3 +135,25 @@ jobs: with: path: /tmp/bench_results.txt destination: "rerun-builds/commit/${{env.SHORT_SHA}}/bench_results.txt" + + - name: Store benchmark result + # https://github.com/benchmark-action/github-action-benchmark + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Rust Benchmark + tool: "cargo" + output-file-path: /tmp/${{ env.SHORT_SHA }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + # Show alert with commit comment on detecting possible performance regression + comment-on-alert: true + alert-threshold: "125%" + fail-on-alert: ${{ inputs.FAIL_ON_ALERT }} + comment-always: false # Generates too much GitHub notification spam + + # Save, results and push to GitHub only on main + save-data-file: ${{ inputs.SAVE_BENCHES }} + auto-push: ${{ inputs.SAVE_BENCHES }} + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench + max-items-in-chart: 30 From 84b568339e022fd97b4e62a68d94c8196deb0c58 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 18:42:58 +0200 Subject: [PATCH 099/106] Actually include benchcmp command --- .github/workflows/reusable_bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index e0d400eda2b1..1eaac03d3136 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -128,7 +128,7 @@ jobs: - name: Install benchcmp if: ${{ inputs.COMPARE_TO != '' }} - run: cargo run /tmp/${{ inputs.COMPARE_TO }} /tmp/${{ env.SHORT_SHA }} > /tmp/bench_results.txt + run: cargo benchcmp /tmp/${{ inputs.COMPARE_TO }} /tmp/${{ env.SHORT_SHA }} > /tmp/bench_results.txt - name: "Upload bench-results to GCS" uses: google-github-actions/upload-cloud-storage@v1 From bf18276bd9ccd3deb25cfac5c9a6dc929c376b61 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 19:08:50 +0200 Subject: [PATCH 100/106] Move benchmp into the docker image --- .github/workflows/reusable_bench.yml | 6 +----- ci_docker/Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 1eaac03d3136..8a9fd1435751 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -122,11 +122,7 @@ jobs: run: | gsutil cp gs://rerun-builds/benches/${{inputs.COMPARE_TO}} /tmp/${{ inputs.COMPARE_TO }} - - name: Install benchcmp - if: ${{ inputs.COMPARE_TO != '' }} - run: cargo install cargo-benchcmp - - - name: Install benchcmp + - name: Compare results with benchcmp if: ${{ inputs.COMPARE_TO != '' }} run: cargo benchcmp /tmp/${{ inputs.COMPARE_TO }} /tmp/${{ env.SHORT_SHA }} > /tmp/bench_results.txt diff --git a/ci_docker/Dockerfile b/ci_docker/Dockerfile index 80c2f73bf820..f3fa89019ce5 100644 --- a/ci_docker/Dockerfile +++ b/ci_docker/Dockerfile @@ -58,7 +58,8 @@ RUN set -eux; \ # Install some cargo tools we know we'll always need # We can't do this until after we've installed rust / cargo above RUN cargo install cargo-deny && \ - cargo install cargo-cranky + cargo install cargo-cranky && \ + cargo install cargo-benchcmp # Install the python build dependencies ADD rerun_py/requirements-build.txt requirements-build.txt From fce25350dd8502b88fe3ad7b194dce2a4ebbbfd5 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 19:25:34 +0200 Subject: [PATCH 101/106] Fix benchmark upload path --- .github/workflows/reusable_bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 8a9fd1435751..bb71d983210a 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -130,7 +130,7 @@ jobs: uses: google-github-actions/upload-cloud-storage@v1 with: path: /tmp/bench_results.txt - destination: "rerun-builds/commit/${{env.SHORT_SHA}}/bench_results.txt" + destination: "rerun-builds/commit/${{env.SHORT_SHA}}/ - name: Store benchmark result # https://github.com/benchmark-action/github-action-benchmark From 732a781a502fd0177c7dad34855ea3709b10392d Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 19:26:23 +0200 Subject: [PATCH 102/106] Add benchmark results to the PR summary page --- scripts/generate_pr_summary.py | 12 +++++++++--- scripts/templates/pr_results_summary.html | 3 +++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/generate_pr_summary.py b/scripts/generate_pr_summary.py index 1d267d5fec78..f3f5296b3e69 100644 --- a/scripts/generate_pr_summary.py +++ b/scripts/generate_pr_summary.py @@ -43,14 +43,20 @@ def generate_pr_summary(github_token: str, github_repository: str, pr_number: in found: Dict[str, Any] = {} # Check if there is a hosted app for the current commit - commit_blob = viewer_bucket.blob(f"commit/{commit_short}/index.html") - if commit_blob.exists(): + app_blob = viewer_bucket.blob(f"commit/{commit_short}/index.html") + if app_blob.exists(): print("Found web assets commit: {}".format(commit_short)) found["hosted_app"] = f"https://app.rerun.io/commit/{commit_short}" + # Check if there are benchmark results + bench_blob = viewer_bucket.blob(f"commit/{commit_short}/bench_results.txt") + if bench_blob.exists(): + print("Found benchmark results: {}".format(commit_short)) + found["bench_results"] = f"https://build.rerun.io/{bench_blob.name}" + # Get the wheel files for the commit wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) - wheels = [f"https://storage.googleapis.com/{blob.bucket.name}/{blob.name}" for blob in wheel_blobs] + wheels = [f"https://build.rerun.io/{blob.name}" for blob in wheel_blobs if blob.name.endswith(".whl")] if wheels: print("Found wheels for commit: {}".format(commit_short)) found["wheels"] = wheels diff --git a/scripts/templates/pr_results_summary.html b/scripts/templates/pr_results_summary.html index 08667bd36e79..14d0881eb22b 100644 --- a/scripts/templates/pr_results_summary.html +++ b/scripts/templates/pr_results_summary.html @@ -31,6 +31,9 @@

Hosted App:

{{ build.hosted_app }}
{% endif %} + {% if build.bench_results %} +

{{ build.bench_results }}

+ {% endif %} {% if build.wheels %}

Wheels:

From ad94a31bc126f9d6442697189be0cdc6a6924da2 Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 19:33:39 +0200 Subject: [PATCH 103/106] Missing " --- .github/workflows/reusable_bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index bb71d983210a..68a79db627dc 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -130,7 +130,7 @@ jobs: uses: google-github-actions/upload-cloud-storage@v1 with: path: /tmp/bench_results.txt - destination: "rerun-builds/commit/${{env.SHORT_SHA}}/ + destination: "rerun-builds/commit/${{env.SHORT_SHA}}/" - name: Store benchmark result # https://github.com/benchmark-action/github-action-benchmark From 3dae2c1f6e1b858b6308b3039ace02ae3f670a2f Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 19:48:38 +0200 Subject: [PATCH 104/106] Reorder the download/uploads in named bench --- .github/workflows/on_push_main.yml | 4 ++++ .github/workflows/reusable_bench.yml | 34 +++++++++++++++------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml index 1314b1d32b30..8302dc738c2a 100644 --- a/.github/workflows/on_push_main.yml +++ b/.github/workflows/on_push_main.yml @@ -18,6 +18,10 @@ jobs: benches: name: Benchmarks uses: ./.github/workflows/reusable_bench.yml + with: + SAVE_BENCHES: true + BENCH_NAME: main + COMPARE_TO: main secrets: inherit deploy-docs: diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml index 68a79db627dc..948e6b259cff 100644 --- a/.github/workflows/reusable_bench.yml +++ b/.github/workflows/reusable_bench.yml @@ -98,35 +98,25 @@ jobs: with: version: '>= 363.0.0' - - name: "Upload bench to GCS" - uses: google-github-actions/upload-cloud-storage@v1 - with: - path: /tmp/${{ env.SHORT_SHA }} - destination: "rerun-builds/benches/" - - # Gcloud upload action depends on local filename for destination filename # TODO(jleibs) make this whole thing a python script - - name: "Copy bench to named file" - if: ${{ inputs.BENCH_NAME != '' }} - run: cp /tmp/${{ env.SHORT_SHA }} /tmp/${{ inputs.BENCH_NAME }} - - - name: "Upload named bench to GCS" - if: ${{ inputs.BENCH_NAME != '' }} + - name: "Upload bench to GCS based on SHA" uses: google-github-actions/upload-cloud-storage@v1 with: - path: /tmp/${{ inputs.BENCH_NAME }} + path: /tmp/${{ env.SHORT_SHA }} destination: "rerun-builds/benches/" - name: Download comparison bench from GCS if: ${{ inputs.COMPARE_TO != '' }} run: | - gsutil cp gs://rerun-builds/benches/${{inputs.COMPARE_TO}} /tmp/${{ inputs.COMPARE_TO }} + mkdir /tmp/compare/ + gsutil cp gs://rerun-builds/benches/${{inputs.COMPARE_TO}} /tmp/compare/${{ inputs.COMPARE_TO }} - name: Compare results with benchcmp if: ${{ inputs.COMPARE_TO != '' }} - run: cargo benchcmp /tmp/${{ inputs.COMPARE_TO }} /tmp/${{ env.SHORT_SHA }} > /tmp/bench_results.txt + run: cargo benchcmp /tmp/compare/${{ inputs.COMPARE_TO }} /tmp/${{ env.SHORT_SHA }} > /tmp/bench_results.txt - name: "Upload bench-results to GCS" + if: ${{ inputs.COMPARE_TO != '' }} uses: google-github-actions/upload-cloud-storage@v1 with: path: /tmp/bench_results.txt @@ -153,3 +143,15 @@ jobs: gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench max-items-in-chart: 30 + + - name: "Copy bench to named file" + if: ${{ inputs.BENCH_NAME != '' }} + run: cp /tmp/${{ env.SHORT_SHA }} /tmp/${{ inputs.BENCH_NAME }} + + # Don't upload the new named bench until the end in case the names are the same + - name: "Upload named bench to GCS" + if: ${{ inputs.BENCH_NAME != '' }} + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: /tmp/${{ inputs.BENCH_NAME }} + destination: "rerun-builds/benches/" From 07ff836d03d5898a906649c7e9a9b285e93a2eca Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 25 Apr 2023 19:57:29 +0200 Subject: [PATCH 105/106] Manual run bench should update the PR summary --- .github/workflows/manual_run_bench.yml | 32 ++++++++++++++++++++++- scripts/generate_pr_summary.py | 8 +++--- scripts/templates/pr_results_summary.html | 2 +- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manual_run_bench.yml b/.github/workflows/manual_run_bench.yml index d32f32a25eaa..5e6fc1c0139d 100644 --- a/.github/workflows/manual_run_bench.yml +++ b/.github/workflows/manual_run_bench.yml @@ -16,10 +16,40 @@ on: jobs: - run_benches: + run-benches: name: Benchmarks uses: ./.github/workflows/reusable_bench.yml with: BENCH_NAME: ${{ github.event.inputs.BENCH_NAME }} COMPARE_TO: ${{ github.event.inputs.COMPARE_TO }} secrets: inherit + + check-for-pr: + runs-on: ubuntu-latest + outputs: + PR_NUMBER: ${{ steps.get_pr.outputs.PR_NUMBER }} + steps: + - name: Check if commit belongs to a PR + id: get_pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pr_number=$(curl --silent --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --url "https://api.github.com/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls" \ + | jq '.[] | .number') + + if [ -z "$pr_number" ]; then + echo "No PR associated with this commit" + exit 1 + else + echo "Commit is associated with PR: $pr_number" + echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT" + fi + + update-pr-summary: + name: 'Update PR Summary' + needs: [check-for-pr, run-benches] + uses: ./.github/workflows/reusable_pr_summary.yml + with: + PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER}} + secrets: inherit diff --git a/scripts/generate_pr_summary.py b/scripts/generate_pr_summary.py index f3f5296b3e69..4043be33c40f 100644 --- a/scripts/generate_pr_summary.py +++ b/scripts/generate_pr_summary.py @@ -34,7 +34,7 @@ def generate_pr_summary(github_token: str, github_repository: str, pr_number: in # Prepare the found_builds list found_builds = [] viewer_bucket = gcs_client.bucket("rerun-web-viewer") - wheels_bucket = gcs_client.bucket("rerun-builds") + builds_bucket = gcs_client.bucket("rerun-builds") for commit in all_commits: commit_short = commit[:7] @@ -49,13 +49,13 @@ def generate_pr_summary(github_token: str, github_repository: str, pr_number: in found["hosted_app"] = f"https://app.rerun.io/commit/{commit_short}" # Check if there are benchmark results - bench_blob = viewer_bucket.blob(f"commit/{commit_short}/bench_results.txt") + bench_blob = builds_bucket.blob(f"commit/{commit_short}/bench_results.txt") if bench_blob.exists(): print("Found benchmark results: {}".format(commit_short)) found["bench_results"] = f"https://build.rerun.io/{bench_blob.name}" # Get the wheel files for the commit - wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) + wheel_blobs = list(builds_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) wheels = [f"https://build.rerun.io/{blob.name}" for blob in wheel_blobs if blob.name.endswith(".whl")] if wheels: print("Found wheels for commit: {}".format(commit_short)) @@ -75,7 +75,7 @@ def generate_pr_summary(github_token: str, github_repository: str, pr_number: in buffer.seek(0) if upload: - upload_blob = wheels_bucket.blob(f"pull_request/{pr_number}/index.html") + upload_blob = builds_bucket.blob(f"pull_request/{pr_number}/index.html") print("Uploading results to {}".format(upload_blob.name)) upload_blob.upload_from_file(buffer, content_type="text/html") diff --git a/scripts/templates/pr_results_summary.html b/scripts/templates/pr_results_summary.html index 14d0881eb22b..5990b71e5134 100644 --- a/scripts/templates/pr_results_summary.html +++ b/scripts/templates/pr_results_summary.html @@ -32,7 +32,7 @@

Hosted App:

{% endif %} {% if build.bench_results %} -

{{ build.bench_results }}

+

Benchmark Results

{% endif %} {% if build.wheels %}
From 4ec98f098b8ffd46d9e1cf474159f62a8171276a Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Wed, 26 Apr 2023 00:25:49 +0200 Subject: [PATCH 106/106] Use new url for pr number --- .github/workflows/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 421727498c7e..e93f751b7327 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -51,7 +51,7 @@ uploaded anywhere. request builds associated with their originating commit, even if the web-build happens on an ephemeral merge-commit. - [reusable_pr_summary.yml](reusable_pr_summary.yml) - This job updates the PR summary with the results of the CI run. - This summary can be found at: - `https://storage.googleapis.com/rerun-builds/pull_request//index.html` + `https://build.rerun.io/pr//` - `PR_NUMBER` - The PR number to update. This will generally be set by the `on_pull_request.yml` workflow using: `${{github.event.pull_request.number}}`