From 59c39b62e043f79b0b67e1992cd976cacb743f72 Mon Sep 17 00:00:00 2001 From: Alexey Orlenko Date: Thu, 19 Dec 2024 15:12:18 +0100 Subject: [PATCH] chore(ci): migrate engine size dashboard to pure bash and GH Actions (#5095) Due to the lack of Nix buy-in from the team and with most of Nix users leaving, and given the increased overhead of keeping multiple separate build systems and workflows in sync after the introduction of the WebAssembly tooling and difficulties in making changes in Nix code for team members who don't use it, it was previously decided to decrease our reliance on Nix and stop using it on CI, leaving it as optional and only for local development, which mostly happened [in February](https://github.com/prisma/prisma-engines/pull/4713). However, the engines size dashboard was still powered by Nix because we ran out of the allocated time for the tech debt task. After the Nix flake was updated last time, the workflow was broken because `wasm-bindgen-cli` in the flake was at 0.2.95 while we are currently pinned to 0.2.93 and are blocked from upgrading to a newer version at the moment. Rather than pinning `wasm-bindgen-cli` to 0.2.93 in the flake by taking it from a different nixpkgs commit, it's a good opportunity to start using the same infrastructure we use for other GitHub Actions jobs instead. With that, and given the fact that our workflows and build scripts are heavily dependent on rustup and we even used rustup within the dev shell instead of the toolchain from `rust-overlay`, there's not much benefit for the local dev shell for Nix users to be a flake, a classic `shell.nix` is more appropriate: pinning the state of the environment in `flake.lock` is no longer useful and only gets in the way. As an added bonus, classic Nix doesn't require copying the sources to the store, which makes the shell startup a bit faster. Fixes: https://github.com/prisma/team-orm/issues/1444 Closes: https://github.com/prisma/prisma-engines/pull/5072 --- .editorconfig | 3 + .envrc | 7 +- .github/workflows/on-push-to-main.yml | 70 +++++-- flake.lock | 121 ------------ flake.nix | 32 ---- nix/README.md | 5 - nix/args.nix | 15 -- nix/publish-engine-size.nix | 264 -------------------------- nix/shell.nix | 32 ---- shell.nix | 51 +++++ 10 files changed, 115 insertions(+), 485 deletions(-) create mode 100644 .editorconfig delete mode 100644 flake.lock delete mode 100644 flake.nix delete mode 100644 nix/README.md delete mode 100644 nix/args.nix delete mode 100644 nix/publish-engine-size.nix delete mode 100644 nix/shell.nix create mode 100644 shell.nix diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000000..b9d280b84033 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,3 @@ +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.envrc b/.envrc index e66331a4edce..5436426aa834 100644 --- a/.envrc +++ b/.envrc @@ -45,14 +45,11 @@ fi # export TEST_CONNECTOR="postgres" # export TEST_CONNECTOR_VERSION="10" -# Set up env vars and build inputs from flake.nix automatically for nix users. +# Set up env vars and build inputs from shell.nix automatically for nix users. # If you don't use nix, you can safely ignore this. # You can set the DISABLE_NIX environment variable if you're in an environment # where nix is pre-installed (e.g. gitpod) but you don't want to use it. if command -v nix &> /dev/null && [ -z ${DISABLE_NIX+x} ] then - if nix flake metadata > /dev/null; then - watch_file nix/shell.nix nix/all-engines.nix nix/args.nix - use flake - fi + use nix fi diff --git a/.github/workflows/on-push-to-main.yml b/.github/workflows/on-push-to-main.yml index 6e01cd513713..cf146732ae0b 100644 --- a/.github/workflows/on-push-to-main.yml +++ b/.github/workflows/on-push-to-main.yml @@ -12,20 +12,68 @@ concurrency: jobs: publish-to-gh-pages: runs-on: ubuntu-latest - strategy: - fail-fast: true + + env: + CSV_PATH: engines-size/data.csv steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v30 - with: - # we need internet access for the moment - extra_nix_config: | - sandbox = false - - run: | + - uses: ./.github/workflows/include/rust-wasm-setup + + - name: Build native engines + run: | + cargo build --release -p query-engine -p query-engine-node-api + mv target/release/libquery_engine.{so,node} + + - name: Build WASM engines + env: + WASM_BUILD_PROFILE: release + PKG_DIR: query-engine/query-engine-wasm/pkg + TARGET_DIR: target/query-engine-wasm + run: | + make build-qe-wasm-gz + mkdir -p $TARGET_DIR + + for provider in "postgresql" "mysql" "sqlite"; do + cp $PKG_DIR/$provider/query_engine_bg.wasm $TARGET_DIR/query-engine-$provider.wasm + cp $PKG_DIR/$provider.gz $TARGET_DIR/query-engine-$provider.wasm.gz + done + + - name: Check out gh-pages branch + run: | + git fetch --depth=1 origin gh-pages + git checkout origin/gh-pages + + - name: Update engines size + run: | + files=( + target/release/query-engine + target/release/libquery_engine.node + target/query-engine-wasm/query-engine-postgresql.wasm.gz + target/query-engine-wasm/query-engine-postgresql.wasm + target/query-engine-wasm/query-engine-mysql.wasm.gz + target/query-engine-wasm/query-engine-mysql.wasm + target/query-engine-wasm/query-engine-sqlite.wasm.gz + target/query-engine-wasm/query-engine-sqlite.wasm + ) + + DATE_TIME="$(date -u --iso-8601=seconds)" + + if [[ ! -f $CSV_PATH ]]; then + echo "date_time,branch,commit,file,size_bytes" > "$CSV_PATH" + fi + + for file in "${files[@]}"; do + file_name=$(basename "$file") + size=$(stat -c %s "$file") + echo "$DATE_TIME,$GITHUB_REF_NAME,$GITHUB_SHA,$file_name,$size" >> "$CSV_PATH" + done + + - name: Commit the changes + run: | git config user.email "prismabots@gmail.com" git config user.name "prisma-bot" - - - name: Publish engines size to gh-pages branch - run: nix run .#publish-engine-size + git add "$CSV_PATH" + git commit --quiet -m "update engines size for $GITHUB_SHA" + git push origin '+HEAD:gh-pages' diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 28dbb14d0b9a..000000000000 --- a/flake.lock +++ /dev/null @@ -1,121 +0,0 @@ -{ - "nodes": { - "crane": { - "locked": { - "lastModified": 1733418579, - "narHash": "sha256-0fJaoI4B9Nn67E1P44usZhZHkSSyWdAI23HU+X+HJCQ=", - "owner": "ipetkov", - "repo": "crane", - "rev": "62e50137688d953557f156f01e2ad2a25b22d66c", - "type": "github" - }, - "original": { - "owner": "ipetkov", - "repo": "crane", - "type": "github" - } - }, - "flake-parts": { - "inputs": { - "nixpkgs-lib": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1709087332, - "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1733392399, - "narHash": "sha256-kEsTJTUQfQFIJOcLYFt/RvNxIK653ZkTBIs4DG+cBns=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "d0797a04b81caeae77bcff10a9dde78bc17f5661", - "type": "github" - }, - "original": { - "id": "nixpkgs", - "ref": "nixos-unstable", - "type": "indirect" - } - }, - "root": { - "inputs": { - "crane": "crane", - "flake-parts": "flake-parts", - "gitignore": "gitignore", - "nixpkgs": "nixpkgs", - "rust-overlay": "rust-overlay", - "systems": "systems" - } - }, - "rust-overlay": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1733452419, - "narHash": "sha256-eh2i2GtqdWVOP7yjiWtB8FMUWktCZ4vjo81n6g5mSiE=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "020701e6057992329a7cfafc6e3c5d5658bbcf79", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index f55ed31a29d9..000000000000 --- a/flake.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ - inputs = { - crane.url = "github:ipetkov/crane"; - flake-parts = { - url = "github:hercules-ci/flake-parts"; - inputs.nixpkgs-lib.follows = "nixpkgs"; - }; - gitignore = { - url = "github:hercules-ci/gitignore.nix"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - rust-overlay = { - url = "github:oxalica/rust-overlay"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - nixpkgs.url = "nixpkgs/nixos-unstable"; - systems.url = "github:nix-systems/default"; - }; - - outputs = inputs@{ self, nixpkgs, rust-overlay, flake-parts, crane, systems, ... }: - flake-parts.lib.mkFlake { inherit inputs; } { - systems = import systems; - perSystem = { config, system, pkgs, craneLib, ... }: { - config._module.args.flakeInputs = inputs; - imports = [ - ./nix/args.nix - ./nix/shell.nix - ./nix/publish-engine-size.nix - ]; - }; - }; -} diff --git a/nix/README.md b/nix/README.md deleted file mode 100644 index 8f95176511b7..000000000000 --- a/nix/README.md +++ /dev/null @@ -1,5 +0,0 @@ -This directory contains a nix shell that is convenient to streamline developement, however, -contributors must not require to depend on nix for any specific workflow. - -Instead, automation should be provided in a combination of bash scripts and docker, exposed over -tasks in the [root's Makefile](/Makefile) diff --git a/nix/args.nix b/nix/args.nix deleted file mode 100644 index 352d07fd5843..000000000000 --- a/nix/args.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ flakeInputs, system, ... }: -{ - config._module.args = - let - overlays = [ - flakeInputs.rust-overlay.overlays.default - ]; - in rec - { - pkgs = import flakeInputs.nixpkgs { inherit system overlays; }; - rustToolchain = pkgs.rust-bin.stable.latest.default.override { - targets = ["wasm32-unknown-unknown"]; - }; - }; -} diff --git a/nix/publish-engine-size.nix b/nix/publish-engine-size.nix deleted file mode 100644 index 6db06c0e84c1..000000000000 --- a/nix/publish-engine-size.nix +++ /dev/null @@ -1,264 +0,0 @@ -/* -* Deprecated: This file is deprecated and will be removed soon. -* See https://github.com/prisma/team-orm/issues/943 -*/ -{ pkgs, flakeInputs, lib, self', rustToolchain, ... }: - -let - stdenv = pkgs.clangStdenv; - srcPath = ../.; - srcFilter = flakeInputs.gitignore.lib.gitignoreFilterWith { - basePath = srcPath; - extraRules = '' - /nix - /flake.* - ''; - }; - src = lib.cleanSourceWith { - filter = srcFilter; - src = srcPath; - name = "prisma-engines-source"; - }; - craneLib = (flakeInputs.crane.mkLib pkgs).overrideToolchain rustToolchain; - deps = craneLib.vendorCargoDeps { inherit src; }; - libSuffix = stdenv.hostPlatform.extensions.sharedLibrary; - fakeGitHash = "0000000000000000000000000000000000000000"; -in -{ - packages.prisma-engines = stdenv.mkDerivation { - name = "prisma-engines"; - inherit src; - - GIT_HASH = "${fakeGitHash}"; - - buildInputs = [ pkgs.openssl.out ]; - nativeBuildInputs = with pkgs; [ - rustToolchain - git # for our build scripts that bake in the git hash - openssl.dev - pkg-config - ] ++ lib.optionals stdenv.isDarwin [ - perl # required to build openssl - darwin.apple_sdk.frameworks.Security - darwin.apple_sdk.frameworks.SystemConfiguration - iconv - ]; - - configurePhase = '' - mkdir .cargo - ln -s ${deps}/config.toml .cargo/config.toml - ''; - - buildPhase = '' - cargo build --release --bins - cargo build --release -p query-engine-node-api - ''; - - installPhase = '' - mkdir -p $out/bin $out/lib - cp target/release/query-engine $out/bin/ - cp target/release/schema-engine $out/bin/ - cp target/release/prisma-fmt $out/bin/ - cp target/release/libquery_engine${libSuffix} $out/lib/libquery_engine.node - ''; - - dontStrip = true; - }; - - packages.test-cli = lib.makeOverridable - ({ profile }: stdenv.mkDerivation { - name = "test-cli"; - inherit src; - inherit (self'.packages.prisma-engines) buildInputs nativeBuildInputs configurePhase dontStrip; - - GIT_HASH = "${fakeGitHash}"; - - buildPhase = "cargo build --profile=${profile} --bin=test-cli"; - - installPhase = '' - set -eu - mkdir -p $out/bin - QE_PATH=$(find target -name 'test-cli') - cp $QE_PATH $out/bin - ''; - }) - { profile = "release"; }; - - packages.query-engine-bin = lib.makeOverridable - ({ profile }: stdenv.mkDerivation { - name = "query-engine-bin"; - inherit src; - inherit (self'.packages.prisma-engines) buildInputs nativeBuildInputs configurePhase dontStrip; - - GIT_HASH = "${fakeGitHash}"; - - buildPhase = "cargo build --profile=${profile} --bin=query-engine"; - - installPhase = '' - set -eu - mkdir -p $out/bin - QE_PATH=$(find target -name 'query-engine') - cp $QE_PATH $out/bin - ''; - }) - { profile = "release"; }; - - # TODO: try to make caching and sharing the build artifacts work with crane. There should be - # separate `query-engine-lib` and `query-engine-bin` derivations instead, but we use this for now - # to make the CI job that uses it faster. - packages.query-engine-bin-and-lib = lib.makeOverridable - ({ profile }: stdenv.mkDerivation { - name = "query-engine-bin-and-lib"; - inherit src; - inherit (self'.packages.prisma-engines) buildInputs nativeBuildInputs configurePhase dontStrip; - - GIT_HASH = "${fakeGitHash}"; - - buildPhase = '' - cargo build --profile=${profile} --bin=query-engine - cargo build --profile=${profile} -p query-engine-node-api - ''; - - installPhase = '' - set -eu - mkdir -p $out/bin $out/lib - cp target/${profile}/query-engine $out/bin/query-engine - cp target/${profile}/libquery_engine${libSuffix} $out/lib/libquery_engine.node - ''; - }) - { profile = "release"; }; - - packages.build-engine-wasm = pkgs.writeShellApplication { - name = "build-engine-wasm"; - runtimeInputs = with pkgs; [ git rustup wasm-bindgen-cli binaryen jq iconv ]; - text = '' - cd query-engine/query-engine-wasm - WASM_BUILD_PROFILE=release ./build.sh "$1" "$2" - ''; - }; - - packages.query-engine-wasm-gz = lib.makeOverridable - ({ profile }: stdenv.mkDerivation { - name = "query-engine-wasm-gz"; - inherit src; - buildInputs = with pkgs; [ iconv ]; - - GIT_HASH = "${fakeGitHash}"; - - buildPhase = '' - export HOME=$(mktemp -dt wasm-engine-home-XXXX) - - OUT_FOLDER=$(mktemp -dt wasm-engine-out-XXXX) - ${self'.packages.build-engine-wasm}/bin/build-engine-wasm "0.0.0" "$OUT_FOLDER" - - for provider in "postgresql" "mysql" "sqlite"; do - gzip -ckn "$OUT_FOLDER/$provider/query_engine_bg.wasm" > "query-engine-$provider.wasm.gz" - done - ''; - - installPhase = '' - set +x - mkdir -p $out - for provider in "postgresql" "mysql" "sqlite"; do - cp "$OUT_FOLDER/$provider/query_engine_bg.wasm" "$out/query-engine-$provider.wasm" - cp "query-engine-$provider.wasm.gz" "$out/" - done - ''; - }) - { profile = "release"; }; - - packages.export-query-engine-wasm = - pkgs.writeShellApplication { - name = "export-query-engine-wasm"; - runtimeInputs = with pkgs; [ jq ]; - text = '' - OUT_VERSION="$1" - OUT_FOLDER="$2" - - mkdir -p "$OUT_FOLDER" - ${self'.packages.build-engine-wasm}/bin/build-engine-wasm "$OUT_VERSION" "$OUT_FOLDER" - chmod -R +rw "$OUT_FOLDER" - mv "$OUT_FOLDER/package.json" "$OUT_FOLDER/package.json.bak" - jq --arg new_version "$OUT_VERSION" '.version = $new_version' "$OUT_FOLDER/package.json.bak" > "$OUT_FOLDER/package.json" - ''; - }; - - /* Publish the size of the Query Engine binary and library to the CSV file - in the `gh-pages` branch of the repository. - - Data: https://github.com/prisma/prisma-engines/blob/gh-pages/engines-size/data.csv - Dashboard: https://prisma.github.io/prisma-engines/engines-size/ - */ - packages.publish-engine-size = pkgs.writeShellApplication { - name = "publish-engine-size"; - text = '' - set -euxo pipefail - - CURRENT_SYSTEM=$(uname -sm) - - if [[ "$CURRENT_SYSTEM" != "Linux x86_64" ]]; then - : This script publishes the built engine size directly to the gh-pages - : branch of the repository. Refusing to run on "$CURRENT_SYSTEM" to - : avoid inconsistent data being published. Please run the script on - : Linux x86_64 if you want to update the data manually, or use - : "nix run .#update-engine-size" for local testing without modifying - : the data in gh-pages branch. - exit 1 - fi - - if ! git diff --exit-code 1> /dev/null; then - : "The workspace is not clean. Please commit or reset, then try again." - exit 1 - fi - - CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) - CURRENT_COMMIT=$(git rev-parse HEAD) - CURRENT_COMMIT_SHORT=$(git rev-parse --short HEAD) - REPO_ROOT=$(git rev-parse --show-toplevel) - CSV_PATH="engines-size/data.csv" - - pushd "$REPO_ROOT" - git fetch --depth=1 origin gh-pages - git checkout origin/gh-pages - - export CSV_PATH - export CURRENT_BRANCH - export CURRENT_COMMIT - - ${self'.packages.update-engine-size}/bin/update-engine-size \ - ${self'.packages.query-engine-bin-and-lib}/bin/query-engine \ - ${self'.packages.query-engine-bin-and-lib}/lib/libquery_engine.node \ - ${self'.packages.query-engine-wasm-gz}/query-engine-postgresql.wasm.gz \ - ${self'.packages.query-engine-wasm-gz}/query-engine-postgresql.wasm \ - ${self'.packages.query-engine-wasm-gz}/query-engine-mysql.wasm.gz \ - ${self'.packages.query-engine-wasm-gz}/query-engine-mysql.wasm \ - ${self'.packages.query-engine-wasm-gz}/query-engine-sqlite.wasm.gz \ - ${self'.packages.query-engine-wasm-gz}/query-engine-sqlite.wasm - - git add "$CSV_PATH" - git commit --quiet -m "update engines size for $CURRENT_COMMIT_SHORT" - git push origin '+HEAD:gh-pages' - git checkout "$CURRENT_BRANCH" - popd - ''; - }; - - packages.update-engine-size = pkgs.writeShellApplication { - name = "update-engine-size"; - text = '' - set -euxo pipefail - - DATE_TIME="$(date -u --iso-8601=seconds)" - - if [[ ! -f $CSV_PATH ]]; then - echo "date_time,branch,commit,file,size_bytes" > "$CSV_PATH" - fi - - for file in "$@"; do - file_name=$(basename "$file") - size=$(stat -c %s "$file") - echo "$DATE_TIME,$CURRENT_BRANCH,$CURRENT_COMMIT,$file_name,$size" >> "$CSV_PATH" - done - ''; - }; -} diff --git a/nix/shell.nix b/nix/shell.nix deleted file mode 100644 index f23b101de8f6..000000000000 --- a/nix/shell.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ self', pkgs, ... }: - -{ - devShells.default = pkgs.mkShell { - packages = with pkgs; [ - rustup - llvmPackages_latest.bintools - - nodejs_22 - pnpm_9 - - binaryen - cargo-insta - cargo-nextest - jq - graphviz - wabt - wasm-bindgen-cli - wasm-pack - ]; - - inputsFrom = [ self'.packages.prisma-engines ]; - shellHook = - let - useLld = "-C link-arg=-fuse-ld=lld"; - in - pkgs.lib.optionalString pkgs.stdenv.isLinux '' - export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS="${useLld}" - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS="${useLld}" - ''; - }; -} diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000000..309f6275660f --- /dev/null +++ b/shell.nix @@ -0,0 +1,51 @@ +{ + pkgs ? import { }, +}: + +pkgs.mkShell { + packages = with pkgs; [ + binaryen + cargo-insta + cargo-nextest + cargo-watch + git + graphviz + jq + llvmPackages_latest.bintools + nodejs_22 + pnpm_9 + rustup + wabt + wasm-bindgen-cli + wasm-pack + ]; + + nativeBuildInputs = + with pkgs; + [ + pkg-config + ] + ++ lib.optionals stdenv.isDarwin [ + perl + ]; + + buildInputs = + with pkgs; + [ + openssl.dev + ] + ++ lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.Security + darwin.apple_sdk.frameworks.SystemConfiguration + iconv + ]; + + shellHook = + let + useLld = "-C link-arg=-fuse-ld=lld"; + in + pkgs.lib.optionalString pkgs.stdenv.isLinux '' + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS="${useLld}" + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS="${useLld}" + ''; +}