From fecfc97c3ec0be1dcfe43c73a12297f1295b5199 Mon Sep 17 00:00:00 2001 From: Luiz Irber Date: Sun, 5 Mar 2023 16:42:33 -0800 Subject: [PATCH] Build pyodide wheels in CI (#2433) Following https://pyodide.org/en/latest/development/building-and-testing-packages.html#build-github-actions-example --- .github/workflows/build_wheel.yml | 29 ++++++- .github/workflows/dev_envs.yml | 9 ++- Cargo.lock | 130 ++++++++++++++++++++++++++++-- src/core/Cargo.toml | 12 ++- src/core/src/errors.rs | 4 +- src/core/src/lib.rs | 3 +- 6 files changed, 172 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 5b4c221024..57feedd66c 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -55,11 +55,38 @@ jobs: with: path: './wheelhouse/sourmash*.whl' + build_wasm: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.10.2 + - run: | + pip install 'pyodide-build>=0.22.0' + pyodide config get emscripten_version # trigger setup + echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV + - uses: mymindstorm/setup-emsdk@v11 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: wasm32-unknown-emscripten + - run: | + export RUSTC_BOOTSTRAP=1 + pyodide build + + - uses: actions/upload-artifact@v3 + with: + path: './dist/sourmash*.whl' + + release: name: Publish wheels runs-on: ubuntu-20.04 if: startsWith(github.ref, 'refs/tags/v') - needs: build_wheels + needs: [build_wheels, build_wasm] steps: - name: Fetch wheels from artifacts diff --git a/.github/workflows/dev_envs.yml b/.github/workflows/dev_envs.yml index e96b9d1756..09b5ae512c 100644 --- a/.github/workflows/dev_envs.yml +++ b/.github/workflows/dev_envs.yml @@ -11,14 +11,19 @@ jobs: with: fetch-depth: 0 - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - uses: cachix/cachix-action@v12 + with: + name: sourmash-bio + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - run: nix run .# -- --version - - run: nix-shell --command "tox -e py39" + - run: nix-shell --command "tox -e py310" mamba: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 8f7529cabe..0b9361e111 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "assert_matches" version = "1.5.0" @@ -171,14 +180,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", "num-traits", "time", + "wasm-bindgen", "winapi", ] @@ -202,6 +213,16 @@ dependencies = [ "csv", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -212,6 +233,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "counter" version = "0.5.7" @@ -332,6 +359,50 @@ dependencies = [ "memchr", ] +[[package]] +name = "cxx" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.6.1" @@ -423,6 +494,30 @@ dependencies = [ "libc", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "indoc" version = "1.0.7" @@ -476,9 +571,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.124" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libm" @@ -486,6 +581,15 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + [[package]] name = "lock_api" version = "0.4.8" @@ -1086,6 +1190,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" + [[package]] name = "semver" version = "1.0.6" @@ -1154,6 +1264,7 @@ dependencies = [ "bytecount", "byteorder", "cfg-if", + "chrono", "counter", "criterion", "finch", @@ -1224,6 +1335,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + [[package]] name = "textwrap" version = "0.11.0" diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index d732f2de62..f1b83e55de 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -78,19 +78,23 @@ harness = false ## Wasm section. Crates only used for WASM, as well as specific configurations -[target.'cfg(all(target_arch = "wasm32", target_vendor="unknown"))'.dependencies.wasm-bindgen] +[target.'cfg(all(target_arch = "wasm32", target_os="unknown"))'.dependencies.wasm-bindgen] version = "0.2.84" features = ["serde-serialize"] -[target.'cfg(all(target_arch = "wasm32", target_vendor="unknown"))'.dependencies.web-sys] +[target.'cfg(all(target_arch = "wasm32", target_os="unknown"))'.dependencies.web-sys] version = "0.3.61" features = ["console", "File"] -[target.'cfg(all(target_arch = "wasm32", target_vendor="unknown"))'.dev-dependencies] +[target.'cfg(all(target_arch = "wasm32"))'.dependencies.chrono] +version = "0.4.23" +features = ["wasmbind"] + +[target.'cfg(all(target_arch = "wasm32", target_os="unknown"))'.dev-dependencies] wasm-bindgen-test = "0.3.34" ### These crates don't compile on wasm -[target.'cfg(not(all(target_arch = "wasm32", target_vendor="unknown")))'.dependencies] +[target.'cfg(not(all(target_arch = "wasm32", target_os="unknown")))'.dependencies] [package.metadata.maturin] name = "sourmash._lowlevel" diff --git a/src/core/src/errors.rs b/src/core/src/errors.rs index fc1eb285d0..cd4ddcfaf1 100644 --- a/src/core/src/errors.rs +++ b/src/core/src/errors.rs @@ -63,7 +63,7 @@ pub enum SourmashError { #[error(transparent)] IOError(#[from] std::io::Error), - #[cfg(not(all(target_arch = "wasm32", target_vendor = "unknown")))] + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] #[error(transparent)] Panic(#[from] crate::ffi::utils::Panic), } @@ -110,7 +110,7 @@ pub enum SourmashErrorCode { NifflerError = 100_005, } -#[cfg(not(all(target_arch = "wasm32", target_vendor = "unknown")))] +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] impl SourmashErrorCode { pub fn from_error(error: &SourmashError) -> SourmashErrorCode { match error { diff --git a/src/core/src/lib.rs b/src/core/src/lib.rs index 14a25ab632..66de82e6a0 100644 --- a/src/core/src/lib.rs +++ b/src/core/src/lib.rs @@ -39,7 +39,8 @@ use cfg_if::cfg_if; use murmurhash3::murmurhash3_x64_128; cfg_if! { - if #[cfg(all(target_arch = "wasm32", target_vendor = "unknown"))] { + if #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] { + // Explicitly keeping emscripten and wasi out of this pub mod wasm; } else { pub mod ffi;