From d17ae4755c0a6174fb733447c9cefe5b01455339 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Sat, 2 Mar 2024 23:16:32 +0900 Subject: [PATCH] support multi binaries --- README.md | 2 +- main.sh | 157 +++++++++++------- ...gen-test-runner.json => wasm-bindgen.json} | 30 +++- .../base/wasm-bindgen-test-runner.json | 28 ---- tools/codegen/base/wasm-bindgen.json | 19 +++ tools/codegen/src/main.rs | 56 +++++-- 6 files changed, 181 insertions(+), 111 deletions(-) rename manifests/{wasm-bindgen-test-runner.json => wasm-bindgen.json} (73%) delete mode 100644 tools/codegen/base/wasm-bindgen-test-runner.json create mode 100644 tools/codegen/base/wasm-bindgen.json diff --git a/README.md b/README.md index 72c0a71ee..d61980ab3 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ https://spdx.org/licenses | [**shfmt**](https://github.com/mvdan/sh) | `/usr/local/bin` | [GitHub Releases](https://github.com/mvdan/sh/releases) | Linux, macOS, Windows | [BSD-3-Clause](https://github.com/mvdan/sh/blob/HEAD/LICENSE) | | [**syft**](https://github.com/anchore/syft) | `/usr/local/bin` | [Github Releases](https://github.com/anchore/syft/releases) | Linux, macOS, Windows | [Apache-2.0](https://github.com/anchore/syft/blob/HEAD/LICENSE) | | [**valgrind**](https://valgrind.org) | `/snap/bin` | [snap](https://snapcraft.io/install/valgrind/ubuntu) | Linux | [GPL-2.0-or-later](https://valgrind.org/docs/manual/license.gpl.html) | -| [**wasm-bindgen-test-runner**](https://github.com/rustwasm/wasm-bindgen) | `$CARGO_HOME/bin` | [GitHub Releases](https://github.com/rustwasm/wasm-bindgen/releases) | Linux, macOS, Windows | [Apache-2.0](https://github.com/rustwasm/wasm-bindgen/blob/HEAD/LICENSE-APACHE) OR [MIT](https://github.com/rustwasm/wasm-bindgen/blob/HEAD/LICENSE-MIT) | +| [**wasm-bindgen**](https://github.com/rustwasm/wasm-bindgen) | `$CARGO_HOME/bin` | [GitHub Releases](https://github.com/rustwasm/wasm-bindgen/releases) | Linux, macOS, Windows | [Apache-2.0](https://github.com/rustwasm/wasm-bindgen/blob/HEAD/LICENSE-APACHE) OR [MIT](https://github.com/rustwasm/wasm-bindgen/blob/HEAD/LICENSE-MIT) | | [**wasm-pack**](https://github.com/rustwasm/wasm-pack) | `$CARGO_HOME/bin` | [GitHub Releases](https://github.com/rustwasm/wasm-pack/releases) | Linux, macOS, Windows | [Apache-2.0](https://github.com/rustwasm/wasm-pack/blob/HEAD/LICENSE-APACHE) OR [MIT](https://github.com/rustwasm/wasm-pack/blob/HEAD/LICENSE-MIT) | | [**wasmtime**](https://github.com/bytecodealliance/wasmtime) | `$CARGO_HOME/bin` | [GitHub Releases](https://github.com/bytecodealliance/wasmtime/releases) | Linux, macOS, Windows | [Apache-2.0 WITH LLVM-exception](https://github.com/bytecodealliance/wasmtime/blob/HEAD/LICENSE) | | [**xbuild**](https://github.com/rust-mobile/xbuild) | `$CARGO_HOME/bin` | [GitHub Releases](https://github.com/rust-mobile/xbuild/releases) | Linux, macOS, Windows | Apache-2.0 OR MIT | diff --git a/main.sh b/main.sh index ce464101e..8b7404fb6 100755 --- a/main.sh +++ b/main.sh @@ -54,18 +54,26 @@ download_and_checksum() { } download_and_extract() { local url="$1" - local checksum="$2" - local bin_dir="$3" - local bin_in_archive="$4" # path to bin in archive + shift + local checksum="$1" + shift + local bin_dir="$1" + shift + local bin_in_archive=("$@") # path to bin in archive if [[ "${bin_dir}" == "${install_action_dir}/bin" ]]; then init_install_action_bin_dir fi - local installed_bin - # xbuild's binary name is "x", as opposed to the usual crate name + installed_bin=() + local tmp case "${tool}" in - xbuild) installed_bin="${bin_dir}/x" ;; - *) installed_bin="${bin_dir}/$(basename "${bin_in_archive}")" ;; + # xbuild's binary name is "x", as opposed to the usual crate name + xbuild) installed_bin=("${bin_dir}/x") ;; + *) + for tmp in "${bin_in_archive[@]}"; do + installed_bin+=("${bin_dir}/$(basename "${tmp}")") + done + ;; esac local tar_args=() @@ -119,19 +127,23 @@ download_and_extract() { download_and_checksum "${url}" "${checksum}" if [[ ${#tar_args[@]} -gt 0 ]]; then tar_args+=("tmp") - local components - components=$(tr <<<"${bin_in_archive}" -cd '/' | wc -c) - if [[ "${components}" != "0" ]]; then - tar_args+=(--strip-components "${components}") - fi - tar "${tar_args[@]}" -C "${bin_dir}" "${bin_in_archive}" + tar "${tar_args[@]}" + for tmp in "${bin_in_archive[@]}"; do + mv "${tmp}" "${bin_dir}/" + done else case "${url}" in *.zip) unzip -q tmp "${bin_in_archive#\./}" - mv "${bin_in_archive}" "${bin_dir}/" + for tmp in "${bin_in_archive[@]}"; do + mv "${tmp}" "${bin_dir}/" + done + ;; + *) + for tmp in "${installed_bin[@]}"; do + mv tmp "${tmp}" + done ;; - *) mv tmp "${installed_bin}" ;; esac fi ) @@ -139,9 +151,11 @@ download_and_extract() { case "${host_os}" in linux | macos) - if [[ ! -x "${installed_bin}" ]]; then - chmod +x "${installed_bin}" - fi + for tmp in "${installed_bin[@]}"; do + if [[ ! -x "${tmp}" ]]; then + chmod +x "${tmp}" + fi + done ;; esac } @@ -198,15 +212,31 @@ read_download_info() { fi checksum=$(call_jq <<<"${download_info}" -r '.checksum') url=$(call_jq <<<"${download_info}" -r '.url') + local tmp + bin_in_archive=() if [[ "${url}" == "null" ]]; then local template template=$(call_jq -r ".template.${host_platform}" "${manifest_dir}/${tool}.json") url=$(call_jq <<<"${template}" -r '.url') url="${url//\$\{version\}/${exact_version}}" - bin_in_archive=$(call_jq <<<"${template}" -r '.bin') - bin_in_archive="${bin_in_archive//\$\{version\}/${exact_version}}" + tmp=$(call_jq <<<"${template}" -r '.bin' | sed -E "s/\\$\\{version\\}/${exact_version}/g") + if [[ "${tmp}" == *"["* ]]; then + # shellcheck disable=SC2207 + bin_in_archive=($(call_jq <<<"${template}" -r '.bin[]' | sed -E "s/\\$\\{version\\}/${exact_version}/g")) + fi else - bin_in_archive=$(call_jq <<<"${download_info}" -r '.bin') + tmp=$(call_jq <<<"${download_info}" -r '.bin') + if [[ "${tmp}" == *"["* ]]; then + # shellcheck disable=SC2207 + bin_in_archive=($(call_jq <<<"${download_info}" -r '.bin[]')) + fi + fi + if [[ ${#bin_in_archive[@]} -eq 0 ]]; then + if [[ "${tmp}" == "null" ]]; then + bin_in_archive=("${tool}${exe}") + else + bin_in_archive=("${tmp}") + fi fi if [[ "${rust_crate}" == "null" ]]; then if [[ "${host_os}" == "windows" ]] || [[ ! -e /usr/local/bin ]]; then @@ -217,9 +247,6 @@ read_download_info() { else bin_dir="${cargo_bin}" fi - if [[ "${bin_in_archive}" == "null" ]]; then - bin_in_archive="${tool}${exe}" - fi } download_from_manifest() { read_manifest "$@" @@ -227,7 +254,7 @@ download_from_manifest() { } download_from_download_info() { read_download_info "$@" - download_and_extract "${url}" "${checksum}" "${bin_dir}" "${bin_in_archive}" + download_and_extract "${url}" "${checksum}" "${bin_dir}" "${bin_in_archive[@]}" } install_cargo_binstall() { local binstall_version @@ -540,6 +567,7 @@ for tool in "${tools[@]}"; do else version="latest" fi + installed_bin=() case "${tool}" in protoc) info "installing ${tool}@${version}" @@ -576,6 +604,7 @@ for tool in "${tools[@]}"; do fi ) rm -rf "${tmp_dir}" + installed_bin=("${tool}${exe}") ;; valgrind) info "installing ${tool}@${version}" @@ -593,6 +622,7 @@ for tool in "${tools[@]}"; do # Use snap to install the latest Valgrind # https://snapcraft.io/install/valgrind/ubuntu snap_install valgrind --classic + installed_bin=("${tool}${exe}") ;; cargo-binstall) case "${version}" in @@ -651,41 +681,50 @@ for tool in "${tools[@]}"; do ;; esac - case "${tool}" in - xbuild) tool_bin="x" ;; - *) tool_bin="${tool}" ;; - esac - installed_at=$(type -P "${tool_bin}${exe}" || echo "") - if [[ -n "${installed_at}" ]]; then - tool_bin="${tool_bin}${exe}" - else + tool_bin_stems=() + for tool_bin in "${installed_bin[@]}"; do + tool_bin=$(basename "${tool_bin}") + tool_bin_stem="${tool_bin%.exe}" installed_at=$(type -P "${tool_bin}" || echo "") - fi - if [[ -n "${installed_at}" ]]; then - info "${tool} installed at ${installed_at}" - else - warn "${tool} should be installed at ${bin_dir:+"${bin_dir}/"}${tool_bin}${exe}; but ${tool_bin}${exe} not found in path" - fi - # cargo-udeps 0.1.30 and wasm-pack 0.12.0 do not support --version option. - case "${tool}" in - biome) rx "${tool_bin}" --version || true ;; # biome up to 1.2.2 exits with 1 on --version and --help - cargo-careful | cargo-machete | wasm-bindgen-test-runner) ;; # these packages support neither --version nor --help option. - cargo-*) - case "${tool}" in - cargo-valgrind) rx "${tool_bin}" "${tool#cargo-}" --help ;; # cargo-valgrind 2.1.0's --version option just calls cargo's --version option - *) - if ! rx "${tool_bin}" "${tool#cargo-}" --version; then - rx "${tool_bin}" "${tool#cargo-}" --help - fi - ;; - esac - ;; - *) - if ! rx "${tool_bin}" --version; then - rx "${tool_bin}" --help - fi - ;; - esac + if [[ -z "${installed_at}" ]]; then + tool_bin="${tool_bin_stem}" + installed_at=$(type -P "${tool_bin}" || echo "") + fi + if [[ -n "${installed_at}" ]]; then + info "${tool_bin_stem} installed at ${installed_at}" + else + warn "${tool_bin_stem} should be installed at ${bin_dir:+"${bin_dir}/"}${tool_bin}${exe}; but ${tool_bin}${exe} not found in path" + fi + tool_bin_stems+=("${tool_bin_stem}") + done + for tool_bin_stem in "${tool_bin_stems[@]}"; do + # cargo-udeps 0.1.30 and wasm-pack 0.12.0 do not support --version flag. + case "${tool_bin_stem}" in + # biome up to 1.2.2 exits with 1 on both --version and --help flags. + # cargo-machete up to 0.6.0 does not support --version flag. + biome | cargo-machete) rx "${tool_bin_stem}" --version || true ;; + # these packages support neither --version nor --help flag. + cargo-careful | wasm-bindgen-test-runner) ;; + # wasm2es6js does not support --version flag and --help flag doesn't contains version info. + wasm2es6js) ;; + cargo-*) + case "${tool_bin_stem}" in + # cargo-valgrind 2.1.0's --version flag just calls cargo's --version flag + cargo-valgrind) rx "${tool_bin_stem}" "${tool_bin_stem#cargo-}" --help ;; + *) + if ! rx "${tool_bin_stem}" "${tool_bin_stem#cargo-}" --version; then + rx "${tool_bin_stem}" "${tool_bin_stem#cargo-}" --help + fi + ;; + esac + ;; + *) + if ! rx "${tool_bin_stem}" --version; then + rx "${tool_bin_stem}" --help + fi + ;; + esac + done echo done diff --git a/manifests/wasm-bindgen-test-runner.json b/manifests/wasm-bindgen.json similarity index 73% rename from manifests/wasm-bindgen-test-runner.json rename to manifests/wasm-bindgen.json index dc60d5dd1..8cda7bfc5 100644 --- a/manifests/wasm-bindgen-test-runner.json +++ b/manifests/wasm-bindgen.json @@ -3,23 +3,43 @@ "template": { "x86_64_linux_musl": { "url": "https://github.com/rustwasm/wasm-bindgen/releases/download/${version}/wasm-bindgen-${version}-x86_64-unknown-linux-musl.tar.gz", - "bin": "wasm-bindgen-${version}-x86_64-unknown-linux-musl/wasm-bindgen-test-runner" + "bin": [ + "wasm-bindgen-${version}-x86_64-unknown-linux-musl/wasm-bindgen-test-runner", + "wasm-bindgen-${version}-x86_64-unknown-linux-musl/wasm-bindgen", + "wasm-bindgen-${version}-x86_64-unknown-linux-musl/wasm2es6js" + ] }, "x86_64_macos": { "url": "https://github.com/rustwasm/wasm-bindgen/releases/download/${version}/wasm-bindgen-${version}-x86_64-apple-darwin.tar.gz", - "bin": "wasm-bindgen-${version}-x86_64-apple-darwin/wasm-bindgen-test-runner" + "bin": [ + "wasm-bindgen-${version}-x86_64-apple-darwin/wasm-bindgen-test-runner", + "wasm-bindgen-${version}-x86_64-apple-darwin/wasm-bindgen", + "wasm-bindgen-${version}-x86_64-apple-darwin/wasm2es6js" + ] }, "x86_64_windows": { "url": "https://github.com/rustwasm/wasm-bindgen/releases/download/${version}/wasm-bindgen-${version}-x86_64-pc-windows-msvc.tar.gz", - "bin": "wasm-bindgen-${version}-x86_64-pc-windows-msvc/wasm-bindgen-test-runner.exe" + "bin": [ + "wasm-bindgen-${version}-x86_64-pc-windows-msvc/wasm-bindgen-test-runner.exe", + "wasm-bindgen-${version}-x86_64-pc-windows-msvc/wasm-bindgen.exe", + "wasm-bindgen-${version}-x86_64-pc-windows-msvc/wasm2es6js.exe" + ] }, "aarch64_linux_gnu": { "url": "https://github.com/rustwasm/wasm-bindgen/releases/download/${version}/wasm-bindgen-${version}-aarch64-unknown-linux-gnu.tar.gz", - "bin": "wasm-bindgen-${version}-aarch64-unknown-linux-gnu/wasm-bindgen-test-runner" + "bin": [ + "wasm-bindgen-${version}-aarch64-unknown-linux-gnu/wasm-bindgen-test-runner", + "wasm-bindgen-${version}-aarch64-unknown-linux-gnu/wasm-bindgen", + "wasm-bindgen-${version}-aarch64-unknown-linux-gnu/wasm2es6js" + ] }, "aarch64_macos": { "url": "https://github.com/rustwasm/wasm-bindgen/releases/download/${version}/wasm-bindgen-${version}-aarch64-apple-darwin.tar.gz", - "bin": "wasm-bindgen-${version}-aarch64-apple-darwin/wasm-bindgen-test-runner" + "bin": [ + "wasm-bindgen-${version}-aarch64-apple-darwin/wasm-bindgen-test-runner", + "wasm-bindgen-${version}-aarch64-apple-darwin/wasm-bindgen", + "wasm-bindgen-${version}-aarch64-apple-darwin/wasm2es6js" + ] } }, "latest": { diff --git a/tools/codegen/base/wasm-bindgen-test-runner.json b/tools/codegen/base/wasm-bindgen-test-runner.json deleted file mode 100644 index 636fb57be..000000000 --- a/tools/codegen/base/wasm-bindgen-test-runner.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "repository": "https://github.com/rustwasm/wasm-bindgen", - "tag_prefix": "", - "rust_crate": "wasm-bindgen-cli", - "version_range": ">= 0.2.88", - "platform": { - "x86_64_linux_musl": { - "asset_name": "wasm-bindgen-${version}-x86_64-unknown-linux-musl.tar.gz", - "bin": "wasm-bindgen-${version}-x86_64-unknown-linux-musl/wasm-bindgen-test-runner" - }, - "x86_64_macos": { - "asset_name": "wasm-bindgen-${version}-x86_64-apple-darwin.tar.gz", - "bin": "wasm-bindgen-${version}-x86_64-apple-darwin/wasm-bindgen-test-runner" - }, - "x86_64_windows": { - "asset_name": "wasm-bindgen-${version}-x86_64-pc-windows-msvc.tar.gz", - "bin": "wasm-bindgen-${version}-x86_64-pc-windows-msvc/wasm-bindgen-test-runner.exe" - }, - "aarch64_linux_gnu": { - "asset_name": "wasm-bindgen-${version}-aarch64-unknown-linux-gnu.tar.gz", - "bin": "wasm-bindgen-${version}-aarch64-unknown-linux-gnu/wasm-bindgen-test-runner" - }, - "aarch64_macos": { - "asset_name": "wasm-bindgen-${version}-aarch64-apple-darwin.tar.gz", - "bin": "wasm-bindgen-${version}-aarch64-apple-darwin/wasm-bindgen-test-runner" - } - } -} diff --git a/tools/codegen/base/wasm-bindgen.json b/tools/codegen/base/wasm-bindgen.json new file mode 100644 index 000000000..13e2892f3 --- /dev/null +++ b/tools/codegen/base/wasm-bindgen.json @@ -0,0 +1,19 @@ +{ + "repository": "https://github.com/rustwasm/wasm-bindgen", + "tag_prefix": "", + "rust_crate": "wasm-bindgen-cli", + "version_range": ">= 0.2.88", + "asset_name": "wasm-bindgen-${version}-${rust_target}.tar.gz", + "bin": [ + "wasm-bindgen-${version}-${rust_target}/wasm-bindgen-test-runner${exe}", + "wasm-bindgen-${version}-${rust_target}/wasm-bindgen${exe}", + "wasm-bindgen-${version}-${rust_target}/wasm2es6js${exe}" + ], + "platform": { + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": {}, + "aarch64_linux_gnu": {}, + "aarch64_macos": {} + } +} diff --git a/tools/codegen/src/main.rs b/tools/codegen/src/main.rs index 1f7d2f652..04564895f 100644 --- a/tools/codegen/src/main.rs +++ b/tools/codegen/src/main.rs @@ -41,6 +41,7 @@ fn main() -> Result<()> { let mut base_info: BaseManifest = serde_json::from_slice(&fs::read( workspace_root.join("tools/codegen/base").join(format!("{package}.json")), )?)?; + base_info.validate(); let repo = base_info .repository .strip_prefix("https://github.com/") @@ -116,7 +117,10 @@ fn main() -> Result<()> { for (platform, d) in &mut manifest.download_info { let template = &template.download_info[platform]; d.url = Some(template.url.replace("${version}", version)); - d.bin = template.bin.as_ref().map(|s| s.replace("${version}", version)); + d.bin = template + .bin + .as_ref() + .map(|s| s.map(|s| s.replace("${version}", version))); } } } @@ -310,11 +314,8 @@ fn main() -> Result<()> { download_info.insert(platform, ManifestDownloadInfo { url: Some(url), checksum: hash, - bin: base_download_info - .bin - .as_ref() - .or(base_info.bin.as_ref()) - .map(|s| { + bin: base_download_info.bin.as_ref().or(base_info.bin.as_ref()).map(|s| { + s.map(|s| { replace_vars( s, package, @@ -322,8 +323,9 @@ fn main() -> Result<()> { Some(platform), base_info.rust_crate.as_deref(), ) + .unwrap() }) - .transpose()?, + }), }); buf.clear(); } @@ -451,7 +453,7 @@ fn main() -> Result<()> { let t = template.as_mut().unwrap(); for (platform, d) in &mut manifest.download_info { let template_url = d.url.take().unwrap().replace(version, "${version}"); - let template_bin = d.bin.take().map(|s| s.replace(version, "${version}")); + let template_bin = d.bin.take().map(|s| s.map(|s| s.replace(version, "${version}"))); if let Some(d) = t.download_info.get(platform) { if template_url != d.url || template_bin != d.bin { template = None; @@ -711,9 +713,9 @@ struct ManifestDownloadInfo { #[serde(skip_serializing_if = "Option::is_none")] url: Option, checksum: String, - /// Default to ${tool}${exe} + /// Path to binaries in archive. Default to `${tool}${exe}`. #[serde(skip_serializing_if = "Option::is_none")] - bin: Option, + bin: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -725,9 +727,9 @@ struct ManifestTemplate { #[derive(Debug, Clone, Serialize, Deserialize)] struct ManifestTemplateDownloadInfo { url: String, - /// Default to ${tool}${exe} + /// Path to binaries in archive. Default to `${tool}${exe}`. #[serde(skip_serializing_if = "Option::is_none")] - bin: Option, + bin: Option, } #[derive(Debug, Deserialize)] @@ -742,14 +744,26 @@ struct BaseManifest { default_major_version: Option, /// Asset name patterns. asset_name: Option, - /// Path to binary in archive. Default to `${tool}${exe}`. - bin: Option, + /// Path to binaries in archive. Default to `${tool}${exe}`. + bin: Option, signing: Option, #[serde(default)] broken: Vec, platform: BTreeMap, version_range: Option, } +impl BaseManifest { + fn validate(&self) { + if let Some(bin) = &self.bin { + assert!(!bin.as_slice().is_empty()); + } + for m in self.platform.values() { + if let Some(bin) = &m.bin { + assert!(!bin.as_slice().is_empty()); + } + } + } +} #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -772,11 +786,11 @@ enum SigningKind { struct BaseManifestPlatformInfo { /// Asset name patterns. Default to the value at `BaseManifest::asset_name`. asset_name: Option, - /// Path to binary in archive. Default to the value at `BaseManifest::bin`. - bin: Option, + /// Path to binaries in archive. Default to the value at `BaseManifest::bin`. + bin: Option, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(untagged)] enum StringOrArray { String(String), @@ -786,8 +800,14 @@ enum StringOrArray { impl StringOrArray { fn as_slice(&self) -> &[String] { match self { - Self::Array(v) => v, Self::String(s) => slice::from_ref(s), + Self::Array(v) => v, + } + } + fn map(&self, mut f: impl FnMut(&String) -> String) -> Self { + match self { + Self::String(s) => Self::String(f(s)), + Self::Array(v) => Self::Array(v.iter().map(f).collect()), } } }