From 71af1950fbb664dad73cc2ee6df9abd966704bb3 Mon Sep 17 00:00:00 2001 From: Dylan Conway <35280289+dylan-conway@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:21:56 -0800 Subject: [PATCH 1/2] bump webkit (#15328) Co-authored-by: dylan-conway Co-authored-by: Jarred Sumner Co-authored-by: Ben Grant Co-authored-by: Meghan Denny Co-authored-by: Ashcon Partovi --- .buildkite/Dockerfile | 170 ++ .buildkite/Dockerfile-bootstrap.sh | 122 + .buildkite/ci.mjs | 72 +- .dockerignore | 3 + CONTRIBUTING.md | 9 +- cmake/CompilerFlags.cmake | 4 + cmake/Globals.cmake | 2 +- cmake/targets/BuildBun.cmake | 39 +- cmake/targets/BuildLibArchive.cmake | 2 +- cmake/tools/SetupCcache.cmake | 8 +- cmake/tools/SetupLLVM.cmake | 18 +- cmake/tools/SetupWebKit.cmake | 2 +- scripts/agent.mjs | 12 +- scripts/bootstrap.sh | 506 ++-- scripts/docker.mjs | 300 +++ scripts/google.mjs | 510 ++++ scripts/machine.mjs | 2299 ++++------------- scripts/orbstack.mjs | 195 ++ scripts/tart.mjs | 283 ++ scripts/utils.mjs | 144 +- src/bun.js/bindings/Base64Helpers.cpp | 8 +- src/bun.js/bindings/BunString.cpp | 20 +- src/bun.js/bindings/JSBuffer.cpp | 12 +- src/bun.js/bindings/JSBufferEncodingType.cpp | 5 +- src/bun.js/bindings/JSBundlerPlugin.cpp | 4 +- src/bun.js/bindings/JSFFIFunction.cpp | 2 +- src/bun.js/bindings/NodeHTTP.cpp | 4 +- src/bun.js/bindings/ZigGlobalObject.cpp | 10 +- src/bun.js/bindings/bindings.cpp | 8 +- src/bun.js/bindings/c-bindings.cpp | 6 +- src/bun.js/bindings/helpers.h | 8 +- src/bun.js/bindings/webcore/HTTPHeaderMap.cpp | 4 +- .../webcore/PerformanceUserTiming.cpp | 42 +- .../webcore/SerializedScriptValue.cpp | 4 +- .../webcrypto/CryptoAlgorithmHMACOpenSSL.cpp | 1 + src/bun.js/bindings/webcrypto/JSCryptoKey.cpp | 6 +- .../bindings/webcrypto/JSCryptoKeyUsage.cpp | 16 +- .../bindings/webcrypto/JSSubtleCrypto.cpp | 8 +- .../bindings/workaround-missing-symbols.cpp | 192 +- src/bun.js/bindings/wtf-bindings.cpp | 9 +- src/c.zig | 18 +- src/codegen/bindgen.ts | 2 +- src/darwin_c.zig | 14 + src/linux_c.zig | 60 + src/windows_c.zig | 14 + test/js/bun/symbols.test.ts | 6 +- 46 files changed, 2898 insertions(+), 2285 deletions(-) create mode 100644 .buildkite/Dockerfile create mode 100644 .buildkite/Dockerfile-bootstrap.sh create mode 100644 scripts/docker.mjs create mode 100644 scripts/google.mjs create mode 100644 scripts/orbstack.mjs create mode 100644 scripts/tart.mjs diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile new file mode 100644 index 00000000000000..2ace3c0f5052ee --- /dev/null +++ b/.buildkite/Dockerfile @@ -0,0 +1,170 @@ +ARG LLVM_VERSION="18" +ARG REPORTED_LLVM_VERSION="18.1.8" +ARG OLD_BUN_VERSION="1.1.38" +ARG DEFAULT_CFLAGS="-mno-omit-leaf-frame-pointer -fno-omit-frame-pointer -ffunction-sections -fdata-sections -faddrsig -fno-unwind-tables -fno-asynchronous-unwind-tables" +ARG DEFAULT_CXXFLAGS="-flto=full -fwhole-program-vtables -fforce-emit-vtables" +ARG BUILDKITE_AGENT_TAGS="queue=linux,os=linux,arch=${TARGETARCH}" + +FROM --platform=$BUILDPLATFORM ubuntu:20.04 as base-arm64 +FROM --platform=$BUILDPLATFORM ubuntu:18.04 as base-amd64 +FROM base-$TARGETARCH as base + +ARG LLVM_VERSION +ARG OLD_BUN_VERSION +ARG TARGETARCH +ARG DEFAULT_CXXFLAGS +ARG DEFAULT_CFLAGS +ARG REPORTED_LLVM_VERSION + +ENV DEBIAN_FRONTEND=noninteractive \ + CI=true \ + DOCKER=true + +RUN echo "Acquire::Queue-Mode \"host\";" > /etc/apt/apt.conf.d/99-apt-queue-mode.conf \ + && echo "Acquire::Timeout \"120\";" >> /etc/apt/apt.conf.d/99-apt-timeout.conf \ + && echo "Acquire::Retries \"3\";" >> /etc/apt/apt.conf.d/99-apt-retries.conf \ + && echo "APT::Install-Recommends \"false\";" >> /etc/apt/apt.conf.d/99-apt-install-recommends.conf \ + && echo "APT::Install-Suggests \"false\";" >> /etc/apt/apt.conf.d/99-apt-install-suggests.conf + +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget curl git python3 python3-pip ninja-build \ + software-properties-common apt-transport-https \ + ca-certificates gnupg lsb-release unzip \ + libxml2-dev ruby ruby-dev bison gawk perl make golang \ + && add-apt-repository ppa:ubuntu-toolchain-r/test \ + && apt-get update \ + && apt-get install -y gcc-13 g++-13 libgcc-13-dev libstdc++-13-dev \ + libasan6 libubsan1 libatomic1 libtsan0 liblsan0 \ + libgfortran5 libc6-dev \ + && wget https://apt.llvm.org/llvm.sh \ + && chmod +x llvm.sh \ + && ./llvm.sh ${LLVM_VERSION} all \ + && rm llvm.sh + + +RUN --mount=type=tmpfs,target=/tmp \ + cmake_version="3.30.5" && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + cmake_url="https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-aarch64.sh"; \ + else \ + cmake_url="https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-x86_64.sh"; \ + fi && \ + wget -O /tmp/cmake.sh "$cmake_url" && \ + sh /tmp/cmake.sh --skip-license --prefix=/usr + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-13 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-13 \ + --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-13 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-13 + +RUN echo "ARCH_PATH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64-linux-gnu" || echo "x86_64-linux-gnu")" >> /etc/environment \ + && echo "BUN_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "x64")" >> /etc/environment + +ENV LD_LIBRARY_PATH=/usr/lib/gcc/${ARCH_PATH}/13:/usr/lib/${ARCH_PATH} \ + LIBRARY_PATH=/usr/lib/gcc/${ARCH_PATH}/13:/usr/lib/${ARCH_PATH} \ + CPLUS_INCLUDE_PATH=/usr/include/c++/13:/usr/include/${ARCH_PATH}/c++/13 \ + C_INCLUDE_PATH=/usr/lib/gcc/${ARCH_PATH}/13/include \ + CFLAGS=${DEFAULT_CFLAGS} \ + CXXFLAGS="${DEFAULT_CFLAGS} ${DEFAULT_CXXFLAGS}" + +RUN if [ "$TARGETARCH" = "arm64" ]; then \ + export ARCH_PATH="aarch64-linux-gnu"; \ + else \ + export ARCH_PATH="x86_64-linux-gnu"; \ + fi \ + && mkdir -p /usr/lib/gcc/${ARCH_PATH}/13 \ + && ln -sf /usr/lib/${ARCH_PATH}/libstdc++.so.6 /usr/lib/gcc/${ARCH_PATH}/13/ \ + && echo "/usr/lib/gcc/${ARCH_PATH}/13" > /etc/ld.so.conf.d/gcc-13.conf \ + && echo "/usr/lib/${ARCH_PATH}" >> /etc/ld.so.conf.d/gcc-13.conf \ + && ldconfig + +RUN for f in /usr/lib/llvm-${LLVM_VERSION}/bin/*; do ln -sf "$f" /usr/bin; done \ + && ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang \ + && ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ \ + && ln -sf /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld \ + && ln -sf /usr/bin/lldb-${LLVM_VERSION} /usr/bin/lldb \ + && ln -sf /usr/bin/clangd-${LLVM_VERSION} /usr/bin/clangd \ + && ln -sf /usr/bin/llvm-ar-${LLVM_VERSION} /usr/bin/llvm-ar \ + && ln -sf /usr/bin/ld.lld /usr/bin/ld \ + && ln -sf /usr/bin/clang /usr/bin/cc \ + && ln -sf /usr/bin/clang++ /usr/bin/c++ + +ENV CC="clang" \ + CXX="clang++" \ + AR="llvm-ar-${LLVM_VERSION}" \ + RANLIB="llvm-ranlib-${LLVM_VERSION}" \ + LD="lld-${LLVM_VERSION}" + +RUN --mount=type=tmpfs,target=/tmp \ + bash -c '\ + set -euxo pipefail && \ + source /etc/environment && \ + echo "Downloading bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip from https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip" && \ + curl -fsSL https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip -o /tmp/bun.zip && \ + unzip /tmp/bun.zip -d /tmp/bun && \ + mv /tmp/bun/*/bun /usr/bin/bun && \ + chmod +x /usr/bin/bun' + +ENV LLVM_VERSION=${REPORTED_LLVM_VERSION} + +WORKDIR /workspace + + +FROM --platform=$BUILDPLATFORM base as buildkite +ARG BUILDKITE_AGENT_TAGS + + +# Install Rust nightly +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && export PATH=$HOME/.cargo/bin:$PATH \ + && rustup install nightly \ + && rustup default nightly + + +RUN ARCH=$(if [ "$TARGETARCH" = "arm64" ]; then echo "arm64"; else echo "amd64"; fi) && \ + echo "Downloading buildkite" && \ + curl -fsSL "https://github.com/buildkite/agent/releases/download/v3.87.0/buildkite-agent-linux-${ARCH}-3.87.0.tar.gz" -o /tmp/buildkite-agent.tar.gz && \ + mkdir -p /tmp/buildkite-agent && \ + tar -xzf /tmp/buildkite-agent.tar.gz -C /tmp/buildkite-agent && \ + mv /tmp/buildkite-agent/buildkite-agent /usr/bin/buildkite-agent + +RUN mkdir -p /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /etc/buildkite-agent /var/lib/buildkite-agent/cache/bun + +COPY ../*/agent.mjs /var/bun/scripts/ + +ENV BUN_INSTALL_CACHE=/var/lib/buildkite-agent/cache/bun +ENV BUILDKITE_AGENT_TAGS=${BUILDKITE_AGENT_TAGS} + + +WORKDIR /var/bun/scripts + +ENV PATH=/root/.cargo/bin:$PATH + + +CMD ["bun", "/var/bun/scripts/agent.mjs", "start"] + +FROM --platform=$BUILDPLATFORM base as bun-build-linux-local + +ARG LLVM_VERSION +WORKDIR /workspace/bun + +COPY . /workspace/bun + + +# Install Rust nightly +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && export PATH=$HOME/.cargo/bin:$PATH \ + && rustup install nightly \ + && rustup default nightly + +ENV PATH=/root/.cargo/bin:$PATH + +ENV LLVM_VERSION=${REPORTED_LLVM_VERSION} + + +RUN --mount=type=tmpfs,target=/workspace/bun/build \ + ls -la \ + && bun run build:release \ + && mkdir -p /target \ + && cp -r /workspace/bun/build/release/bun /target/bun \ No newline at end of file diff --git a/.buildkite/Dockerfile-bootstrap.sh b/.buildkite/Dockerfile-bootstrap.sh new file mode 100644 index 00000000000000..b8ccba7d17dded --- /dev/null +++ b/.buildkite/Dockerfile-bootstrap.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + echo "error: must run as root" + exit 1 +fi + +# Check OS compatibility +if ! command -v dnf &> /dev/null; then + echo "error: this script requires dnf (RHEL/Fedora/CentOS)" + exit 1 +fi + +# Ensure /tmp/agent.mjs, /tmp/Dockerfile are present +if [ ! -f /tmp/agent.mjs ] || [ ! -f /tmp/Dockerfile ]; then + # Print each missing file + if [ ! -f /tmp/agent.mjs ]; then + echo "error: /tmp/agent.mjs is missing" + fi + if [ ! -f /tmp/Dockerfile ]; then + echo "error: /tmp/Dockerfile is missing" + fi + exit 1 +fi + +# Install Docker +dnf update -y +dnf install -y docker + +systemctl enable docker +systemctl start docker || { + echo "error: failed to start Docker" + exit 1 +} + +# Create builder +docker buildx create --name builder --driver docker-container --bootstrap --use || { + echo "error: failed to create Docker buildx builder" + exit 1 +} + +# Set up Docker to start on boot +cat << 'EOF' > /etc/systemd/system/buildkite-agent.service +[Unit] +Description=Buildkite Docker Container +After=docker.service network-online.target +Requires=docker.service network-online.target + +[Service] +TimeoutStartSec=0 +Restart=always +RestartSec=5 +ExecStartPre=-/usr/bin/docker stop buildkite +ExecStartPre=-/usr/bin/docker rm buildkite +ExecStart=/usr/bin/docker run \ + --name buildkite \ + --restart=unless-stopped \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + buildkite:latest + +[Install] +WantedBy=multi-user.target +EOF + +echo "Building Buildkite image" + +# Clean up any previous build artifacts +rm -rf /tmp/fakebun +mkdir -p /tmp/fakebun/scripts /tmp/fakebun/.buildkite + +# Copy required files +cp /tmp/agent.mjs /tmp/fakebun/scripts/ || { + echo "error: failed to copy agent.mjs" + exit 1 +} +cp /tmp/Dockerfile /tmp/fakebun/.buildkite/Dockerfile || { + echo "error: failed to copy Dockerfile" + exit 1 +} + +cd /tmp/fakebun || { + echo "error: failed to change directory" + exit 1 +} + +# Build the Buildkite image +docker buildx build \ + --platform $(uname -m | sed 's/aarch64/linux\/arm64/;s/x86_64/linux\/amd64/') \ + --tag buildkite:latest \ + --target buildkite \ + -f .buildkite/Dockerfile \ + --load \ + . || { + echo "error: Docker build failed" + exit 1 +} + +# Create container to ensure image is cached in AMI +docker container create \ + --name buildkite \ + --restart=unless-stopped \ + buildkite:latest || { + echo "error: failed to create buildkite container" + exit 1 +} + +# Reload systemd to pick up new service +systemctl daemon-reload + +# Enable the service, but don't start it yet +systemctl enable buildkite-agent || { + echo "error: failed to enable buildkite-agent service" + exit 1 +} + +echo "Bootstrap complete" +echo "To start the Buildkite agent, run: " +echo " systemctl start buildkite-agent" \ No newline at end of file diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index 240ab9f16e8d7c..9f39a72a3d0948 100755 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -95,6 +95,7 @@ function getTargetLabel(target) { * @property {Distro} [distro] * @property {string} release * @property {Tier} [tier] + * @property {string[]} [features] */ /** @@ -102,10 +103,10 @@ function getTargetLabel(target) { */ const buildPlatforms = [ { os: "darwin", arch: "aarch64", release: "14" }, - // { os: "darwin", arch: "x64", release: "14" }, - { os: "linux", arch: "aarch64", distro: "debian", release: "11" }, - { os: "linux", arch: "x64", distro: "debian", release: "11" }, - { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" }, + { os: "darwin", arch: "x64", release: "14" }, + { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023", features: ["docker"] }, + { os: "linux", arch: "x64", distro: "amazonlinux", release: "2023", features: ["docker"] }, + { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023", features: ["docker"] }, { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" }, { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20" }, { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20" }, @@ -119,8 +120,8 @@ const buildPlatforms = [ const testPlatforms = [ { os: "darwin", arch: "aarch64", release: "14", tier: "latest" }, { os: "darwin", arch: "aarch64", release: "13", tier: "previous" }, - // { os: "darwin", arch: "x64", release: "14", tier: "latest" }, - // { os: "darwin", arch: "x64", release: "13", tier: "previous" }, + { os: "darwin", arch: "x64", release: "14", tier: "latest" }, + { os: "darwin", arch: "x64", release: "13", tier: "previous" }, { os: "linux", arch: "aarch64", distro: "debian", release: "12", tier: "latest" }, { os: "linux", arch: "x64", distro: "debian", release: "12", tier: "latest" }, { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "12", tier: "latest" }, @@ -175,12 +176,21 @@ function getPlatformLabel(platform) { * @returns {string} */ function getImageKey(platform) { - const { os, arch, distro, release } = platform; + const { os, arch, distro, release, features, abi } = platform; const version = release.replace(/\./g, ""); + let key = `${os}-${arch}-${version}`; if (distro) { - return `${os}-${arch}-${distro}-${version}`; + key += `-${distro}`; + } + if (features?.length) { + key += `-with-${features.join("-")}`; + } + + if (abi) { + key += `-${abi}`; } - return `${os}-${arch}-${version}`; + + return key; } /** @@ -198,13 +208,15 @@ function getImageLabel(platform) { * @returns {string} */ function getImageName(platform, options) { - const { os, arch, distro, release } = platform; + const { os } = platform; const { buildImages, publishImages } = options; - const name = distro ? `${os}-${arch}-${distro}-${release}` : `${os}-${arch}-${release}`; + + const name = getImageKey(platform); if (buildImages && !publishImages) { return `${name}-build-${getBuildNumber()}`; } + return `${name}-v${getBootstrapVersion(os)}`; } @@ -253,6 +265,7 @@ function getPriority() { * @property {string} instanceType * @property {number} cpuCount * @property {number} threadsPerCore + * @property {boolean} dryRun */ /** @@ -270,8 +283,6 @@ function getEc2Agent(platform, options, ec2Options) { abi, distro, release, - // The agent is created by robobun, see more details here: - // https://github.com/oven-sh/robobun/blob/d46c07e0ac5ac0f9ffe1012f0e98b59e1a0d387a/src/robobun.ts#L1707 robobun: true, robobun2: true, "image-name": getImageName(platform, options), @@ -288,7 +299,7 @@ function getEc2Agent(platform, options, ec2Options) { * @returns {string} */ function getCppAgent(platform, options) { - const { os, arch } = platform; + const { os, arch, distro } = platform; if (os === "darwin") { return { @@ -312,25 +323,9 @@ function getCppAgent(platform, options) { */ function getZigAgent(platform, options) { const { arch } = platform; - return { queue: "build-zig", }; - - // return getEc2Agent( - // { - // os: "linux", - // arch, - // distro: "debian", - // release: "11", - // }, - // options, - // { - // instanceType: arch === "aarch64" ? "c8g.2xlarge" : "c7i.2xlarge", - // cpuCount: 4, - // threadsPerCore: 1, - // }, - // ); } /** @@ -517,6 +512,7 @@ function getBuildBunStep(platform, options) { * @property {string} [buildId] * @property {boolean} [unifiedTests] * @property {string[]} [testFiles] + * @property {boolean} [dryRun] */ /** @@ -564,9 +560,10 @@ function getTestBunStep(platform, options, testOptions = {}) { * @returns {Step} */ function getBuildImageStep(platform, options) { - const { os, arch, distro, release } = platform; + const { os, arch, distro, release, features } = platform; const { publishImages } = options; const action = publishImages ? "publish-image" : "create-image"; + const command = [ "node", "./scripts/machine.mjs", @@ -579,6 +576,10 @@ function getBuildImageStep(platform, options) { "--ci", "--authorized-org=oven-sh", ]; + for (const feature of features || []) { + command.push(`--feature=${feature}`); + } + return { key: `${getImageKey(platform)}-build-image`, label: `${getImageLabel(platform)} - build-image`, @@ -947,9 +948,9 @@ async function getPipelineOptions() { skipBuilds: parseBoolean(options["skip-builds"]), forceBuilds: parseBoolean(options["force-builds"]), skipTests: parseBoolean(options["skip-tests"]), - testFiles: parseArray(options["test-files"]), buildImages: parseBoolean(options["build-images"]), publishImages: parseBoolean(options["publish-images"]), + testFiles: parseArray(options["test-files"]), unifiedBuilds: parseBoolean(options["unified-builds"]), unifiedTests: parseBoolean(options["unified-tests"]), buildProfiles: parseArray(options["build-profiles"]), @@ -959,6 +960,7 @@ async function getPipelineOptions() { testPlatforms: testPlatformKeys?.length ? testPlatformKeys.map(key => testPlatformsMap.get(key)) : Array.from(testPlatformsMap.values()), + dryRun: parseBoolean(options["dry-run"]), }; } @@ -986,6 +988,9 @@ async function getPipelineOptions() { skipBuilds: parseOption(/\[(skip builds?|no builds?|only tests?)\]/i), forceBuilds: parseOption(/\[(force builds?)\]/i), skipTests: parseOption(/\[(skip tests?|no tests?|only builds?)\]/i), + buildImages: parseOption(/\[(build images?)\]/i), + dryRun: parseOption(/\[(dry run)\]/i), + publishImages: parseOption(/\[(publish images?)\]/i), buildPlatforms: Array.from(buildPlatformsMap.values()), testPlatforms: Array.from(testPlatformsMap.values()), buildProfiles: ["release"], @@ -1031,7 +1036,8 @@ async function getPipeline(options = {}) { }); } - const { skipBuilds, forceBuilds, unifiedBuilds } = options; + let { skipBuilds, forceBuilds, unifiedBuilds, dryRun } = options; + dryRun = dryRun || !!buildImages; /** @type {string | undefined} */ let buildId; diff --git a/.dockerignore b/.dockerignore index 6a0ae98134ec53..d76783768281be 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,6 @@ zig-out build vendor node_modules +*.trace + +packages/bun-uws/fuzzing \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a7f41ae5c0080..d483bf06844619 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,11 +2,6 @@ Configuring a development environment for Bun can take 10-30 minutes depending o If you are using Windows, please refer to [this guide](/docs/project/building-windows.md) -{% details summary="For Ubuntu users" %} -TL;DR: Ubuntu 22.04 is suggested. -Bun currently requires `glibc >=2.32` in development which means if you're on Ubuntu 20.04 (glibc == 2.31), you may likely meet `error: undefined symbol: __libc_single_threaded `. You need to take extra configurations. Also, according to this [issue](https://github.com/llvm/llvm-project/issues/97314), LLVM 16 is no longer maintained on Ubuntu 24.04 (noble). And instead, you might want `brew` to install LLVM 16 for your Ubuntu 24.04. -{% /details %} - ## Install Dependencies Using your system's package manager, install Bun's dependencies: @@ -58,7 +53,7 @@ $ brew install bun ## Install LLVM -Bun requires LLVM 16 (`clang` is part of LLVM). This version requirement is to match WebKit (precompiled), as mismatching versions will cause memory allocation failures at runtime. In most cases, you can install LLVM through your system package manager: +Bun requires LLVM 18 (`clang` is part of LLVM). This version requirement is to match WebKit (precompiled), as mismatching versions will cause memory allocation failures at runtime. In most cases, you can install LLVM through your system package manager: {% codetabs group="os" %} @@ -89,7 +84,7 @@ $ sudo zypper install clang16 lld16 llvm16 If none of the above solutions apply, you will have to install it [manually](https://github.com/llvm/llvm-project/releases/tag/llvmorg-16.0.6). -Make sure Clang/LLVM 16 is in your path: +Make sure Clang/LLVM 18 is in your path: ```bash $ which clang-16 diff --git a/cmake/CompilerFlags.cmake b/cmake/CompilerFlags.cmake index 31d738134a0af1..847b365ddae467 100644 --- a/cmake/CompilerFlags.cmake +++ b/cmake/CompilerFlags.cmake @@ -176,6 +176,10 @@ if(LINUX) DESCRIPTION "Disable relocation read-only (RELRO)" -Wl,-z,norelro ) + register_compiler_flags( + DESCRIPTION "Disable semantic interposition" + -fno-semantic-interposition + ) endif() # --- Assertions --- diff --git a/cmake/Globals.cmake b/cmake/Globals.cmake index 3066bb2033dc67..af66b00f081762 100644 --- a/cmake/Globals.cmake +++ b/cmake/Globals.cmake @@ -291,7 +291,7 @@ function(find_command) set_property(GLOBAL PROPERTY ${FIND_NAME} "${exe}: ${reason}" APPEND) if(version) - satisfies_range(${version} ${${FIND_VERSION_VARIABLE}} ${variable}) + satisfies_range(${version} ${FIND_VERSION} ${variable}) set(${variable} ${${variable}} PARENT_SCOPE) endif() endfunction() diff --git a/cmake/targets/BuildBun.cmake b/cmake/targets/BuildBun.cmake index 24ec57d409bf4b..a846750ffad0cb 100644 --- a/cmake/targets/BuildBun.cmake +++ b/cmake/targets/BuildBun.cmake @@ -886,49 +886,21 @@ endif() if(LINUX) if(NOT ABI STREQUAL "musl") - if(ARCH STREQUAL "aarch64") - target_link_options(${bun} PUBLIC - -Wl,--wrap=fcntl64 - -Wl,--wrap=statx - ) - endif() - - if(ARCH STREQUAL "x64") - target_link_options(${bun} PUBLIC - -Wl,--wrap=fcntl - -Wl,--wrap=fcntl64 - -Wl,--wrap=fstat - -Wl,--wrap=fstat64 - -Wl,--wrap=fstatat - -Wl,--wrap=fstatat64 - -Wl,--wrap=lstat - -Wl,--wrap=lstat64 - -Wl,--wrap=mknod - -Wl,--wrap=mknodat - -Wl,--wrap=stat - -Wl,--wrap=stat64 - -Wl,--wrap=statx - ) - endif() - + # on arm64 + if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm|ARM|arm64|ARM64|aarch64|AARCH64") target_link_options(${bun} PUBLIC - -Wl,--wrap=cosf -Wl,--wrap=exp -Wl,--wrap=expf - -Wl,--wrap=fmod - -Wl,--wrap=fmodf -Wl,--wrap=log - -Wl,--wrap=log10f -Wl,--wrap=log2 -Wl,--wrap=log2f -Wl,--wrap=logf -Wl,--wrap=pow -Wl,--wrap=powf - -Wl,--wrap=sincosf - -Wl,--wrap=sinf - -Wl,--wrap=tanf + -Wl,--wrap=fcntl64 ) endif() + endif() if(NOT ABI STREQUAL "musl") target_link_options(${bun} PUBLIC @@ -956,7 +928,7 @@ if(LINUX) -Wl,-z,combreloc -Wl,--no-eh-frame-hdr -Wl,--sort-section=name - -Wl,--hash-style=gnu + -Wl,--hash-style=both -Wl,--build-id=sha1 # Better for debugging than default -Wl,-Map=${bun}.linker-map ) @@ -968,6 +940,7 @@ if(WIN32) set(BUN_SYMBOLS_PATH ${CWD}/src/symbols.def) target_link_options(${bun} PUBLIC /DEF:${BUN_SYMBOLS_PATH}) elseif(APPLE) + set(BUN_SYMBOLS_PATH ${CWD}/src/symbols.txt) target_link_options(${bun} PUBLIC -exported_symbols_list ${BUN_SYMBOLS_PATH}) else() diff --git a/cmake/targets/BuildLibArchive.cmake b/cmake/targets/BuildLibArchive.cmake index e0cffd020be574..da8bfcb7cd5c01 100644 --- a/cmake/targets/BuildLibArchive.cmake +++ b/cmake/targets/BuildLibArchive.cmake @@ -18,7 +18,7 @@ register_cmake_command( -DENABLE_INSTALL=OFF -DENABLE_TEST=OFF -DENABLE_WERROR=OFF - -DENABLE_BZIP2=OFF + -DENABLE_BZip2=OFF -DENABLE_CAT=OFF -DENABLE_EXPAT=OFF -DENABLE_ICONV=OFF diff --git a/cmake/tools/SetupCcache.cmake b/cmake/tools/SetupCcache.cmake index d2367205c87d72..a128fac98bd690 100644 --- a/cmake/tools/SetupCcache.cmake +++ b/cmake/tools/SetupCcache.cmake @@ -5,6 +5,11 @@ if(NOT ENABLE_CCACHE OR CACHE_STRATEGY STREQUAL "none") return() endif() +if (CI AND NOT APPLE) + setenv(CCACHE_DISABLE 1) + return() +endif() + find_command( VARIABLE CCACHE_PROGRAM @@ -38,7 +43,8 @@ setenv(CCACHE_FILECLONE 1) setenv(CCACHE_STATSLOG ${BUILD_PATH}/ccache.log) if(CI) - setenv(CCACHE_SLOPPINESS "pch_defines,time_macros,locale,clang_index_store,gcno_cwd,include_file_ctime,include_file_mtime") + # FIXME: Does not work on Ubuntu 18.04 + # setenv(CCACHE_SLOPPINESS "pch_defines,time_macros,locale,clang_index_store,gcno_cwd,include_file_ctime,include_file_mtime") else() setenv(CCACHE_SLOPPINESS "pch_defines,time_macros,locale,random_seed,clang_index_store,gcno_cwd") endif() diff --git a/cmake/tools/SetupLLVM.cmake b/cmake/tools/SetupLLVM.cmake index 9db637b60d5fce..2bcc97ceed0443 100644 --- a/cmake/tools/SetupLLVM.cmake +++ b/cmake/tools/SetupLLVM.cmake @@ -1,14 +1,18 @@ -optionx(ENABLE_LLVM BOOL "If LLVM should be used for compilation" DEFAULT ON) + +set(DEFAULT_ENABLE_LLVM ON) + +# if target is bun-zig, set ENABLE_LLVM to OFF +if(TARGET bun-zig) + set(DEFAULT_ENABLE_LLVM OFF) +endif() + +optionx(ENABLE_LLVM BOOL "If LLVM should be used for compilation" DEFAULT ${DEFAULT_ENABLE_LLVM}) if(NOT ENABLE_LLVM) return() endif() -if(CMAKE_HOST_WIN32 OR CMAKE_HOST_APPLE OR EXISTS "/etc/alpine-release") - set(DEFAULT_LLVM_VERSION "18.1.8") -else() - set(DEFAULT_LLVM_VERSION "16.0.6") -endif() +set(DEFAULT_LLVM_VERSION "18.1.8") optionx(LLVM_VERSION STRING "The version of LLVM to use" DEFAULT ${DEFAULT_LLVM_VERSION}) @@ -73,7 +77,7 @@ macro(find_llvm_command variable command) VERSION_VARIABLE LLVM_VERSION COMMAND ${commands} PATHS ${LLVM_PATHS} - VERSION ${LLVM_VERSION} + VERSION >=${LLVM_VERSION_MAJOR}.1.0 ) list(APPEND CMAKE_ARGS -D${variable}=${${variable}}) endmacro() diff --git a/cmake/tools/SetupWebKit.cmake b/cmake/tools/SetupWebKit.cmake index e7cb26be5e59b4..50c1e2c7b93224 100644 --- a/cmake/tools/SetupWebKit.cmake +++ b/cmake/tools/SetupWebKit.cmake @@ -2,7 +2,7 @@ option(WEBKIT_VERSION "The version of WebKit to use") option(WEBKIT_LOCAL "If a local version of WebKit should be used instead of downloading") if(NOT WEBKIT_VERSION) - set(WEBKIT_VERSION 8f9ae4f01a047c666ef548864294e01df731d4ea) + set(WEBKIT_VERSION e17d16e0060b3d80ae40e78353d19575c9a8f3af) endif() if(WEBKIT_LOCAL) diff --git a/scripts/agent.mjs b/scripts/agent.mjs index e94f0658d0e071..ece3359cc79111 100755 --- a/scripts/agent.mjs +++ b/scripts/agent.mjs @@ -20,7 +20,6 @@ import { getEnv, writeFile, spawnSafe, - spawn, mkdir, } from "./utils.mjs"; import { parseArgs } from "node:util"; @@ -76,10 +75,10 @@ async function doBuildkiteAgent(action) { command_user=${escape(username)} pidfile=${escape(pidPath)} - start_stop_daemon_args=" \ - --background \ - --make-pidfile \ - --stdout ${escape(agentLogPath)} \ + start_stop_daemon_args=" \\ + --background \\ + --make-pidfile \\ + --stdout ${escape(agentLogPath)} \\ --stderr ${escape(agentLogPath)}" depend() { @@ -88,7 +87,6 @@ async function doBuildkiteAgent(action) { } `; writeFile(servicePath, service, { mode: 0o755 }); - writeFile(`/etc/conf.d/buildkite-agent`, `rc_ulimit="-n 262144"`); await spawnSafe(["rc-update", "add", "buildkite-agent", "default"], { stdio: "inherit", privileged: true }); } @@ -143,7 +141,7 @@ async function doBuildkiteAgent(action) { shell = `"${cmd}" /S /C`; } else { const sh = which("sh", { required: true }); - shell = `${sh} -e -c`; + shell = `${sh} -elc`; } const flags = ["enable-job-log-tmpfile", "no-feature-reporting"]; diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 18defcfe8854f1..42988c4f338f60 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Version: 7 +# Version: 8 # A script that installs the dependencies needed to build and test Bun. # This should work on macOS and Linux with a POSIX shell. @@ -11,15 +11,17 @@ # increment the version comment to indicate that a new image should be built. # Otherwise, the existing image will be retroactively updated. -pid=$$ +pid="$$" print() { echo "$@" } error() { - echo "error: $@" >&2 - kill -s TERM "$pid" + print "error: $@" >&2 + if ! [ "$$" = "$pid" ]; then + kill -s TERM "$pid" + fi exit 1 } @@ -39,24 +41,32 @@ execute_sudo() { } execute_as_user() { + sh="$(require sh)" + if [ "$sudo" = "1" ] || [ "$can_sudo" = "1" ]; then if [ -f "$(which sudo)" ]; then - execute sudo -n -u "$user" /bin/sh -c "$*" + execute sudo -n -u "$user" "$sh" -lc "$*" elif [ -f "$(which doas)" ]; then - execute doas -u "$user" /bin/sh -c "$*" + execute doas -u "$user" "$sh" -lc "$*" elif [ -f "$(which su)" ]; then - execute su -s /bin/sh "$user" -c "$*" + execute su -s "$sh" "$user" -lc "$*" else - execute /bin/sh -c "$*" + execute "$sh" -lc "$*" fi else - execute /bin/sh -c "$*" + execute "$sh" -lc "$*" fi } grant_to_user() { path="$1" - execute_sudo chown -R "$user:$group" "$path" + if ! [ -f "$path" ] && ! [ -d "$path" ]; then + error "Could not find file or directory: \"$path\"" + fi + + chown="$(require chown)" + execute_sudo "$chown" -R "$user:$group" "$path" + execute_sudo chmod -R 777 "$path" } which() { @@ -68,15 +78,15 @@ require() { if ! [ -f "$path" ]; then error "Command \"$1\" is required, but is not installed." fi - echo "$path" + print "$path" } fetch() { - curl=$(which curl) + curl="$(which curl)" if [ -f "$curl" ]; then execute "$curl" -fsSL "$1" else - wget=$(which wget) + wget="$(which wget)" if [ -f "$wget" ]; then execute "$wget" -qO- "$1" else @@ -85,78 +95,115 @@ fetch() { fi } -download_file() { - url="$1" - filename="${2:-$(basename "$url")}" - tmp="$(execute mktemp -d)" - execute chmod 755 "$tmp" - - path="$tmp/$filename" - fetch "$url" >"$path" - execute chmod 644 "$path" - - print "$path" -} - compare_version() { if [ "$1" = "$2" ]; then - echo "0" + print "0" elif [ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]; then - echo "-1" + print "-1" else - echo "1" + print "1" fi } -append_to_file() { - file="$1" - content="$2" +create_directory() { + path="$1" + path_dir="$path" + while ! [ -d "$path_dir" ]; do + path_dir="$(dirname "$path_dir")" + done - file_needs_sudo="0" - if [ -f "$file" ]; then - if ! [ -r "$file" ] || ! [ -w "$file" ]; then - file_needs_sudo="1" - fi + path_needs_sudo="0" + if ! [ -r "$path_dir" ] || ! [ -w "$path_dir" ]; then + path_needs_sudo="1" + fi + + mkdir="$(require mkdir)" + if [ "$path_needs_sudo" = "1" ]; then + execute_sudo "$mkdir" -p "$path" else - execute_as_user mkdir -p "$(dirname "$file")" - execute_as_user touch "$file" + execute "$mkdir" -p "$path" fi - echo "$content" | while read -r line; do - if ! grep -q "$line" "$file"; then - if [ "$file_needs_sudo" = "1" ]; then - execute_sudo sh -c "echo '$line' >> '$file'" - else - echo "$line" >>"$file" - fi - fi - done + grant_to_user "$path" } -append_to_file_sudo() { - file="$1" +create_tmp_directory() { + mktemp="$(require mktemp)" + path="$(execute "$mktemp" -d)" + grant_to_user "$path" + print "$path" +} + +create_file() { + path="$1" + path_dir="$(dirname "$path")" + if ! [ -d "$path_dir" ]; then + create_directory "$path_dir" + fi + + path_needs_sudo="0" + if ! [ -r "$path" ] || ! [ -w "$path" ]; then + path_needs_sudo="1" + fi + + if [ "$path_needs_sudo" = "1" ]; then + execute_sudo touch "$path" + else + execute touch "$path" + fi + content="$2" + if [ -n "$content" ]; then + append_file "$path" "$content" + fi - if ! [ -f "$file" ]; then - execute_sudo mkdir -p "$(dirname "$file")" - execute_sudo touch "$file" + grant_to_user "$path" +} + +append_file() { + path="$1" + if ! [ -f "$path" ]; then + create_file "$path" + fi + + path_needs_sudo="0" + if ! [ -r "$path" ] || ! [ -w "$path" ]; then + path_needs_sudo="1" fi - echo "$content" | while read -r line; do - if ! grep -q "$line" "$file"; then - echo "$line" | execute_sudo tee "$file" >/dev/null + content="$2" + print "$content" | while read -r line; do + if ! grep -q "$line" "$path"; then + sh="$(require sh)" + if [ "$path_needs_sudo" = "1" ]; then + execute_sudo "$sh" -c "echo '$line' >> '$path'" + else + execute "$sh" -c "echo '$line' >> '$path'" + fi fi done } +download_file() { + file_url="$1" + file_tmp_dir="$(create_tmp_directory)" + file_tmp_path="$file_tmp_dir/$(basename "$file_url")" + + fetch "$file_url" >"$file_tmp_path" + grant_to_user "$file_tmp_path" + + print "$file_tmp_path" +} + append_to_profile() { content="$1" profiles=".profile .zprofile .bash_profile .bashrc .zshrc" for profile in $profiles; do - file="$home/$profile" - if [ "$ci" = "1" ] || [ -f "$file" ]; then - append_to_file "$file" "$content" - fi + for profile_path in "$current_home/$profile" "$home/$profile"; do + if [ "$ci" = "1" ] || [ -f "$profile_path" ]; then + append_file "$profile_path" "$content" + fi + done done } @@ -190,19 +237,22 @@ move_to_bin() { check_features() { print "Checking features..." - case "$CI" in - true | 1) - ci=1 - print "CI: enabled" - ;; - esac - - case "$@" in - *--ci*) - ci=1 - print "CI: enabled" - ;; - esac + for arg in "$@"; do + case "$arg" in + *--ci*) + ci=1 + print "CI: enabled" + ;; + *--osxcross*) + osxcross=1 + print "Cross-compiling to macOS: enabled" + ;; + *--gcc-13*) + gcc_version="13" + print "GCC 13: enabled" + ;; + esac + done } check_operating_system() { @@ -211,17 +261,29 @@ check_operating_system() { os="$("$uname" -s)" case "$os" in - Linux*) os="linux" ;; - Darwin*) os="darwin" ;; - *) error "Unsupported operating system: $os" ;; + Linux*) + os="linux" + ;; + Darwin*) + os="darwin" + ;; + *) + error "Unsupported operating system: $os" + ;; esac print "Operating System: $os" arch="$("$uname" -m)" case "$arch" in - x86_64 | x64 | amd64) arch="x64" ;; - aarch64 | arm64) arch="aarch64" ;; - *) error "Unsupported architecture: $arch" ;; + x86_64 | x64 | amd64) + arch="x64" + ;; + aarch64 | arm64) + arch="aarch64" + ;; + *) + error "Unsupported architecture: $arch" + ;; esac print "Architecture: $arch" @@ -235,7 +297,7 @@ check_operating_system() { abi="musl" alpine="$(cat /etc/alpine-release)" if [ "$alpine" ~ "_" ]; then - release="$(echo "$alpine" | cut -d_ -f1)-edge" + release="$(print "$alpine" | cut -d_ -f1)-edge" else release="$alpine" fi @@ -255,6 +317,7 @@ check_operating_system() { distro="$("$sw_vers" -productName)" release="$("$sw_vers" -productVersion)" fi + case "$arch" in x64) sysctl="$(which sysctl)" @@ -277,7 +340,7 @@ check_operating_system() { ldd="$(which ldd)" if [ -f "$ldd" ]; then ldd_version="$($ldd --version 2>&1)" - abi_version="$(echo "$ldd_version" | grep -o -E '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -n 1)" + abi_version="$(print "$ldd_version" | grep -o -E '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -n 1)" case "$ldd_version" in *musl*) abi="musl" @@ -394,6 +457,10 @@ check_user() { can_sudo=1 print "Sudo: can be used" fi + + current_user="$user" + current_group="$group" + current_home="$home" } check_ulimit() { @@ -405,15 +472,12 @@ check_ulimit() { systemd_conf="/etc/systemd/system.conf" if [ -f "$systemd_conf" ]; then limits_conf="/etc/security/limits.d/99-unlimited.conf" - if ! [ -f "$limits_conf" ]; then - execute_sudo mkdir -p "$(dirname "$limits_conf")" - execute_sudo touch "$limits_conf" - fi + create_file "$limits_conf" fi limits="core data fsize memlock nofile rss stack cpu nproc as locks sigpending msgqueue" for limit in $limits; do - limit_upper="$(echo "$limit" | tr '[:lower:]' '[:upper:]')" + limit_upper="$(print "$limit" | tr '[:lower:]' '[:upper:]')" limit_value="unlimited" case "$limit" in @@ -425,13 +489,13 @@ check_ulimit() { if [ -f "$limits_conf" ]; then limit_users="root *" for limit_user in $limit_users; do - append_to_file "$limits_conf" "$limit_user soft $limit $limit_value" - append_to_file "$limits_conf" "$limit_user hard $limit $limit_value" + append_file "$limits_conf" "$limit_user soft $limit $limit_value" + append_file "$limits_conf" "$limit_user hard $limit $limit_value" done fi if [ -f "$systemd_conf" ]; then - append_to_file "$systemd_conf" "DefaultLimit$limit_upper=$limit_value" + append_file "$systemd_conf" "DefaultLimit$limit_upper=$limit_value" fi done @@ -448,13 +512,13 @@ check_ulimit() { esac rc_ulimit="$rc_ulimit -$limit_flag $limit_value" done - append_to_file "$rc_conf" "rc_ulimit=\"$rc_ulimit\"" + append_file "$rc_conf" "rc_ulimit=\"$rc_ulimit\"" fi pam_confs="/etc/pam.d/common-session /etc/pam.d/common-session-noninteractive" for pam_conf in $pam_confs; do if [ -f "$pam_conf" ]; then - append_to_file "$pam_conf" "session optional pam_limits.so" + append_file "$pam_conf" "session optional pam_limits.so" fi done @@ -462,6 +526,24 @@ check_ulimit() { if [ -f "$systemctl" ]; then execute_sudo "$systemctl" daemon-reload fi + + # Configure dpkg and apt for faster operation in CI environments + if [ "$ci" = "1" ] && [ "$pm" = "apt" ]; then + dpkg_conf="/etc/dpkg/dpkg.cfg.d/01-ci-options" + execute_sudo create_directory "$(dirname "$dpkg_conf")" + append_file "$dpkg_conf" "force-unsafe-io" + append_file "$dpkg_conf" "no-debsig" + + apt_conf="/etc/apt/apt.conf.d/99-ci-options" + execute_sudo create_directory "$(dirname "$apt_conf")" + append_file "$apt_conf" 'Acquire::Languages "none";' + append_file "$apt_conf" 'Acquire::GzipIndexes "true";' + append_file "$apt_conf" 'Acquire::CompressionTypes::Order:: "gz";' + append_file "$apt_conf" 'APT::Get::Install-Recommends "false";' + append_file "$apt_conf" 'APT::Get::Install-Suggests "false";' + append_file "$apt_conf" 'Dpkg::Options { "--force-confdef"; "--force-confold"; }' + fi + } package_manager() { @@ -557,7 +639,7 @@ install_brew() { bash="$(require bash)" script=$(download_file "https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh") - NONINTERACTIVE=1 execute_as_user "$bash" "$script" + execute_as_user "$bash" -lc "NONINTERACTIVE=1 $script" case "$arch" in x64) @@ -638,7 +720,7 @@ nodejs_version_exact() { } nodejs_version() { - echo "$(nodejs_version_exact)" | cut -d. -f1 + print "$(nodejs_version_exact)" | cut -d. -f1 } install_nodejs() { @@ -674,14 +756,21 @@ install_nodejs() { } install_nodejs_headers() { - headers_tar="$(download_file "https://nodejs.org/download/release/v$(nodejs_version_exact)/node-v$(nodejs_version_exact)-headers.tar.gz")" - headers_dir="$(dirname "$headers_tar")" - execute tar -xzf "$headers_tar" -C "$headers_dir" - headers_include="$headers_dir/node-v$(nodejs_version_exact)/include" - execute_sudo cp -R "$headers_include/" "/usr" + nodejs_headers_tar="$(download_file "https://nodejs.org/download/release/v$(nodejs_version_exact)/node-v$(nodejs_version_exact)-headers.tar.gz")" + nodejs_headers_dir="$(dirname "$nodejs_headers_tar")" + execute tar -xzf "$nodejs_headers_tar" -C "$nodejs_headers_dir" + + nodejs_headers_include="$nodejs_headers_dir/node-v$(nodejs_version_exact)/include" + execute_sudo cp -R "$nodejs_headers_include/" "/usr" +} + +bun_version_exact() { + print "1.1.38" } install_bun() { + install_packages unzip + case "$pm" in apk) install_packages \ @@ -690,23 +779,24 @@ install_bun() { ;; esac - bash="$(require bash)" - script=$(download_file "https://bun.sh/install") - - version="${1:-"latest"}" - case "$version" in - latest) - execute_as_user "$bash" "$script" + case "$abi" in + musl) + bun_triplet="bun-$os-$arch-$abi" ;; *) - execute_as_user "$bash" "$script" -s "$version" + bun_triplet="bun-$os-$arch" ;; esac - move_to_bin "$home/.bun/bin/bun" - bun_path="$(which bun)" - bunx_path="$(dirname "$bun_path")/bunx" - execute_sudo ln -sf "$bun_path" "$bunx_path" + unzip="$(require unzip)" + bun_download_url="https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/bun-v$(bun_version_exact)/$bun_triplet.zip" + bun_zip="$(download_file "$bun_download_url")" + bun_tmpdir="$(dirname "$bun_zip")" + execute "$unzip" -o "$bun_zip" -d "$bun_tmpdir" + + move_to_bin "$bun_tmpdir/$bun_triplet/bun" + bun_path="$(require bun)" + execute_sudo ln -sf "$bun_path" "$(dirname "$bun_path")/bunx" } install_cmake() { @@ -799,24 +889,19 @@ install_build_essentials() { install_cmake install_llvm + install_osxcross + install_gcc install_ccache install_rust install_docker } llvm_version_exact() { - case "$os-$abi" in - darwin-* | windows-* | linux-musl) - print "18.1.8" - ;; - linux-*) - print "16.0.6" - ;; - esac + print "18.1.8" } llvm_version() { - echo "$(llvm_version_exact)" | cut -d. -f1 + print "$(llvm_version_exact)" | cut -d. -f1 } install_llvm() { @@ -824,14 +909,7 @@ install_llvm() { apt) bash="$(require bash)" llvm_script="$(download_file "https://apt.llvm.org/llvm.sh")" - case "$distro-$release" in - ubuntu-24*) - execute_sudo "$bash" "$llvm_script" "$(llvm_version)" all -njammy - ;; - *) - execute_sudo "$bash" "$llvm_script" "$(llvm_version)" all - ;; - esac + execute_sudo "$bash" "$llvm_script" "$(llvm_version)" all ;; brew) install_packages "llvm@$(llvm_version)" @@ -849,6 +927,77 @@ install_llvm() { esac } +install_gcc() { + if ! [ "$os" = "linux" ] || ! [ "$distro" = "ubuntu" ] || [ -z "$gcc_version" ]; then + return + fi + + # Taken from WebKit's Dockerfile. + # https://github.com/oven-sh/WebKit/blob/816a3c02e0f8b53f8eec06b5ed911192589b51e2/Dockerfile + + execute_sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y + execute_sudo apt update -y + execute_sudo apt install -y \ + "gcc-$gcc_version" \ + "g++-$gcc_version" \ + "libgcc-$gcc_version-dev" \ + "libstdc++-$gcc_version-dev" \ + libasan6 \ + libubsan1 \ + libatomic1 \ + libtsan0 \ + liblsan0 \ + libgfortran5 \ + libc6-dev + + execute_sudo update-alternatives \ + --install /usr/bin/gcc gcc "/usr/bin/gcc-$gcc_version" 130 \ + --slave /usr/bin/g++ g++ "/usr/bin/g++-$gcc_version" \ + --slave /usr/bin/gcc-ar gcc-ar "/usr/bin/gcc-ar-$gcc_version" \ + --slave /usr/bin/gcc-nm gcc-nm "/usr/bin/gcc-nm-$gcc_version" \ + --slave /usr/bin/gcc-ranlib gcc-ranlib "/usr/bin/gcc-ranlib-$gcc_version" + + case "$arch" in + x64) + arch_path="x86_64-linux-gnu" + ;; + aarch64) + arch_path="aarch64-linux-gnu" + ;; + esac + + llvm_v="18" + + append_to_profile "export CC=clang-${llvm_v}" + append_to_profile "export CXX=clang++-${llvm_v}" + append_to_profile "export AR=llvm-ar-${llvm_v}" + append_to_profile "export RANLIB=llvm-ranlib-${llvm_v}" + append_to_profile "export LD=lld-${llvm_v}" + append_to_profile "export LD_LIBRARY_PATH=/usr/lib/gcc/${arch_path}/${gcc_version}:/usr/lib/${arch_path}" + append_to_profile "export LIBRARY_PATH=/usr/lib/gcc/${arch_path}/${gcc_version}:/usr/lib/${arch_path}" + append_to_profile "export CPLUS_INCLUDE_PATH=/usr/include/c++/${gcc_version}:/usr/include/${arch_path}/c++/${gcc_version}" + append_to_profile "export C_INCLUDE_PATH=/usr/lib/gcc/${arch_path}/${gcc_version}/include" + + gcc_path="/usr/lib/gcc/$arch_path/$gcc_version" + create_directory "$gcc_path" + execute_sudo ln -sf /usr/lib/$arch_path/libstdc++.so.6 "$gcc_path/libstdc++.so.6" + + ld_conf_path="/etc/ld.so.conf.d/gcc-$gcc_version.conf" + append_file "$ld_conf_path" "$gcc_path" + append_file "$ld_conf_path" "/usr/lib/$arch_path" + execute_sudo ldconfig + + execute_sudo ln -sf $(which clang-$llvm_v) /usr/bin/clang + execute_sudo ln -sf $(which clang++-$llvm_v) /usr/bin/clang++ + execute_sudo ln -sf $(which lld-$llvm_v) /usr/bin/lld + execute_sudo ln -sf $(which lldb-$llvm_v) /usr/bin/lldb + execute_sudo ln -sf $(which clangd-$llvm_v) /usr/bin/clangd + execute_sudo ln -sf $(which llvm-ar-$llvm_v) /usr/bin/llvm-ar + execute_sudo ln -sf $(which ld.lld-$llvm_v) /usr/bin/ld + execute_sudo ln -sf $(which clang) /usr/bin/cc + execute_sudo ln -sf $(which clang++) /usr/bin/c++ +} + install_ccache() { case "$pm" in apt | apk | brew) @@ -865,9 +1014,23 @@ install_rust() { cargo ;; *) + rust_home="/opt/rust" + create_directory "$rust_home" + append_to_profile "export RUSTUP_HOME=$rust_home" + append_to_profile "export CARGO_HOME=$rust_home" + sh="$(require sh)" - script=$(download_file "https://sh.rustup.rs") - execute_as_user "$sh" "$script" -y + rustup_script=$(download_file "https://sh.rustup.rs") + execute "$sh" -lc "$rustup_script -y --no-modify-path" + append_to_path "$rust_home/bin" + ;; + esac + + case "$osxcross" in + 1) + rustup="$(require rustup)" + execute_as_user "$rustup" target add aarch64-apple-darwin + execute_as_user "$rustup" target add x86_64-apple-darwin ;; esac } @@ -910,6 +1073,46 @@ install_docker() { fi } +macos_sdk_version() { + # https://github.com/alexey-lysiuk/macos-sdk/releases + print "13.3" +} + +install_osxcross() { + if ! [ "$os" = "linux" ] || ! [ "$osxcross" = "1" ]; then + return + fi + + install_packages \ + libssl-dev \ + lzma-dev \ + libxml2-dev \ + zlib1g-dev \ + bzip2 \ + cpio + + osxcross_path="/opt/osxcross" + create_directory "$osxcross_path" + + osxcross_commit="29fe6dd35522073c9df5800f8cd1feb4b9a993a8" + osxcross_tar="$(download_file "https://github.com/tpoechtrager/osxcross/archive/$osxcross_commit.tar.gz")" + execute tar -xzf "$osxcross_tar" -C "$osxcross_path" + + osxcross_build_path="$osxcross_path/build" + execute mv "$osxcross_path/osxcross-$osxcross_commit" "$osxcross_build_path" + + osxcross_sdk_tar="$(download_file "https://github.com/alexey-lysiuk/macos-sdk/releases/download/$(macos_sdk_version)/MacOSX$(macos_sdk_version).tar.xz")" + execute mv "$osxcross_sdk_tar" "$osxcross_build_path/tarballs/MacOSX$(macos_sdk_version).sdk.tar.xz" + + bash="$(require bash)" + execute_sudo ln -sf "$(which clang-$(llvm_version))" /usr/bin/clang + execute_sudo ln -sf "$(which clang++-$(llvm_version))" /usr/bin/clang++ + execute_sudo "$bash" -lc "UNATTENDED=1 TARGET_DIR='$osxcross_path' $osxcross_build_path/build.sh" + + execute_sudo rm -rf "$osxcross_build_path" + grant_to_user "$osxcross_path" +} + install_tailscale() { if [ "$docker" = "1" ]; then return @@ -975,14 +1178,12 @@ create_buildkite_user() { buildkite_paths="$home /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /var/run/buildkite-agent/buildkite-agent.sock" for path in $buildkite_paths; do - execute_sudo mkdir -p "$path" - execute_sudo chown -R "$user:$group" "$path" + create_directory "$path" done buildkite_files="/var/run/buildkite-agent/buildkite-agent.pid" for file in $buildkite_files; do - execute_sudo touch "$file" - execute_sudo chown "$user:$group" "$file" + create_file "$file" done } @@ -992,27 +1193,22 @@ install_buildkite() { fi buildkite_version="3.87.0" - case "$os-$arch" in - linux-aarch64) - buildkite_filename="buildkite-agent-linux-arm64-$buildkite_version.tar.gz" - ;; - linux-x64) - buildkite_filename="buildkite-agent-linux-amd64-$buildkite_version.tar.gz" - ;; - darwin-aarch64) - buildkite_filename="buildkite-agent-darwin-arm64-$buildkite_version.tar.gz" + case "$arch" in + aarch64) + buildkite_arch="arm64" ;; - darwin-x64) - buildkite_filename="buildkite-agent-darwin-amd64-$buildkite_version.tar.gz" + x64) + buildkite_arch="amd64" ;; esac + + buildkite_filename="buildkite-agent-$os-$buildkite_arch-$buildkite_version.tar.gz" buildkite_url="https://github.com/buildkite/agent/releases/download/v$buildkite_version/$buildkite_filename" - buildkite_filepath="$(download_file "$buildkite_url" "$buildkite_filename")" - buildkite_tmpdir="$(dirname "$buildkite_filepath")" + buildkite_tar="$(download_file "$buildkite_url")" + buildkite_tmpdir="$(dirname "$buildkite_tar")" - execute tar -xzf "$buildkite_filepath" -C "$buildkite_tmpdir" + execute tar -xzf "$buildkite_tar" -C "$buildkite_tmpdir" move_to_bin "$buildkite_tmpdir/buildkite-agent" - execute rm -rf "$buildkite_tmpdir" } install_chromium() { @@ -1103,6 +1299,19 @@ install_chromium() { esac } +clean_system() { + if ! [ "$ci" = "1" ]; then + return + fi + + print "Cleaning system..." + + tmp_paths="/tmp /var/tmp" + for path in $tmp_paths; do + execute_sudo rm -rf "$path"/* + done +} + main() { check_features "$@" check_operating_system @@ -1114,6 +1323,7 @@ main() { install_common_software install_build_essentials install_chromium + clean_system } main "$@" diff --git a/scripts/docker.mjs b/scripts/docker.mjs new file mode 100644 index 00000000000000..60c9aa2ea54030 --- /dev/null +++ b/scripts/docker.mjs @@ -0,0 +1,300 @@ +import { inspect } from "node:util"; +import { $, isCI, spawn, spawnSafe, which } from "./utils.mjs"; + +export const docker = { + get name() { + return "docker"; + }, + + /** + * @typedef {"linux" | "darwin" | "windows"} DockerOs + * @typedef {"amd64" | "arm64"} DockerArch + * @typedef {`${DockerOs}/${DockerArch}`} DockerPlatform + */ + + /** + * @param {Platform} platform + * @returns {DockerPlatform} + */ + getPlatform(platform) { + const { os, arch } = platform; + if (arch === "aarch64") { + return `${os}/arm64`; + } else if (arch === "x64") { + return `${os}/amd64`; + } + throw new Error(`Unsupported platform: ${inspect(platform)}`); + }, + + /** + * @typedef DockerSpawnOptions + * @property {DockerPlatform} [platform] + * @property {boolean} [json] + */ + + /** + * @param {string[]} args + * @param {DockerSpawnOptions & import("./utils.mjs").SpawnOptions} [options] + * @returns {Promise} + */ + async spawn(args, options = {}) { + const docker = which("docker", { required: true }); + + let env = { ...process.env }; + if (isCI) { + env["BUILDKIT_PROGRESS"] = "plain"; + } + + const { json, platform } = options; + if (json) { + args.push("--format=json"); + } + if (platform) { + args.push(`--platform=${platform}`); + } + + const { error, stdout } = await spawnSafe($`${docker} ${args}`, { env, ...options }); + if (error) { + return; + } + if (!json) { + return stdout; + } + + try { + return JSON.parse(stdout); + } catch { + return; + } + }, + + /** + * @typedef {Object} DockerImage + * @property {string} Id + * @property {string[]} RepoTags + * @property {string[]} RepoDigests + * @property {string} Created + * @property {DockerOs} Os + * @property {DockerArch} Architecture + * @property {number} Size + */ + + /** + * @param {string} url + * @param {DockerPlatform} [platform] + * @returns {Promise} + */ + async pullImage(url, platform) { + const done = await this.spawn($`pull ${url}`, { + platform, + throwOnError: error => !/No such image|manifest unknown/i.test(inspect(error)), + }); + return !!done; + }, + + /** + * @param {string} url + * @param {DockerPlatform} [platform] + * @returns {Promise} + */ + async inspectImage(url, platform) { + /** @type {DockerImage[]} */ + const images = await this.spawn($`image inspect ${url}`, { + json: true, + throwOnError: error => !/No such image/i.test(inspect(error)), + }); + + if (!images) { + const pulled = await this.pullImage(url, platform); + if (pulled) { + return this.inspectImage(url, platform); + } + } + + const { os, arch } = platform || {}; + return images + ?.filter(({ Os, Architecture }) => !os || !arch || (Os === os && Architecture === arch)) + ?.find((a, b) => (a.Created < b.Created ? 1 : -1)); + }, + + /** + * @typedef {Object} DockerContainer + * @property {string} Id + * @property {string} Name + * @property {string} Image + * @property {string} Created + * @property {DockerContainerState} State + * @property {DockerContainerNetworkSettings} NetworkSettings + */ + + /** + * @typedef {Object} DockerContainerState + * @property {"exited" | "running"} Status + * @property {number} [Pid] + * @property {number} ExitCode + * @property {string} [Error] + * @property {string} StartedAt + * @property {string} FinishedAt + */ + + /** + * @typedef {Object} DockerContainerNetworkSettings + * @property {string} [IPAddress] + */ + + /** + * @param {string} containerId + * @returns {Promise} + */ + async inspectContainer(containerId) { + const containers = await this.spawn($`container inspect ${containerId}`, { json: true }); + return containers?.find(a => a.Id === containerId); + }, + + /** + * @returns {Promise} + */ + async listContainers() { + const containers = await this.spawn($`container ls --all`, { json: true }); + return containers || []; + }, + + /** + * @typedef {Object} DockerRunOptions + * @property {string[]} [command] + * @property {DockerPlatform} [platform] + * @property {string} [name] + * @property {boolean} [detach] + * @property {"always" | "never"} [pull] + * @property {boolean} [rm] + * @property {"no" | "on-failure" | "always"} [restart] + */ + + /** + * @param {string} url + * @param {DockerRunOptions} [options] + * @returns {Promise} + */ + async runContainer(url, options = {}) { + const { detach, command = [], ...containerOptions } = options; + const args = Object.entries(containerOptions) + .filter(([_, value]) => typeof value !== "undefined") + .map(([key, value]) => (typeof value === "boolean" ? `--${key}` : `--${key}=${value}`)); + if (detach) { + args.push("--detach"); + } else { + args.push("--tty", "--interactive"); + } + + const stdio = detach ? "pipe" : "inherit"; + const result = await this.spawn($`run ${args} ${url} ${command}`, { stdio }); + if (!detach) { + return; + } + + const containerId = result.trim(); + const container = await this.inspectContainer(containerId); + if (!container) { + throw new Error(`Failed to run container: ${inspect(result)}`); + } + return container; + }, + + /** + * @param {Platform} platform + * @returns {Promise} + */ + async getBaseImage(platform) { + const { os, distro, release } = platform; + const dockerPlatform = this.getPlatform(platform); + + let url; + if (os === "linux") { + if (distro === "debian" || distro === "ubuntu" || distro === "alpine") { + url = `docker.io/library/${distro}:${release}`; + } else if (distro === "amazonlinux") { + url = `public.ecr.aws/amazonlinux/amazonlinux:${release}`; + } + } + + if (url) { + const image = await this.inspectImage(url, dockerPlatform); + if (image) { + return image; + } + } + + throw new Error(`Unsupported platform: ${inspect(platform)}`); + }, + + /** + * @param {DockerContainer} container + * @param {MachineOptions} [options] + * @returns {Machine} + */ + toMachine(container, options = {}) { + const { Id: containerId } = container; + + const exec = (command, options) => { + return spawn(["docker", "exec", containerId, ...command], options); + }; + + const execSafe = (command, options) => { + return spawnSafe(["docker", "exec", containerId, ...command], options); + }; + + const upload = async (source, destination) => { + await spawn(["docker", "cp", source, `${containerId}:${destination}`]); + }; + + const attach = async () => { + const { exitCode, error } = await spawn(["docker", "exec", "-it", containerId, "sh"], { + stdio: "inherit", + }); + + if (exitCode === 0 || exitCode === 130) { + return; + } + + throw error; + }; + + const snapshot = async name => { + await spawn(["docker", "commit", containerId]); + }; + + const kill = async () => { + await spawn(["docker", "kill", containerId]); + }; + + return { + cloud: "docker", + id: containerId, + spawn: exec, + spawnSafe: execSafe, + upload, + attach, + snapshot, + close: kill, + [Symbol.asyncDispose]: kill, + }; + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { Id: imageId, Os, Architecture } = await docker.getBaseImage(options); + + const container = await docker.runContainer(imageId, { + platform: `${Os}/${Architecture}`, + command: ["sleep", "1d"], + detach: true, + rm: true, + restart: "no", + }); + + return this.toMachine(container, options); + }, +}; diff --git a/scripts/google.mjs b/scripts/google.mjs new file mode 100644 index 00000000000000..f5fb1daf552712 --- /dev/null +++ b/scripts/google.mjs @@ -0,0 +1,510 @@ +import { $, spawnSafe, which, getUsernameForDistro } from "./utils.mjs"; + +export const google = { + get cloud() { + return "google"; + }, + + /** + * @param {string[]} args + * @param {import("./utils.mjs").SpawnOptions} [options] + * @returns {Promise} + */ + async spawn(args, options = {}) { + const gcloud = which("gcloud", { required: true }); + + let env = { ...process.env }; + // if (isCI) { + // env; // TODO: Add Google Cloud credentials + // } else { + // env["TERM"] = "dumb"; + // } + + const { stdout } = await spawnSafe($`${gcloud} ${args} --format json`, { + env, + ...options, + }); + try { + return JSON.parse(stdout); + } catch { + return; + } + }, + + /** + * @param {Record} [options] + * @returns {string[]} + */ + getFilters(options = {}) { + const filter = Object.entries(options) + .filter(([, value]) => value !== undefined) + .map(([key, value]) => [value.includes("*") ? `${key}~${value}` : `${key}=${value}`]) + .join(" AND "); + return filter ? ["--filter", filter] : []; + }, + + /** + * @param {Record} options + * @returns {string[]} + */ + getFlags(options) { + return Object.entries(options) + .filter(([, value]) => value !== undefined) + .flatMap(([key, value]) => { + if (typeof value === "boolean") { + return value ? [`--${key}`] : []; + } + return [`--${key}=${value}`]; + }); + }, + + /** + * @param {Record} options + * @returns {string} + * @link https://cloud.google.com/sdk/gcloud/reference/topic/escaping + */ + getMetadata(options) { + const delimiter = Math.random().toString(36).substring(2, 15); + const entries = Object.entries(options) + .map(([key, value]) => `${key}=${value}`) + .join(delimiter); + return `^${delimiter}^${entries}`; + }, + + /** + * @param {string} name + * @returns {string} + */ + getLabel(name) { + return name.replace(/[^a-z0-9_-]/g, "-").toLowerCase(); + }, + + /** + * @typedef {Object} GoogleImage + * @property {string} id + * @property {string} name + * @property {string} family + * @property {"X86_64" | "ARM64"} architecture + * @property {string} diskSizeGb + * @property {string} selfLink + * @property {"READY"} status + * @property {string} creationTimestamp + */ + + /** + * @param {Partial} [options] + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/images/list + */ + async listImages(options) { + const filters = google.getFilters(options); + const images = await google.spawn($`compute images list ${filters} --preview-images --show-deprecated`); + return images.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + }, + + /** + * @param {Record} options + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/images/create + */ + async createImage(options) { + const { name, ...otherOptions } = options; + const flags = this.getFlags(otherOptions); + const imageId = name || "i-" + Math.random().toString(36).substring(2, 15); + return this.spawn($`compute images create ${imageId} ${flags}`); + }, + + /** + * @typedef {Object} GoogleInstance + * @property {string} id + * @property {string} name + * @property {"RUNNING"} status + * @property {string} machineType + * @property {string} zone + * @property {GoogleDisk[]} disks + * @property {GoogleNetworkInterface[]} networkInterfaces + * @property {object} [scheduling] + * @property {"STANDARD" | "SPOT"} [scheduling.provisioningModel] + * @property {boolean} [scheduling.preemptible] + * @property {Record} [labels] + * @property {string} selfLink + * @property {string} creationTimestamp + */ + + /** + * @typedef {Object} GoogleDisk + * @property {string} deviceName + * @property {boolean} boot + * @property {"X86_64" | "ARM64"} architecture + * @property {string[]} [licenses] + * @property {number} diskSizeGb + */ + + /** + * @typedef {Object} GoogleNetworkInterface + * @property {"IPV4_ONLY" | "IPV4_IPV6" | "IPV6_ONLY"} stackType + * @property {string} name + * @property {string} network + * @property {string} networkIP + * @property {string} subnetwork + * @property {GoogleAccessConfig[]} accessConfigs + */ + + /** + * @typedef {Object} GoogleAccessConfig + * @property {string} name + * @property {"ONE_TO_ONE_NAT" | "INTERNAL_NAT"} type + * @property {string} [natIP] + */ + + /** + * @param {Record} options + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/create + */ + async createInstance(options) { + const { name, ...otherOptions } = options || {}; + const flags = this.getFlags(otherOptions); + const instanceId = name || "i-" + Math.random().toString(36).substring(2, 15); + const [instance] = await this.spawn($`compute instances create ${instanceId} ${flags}`); + return instance; + }, + + /** + * @param {string} instanceId + * @param {string} zoneId + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/stop + */ + async stopInstance(instanceId, zoneId) { + await this.spawn($`compute instances stop ${instanceId} --zone=${zoneId}`); + }, + + /** + * @param {string} instanceId + * @param {string} zoneId + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/delete + */ + async deleteInstance(instanceId, zoneId) { + await this.spawn($`compute instances delete ${instanceId} --delete-disks=all --zone=${zoneId}`, { + throwOnError: error => !/not found/i.test(inspect(error)), + }); + }, + + /** + * @param {string} instanceId + * @param {string} username + * @param {string} zoneId + * @param {object} [options] + * @param {boolean} [options.wait] + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/reset-windows-password + */ + async resetWindowsPassword(instanceId, username, zoneId, options = {}) { + const attempts = options.wait ? 15 : 1; + for (let i = 0; i < attempts; i++) { + const result = await this.spawn( + $`compute reset-windows-password ${instanceId} --user=${username} --zone=${zoneId}`, + { + throwOnError: error => !/instance may not be ready for use/i.test(inspect(error)), + }, + ); + if (result) { + const { password } = result; + if (password) { + return password; + } + } + await new Promise(resolve => setTimeout(resolve, 60000 * i)); + } + }, + + /** + * @param {Partial} options + * @returns {Promise} + */ + async listInstances(options) { + const filters = this.getFilters(options); + const instances = await this.spawn($`compute instances list ${filters}`); + return instances.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async getMachineImage(options) { + const { os, arch, distro, release } = options; + const architecture = arch === "aarch64" ? "ARM64" : "X86_64"; + + /** @type {string | undefined} */ + let family; + if (os === "linux") { + if (!distro || distro === "debian") { + family = `debian-${release || "*"}`; + } else if (distro === "ubuntu") { + family = `ubuntu-${release?.replace(/\./g, "") || "*"}`; + } else if (distro === "fedora") { + family = `fedora-coreos-${release || "*"}`; + } else if (distro === "rhel") { + family = `rhel-${release || "*"}`; + } + } else if (os === "windows" && arch === "x64") { + if (!distro || distro === "server") { + family = `windows-${release || "*"}`; + } + } + + if (family) { + const images = await this.listImages({ family, architecture }); + if (images.length) { + const [image] = images; + return image; + } + } + + throw new Error(`Unsupported platform: ${inspect(options)}`); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { name, os, arch, distro, instanceType, tags, preemptible, detached } = options; + const image = await google.getMachineImage(options); + const { selfLink: imageUrl } = image; + + const username = getUsername(distro || os); + const userData = getUserData({ ...options, username }); + + /** @type {Record} */ + let metadata; + if (os === "windows") { + metadata = { + "enable-windows-ssh": "TRUE", + "sysprep-specialize-script-ps1": userData, + }; + } else { + metadata = { + "user-data": userData, + }; + } + + const instance = await google.createInstance({ + "name": name, + "zone": "us-central1-a", + "image": imageUrl, + "machine-type": instanceType || (arch === "aarch64" ? "t2a-standard-2" : "t2d-standard-2"), + "boot-disk-auto-delete": true, + "boot-disk-size": `${getDiskSize(options)}GB`, + "metadata": this.getMetadata(metadata), + "labels": Object.entries(tags || {}) + .filter(([, value]) => value !== undefined) + .map(([key, value]) => `${this.getLabel(key)}=${value}`) + .join(","), + "provisioning-model": preemptible ? "SPOT" : "STANDARD", + "instance-termination-action": preemptible || !detached ? "DELETE" : undefined, + "no-restart-on-failure": true, + "threads-per-core": 1, + "max-run-duration": detached ? undefined : "6h", + }); + + return this.toMachine(instance, options); + }, + + /** + * @param {GoogleInstance} instance + * @param {MachineOptions} [options] + * @returns {Machine} + */ + toMachine(instance, options = {}) { + const { id: instanceId, name, zone: zoneUrl, machineType: machineTypeUrl, labels } = instance; + const machineType = machineTypeUrl.split("/").pop(); + const zoneId = zoneUrl.split("/").pop(); + + let os, arch, distro, release; + const { disks = [] } = instance; + for (const { boot, architecture, licenses = [] } of disks) { + if (!boot) { + continue; + } + + if (architecture === "X86_64") { + arch = "x64"; + } else if (architecture === "ARM64") { + arch = "aarch64"; + } + + for (const license of licenses) { + const linuxMatch = /(debian|ubuntu|fedora|rhel)-(\d+)/i.exec(license); + if (linuxMatch) { + os = "linux"; + [, distro, release] = linuxMatch; + } else { + const windowsMatch = /windows-server-(\d+)-dc-core/i.exec(license); + if (windowsMatch) { + os = "windows"; + distro = "windowsserver"; + [, release] = windowsMatch; + } + } + } + } + + let publicIp; + const { networkInterfaces = [] } = instance; + for (const { accessConfigs = [] } of networkInterfaces) { + for (const { type, natIP } of accessConfigs) { + if (type === "ONE_TO_ONE_NAT" && natIP) { + publicIp = natIP; + } + } + } + + let preemptible; + const { scheduling } = instance; + if (scheduling) { + const { provisioningModel, preemptible: isPreemptible } = scheduling; + preemptible = provisioningModel === "SPOT" || isPreemptible; + } + + /** + * @returns {SshOptions} + */ + const connect = () => { + if (!publicIp) { + throw new Error(`Failed to find public IP for instance: ${name}`); + } + + /** @type {string | undefined} */ + let username; + + const { os, distro } = options; + if (os || distro) { + username = getUsernameForDistro(distro || os); + } + + return { hostname: publicIp, username }; + }; + + const spawn = async (command, options) => { + const connectOptions = connect(); + return spawnSsh({ ...connectOptions, command }, options); + }; + + const spawnSafe = async (command, options) => { + const connectOptions = connect(); + return spawnSshSafe({ ...connectOptions, command }, options); + }; + + const rdp = async () => { + const { hostname, username } = connect(); + const rdpUsername = `${username}-rdp`; + const password = await google.resetWindowsPassword(instanceId, rdpUsername, zoneId, { wait: true }); + return { hostname, username: rdpUsername, password }; + }; + + const attach = async () => { + const connectOptions = connect(); + await spawnSshSafe({ ...connectOptions }); + }; + + const upload = async (source, destination) => { + const connectOptions = connect(); + await spawnScp({ ...connectOptions, source, destination }); + }; + + const snapshot = async name => { + const stopResult = await this.stopInstance(instanceId, zoneId); + console.log(stopResult); + const image = await this.createImage({ + ["source-disk"]: instanceId, + ["zone"]: zoneId, + ["name"]: name || `${instanceId}-snapshot-${Date.now()}`, + }); + console.log(image); + return; + }; + + const terminate = async () => { + await google.deleteInstance(instanceId, zoneId); + }; + + return { + cloud: "google", + os, + arch, + distro, + release, + id: instanceId, + imageId: undefined, + name, + instanceType: machineType, + region: zoneId, + publicIp, + preemptible, + labels, + spawn, + spawnSafe, + rdp, + attach, + upload, + snapshot, + close: terminate, + [Symbol.asyncDispose]: terminate, + }; + }, + + /** + * @param {Record} [labels] + * @returns {Promise} + */ + async getMachines(labels) { + const filters = labels ? this.getFilters({ labels }) : {}; + const instances = await google.listInstances(filters); + return instances.map(instance => this.toMachine(instance)); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async getImage(options) { + const { os, arch, distro, release } = options; + const architecture = arch === "aarch64" ? "ARM64" : "X86_64"; + + let name; + let username; + if (os === "linux") { + if (distro === "debian") { + name = `debian-${release}-*`; + username = "admin"; + } else if (distro === "ubuntu") { + name = `ubuntu-${release.replace(/\./g, "")}-*`; + username = "ubuntu"; + } + } else if (os === "windows" && arch === "x64") { + if (distro === "server") { + name = `windows-server-${release}-dc-core-*`; + username = "administrator"; + } + } + + if (name && username) { + const images = await google.listImages({ name, architecture }); + if (images.length) { + const [image] = images; + const { name, selfLink } = image; + return { + id: selfLink, + name, + username, + }; + } + } + + throw new Error(`Unsupported platform: ${inspect(platform)}`); + }, +}; diff --git a/scripts/machine.mjs b/scripts/machine.mjs index 479dbb4cfd7844..8d0ae49ec0dbd5 100755 --- a/scripts/machine.mjs +++ b/scripts/machine.mjs @@ -14,6 +14,8 @@ import { spawnSafe, spawnSyncSafe, startGroup, + spawnSshSafe, + spawnSsh, tmpdir, waitForPort, which, @@ -29,798 +31,39 @@ import { rm, homedir, isWindows, + setupUserData, sha256, isPrivileged, + getUsernameForDistro, } from "./utils.mjs"; import { basename, extname, join, relative, resolve } from "node:path"; import { existsSync, mkdtempSync, readdirSync } from "node:fs"; import { fileURLToPath } from "node:url"; +import { orbstack } from "./orbstack.mjs"; +import { docker } from "./docker.mjs"; +import { google } from "./google.mjs"; +import { tart } from "./tart.mjs"; -/** - * @link https://tart.run/ - * @link https://github.com/cirruslabs/tart - */ -const tart = { - get name() { - return "tart"; - }, - - /** - * @param {string[]} args - * @param {import("./utils.mjs").SpawnOptions} options - * @returns {Promise} - */ - async spawn(args, options) { - const tart = which("tart", { required: true }); - const { json } = options || {}; - const command = json ? [tart, ...args, "--format=json"] : [tart, ...args]; - - const { stdout } = await spawnSafe(command, options); - if (!json) { - return stdout; - } - - try { - return JSON.parse(stdout); - } catch { - return; - } - }, - - /** - * @typedef {"sequoia" | "sonoma" | "ventura" | "monterey"} TartDistro - * @typedef {`ghcr.io/cirruslabs/macos-${TartDistro}-xcode`} TartImage - * @link https://github.com/orgs/cirruslabs/packages?repo_name=macos-image-templates - */ - - /** - * @param {Platform} platform - * @returns {TartImage} - */ - getImage(platform) { - const { os, arch, release } = platform; - if (os !== "darwin" || arch !== "aarch64") { - throw new Error(`Unsupported platform: ${inspect(platform)}`); - } - const distros = { - "15": "sequoia", - "14": "sonoma", - "13": "ventura", - "12": "monterey", - }; - const distro = distros[release]; - if (!distro) { - throw new Error(`Unsupported macOS release: ${distro}`); - } - return `ghcr.io/cirruslabs/macos-${distro}-xcode`; - }, - - /** - * @typedef {Object} TartVm - * @property {string} Name - * @property {"running" | "stopped"} State - * @property {"local"} Source - * @property {number} Size - * @property {number} Disk - * @property {number} [CPU] - * @property {number} [Memory] - */ - - /** - * @returns {Promise} - */ - async listVms() { - return this.spawn(["list"], { json: true }); - }, - - /** - * @param {string} name - * @returns {Promise} - */ - async getVm(name) { - const result = await this.spawn(["get", name], { - json: true, - throwOnError: error => !/does not exist/i.test(inspect(error)), - }); - return { - Name: name, - ...result, - }; - }, - - /** - * @param {string} name - * @returns {Promise} - */ - async stopVm(name) { - await this.spawn(["stop", name, "--timeout=0"], { - throwOnError: error => !/does not exist|is not running/i.test(inspect(error)), - }); - }, - - /** - * @param {string} name - * @returns {Promise} - */ - async deleteVm(name) { - await this.stopVm(name); - await this.spawn(["delete", name], { - throwOnError: error => !/does not exist/i.test(inspect(error)), - }); - }, - - /** - * @param {string} name - * @param {TartImage} image - * @returns {Promise} - */ - async cloneVm(name, image) { - const localName = image.split("/").pop(); - const localVm = await this.getVm(localName); - if (localVm) { - const { Name } = localVm; - await this.spawn(["clone", Name, name]); - return; - } - - console.log(`Cloning macOS image: ${image} (this will take a long time)`); - await this.spawn(["clone", image, localName]); - await this.spawn(["clone", localName, name]); - }, - - /** - * @typedef {Object} TartMount - * @property {boolean} [readOnly] - * @property {string} source - * @property {string} destination - */ - - /** - * @typedef {Object} TartVmOptions - * @property {number} [cpuCount] - * @property {number} [memoryGb] - * @property {number} [diskSizeGb] - * @property {boolean} [no-graphics] - * @property {boolean} [no-audio] - * @property {boolean} [no-clipboard] - * @property {boolean} [recovery] - * @property {boolean} [vnc] - * @property {boolean} [vnc-experimental] - * @property {boolean} [net-softnet] - * @property {TartMount[]} [dir] - */ - - /** - * @param {string} name - * @param {TartVmOptions} options - * @returns {Promise} - */ - async runVm(name, options = {}) { - const { cpuCount, memoryGb, diskSizeGb, dir, ...vmOptions } = options; - - const setArgs = ["--random-mac", "--random-serial"]; - if (cpuCount) { - setArgs.push(`--cpu=${cpuCount}`); - } - if (memoryGb) { - setArgs.push(`--memory=${memoryGb}`); - } - if (diskSizeGb) { - setArgs.push(`--disk-size=${diskSizeGb}`); - } - await this.spawn(["set", name, ...setArgs]); - - const args = Object.entries(vmOptions) - .filter(([, value]) => value !== undefined) - .flatMap(([key, value]) => (typeof value === "boolean" ? (value ? [`--${key}`] : []) : [`--${key}=${value}`])); - if (dir?.length) { - args.push( - ...dir.map(({ source, destination, readOnly }) => `--dir=${source}:${destination}${readOnly ? ":ro" : ""}`), - ); - } - - // This command is blocking, so it needs to be detached and not awaited - this.spawn(["run", name, ...args], { detached: true }); - }, - - /** - * @param {string} name - * @returns {Promise} - */ - async getVmIp(name) { - const stdout = await this.spawn(["ip", name], { - retryOnError: error => /no IP address found/i.test(inspect(error)), - throwOnError: error => !/does not exist/i.test(inspect(error)), - }); - return stdout?.trim(); - }, - - /** - * @param {MachineOptions} options - * @returns {Promise} - */ - async createMachine(options) { - const { name, imageName, cpuCount, memoryGb, diskSizeGb, rdp } = options; - - const image = imageName || this.getImage(options); - const machineId = name || `i-${Math.random().toString(36).slice(2, 11)}`; - await this.cloneVm(machineId, image); - - await this.runVm(machineId, { - cpuCount, - memoryGb, - diskSizeGb, - "net-softnet": isPrivileged(), - "no-audio": true, - "no-clipboard": true, - "no-graphics": true, - "vnc-experimental": rdp, - }); - - return this.toMachine(machineId); - }, - - /** - * @param {string} name - * @returns {Machine} - */ - toMachine(name) { - const connect = async () => { - const hostname = await this.getVmIp(name); - return { - hostname, - // hardcoded by base images - username: "admin", - password: "admin", - }; - }; - - const exec = async (command, options) => { - const connectOptions = await connect(); - return spawnSsh({ ...connectOptions, command }, options); - }; - - const execSafe = async (command, options) => { - const connectOptions = await connect(); - return spawnSshSafe({ ...connectOptions, command }, options); - }; - - const attach = async () => { - const connectOptions = await connect(); - await spawnSshSafe({ ...connectOptions }); - }; - - const upload = async (source, destination) => { - const connectOptions = await connect(); - await spawnScp({ ...connectOptions, source, destination }); - }; - - const rdp = async () => { - const connectOptions = await connect(); - await spawnRdp({ ...connectOptions }); - }; - - const close = async () => { - await this.deleteVm(name); - }; - - return { - cloud: "tart", - id: name, - spawn: exec, - spawnSafe: execSafe, - attach, - upload, - close, - [Symbol.asyncDispose]: close, - }; - }, -}; - -/** - * @link https://docs.orbstack.dev/ - */ -const orbstack = { - get name() { - return "orbstack"; - }, - - /** - * @typedef {Object} OrbstackImage - * @property {string} distro - * @property {string} version - * @property {string} arch - */ - - /** - * @param {Platform} platform - * @returns {OrbstackImage} - */ - getImage(platform) { - const { os, arch, distro, release } = platform; - if (os !== "linux" || !/^debian|ubuntu|alpine|fedora|centos$/.test(distro)) { - throw new Error(`Unsupported platform: ${inspect(platform)}`); - } - - return { - distro, - version: release, - arch: arch === "aarch64" ? "arm64" : "amd64", - }; - }, - - /** - * @typedef {Object} OrbstackVm - * @property {string} id - * @property {string} name - * @property {"running"} state - * @property {OrbstackImage} image - * @property {OrbstackConfig} config - */ - - /** - * @typedef {Object} OrbstackConfig - * @property {string} default_username - * @property {boolean} isolated - */ - - /** - * @typedef {Object} OrbstackVmOptions - * @property {string} [name] - * @property {OrbstackImage} image - * @property {string} [username] - * @property {string} [password] - * @property {string} [userData] - */ - - /** - * @param {OrbstackVmOptions} options - * @returns {Promise} - */ - async createVm(options) { - const { name, image, username, password, userData } = options; - const { distro, version, arch } = image; - const uniqueId = name || `linux-${distro}-${version}-${arch}-${Math.random().toString(36).slice(2, 11)}`; - - const args = [`--arch=${arch}`, `${distro}:${version}`, uniqueId]; - if (username) { - args.push(`--user=${username}`); - } - if (password) { - args.push(`--set-password=${password}`); - } - - let userDataPath; - if (userData) { - userDataPath = mkdtemp("orbstack-user-data-", "user-data.txt"); - writeFile(userDataPath, userData); - args.push(`--user-data=${userDataPath}`); - } - - try { - await spawnSafe($`orbctl create ${args}`); - } finally { - if (userDataPath) { - rm(userDataPath); - } - } - - return this.inspectVm(uniqueId); - }, - - /** - * @param {string} name - */ - async deleteVm(name) { - await spawnSafe($`orbctl delete ${name}`, { - throwOnError: error => !/machine not found/i.test(inspect(error)), - }); - }, - - /** - * @param {string} name - * @returns {Promise} - */ - async inspectVm(name) { - const { exitCode, stdout } = await spawnSafe($`orbctl info ${name} --format=json`, { - throwOnError: error => !/machine not found/i.test(inspect(error)), - }); - if (exitCode === 0) { - return JSON.parse(stdout); - } - }, - - /** - * @returns {Promise} - */ - async listVms() { - const { stdout } = await spawnSafe($`orbctl list --format=json`); - return JSON.parse(stdout); - }, - - /** - * @param {MachineOptions} options - * @returns {Promise} - */ - async createMachine(options) { - const { distro } = options; - const username = getUsername(distro); - const userData = getUserData({ ...options, username }); - - const image = this.getImage(options); - const vm = await this.createVm({ - image, - username, - userData, - }); - - return this.toMachine(vm, options); - }, - - /** - * @param {OrbstackVm} vm - * @returns {Machine} - */ - toMachine(vm) { - const { id, name, config } = vm; - - const { default_username: username } = config; - const connectOptions = { - username, - hostname: `${name}@orb`, - }; - - const exec = async (command, options) => { - return spawnSsh({ ...connectOptions, command }, options); - }; - - const execSafe = async (command, options) => { - return spawnSshSafe({ ...connectOptions, command }, options); - }; - - const attach = async () => { - await spawnSshSafe({ ...connectOptions }); - }; - - const upload = async (source, destination) => { - await spawnSafe(["orbctl", "push", `--machine=${name}`, source, destination]); - }; - - const close = async () => { - await this.deleteVm(name); - }; - - return { - cloud: "orbstack", - id, - name, - spawn: exec, - spawnSafe: execSafe, - upload, - attach, - close, - [Symbol.asyncDispose]: close, - }; - }, -}; - -const docker = { +const aws = { get name() { - return "docker"; - }, - - /** - * @typedef {"linux" | "darwin" | "windows"} DockerOs - * @typedef {"amd64" | "arm64"} DockerArch - * @typedef {`${DockerOs}/${DockerArch}`} DockerPlatform - */ - - /** - * @param {Platform} platform - * @returns {DockerPlatform} - */ - getPlatform(platform) { - const { os, arch } = platform; - if (arch === "aarch64") { - return `${os}/arm64`; - } else if (arch === "x64") { - return `${os}/amd64`; - } - throw new Error(`Unsupported platform: ${inspect(platform)}`); + return "aws"; }, - /** - * @typedef DockerSpawnOptions - * @property {DockerPlatform} [platform] - * @property {boolean} [json] - */ - /** * @param {string[]} args - * @param {DockerSpawnOptions & import("./utils.mjs").SpawnOptions} [options] + * @param {import("./utils.mjs").SpawnOptions} [options] * @returns {Promise} */ async spawn(args, options = {}) { - const docker = which("docker", { required: true }); + const aws = which("aws", { required: true }); - let env = { ...process.env }; + let env; if (isCI) { - env["BUILDKIT_PROGRESS"] = "plain"; - } - - const { json, platform } = options; - if (json) { - args.push("--format=json"); - } - if (platform) { - args.push(`--platform=${platform}`); - } - - const { error, stdout } = await spawnSafe($`${docker} ${args}`, { env, ...options }); - if (error) { - return; - } - if (!json) { - return stdout; - } - - try { - return JSON.parse(stdout); - } catch { - return; - } - }, - - /** - * @typedef {Object} DockerImage - * @property {string} Id - * @property {string[]} RepoTags - * @property {string[]} RepoDigests - * @property {string} Created - * @property {DockerOs} Os - * @property {DockerArch} Architecture - * @property {number} Size - */ - - /** - * @param {string} url - * @param {DockerPlatform} [platform] - * @returns {Promise} - */ - async pullImage(url, platform) { - const done = await this.spawn($`pull ${url}`, { - platform, - throwOnError: error => !/No such image|manifest unknown/i.test(inspect(error)), - }); - return !!done; - }, - - /** - * @param {string} url - * @param {DockerPlatform} [platform] - * @returns {Promise} - */ - async inspectImage(url, platform) { - /** @type {DockerImage[]} */ - const images = await this.spawn($`image inspect ${url}`, { - json: true, - throwOnError: error => !/No such image/i.test(inspect(error)), - }); - - if (!images) { - const pulled = await this.pullImage(url, platform); - if (pulled) { - return this.inspectImage(url, platform); - } - } - - const { os, arch } = platform || {}; - return images - ?.filter(({ Os, Architecture }) => !os || !arch || (Os === os && Architecture === arch)) - ?.find((a, b) => (a.Created < b.Created ? 1 : -1)); - }, - - /** - * @typedef {Object} DockerContainer - * @property {string} Id - * @property {string} Name - * @property {string} Image - * @property {string} Created - * @property {DockerContainerState} State - * @property {DockerContainerNetworkSettings} NetworkSettings - */ - - /** - * @typedef {Object} DockerContainerState - * @property {"exited" | "running"} Status - * @property {number} [Pid] - * @property {number} ExitCode - * @property {string} [Error] - * @property {string} StartedAt - * @property {string} FinishedAt - */ - - /** - * @typedef {Object} DockerContainerNetworkSettings - * @property {string} [IPAddress] - */ - - /** - * @param {string} containerId - * @returns {Promise} - */ - async inspectContainer(containerId) { - const containers = await this.spawn($`container inspect ${containerId}`, { json: true }); - return containers?.find(a => a.Id === containerId); - }, - - /** - * @returns {Promise} - */ - async listContainers() { - const containers = await this.spawn($`container ls --all`, { json: true }); - return containers || []; - }, - - /** - * @typedef {Object} DockerRunOptions - * @property {string[]} [command] - * @property {DockerPlatform} [platform] - * @property {string} [name] - * @property {boolean} [detach] - * @property {"always" | "never"} [pull] - * @property {boolean} [rm] - * @property {"no" | "on-failure" | "always"} [restart] - */ - - /** - * @param {string} url - * @param {DockerRunOptions} [options] - * @returns {Promise} - */ - async runContainer(url, options = {}) { - const { detach, command = [], ...containerOptions } = options; - const args = Object.entries(containerOptions) - .filter(([_, value]) => typeof value !== "undefined") - .map(([key, value]) => (typeof value === "boolean" ? `--${key}` : `--${key}=${value}`)); - if (detach) { - args.push("--detach"); - } else { - args.push("--tty", "--interactive"); - } - - const stdio = detach ? "pipe" : "inherit"; - const result = await this.spawn($`run ${args} ${url} ${command}`, { stdio }); - if (!detach) { - return; - } - - const containerId = result.trim(); - const container = await this.inspectContainer(containerId); - if (!container) { - throw new Error(`Failed to run container: ${inspect(result)}`); - } - return container; - }, - - /** - * @param {Platform} platform - * @returns {Promise} - */ - async getBaseImage(platform) { - const { os, distro, release } = platform; - const dockerPlatform = this.getPlatform(platform); - - let url; - if (os === "linux") { - if (distro === "debian" || distro === "ubuntu" || distro === "alpine") { - url = `docker.io/library/${distro}:${release}`; - } else if (distro === "amazonlinux") { - url = `public.ecr.aws/amazonlinux/amazonlinux:${release}`; - } - } - - if (url) { - const image = await this.inspectImage(url, dockerPlatform); - if (image) { - return image; - } - } - - throw new Error(`Unsupported platform: ${inspect(platform)}`); - }, - - /** - * @param {DockerContainer} container - * @param {MachineOptions} [options] - * @returns {Machine} - */ - toMachine(container, options = {}) { - const { Id: containerId } = container; - - const exec = (command, options) => { - return spawn(["docker", "exec", containerId, ...command], options); - }; - - const execSafe = (command, options) => { - return spawnSafe(["docker", "exec", containerId, ...command], options); - }; - - const upload = async (source, destination) => { - await spawn(["docker", "cp", source, `${containerId}:${destination}`]); - }; - - const attach = async () => { - const { exitCode, error } = await spawn(["docker", "exec", "-it", containerId, "sh"], { - stdio: "inherit", - }); - - if (exitCode === 0 || exitCode === 130) { - return; - } - - throw error; - }; - - const snapshot = async name => { - await spawn(["docker", "commit", containerId]); - }; - - const kill = async () => { - await spawn(["docker", "kill", containerId]); - }; - - return { - cloud: "docker", - id: containerId, - spawn: exec, - spawnSafe: execSafe, - upload, - attach, - close: kill, - [Symbol.asyncDispose]: kill, - }; - }, - - /** - * @param {MachineOptions} options - * @returns {Promise} - */ - async createMachine(options) { - const { Id: imageId, Os, Architecture } = await docker.getBaseImage(options); - - const container = await docker.runContainer(imageId, { - platform: `${Os}/${Architecture}`, - command: ["sleep", "1d"], - detach: true, - rm: true, - restart: "no", - }); - - return this.toMachine(container, options); - }, -}; - -const aws = { - get name() { - return "aws"; - }, - - /** - * @param {string[]} args - * @param {import("./utils.mjs").SpawnOptions} [options] - * @returns {Promise} - */ - async spawn(args, options = {}) { - const aws = which("aws", { required: true }); - - let env; - if (isCI) { - env = { - AWS_ACCESS_KEY_ID: getSecret("EC2_ACCESS_KEY_ID", { required: true }), - AWS_SECRET_ACCESS_KEY: getSecret("EC2_SECRET_ACCESS_KEY", { required: true }), - AWS_REGION: getSecret("EC2_REGION", { required: false }) || "us-east-1", - }; + env = { + AWS_ACCESS_KEY_ID: getSecret("EC2_ACCESS_KEY_ID", { required: true }), + AWS_SECRET_ACCESS_KEY: getSecret("EC2_SECRET_ACCESS_KEY", { required: true }), + AWS_REGION: getSecret("EC2_REGION", { required: false }) || "us-east-1", + }; } const { stdout } = await spawnSafe($`${aws} ${args} --output json`, { env, ...options }); @@ -836,778 +79,335 @@ const aws = { * @returns {string[]} */ getFilters(options = {}) { - return Object.entries(options) - .filter(([_, value]) => typeof value !== "undefined") - .map(([key, value]) => `Name=${key},Values=${value}`); - }, - - /** - * @param {Record} [options] - * @returns {string[]} - */ - getFlags(options = {}) { - return Object.entries(options) - .filter(([_, value]) => typeof value !== "undefined") - .map(([key, value]) => `--${key}=${value}`); - }, - - /** - * @typedef AwsInstance - * @property {string} InstanceId - * @property {string} ImageId - * @property {string} InstanceType - * @property {string} [PublicIpAddress] - * @property {string} [PlatformDetails] - * @property {string} [Architecture] - * @property {object} [Placement] - * @property {string} [Placement.AvailabilityZone] - * @property {string} LaunchTime - */ - - /** - * @param {Record} [options] - * @returns {Promise} - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-instances.html - */ - async describeInstances(options) { - const filters = aws.getFilters(options); - const { Reservations } = await aws.spawn($`ec2 describe-instances --filters ${filters}`); - return Reservations.flatMap(({ Instances }) => Instances).sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1)); - }, - - /** - * @param {Record} [options] - * @returns {Promise} - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/run-instances.html - */ - async runInstances(options) { - for (let i = 0; i < 3; i++) { - const flags = aws.getFlags(options); - const result = await aws.spawn($`ec2 run-instances ${flags}`, { - throwOnError: error => { - if (options["instance-market-options"] && /InsufficientInstanceCapacity/i.test(inspect(error))) { - delete options["instance-market-options"]; - const instanceType = options["instance-type"] || "default"; - console.warn(`There is not enough capacity for ${instanceType} spot instances, retrying with on-demand...`); - return false; - } - return true; - }, - }); - if (result) { - const { Instances } = result; - if (Instances.length) { - return Instances.sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1)); - } - } - await new Promise(resolve => setTimeout(resolve, i * Math.random() * 15_000)); - } - throw new Error(`Failed to run instances: ${inspect(instanceOptions)}`); - }, - - /** - * @param {...string} instanceIds - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/stop-instances.html - */ - async stopInstances(...instanceIds) { - await aws.spawn($`ec2 stop-instances --no-hibernate --force --instance-ids ${instanceIds}`); - }, - - /** - * @param {...string} instanceIds - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/terminate-instances.html - */ - async terminateInstances(...instanceIds) { - await aws.spawn($`ec2 terminate-instances --instance-ids ${instanceIds}`, { - throwOnError: error => !/InvalidInstanceID\.NotFound/i.test(inspect(error)), - }); - }, - - /** - * @param {"instance-running" | "instance-stopped" | "instance-terminated"} action - * @param {...string} instanceIds - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait.html - */ - async waitInstances(action, ...instanceIds) { - await aws.spawn($`ec2 wait ${action} --instance-ids ${instanceIds}`, { - retryOnError: error => /max attempts exceeded/i.test(inspect(error)), - }); - }, - - /** - * @param {string} instanceId - * @param {string} privateKeyPath - * @param {object} [passwordOptions] - * @param {boolean} [passwordOptions.wait] - * @returns {Promise} - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/get-password-data.html - */ - async getPasswordData(instanceId, privateKeyPath, passwordOptions = {}) { - const attempts = passwordOptions.wait ? 15 : 1; - for (let i = 0; i < attempts; i++) { - const { PasswordData } = await aws.spawn($`ec2 get-password-data --instance-id ${instanceId}`); - if (PasswordData) { - return decryptPassword(PasswordData, privateKeyPath); - } - await new Promise(resolve => setTimeout(resolve, 60000 * i)); - } - throw new Error(`Failed to get password data for instance: ${instanceId}`); - }, - - /** - * @typedef AwsImage - * @property {string} ImageId - * @property {string} Name - * @property {string} State - * @property {string} CreationDate - */ - - /** - * @param {Record} [options] - * @returns {Promise} - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-images.html - */ - async describeImages(options = {}) { - const { ["owner-alias"]: owners, ...filterOptions } = options; - const filters = aws.getFilters(filterOptions); - if (owners) { - filters.push(`--owners=${owners}`); - } - const { Images } = await aws.spawn($`ec2 describe-images --filters ${filters}`); - return Images.sort((a, b) => (a.CreationDate < b.CreationDate ? 1 : -1)); - }, - - /** - * @param {Record} [options] - * @returns {Promise} - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/create-image.html - */ - async createImage(options) { - const flags = aws.getFlags(options); - - /** @type {string | undefined} */ - let existingImageId; - - /** @type {AwsImage | undefined} */ - const image = await aws.spawn($`ec2 create-image ${flags}`, { - throwOnError: error => { - const match = /already in use by AMI (ami-[a-z0-9]+)/i.exec(inspect(error)); - if (!match) { - return true; - } - const [, imageId] = match; - existingImageId = imageId; - return false; - }, - }); - - if (!existingImageId) { - const { ImageId } = image; - return ImageId; - } - - await aws.spawn($`ec2 deregister-image --image-id ${existingImageId}`); - const { ImageId } = await aws.spawn($`ec2 create-image ${flags}`); - return ImageId; - }, - - /** - * @param {Record} options - * @returns {Promise} - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/copy-image.html - */ - async copyImage(options) { - const flags = aws.getFlags(options); - const { ImageId } = await aws.spawn($`ec2 copy-image ${flags}`); - return ImageId; - }, - - /** - * @param {"image-available"} action - * @param {...string} imageIds - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait/image-available.html - */ - async waitImage(action, ...imageIds) { - await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`, { - retryOnError: error => /max attempts exceeded/i.test(inspect(error)), - }); - }, - - /** - * @typedef {Object} AwsKeyPair - * @property {string} KeyPairId - * @property {string} KeyName - * @property {string} KeyFingerprint - * @property {string} [PublicKeyMaterial] - */ - - /** - * @param {string[]} [names] - * @returns {Promise} - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-key-pairs.html - */ - async describeKeyPairs(names) { - const command = names - ? $`ec2 describe-key-pairs --include-public-key --key-names ${names}` - : $`ec2 describe-key-pairs --include-public-key`; - const { KeyPairs } = await aws.spawn(command); - return KeyPairs; - }, - - /** - * @param {string | Buffer} publicKey - * @param {string} [name] - * @returns {Promise} - * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/import-key-pair.html - */ - async importKeyPair(publicKey, name) { - const keyName = name || `key-pair-${sha256(publicKey)}`; - const publicKeyBase64 = Buffer.from(publicKey).toString("base64"); - - /** @type {AwsKeyPair | undefined} */ - const keyPair = await aws.spawn( - $`ec2 import-key-pair --key-name ${keyName} --public-key-material ${publicKeyBase64}`, - { - throwOnError: error => !/InvalidKeyPair\.Duplicate/i.test(inspect(error)), - }, - ); - - if (keyPair) { - return keyPair; - } - - const keyPairs = await aws.describeKeyPairs(keyName); - if (keyPairs.length) { - return keyPairs[0]; - } - - throw new Error(`Failed to import key pair: ${keyName}`); - }, - - /** - * @param {AwsImage | string} imageOrImageId - * @returns {Promise} - */ - async getAvailableImage(imageOrImageId) { - let imageId = imageOrImageId; - if (typeof imageOrImageId === "object") { - const { ImageId, State } = imageOrImageId; - if (State === "available") { - return imageOrImageId; - } - imageId = ImageId; - } - - await aws.waitImage("image-available", imageId); - const [availableImage] = await aws.describeImages({ - "state": "available", - "image-id": imageId, - }); - - if (!availableImage) { - throw new Error(`Failed to find available image: ${imageId}`); - } - - return availableImage; - }, - - /** - * @param {MachineOptions} options - * @returns {Promise} - */ - async getBaseImage(options) { - const { os, arch, distro, release } = options; - - let name, owner; - if (os === "linux") { - if (!distro || distro === "debian") { - owner = "amazon"; - name = `debian-${release || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-*`; - } else if (distro === "ubuntu") { - owner = "099720109477"; - name = `ubuntu/images/hvm-ssd*/ubuntu-*-${release || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-server-*`; - } else if (distro === "amazonlinux") { - owner = "amazon"; - if (release === "1" && arch === "x64") { - name = `amzn-ami-2018.03.*`; - } else if (release === "2") { - name = `amzn2-ami-hvm-*-${arch === "aarch64" ? "arm64" : "x86_64"}-gp2`; - } else { - name = `al${release || "*"}-ami-*-${arch === "aarch64" ? "arm64" : "x86_64"}`; - } - } else if (distro === "alpine") { - owner = "538276064493"; - name = `alpine-${release || "*"}.*-${arch === "aarch64" ? "aarch64" : "x86_64"}-uefi-cloudinit-*`; - } else if (distro === "centos") { - owner = "aws-marketplace"; - name = `CentOS-Stream-ec2-${release || "*"}-*.${arch === "aarch64" ? "aarch64" : "x86_64"}-*`; - } - } else if (os === "windows") { - if (!distro || distro === "server") { - owner = "amazon"; - name = `Windows_Server-${release || "*"}-English-Full-Base-*`; - } - } - - if (!name) { - throw new Error(`Unsupported platform: ${inspect(options)}`); - } - - const baseImages = await aws.describeImages({ - "state": "available", - "owner-alias": owner, - "name": name, - }); - - if (!baseImages.length) { - throw new Error(`No base image found: ${inspect(options)}`); - } - - const [baseImage] = baseImages; - return aws.getAvailableImage(baseImage); - }, - - /** - * @param {MachineOptions} options - * @returns {Promise} - */ - async createMachine(options) { - const { os, arch, imageId, instanceType, tags, sshKeys, preemptible } = options; - - /** @type {AwsImage} */ - let image; - if (imageId) { - image = await aws.getAvailableImage(imageId); - } else { - image = await aws.getBaseImage(options); - } - - const { ImageId, Name, RootDeviceName, BlockDeviceMappings } = image; - const blockDeviceMappings = BlockDeviceMappings.map(device => { - const { DeviceName } = device; - if (DeviceName === RootDeviceName) { - return { - ...device, - Ebs: { - VolumeSize: getDiskSize(options), - }, - }; - } - return device; - }); - - const username = getUsername(Name); - - let userData = getUserData({ ...options, username }); - if (os === "windows") { - userData = `${userData}-ExecutionPolicy Unrestricted -NoProfile -NonInteractivetrue`; - } - - let tagSpecification = []; - if (tags) { - tagSpecification = ["instance", "volume"].map(resourceType => { - return { - ResourceType: resourceType, - Tags: Object.entries(tags).map(([Key, Value]) => ({ Key, Value: String(Value) })), - }; - }); - } - - /** @type {string | undefined} */ - let keyName, keyPath; - if (os === "windows") { - const sshKey = sshKeys.find(({ privatePath }) => existsSync(privatePath)); - if (sshKey) { - const { publicKey, privatePath } = sshKey; - const { KeyName } = await aws.importKeyPair(publicKey); - keyName = KeyName; - keyPath = privatePath; - } - } - - let marketOptions; - if (preemptible) { - marketOptions = JSON.stringify({ - MarketType: "spot", - SpotOptions: { - InstanceInterruptionBehavior: "terminate", - SpotInstanceType: "one-time", - }, - }); - } - - const [instance] = await aws.runInstances({ - ["image-id"]: ImageId, - ["instance-type"]: instanceType || (arch === "aarch64" ? "t4g.large" : "t3.large"), - ["user-data"]: userData, - ["block-device-mappings"]: JSON.stringify(blockDeviceMappings), - ["metadata-options"]: JSON.stringify({ - "HttpTokens": "optional", - "HttpEndpoint": "enabled", - "HttpProtocolIpv6": "enabled", - "InstanceMetadataTags": "enabled", - }), - ["tag-specifications"]: JSON.stringify(tagSpecification), - ["key-name"]: keyName, - ["instance-market-options"]: marketOptions, - }); - - return aws.toMachine(instance, { ...options, username, keyPath }); - }, - - /** - * @param {AwsInstance} instance - * @param {MachineOptions} [options] - * @returns {Machine} - */ - toMachine(instance, options = {}) { - let { InstanceId, ImageId, InstanceType, Placement, PublicIpAddress } = instance; - - const connect = async () => { - if (!PublicIpAddress) { - await aws.waitInstances("instance-running", InstanceId); - const [{ PublicIpAddress: IpAddress }] = await aws.describeInstances({ - ["instance-id"]: InstanceId, - }); - PublicIpAddress = IpAddress; - } - - const { username, sshKeys } = options; - const identityPaths = sshKeys - ?.filter(({ privatePath }) => existsSync(privatePath)) - ?.map(({ privatePath }) => privatePath); - - return { hostname: PublicIpAddress, username, identityPaths }; - }; - - const spawn = async (command, options) => { - const connectOptions = await connect(); - return spawnSsh({ ...connectOptions, command }, options); - }; - - const spawnSafe = async (command, options) => { - const connectOptions = await connect(); - return spawnSshSafe({ ...connectOptions, command }, options); - }; - - const rdp = async () => { - const { keyPath } = options; - const { hostname, username } = await connect(); - const password = await aws.getPasswordData(InstanceId, keyPath, { wait: true }); - return { hostname, username, password }; - }; - - const attach = async () => { - const connectOptions = await connect(); - await spawnSshSafe({ ...connectOptions }); - }; - - const upload = async (source, destination) => { - const connectOptions = await connect(); - await spawnScp({ ...connectOptions, source, destination }); - }; - - const snapshot = async name => { - await aws.stopInstances(InstanceId); - await aws.waitInstances("instance-stopped", InstanceId); - const imageId = await aws.createImage({ - ["instance-id"]: InstanceId, - ["name"]: name || `${InstanceId}-snapshot-${Date.now()}`, - }); - await aws.waitImage("image-available", imageId); - return imageId; - }; - - const terminate = async () => { - await aws.terminateInstances(InstanceId); - }; - - return { - cloud: "aws", - id: InstanceId, - imageId: ImageId, - instanceType: InstanceType, - region: Placement?.AvailabilityZone, - get publicIp() { - return PublicIpAddress; - }, - spawn, - spawnSafe, - upload, - attach, - rdp, - snapshot, - close: terminate, - [Symbol.asyncDispose]: terminate, - }; - }, -}; - -const google = { - get cloud() { - return "google"; - }, - - /** - * @param {string[]} args - * @param {import("./utils.mjs").SpawnOptions} [options] - * @returns {Promise} - */ - async spawn(args, options = {}) { - const gcloud = which("gcloud", { required: true }); - - let env = { ...process.env }; - // if (isCI) { - // env; // TODO: Add Google Cloud credentials - // } else { - // env["TERM"] = "dumb"; - // } - - const { stdout } = await spawnSafe($`${gcloud} ${args} --format json`, { - env, - ...options, - }); - try { - return JSON.parse(stdout); - } catch { - return; - } - }, - - /** - * @param {Record} [options] - * @returns {string[]} - */ - getFilters(options = {}) { - const filter = Object.entries(options) - .filter(([, value]) => value !== undefined) - .map(([key, value]) => [value.includes("*") ? `${key}~${value}` : `${key}=${value}`]) - .join(" AND "); - return filter ? ["--filter", filter] : []; + return Object.entries(options) + .filter(([_, value]) => typeof value !== "undefined") + .map(([key, value]) => `Name=${key},Values=${value}`); }, /** - * @param {Record} options + * @param {Record} [options] * @returns {string[]} */ - getFlags(options) { + getFlags(options = {}) { return Object.entries(options) - .filter(([, value]) => value !== undefined) - .flatMap(([key, value]) => { - if (typeof value === "boolean") { - return value ? [`--${key}`] : []; - } - return [`--${key}=${value}`]; - }); + .filter(([_, value]) => typeof value !== "undefined") + .map(([key, value]) => `--${key}=${value}`); }, /** - * @param {Record} options - * @returns {string} - * @link https://cloud.google.com/sdk/gcloud/reference/topic/escaping + * @typedef AwsInstance + * @property {string} InstanceId + * @property {string} ImageId + * @property {string} InstanceType + * @property {string} [PublicIpAddress] + * @property {string} [PlatformDetails] + * @property {string} [Architecture] + * @property {object} [Placement] + * @property {string} [Placement.AvailabilityZone] + * @property {string} LaunchTime */ - getMetadata(options) { - const delimiter = Math.random().toString(36).substring(2, 15); - const entries = Object.entries(options) - .map(([key, value]) => `${key}=${value}`) - .join(delimiter); - return `^${delimiter}^${entries}`; - }, /** - * @param {string} name - * @returns {string} + * @param {Record} [options] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-instances.html */ - getLabel(name) { - return name.replace(/[^a-z0-9_-]/g, "-").toLowerCase(); + async describeInstances(options) { + const filters = aws.getFilters(options); + const { Reservations } = await aws.spawn($`ec2 describe-instances --filters ${filters}`); + return Reservations.flatMap(({ Instances }) => Instances).sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1)); }, /** - * @typedef {Object} GoogleImage - * @property {string} id - * @property {string} name - * @property {string} family - * @property {"X86_64" | "ARM64"} architecture - * @property {string} diskSizeGb - * @property {string} selfLink - * @property {"READY"} status - * @property {string} creationTimestamp + * @param {Record} [options] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/run-instances.html */ + async runInstances(options) { + for (let i = 0; i < 3; i++) { + const flags = aws.getFlags(options); + const result = await aws.spawn($`ec2 run-instances ${flags}`, { + throwOnError: error => { + if (options["instance-market-options"] && /InsufficientInstanceCapacity/i.test(inspect(error))) { + delete options["instance-market-options"]; + const instanceType = options["instance-type"] || "default"; + console.warn(`There is not enough capacity for ${instanceType} spot instances, retrying with on-demand...`); + return false; + } + return true; + }, + }); + if (result) { + const { Instances } = result; + if (Instances.length) { + return Instances.sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1)); + } + } + await new Promise(resolve => setTimeout(resolve, i * Math.random() * 15_000)); + } + throw new Error(`Failed to run instances: ${inspect(instanceOptions)}`); + }, /** - * @param {Partial} [options] - * @returns {Promise} - * @link https://cloud.google.com/sdk/gcloud/reference/compute/images/list + * @param {...string} instanceIds + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/stop-instances.html */ - async listImages(options) { - const filters = google.getFilters(options); - const images = await google.spawn($`compute images list ${filters} --preview-images --show-deprecated`); - return images.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + async stopInstances(...instanceIds) { + await aws.spawn($`ec2 stop-instances --no-hibernate --force --instance-ids ${instanceIds}`); }, /** - * @param {Record} options - * @returns {Promise} - * @link https://cloud.google.com/sdk/gcloud/reference/compute/images/create + * @param {...string} instanceIds + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/terminate-instances.html */ - async createImage(options) { - const { name, ...otherOptions } = options; - const flags = this.getFlags(otherOptions); - const imageId = name || "i-" + Math.random().toString(36).substring(2, 15); - return this.spawn($`compute images create ${imageId} ${flags}`); + async terminateInstances(...instanceIds) { + await aws.spawn($`ec2 terminate-instances --instance-ids ${instanceIds}`, { + throwOnError: error => !/InvalidInstanceID\.NotFound/i.test(inspect(error)), + }); }, /** - * @typedef {Object} GoogleInstance - * @property {string} id - * @property {string} name - * @property {"RUNNING"} status - * @property {string} machineType - * @property {string} zone - * @property {GoogleDisk[]} disks - * @property {GoogleNetworkInterface[]} networkInterfaces - * @property {object} [scheduling] - * @property {"STANDARD" | "SPOT"} [scheduling.provisioningModel] - * @property {boolean} [scheduling.preemptible] - * @property {Record} [labels] - * @property {string} selfLink - * @property {string} creationTimestamp + * @param {"instance-running" | "instance-stopped" | "instance-terminated"} action + * @param {...string} instanceIds + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait.html */ + async waitInstances(action, ...instanceIds) { + await aws.spawn($`ec2 wait ${action} --instance-ids ${instanceIds}`, { + retryOnError: error => /max attempts exceeded/i.test(inspect(error)), + }); + }, /** - * @typedef {Object} GoogleDisk - * @property {string} deviceName - * @property {boolean} boot - * @property {"X86_64" | "ARM64"} architecture - * @property {string[]} [licenses] - * @property {number} diskSizeGb + * @param {string} instanceId + * @param {string} privateKeyPath + * @param {object} [passwordOptions] + * @param {boolean} [passwordOptions.wait] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/get-password-data.html */ + async getPasswordData(instanceId, privateKeyPath, passwordOptions = {}) { + const attempts = passwordOptions.wait ? 15 : 1; + for (let i = 0; i < attempts; i++) { + const { PasswordData } = await aws.spawn($`ec2 get-password-data --instance-id ${instanceId}`); + if (PasswordData) { + return decryptPassword(PasswordData, privateKeyPath); + } + await new Promise(resolve => setTimeout(resolve, 60000 * i)); + } + throw new Error(`Failed to get password data for instance: ${instanceId}`); + }, /** - * @typedef {Object} GoogleNetworkInterface - * @property {"IPV4_ONLY" | "IPV4_IPV6" | "IPV6_ONLY"} stackType - * @property {string} name - * @property {string} network - * @property {string} networkIP - * @property {string} subnetwork - * @property {GoogleAccessConfig[]} accessConfigs + * @typedef AwsImage + * @property {string} ImageId + * @property {string} Name + * @property {string} State + * @property {string} CreationDate */ /** - * @typedef {Object} GoogleAccessConfig - * @property {string} name - * @property {"ONE_TO_ONE_NAT" | "INTERNAL_NAT"} type - * @property {string} [natIP] + * @param {Record} [options] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-images.html */ + async describeImages(options = {}) { + const { ["owner-alias"]: owners, ...filterOptions } = options; + const filters = aws.getFilters(filterOptions); + if (owners) { + filters.push(`--owners=${owners}`); + } + const { Images } = await aws.spawn($`ec2 describe-images --filters ${filters}`); + return Images.sort((a, b) => (a.CreationDate < b.CreationDate ? 1 : -1)); + }, /** - * @param {Record} options - * @returns {Promise} - * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/create + * @param {Record} [options] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/create-image.html */ - async createInstance(options) { - const { name, ...otherOptions } = options || {}; - const flags = this.getFlags(otherOptions); - const instanceId = name || "i-" + Math.random().toString(36).substring(2, 15); - const [instance] = await this.spawn($`compute instances create ${instanceId} ${flags}`); - return instance; + async createImage(options) { + const flags = aws.getFlags(options); + + /** @type {string | undefined} */ + let existingImageId; + + /** @type {AwsImage | undefined} */ + const image = await aws.spawn($`ec2 create-image ${flags}`, { + throwOnError: error => { + const match = /already in use by AMI (ami-[a-z0-9]+)/i.exec(inspect(error)); + if (!match) { + return true; + } + const [, imageId] = match; + existingImageId = imageId; + return false; + }, + }); + + if (!existingImageId) { + const { ImageId } = image; + return ImageId; + } + + await aws.spawn($`ec2 deregister-image --image-id ${existingImageId}`); + const { ImageId } = await aws.spawn($`ec2 create-image ${flags}`); + return ImageId; }, /** - * @param {string} instanceId - * @param {string} zoneId - * @returns {Promise} - * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/stop + * @param {Record} options + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/copy-image.html */ - async stopInstance(instanceId, zoneId) { - await this.spawn($`compute instances stop ${instanceId} --zone=${zoneId}`); + async copyImage(options) { + const flags = aws.getFlags(options); + const { ImageId } = await aws.spawn($`ec2 copy-image ${flags}`); + return ImageId; }, /** - * @param {string} instanceId - * @param {string} zoneId - * @returns {Promise} - * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/delete + * @param {"image-available"} action + * @param {...string} imageIds + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait/image-available.html */ - async deleteInstance(instanceId, zoneId) { - await this.spawn($`compute instances delete ${instanceId} --delete-disks=all --zone=${zoneId}`, { - throwOnError: error => !/not found/i.test(inspect(error)), + async waitImage(action, ...imageIds) { + await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`, { + retryOnError: error => /max attempts exceeded/i.test(inspect(error)), }); }, /** - * @param {string} instanceId - * @param {string} username - * @param {string} zoneId - * @param {object} [options] - * @param {boolean} [options.wait] - * @returns {Promise} - * @link https://cloud.google.com/sdk/gcloud/reference/compute/reset-windows-password + * @typedef {Object} AwsKeyPair + * @property {string} KeyPairId + * @property {string} KeyName + * @property {string} KeyFingerprint + * @property {string} [PublicKeyMaterial] */ - async resetWindowsPassword(instanceId, username, zoneId, options = {}) { - const attempts = options.wait ? 15 : 1; - for (let i = 0; i < attempts; i++) { - const result = await this.spawn( - $`compute reset-windows-password ${instanceId} --user=${username} --zone=${zoneId}`, - { - throwOnError: error => !/instance may not be ready for use/i.test(inspect(error)), - }, - ); - if (result) { - const { password } = result; - if (password) { - return password; - } - } - await new Promise(resolve => setTimeout(resolve, 60000 * i)); + + /** + * @param {string[]} [names] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-key-pairs.html + */ + async describeKeyPairs(names) { + const command = names + ? $`ec2 describe-key-pairs --include-public-key --key-names ${names}` + : $`ec2 describe-key-pairs --include-public-key`; + const { KeyPairs } = await aws.spawn(command); + return KeyPairs; + }, + + /** + * @param {string | Buffer} publicKey + * @param {string} [name] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/import-key-pair.html + */ + async importKeyPair(publicKey, name) { + const keyName = name || `key-pair-${sha256(publicKey)}`; + const publicKeyBase64 = Buffer.from(publicKey).toString("base64"); + + /** @type {AwsKeyPair | undefined} */ + const keyPair = await aws.spawn( + $`ec2 import-key-pair --key-name ${keyName} --public-key-material ${publicKeyBase64}`, + { + throwOnError: error => !/InvalidKeyPair\.Duplicate/i.test(inspect(error)), + }, + ); + + if (keyPair) { + return keyPair; + } + + const keyPairs = await aws.describeKeyPairs(keyName); + if (keyPairs.length) { + return keyPairs[0]; } + + throw new Error(`Failed to import key pair: ${keyName}`); }, /** - * @param {Partial} options - * @returns {Promise} + * @param {AwsImage | string} imageOrImageId + * @returns {Promise} */ - async listInstances(options) { - const filters = this.getFilters(options); - const instances = await this.spawn($`compute instances list ${filters}`); - return instances.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + async getAvailableImage(imageOrImageId) { + let imageId = imageOrImageId; + if (typeof imageOrImageId === "object") { + const { ImageId, State } = imageOrImageId; + if (State === "available") { + return imageOrImageId; + } + imageId = ImageId; + } + + await aws.waitImage("image-available", imageId); + const [availableImage] = await aws.describeImages({ + "state": "available", + "image-id": imageId, + }); + + if (!availableImage) { + throw new Error(`Failed to find available image: ${imageId}`); + } + + return availableImage; }, /** * @param {MachineOptions} options - * @returns {Promise} + * @returns {Promise} */ - async getMachineImage(options) { + async getBaseImage(options) { const { os, arch, distro, release } = options; - const architecture = arch === "aarch64" ? "ARM64" : "X86_64"; - /** @type {string | undefined} */ - let family; + let name, owner; if (os === "linux") { if (!distro || distro === "debian") { - family = `debian-${release || "*"}`; + owner = "amazon"; + name = `debian-${release || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-*`; } else if (distro === "ubuntu") { - family = `ubuntu-${release?.replace(/\./g, "") || "*"}`; - } else if (distro === "fedora") { - family = `fedora-coreos-${release || "*"}`; - } else if (distro === "rhel") { - family = `rhel-${release || "*"}`; + owner = "099720109477"; + name = `ubuntu/images/hvm-ssd*/ubuntu-*-${release || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-server-*`; + } else if (distro === "amazonlinux") { + owner = "amazon"; + if (release === "1" && arch === "x64") { + name = `amzn-ami-2018.03.*`; + } else if (release === "2") { + name = `amzn2-ami-hvm-*-${arch === "aarch64" ? "arm64" : "x86_64"}-gp2`; + } else { + name = `al${release || "*"}-ami-*-${arch === "aarch64" ? "arm64" : "x86_64"}`; + } + } else if (distro === "alpine") { + owner = "538276064493"; + name = `alpine-${release || "*"}.*-${arch === "aarch64" ? "aarch64" : "x86_64"}-uefi-cloudinit-*`; + } else if (distro === "centos") { + owner = "aws-marketplace"; + name = `CentOS-Stream-ec2-${release || "*"}-*.${arch === "aarch64" ? "aarch64" : "x86_64"}-*`; } - } else if (os === "windows" && arch === "x64") { + } else if (os === "windows") { if (!distro || distro === "server") { - family = `windows-${release || "*"}`; + owner = "amazon"; + name = `Windows_Server-${release || "*"}-English-Full-Base-*`; } } - if (family) { - const images = await this.listImages({ family, architecture }); - if (images.length) { - const [image] = images; - return image; - } + if (!name) { + throw new Error(`Unsupported platform: ${inspect(options)}`); + } + + const baseImages = await aws.describeImages({ + "state": "available", + "owner-alias": owner, + "name": name, + }); + + if (!baseImages.length) { + throw new Error(`No base image found: ${inspect(options)}`); } - throw new Error(`Unsupported platform: ${inspect(options)}`); + const [baseImage] = baseImages; + return aws.getAvailableImage(baseImage); }, /** @@ -1615,241 +415,204 @@ const google = { * @returns {Promise} */ async createMachine(options) { - const { name, os, arch, distro, instanceType, tags, preemptible, detached } = options; - const image = await google.getMachineImage(options); - const { selfLink: imageUrl } = image; + const { os, arch, imageId, instanceType, tags, sshKeys, preemptible } = options; + + /** @type {AwsImage} */ + let image; + if (imageId) { + image = await aws.getAvailableImage(imageId); + } else { + image = await aws.getBaseImage(options); + } + + const { ImageId, Name, RootDeviceName, BlockDeviceMappings } = image; + const blockDeviceMappings = BlockDeviceMappings.map(device => { + const { DeviceName } = device; + if (DeviceName === RootDeviceName) { + return { + ...device, + Ebs: { + VolumeSize: getDiskSize(options), + }, + }; + } + return device; + }); - const username = getUsername(distro || os); - const userData = getUserData({ ...options, username }); + const username = getUsernameForDistro(Name); - /** @type {Record} */ - let metadata; + // Only include minimal cloud-init for SSH access + let userData = getUserData({ ...options, username }); if (os === "windows") { - metadata = { - "enable-windows-ssh": "TRUE", - "sysprep-specialize-script-ps1": userData, - }; - } else { - metadata = { - "user-data": userData, - }; + userData = `${userData}-ExecutionPolicy Unrestricted -NoProfile -NonInteractivetrue`; } - const instance = await google.createInstance({ - "name": name, - "zone": "us-central1-a", - "image": imageUrl, - "machine-type": instanceType || (arch === "aarch64" ? "t2a-standard-2" : "t2d-standard-2"), - "boot-disk-auto-delete": true, - "boot-disk-size": `${getDiskSize(options)}GB`, - "metadata": this.getMetadata(metadata), - "labels": Object.entries(tags || {}) - .filter(([, value]) => value !== undefined) - .map(([key, value]) => `${this.getLabel(key)}=${value}`) - .join(","), - "provisioning-model": preemptible ? "SPOT" : "STANDARD", - "instance-termination-action": preemptible || !detached ? "DELETE" : undefined, - "no-restart-on-failure": true, - "threads-per-core": 1, - "max-run-duration": detached ? undefined : "6h", + let tagSpecification = []; + if (tags) { + tagSpecification = ["instance", "volume"].map(resourceType => { + return { + ResourceType: resourceType, + Tags: Object.entries(tags).map(([Key, Value]) => ({ Key, Value: String(Value) })), + }; + }); + } + + /** @type {string | undefined} */ + let keyName, keyPath; + if (os === "windows") { + const sshKey = sshKeys.find(({ privatePath }) => existsSync(privatePath)); + if (sshKey) { + const { publicKey, privatePath } = sshKey; + const { KeyName } = await aws.importKeyPair(publicKey); + keyName = KeyName; + keyPath = privatePath; + } + } + + let marketOptions; + if (preemptible) { + marketOptions = JSON.stringify({ + MarketType: "spot", + SpotOptions: { + InstanceInterruptionBehavior: "terminate", + SpotInstanceType: "one-time", + }, + }); + } + + const [instance] = await aws.runInstances({ + ["image-id"]: ImageId, + ["instance-type"]: instanceType || (arch === "aarch64" ? "t4g.large" : "t3.large"), + ["user-data"]: userData, + ["block-device-mappings"]: JSON.stringify(blockDeviceMappings), + ["metadata-options"]: JSON.stringify({ + "HttpTokens": "optional", + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "enabled", + "InstanceMetadataTags": "enabled", + }), + ["tag-specifications"]: JSON.stringify(tagSpecification), + ["key-name"]: keyName, + ["instance-market-options"]: marketOptions, }); - return this.toMachine(instance, options); + const machine = aws.toMachine(instance, { ...options, username, keyPath }); + + await setupUserData(machine, options); + + return machine; }, /** - * @param {GoogleInstance} instance + * @param {AwsInstance} instance * @param {MachineOptions} [options] * @returns {Machine} */ toMachine(instance, options = {}) { - const { id: instanceId, name, zone: zoneUrl, machineType: machineTypeUrl, labels } = instance; - const machineType = machineTypeUrl.split("/").pop(); - const zoneId = zoneUrl.split("/").pop(); - - let os, arch, distro, release; - const { disks = [] } = instance; - for (const { boot, architecture, licenses = [] } of disks) { - if (!boot) { - continue; - } - - if (architecture === "X86_64") { - arch = "x64"; - } else if (architecture === "ARM64") { - arch = "aarch64"; - } - - for (const license of licenses) { - const linuxMatch = /(debian|ubuntu|fedora|rhel)-(\d+)/i.exec(license); - if (linuxMatch) { - os = "linux"; - [, distro, release] = linuxMatch; - } else { - const windowsMatch = /windows-server-(\d+)-dc-core/i.exec(license); - if (windowsMatch) { - os = "windows"; - distro = "windowsserver"; - [, release] = windowsMatch; - } - } - } - } + let { InstanceId, ImageId, InstanceType, Placement, PublicIpAddress } = instance; - let publicIp; - const { networkInterfaces = [] } = instance; - for (const { accessConfigs = [] } of networkInterfaces) { - for (const { type, natIP } of accessConfigs) { - if (type === "ONE_TO_ONE_NAT" && natIP) { - publicIp = natIP; - } + const connect = async () => { + if (!PublicIpAddress) { + await aws.waitInstances("instance-running", InstanceId); + const [{ PublicIpAddress: IpAddress }] = await aws.describeInstances({ + ["instance-id"]: InstanceId, + }); + PublicIpAddress = IpAddress; } - } - let preemptible; - const { scheduling } = instance; - if (scheduling) { - const { provisioningModel, preemptible: isPreemptible } = scheduling; - preemptible = provisioningModel === "SPOT" || isPreemptible; - } - - /** - * @returns {SshOptions} - */ - const connect = () => { - if (!publicIp) { - throw new Error(`Failed to find public IP for instance: ${name}`); - } + const { username, sshKeys } = options; + const identityPaths = sshKeys + ?.filter(({ privatePath }) => existsSync(privatePath)) + ?.map(({ privatePath }) => privatePath); - /** @type {string | undefined} */ - let username; + return { hostname: PublicIpAddress, username, identityPaths }; + }; - const { os, distro } = options; - if (os || distro) { - username = getUsername(distro || os); + const waitForSsh = async () => { + const connectOptions = await connect(); + const { hostname, username, identityPaths } = connectOptions; + + // Try to connect until it succeeds + for (let i = 0; i < 30; i++) { + try { + await spawnSshSafe({ + hostname, + username, + identityPaths, + command: ["true"], + }); + return; + } catch (error) { + if (i === 29) { + throw error; + } + await new Promise(resolve => setTimeout(resolve, 5000)); + } } - - return { hostname: publicIp, username }; }; const spawn = async (command, options) => { - const connectOptions = connect(); + const connectOptions = await connect(); return spawnSsh({ ...connectOptions, command }, options); }; const spawnSafe = async (command, options) => { - const connectOptions = connect(); + const connectOptions = await connect(); return spawnSshSafe({ ...connectOptions, command }, options); }; const rdp = async () => { - const { hostname, username } = connect(); - const rdpUsername = `${username}-rdp`; - const password = await google.resetWindowsPassword(instanceId, rdpUsername, zoneId, { wait: true }); - return { hostname, username: rdpUsername, password }; + const { keyPath } = options; + const { hostname, username } = await connect(); + const password = await aws.getPasswordData(InstanceId, keyPath, { wait: true }); + return { hostname, username, password }; }; const attach = async () => { - const connectOptions = connect(); + const connectOptions = await connect(); await spawnSshSafe({ ...connectOptions }); }; const upload = async (source, destination) => { - const connectOptions = connect(); + const connectOptions = await connect(); await spawnScp({ ...connectOptions, source, destination }); }; const snapshot = async name => { - const stopResult = await this.stopInstance(instanceId, zoneId); - console.log(stopResult); - const image = await this.createImage({ - ["source-disk"]: instanceId, - ["zone"]: zoneId, - ["name"]: name || `${instanceId}-snapshot-${Date.now()}`, + await aws.stopInstances(InstanceId); + await aws.waitInstances("instance-stopped", InstanceId); + const imageId = await aws.createImage({ + ["instance-id"]: InstanceId, + ["name"]: name || `${InstanceId}-snapshot-${Date.now()}`, }); - console.log(image); - return; + await aws.waitImage("image-available", imageId); + return imageId; }; const terminate = async () => { - await google.deleteInstance(instanceId, zoneId); + await aws.terminateInstances(InstanceId); }; return { - cloud: "google", - os, - arch, - distro, - release, - id: instanceId, - imageId: undefined, - name, - instanceType: machineType, - region: zoneId, - publicIp, - preemptible, - labels, + cloud: "aws", + id: InstanceId, + imageId: ImageId, + instanceType: InstanceType, + region: Placement?.AvailabilityZone, + get publicIp() { + return PublicIpAddress; + }, spawn, spawnSafe, - rdp, - attach, upload, + attach, + rdp, snapshot, + waitForSsh, close: terminate, [Symbol.asyncDispose]: terminate, }; }, - - /** - * @param {Record} [labels] - * @returns {Promise} - */ - async getMachines(labels) { - const filters = labels ? this.getFilters({ labels }) : {}; - const instances = await google.listInstances(filters); - return instances.map(instance => this.toMachine(instance)); - }, - - /** - * @param {MachineOptions} options - * @returns {Promise} - */ - async getImage(options) { - const { os, arch, distro, release } = options; - const architecture = arch === "aarch64" ? "ARM64" : "X86_64"; - - let name; - let username; - if (os === "linux") { - if (distro === "debian") { - name = `debian-${release}-*`; - username = "admin"; - } else if (distro === "ubuntu") { - name = `ubuntu-${release.replace(/\./g, "")}-*`; - username = "ubuntu"; - } - } else if (os === "windows" && arch === "x64") { - if (distro === "server") { - name = `windows-server-${release}-dc-core-*`; - username = "administrator"; - } - } - - if (name && username) { - const images = await google.listImages({ name, architecture }); - if (images.length) { - const [image] = images; - const { name, selfLink } = image; - return { - id: selfLink, - name, - username, - }; - } - } - - throw new Error(`Unsupported platform: ${inspect(platform)}`); - }, }; /** @@ -1864,11 +627,15 @@ const google = { * @param {CloudInit} cloudInit * @returns {string} */ -function getUserData(cloudInit) { - const { os } = cloudInit; +export function getUserData(cloudInit) { + const { os, userData } = cloudInit; + + // For Windows, use PowerShell script if (os === "windows") { return getWindowsStartupScript(cloudInit); } + + // For Linux, just set up SSH access return getCloudInit(cloudInit); } @@ -1879,7 +646,7 @@ function getUserData(cloudInit) { function getCloudInit(cloudInit) { const username = cloudInit["username"] || "root"; const password = cloudInit["password"] || crypto.randomUUID(); - const authorizedKeys = JSON.stringify(cloudInit["sshKeys"]?.map(({ publicKey }) => publicKey) || []); + const authorizedKeys = cloudInit["sshKeys"]?.map(({ publicKey }) => publicKey) || []; let sftpPath = "/usr/lib/openssh/sftp-server"; switch (cloudInit["distro"]) { @@ -1902,22 +669,36 @@ function getCloudInit(cloudInit) { // https://cloudinit.readthedocs.io/en/stable/ return `#cloud-config - write_files: - - path: /etc/ssh/sshd_config - content: | - PermitRootLogin yes - PasswordAuthentication no - PubkeyAuthentication yes - UsePAM yes - UseLogin yes - Subsystem sftp ${sftpPath} - chpasswd: - expire: false - list: ${JSON.stringify(users)} - disable_root: false - ssh_pwauth: true - ssh_authorized_keys: ${authorizedKeys} - `; +users: + - name: ${username} + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: +${authorizedKeys.map(key => ` - ${key}`).join("\n")} + +write_files: + - path: /etc/ssh/sshd_config + permissions: '0644' + owner: root:root + content: | + Port 22 + Protocol 2 + HostKey /etc/ssh/ssh_host_rsa_key + HostKey /etc/ssh/ssh_host_ecdsa_key + HostKey /etc/ssh/ssh_host_ed25519_key + SyslogFacility AUTHPRIV + PermitRootLogin yes + AuthorizedKeysFile .ssh/authorized_keys + PasswordAuthentication no + ChallengeResponseAuthentication no + GSSAPIAuthentication yes + GSSAPICleanupCredentials no + UsePAM yes + X11Forwarding yes + PrintMotd no + AcceptEnv LANG LC_* + Subsystem sftp ${sftpPath} +`; } /** @@ -2002,39 +783,11 @@ function getWindowsStartupScript(cloudInit) { `; } -/** - * @param {string} distro - * @returns {string} - */ -function getUsername(distro) { - if (/windows/i.test(distro)) { - return "administrator"; - } - - if (/alpine|centos/i.test(distro)) { - return "root"; - } - - if (/debian/i.test(distro)) { - return "admin"; - } - - if (/ubuntu/i.test(distro)) { - return "ubuntu"; - } - - if (/amazon|amzn|al\d+|rhel/i.test(distro)) { - return "ec2-user"; - } - - throw new Error(`Unsupported distro: ${distro}`); -} - /** * @param {MachineOptions} options * @returns {number} */ -function getDiskSize(options) { +export function getDiskSize(options) { const { os, diskSizeGb } = options; if (diskSizeGb) { @@ -2166,80 +919,6 @@ async function getGithubOrgSshKeys(organization) { * @property {number} [retries] */ -/** - * @param {SshOptions} options - * @param {import("./utils.mjs").SpawnOptions} [spawnOptions] - * @returns {Promise} - */ -async function spawnSsh(options, spawnOptions = {}) { - const { hostname, port, username, identityPaths, password, retries = 10, command: spawnCommand } = options; - - if (!hostname.includes("@")) { - await waitForPort({ - hostname, - port: port || 22, - }); - } - - const logPath = mkdtemp("ssh-", "ssh.log"); - const command = ["ssh", hostname, "-v", "-C", "-E", logPath, "-o", "StrictHostKeyChecking=no"]; - if (!password) { - command.push("-o", "BatchMode=yes"); - } - if (port) { - command.push("-p", port); - } - if (username) { - command.push("-l", username); - } - if (password) { - const sshPass = which("sshpass", { required: true }); - command.unshift(sshPass, "-p", password); - } else if (identityPaths) { - command.push(...identityPaths.flatMap(path => ["-i", path])); - } - const stdio = spawnCommand ? "pipe" : "inherit"; - if (spawnCommand) { - command.push(...spawnCommand); - } - - /** @type {import("./utils.mjs").SpawnResult} */ - let result; - for (let i = 0; i < retries; i++) { - result = await spawn(command, { stdio, ...spawnOptions, throwOnError: undefined }); - - const { exitCode } = result; - if (exitCode !== 255) { - break; - } - - const sshLogs = readFile(logPath, { encoding: "utf-8" }); - if (sshLogs.includes("Authenticated")) { - break; - } - - await new Promise(resolve => setTimeout(resolve, (i + 1) * 15000)); - } - - if (spawnOptions?.throwOnError) { - const { error } = result; - if (error) { - throw error; - } - } - - return result; -} - -/** - * @param {SshOptions} options - * @param {import("./utils.mjs").SpawnOptions} [spawnOptions] - * @returns {Promise} - */ -async function spawnSshSafe(options, spawnOptions = {}) { - return spawnSsh(options, { throwOnError: true, ...spawnOptions }); -} - /** * @typedef ScpOptions * @property {string} hostname @@ -2400,6 +1079,7 @@ function getCloud(name) { * @property {string} [publicIp] * @property {boolean} [preemptible] * @property {Record} tags + * @property {string} [userData] * @property {(command: string[], options?: import("./utils.mjs").SpawnOptions) => Promise} spawn * @property {(command: string[], options?: import("./utils.mjs").SpawnOptions) => Promise} spawnSafe * @property {(source: string, destination: string) => Promise} upload @@ -2429,6 +1109,7 @@ function getCloud(name) { * @property {boolean} [bootstrap] * @property {boolean} [ci] * @property {boolean} [rdp] + * @property {string} [userData] * @property {SshKey[]} sshKeys */ @@ -2466,11 +1147,14 @@ async function main() { "ci": { type: "boolean" }, "rdp": { type: "boolean" }, "vnc": { type: "boolean" }, + "feature": { type: "string", multiple: true }, + "user-data": { type: "string" }, "authorized-user": { type: "string", multiple: true }, "authorized-org": { type: "string", multiple: true }, "no-bootstrap": { type: "boolean" }, "buildkite-token": { type: "string" }, "tailscale-authkey": { type: "string" }, + "docker": { type: "boolean" }, }, }); @@ -2513,16 +1197,38 @@ async function main() { detached: !!args["detached"], bootstrap: args["no-bootstrap"] !== true, ci: !!args["ci"], + features: args["feature"], rdp: !!args["rdp"] || !!args["vnc"], sshKeys, + userData: args["user-data"] ? readFile(args["user-data"]) : undefined, }; - const { detached, bootstrap, ci, os, arch, distro, release } = options; - const name = distro ? `${os}-${arch}-${distro}-${release}` : `${os}-${arch}-${release}`; + let { detached, bootstrap, ci, os, arch, distro, release, features } = options; + + let name = `${os}-${arch}-${(release || "").replace(/\./g, "")}`; + + if (distro) { + name += `-${distro}`; + } + + if (distro === "alpine") { + name += `-musl`; + } + + if (features?.length) { + name += `-with-${features.join("-")}`; + } - let bootstrapPath, agentPath; + let bootstrapPath, agentPath, dockerfilePath; if (bootstrap) { - bootstrapPath = resolve(import.meta.dirname, os === "windows" ? "bootstrap.ps1" : "bootstrap.sh"); + bootstrapPath = resolve( + import.meta.dirname, + os === "windows" + ? "bootstrap.ps1" + : features?.includes("docker") + ? "../.buildkite/Dockerfile-bootstrap.sh" + : "bootstrap.sh", + ); if (!existsSync(bootstrapPath)) { throw new Error(`Script not found: ${bootstrapPath}`); } @@ -2536,6 +1242,14 @@ async function main() { agentPath = join(tmpPath, "agent.mjs"); await spawnSafe($`${npx} esbuild ${entryPath} --bundle --platform=node --format=esm --outfile=${agentPath}`); } + + if (features?.includes("docker")) { + dockerfilePath = resolve(import.meta.dirname, "../.buildkite/Dockerfile"); + + if (!existsSync(dockerfilePath)) { + throw new Error(`Dockerfile not found: ${dockerfilePath}`); + } + } } /** @type {Machine} */ @@ -2625,12 +1339,31 @@ async function main() { await machine.spawnSafe(["powershell", remotePath, ...args], { stdio: "inherit" }); }); } else { - const remotePath = "/tmp/bootstrap.sh"; - const args = ci ? ["--ci"] : []; - await startGroup("Running bootstrap...", async () => { - await machine.upload(bootstrapPath, remotePath); - await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" }); - }); + if (!features?.includes("docker")) { + const remotePath = "/tmp/bootstrap.sh"; + const args = ci ? ["--ci"] : []; + for (const feature of features || []) { + args.push(`--${feature}`); + } + await startGroup("Running bootstrap...", async () => { + await machine.upload(bootstrapPath, remotePath); + await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" }); + }); + } else if (dockerfilePath) { + const remotePath = "/tmp/bootstrap.sh"; + + await startGroup("Running Docker bootstrap...", async () => { + await machine.upload(bootstrapPath, remotePath); + console.log("Uploaded bootstrap.sh"); + await machine.upload(dockerfilePath, "/tmp/Dockerfile"); + console.log("Uploaded Dockerfile"); + await machine.upload(agentPath, "/tmp/agent.mjs"); + console.log("Uploaded agent.mjs"); + agentPath = ""; + bootstrapPath = ""; + await machine.spawnSafe(["sudo", "bash", remotePath], { stdio: "inherit", cwd: "/tmp" }); + }); + } } } @@ -2639,7 +1372,7 @@ async function main() { const remotePath = "C:\\buildkite-agent\\agent.mjs"; await startGroup("Installing agent...", async () => { await machine.upload(agentPath, remotePath); - if (cloud.name === "docker") { + if (cloud.name === "docker" || features?.includes("docker")) { return; } await machine.spawnSafe(["node", remotePath, "install"], { stdio: "inherit" }); diff --git a/scripts/orbstack.mjs b/scripts/orbstack.mjs new file mode 100644 index 00000000000000..905c3a40a59142 --- /dev/null +++ b/scripts/orbstack.mjs @@ -0,0 +1,195 @@ +import { inspect } from "node:util"; +import { $, mkdtemp, rm, spawnSafe, writeFile, getUsernameForDistro, spawnSshSafe, setupUserData } from "./utils.mjs"; +import { getUserData } from "./machine.mjs"; + +/** + * @link https://docs.orbstack.dev/ + */ +export const orbstack = { + get name() { + return "orbstack"; + }, + + /** + * @typedef {Object} OrbstackImage + * @property {string} distro + * @property {string} version + * @property {string} arch + */ + + /** + * @param {Platform} platform + * @returns {OrbstackImage} + */ + getImage(platform) { + const { os, arch, distro, release } = platform; + if (os !== "linux" || !/^debian|ubuntu|alpine|fedora|centos$/.test(distro)) { + throw new Error(`Unsupported platform: ${inspect(platform)}`); + } + + return { + distro, + version: release, + arch: arch === "aarch64" ? "arm64" : "amd64", + }; + }, + + /** + * @typedef {Object} OrbstackVm + * @property {string} id + * @property {string} name + * @property {"running"} state + * @property {OrbstackImage} image + * @property {OrbstackConfig} config + */ + + /** + * @typedef {Object} OrbstackConfig + * @property {string} default_username + * @property {boolean} isolated + */ + + /** + * @typedef {Object} OrbstackVmOptions + * @property {string} [name] + * @property {OrbstackImage} image + * @property {string} [username] + * @property {string} [password] + * @property {string} [userData] + */ + + /** + * @param {OrbstackVmOptions} options + * @returns {Promise} + */ + async createVm(options) { + const { name, image, username, password, userData } = options; + const { distro, version, arch } = image; + const uniqueId = name || `linux-${distro}-${version}-${arch}-${Math.random().toString(36).slice(2, 11)}`; + + const args = [`--arch=${arch}`, `${distro}:${version}`, uniqueId]; + if (username) { + args.push(`--user=${username}`); + } + if (password) { + args.push(`--set-password=${password}`); + } + + let userDataPath; + if (userData) { + userDataPath = mkdtemp("orbstack-user-data-", "user-data.txt"); + console.log("User data path:", userData); + writeFile(userDataPath, userData); + args.push(`--user-data=${userDataPath}`); + } + + try { + await spawnSafe($`orbctl create ${args}`); + } finally { + if (userDataPath) { + rm(userDataPath); + } + } + + return this.inspectVm(uniqueId); + }, + + /** + * @param {string} name + */ + async deleteVm(name) { + await spawnSafe($`orbctl delete ${name}`, { + throwOnError: error => !/machine not found/i.test(inspect(error)), + }); + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async inspectVm(name) { + const { exitCode, stdout } = await spawnSafe($`orbctl info ${name} --format=json`, { + throwOnError: error => !/machine not found/i.test(inspect(error)), + }); + if (exitCode === 0) { + return JSON.parse(stdout); + } + }, + + /** + * @returns {Promise} + */ + async listVms() { + const { stdout } = await spawnSafe($`orbctl list --format=json`); + return JSON.parse(stdout); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { distro } = options; + const username = getUsernameForDistro(distro); + const userData = getUserData({ ...options, username }); + + const image = this.getImage(options); + const vm = await this.createVm({ + image, + username, + userData, + }); + + const machine = this.toMachine(vm, options); + + await setupUserData(machine, options); + + return machine; + }, + + /** + * @param {OrbstackVm} vm + * @returns {Machine} + */ + toMachine(vm) { + const { id, name, config } = vm; + + const { default_username: username } = config; + const connectOptions = { + username, + hostname: `${name}@orb`, + }; + + const exec = async (command, options) => { + return spawnSsh({ ...connectOptions, command }, options); + }; + + const execSafe = async (command, options) => { + return spawnSshSafe({ ...connectOptions, command }, options); + }; + + const attach = async () => { + await spawnSshSafe({ ...connectOptions }); + }; + + const upload = async (source, destination) => { + await spawnSafe(["orbctl", "push", `--machine=${name}`, source, destination]); + }; + + const close = async () => { + await this.deleteVm(name); + }; + + return { + cloud: "orbstack", + id, + name, + spawn: exec, + spawnSafe: execSafe, + upload, + attach, + close, + [Symbol.asyncDispose]: close, + }; + }, +}; diff --git a/scripts/tart.mjs b/scripts/tart.mjs new file mode 100644 index 00000000000000..e8701b2a48ce6f --- /dev/null +++ b/scripts/tart.mjs @@ -0,0 +1,283 @@ +import { inspect } from "node:util"; +import { isPrivileged, spawnSafe, which } from "./utils.mjs"; + +/** + * @link https://tart.run/ + * @link https://github.com/cirruslabs/tart + */ +export const tart = { + get name() { + return "tart"; + }, + + /** + * @param {string[]} args + * @param {import("./utils.mjs").SpawnOptions} options + * @returns {Promise} + */ + async spawn(args, options) { + const tart = which("tart", { required: true }); + const { json } = options || {}; + const command = json ? [tart, ...args, "--format=json"] : [tart, ...args]; + + const { stdout } = await spawnSafe(command, options); + if (!json) { + return stdout; + } + + try { + return JSON.parse(stdout); + } catch { + return; + } + }, + + /** + * @typedef {"sequoia" | "sonoma" | "ventura" | "monterey"} TartDistro + * @typedef {`ghcr.io/cirruslabs/macos-${TartDistro}-xcode`} TartImage + * @link https://github.com/orgs/cirruslabs/packages?repo_name=macos-image-templates + */ + + /** + * @param {Platform} platform + * @returns {TartImage} + */ + getImage(platform) { + const { os, arch, release } = platform; + if (os !== "darwin" || arch !== "aarch64") { + throw new Error(`Unsupported platform: ${inspect(platform)}`); + } + const distros = { + "15": "sequoia", + "14": "sonoma", + "13": "ventura", + "12": "monterey", + }; + const distro = distros[release]; + if (!distro) { + throw new Error(`Unsupported macOS release: ${distro}`); + } + return `ghcr.io/cirruslabs/macos-${distro}-xcode`; + }, + + /** + * @typedef {Object} TartVm + * @property {string} Name + * @property {"running" | "stopped"} State + * @property {"local"} Source + * @property {number} Size + * @property {number} Disk + * @property {number} [CPU] + * @property {number} [Memory] + */ + + /** + * @returns {Promise} + */ + async listVms() { + return this.spawn(["list"], { json: true }); + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async getVm(name) { + const result = await this.spawn(["get", name], { + json: true, + throwOnError: error => !/does not exist/i.test(inspect(error)), + }); + return { + Name: name, + ...result, + }; + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async stopVm(name) { + await this.spawn(["stop", name, "--timeout=0"], { + throwOnError: error => !/does not exist|is not running/i.test(inspect(error)), + }); + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async deleteVm(name) { + await this.stopVm(name); + await this.spawn(["delete", name], { + throwOnError: error => !/does not exist/i.test(inspect(error)), + }); + }, + + /** + * @param {string} name + * @param {TartImage} image + * @returns {Promise} + */ + async cloneVm(name, image) { + const localName = image.split("/").pop(); + const localVm = await this.getVm(localName); + if (localVm) { + const { Name } = localVm; + await this.spawn(["clone", Name, name]); + return; + } + + console.log(`Cloning macOS image: ${image} (this will take a long time)`); + await this.spawn(["clone", image, localName]); + await this.spawn(["clone", localName, name]); + }, + + /** + * @typedef {Object} TartMount + * @property {boolean} [readOnly] + * @property {string} source + * @property {string} destination + */ + + /** + * @typedef {Object} TartVmOptions + * @property {number} [cpuCount] + * @property {number} [memoryGb] + * @property {number} [diskSizeGb] + * @property {boolean} [no-graphics] + * @property {boolean} [no-audio] + * @property {boolean} [no-clipboard] + * @property {boolean} [recovery] + * @property {boolean} [vnc] + * @property {boolean} [vnc-experimental] + * @property {boolean} [net-softnet] + * @property {TartMount[]} [dir] + */ + + /** + * @param {string} name + * @param {TartVmOptions} options + * @returns {Promise} + */ + async runVm(name, options = {}) { + const { cpuCount, memoryGb, diskSizeGb, dir, ...vmOptions } = options; + + const setArgs = ["--random-mac", "--random-serial"]; + if (cpuCount) { + setArgs.push(`--cpu=${cpuCount}`); + } + if (memoryGb) { + setArgs.push(`--memory=${memoryGb}`); + } + if (diskSizeGb) { + setArgs.push(`--disk-size=${diskSizeGb}`); + } + await this.spawn(["set", name, ...setArgs]); + + const args = Object.entries(vmOptions) + .filter(([, value]) => value !== undefined) + .flatMap(([key, value]) => (typeof value === "boolean" ? (value ? [`--${key}`] : []) : [`--${key}=${value}`])); + if (dir?.length) { + args.push( + ...dir.map(({ source, destination, readOnly }) => `--dir=${source}:${destination}${readOnly ? ":ro" : ""}`), + ); + } + + // This command is blocking, so it needs to be detached and not awaited + this.spawn(["run", name, ...args], { detached: true }); + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async getVmIp(name) { + const stdout = await this.spawn(["ip", name], { + retryOnError: error => /no IP address found/i.test(inspect(error)), + throwOnError: error => !/does not exist/i.test(inspect(error)), + }); + return stdout?.trim(); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { name, imageName, cpuCount, memoryGb, diskSizeGb, rdp } = options; + + const image = imageName || this.getImage(options); + const machineId = name || `i-${Math.random().toString(36).slice(2, 11)}`; + await this.cloneVm(machineId, image); + + await this.runVm(machineId, { + cpuCount, + memoryGb, + diskSizeGb, + "net-softnet": isPrivileged(), + "no-audio": true, + "no-clipboard": true, + "no-graphics": true, + "vnc-experimental": rdp, + }); + + return this.toMachine(machineId); + }, + + /** + * @param {string} name + * @returns {Machine} + */ + toMachine(name) { + const connect = async () => { + const hostname = await this.getVmIp(name); + return { + hostname, + // hardcoded by base images + username: "admin", + password: "admin", + }; + }; + + const exec = async (command, options) => { + const connectOptions = await connect(); + return spawnSsh({ ...connectOptions, command }, options); + }; + + const execSafe = async (command, options) => { + const connectOptions = await connect(); + return spawnSshSafe({ ...connectOptions, command }, options); + }; + + const attach = async () => { + const connectOptions = await connect(); + await spawnSshSafe({ ...connectOptions }); + }; + + const upload = async (source, destination) => { + const connectOptions = await connect(); + await spawnScp({ ...connectOptions, source, destination }); + }; + + const rdp = async () => { + const connectOptions = await connect(); + await spawnRdp({ ...connectOptions }); + }; + + const close = async () => { + await this.deleteVm(name); + }; + + return { + cloud: "tart", + id: name, + spawn: exec, + spawnSafe: execSafe, + attach, + upload, + close, + [Symbol.asyncDispose]: close, + }; + }, +}; diff --git a/scripts/utils.mjs b/scripts/utils.mjs index b391dc2c2d9856..18aa8c51c83a27 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -243,7 +243,7 @@ export async function spawn(command, options = {}) { cwd: options["cwd"] ?? process.cwd(), timeout: options["timeout"] ?? undefined, env: options["env"] ?? undefined, - stdio: [stdin ? "pipe" : "ignore", "pipe", "pipe"], + stdio: stdin === "inherit" ? "inherit" : [stdin ? "pipe" : "ignore", "pipe", "pipe"], ...options, }; @@ -355,7 +355,7 @@ export function spawnSync(command, options = {}) { cwd: options["cwd"] ?? process.cwd(), timeout: options["timeout"] ?? undefined, env: options["env"] ?? undefined, - stdio: [typeof stdin === "undefined" ? "ignore" : "pipe", "pipe", "pipe"], + stdio: stdin === "inherit" ? "inherit" : [typeof stdin === "undefined" ? "ignore" : "pipe", "pipe", "pipe"], input: stdin, ...options, }; @@ -379,8 +379,8 @@ export function spawnSync(command, options = {}) { } else { exitCode = status ?? 1; signalCode = signal || undefined; - stdout = stdoutBuffer?.toString(); - stderr = stderrBuffer?.toString(); + stdout = stdoutBuffer?.toString?.() ?? ""; + stderr = stderrBuffer?.toString?.() ?? ""; } if (exitCode !== 0 && isWindows) { @@ -1861,6 +1861,34 @@ export function getUsername() { return username; } +/** + * @param {string} distro + * @returns {string} + */ +export function getUsernameForDistro(distro) { + if (/windows/i.test(distro)) { + return "administrator"; + } + + if (/alpine|centos/i.test(distro)) { + return "root"; + } + + if (/debian/i.test(distro)) { + return "admin"; + } + + if (/ubuntu/i.test(distro)) { + return "ubuntu"; + } + + if (/amazon|amzn|al\d+|rhel/i.test(distro)) { + return "ec2-user"; + } + + throw new Error(`Unsupported distro: ${distro}`); +} + /** * @typedef {object} User * @property {string} username @@ -2723,7 +2751,7 @@ export function getLoggedInUserCountOrDetails() { return 0; } - let message = users.length + " currently logged in users:"; + let message = `${users.length} currently logged in users:`; for (const user of users) { message += `\n- ${user.username} on ${user.terminal} since ${user.datetime}${user.ip ? ` from ${user.ip}` : ""}`; @@ -2743,6 +2771,7 @@ const emojiMap = { alpine: ["🐧", "alpine"], aws: ["☁️", "aws"], amazonlinux: ["🐧", "aws"], + nix: ["🐧", "nix"], windows: ["🪟", "windows"], true: ["✅", "white_check_mark"], false: ["❌", "x"], @@ -2772,3 +2801,108 @@ export function getBuildkiteEmoji(emoji) { const [, name] = emojiMap[emoji] || []; return name ? `:${name}:` : ""; } + +/** + * @param {SshOptions} options + * @param {import("./utils.mjs").SpawnOptions} [spawnOptions] + * @returns {Promise} + */ +export async function spawnSshSafe(options, spawnOptions = {}) { + return spawnSsh(options, { throwOnError: true, ...spawnOptions }); +} + +/** + * @param {SshOptions} options + * @param {import("./utils.mjs").SpawnOptions} [spawnOptions] + * @returns {Promise} + */ +export async function spawnSsh(options, spawnOptions = {}) { + const { hostname, port, username, identityPaths, password, retries = 10, command: spawnCommand } = options; + + if (!hostname.includes("@")) { + await waitForPort({ + hostname, + port: port || 22, + }); + } + + const logPath = mkdtemp("ssh-", "ssh.log"); + const command = ["ssh", hostname, "-v", "-C", "-E", logPath, "-o", "StrictHostKeyChecking=no"]; + if (!password) { + command.push("-o", "BatchMode=yes"); + } + if (port) { + command.push("-p", port); + } + if (username) { + command.push("-l", username); + } + if (password) { + const sshPass = which("sshpass", { required: true }); + command.unshift(sshPass, "-p", password); + } else if (identityPaths) { + command.push(...identityPaths.flatMap(path => ["-i", path])); + } + const stdio = spawnCommand ? "pipe" : "inherit"; + if (spawnCommand) { + command.push(...spawnCommand); + } + + /** @type {import("./utils.mjs").SpawnResult} */ + let result; + for (let i = 0; i < retries; i++) { + result = await spawn(command, { stdio, ...spawnOptions, throwOnError: undefined }); + + const { exitCode } = result; + if (exitCode !== 255) { + break; + } + + const sshLogs = readFile(logPath, { encoding: "utf-8" }); + if (sshLogs.includes("Authenticated")) { + break; + } + + await new Promise(resolve => setTimeout(resolve, (i + 1) * 15000)); + } + + if (spawnOptions?.throwOnError) { + const { error } = result; + if (error) { + throw error; + } + } + + return result; +} + +/** + * @param {MachineOptions} options + * @returns {Promise} + */ +export async function setupUserData(machine, options) { + const { os, userData } = options; + if (!userData) { + return; + } + + // Write user data to a temporary file + const tmpFile = mkdtemp("user-data-", os === "windows" ? "setup.ps1" : "setup.sh"); + await writeFile(tmpFile, userData); + + try { + // Upload the script + const remotePath = os === "windows" ? "C:\\Windows\\Temp\\setup.ps1" : "/tmp/setup.sh"; + await machine.upload(tmpFile, remotePath); + + // Execute the script + if (os === "windows") { + await machine.spawnSafe(["powershell", remotePath], { stdio: "inherit" }); + } else { + await machine.spawnSafe(["bash", remotePath], { stdio: "inherit" }); + } + } finally { + // Clean up the temporary file + rm(tmpFile); + } +} diff --git a/src/bun.js/bindings/Base64Helpers.cpp b/src/bun.js/bindings/Base64Helpers.cpp index 1e13730f47b568..b502ef32cf4638 100644 --- a/src/bun.js/bindings/Base64Helpers.cpp +++ b/src/bun.js/bindings/Base64Helpers.cpp @@ -18,13 +18,13 @@ ExceptionOr atob(const String& encodedString) if (!encodedString.is8Bit()) { const auto span = encodedString.span16(); size_t expected_length = simdutf::latin1_length_from_utf16(span.size()); - LChar* ptr; + std::span ptr; WTF::String convertedString = WTF::String::createUninitialized(expected_length, ptr); if (UNLIKELY(convertedString.isNull())) { return WebCore::Exception { OutOfMemoryError }; } - auto result = simdutf::convert_utf16le_to_latin1_with_errors(span.data(), span.size(), reinterpret_cast(ptr)); + auto result = simdutf::convert_utf16le_to_latin1_with_errors(span.data(), span.size(), reinterpret_cast(ptr.data())); if (result.error) { return WebCore::Exception { InvalidCharacterError }; @@ -34,12 +34,12 @@ ExceptionOr atob(const String& encodedString) const auto span = encodedString.span8(); size_t result_length = simdutf::maximal_binary_length_from_base64(reinterpret_cast(span.data()), encodedString.length()); - LChar* ptr; + std::span ptr; WTF::String outString = WTF::String::createUninitialized(result_length, ptr); if (UNLIKELY(outString.isNull())) { return WebCore::Exception { OutOfMemoryError }; } - auto result = simdutf::base64_to_binary(reinterpret_cast(span.data()), span.size(), reinterpret_cast(ptr), simdutf::base64_default); + auto result = simdutf::base64_to_binary(reinterpret_cast(span.data()), span.size(), reinterpret_cast(ptr.data()), simdutf::base64_default); if (result.error != simdutf::error_code::SUCCESS) { return WebCore::Exception { InvalidCharacterError }; } diff --git a/src/bun.js/bindings/BunString.cpp b/src/bun.js/bindings/BunString.cpp index 4d9ce102f3fb02..17200b35c30552 100644 --- a/src/bun.js/bindings/BunString.cpp +++ b/src/bun.js/bindings/BunString.cpp @@ -226,7 +226,7 @@ extern "C" JSC::EncodedJSValue BunString__toJS(JSC::JSGlobalObject* globalObject extern "C" BunString BunString__fromUTF16Unitialized(size_t length) { ASSERT(length > 0); - UChar* ptr; + std::span ptr; auto impl = WTF::StringImpl::tryCreateUninitialized(length, ptr); if (UNLIKELY(!impl)) { return { .tag = BunStringTag::Dead }; @@ -237,7 +237,7 @@ extern "C" BunString BunString__fromUTF16Unitialized(size_t length) extern "C" BunString BunString__fromLatin1Unitialized(size_t length) { ASSERT(length > 0); - LChar* ptr; + std::span ptr; auto impl = WTF::StringImpl::tryCreateUninitialized(length, ptr); if (UNLIKELY(!impl)) { return { .tag = BunStringTag::Dead }; @@ -250,12 +250,12 @@ extern "C" BunString BunString__fromUTF8(const char* bytes, size_t length) ASSERT(length > 0); if (simdutf::validate_utf8(bytes, length)) { size_t u16Length = simdutf::utf16_length_from_utf8(bytes, length); - UChar* ptr; + std::span ptr; auto impl = WTF::StringImpl::tryCreateUninitialized(static_cast(u16Length), ptr); if (UNLIKELY(!impl)) { return { .tag = BunStringTag::Dead }; } - RELEASE_ASSERT(simdutf::convert_utf8_to_utf16(bytes, length, ptr) == u16Length); + RELEASE_ASSERT(simdutf::convert_utf8_to_utf16(bytes, length, ptr.data()) == u16Length); impl->ref(); return { BunStringTag::WTFStringImpl, { .wtf = impl.leakRef() } }; } @@ -271,12 +271,12 @@ extern "C" BunString BunString__fromUTF8(const char* bytes, size_t length) extern "C" BunString BunString__fromLatin1(const char* bytes, size_t length) { ASSERT(length > 0); - LChar* ptr; + std::span ptr; auto impl = WTF::StringImpl::tryCreateUninitialized(length, ptr); if (UNLIKELY(!impl)) { return { .tag = BunStringTag::Dead }; } - memcpy(ptr, bytes, length); + memcpy(ptr.data(), bytes, length); return { BunStringTag::WTFStringImpl, { .wtf = impl.leakRef() } }; } @@ -286,13 +286,13 @@ extern "C" BunString BunString__fromUTF16ToLatin1(const char16_t* bytes, size_t ASSERT(length > 0); ASSERT_WITH_MESSAGE(simdutf::validate_utf16le(bytes, length), "This function only accepts ascii UTF16 strings"); size_t outLength = simdutf::latin1_length_from_utf16(length); - LChar* ptr = nullptr; + std::span ptr; auto impl = WTF::StringImpl::tryCreateUninitialized(outLength, ptr); if (UNLIKELY(!impl)) { return { BunStringTag::Dead }; } - size_t latin1_length = simdutf::convert_valid_utf16le_to_latin1(bytes, length, reinterpret_cast(ptr)); + size_t latin1_length = simdutf::convert_valid_utf16le_to_latin1(bytes, length, reinterpret_cast(ptr.data())); ASSERT_WITH_MESSAGE(latin1_length == outLength, "Failed to convert UTF16 to Latin1"); return { BunStringTag::WTFStringImpl, { .wtf = impl.leakRef() } }; } @@ -300,12 +300,12 @@ extern "C" BunString BunString__fromUTF16ToLatin1(const char16_t* bytes, size_t extern "C" BunString BunString__fromUTF16(const char16_t* bytes, size_t length) { ASSERT(length > 0); - UChar* ptr; + std::span ptr; auto impl = WTF::StringImpl::tryCreateUninitialized(length, ptr); if (UNLIKELY(!impl)) { return { .tag = BunStringTag::Dead }; } - memcpy(ptr, bytes, length * sizeof(char16_t)); + memcpy(ptr.data(), bytes, length * sizeof(char16_t)); return { BunStringTag::WTFStringImpl, { .wtf = impl.leakRef() } }; } diff --git a/src/bun.js/bindings/JSBuffer.cpp b/src/bun.js/bindings/JSBuffer.cpp index d29fbbc0957660..fb0c0b5fe9dfb9 100644 --- a/src/bun.js/bindings/JSBuffer.cpp +++ b/src/bun.js/bindings/JSBuffer.cpp @@ -1547,21 +1547,21 @@ static inline JSC::EncodedJSValue jsBufferToString(JSC::VM& vm, JSC::JSGlobalObj switch (encoding) { case WebCore::BufferEncodingType::latin1: { - LChar* data = nullptr; + std::span data; auto str = String::createUninitialized(length, data); - memcpy(data, reinterpret_cast(castedThis->vector()) + offset, length); + memcpy(data.data(), reinterpret_cast(castedThis->vector()) + offset, length); return JSC::JSValue::encode(JSC::jsString(vm, WTFMove(str))); } case WebCore::BufferEncodingType::ucs2: case WebCore::BufferEncodingType::utf16le: { - UChar* data = nullptr; + std::span data; size_t u16length = length / 2; if (u16length == 0) { return JSC::JSValue::encode(JSC::jsEmptyString(vm)); } else { auto str = String::createUninitialized(u16length, data); - memcpy(reinterpret_cast(data), reinterpret_cast(castedThis->vector()) + offset, u16length * 2); + memcpy(reinterpret_cast(data.data()), reinterpret_cast(castedThis->vector()) + offset, u16length * 2); return JSC::JSValue::encode(JSC::jsString(vm, str)); } @@ -1571,9 +1571,9 @@ static inline JSC::EncodedJSValue jsBufferToString(JSC::VM& vm, JSC::JSGlobalObj case WebCore::BufferEncodingType::ascii: { // ascii: we always know the length // so we might as well allocate upfront - LChar* data = nullptr; + std::span data; auto str = String::createUninitialized(length, data); - Bun__encoding__writeLatin1(reinterpret_cast(castedThis->vector()) + offset, length, data, length, static_cast(encoding)); + Bun__encoding__writeLatin1(reinterpret_cast(castedThis->vector()) + offset, length, data.data(), length, static_cast(encoding)); return JSC::JSValue::encode(JSC::jsString(vm, WTFMove(str))); } diff --git a/src/bun.js/bindings/JSBufferEncodingType.cpp b/src/bun.js/bindings/JSBufferEncodingType.cpp index e97504fb707072..903ba949518e6c 100644 --- a/src/bun.js/bindings/JSBufferEncodingType.cpp +++ b/src/bun.js/bindings/JSBufferEncodingType.cpp @@ -31,7 +31,8 @@ using namespace JSC; String convertEnumerationToString(BufferEncodingType enumerationValue) { - static const NeverDestroyed values[] = { + + static const std::array, 8> values = { MAKE_STATIC_STRING_IMPL("utf8"), MAKE_STATIC_STRING_IMPL("ucs2"), MAKE_STATIC_STRING_IMPL("utf16le"), @@ -56,7 +57,7 @@ template<> std::optional parseEnumerationgetString(&lexicalGlobalObject)); + return parseEnumeration2(lexicalGlobalObject, arg.toWTFString(&lexicalGlobalObject)); } std::optional parseEnumeration2(JSGlobalObject& lexicalGlobalObject, WTF::String encoding) diff --git a/src/bun.js/bindings/JSBundlerPlugin.cpp b/src/bun.js/bindings/JSBundlerPlugin.cpp index 80b25b4373f44d..fea22a07a47b9d 100644 --- a/src/bun.js/bindings/JSBundlerPlugin.cpp +++ b/src/bun.js/bindings/JSBundlerPlugin.cpp @@ -39,8 +39,8 @@ namespace Bun { extern "C" int OnBeforeParsePlugin__isDone(void* context); extern "C" void OnBeforeParseResult__reset(OnBeforeParseResult* result); -#define WRAP_BUNDLER_PLUGIN(argName) jsDoubleNumber(bitwise_cast(reinterpret_cast(argName))) -#define UNWRAP_BUNDLER_PLUGIN(callFrame) reinterpret_cast(bitwise_cast(callFrame->argument(0).asDouble())) +#define WRAP_BUNDLER_PLUGIN(argName) jsDoubleNumber(std::bit_cast(reinterpret_cast(argName))) +#define UNWRAP_BUNDLER_PLUGIN(callFrame) reinterpret_cast(std::bit_cast(callFrame->argument(0).asDouble())) /// These are callbacks defined in Zig and to be run after their associated JS version is run extern "C" void JSBundlerPlugin__addError(void*, void*, JSC::EncodedJSValue, JSC::EncodedJSValue); diff --git a/src/bun.js/bindings/JSFFIFunction.cpp b/src/bun.js/bindings/JSFFIFunction.cpp index 9533ed444ee14e..b58d4ed14487d7 100644 --- a/src/bun.js/bindings/JSFFIFunction.cpp +++ b/src/bun.js/bindings/JSFFIFunction.cpp @@ -121,7 +121,7 @@ extern "C" JSC::EncodedJSValue Bun__CreateFFIFunctionValue(Zig::GlobalObject* gl // We should only expose the "ptr" field when it's a JSCallback for bun:ffi. // Not for internal usages of this function type. // We should also consider a separate JSFunction type for our usage to not have this branch in the first place... - function->putDirect(vm, JSC::Identifier::fromString(vm, String("ptr"_s)), JSC::jsNumber(bitwise_cast(functionPointer)), JSC::PropertyAttribute::ReadOnly | 0); + function->putDirect(vm, JSC::Identifier::fromString(vm, String("ptr"_s)), JSC::jsNumber(std::bit_cast(functionPointer)), JSC::PropertyAttribute::ReadOnly | 0); function->symbolFromDynamicLibrary = symbolFromDynamicLibrary; return JSC::JSValue::encode(function); } diff --git a/src/bun.js/bindings/NodeHTTP.cpp b/src/bun.js/bindings/NodeHTTP.cpp index 2ff078e67d62ed..1bef336c82c3cf 100644 --- a/src/bun.js/bindings/NodeHTTP.cpp +++ b/src/bun.js/bindings/NodeHTTP.cpp @@ -212,10 +212,10 @@ static EncodedJSValue assignHeadersFromUWebSockets(uWS::HttpRequest* request, JS for (auto it = request->begin(); it != request->end(); ++it) { auto pair = *it; StringView nameView = StringView(std::span { reinterpret_cast(pair.first.data()), pair.first.length() }); - LChar* data = nullptr; + std::span data; auto value = String::createUninitialized(pair.second.length(), data); if (pair.second.length() > 0) - memcpy(data, pair.second.data(), pair.second.length()); + memcpy(data.data(), pair.second.data(), pair.second.length()); HTTPHeaderName name; WTF::String nameString; diff --git a/src/bun.js/bindings/ZigGlobalObject.cpp b/src/bun.js/bindings/ZigGlobalObject.cpp index daf8b8c194bc63..8315670db234ee 100644 --- a/src/bun.js/bindings/ZigGlobalObject.cpp +++ b/src/bun.js/bindings/ZigGlobalObject.cpp @@ -1460,8 +1460,8 @@ JSC_DEFINE_HOST_FUNCTION(functionNativeMicrotaskTrampoline, double cellPtr = callFrame->uncheckedArgument(0).asNumber(); double callbackPtr = callFrame->uncheckedArgument(1).asNumber(); - void* cell = reinterpret_cast(bitwise_cast(cellPtr)); - auto* callback = reinterpret_cast(bitwise_cast(callbackPtr)); + void* cell = reinterpret_cast(__bit_cast(cellPtr)); + auto* callback = reinterpret_cast(__bit_cast(callbackPtr)); callback(cell); return JSValue::encode(jsUndefined()); } @@ -1708,14 +1708,14 @@ JSC_DEFINE_HOST_FUNCTION(functionBTOA, // That means even though this looks like the wrong thing to do, // we should be converting to latin1, not utf8. if (!encodedString.is8Bit()) { - LChar* ptr; + std::span ptr; unsigned length = encodedString.length(); auto dest = WTF::String::createUninitialized(length, ptr); if (UNLIKELY(dest.isNull())) { throwOutOfMemoryError(globalObject, throwScope); return {}; } - WTF::StringImpl::copyCharacters(ptr, encodedString.span16()); + WTF::StringImpl::copyCharacters(ptr.data(), encodedString.span16()); encodedString = WTFMove(dest); } @@ -4043,7 +4043,7 @@ extern "C" void JSC__JSGlobalObject__queueMicrotaskCallback(Zig::GlobalObject* g JSFunction* function = globalObject->nativeMicrotaskTrampoline(); // Do not use JSCell* here because the GC will try to visit it. - globalObject->queueMicrotask(function, JSValue(bitwise_cast(reinterpret_cast(ptr))), JSValue(bitwise_cast(reinterpret_cast(callback))), jsUndefined(), jsUndefined()); + globalObject->queueMicrotask(function, JSValue(__bit_cast(reinterpret_cast(ptr))), JSValue(__bit_cast(reinterpret_cast(callback))), jsUndefined(), jsUndefined()); } JSC::Identifier GlobalObject::moduleLoaderResolve(JSGlobalObject* jsGlobalObject, diff --git a/src/bun.js/bindings/bindings.cpp b/src/bun.js/bindings/bindings.cpp index 8c5369b15aa1bb..1d6d42150314a3 100644 --- a/src/bun.js/bindings/bindings.cpp +++ b/src/bun.js/bindings/bindings.cpp @@ -1675,9 +1675,9 @@ WebCore::FetchHeaders* WebCore__FetchHeaders__createFromPicoHeaders_(const void* StringView nameView = StringView(std::span { reinterpret_cast(header.name.ptr), header.name.len }); - LChar* data = nullptr; + std::span data; auto value = String::createUninitialized(header.value.len, data); - memcpy(data, header.value.ptr, header.value.len); + memcpy(data.data(), header.value.ptr, header.value.len); HTTPHeaderName name; @@ -1709,10 +1709,10 @@ WebCore::FetchHeaders* WebCore__FetchHeaders__createFromUWS(void* arg1) for (const auto& header : req) { StringView nameView = StringView(std::span { reinterpret_cast(header.first.data()), header.first.length() }); - LChar* data = nullptr; + std::span data; auto value = String::createUninitialized(header.second.length(), data); if (header.second.length() > 0) - memcpy(data, header.second.data(), header.second.length()); + memcpy(data.data(), header.second.data(), header.second.length()); HTTPHeaderName name; diff --git a/src/bun.js/bindings/c-bindings.cpp b/src/bun.js/bindings/c-bindings.cpp index 18bf169ae7f2bb..38f74c731f46d6 100644 --- a/src/bun.js/bindings/c-bindings.cpp +++ b/src/bun.js/bindings/c-bindings.cpp @@ -688,7 +688,11 @@ extern "C" int ffi_fscanf(FILE* stream, const char* fmt, ...) extern "C" int ffi_vsscanf(const char* str, const char* fmt, va_list ap) { - return vsscanf(str, fmt, ap); + va_list ap_copy; + va_copy(ap_copy, ap); + int result = vsscanf(str, fmt, ap_copy); + va_end(ap_copy); + return result; } extern "C" int ffi_sscanf(const char* str, const char* fmt, ...) diff --git a/src/bun.js/bindings/helpers.h b/src/bun.js/bindings/helpers.h index 5aeca386957d2c..cf5a5e875fbd4b 100644 --- a/src/bun.js/bindings/helpers.h +++ b/src/bun.js/bindings/helpers.h @@ -168,18 +168,18 @@ static const WTF::String toStringCopy(ZigString str) } if (isTaggedUTF16Ptr(str.ptr)) { - UChar* out = nullptr; + std::span out; auto impl = WTF::StringImpl::tryCreateUninitialized(str.len, out); if (UNLIKELY(!impl)) return WTF::String(); - memcpy(out, untag(str.ptr), str.len * sizeof(UChar)); + memcpy(out.data(), untag(str.ptr), str.len * sizeof(UChar)); return WTF::String(WTFMove(impl)); } else { - LChar* out = nullptr; + std::span out; auto impl = WTF::StringImpl::tryCreateUninitialized(str.len, out); if (UNLIKELY(!impl)) return WTF::String(); - memcpy(out, untag(str.ptr), str.len * sizeof(LChar)); + memcpy(out.data(), untag(str.ptr), str.len * sizeof(LChar)); return WTF::String(WTFMove(impl)); } } diff --git a/src/bun.js/bindings/webcore/HTTPHeaderMap.cpp b/src/bun.js/bindings/webcore/HTTPHeaderMap.cpp index d2765a7b8f6122..577485117be0ef 100644 --- a/src/bun.js/bindings/webcore/HTTPHeaderMap.cpp +++ b/src/bun.js/bindings/webcore/HTTPHeaderMap.cpp @@ -153,9 +153,9 @@ void HTTPHeaderMap::setUncommonHeaderCloneName(const StringView name, const Stri return equalIgnoringASCIICase(header.key, name); }); if (index == notFound) { - LChar* ptr = nullptr; + std::span ptr; auto nameCopy = WTF::String::createUninitialized(name.length(), ptr); - memcpy(ptr, name.span8().data(), name.length()); + memcpy(ptr.data(), name.span8().data(), name.length()); m_uncommonHeaders.append(UncommonHeader { nameCopy, value }); } else m_uncommonHeaders[index].value = value; diff --git a/src/bun.js/bindings/webcore/PerformanceUserTiming.cpp b/src/bun.js/bindings/webcore/PerformanceUserTiming.cpp index 0ea5967ec1f4a5..81061abfb22b4c 100644 --- a/src/bun.js/bindings/webcore/PerformanceUserTiming.cpp +++ b/src/bun.js/bindings/webcore/PerformanceUserTiming.cpp @@ -44,27 +44,27 @@ namespace WebCore { using NavigationTimingFunction = unsigned long long (PerformanceTiming::*)() const; static constexpr std::pair restrictedMarkMappings[] = { - { "connectEnd", &PerformanceTiming::connectEnd }, - { "connectStart", &PerformanceTiming::connectStart }, - { "domComplete", &PerformanceTiming::domComplete }, - { "domContentLoadedEventEnd", &PerformanceTiming::domContentLoadedEventEnd }, - { "domContentLoadedEventStart", &PerformanceTiming::domContentLoadedEventStart }, - { "domInteractive", &PerformanceTiming::domInteractive }, - { "domLoading", &PerformanceTiming::domLoading }, - { "domainLookupEnd", &PerformanceTiming::domainLookupEnd }, - { "domainLookupStart", &PerformanceTiming::domainLookupStart }, - { "fetchStart", &PerformanceTiming::fetchStart }, - { "loadEventEnd", &PerformanceTiming::loadEventEnd }, - { "loadEventStart", &PerformanceTiming::loadEventStart }, - { "navigationStart", &PerformanceTiming::navigationStart }, - { "redirectEnd", &PerformanceTiming::redirectEnd }, - { "redirectStart", &PerformanceTiming::redirectStart }, - { "requestStart", &PerformanceTiming::requestStart }, - { "responseEnd", &PerformanceTiming::responseEnd }, - { "responseStart", &PerformanceTiming::responseStart }, - { "secureConnectionStart", &PerformanceTiming::secureConnectionStart }, - { "unloadEventEnd", &PerformanceTiming::unloadEventEnd }, - { "unloadEventStart", &PerformanceTiming::unloadEventStart }, + { "connectEnd"_s, &PerformanceTiming::connectEnd }, + { "connectStart"_s, &PerformanceTiming::connectStart }, + { "domComplete"_s, &PerformanceTiming::domComplete }, + { "domContentLoadedEventEnd"_s, &PerformanceTiming::domContentLoadedEventEnd }, + { "domContentLoadedEventStart"_s, &PerformanceTiming::domContentLoadedEventStart }, + { "domInteractive"_s, &PerformanceTiming::domInteractive }, + { "domLoading"_s, &PerformanceTiming::domLoading }, + { "domainLookupEnd"_s, &PerformanceTiming::domainLookupEnd }, + { "domainLookupStart"_s, &PerformanceTiming::domainLookupStart }, + { "fetchStart"_s, &PerformanceTiming::fetchStart }, + { "loadEventEnd"_s, &PerformanceTiming::loadEventEnd }, + { "loadEventStart"_s, &PerformanceTiming::loadEventStart }, + { "navigationStart"_s, &PerformanceTiming::navigationStart }, + { "redirectEnd"_s, &PerformanceTiming::redirectEnd }, + { "redirectStart"_s, &PerformanceTiming::redirectStart }, + { "requestStart"_s, &PerformanceTiming::requestStart }, + { "responseEnd"_s, &PerformanceTiming::responseEnd }, + { "responseStart"_s, &PerformanceTiming::responseStart }, + { "secureConnectionStart"_s, &PerformanceTiming::secureConnectionStart }, + { "unloadEventEnd"_s, &PerformanceTiming::unloadEventEnd }, + { "unloadEventStart"_s, &PerformanceTiming::unloadEventStart }, }; static constexpr SortedArrayMap restrictedMarkFunctions { restrictedMarkMappings }; diff --git a/src/bun.js/bindings/webcore/SerializedScriptValue.cpp b/src/bun.js/bindings/webcore/SerializedScriptValue.cpp index c3b936459e8fd1..c749770c3328df 100644 --- a/src/bun.js/bindings/webcore/SerializedScriptValue.cpp +++ b/src/bun.js/bindings/webcore/SerializedScriptValue.cpp @@ -3223,7 +3223,7 @@ class CloneDeserializer : CloneBase { str = String({ reinterpret_cast(ptr), length }); ptr += length * sizeof(UChar); #else - UChar* characters; + std::span characters; str = String::createUninitialized(length, characters); for (unsigned i = 0; i < length; ++i) { uint16_t c; @@ -3269,7 +3269,7 @@ class CloneDeserializer : CloneBase { str = Identifier::fromString(vm, { reinterpret_cast(ptr), length }); ptr += length * sizeof(UChar); #else - UChar* characters; + std::span characters; str = String::createUninitialized(length, characters); for (unsigned i = 0; i < length; ++i) { uint16_t c; diff --git a/src/bun.js/bindings/webcrypto/CryptoAlgorithmHMACOpenSSL.cpp b/src/bun.js/bindings/webcrypto/CryptoAlgorithmHMACOpenSSL.cpp index 2caf8149e852f6..78cbc6f8e86740 100644 --- a/src/bun.js/bindings/webcrypto/CryptoAlgorithmHMACOpenSSL.cpp +++ b/src/bun.js/bindings/webcrypto/CryptoAlgorithmHMACOpenSSL.cpp @@ -97,6 +97,7 @@ ExceptionOr CryptoAlgorithmHMAC::platformVerifyWithAlgorithm(const CryptoK // Using a constant time comparison to prevent timing attacks. return signature.size() == expectedSignature->size() && !constantTimeMemcmp(expectedSignature->data(), signature.data(), expectedSignature->size()); } + ExceptionOr CryptoAlgorithmHMAC::platformVerify(const CryptoKeyHMAC& key, const Vector& signature, const Vector& data) { auto algorithm = digestAlgorithm(key.hashAlgorithmIdentifier()); diff --git a/src/bun.js/bindings/webcrypto/JSCryptoKey.cpp b/src/bun.js/bindings/webcrypto/JSCryptoKey.cpp index 1d3d32e6683bb5..ff2140d0d7fccf 100644 --- a/src/bun.js/bindings/webcrypto/JSCryptoKey.cpp +++ b/src/bun.js/bindings/webcrypto/JSCryptoKey.cpp @@ -88,9 +88,9 @@ template<> std::optional parseEnumeration(JSGl { auto stringValue = value.toWTFString(&lexicalGlobalObject); static constexpr std::pair mappings[] = { - { "private", CryptoKey::Type::Private }, - { "public", CryptoKey::Type::Public }, - { "secret", CryptoKey::Type::Secret }, + { "private"_s, CryptoKey::Type::Private }, + { "public"_s, CryptoKey::Type::Public }, + { "secret"_s, CryptoKey::Type::Secret }, }; static constexpr SortedArrayMap enumerationMapping { mappings }; if (auto* enumerationValue = enumerationMapping.tryGet(stringValue); LIKELY(enumerationValue)) diff --git a/src/bun.js/bindings/webcrypto/JSCryptoKeyUsage.cpp b/src/bun.js/bindings/webcrypto/JSCryptoKeyUsage.cpp index d749d879c2f0db..742f63ea6e62e4 100644 --- a/src/bun.js/bindings/webcrypto/JSCryptoKeyUsage.cpp +++ b/src/bun.js/bindings/webcrypto/JSCryptoKeyUsage.cpp @@ -65,14 +65,14 @@ template<> std::optional parseEnumeration(JSGlob { auto stringValue = value.toWTFString(&lexicalGlobalObject); static constexpr std::pair mappings[] = { - { "decrypt", CryptoKeyUsage::Decrypt }, - { "deriveBits", CryptoKeyUsage::DeriveBits }, - { "deriveKey", CryptoKeyUsage::DeriveKey }, - { "encrypt", CryptoKeyUsage::Encrypt }, - { "sign", CryptoKeyUsage::Sign }, - { "unwrapKey", CryptoKeyUsage::UnwrapKey }, - { "verify", CryptoKeyUsage::Verify }, - { "wrapKey", CryptoKeyUsage::WrapKey }, + { "decrypt"_s, CryptoKeyUsage::Decrypt }, + { "deriveBits"_s, CryptoKeyUsage::DeriveBits }, + { "deriveKey"_s, CryptoKeyUsage::DeriveKey }, + { "encrypt"_s, CryptoKeyUsage::Encrypt }, + { "sign"_s, CryptoKeyUsage::Sign }, + { "unwrapKey"_s, CryptoKeyUsage::UnwrapKey }, + { "verify"_s, CryptoKeyUsage::Verify }, + { "wrapKey"_s, CryptoKeyUsage::WrapKey }, }; static constexpr SortedArrayMap enumerationMapping { mappings }; if (auto* enumerationValue = enumerationMapping.tryGet(stringValue); LIKELY(enumerationValue)) diff --git a/src/bun.js/bindings/webcrypto/JSSubtleCrypto.cpp b/src/bun.js/bindings/webcrypto/JSSubtleCrypto.cpp index edd11f56ab2abd..92780d31add505 100644 --- a/src/bun.js/bindings/webcrypto/JSSubtleCrypto.cpp +++ b/src/bun.js/bindings/webcrypto/JSSubtleCrypto.cpp @@ -96,10 +96,10 @@ template<> std::optional parseEnumeration mappings[] = { - { "jwk", SubtleCrypto::KeyFormat::Jwk }, - { "pkcs8", SubtleCrypto::KeyFormat::Pkcs8 }, - { "raw", SubtleCrypto::KeyFormat::Raw }, - { "spki", SubtleCrypto::KeyFormat::Spki }, + { "jwk"_s, SubtleCrypto::KeyFormat::Jwk }, + { "pkcs8"_s, SubtleCrypto::KeyFormat::Pkcs8 }, + { "raw"_s, SubtleCrypto::KeyFormat::Raw }, + { "spki"_s, SubtleCrypto::KeyFormat::Spki }, }; static constexpr SortedArrayMap enumerationMapping { mappings }; if (auto* enumerationValue = enumerationMapping.tryGet(stringValue); LIKELY(enumerationValue)) diff --git a/src/bun.js/bindings/workaround-missing-symbols.cpp b/src/bun.js/bindings/workaround-missing-symbols.cpp index b1fcc12a637fdc..3cfb58408616d1 100644 --- a/src/bun.js/bindings/workaround-missing-symbols.cpp +++ b/src/bun.js/bindings/workaround-missing-symbols.cpp @@ -1,5 +1,3 @@ - - #if defined(WIN32) #include @@ -66,6 +64,11 @@ extern "C" int kill(int pid, int sig) #include #include #include +#include +#include +#include +#include +#include #ifndef _STAT_VER #if defined(__aarch64__) @@ -78,38 +81,16 @@ extern "C" int kill(int pid, int sig) #endif #if defined(__x86_64__) -__asm__(".symver cosf,cosf@GLIBC_2.2.5"); -__asm__(".symver exp,exp@GLIBC_2.2.5"); __asm__(".symver expf,expf@GLIBC_2.2.5"); -__asm__(".symver fcntl,fcntl@GLIBC_2.2.5"); -__asm__(".symver fmod,fmod@GLIBC_2.2.5"); -__asm__(".symver fmodf,fmodf@GLIBC_2.2.5"); -__asm__(".symver log,log@GLIBC_2.2.5"); -__asm__(".symver log10f,log10f@GLIBC_2.2.5"); -__asm__(".symver log2,log2@GLIBC_2.2.5"); -__asm__(".symver log2f,log2f@GLIBC_2.2.5"); -__asm__(".symver logf,logf@GLIBC_2.2.5"); -__asm__(".symver pow,pow@GLIBC_2.2.5"); -__asm__(".symver powf,powf@GLIBC_2.2.5"); -__asm__(".symver sincosf,sincosf@GLIBC_2.2.5"); -__asm__(".symver sinf,sinf@GLIBC_2.2.5"); -__asm__(".symver tanf,tanf@GLIBC_2.2.5"); #elif defined(__aarch64__) -__asm__(".symver cosf,cosf@GLIBC_2.17"); -__asm__(".symver exp,exp@GLIBC_2.17"); __asm__(".symver expf,expf@GLIBC_2.17"); -__asm__(".symver fmod,fmod@GLIBC_2.17"); -__asm__(".symver fmodf,fmodf@GLIBC_2.17"); +__asm__(".symver powf,powf@GLIBC_2.17"); +__asm__(".symver pow,pow@GLIBC_2.17"); __asm__(".symver log,log@GLIBC_2.17"); -__asm__(".symver log10f,log10f@GLIBC_2.17"); -__asm__(".symver log2,log2@GLIBC_2.17"); -__asm__(".symver log2f,log2f@GLIBC_2.17"); +__asm__(".symver exp,exp@GLIBC_2.17"); __asm__(".symver logf,logf@GLIBC_2.17"); -__asm__(".symver pow,pow@GLIBC_2.17"); -__asm__(".symver powf,powf@GLIBC_2.17"); -__asm__(".symver sincosf,sincosf@GLIBC_2.17"); -__asm__(".symver sinf,sinf@GLIBC_2.17"); -__asm__(".symver tanf,tanf@GLIBC_2.17"); +__asm__(".symver log2f,log2f@GLIBC_2.17"); +__asm__(".symver log2,log2@GLIBC_2.17"); #endif #if defined(__x86_64__) || defined(__aarch64__) @@ -119,36 +100,43 @@ __asm__(".symver tanf,tanf@GLIBC_2.17"); #endif extern "C" { -double BUN_WRAP_GLIBC_SYMBOL(exp)(double); -double BUN_WRAP_GLIBC_SYMBOL(fmod)(double, double); -double BUN_WRAP_GLIBC_SYMBOL(log)(double); -double BUN_WRAP_GLIBC_SYMBOL(log2)(double); -double BUN_WRAP_GLIBC_SYMBOL(pow)(double, double); -float BUN_WRAP_GLIBC_SYMBOL(cosf)(float); + float BUN_WRAP_GLIBC_SYMBOL(expf)(float); -float BUN_WRAP_GLIBC_SYMBOL(fmodf)(float, float); -float BUN_WRAP_GLIBC_SYMBOL(log10f)(float); -float BUN_WRAP_GLIBC_SYMBOL(log2f)(float); + +#if defined(__aarch64__) + +float BUN_WRAP_GLIBC_SYMBOL(powf)(float, float); +double BUN_WRAP_GLIBC_SYMBOL(pow)(double, double); +double BUN_WRAP_GLIBC_SYMBOL(log)(double); +double BUN_WRAP_GLIBC_SYMBOL(exp)(double); float BUN_WRAP_GLIBC_SYMBOL(logf)(float); -float BUN_WRAP_GLIBC_SYMBOL(sinf)(float); -float BUN_WRAP_GLIBC_SYMBOL(tanf)(float); -int BUN_WRAP_GLIBC_SYMBOL(fcntl)(int, int, ...); +float BUN_WRAP_GLIBC_SYMBOL(log2f)(float); +double BUN_WRAP_GLIBC_SYMBOL(log2)(double); int BUN_WRAP_GLIBC_SYMBOL(fcntl64)(int, int, ...); -void BUN_WRAP_GLIBC_SYMBOL(sincosf)(float, float*, float*); -} -extern "C" { +#endif #if defined(__x86_64__) || defined(__aarch64__) -int __wrap_fcntl(int fd, int cmd, ...) -{ - va_list args; - va_start(args, cmd); - void* arg = va_arg(args, void*); - va_end(args); - return fcntl(fd, cmd, arg); -} +float __wrap_expf(float x) { return expf(x); } + +#if defined(__aarch64__) + +float __wrap_powf(float x, float y) { return powf(x, y); } +double __wrap_pow(double x, double y) { return pow(x, y); } +double __wrap_log(double x) { return log(x); } +double __wrap_exp(double x) { return exp(x); } +float __wrap_logf(float x) { return logf(x); } +float __wrap_log2f(float x) { return log2f(x); } +double __wrap_log2(double x) { return log2(x); } + +#endif + +#endif // x86_64 or aarch64 + +} // extern "C" + +#if defined(__aarch64__) typedef int (*fcntl64_func)(int fd, int cmd, ...); @@ -242,104 +230,8 @@ extern "C" int __wrap_fcntl64(int fd, int cmd, ...) #endif -#if defined(__x86_64__) - -#ifndef _MKNOD_VER -#define _MKNOD_VER 1 -#endif - -extern "C" int __lxstat(int ver, const char* filename, struct stat* stat); -extern "C" int __wrap_lstat(const char* filename, struct stat* stat) -{ - return __lxstat(_STAT_VER, filename, stat); -} - -extern "C" int __xstat(int ver, const char* filename, struct stat* stat); -extern "C" int __wrap_stat(const char* filename, struct stat* stat) -{ - return __xstat(_STAT_VER, filename, stat); -} - -extern "C" int __fxstat(int ver, int fd, struct stat* stat); -extern "C" int __wrap_fstat(int fd, struct stat* stat) -{ - return __fxstat(_STAT_VER, fd, stat); -} - -extern "C" int __fxstatat(int ver, int dirfd, const char* path, struct stat* stat, int flags); -extern "C" int __wrap_fstatat(int dirfd, const char* path, struct stat* stat, int flags) -{ - return __fxstatat(_STAT_VER, dirfd, path, stat, flags); -} - -extern "C" int __lxstat64(int ver, const char* filename, struct stat64* stat); -extern "C" int __wrap_lstat64(const char* filename, struct stat64* stat) -{ - return __lxstat64(_STAT_VER, filename, stat); -} - -extern "C" int __xstat64(int ver, const char* filename, struct stat64* stat); -extern "C" int __wrap_stat64(const char* filename, struct stat64* stat) -{ - return __xstat64(_STAT_VER, filename, stat); -} - -extern "C" int __fxstat64(int ver, int fd, struct stat64* stat); -extern "C" int __wrap_fstat64(int fd, struct stat64* stat) -{ - return __fxstat64(_STAT_VER, fd, stat); -} - -extern "C" int __fxstatat64(int ver, int dirfd, const char* path, struct stat64* stat, int flags); -extern "C" int __wrap_fstatat64(int dirfd, const char* path, struct stat64* stat, int flags) -{ - return __fxstatat64(_STAT_VER, dirfd, path, stat, flags); -} - -extern "C" int __xmknod(int ver, const char* path, mode_t mode, dev_t dev); -extern "C" int __wrap_mknod(const char* path, mode_t mode, dev_t dev) -{ - return __xmknod(_MKNOD_VER, path, mode, dev); -} - -extern "C" int __xmknodat(int ver, int dirfd, const char* path, mode_t mode, dev_t dev); -extern "C" int __wrap_mknodat(int dirfd, const char* path, mode_t mode, dev_t dev) -{ - return __xmknodat(_MKNOD_VER, dirfd, path, mode, dev); -} - -#endif - -double __wrap_exp(double x) -{ - return exp(x); -} -double __wrap_fmod(double x, double y) { return fmod(x, y); } -double __wrap_log(double x) { return log(x); } -double __wrap_log2(double x) { return log2(x); } -double __wrap_pow(double x, double y) { return pow(x, y); } -float __wrap_powf(float x, float y) { return powf(x, y); } -float __wrap_cosf(float x) { return cosf(x); } -float __wrap_expf(float x) { return expf(x); } -float __wrap_fmodf(float x, float y) { return fmodf(x, y); } -float __wrap_log10f(float x) { return log10f(x); } -float __wrap_log2f(float x) { return log2f(x); } -float __wrap_logf(float x) { return logf(x); } -float __wrap_sinf(float x) { return sinf(x); } -float __wrap_tanf(float x) { return tanf(x); } -void __wrap_sincosf(float x, float* sin_x, float* cos_x) { sincosf(x, sin_x, cos_x); } -} - -// ban statx, for now -extern "C" int __wrap_statx(int fd, const char* path, int flags, - unsigned int mask, struct statx* buf) -{ - errno = ENOSYS; -#ifdef BUN_DEBUG - abort(); -#endif - return -1; -} +extern "C" __attribute__((used)) char _libc_single_threaded = 0; +extern "C" __attribute__((used)) char __libc_single_threaded = 0; #endif // glibc diff --git a/src/bun.js/bindings/wtf-bindings.cpp b/src/bun.js/bindings/wtf-bindings.cpp index 848e7d143d1e72..52de7d5a7c0b0c 100644 --- a/src/bun.js/bindings/wtf-bindings.cpp +++ b/src/bun.js/bindings/wtf-bindings.cpp @@ -195,13 +195,10 @@ String base64URLEncodeToString(Vector data) if (!encodedLength) return String(); - LChar* ptr; + std::span ptr; auto result = String::createUninitialized(encodedLength, ptr); - if (UNLIKELY(!ptr)) { - RELEASE_ASSERT_NOT_REACHED(); - return String(); - } - encodedLength = WTF__base64URLEncode(reinterpret_cast(data.data()), data.size(), reinterpret_cast(ptr), encodedLength); + + encodedLength = WTF__base64URLEncode(reinterpret_cast(data.data()), data.size(), reinterpret_cast(ptr.data()), encodedLength); if (result.length() != encodedLength) { return result.substringSharingImpl(0, encodedLength); } diff --git a/src/c.zig b/src/c.zig index 1579667ab9f0e5..e0b8238fca1227 100644 --- a/src/c.zig +++ b/src/c.zig @@ -42,29 +42,13 @@ pub extern "c" fn memchr(s: [*]const u8, c: u8, n: usize) ?[*]const u8; pub extern "c" fn strchr(str: [*]const u8, char: u8) ?[*]const u8; -pub const lstat = blk: { - const T = *const fn ([*c]const u8, [*c]libc_stat) callconv(.C) c_int; // TODO: this is wrong on Windows - if (bun.Environment.isMusl) break :blk @extern(T, .{ .library_name = "c", .name = "lstat" }); - break :blk @extern(T, .{ .name = "lstat64" }); -}; -pub const fstat = blk: { - const T = *const fn (c_int, [*c]libc_stat) callconv(.C) c_int; // TODO: this is wrong on Windows - if (bun.Environment.isMusl) break :blk @extern(T, .{ .library_name = "c", .name = "fstat" }); - break :blk @extern(T, .{ .name = "fstat64" }); -}; -pub const stat = blk: { - const T = *const fn ([*c]const u8, [*c]libc_stat) callconv(.C) c_int; // TODO: this is wrong on Windows - if (bun.Environment.isMusl) break :blk @extern(T, .{ .library_name = "c", .name = "stat" }); - break :blk @extern(T, .{ .name = "stat64" }); -}; - pub fn lstat_absolute(path: [:0]const u8) !Stat { if (builtin.os.tag == .windows) { @compileError("Not implemented yet, conside using bun.sys.lstat()"); } var st = zeroes(libc_stat); - switch (errno(lstat(path.ptr, &st))) { + switch (errno(bun.C.lstat(path.ptr, &st))) { .SUCCESS => {}, .NOENT => return error.FileNotFound, // .EINVAL => unreachable, diff --git a/src/codegen/bindgen.ts b/src/codegen/bindgen.ts index d548a9a7cde09f..ef6d870908043e 100644 --- a/src/codegen/bindgen.ts +++ b/src/codegen/bindgen.ts @@ -683,7 +683,7 @@ function emitConvertEnumFunction(w: CodeWriter, type: TypeImpl) { w.line(`{`); w.line(` static constexpr std::pair mappings[] = {`); for (const value of type.data) { - w.line(` { ${str(value)}, ${name}::${pascal(value)} },`); + w.line(` { ${str(value)}_s, ${name}::${pascal(value)} },`); } w.line(` };`); w.line(` static constexpr SortedArrayMap enumerationMapping { mappings };`); diff --git a/src/darwin_c.zig b/src/darwin_c.zig index 2c0268058f8eab..7fb07e64d9e340 100644 --- a/src/darwin_c.zig +++ b/src/darwin_c.zig @@ -73,6 +73,20 @@ pub extern "c" fn fclonefileat(c_int, c_int, [*:0]const u8, uint32_t: c_int) c_i // int clonefile(const char * src, const char * dst, int flags); pub extern "c" fn clonefile(src: [*:0]const u8, dest: [*:0]const u8, flags: c_int) c_int; +pub const lstat = blk: { + const T = *const fn ([*c]const u8, [*c]std.c.Stat) callconv(.C) c_int; + break :blk @extern(T, .{ .name = "lstat64" }); +}; + +pub const fstat = blk: { + const T = *const fn ([*c]const u8, [*c]std.c.Stat) callconv(.C) c_int; + break :blk @extern(T, .{ .name = "fstat64" }); +}; +pub const stat = blk: { + const T = *const fn ([*c]const u8, [*c]std.c.Stat) callconv(.C) c_int; + break :blk @extern(T, .{ .name = "stat64" }); +}; + // pub fn stat_absolute(path: [:0]const u8) StatError!Stat { // if (builtin.os.tag == .windows) { // var io_status_block: windows.IO_STATUS_BLOCK = undefined; diff --git a/src/linux_c.zig b/src/linux_c.zig index 32292c6426cf66..bf99c082710ee5 100644 --- a/src/linux_c.zig +++ b/src/linux_c.zig @@ -785,3 +785,63 @@ export fn sys_epoll_pwait2(epfd: i32, events: ?[*]std.os.linux.epoll_event, maxe ), ); } + +// ********************************************************************************* +// libc overrides +// ********************************************************************************* + +fn simulateLibcErrno(rc: usize) c_int { + const signed: isize = @bitCast(rc); + const int: c_int = @intCast(if (signed > -4096 and signed < 0) -signed else 0); + std.c._errno().* = int; + return if (signed > -4096 and signed < 0) -1 else int; +} + +pub export fn stat(path: [*:0]const u8, buf: *std.os.linux.Stat) c_int { + // https://git.musl-libc.org/cgit/musl/tree/src/stat/stat.c + const rc = std.os.linux.fstatat(std.os.linux.AT.FDCWD, path, buf, 0); + return simulateLibcErrno(rc); +} + +pub const stat64 = stat; +pub const lstat64 = lstat; +pub const fstat64 = fstat; +pub const fstatat64 = fstatat; + +pub export fn lstat(path: [*:0]const u8, buf: *std.os.linux.Stat) c_int { + // https://git.musl-libc.org/cgit/musl/tree/src/stat/lstat.c + const rc = std.os.linux.fstatat(std.os.linux.AT.FDCWD, path, buf, std.os.linux.AT.SYMLINK_NOFOLLOW); + return simulateLibcErrno(rc); +} + +pub export fn fstat(fd: c_int, buf: *std.os.linux.Stat) c_int { + const rc = std.os.linux.fstat(fd, buf); + return simulateLibcErrno(rc); +} + +pub export fn fstatat(dirfd: i32, path: [*:0]const u8, buf: *std.os.linux.Stat, flags: u32) c_int { + const rc = std.os.linux.fstatat(dirfd, path, buf, flags); + return simulateLibcErrno(rc); +} + +pub export fn statx(dirfd: i32, path: [*:0]const u8, flags: u32, mask: u32, buf: *std.os.linux.Statx) c_int { + const rc = std.os.linux.statx(dirfd, path, flags, mask, buf); + return simulateLibcErrno(rc); +} + +comptime { + _ = stat; + _ = stat64; + _ = lstat; + _ = lstat64; + _ = fstat; + _ = fstat64; + _ = fstatat; + _ = statx; + @export(stat, .{ .name = "stat64" }); + @export(lstat, .{ .name = "lstat64" }); + @export(fstat, .{ .name = "fstat64" }); + @export(fstatat, .{ .name = "fstatat64" }); +} + +// ********************************************************************************* diff --git a/src/windows_c.zig b/src/windows_c.zig index 7c0c5d0d9eea11..86d3c32f95311e 100644 --- a/src/windows_c.zig +++ b/src/windows_c.zig @@ -8,6 +8,20 @@ const Stat = std.fs.File.Stat; const Kind = std.fs.File.Kind; const StatError = std.fs.File.StatError; +pub const lstat = blk: { + const T = *const fn ([*c]const u8, [*c]std.c.Stat) callconv(.C) c_int; + break :blk @extern(T, .{ .name = "lstat64" }); +}; + +pub const fstat = blk: { + const T = *const fn ([*c]const u8, [*c]std.c.Stat) callconv(.C) c_int; + break :blk @extern(T, .{ .name = "fstat64" }); +}; +pub const stat = blk: { + const T = *const fn ([*c]const u8, [*c]std.c.Stat) callconv(.C) c_int; + break :blk @extern(T, .{ .name = "stat64" }); +}; + pub fn getTotalMemory() usize { return uv.uv_get_total_memory(); } diff --git a/test/js/bun/symbols.test.ts b/test/js/bun/symbols.test.ts index 67127e3555b3b1..e255954ec606cb 100644 --- a/test/js/bun/symbols.test.ts +++ b/test/js/bun/symbols.test.ts @@ -6,7 +6,7 @@ import { semver } from "bun"; const BUN_EXE = bunExe(); if (process.platform === "linux") { - test("objdump -T does not include symbols from glibc > 2.27", async () => { + test("objdump -T does not include symbols from glibc > 2.26", async () => { const objdump = Bun.which("objdump") || Bun.which("llvm-objdump"); if (!objdump) { throw new Error("objdump executable not found. Please install it."); @@ -22,7 +22,7 @@ if (process.platform === "linux") { if (version.startsWith("2..")) { version = "2." + version.slice(3); } - if (semver.order(version, "2.27.0") >= 0) { + if (semver.order(version, "2.26.0") > 0) { errors.push({ symbol: line.slice(line.lastIndexOf(")") + 1).trim(), "glibc version": version, @@ -31,7 +31,7 @@ if (process.platform === "linux") { } } if (errors.length) { - throw new Error(`Found glibc symbols >= 2.27. This breaks Amazon Linux 2 and Vercel. + throw new Error(`Found glibc symbols > 2.26. This breaks Amazon Linux 2 and Vercel. ${Bun.inspect.table(errors, { colors: true })} To fix this, add it to -Wl,-wrap=symbol in the linker flags and update workaround-missing-symbols.cpp.`); From 7ded57854773912771f7a2731f3a1b90f8a457bc Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Thu, 12 Dec 2024 03:22:45 -0800 Subject: [PATCH 2/2] [publish images] --- scripts/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 42988c4f338f60..f5f793fa3aa2a8 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Version: 8 +# Version: 9 # A script that installs the dependencies needed to build and test Bun. # This should work on macOS and Linux with a POSIX shell.