Skip to content

Commit

Permalink
Esbuild bundle the Python pool setup code
Browse files Browse the repository at this point in the history
We want to execute a single standalone JavaScript file in the pool scope. It
should work in a normal v8 isolate with no modifications.
This adds the esbuild bundle step, but keeps everything else working the same.
  • Loading branch information
hoodmane committed Oct 10, 2024
1 parent e6f45b2 commit 4cdfbd3
Show file tree
Hide file tree
Showing 8 changed files with 201 additions and 6 deletions.
18 changes: 18 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,13 @@ http_archive(
url = "https://github.com/aspect-build/rules_ts/releases/download/v3.0.0/rules_ts-v3.0.0.tar.gz",
)

http_archive(
name = "aspect_rules_esbuild",
sha256 = "550e33ddeb86a564b22b2c5d3f84748c6639b1b2b71fae66bf362c33392cbed8",
strip_prefix = "rules_esbuild-0.21.0",
url = "https://github.com/aspect-build/rules_esbuild/releases/download/v0.21.0/rules_esbuild-v0.21.0.tar.gz",
)

load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies")

rules_js_dependencies()
Expand Down Expand Up @@ -312,6 +319,17 @@ load("@npm//:repositories.bzl", "npm_repositories")

npm_repositories()

load("@aspect_rules_esbuild//esbuild:dependencies.bzl", "rules_esbuild_dependencies")

rules_esbuild_dependencies()

load("@aspect_rules_esbuild//esbuild:repositories.bzl", "esbuild_register_toolchains")

esbuild_register_toolchains(
name = "esbuild",
esbuild_version = "0.23.0",
)

# ========================================================================================
# V8 and its dependencies
#
Expand Down
103 changes: 103 additions & 0 deletions build/js_file.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
"""
Give a collection of js files a JsInfo provider so it can be used as a dependency for aspect_rules.
"""

load("@aspect_bazel_lib//lib:copy_to_bin.bzl", "COPY_FILE_TO_BIN_TOOLCHAINS", "copy_file_to_bin_action")
load("@aspect_rules_js//js:providers.bzl", "JsInfo", "js_info")

_ATTRS = {
"srcs": attr.label_list(
allow_files = True,
),
"deps": attr.label_list(),
}

def _gather_sources_and_types(ctx, targets, files):
"""Gathers sources and types from a list of targets
Args:
ctx: the rule context
targets: List of targets to gather sources and types from their JsInfo providers.
These typically come from the `srcs` and/or `data` attributes of a rule
files: List of files to gather as sources and types.
These typically come from the `srcs` and/or `data` attributes of a rule
Returns:
Sources & declaration files depsets in the sequence (sources, types)
"""
sources = []
types = []

for file in files:
if file.is_source:
file = copy_file_to_bin_action(ctx, file)

if file.is_directory:
# assume a directory contains types since we can't know that it doesn't
types.append(file)
sources.append(file)
elif (
file.path.endswith((".d.ts", ".d.ts.map", ".d.mts", ".d.mts.map", ".d.cts", ".d.cts.map"))
):
types.append(file)
elif file.path.endswith(".json"):
# Any .json can produce types: https://www.typescriptlang.org/tsconfig/#resolveJsonModule
# package.json may be required to resolve types with the "typings" key
types.append(file)
sources.append(file)
else:
sources.append(file)

# sources as depset
sources = depset(sources, transitive = [
target[JsInfo].sources
for target in targets
if JsInfo in target
])

# types as depset
types = depset(types, transitive = [
target[JsInfo].types
for target in targets
if JsInfo in target
])

return (sources, types)

def _js_file_impl(ctx):
sources, types = _gather_sources_and_types(
ctx = ctx,
targets = ctx.attr.srcs,
files = ctx.files.srcs,
)

return [
js_info(
target = ctx.label,
sources = sources,
types = types,
),
DefaultInfo(
files = sources,
),
OutputGroupInfo(
types = types,
),
]

js_file_lib = struct(
attrs = _ATTRS,
implementation = _js_file_impl,
provides = [DefaultInfo, JsInfo, OutputGroupInfo],
)

js_file = rule(
implementation = js_file_lib.implementation,
attrs = js_file_lib.attrs,
provides = js_file_lib.provides,
toolchains = COPY_FILE_TO_BIN_TOOLCHAINS,
)
43 changes: 39 additions & 4 deletions src/pyodide/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
load("@aspect_rules_esbuild//esbuild:defs.bzl", "esbuild")
load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
load("@bazel_skylib//rules:write_file.bzl", "write_file")
load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
load("//:build/capnp_embed.bzl", "capnp_embed")
load("//:build/js_file.bzl", "js_file")
load("//:build/pyodide_bucket.bzl", "PYODIDE_PACKAGE_BUCKET_URL")
load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle_capnp")

Expand Down Expand Up @@ -82,7 +84,7 @@ copy_file(
# TODO: all of these should be fixed by linking our own Pyodide or by upstreaming.

PRELUDE = """
import { newWasmModule, monotonicDateNow, wasmInstantiate, getRandomValues } from "pyodide-internal:builtin_wrappers";
import { newWasmModule, monotonicDateNow, wasmInstantiate, getRandomValues } from "pyodide-internal:pool/builtin_wrappers";
// Pyodide uses `new URL(some_url, location)` to resolve the path in `loadPackage`. Setting
// `location = undefined` makes this throw an error if some_url is not an absolute url. Which is what
Expand Down Expand Up @@ -145,6 +147,38 @@ expand_template(
template = "@pyodide//:pyodide/pyodide.asm.js",
)

js_file(
name = "pyodide.asm.js@rule_js",
srcs = ["generated/pyodide.asm.js"],
deps = ["pyodide.asm.js@rule"],
)

esbuild(
name = "generated/emscriptenSetup",
srcs = glob([
"internal/pool/*",
]) + [
"generated/pyodide.asm.js",
"internal/util.ts",
],
config = "internal/pool/esbuild.config.mjs",
entry_point = "internal/pool/emscriptenSetup.ts",
external = [
"fs",
"path",
"url",
"vm",
"path",
"crypto",
"ws",
"child_process",
],
format = "esm",
output = "generated/emscriptenSetup.js",
target = "esnext",
deps = ["pyodide.asm.js@rule_js"],
)

data = wd_ts_bundle_capnp(
name = "pyodide.capnp",
eslintrc_json = "eslint.config.mjs",
Expand All @@ -159,20 +193,21 @@ data = wd_ts_bundle_capnp(
"generated/pyodide-bucket.json",
],
internal_modules = [
"generated/pyodide.asm.js",
"generated/emscriptenSetup.js",
] + glob(
[
"internal/*.ts",
"internal/*.js",
"internal/topLevelEntropy/*.ts",
"internal/topLevelEntropy/*.js",
# The pool directory is only needed by typescript, it shouldn't be used at runtime.
"internal/pool/*.ts",
"types/*.ts",
"types/*/*.ts",
],
allow_empty = True,
),
internal_wasm_modules = ["generated/pyodide.asm.wasm"],
js_deps = [
"generated/emscriptenSetup",
"pyodide.asm.js@rule",
"pyodide.asm.wasm@rule",
"pyodide-lock.js@rule",
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { _createPyodideModule } from 'pyodide-internal:generated/pyodide.asm';
export {
setUnsafeEval,
setGetRandomValues,
} from 'pyodide-internal:builtin_wrappers';
} from 'pyodide-internal:pool/builtin_wrappers';

/**
* A preRun hook. Make sure environment variables are visible at runtime.
Expand Down
34 changes: 34 additions & 0 deletions src/pyodide/internal/pool/esbuild.config.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import { dirname, join } from 'node:path';
import { existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';

const pyodideRootDir = dirname(
dirname(dirname(new URL(import.meta.url).pathname))
);

let resolvePlugin = {
name: 'example',
setup(build) {
// Redirect all paths starting with "images/" to "./public/images/"
build.onResolve({ filter: /pyodide-internal:.*/ }, (args) => {
let rest = args.path.split(':')[1];
let path;
if (rest.startsWith('generated')) {
path = join(pyodideRootDir, rest);
if (!existsSync(path)) {
path += '.js';
}
} else {
path = join(pyodideRootDir, 'internal', rest);
if (!existsSync(path)) {
path += '.ts';
}
}
return { path };
});
},
};

export default {
plugins: [resolvePlugin],
};
2 changes: 1 addition & 1 deletion src/pyodide/internal/python.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ import {
instantiateEmscriptenModule,
setUnsafeEval,
setGetRandomValues,
} from 'pyodide-internal:emscriptenSetup';
} from 'pyodide-internal:generated/emscriptenSetup';

/**
* After running `instantiateEmscriptenModule` but before calling into any C
Expand Down
5 changes: 5 additions & 0 deletions src/pyodide/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@
"paths": {
"pyodide:*": ["./*"],
"pyodide-internal:*": ["./internal/*", "./types/*"],
// generated/emscriptenSetup is an esbuild-bundled version of pool/emscriptenSetup. They have
// the same exports, so tell TypeScript to redirect the import.
"pyodide-internal:generated/emscriptenSetup": [
"./internal/pool/emscriptenSetup.ts"
],
"internal:*": ["./types/*"]
},
"typeRoots": ["./types"],
Expand Down

0 comments on commit 4cdfbd3

Please sign in to comment.