Skip to content

Commit

Permalink
feat: split js_image_layer into 5 layers
Browse files Browse the repository at this point in the history
  • Loading branch information
@gregmagolan
gregmagolan committed May 8, 2024
1 parent 66a2a55 commit 95db098
Show file tree
Hide file tree
Showing 23 changed files with 703 additions and 191 deletions.
4 changes: 2 additions & 2 deletions .aspect/rules/external_repository_action_cache/npm_translate_lock_LTE4Nzc1MDcwNjU=
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Input hashes for repository rule npm_translate_lock(name = "npm", pnpm_lock = "@@//:pnpm-lock.yaml").
# This file should be checked into version control along with the pnpm-lock.yaml file.
.npmrc=-2065072158
pnpm-lock.yaml=1142601305
pnpm-lock.yaml=-481966245
examples/npm_deps/patches/meaning-of-life@1.0.0-pnpm.patch=-442666336
package.json=-275319675
pnpm-workspace.yaml=-1178830835
Expand All @@ -23,5 +23,5 @@ npm/private/test/vendored/semver-max/package.json=578664053
examples/linked_empty_node_modules/package.json=-1039372825
examples/npm_package/packages/pkg_d/package.json=1110895851
js/private/image/package.json=-1260474848
js/private/test/image/package.json=1286417612
js/private/test/image/package.json=-687546763
js/private/test/js_run_devserver/package.json=-260856079
54 changes: 45 additions & 9 deletions docs/js_image_layer.md
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

54 changes: 45 additions & 9 deletions e2e/js_image_docker/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,25 +47,58 @@ js_image_layer(
)

filegroup(
name = "app_tar",
name = "node_tar",
srcs = [":layers"],
output_group = "app",
output_group = "node",
)

container_layer(
name = "app_layer",
tars = [":app_tar"],
name = "node_layer",
tars = [":node_tar"],
)

filegroup(
name = "node_modules_tar",
name = "package_store_3p_tar",
srcs = [":layers"],
output_group = "node_modules",
output_group = "package_store_3p",
)

container_layer(
name = "node_modules_layer",
tars = [":node_modules_tar"],
name = "package_store_3p_layer",
tars = [":package_store_3p_tar"],
)

filegroup(
name = "package_store_1p_tar",
srcs = [":layers"],
output_group = "package_store_1p",
)

container_layer(
name = "package_store_1p_layer",
tars = [":package_store_1p_tar"],
)

filegroup(
name = "node_modules_links_tar",
srcs = [":layers"],
output_group = "node_modules_links",
)

container_layer(
name = "node_modules_links_layer",
tars = [":node_modules_links_tar"],
)

filegroup(
name = "app_tar",
srcs = [":layers"],
output_group = "app",
)

container_layer(
name = "app_layer",
tars = [":app_tar"],
)

container_image(
Expand All @@ -77,8 +110,11 @@ container_image(
cmd = ["/app/bin"],
entrypoint = ["bash"],
layers = [
":node_layer",
":package_store_3p_layer",
":package_store_1p_layer",
":node_modules_links_layer",
":app_layer",
":node_modules_layer",
],
# This is `cmd` + `.runfiles/[workspace name]`
workdir = select({
Expand Down
168 changes: 137 additions & 31 deletions js/private/js_image_layer.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,34 +149,70 @@ js_image_layer(
)
filegroup(
name = "app_tar",
srcs = [":layers"],
output_group = "app",
name = "node_tar",
srcs = [":layers"],
output_group = "node",
)
container_layer(
name = "app_layer",
tars = [":app_tar"],
name = "node_layer",
tars = [":node_tar"],
)
filegroup(
name = "package_store_3p_tar",
srcs = [":layers"],
output_group = "package_store_3p",
)
container_layer(
name = "package_store_3p_layer",
tars = [":package_store_3p_tar"],
)
filegroup(
name = "node_modules_tar",
srcs = [":layers"],
output_group = "node_modules",
name = "package_store_1p_tar",
srcs = [":layers"],
output_group = "package_store_1p",
)
container_layer(
name = "node_modules_layer",
tars = [":node_modules_tar"],
name = "package_store_1p_layer",
tars = [":package_store_1p_tar"],
)
filegroup(
name = "node_modules_links_tar",
srcs = [":layers"],
output_group = "node_modules_links",
)
container_layer(
name = "node_modules_links_layer",
tars = [":node_modules_links_tar"],
)
filegroup(
name = "app_tar",
srcs = [":layers"],
output_group = "app",
)
container_layer(
name = "app_layer",
tars = [":app_tar"],
)
container_image(
name = "image",
cmd = ["/app/bin"],
entrypoint = ["bash"],
layers = [
":node_layer",
":package_store_3p_layer",
":package_store_1p_layer",
":node_modules_links_layer",
":app_layer",
":node_modules_layer",
],
workdir = select({
"@aspect_bazel_lib//lib:bzlmod": "/app/bin.runfiles/_main",
Expand Down Expand Up @@ -236,11 +272,22 @@ def _build_layer(ctx, type, all_entries_json, entries, inputs):

return output

def _should_be_in_node_modules_layer(destination, file):
def _select_layer(layers, destination, file):
is_node = file.owner.workspace_name != "" and "/bin/nodejs/" in destination
is_node_modules = "/node_modules/" in destination
is_js_patches = "/js/private/node-patches" in destination
return is_node or is_node_modules or is_js_patches
if is_node or is_js_patches:
return layers.node
is_package_store = "/.aspect_rules_js/" in destination
if is_package_store:
is_1p_dep = "@0.0.0/node_modules/" in destination
if is_1p_dep:
return layers.package_store_1p
else:
return layers.package_store_3p
is_node_modules = "/node_modules/" in destination
if is_node_modules:
return layers.node_modules_links
return layers.app

def _js_image_layer_impl(ctx):
if len(ctx.attr.binary) != 1:
Expand All @@ -260,15 +307,31 @@ def _js_image_layer_impl(ctx):
launcher = _write_laucher(ctx, real_binary_path)

all_files = depset(transitive = [binary_default_info.files, binary_default_info.default_runfiles.files])

app_entries = {binary_path: {"dest": launcher.path, "root": launcher.root.path}}
app_inputs = [launcher]

node_modules_entries = {}
node_modules_inputs = []

all_entries = {}

layers = struct(
node = struct(
entries = {},
inputs = [],
),
package_store_3p = struct(
entries = {},
inputs = [],
),
package_store_1p = struct(
entries = {},
inputs = [],
),
node_modules_links = struct(
entries = {},
inputs = [],
),
app = struct(
entries = {binary_path: {"dest": launcher.path, "root": launcher.root.path}},
inputs = [launcher],
),
)

for file in all_files.to_list():
destination = _runfile_path(ctx, file, runfiles_dir)
entry = {
Expand All @@ -282,22 +345,65 @@ def _js_image_layer_impl(ctx):
entry["remove_non_hermetic_lines"] = True

all_entries[destination] = entry
if _should_be_in_node_modules_layer(destination, file):
node_modules_entries[destination] = entry
node_modules_inputs.append(file)
else:
app_entries[destination] = entry
app_inputs.append(file)

layer = _select_layer(layers, destination, file)
layer.entries[destination] = entry
layer.inputs.append(file)

all_entries_json = ctx.actions.declare_file("{}_all_entries.json".format(ctx.label.name))
ctx.actions.write(all_entries_json, content = json.encode(all_entries))

app = _build_layer(ctx, type = "app", all_entries_json = all_entries_json, entries = app_entries, inputs = app_inputs)
node_modules = _build_layer(ctx, type = "node_modules", all_entries_json = all_entries_json, entries = node_modules_entries, inputs = node_modules_inputs)
node = _build_layer(
ctx,
type = "node",
all_entries_json = all_entries_json,
entries = layers.node.entries,
inputs = layers.node.inputs,
)
package_store_3p = _build_layer(
ctx,
type = "package_store_3p",
all_entries_json = all_entries_json,
entries = layers.package_store_3p.entries,
inputs = layers.package_store_3p.inputs,
)
package_store_1p = _build_layer(
ctx,
type = "package_store_1p",
all_entries_json = all_entries_json,
entries = layers.package_store_1p.entries,
inputs = layers.package_store_1p.inputs,
)
node_modules_links = _build_layer(
ctx,
type = "node_modules_links",
all_entries_json = all_entries_json,
entries = layers.node_modules_links.entries,
inputs = layers.node_modules_links.inputs,
)
app = _build_layer(
ctx,
type = "app",
all_entries_json = all_entries_json,
entries = layers.app.entries,
inputs = layers.app.inputs,
)

return [
DefaultInfo(files = depset([node_modules, app])),
OutputGroupInfo(app = depset([app]), node_modules = depset([node_modules])),
DefaultInfo(files = depset([
node,
package_store_3p,
package_store_1p,
node_modules_links,
app,
])),
OutputGroupInfo(
node = depset([node]),
package_store_3p = depset([package_store_3p]),
package_store_1p = depset([package_store_1p]),
node_modules_links = depset([node_modules_links]),
app = depset([app]),
),
]

def _js_image_layer_transition_impl(settings, attr):
Expand Down
Loading

0 comments on commit 95db098

Please sign in to comment.