Merge branch 'master' of github.com:bazelbuild/rules_docker into dock…

…er_run_args

.bazelci/presubmit.yml

@@ -14,6 +14,16 @@ tasks:
   default_workspace_macos:
     platform: macos
     test_targets:
+      # We don't know why this test started failing. Disable this until
+      # I have some time to look into it:
+      - -//tests/container:alpine_custom_attr_digest_test
+      - -//tests/container:image_test
+      - -//tests/container:test_digest_output1
+      - -//tests/container:test_digest_output1_impl
+      - -//tests/container:test_digest_output2
+      - -//tests/container:test_digest_output2_impl
+      - -//tests/container:test_push_digest_output
+      - -//tests/container:test_push_digest_output_impl
       # Use "bazel query 'kind(".*_test", ...)'" to refresh this list.
       #
       # NOTE: per Philipp Wollermann, docker is not installed on the macos
@@ -59,12 +69,15 @@ tasks:
       - -//tests/container/go:go_image_test
       - -//tests/container/go:go_static_image_test
       - -//tests/container/groovy:groovy_image_test
+      - -//tests/container/groovy:groovy_classpath_as_file_image_test
       - -//tests/container/java:java_image_test
+      - -//tests/container/java:java_classpath_as_file_image_test
       - -//tests/container/java:java_partial_entrypoint_image_test
       - -//tests/container/java:java_runfiles_as_lib_image_test
       - -//tests/container/java:java_runfiles_image_test
       - -//tests/container/java:simple_java_entrypoint_image_test
       - -//tests/container/kotlin:kotlin_image_test
+      - -//tests/container/kotlin:kotlin_classpath_as_file_image_test
       - -//tests/container/nodejs:nodejs_image_custom_binary_test
       - -//tests/container/nodejs:nodejs_image_custom_binary_with_args_test
       - -//tests/container/nodejs:nodejs_image_empty_list_args_test
@@ -76,6 +89,7 @@ tasks:
       - -//tests/container/python3:py3_image_test
       - -//tests/container/rust:rust_image_test
       - -//tests/container/scala:scala_image_test
+      - -//tests/container/scala:scala_classpath_as_file_image_test
       - -//tests/contrib:derivative_with_volume_repro_test
       - -//tests/contrib:random_file_img_non_repro_test
       - -//tests/contrib:rbe-test-xenial_repro_test
@@ -113,7 +127,6 @@ tasks:
       - //container/go/pkg/oci:go_default_test
       - //docker/util:config_stripper_test
       - //docs:all
-      - //tests/container:alpine_custom_attr_digest_test
       - //tests/container:alpine_custom_attr_digest_test_impl
       - //tests/container:alpine_linux_armv6_tar_test_image_tar
       - //tests/container:alpine_linux_armv6_tar_test_image_tar_impl
@@ -124,7 +137,6 @@ tasks:
       - //tests/container:distroless_fixed_id_digest_test_impl
       - //tests/container:distroless_fixed_id_image_digest_test
       - //tests/container:distroless_fixed_id_image_digest_test_impl
-      - //tests/container:image_test
       - //tests/container:k8s_pause_arm64_digest_test
       - //tests/container:k8s_pause_arm64_digest_test_impl
       - //tests/container:pause_tar_test_0_tar_gz
@@ -136,12 +148,6 @@ tasks:
       - //tests/container:pull_info_validation_test
       - //tests/container:set_cmd_test_host
       - //tests/container:special_characters_test
-      - //tests/container:test_digest_output1
-      - //tests/container:test_digest_output1_impl
-      - //tests/container:test_digest_output2
-      - //tests/container:test_digest_output2_impl
-      - //tests/container:test_push_digest_output
-      - //tests/container:test_push_digest_output_impl
       - //tests/contrib:rename_image_test
       - //tests/contrib:test_id_compare_ids_test
       - //tests/docker/package_managers:test_download_pkgs_docker_cp

README.md

@@ -139,6 +139,9 @@ docker_toolchain_configure(
   # OPTIONAL: Path to the xz binary.
   # Should be set explicitly for remote execution.
   xz_path="<enter absolute path to the xz binary (in the remote exec env) here>",
+  # OPTIONAL: Bazel target for the xz tool.
+  # Either xz_path or xz_target should be set explicitly for remote execution.
+  xz_target="<enter absolute path (i.e., must start with repo name @...//:...) to an executable xz target>",
   # OPTIONAL: List of additional flags to pass to the docker command.
   docker_flags = [
     "--tls",

WORKSPACE

@@ -297,15 +297,25 @@ jvm_maven_import_external(
 # For our scala_image test.
 http_archive(
     name = "io_bazel_rules_scala",
-    sha256 = "ed1a62f9fb2cb8930dd026b761ff900599b4c786c6cb6b7b5f9ad418f312c272",
-    strip_prefix = "rules_scala-0366fb23cb91fee2847a8358472278ddc9940c5f",
-    urls = ["https://github.com/bazelbuild/rules_scala/archive/0366fb23cb91fee2847a8358472278ddc9940c5f.tar.gz"],
+    sha256 = "ccf19e8f966022eaaca64da559c6140b23409829cb315f2eff5dc3e757fb6ad8",
+    strip_prefix = "rules_scala-e4560ac332e9da731c1e50a76af2579c55836a5c",
+    urls = ["https://github.com/bazelbuild/rules_scala/archive/e4560ac332e9da731c1e50a76af2579c55836a5c.zip"],
 )
 
+load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config")
+
+scala_config()
+
 load("@io_bazel_rules_scala//scala:scala.bzl", "scala_repositories")
 
 scala_repositories()
 
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+
+rules_proto_dependencies()
+
+rules_proto_toolchains()
+
 load("@io_bazel_rules_scala//scala:toolchains.bzl", "scala_register_toolchains")
 
 scala_register_toolchains()
@@ -333,11 +343,11 @@ _go_image_repos()
 # For our rust_image test
 http_archive(
     name = "rules_rust",
-    sha256 = "42e60f81e2b269d28334b73b70d02fb516c8de0c16242f5d376bfe6d94a3509f",
-    strip_prefix = "rules_rust-58f709ffec90da93c4e622d8d94f0cd55cd2ef54",
+    sha256 = "d6a8bc37502f252ef190b37945c922e0d0104dc2250940a3ea5f9c42e7a0dc10",
+    strip_prefix = "rules_rust-e2f0fccda912daac686b533ad77c5bc5d2f2ddb7",
     urls = [
-        # Master branch as of 2021-02-04
-        "https://github.com/bazelbuild/rules_rust/archive/58f709ffec90da93c4e622d8d94f0cd55cd2ef54.tar.gz",
+        # `main` branch as of 2021-11-05
+        "https://github.com/bazelbuild/rules_rust/archive/e2f0fccda912daac686b533ad77c5bc5d2f2ddb7.tar.gz",
     ],
 )
 

cc/cc.bzl

@@ -19,8 +19,8 @@
 # git repository.
 
 DIGESTS = {
-    # "gcr.io/distroless/cc:debug" circa 2021-03-14 11:14 -0400
-    "debug": "sha256:2b083aa36dbfe76eff429ee21596d53a6ca5ab69ec48aef375264c267e3da5a0",
-    # "gcr.io/distroless/cc:latest" circa 2021-03-14 11:14 -0400
-    "latest": "sha256:937aa8c7edd91618d8ed4510a0fd06a9dd97db3a786b23e953213cdcf88ccfde",
+    # "gcr.io/distroless/cc:debug" circa 2021-09-25 14:45 -0400
+    "debug": "sha256:3680c61e81f68fc00bfb5e1ec65e8e678aaafa7c5f056bc2681c29527ebbb30c",
+    # "gcr.io/distroless/cc:latest" circa 2021-09-25 14:45 -0400
+    "latest": "sha256:2c4bb6b7236db0a55ec54ba8845e4031f5db2be957ac61867872bf42e56c4deb",
 }

container/build_tar.py

@@ -42,7 +42,7 @@ class DebError(Exception):
 
   @staticmethod
   def parse_pkg_name(metadata, filename):
-    pkg_name_match = TarFile.PKG_NAME_RE.match(str(metadata))
+    pkg_name_match = TarFile.PKG_NAME_RE.match(metadata)
     if pkg_name_match:
       return pkg_name_match.group('pkg_name')
     else:
@@ -231,7 +231,7 @@ def add_pkg_metadata(self, metadata_tar, deb):
         control_file = tar.extractfile(control_file_member[0])
         metadata = b''.join(control_file.readlines())
         destination_file = os.path.join(TarFile.DPKG_STATUS_DIR,
-                                        TarFile.parse_pkg_name(metadata, deb))
+                                        TarFile.parse_pkg_name(metadata.decode("utf-8"), deb))
         with self.write_temp_file(data=metadata) as metadata_file:
           self.add_file(metadata_file, destination_file)
     except (KeyError, TypeError) as e:

container/bundle.bzl

@@ -73,10 +73,7 @@ def _container_bundle_impl(ctx):
         ctx,
         images,
         ctx.outputs.tar_output,
-        # Experiment: currently only support experimental_tarball_format in
-        # container_image for testing optimization.
-        # TODO(#1695): Update this.
-        "legacy",
+        ctx.attr.experimental_tarball_format,
         stamp = stamp,
     )
 
@@ -109,6 +106,19 @@ container_bundle_ = rule(
             mandatory = False,
         ),
         "tar_output": attr.output(),
+        "experimental_tarball_format": attr.string(
+            values = [
+                "legacy",
+                "compressed",
+            ],
+            default = "legacy",
+            doc = ("The tarball format to use when producing an image .tar file. " +
+                   "Defaults to \"legacy\", which contains uncompressed layers. " +
+                   "If set to \"compressed\", the resulting tarball will contain " +
+                   "compressed layers, but is only loadable by newer versions of " +
+                   "docker. This is an experimental attribute, which is subject " +
+                   "to change or removal: do not depend on its exact behavior."),
+        ),
     }, _layer_tools),
     executable = True,
     toolchains = ["@io_bazel_rules_docker//toolchains/docker:toolchain_type"],

container/go/cmd/join_layers/join_layers.go

@@ -85,17 +85,19 @@ func writeOutput(outputTarball string, tarballFormat string, tagToConfigs, tagTo
 	if err != nil {
 		return errors.Wrap(err, "unable to load images from the given tarballs")
 	}
+	parts := compat.ImageParts{
+		Images: images,
+		Layers: layerParts,
+	}
+	r := compat.Reader{Parts: parts}
 	for tag, configFile := range tagToConfigs {
 		// Manifest file may not have been specified and this is ok as it's
 		// only required if the base images has foreign layers.
 		manifestFile := tagToBaseManifests[tag]
-		parts := compat.ImageParts{
-			Config:       configFile,
-			BaseManifest: manifestFile,
-			Images:       images,
-			Layers:       layerParts,
-		}
-		img, err := compat.ReadImage(parts)
+		r.Parts.Config = configFile
+		r.Parts.BaseManifest = manifestFile
+
+		img, err := r.ReadImage()
 		if err != nil {
 			return errors.Wrapf(err, "unable to load image %v corresponding to config %s", tag, configFile)
 		}

container/go/pkg/compat/reader.go

@@ -117,11 +117,11 @@ func ImagePartsFromArgs(config, baseManifest, imgTarball string, layers []string
 	return result, nil
 }
 
-// reader maintains the state necessary to build a legacyImage object from an
+// Reader maintains the state necessary to build a legacyImage object from an
 // ImageParts object.
-type reader struct {
+type Reader struct {
 	// parts is the ImageParts being loaded.
-	parts ImageParts
+	Parts ImageParts
 	// baseManifest is the manifest of the very first base image in the chain
 	// of images being loaded.
 	baseManifest *v1.Manifest
@@ -130,32 +130,35 @@ type reader struct {
 	// layerLookup is a map from the diffID of a layer to the layer
 	// itself.
 	layerLookup map[v1.Hash]v1.Layer
+	// loadedImageCache is a cache of all images that have been loaded into memory,
+	// to prevent costly reloads.
+	loadedImageCache map[v1.Hash]bool
 }
 
 // loadMetadata loads the image metadata for the image parts in the given
 // reader.
-func (r *reader) loadMetadata() error {
-	cf, err := os.Open(r.parts.Config)
+func (r *Reader) loadMetadata() error {
+	cf, err := os.Open(r.Parts.Config)
 	if err != nil {
-		return errors.Wrapf(err, "unable to open image config file %s", r.parts.Config)
+		return errors.Wrapf(err, "unable to open image config file %s", r.Parts.Config)
 	}
 	c, err := v1.ParseConfigFile(cf)
 	if err != nil {
-		return errors.Wrapf(err, "unable to parse image config from %s", r.parts.Config)
+		return errors.Wrapf(err, "unable to parse image config from %s", r.Parts.Config)
 	}
 	r.config = c
-	if r.parts.BaseManifest == "" {
+	if r.Parts.BaseManifest == "" {
 		// Base manifest is optional. It's only needed for images whose base
 		// manifests have foreign layers.
 		return nil
 	}
-	mf, err := os.Open(r.parts.BaseManifest)
+	mf, err := os.Open(r.Parts.BaseManifest)
 	if err != nil {
-		return errors.Wrapf(err, "unable to open base image manifest file %s", r.parts.BaseManifest)
+		return errors.Wrapf(err, "unable to open base image manifest file %s", r.Parts.BaseManifest)
 	}
 	m, err := v1.ParseManifest(mf)
 	if err != nil {
-		return errors.Wrapf(err, "unable to parse base image manifest from %s", r.parts.BaseManifest)
+		return errors.Wrapf(err, "unable to parse base image manifest from %s", r.Parts.BaseManifest)
 	}
 	r.baseManifest = m
 	return nil
@@ -209,7 +212,7 @@ func (l *foreignLayer) MediaType() (types.MediaType, error) {
 
 // loadForeignLayers loads the foreign layers from the base manifest in the
 // given reader into the layer lookup.
-func (r *reader) loadForeignLayers() error {
+func (r *Reader) loadForeignLayers() error {
 	if r.baseManifest == nil {
 		// No base manifest so no foreign layers to load.
 		return nil
@@ -237,8 +240,12 @@ func (r *reader) loadForeignLayers() error {
 
 // loadImages loads the layers from the given images into the layers lookup
 // in the given reader.
-func (r *reader) loadImages(images []v1.Image) error {
+func (r *Reader) loadImages(images []v1.Image) error {
 	for _, img := range images {
+		digest, _ := img.Digest()
+		if r.loadedImageCache[digest] {
+			continue
+		}
 		layers, err := img.Layers()
 		if err != nil {
 			return errors.Wrap(err, "unable to get the layers in image")
@@ -250,31 +257,32 @@ func (r *reader) loadImages(images []v1.Image) error {
 			}
 			r.layerLookup[diffID] = l
 		}
+		r.loadedImageCache[digest] = true
 	}
 	return nil
 }
 
 // loadImgTarball loads the layers from the image tarball in the parts section
 // of the given reader if one was specified into the layers lookup in the given
 // reader.
-func (r *reader) loadImgTarball() error {
-	if r.parts.ImageTarball == "" {
+func (r *Reader) loadImgTarball() error {
+	if r.Parts.ImageTarball == "" {
 		return nil
 	}
-	img, err := tarball.ImageFromPath(r.parts.ImageTarball, nil)
+	img, err := tarball.ImageFromPath(r.Parts.ImageTarball, nil)
 	if err != nil {
-		return errors.Wrapf(err, "unable to load image from tarball %s", r.parts.ImageTarball)
+		return errors.Wrapf(err, "unable to load image from tarball %s", r.Parts.ImageTarball)
 	}
 	if err := r.loadImages([]v1.Image{img}); err != nil {
-		return errors.Wrapf(err, "unable to load the layers from image loaded from tarball %s", r.parts.ImageTarball)
+		return errors.Wrapf(err, "unable to load the layers from image loaded from tarball %s", r.Parts.ImageTarball)
 	}
 	return nil
 }
 
 // loadLayers loads layers specified as parts in the ImageParts section in the
 // given reader.
-func (r *reader) loadLayers() error {
-	for _, l := range r.parts.Layers {
+func (r *Reader) loadLayers() error {
+	for _, l := range r.Parts.Layers {
 		layer, err := l.V1Layer()
 		if err != nil {
 			return errors.Wrap(err, "unable to build a v1.Layer from the specified parts")
@@ -288,24 +296,28 @@ func (r *reader) loadLayers() error {
 	return nil
 }
 
-// ReadImage loads a v1.Image from the given ImageParts
-func ReadImage(parts ImageParts) (v1.Image, error) {
+// ReadImage loads a v1.Image from the ImageParts section in the reader.
+func (r *Reader) ReadImage() (v1.Image, error) {
 	// Special case: if we only have a tarball, we can instantiate the image
 	// directly from that. Otherwise, we'll process the image layers
 	// individually as specified in the config.
-	if parts.ImageTarball != "" && parts.Config == "" {
-		return tarball.ImageFromPath(parts.ImageTarball, nil)
+	if r.Parts.ImageTarball != "" && r.Parts.Config == "" {
+		return tarball.ImageFromPath(r.Parts.ImageTarball, nil)
 	}
 
-	r := reader{parts: parts}
-	r.layerLookup = make(map[v1.Hash]v1.Layer)
+	if r.layerLookup == nil {
+		r.layerLookup = make(map[v1.Hash]v1.Layer)
+	}
+	if r.loadedImageCache == nil {
+		r.loadedImageCache = make(map[v1.Hash]bool)
+	}
 	if err := r.loadMetadata(); err != nil {
 		return nil, errors.Wrap(err, "unable to load image metadata")
 	}
 	if err := r.loadForeignLayers(); err != nil {
 		return nil, errors.Wrap(err, "unable to load foreign layers specified in the base manifest")
 	}
-	if err := r.loadImages(r.parts.Images); err != nil {
+	if err := r.loadImages(r.Parts.Images); err != nil {
 		return nil, errors.Wrap(err, "unable to load layers from the images in the given image parts")
 	}
 	if err := r.loadImgTarball(); err != nil {
@@ -318,16 +330,23 @@ func ReadImage(parts ImageParts) (v1.Image, error) {
 	for _, diffID := range r.config.RootFS.DiffIDs {
 		layer, ok := r.layerLookup[diffID]
 		if !ok {
-			return nil, errors.Errorf("unable to locate layer with diffID %v as indicated in image config %s", diffID, parts.Config)
+			return nil, errors.Errorf("unable to locate layer with diffID %v as indicated in image config %s", diffID, r.Parts.Config)
 		}
 		layers = append(layers, layer)
 	}
 	img := &legacyImage{
-		configPath: parts.Config,
+		configPath: r.Parts.Config,
 		layers:     layers,
 	}
 	if err := img.init(); err != nil {
 		return nil, errors.Wrap(err, "unable to initialize image from parts")
 	}
 	return img, nil
 }
+
+// ReadImage loads a v1.Image from the given ImageParts
+func ReadImage(parts ImageParts) (v1.Image, error) {
+	r := Reader{Parts: parts}
+	img, err := r.ReadImage()
+	return img, err
+}