testing changes to platforms

.bazelci/presubmit.yml

@@ -71,6 +71,7 @@ platforms:
     # For tests/container:set_env_make_vars_test
     - "--define=ENV_KEY=my_key"
     - "--define=ENV_VALUE=my_value"
+    - "--extra_execution_platforms=@local_config_platform//:host,@io_bazel_rules_docker//platforms:local_container_platform"
     test_targets:
     - "--"
     - "//tests/container:alpine_custom_attr_digest_test"
@@ -111,6 +112,7 @@ platforms:
     - "--define=ENV_KEY=my_key"
     - "--define=ENV_VALUE=my_value"
     - "--test_output=errors"
+    - "--extra_execution_platforms=@local_config_platform//:host,@io_bazel_rules_docker//platforms:local_container_platform"
   rbe_ubuntu1604:
     build_targets:
     - "--"

.bazelrc

@@ -15,7 +15,6 @@
 # The following flags are set to test use of new features for python toolchains
 # These flags will only work with Bazel 0.25.0 or above and are only needed for
 #  //tests/docker/security/... & the docker/package_managers rules.
-build --host_force_python=PY2
 build --incompatible_use_python_toolchains
 test --incompatible_use_python_toolchains
-test --host_force_python=PY2
+

WORKSPACE

@@ -401,16 +401,15 @@ dockerfile_image(
 ]]
 
 # Register the default py_toolchain / platform for containerized execution
-load("//toolchains:py_toolchains.bzl", "py_toolchains")
-
-py_toolchains(name = "container_py_toolchain")
-
 register_toolchains(
     "//toolchains:container_py_toolchain",
-    "@container_py_toolchain//:container_cc_toolchain",
+    "@bazel_tools//tools/python:autodetecting_toolchain",
 )
 
-register_execution_platforms("//platforms:local_container_platform")
+register_execution_platforms(
+    "@local_config_platform//:host",
+    "//platforms:local_container_platform",
+)
 
 http_archive(
     name = "bazel_toolchains",

docker/security/cmd/json_to_yaml/BUILD

@@ -0,0 +1,28 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["json_to_yaml.go"],
+    importpath = "github.com/bazelbuild/rules_docker/docker/security/cmd/json_to_yaml",
+    visibility = ["//visibility:private"],
+    deps = ["@com_github_ghodss_yaml//:go_default_library"],
+)
+
+go_binary(
+    name = "json_to_yaml",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

docker/security/cmd/json_to_yaml/json_to_yaml.go

@@ -0,0 +1,50 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/ghodss/yaml"
+)
+
+var (
+	inJSON  = flag.String("in-json", "", "Path to input JSON file that will be converted to YAML.")
+	outYAML = flag.String("out-yaml", "", "Path to output YAML file.")
+)
+
+func main() {
+	flag.Parse()
+	if *inJSON == "" {
+		log.Fatalf("--in-json is required.")
+	}
+	if *outYAML == "" {
+		log.Fatalf("--out-yaml is required.")
+	}
+
+	j, err := ioutil.ReadFile(*inJSON)
+	if err != nil {
+		log.Fatalf("Unable to read input JSON file %q: %v", *inJSON, err)
+	}
+	y, err := yaml.JSONToYAML(j)
+	if err != nil {
+		log.Fatalf("Unable to convert JSON data loaded from %q to YAML: %v", *inJSON, err)
+	}
+	if err := ioutil.WriteFile(*outYAML, y, os.ModePerm); err != nil {
+		log.Fatalf("Unable to write output YAML to %q: %v", *outYAML, err)
+	}
+}

docker/security/security_check.bzl

@@ -19,7 +19,7 @@ def _impl(ctx):
     output_yaml = ctx.outputs.yaml
     args = ctx.actions.args()
     args.add(ctx.attr.image)
-    args.add("--output-yaml", ctx.outputs.yaml)
+    args.add("--output-json", ctx.outputs.json)
     args.add("--severity", ctx.attr.severity)
     if ctx.attr.whitelist != None:
         files = ctx.attr.whitelist.files.to_list()
@@ -35,7 +35,7 @@ def _impl(ctx):
     ctx.actions.run(
         executable = ctx.executable._security_check,
         arguments = [args],
-        outputs = [ctx.outputs.yaml],
+        outputs = [ctx.outputs.json],
         mnemonic = "ImageSecurityCheck",
         use_default_shell_env = True,
         execution_requirements = {
@@ -46,6 +46,16 @@ def _impl(ctx):
             "no-sandbox": "True",
         },
     )
+    args = ctx.actions.args()
+    args.add("--in-json", ctx.outputs.json)
+    args.add("--out-yaml", ctx.outputs.yaml)
+    ctx.actions.run(
+        executable = ctx.executable._json_to_yaml,
+        arguments = [args],
+        inputs = [ctx.outputs.json],
+        outputs = [ctx.outputs.yaml],
+        mnemonic = "JSONToYAML",
+    )
 
 # Run the security_check.py script on the given docker image to generate a
 # YAML output file with information about the types of vulnerabilities
@@ -68,6 +78,13 @@ security_check = rule(
             default = Label("@io_bazel_rules_docker//docker/security:security_check_whitelist.json"),
             allow_single_file = True,
         ),
+        # JSON to YAML converter.
+        "_json_to_yaml": attr.label(
+            default = Label("@io_bazel_rules_docker//docker/security/cmd/json_to_yaml"),
+            cfg = "host",
+            executable = True,
+            allow_files = True,
+        ),
         # The security checker python executable.
         "_security_check": attr.label(
             default = Label("@io_bazel_rules_docker//docker/security:security_check"),
@@ -77,6 +94,7 @@ security_check = rule(
         ),
     },
     outputs = {
+        "json": "%{name}.json",
         "yaml": "%{name}.yaml",
     },
 )

docker/security/security_check.py

@@ -19,7 +19,6 @@
 import subprocess
 import sys
 import logging
-import yaml
 
 import distutils.version as ver
 
@@ -265,17 +264,17 @@ def _get_version_number(version_obj):
 
   return ''.join([str(epoch), delimiter1, name, delimiter2, str(revision)])
 
-def _generate_yaml_output(output_yaml, vulnerabilities):
-  """Generate a YAML file mapping the key "tags" to the list of types of
+def _generate_json_output(output_json, vulnerabilities):
+  """Generate a JSON file mapping the key "tags" to the list of types of
   vulnerabilities found.
 
   Args:
-    output_yaml: Path to the output YAML file to generate.
+    output_json: Path to the output JSON file to generate.
     vulnerabilities: A dictionary mapping the name of the CVE entry to details
                      about the vulnerability.
   """
   tags = set()
-  for v in vulnerabilities.itervalues():
+  for v in vulnerabilities.values():
     details = v["vulnerabilityDetails"]
     # The service that consumes the metadata expects the tags as follows:
     # LOW -> cveLow
@@ -284,19 +283,19 @@ def _generate_yaml_output(output_yaml, vulnerabilities):
     sev = str(details['severity'])
     tags.add("cve{}".format(sev.lower().capitalize()))
   result = {"tags": list(tags)}
-  logging.info("Creating YAML output {}".format(output_yaml))
-  with open(output_yaml, "w") as ofp:
-    ofp.write(yaml.dump(result))
+  logging.info("Creating JSON output {}".format(output_json))
+  with open(output_json, "w") as ofp:
+    json.dump(result, ofp)
 
 def security_check(image, severity=_MEDIUM, whitelist_file='whitelist.json',
-                   output_yaml=None):
+                   output_json=None):
   """Main security check function.
 
   Args:
     image: full name of the docker image
     severity: the severity of vulnerability to trigger failure
     whitelist_file: file with list of whitelisted CVE
-    output_yaml: Output file which will be populated with a list of types of
+    output_json: Output file which will be populated with a list of types of
                  vulnerability that exist for the given image.
 
   Returns:
@@ -312,9 +311,9 @@ def security_check(image, severity=_MEDIUM, whitelist_file='whitelist.json',
 
   result = _check_for_vulnz(_sub_image(image), severity, whitelist)
 
-  if output_yaml:
-    logging.info("Creating YAML output {}".format(output_yaml))
-    _generate_yaml_output(output_yaml, result)
+  if output_json:
+    logging.info("Creating JSON output {}".format(output_json))
+    _generate_json_output(output_json, result)
   return result
 
 
@@ -330,13 +329,13 @@ def _main():
   parser.add_argument('--whitelist-file', dest='whitelist',
                       help='The path to the whitelist json file',
                       default='whitelist.json')
-  parser.add_argument('--output-yaml', dest='output_yaml',
-                      help='The path to the output YAML file to'+\
+  parser.add_argument('--output-json', dest='output_json',
+                      help='The path to the output JSON file to'+\
                       ' generate with a list of tags indicating the types of'+\
                       ' vulnerability fixes available for the given image.')
   args = parser.parse_args()
   security_check(args.image, args.severity, args.whitelist,
-                            args.output_yaml)
+                            args.output_json)
 
 
 if __name__ == '__main__':

docker/util/BUILD

@@ -21,11 +21,13 @@ licenses(["notice"])  # Apache 2.0
 py_binary(
     name = "config_stripper",
     srcs = ["config_stripper.py"],
+    python_version = "PY3",
 )
 
 py_binary(
     name = "to_json",
     srcs = ["to_json.py"],
+    python_version = "PY3",
 )
 
 exports_files([

docker/util/config_stripper.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import argparse
-import cStringIO
+import io
 import hashlib
 import json
 import os
@@ -39,6 +39,8 @@ def main():
                         required=True)
     args = parser.parse_args()
 
+    os.environ["PYTHONIOENCODING"] = "utf-8"
+
     return strip_tar(args.in_tar_path, args.out_tar_path)
 
 
@@ -101,14 +103,14 @@ def strip_layer(path):
     # working directory is one level up from where layer.tar is.
     original_dir = os.path.normpath(os.path.join(os.path.dirname(path), '..'))
 
-    buf = cStringIO.StringIO()
+    buf = io.BytesIO()
 
     # Go through each file/dir in the layer
     # Set its mtime to 0
     # If it's a file, add its content to the running buffer
     # Add it to the new gzip'd tar.
     with tarfile.open(name=path, mode='r') as it:
-      with tarfile.open(fileobj=buf, mode='w') as ot:
+      with tarfile.open(fileobj=buf, encoding='utf-8', mode='w') as ot:
         for tarinfo in it:
           # Use a deterministic mtime that doesn't confuse other programs,
           # e.g. Python.
@@ -137,7 +139,7 @@ def strip_layer(path):
     # Calculate sha of layer
     sha = hashlib.sha256(gz).hexdigest()
     new_name = 'sha256:%s' % sha
-    with open(os.path.join(original_dir, new_name), 'w') as out:
+    with open(os.path.join(original_dir, new_name), 'wb') as out:
       out.write(gz)
 
     shutil.rmtree(os.path.dirname(path))
@@ -169,7 +171,7 @@ def strip_config(path, new_diff_ids):
         f.write(config_str)
 
     # Calculate the new file path
-    sha = hashlib.sha256(config_str).hexdigest()
+    sha = hashlib.sha256(config_str.encode("utf-8")).hexdigest()
     new_path = 'sha256:%s' % sha
     os.rename(path, os.path.join(os.path.dirname(path), new_path))
     return new_path

python/image.bzl

@@ -29,10 +29,6 @@ load(
     "//repositories:go_repositories.bzl",
     _go_deps = "go_deps",
 )
-load(
-    "//toolchains:py_toolchains.bzl",
-    _py_toolchains = "py_toolchains",
-)
 
 # Load the resolved digests.
 load(":python.bzl", "DIGESTS")
@@ -46,13 +42,14 @@ def repositories():
     _go_deps()
 
     # Register the default py_toolchain / platform for containerized execution
-    if "container_py_toolchain" not in native.existing_rules().keys():
-        _py_toolchains(name = "container_py_toolchain")
     native.register_toolchains(
         "@io_bazel_rules_docker//toolchains:container_py_toolchain",
-        "@container_py_toolchain//:container_cc_toolchain",
+        "@bazel_tools//tools/python:autodetecting_toolchain",
+    )
+    native.register_execution_platforms(
+        "@local_config_platform//:host",
+        "@io_bazel_rules_docker//platforms:local_container_platform",
     )
-    native.register_execution_platforms("@io_bazel_rules_docker//platforms:local_container_platform")
 
     excludes = native.existing_rules().keys()
     if "py_image_base" not in excludes:

python3/image.bzl

@@ -28,10 +28,6 @@ load(
     "//repositories:go_repositories.bzl",
     _go_deps = "go_deps",
 )
-load(
-    "//toolchains:py_toolchains.bzl",
-    _py_toolchains = "py_toolchains",
-)
 
 # Load the resolved digests.
 load(":python3.bzl", "DIGESTS")
@@ -45,13 +41,14 @@ def repositories():
     _go_deps()
 
     # Register the default py_toolchain / platform for containerized execution
-    if "container_py_toolchain" not in native.existing_rules().keys():
-        _py_toolchains(name = "container_py_toolchain")
     native.register_toolchains(
         "@io_bazel_rules_docker//toolchains:container_py_toolchain",
-        "@container_py_toolchain//:container_cc_toolchain",
+        "@bazel_tools//tools/python:autodetecting_toolchain",
+    )
+    native.register_execution_platforms(
+        "@local_config_platform//:host",
+        "@io_bazel_rules_docker//platforms:local_container_platform",
     )
-    native.register_execution_platforms("@io_bazel_rules_docker//platforms:local_container_platform")
 
     excludes = native.existing_rules().keys()
     if "py3_image_base" not in excludes: