diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 2f4df028c..4848158e8 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -71,6 +71,7 @@ platforms: # For tests/container:set_env_make_vars_test - "--define=ENV_KEY=my_key" - "--define=ENV_VALUE=my_value" + - "--extra_execution_platforms=@local_config_platform//:host,@io_bazel_rules_docker//platforms:local_container_platform" test_targets: - "--" - "//tests/container:alpine_custom_attr_digest_test" @@ -111,6 +112,7 @@ platforms: - "--define=ENV_KEY=my_key" - "--define=ENV_VALUE=my_value" - "--test_output=errors" + - "--extra_execution_platforms=@local_config_platform//:host,@io_bazel_rules_docker//platforms:local_container_platform" rbe_ubuntu1604: build_targets: - "--" diff --git a/.bazelrc b/.bazelrc index 056d9a7b2..a607b74f9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -15,7 +15,6 @@ # The following flags are set to test use of new features for python toolchains # These flags will only work with Bazel 0.25.0 or above and are only needed for # //tests/docker/security/... & the docker/package_managers rules. -build --host_force_python=PY2 build --incompatible_use_python_toolchains test --incompatible_use_python_toolchains -test --host_force_python=PY2 + diff --git a/WORKSPACE b/WORKSPACE index 771d373be..ae91a8abe 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -401,16 +401,15 @@ dockerfile_image( ]] # Register the default py_toolchain / platform for containerized execution -load("//toolchains:py_toolchains.bzl", "py_toolchains") - -py_toolchains(name = "container_py_toolchain") - register_toolchains( "//toolchains:container_py_toolchain", - "@container_py_toolchain//:container_cc_toolchain", + "@bazel_tools//tools/python:autodetecting_toolchain", ) -register_execution_platforms("//platforms:local_container_platform") +register_execution_platforms( + "@local_config_platform//:host", + "//platforms:local_container_platform", +) http_archive( name = "bazel_toolchains", diff --git a/docker/security/cmd/json_to_yaml/BUILD b/docker/security/cmd/json_to_yaml/BUILD new file mode 100644 index 000000000..023770c7b --- /dev/null +++ b/docker/security/cmd/json_to_yaml/BUILD @@ -0,0 +1,28 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["json_to_yaml.go"], + importpath = "github.com/bazelbuild/rules_docker/docker/security/cmd/json_to_yaml", + visibility = ["//visibility:private"], + deps = ["@com_github_ghodss_yaml//:go_default_library"], +) + +go_binary( + name = "json_to_yaml", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/docker/security/cmd/json_to_yaml/json_to_yaml.go b/docker/security/cmd/json_to_yaml/json_to_yaml.go new file mode 100644 index 000000000..3bfbc7415 --- /dev/null +++ b/docker/security/cmd/json_to_yaml/json_to_yaml.go @@ -0,0 +1,50 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "flag" + "io/ioutil" + "log" + "os" + + "github.com/ghodss/yaml" +) + +var ( + inJSON = flag.String("in-json", "", "Path to input JSON file that will be converted to YAML.") + outYAML = flag.String("out-yaml", "", "Path to output YAML file.") +) + +func main() { + flag.Parse() + if *inJSON == "" { + log.Fatalf("--in-json is required.") + } + if *outYAML == "" { + log.Fatalf("--out-yaml is required.") + } + + j, err := ioutil.ReadFile(*inJSON) + if err != nil { + log.Fatalf("Unable to read input JSON file %q: %v", *inJSON, err) + } + y, err := yaml.JSONToYAML(j) + if err != nil { + log.Fatalf("Unable to convert JSON data loaded from %q to YAML: %v", *inJSON, err) + } + if err := ioutil.WriteFile(*outYAML, y, os.ModePerm); err != nil { + log.Fatalf("Unable to write output YAML to %q: %v", *outYAML, err) + } +} diff --git a/docker/security/security_check.bzl b/docker/security/security_check.bzl index 68f7a7107..905f7dfe9 100644 --- a/docker/security/security_check.bzl +++ b/docker/security/security_check.bzl @@ -19,7 +19,7 @@ def _impl(ctx): output_yaml = ctx.outputs.yaml args = ctx.actions.args() args.add(ctx.attr.image) - args.add("--output-yaml", ctx.outputs.yaml) + args.add("--output-json", ctx.outputs.json) args.add("--severity", ctx.attr.severity) if ctx.attr.whitelist != None: files = ctx.attr.whitelist.files.to_list() @@ -35,7 +35,7 @@ def _impl(ctx): ctx.actions.run( executable = ctx.executable._security_check, arguments = [args], - outputs = [ctx.outputs.yaml], + outputs = [ctx.outputs.json], mnemonic = "ImageSecurityCheck", use_default_shell_env = True, execution_requirements = { @@ -46,6 +46,16 @@ def _impl(ctx): "no-sandbox": "True", }, ) + args = ctx.actions.args() + args.add("--in-json", ctx.outputs.json) + args.add("--out-yaml", ctx.outputs.yaml) + ctx.actions.run( + executable = ctx.executable._json_to_yaml, + arguments = [args], + inputs = [ctx.outputs.json], + outputs = [ctx.outputs.yaml], + mnemonic = "JSONToYAML", + ) # Run the security_check.py script on the given docker image to generate a # YAML output file with information about the types of vulnerabilities @@ -68,6 +78,13 @@ security_check = rule( default = Label("@io_bazel_rules_docker//docker/security:security_check_whitelist.json"), allow_single_file = True, ), + # JSON to YAML converter. + "_json_to_yaml": attr.label( + default = Label("@io_bazel_rules_docker//docker/security/cmd/json_to_yaml"), + cfg = "host", + executable = True, + allow_files = True, + ), # The security checker python executable. "_security_check": attr.label( default = Label("@io_bazel_rules_docker//docker/security:security_check"), @@ -77,6 +94,7 @@ security_check = rule( ), }, outputs = { + "json": "%{name}.json", "yaml": "%{name}.yaml", }, ) diff --git a/docker/security/security_check.py b/docker/security/security_check.py index 15a3dc471..701d1f2f3 100644 --- a/docker/security/security_check.py +++ b/docker/security/security_check.py @@ -19,7 +19,6 @@ import subprocess import sys import logging -import yaml import distutils.version as ver @@ -265,17 +264,17 @@ def _get_version_number(version_obj): return ''.join([str(epoch), delimiter1, name, delimiter2, str(revision)]) -def _generate_yaml_output(output_yaml, vulnerabilities): - """Generate a YAML file mapping the key "tags" to the list of types of +def _generate_json_output(output_json, vulnerabilities): + """Generate a JSON file mapping the key "tags" to the list of types of vulnerabilities found. Args: - output_yaml: Path to the output YAML file to generate. + output_json: Path to the output JSON file to generate. vulnerabilities: A dictionary mapping the name of the CVE entry to details about the vulnerability. """ tags = set() - for v in vulnerabilities.itervalues(): + for v in vulnerabilities.values(): details = v["vulnerabilityDetails"] # The service that consumes the metadata expects the tags as follows: # LOW -> cveLow @@ -284,19 +283,19 @@ def _generate_yaml_output(output_yaml, vulnerabilities): sev = str(details['severity']) tags.add("cve{}".format(sev.lower().capitalize())) result = {"tags": list(tags)} - logging.info("Creating YAML output {}".format(output_yaml)) - with open(output_yaml, "w") as ofp: - ofp.write(yaml.dump(result)) + logging.info("Creating JSON output {}".format(output_json)) + with open(output_json, "w") as ofp: + json.dump(result, ofp) def security_check(image, severity=_MEDIUM, whitelist_file='whitelist.json', - output_yaml=None): + output_json=None): """Main security check function. Args: image: full name of the docker image severity: the severity of vulnerability to trigger failure whitelist_file: file with list of whitelisted CVE - output_yaml: Output file which will be populated with a list of types of + output_json: Output file which will be populated with a list of types of vulnerability that exist for the given image. Returns: @@ -312,9 +311,9 @@ def security_check(image, severity=_MEDIUM, whitelist_file='whitelist.json', result = _check_for_vulnz(_sub_image(image), severity, whitelist) - if output_yaml: - logging.info("Creating YAML output {}".format(output_yaml)) - _generate_yaml_output(output_yaml, result) + if output_json: + logging.info("Creating JSON output {}".format(output_json)) + _generate_json_output(output_json, result) return result @@ -330,13 +329,13 @@ def _main(): parser.add_argument('--whitelist-file', dest='whitelist', help='The path to the whitelist json file', default='whitelist.json') - parser.add_argument('--output-yaml', dest='output_yaml', - help='The path to the output YAML file to'+\ + parser.add_argument('--output-json', dest='output_json', + help='The path to the output JSON file to'+\ ' generate with a list of tags indicating the types of'+\ ' vulnerability fixes available for the given image.') args = parser.parse_args() security_check(args.image, args.severity, args.whitelist, - args.output_yaml) + args.output_json) if __name__ == '__main__': diff --git a/docker/util/BUILD b/docker/util/BUILD index a7f038d36..6a5454582 100644 --- a/docker/util/BUILD +++ b/docker/util/BUILD @@ -21,11 +21,13 @@ licenses(["notice"]) # Apache 2.0 py_binary( name = "config_stripper", srcs = ["config_stripper.py"], + python_version = "PY3", ) py_binary( name = "to_json", srcs = ["to_json.py"], + python_version = "PY3", ) exports_files([ diff --git a/docker/util/config_stripper.py b/docker/util/config_stripper.py index e88259806..739cdc770 100644 --- a/docker/util/config_stripper.py +++ b/docker/util/config_stripper.py @@ -15,7 +15,7 @@ # limitations under the License. import argparse -import cStringIO +import io import hashlib import json import os @@ -39,6 +39,8 @@ def main(): required=True) args = parser.parse_args() + os.environ["PYTHONIOENCODING"] = "utf-8" + return strip_tar(args.in_tar_path, args.out_tar_path) @@ -101,14 +103,14 @@ def strip_layer(path): # working directory is one level up from where layer.tar is. original_dir = os.path.normpath(os.path.join(os.path.dirname(path), '..')) - buf = cStringIO.StringIO() + buf = io.BytesIO() # Go through each file/dir in the layer # Set its mtime to 0 # If it's a file, add its content to the running buffer # Add it to the new gzip'd tar. with tarfile.open(name=path, mode='r') as it: - with tarfile.open(fileobj=buf, mode='w') as ot: + with tarfile.open(fileobj=buf, encoding='utf-8', mode='w') as ot: for tarinfo in it: # Use a deterministic mtime that doesn't confuse other programs, # e.g. Python. @@ -137,7 +139,7 @@ def strip_layer(path): # Calculate sha of layer sha = hashlib.sha256(gz).hexdigest() new_name = 'sha256:%s' % sha - with open(os.path.join(original_dir, new_name), 'w') as out: + with open(os.path.join(original_dir, new_name), 'wb') as out: out.write(gz) shutil.rmtree(os.path.dirname(path)) @@ -169,7 +171,7 @@ def strip_config(path, new_diff_ids): f.write(config_str) # Calculate the new file path - sha = hashlib.sha256(config_str).hexdigest() + sha = hashlib.sha256(config_str.encode("utf-8")).hexdigest() new_path = 'sha256:%s' % sha os.rename(path, os.path.join(os.path.dirname(path), new_path)) return new_path diff --git a/python/image.bzl b/python/image.bzl index 662c95cbb..5c9baa887 100644 --- a/python/image.bzl +++ b/python/image.bzl @@ -29,10 +29,6 @@ load( "//repositories:go_repositories.bzl", _go_deps = "go_deps", ) -load( - "//toolchains:py_toolchains.bzl", - _py_toolchains = "py_toolchains", -) # Load the resolved digests. load(":python.bzl", "DIGESTS") @@ -46,13 +42,14 @@ def repositories(): _go_deps() # Register the default py_toolchain / platform for containerized execution - if "container_py_toolchain" not in native.existing_rules().keys(): - _py_toolchains(name = "container_py_toolchain") native.register_toolchains( "@io_bazel_rules_docker//toolchains:container_py_toolchain", - "@container_py_toolchain//:container_cc_toolchain", + "@bazel_tools//tools/python:autodetecting_toolchain", + ) + native.register_execution_platforms( + "@local_config_platform//:host", + "@io_bazel_rules_docker//platforms:local_container_platform", ) - native.register_execution_platforms("@io_bazel_rules_docker//platforms:local_container_platform") excludes = native.existing_rules().keys() if "py_image_base" not in excludes: diff --git a/python3/image.bzl b/python3/image.bzl index 24eeb911e..b7673aee8 100644 --- a/python3/image.bzl +++ b/python3/image.bzl @@ -28,10 +28,6 @@ load( "//repositories:go_repositories.bzl", _go_deps = "go_deps", ) -load( - "//toolchains:py_toolchains.bzl", - _py_toolchains = "py_toolchains", -) # Load the resolved digests. load(":python3.bzl", "DIGESTS") @@ -45,13 +41,14 @@ def repositories(): _go_deps() # Register the default py_toolchain / platform for containerized execution - if "container_py_toolchain" not in native.existing_rules().keys(): - _py_toolchains(name = "container_py_toolchain") native.register_toolchains( "@io_bazel_rules_docker//toolchains:container_py_toolchain", - "@container_py_toolchain//:container_cc_toolchain", + "@bazel_tools//tools/python:autodetecting_toolchain", + ) + native.register_execution_platforms( + "@local_config_platform//:host", + "@io_bazel_rules_docker//platforms:local_container_platform", ) - native.register_execution_platforms("@io_bazel_rules_docker//platforms:local_container_platform") excludes = native.existing_rules().keys() if "py3_image_base" not in excludes: diff --git a/toolchains/py_toolchains.bzl b/toolchains/py_toolchains.bzl deleted file mode 100644 index e07de1c58..000000000 --- a/toolchains/py_toolchains.bzl +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2017 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Repo rule to register toolchains required for py_image and py3_image rules. -""" - -load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "get_cpu_value", - "resolve_labels", -) -load("@bazel_tools//tools/osx:xcode_configure.bzl", "run_xcode_locator") - -def _impl(repository_ctx): - """Core implementation of _py_toolchains.""" - - cpu_value = get_cpu_value(repository_ctx) - env = repository_ctx.os.environ - if "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN" in env and env["BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN"] == "1": - # Create an alias to the bazel_tools local toolchain as no cpp toolchain will be produced - repository_ctx.file("BUILD", content = ("""# Alias to local toolchain -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) # Apache 2.0 - -alias( - name = "container_cc_toolchain", - actual = "@bazel_tools//tools/cpp:cc-toolchain-local", -) - -"""), executable = False) - - else: - if cpu_value == "x64_windows": - # Note this is not well tested - cpu_value = "x64_windows_msys" - toolchain = "@local_config_cc//:cc-compiler-%s" % cpu_value - if cpu_value == "darwin": - # This needs to be carefully kept in sync with bazel/tools/cpp/cc_configure.bzl - should_use_xcode = "BAZEL_USE_XCODE_TOOLCHAIN" in env and env["BAZEL_USE_XCODE_TOOLCHAIN"] == "1" - xcode_toolchains = [] - paths = resolve_labels(repository_ctx, [ - "@bazel_tools//tools/osx:xcode_locator.m", - ]) - if not should_use_xcode: - (xcode_toolchains, _xcodeloc_err) = run_xcode_locator( - repository_ctx, - paths["@bazel_tools//tools/osx:xcode_locator.m"], - ) - if should_use_xcode or xcode_toolchains: - toolchain = "@local_config_cc//:cc-compiler-darwin_x86_64" - else: - toolchain = "@local_config_cc//:cc-compiler-darwin" - - repository_ctx.file("BUILD", content = ("""# Toolchain required for xx_image targets that rely on xx_binary -# which transitively require a C/C++ toolchain (currently only -# py_binary). This one is for local execution and will be required -# with versions of Bazel > 1.0.0 -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) # Apache 2.0 - -load("@local_config_platform//:constraints.bzl", "HOST_CONSTRAINTS") - -toolchain( - name = "container_cc_toolchain", - exec_compatible_with = HOST_CONSTRAINTS + ["@io_bazel_rules_docker//platforms:run_in_container"], - target_compatible_with = HOST_CONSTRAINTS, - toolchain = "%s", - toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", -) -""") % toolchain, executable = False) - -py_toolchains = repository_rule( - attrs = {}, - implementation = _impl, -)