Add macro that faciliates "bazel run" of binary generated by rules_foreign_cc

.bazelci/config.yaml

@@ -92,6 +92,9 @@ tasks:
     macos_targets: &macos_targets
       - "//..."
       - "//:third_party_examples_macos_tests"
+      # Remove tests that depend on shared libraries, which currently doesn't work on sandboxed MacOS - https://github.com/bazelbuild/bazel/issues/10254
+      - "-@rules_foreign_cc_examples_third_party//curl:curl_test"
+      - "-@rules_foreign_cc_examples_third_party//openssl:openssl_test"
     build_targets: *macos_targets
     build_flags:
       - "--noincompatible_enable_cc_toolchain_resolution"

examples/third_party/openssl/BUILD.bazel

@@ -25,14 +25,25 @@ build_test(
     name = "build_test",
     targets = [
         "@openssl//:openssl",
+        "@openssl//:runnable_openssl",
     ],
     visibility = ["//:__pkg__"],
 )
 
+sh_test(
+    name = "openssl_launch_test",
+    srcs = ["openssl_test.sh"],
+    data = ["@openssl//:runnable_openssl"],
+    env = {
+        "OPENSSL": "$(rootpath @openssl//:runnable_openssl)",
+    },
+)
+
 test_suite(
     name = "openssl_test_suite",
     tests = [
         ":build_test",
+        ":openssl_launch_test",
         ":openssl_test",
     ],
     visibility = ["//:__pkg__"],

examples/third_party/openssl/BUILD.openssl.bazel

@@ -5,7 +5,7 @@ Note that the $(PERL) "make variable" (https://docs.bazel.build/versions/main/be
 is populated by the perl toolchain provided by rules_perl.
 """
 
-load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make", "configure_make_variant")
+load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make", "configure_make_variant", "runnable_binary")
 
 # Read https://wiki.openssl.org/index.php/Compilation_and_Installation
 
@@ -21,14 +21,13 @@ CONFIGURE_OPTIONS = [
     "no-comp",
     "no-idea",
     "no-weak-ssl-ciphers",
-    "no-shared",
 ]
 
 LIB_NAME = "openssl"
 
 MAKE_TARGETS = [
-    "build_libs",
-    "install_dev",
+    "build_programs",
+    "install_sw",
 ]
 
 config_setting(
@@ -65,15 +64,20 @@ configure_make_variant(
     env = {
         # The Zi flag must be set otherwise OpenSSL fails to build due to missing .pdb files
         "CFLAGS": "-Zi",
-        "PATH": "$$(dirname $(execpath @nasm//:nasm)):$$PATH",
+        "PATH": "$$(dirname $(execpath @nasm//:nasm))",
         "PERL": "$$EXT_BUILD_ROOT$$/$(PERL)",
     },
     lib_name = LIB_NAME,
     lib_source = ":all_srcs",
-    out_static_libs = [
+    out_binaries = ["openssl.exe"],
+    out_interface_libs = [
         "libssl.lib",
         "libcrypto.lib",
     ],
+    out_shared_libs = [
+        "libssl-1_1-x64.dll",
+        "libcrypto-1_1-x64.dll",
+    ],
     targets = MAKE_TARGETS,
     toolchain = "@rules_foreign_cc//toolchains:preinstalled_nmake_toolchain",
     toolchains = ["@rules_perl//:current_toolchain"],
@@ -95,12 +99,19 @@ configure_make(
     }),
     lib_name = LIB_NAME,
     lib_source = ":all_srcs",
+    out_binaries = ["openssl"],
     # Note that for Linux builds, libssl must come before libcrypto on the linker command-line.
     # As such, libssl must be listed before libcrypto
-    out_static_libs = [
-        "libssl.a",
-        "libcrypto.a",
-    ],
+    out_shared_libs = select({
+        "@platforms//os:macos": [
+            "libssl.1.1.dylib",
+            "libcrypto.1.1.dylib",
+        ],
+        "//conditions:default": [
+            "libssl.so.1.1",
+            "libcrypto.so.1.1",
+        ],
+    }),
     targets = MAKE_TARGETS,
     toolchains = ["@rules_perl//:current_toolchain"],
 )
@@ -111,3 +122,13 @@ filegroup(
     output_group = "gen_dir",
     visibility = ["//visibility:public"],
 )
+
+runnable_binary(
+    name = "runnable_openssl",
+    binary = select({
+        "@platforms//os:windows": "openssl.exe",
+        "//conditions:default": "openssl",
+    }),
+    foreign_cc_target = "@openssl//:openssl",
+    visibility = ["//visibility:public"],
+)

examples/third_party/openssl/openssl_repositories.bzl

@@ -8,8 +8,8 @@ def openssl_repositories():
         http_archive,
         name = "openssl",
         build_file = Label("//openssl:BUILD.openssl.bazel"),
-        sha256 = "0f745b85519aab2ce444a3dcada93311ba926aea2899596d01e7f948dbd99981",
-        strip_prefix = "openssl-OpenSSL_1_1_1o",
+        sha256 = "9384a2b0570dd80358841464677115df785edb941c71211f75076d72fe6b438f",
+        strip_prefix = "openssl-1.1.1o",
         urls = [
             "https://mirror.bazel.build/www.openssl.org/source/openssl-1.1.1o.tar.gz",
             "https://www.openssl.org/source/openssl-1.1.1o.tar.gz",

examples/third_party/openssl/openssl_test.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [[ ! -e "$OPENSSL" ]]; then
+    echo "openssl does not exist"
+    exit 1
+fi
+
+exec $OPENSSL help

foreign_cc/defs.bzl

@@ -5,6 +5,7 @@ load(":cmake.bzl", _cmake = "cmake", _cmake_variant = "cmake_variant")
 load(":configure.bzl", _configure_make = "configure_make", _configure_make_variant = "configure_make_variant")
 load(":make.bzl", _make = "make", _make_variant = "make_variant")
 load(":ninja.bzl", _ninja = "ninja")
+load(":utils.bzl", _runnable_binary = "runnable_binary")
 
 boost_build = _boost_build
 cmake = _cmake
@@ -14,3 +15,4 @@ configure_make_variant = _configure_make_variant
 make_variant = _make_variant
 make = _make
 ninja = _ninja
+runnable_binary = _runnable_binary

foreign_cc/private/BUILD.bazel

@@ -1,5 +1,9 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
 
+exports_files([
+    "runnable_binary_wrapper.sh",
+])
+
 bzl_library(
     name = "bzl_srcs",
     srcs = glob(["**/*.bzl"]),

foreign_cc/private/framework.bzl

@@ -315,12 +315,6 @@ def get_env_prelude(ctx, lib_name, data_dependencies, target_root):
             "export CMAKE_OSX_ARCHITECTURES={}".format(ctx.fragments.apple.single_arch_cpu),
         ])
 
-    cc_toolchain = find_cpp_toolchain(ctx)
-    if cc_toolchain.compiler == "msvc-cl":
-        # Prepend PATH environment variable with the path to the toolchain linker, which prevents MSYS using its linker (/usr/bin/link.exe) rather than the MSVC linker (both are named "link.exe")
-        linker_path = paths.dirname(cc_toolchain.ld_executable)
-        env.update({"PATH": _normalize_path(linker_path) + ":" + env.get("PATH")})
-
     # Add all user defined variables
     user_vars = expand_locations_and_make_variables(ctx, ctx.attr.env, "env", data_dependencies)
     env.update(user_vars)
@@ -330,6 +324,12 @@ def get_env_prelude(ctx, lib_name, data_dependencies, target_root):
         if "PATH" in user_var and cc_env.get(user_var):
             env.update({user_var: user_vars.get(user_var) + ":" + cc_env.get(user_var)})
 
+    cc_toolchain = find_cpp_toolchain(ctx)
+    if cc_toolchain.compiler == "msvc-cl":
+        # Prepend PATH environment variable with the path to the toolchain linker, which prevents MSYS using its linker (/usr/bin/link.exe) rather than the MSVC linker (both are named "link.exe")
+        linker_path = paths.dirname(cc_toolchain.ld_executable)
+        env.update({"PATH": _normalize_path(linker_path) + ":" + env.get("PATH")})
+
     env_snippet.extend(["export {}=\"{}\"".format(key, escape_dquote_bash(val)) for key, val in env.items()])
 
     return env_snippet
@@ -496,7 +496,7 @@ def cc_external_rule_impl(ctx, attrs):
 
     # Gather runfiles transitively as per the documentation in:
     # https://docs.bazel.build/versions/master/skylark/rules.html#runfiles
-    runfiles = ctx.runfiles(files = ctx.files.data)
+    runfiles = ctx.runfiles(files = ctx.files.data + outputs.libraries.shared_libraries)
     for target in [ctx.attr.lib_source] + ctx.attr.deps + ctx.attr.data:
         runfiles = runfiles.merge(target[DefaultInfo].default_runfiles)
 

foreign_cc/private/runnable_binary_wrapper.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# --- begin runfiles.bash initialization v2 ---
+# Copy-pasted from the Bazel Bash runfiles library v2. (@bazel_tools//tools/bash/runfiles)
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+source "$0.runfiles/$f" 2>/dev/null || \
+source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+{ echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+    SHARED_LIB_SUFFIX=".so*"
+    LIB_PATH_VAR=LD_LIBRARY_PATH
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+    SHARED_LIB_SUFFIX=".dylib"
+    LIB_PATH_VAR=DYLD_LIBRARY_PATH
+elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
+    SHARED_LIB_SUFFIX=".dll"
+    LIB_PATH_VAR=PATH
+fi
+
+# Add paths to shared libraries to SHARED_LIBS_ARRAY
+SHARED_LIBS_ARRAY=()
+while IFS=  read -r -d $'\0'; do
+    SHARED_LIBS_ARRAY+=("$REPLY")
+done < <(find . -name "*${SHARED_LIB_SUFFIX}" -print0)
+
+# Add paths to shared library directories to SHARED_LIBS_DIRS_ARRAY
+SHARED_LIBS_DIRS_ARRAY=()
+for lib in "${SHARED_LIBS_ARRAY[@]}"; do
+    SHARED_LIBS_DIRS_ARRAY+=($(dirname $(realpath $lib)))
+done
+
+# Remove duplicates from array
+IFS=" " read -r -a SHARED_LIBS_DIRS_ARRAY <<< "$(tr ' ' '\n' <<< "${SHARED_LIBS_DIRS_ARRAY[@]}" | sort -u | tr '\n' ' ')"
+
+# Allow unbound variable here, in case LD_LIBRARY_PATH or similar is not already set
+set +u
+for dir in "${SHARED_LIBS_DIRS_ARRAY[@]}"; do
+    export ${LIB_PATH_VAR}="${!LIB_PATH_VAR}":"$dir"
+done
+set -u
+
+EXE=BIN
+$(rlocation "${EXE#external/}") $@

foreign_cc/utils.bzl

@@ -0,0 +1,48 @@
+""" This file contains useful utilities """
+
+def _full_label(label):
+    return native.repository_name() + "//" + native.package_name() + ":" + label
+
+def runnable_binary(name, binary, foreign_cc_target, **kwargs):
+    """
+    Macro that provides a wrapper script around a binary generated by a rules_foreign_cc rule that can be run using "bazel run".
+
+    The wrapper script also facilitates the running of binaries that are dynamically linked to shared libraries also built by rules_foreign_cc. The runnable bin could be used as a tool in a dependent bazel target
+
+    Note that this macro only works on foreign_cc_targets in external repositories, not in the main repository. This is due to the issue described here: https://github.com/bazelbuild/bazel/issues/10923
+    Also note that the macro requires the `--enable_runfiles` option to be set on Windows.
+
+    Args:
+        name: The target name
+        binary: The name of the binary generated by rules_foreign_cc
+        foreign_cc_target: The target that generates the binary
+        **kwargs: Remaining keyword arguments
+    """
+
+    tags = kwargs.pop("tags", [])
+
+    native.filegroup(
+        name = name + "_fg",
+        srcs = [foreign_cc_target],
+        tags = tags + ["manual"],
+        output_group = binary,
+    )
+
+    native.genrule(
+        name = name + "_wrapper",
+        srcs = ["@rules_foreign_cc//foreign_cc/private:runnable_binary_wrapper.sh", name + "_fg"],
+        outs = [name + "_wrapper.sh"],
+        cmd = "sed s@BIN@$(rootpath {})@g $(location @rules_foreign_cc//foreign_cc/private:runnable_binary_wrapper.sh) > $@".format(_full_label(name + "_fg")),
+        tags = tags + ["manual"],
+    )
+
+    native.sh_binary(
+        name = name,
+        deps = ["@bazel_tools//tools/bash/runfiles"],
+        srcs = [name + "_wrapper"],
+        data = [
+            name + "_fg",
+            foreign_cc_target,
+        ],
+        **kwargs
+    )