From 3a1e1c9d65e54008b2ceec8fed9b5b94e5ec3763 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Mon, 27 Feb 2023 15:06:54 -0800 Subject: [PATCH] Modify release artifacts for protoc to statically link system libraries. Closes #12063 PiperOrigin-RevId: 512742784 --- .github/workflows/test_cpp.yml | 10 ++++---- .github/workflows/test_csharp.yml | 2 +- .github/workflows/test_java.yml | 6 ++--- .github/workflows/test_php.yml | 4 +++- .github/workflows/test_python.yml | 6 ++--- .github/workflows/test_ruby.yml | 5 ++-- BUILD.bazel | 4 ++-- build_defs/cpp_opts.bzl | 7 ------ src/google/protobuf/compiler/BUILD.bazel | 4 ++-- toolchain/cc_toolchain_config.bzl | 1 + toolchain/toolchains.bazelrc | 30 ++++++++++++++---------- 11 files changed, 41 insertions(+), 38 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 1a63692a59dc..d9faca0d90ff 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -23,19 +23,19 @@ jobs: - { name: No-RTTI, flags: --cxxopt=-fno-rtti } include: # Set defaults - - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:dbd2f15fb69734d72c3fd10cb819bbe2ce4890acf49e9a2f9403983fe48e8807 + - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:309dae3122031447d714414814d262e5f31cb93c0e248e9c02f9d8cdafd7e3b9 - targets: //pkg/... //src/... @com_google_protobuf_examples//... # Override cases with custom images - config: { name: "TCMalloc" } - image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:9d975616c3fd44d5a091aeb60ee94f37e22fb367d471d258fc18cb4a2387c943" + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:4df3b4749e787ba0a671ec0b783d0f1ba05f60be4c9e9fd72c875550a0cde1ea" targets: "//src/..." - - config: { name: "aarch64" } + - config: { name: "aarch64", flags: "--linkopt='-static'" } targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test" - image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b" + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-eb065d690a4832bb0abc630b13f1a690242bf0ae" - config: { name: "Bazel4" } targets: "//src/..." - image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-3b71de326b62f67bf754c4dc4016d6a2fa9dd664" + image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-ed17d84c969effa6cbf32f533bed71c8109f5a3f" name: Linux ${{ matrix.config.name }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index d85867f3e9fb..58f88a71d566 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -20,7 +20,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-65526ea124d1034eac33e7c37cc6d65c5bef054f + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-eb065d690a4832bb0abc630b13f1a690242bf0ae credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: csharp_linux bazel: test //csharp/... --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=/home/bazel diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index bb6ea697e90a..9c9a9d2b2b1a 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -16,15 +16,15 @@ jobs: include: - name: OpenJDK 11 version: '11' - image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-ed17d84c969effa6cbf32f533bed71c8109f5a3f targets: //java/... //java/internal:java_version - name: OpenJDK 17 version: '17' - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-65526ea124d1034eac33e7c37cc6d65c5bef054f + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-eb065d690a4832bb0abc630b13f1a690242bf0ae targets: //java/... //java/internal:java_version - name: aarch64 version: 'aarch64' - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-eb065d690a4832bb0abc630b13f1a690242bf0ae targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test name: Linux ${{ matrix.name }} diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 4b9a6d02efa4..221a1da75e1b 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -45,7 +45,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-eb065d690a4832bb0abc630b13f1a690242bf0ae credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: ${{ matrix.command }} @@ -80,6 +80,7 @@ jobs: id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-ed17d84c969effa6cbf32f533bed71c8109f5a3f credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-i386 @@ -111,6 +112,7 @@ jobs: id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-ed17d84c969effa6cbf32f533bed71c8109f5a3f credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 13e5575b24c2..04185b58e375 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -26,8 +26,8 @@ jobs: version: aarch64 targets: //python/... //python:aarch64_test # TODO(b/262628111) Enable this once conformance tests are fixed. - flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b + flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance --linkopt='-static' + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-eb065d690a4832bb0abc630b13f1a690242bf0ae name: Linux ${{ matrix.type }} ${{ matrix.version }} runs-on: ubuntu-latest @@ -39,7 +39,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: - image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }} + image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-eb065d690a4832bb0abc630b13f1a690242bf0ae', matrix.version) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }} bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 0f9392f526dd..c5cf04c63438 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -34,7 +34,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: - image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }} + image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-eb065d690a4832bb0abc630b13f1a690242bf0ae', matrix.ruby, matrix.bazel) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }} bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION @@ -52,6 +52,7 @@ jobs: id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-ed17d84c969effa6cbf32f533bed71c8109f5a3f credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 @@ -124,7 +125,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-eb065d690a4832bb0abc630b13f1a690242bf0ae credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }} bash: > diff --git a/BUILD.bazel b/BUILD.bazel index 69a1ba96c86a..568a1b576985 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -4,7 +4,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library") load("@rules_java//java:defs.bzl", "java_lite_proto_library", "java_proto_library") -load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") load(":protobuf.bzl", "internal_objc_proto_library", "internal_php_proto_library", "internal_py_proto_library", "internal_ruby_proto_library") licenses(["notice"]) @@ -168,7 +168,7 @@ alias( cc_binary( name = "protoc", copts = COPTS, - linkopts = LINK_OPTS + PROTOC_LINK_OPTS, + linkopts = LINK_OPTS, visibility = ["//visibility:public"], deps = ["//src/google/protobuf/compiler:protoc_lib"], ) diff --git a/build_defs/cpp_opts.bzl b/build_defs/cpp_opts.bzl index 33e157875bd1..f95c3e257902 100644 --- a/build_defs/cpp_opts.bzl +++ b/build_defs/cpp_opts.bzl @@ -41,10 +41,3 @@ LINK_OPTS = select({ "-lm", ], }) - -# When cross-compiling for Windows we need to statically link pthread and the C++ library. -PROTOC_LINK_OPTS = select({ - "//build_defs:config_win32": ["-static"], - "//build_defs:config_win64": ["-static"], - "//conditions:default": [], -}) diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel index b1b544cede80..b8d9e2d3db40 100644 --- a/src/google/protobuf/compiler/BUILD.bazel +++ b/src/google/protobuf/compiler/BUILD.bazel @@ -11,7 +11,7 @@ load( ) load("@rules_proto//proto:defs.bzl", "proto_library") load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test") -load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") proto_library( name = "plugin_proto", @@ -123,7 +123,7 @@ cc_library( cc_binary( name = "protoc_nowkt", copts = COPTS, - linkopts = LINK_OPTS + PROTOC_LINK_OPTS, + linkopts = LINK_OPTS, visibility = [ "//src/google/protobuf:__pkg__", ], diff --git a/toolchain/cc_toolchain_config.bzl b/toolchain/cc_toolchain_config.bzl index b610a8ee9d96..3bfe1bfa9337 100644 --- a/toolchain/cc_toolchain_config.bzl +++ b/toolchain/cc_toolchain_config.bzl @@ -98,6 +98,7 @@ def _impl(ctx): flags = [ "-B" + ctx.attr.linker_path, "-lstdc++", + "-lm", "--target=" + ctx.attr.target_full_name, ] + ctx.attr.extra_linker_flags, ), diff --git a/toolchain/toolchains.bazelrc b/toolchain/toolchains.bazelrc index 908a26e12ea0..838a5ce54263 100644 --- a/toolchain/toolchains.bazelrc +++ b/toolchain/toolchains.bazelrc @@ -1,15 +1,21 @@ build:cross_config --crosstool_top=//toolchain:clang_suite build:cross_config --host_crosstool_top=@bazel_tools//tools/cpp:toolchain -build:linux-aarch_64 --config=cross_config --cpu=linux-aarch_64 -build:linux-aarch64 --config=cross_config --cpu=linux-aarch_64 -build:linux-ppcle_64 --config=cross_config --cpu=linux-ppcle_64 -build:linux-ppc64le --config=cross_config --cpu=linux-ppcle_64 -build:linux-s390_64 --config=cross_config --cpu=linux-s390_64 -build:linux-x86_32 --config=cross_config --cpu=linux-x86_32 -build:linux-i386 --config=cross_config --cpu=linux-x86_32 -build:linux-x86_64 --config=cross_config --cpu=linux-x86_64 -build:osx-aarch_64 --config=cross_config --action_env=MACOSX_DEPLOYMENT_TARGET=10.9 --cpu=osx-aarch_64 -build:osx-x86_64 --config=cross_config --action_env=MACOSX_DEPLOYMENT_TARGET=10.9 --cpu=osx-x86_64 -build:win32 --config=cross_config --cpu=win32 -build:win64 --config=cross_config --cpu=win64 +# When cross-compiling we need to statically link all C++ libraries. +build:cross_linux --config=cross_config --linkopt="-static" +build:cross_windows --config=cross_config --linkopt="-static" +# This isn't possible on mac because there is no static library for lcrt0.o +build:cross_osx --config=cross_config + +build:linux-aarch_64 --config=cross_linux --cpu=linux-aarch_64 +build:linux-aarch64 --config=cross_linux --cpu=linux-aarch_64 +build:linux-ppcle_64 --config=cross_linux --cpu=linux-ppcle_64 +build:linux-ppc64le --config=cross_linux --cpu=linux-ppcle_64 +build:linux-s390_64 --config=cross_linux --cpu=linux-s390_64 +build:linux-x86_32 --config=cross_linux --cpu=linux-x86_32 +build:linux-i386 --config=cross_linux --cpu=linux-x86_32 +build:linux-x86_64 --config=cross_linux --cpu=linux-x86_64 +build:osx-aarch_64 --config=cross_osx --action_env=MACOSX_DEPLOYMENT_TARGET=10.9 --cpu=osx-aarch_64 +build:osx-x86_64 --config=cross_osx --action_env=MACOSX_DEPLOYMENT_TARGET=10.9 --cpu=osx-x86_64 +build:win32 --config=cross_windows --cpu=win32 +build:win64 --config=cross_windows --cpu=win64