tools/test: adding tools for splitting up mock headers (#12150)

We need to reduce resource consumption of test compilation by simplifying mock library inclusions. #10917
One way to do that is to break the monolithic mock headers into different mock classes and then refactor the code base (like #12053 #12051 #11797)
It's hard to refactor them manually. So I wrote headersplit, a tool based on libclang and python to help me divide the monolithic mock header and replace includes after dividing automatically. This tool can also generate building time comparison between before/after refactoring.

Risk level: low
Testing: Build succeeds.

Signed-off-by: Muge Chen <mugechen@google.com>

bazel/dependency_imports.bzl

@@ -7,6 +7,7 @@ load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependenci
 load("@upb//bazel:repository_defs.bzl", upb_bazel_version_repository = "bazel_version_repository")
 load("@config_validation_pip3//:requirements.bzl", config_validation_pip_install = "pip_install")
 load("@protodoc_pip3//:requirements.bzl", protodoc_pip_install = "pip_install")
+load("@headersplit_pip3//:requirements.bzl", headersplit_pip_install = "pip_install")
 load("@rules_antlr//antlr:deps.bzl", "antlr_dependencies")
 
 # go version for rules_go
@@ -53,3 +54,4 @@ def envoy_dependency_imports(go_version = GO_VERSION):
 
     config_validation_pip_install()
     protodoc_pip_install()
+    headersplit_pip_install()

bazel/repositories_extra.bzl

@@ -14,6 +14,10 @@ def _python_deps():
         name = "protodoc_pip3",
         requirements = "@envoy//tools/protodoc:requirements.txt",
     )
+    pip3_import(
+        name = "headersplit_pip3",
+        requirements = "@envoy//tools/envoy_headersplit:requirements.txt",
+    )
 
 # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies().
 def envoy_dependencies_extra():

ci/do_ci.sh

@@ -247,6 +247,9 @@ elif [[ "$CI_TARGET" == "bazel.dev" ]]; then
 
   echo "Building and testing ${TEST_TARGETS}"
   bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c fastbuild ${TEST_TARGETS}
+  # TODO(foreseeable): consolidate this and the API tool tests in a dedicated target.
+  bazel_with_collection //tools/envoy_headersplit:headersplit_test --spawn_strategy=local
+  bazel_with_collection //tools/envoy_headersplit:replace_includes_test --spawn_strategy=local
   exit 0
 elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
   # Right now, none of the available compile-time options conflict with each other. If this

ci/run_clang_tidy.sh

@@ -45,6 +45,11 @@ function exclude_testdata() {
   grep -v tools/testdata/check_format/
 }
 
+# Do not run clang-tidy on envoy_headersplit testdata files.
+function exclude_testdata() {
+  grep -v tools/envoy_headersplit/
+}
+
 # Exclude files in third_party which are temporary forks from other OSS projects.
 function exclude_third_party() {
   grep -v third_party/

tools/envoy_headersplit/BUILD

@@ -0,0 +1,67 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+load("@headersplit_pip3//:requirements.bzl", "requirement")
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+py_binary(
+    name = "headersplit",
+    srcs = [
+        "headersplit.py",
+    ],
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("clang"),
+    ],
+)
+
+py_binary(
+    name = "replace_includes",
+    srcs = [
+        "replace_includes.py",
+    ],
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":headersplit",
+    ],
+)
+
+py_test(
+    name = "headersplit_test",
+    srcs = [
+        "headersplit_test.py",
+    ],
+    data = glob(["code_corpus/**"]),
+    python_version = "PY3",
+    srcs_version = "PY3",
+    tags = ["no-sandbox"],  # TODO (foreseeable): make this test run under sandbox
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("clang"),
+        ":headersplit",
+    ],
+)
+
+py_test(
+    name = "replace_includes_test",
+    srcs = [
+        "replace_includes_test.py",
+    ],
+    data = glob(["code_corpus/**"]),
+    python_version = "PY3",
+    srcs_version = "PY3",
+    tags = ["no-sandbox"],  # TODO (foreseeable): make this test run under sandbox
+    visibility = ["//visibility:public"],
+    deps = [
+        ":replace_includes",
+    ],
+)

tools/envoy_headersplit/README.md

@@ -0,0 +1,24 @@
+# Envoy Header Split
+Tool for spliting monolithic header files in Envoy to speed up compilation
+
+Steps to divide Envoy mock headers:
+
+1. Run `headersplit.py` to divide the monolithic mock header into different classes
+
+Example (to split monolithic mock header test/mocks/network/mocks.h):
+
+```
+cd ${ENVOY_SRCDIR}/test/mocks/network/
+python3 ${ENVOY_SRCDIR}/tools/envoy_headersplit/headersplit.py -i mocks.cc -d mocks.h
+```
+
+2. Remove unused `#includes` from the new mock headers, and write Bazel dependencies for the newly divided mock classes. (this step needs to be done manually)
+
+3. Run `replace_includes.py` to replace superfluous `#includes` in Envoy directory after dividing. It will also modify the corresponding Bazel `BUILD` file.
+
+Example (to replace `#includes` after dividing mock header test/mocks/network/mocks.h):
+
+```
+cd ${ENVOY_SRCDIR}
+python3 ${ENVOY_SRCDIR}/tools/envoy_headersplit/replace_includes.py -m network
+```

tools/envoy_headersplit/code_corpus/class_defn.h

@@ -0,0 +1,20 @@
+#include "envoy/split"
+
+// NOLINT(namespace-envoy)
+
+namespace {
+
+class Foo {};
+
+class Bar {
+  Foo getFoo();
+};
+
+class FooBar : Foo, Bar {};
+
+class DeadBeaf {
+public:
+  int val();
+  FooBar foobar;
+};
+} // namespace

tools/envoy_headersplit/code_corpus/class_defn_without_namespace.h

@@ -0,0 +1,17 @@
+#include "envoy/split"
+
+// NOLINT(namespace-envoy)
+
+class Foo {};
+
+class Bar {
+  Foo getFoo();
+};
+
+class FooBar : Foo, Bar {};
+
+class DeadBeaf {
+public:
+  int val();
+  FooBar foobar;
+};

tools/envoy_headersplit/code_corpus/class_impl.cc

@@ -0,0 +1,14 @@
+#include "class_defn.h"
+
+// NOLINT(namespace-envoy)
+
+namespace {
+Foo Bar::getFoo() {
+  Foo foo;
+  return foo;
+}
+
+int DeadBeaf::val() { return 42; }
+
+DeadBeaf::DeadBeaf() = default;
+} // namespace

tools/envoy_headersplit/code_corpus/fail_mocks.cc

@@ -0,0 +1,51 @@
+#include "fail_mocks.h"
+
+// NOLINT(namespace-envoy)
+
+#include <string>
+
+#include "envoy/admin/v3/server_info.pb.h"
+#include "envoy/config/core/v3/base.pb.h"
+
+#include "common/singleton/manager_impl.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using testing::_;
+using testing::Invoke;
+using testing::Return;
+using testing::ReturnPointee;
+using testing::ReturnRef;
+using testing::SaveArg;
+
+namespace Envoy {
+namespace Server {
+
+MockConfigTracker::MockConfigTracker() {
+  ON_CALL(*this, add_(_, _))
+      .WillByDefault(Invoke([this](const std::string& key, Cb callback) -> EntryOwner* {
+        EXPECT_TRUE(config_tracker_callbacks_.find(key) == config_tracker_callbacks_.end());
+        config_tracker_callbacks_[key] = callback;
+        return new MockEntryOwner();
+      }));
+}
+MockConfigTracker::~MockConfigTracker() = default;
+
+MockListenerComponentFactory::MockListenerComponentFactory()
+    : socket_(std::make_shared<NiceMock<Network::MockListenSocket>>()) {
+  ON_CALL(*this, createListenSocket(_, _, _, _))
+      .WillByDefault(Invoke([&](Network::Address::InstanceConstSharedPtr, Network::Socket::Type,
+                                const Network::Socket::OptionsSharedPtr& options,
+                                const ListenSocketCreationParams&) -> Network::SocketSharedPtr {
+        if (!Network::Socket::applyOptions(options, *socket_,
+                                           envoy::config::core::v3::SocketOption::STATE_PREBIND)) {
+          throw EnvoyException("MockListenerComponentFactory: Setting socket options failed");
+        }
+        return socket_;
+      }));
+}
+MockListenerComponentFactory::~MockListenerComponentFactory() = default;
+
+} // namespace Server
+} // namespace Envoy

tools/envoy_headersplit/code_corpus/fail_mocks.h

@@ -0,0 +1,64 @@
+#pragma once
+// NOLINT(namespace-envoy)
+
+#include <chrono>
+#include <cstdint>
+#include <list>
+#include <string>
+
+namespace Envoy {
+namespace Server {
+
+class MockConfigTracker : public ConfigTracker {
+public:
+  MockConfigTracker();
+  ~MockConfigTracker() override;
+
+  struct MockEntryOwner : public EntryOwner {};
+
+  MOCK_METHOD(EntryOwner*, add_, (std::string, Cb));
+
+  // Server::ConfigTracker
+  MOCK_METHOD(const CbsMap&, getCallbacksMap, (), (const));
+  EntryOwnerPtr add(const std::string& key, Cb callback) override {
+    return EntryOwnerPtr{add_(key, std::move(callback))};
+  }
+
+  absl::node_hash_map<std::string, Cb> config_tracker_callbacks_;
+};
+
+class MockListenerComponentFactory : public ListenerComponentFactory {
+public:
+  MockListenerComponentFactory();
+  ~MockListenerComponentFactory() override;
+
+  DrainManagerPtr
+  createDrainManager(envoy::config::listener::v3::Listener::DrainType drain_type) override {
+    return DrainManagerPtr{createDrainManager_(drain_type)};
+  }
+  LdsApiPtr createLdsApi(const envoy::config::core::v3::ConfigSource& lds_config) override {
+    return LdsApiPtr{createLdsApi_(lds_config)};
+  }
+
+  MOCK_METHOD(LdsApi*, createLdsApi_, (const envoy::config::core::v3::ConfigSource& lds_config));
+  MOCK_METHOD(std::vector<Network::FilterFactoryCb>, createNetworkFilterFactoryList,
+              (const Protobuf::RepeatedPtrField<envoy::config::listener::v3::Filter>& filters,
+               Configuration::FilterChainFactoryContext& filter_chain_factory_context));
+  MOCK_METHOD(std::vector<Network::ListenerFilterFactoryCb>, createListenerFilterFactoryList,
+              (const Protobuf::RepeatedPtrField<envoy::config::listener::v3::ListenerFilter>&,
+               Configuration::ListenerFactoryContext& context));
+  MOCK_METHOD(std::vector<Network::UdpListenerFilterFactoryCb>, createUdpListenerFilterFactoryList,
+              (const Protobuf::RepeatedPtrField<envoy::config::listener::v3::ListenerFilter>&,
+               Configuration::ListenerFactoryContext& context));
+  MOCK_METHOD(Network::SocketSharedPtr, createListenSocket,
+              (Network::Address::InstanceConstSharedPtr address, Network::Socket::Type socket_type,
+               const Network::Socket::OptionsSharedPtr& options,
+               const ListenSocketCreationParams& params));
+  MOCK_METHOD(DrainManager*, createDrainManager_,
+              (envoy::config::listener::v3::Listener::DrainType drain_type));
+  MOCK_METHOD(uint64_t, nextListenerTag, ());
+
+  std::shared_ptr<Network::MockListenSocket> socket_;
+};
+} // namespace Server
+} // namespace Envoy

tools/envoy_headersplit/code_corpus/fake_build

@@ -0,0 +1,24 @@
+envoy_cc_test(
+    name = "async_client_impl_test",
+    srcs = ["async_client_impl_test.cc"],
+    deps = [
+        ":common_lib",
+        "//source/common/buffer:buffer_lib",
+        "//source/common/http:async_client_lib",
+        "//source/common/http:context_lib",
+        "//source/common/http:headers_lib",
+        "//source/common/http:utility_lib",
+        "//source/extensions/upstreams/http/generic:config",
+        "//test/mocks:common_lib",
+        "//test/mocks/buffer:buffer_mocks",
+        "//test/mocks/http:http_mocks",
+        "//test/mocks/local_info:local_info_mocks",
+        "//test/mocks/router:router_mocks",
+        "//test/mocks/runtime:runtime_mocks",
+        "//test/mocks/stats:stats_mocks",
+        "//test/mocks/upstream:upstream_mocks",
+        "//test/test_common:test_time_lib",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+        "@envoy_api//envoy/config/route/v3:pkg_cc_proto",
+    ],
+)

tools/envoy_headersplit/code_corpus/fake_source_code.cc

@@ -0,0 +1,32 @@
+// NOLINT(namespace-envoy)
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "envoy/config/core/v3/base.pb.h"
+#include "envoy/config/route/v3/route_components.pb.h"
+
+#include "common/buffer/buffer_impl.h"
+#include "common/http/async_client_impl.h"
+#include "common/http/context_impl.h"
+#include "common/http/headers.h"
+#include "common/http/utility.h"
+
+#include "test/common/http/common.h"
+#include "test/mocks/buffer/mocks.h"
+#include "test/mocks/common.h"
+#include "test/mocks/http/mocks.h"
+#include "test/mocks/local_info/mocks.h"
+#include "test/mocks/router/mocks.h"
+#include "test/mocks/runtime/mocks.h"
+#include "test/mocks/stats/mocks.h"
+#include "test/mocks/upstream/mocks.h"
+#include "test/test_common/printers.h"
+
+....useless stuff...
+
+    NiceMock<Upstream::MockClusterManager>
+        cm_;
+
+... uninteresting stuff..

tools/envoy_headersplit/code_corpus/hello.h

@@ -0,0 +1,11 @@
+// your first c++ program
+// NOLINT(namespace-envoy)
+#include <iostream>
+
+// random strings
+
+#include "foo/bar"
+
+class test {
+  test() { std::cout << "Hello World" << std::endl; }
+};