From e9836c480ff395a31a33f8430514010ff6ff635b Mon Sep 17 00:00:00 2001 From: vitsai Date: Tue, 20 Jun 2023 11:51:59 -0700 Subject: [PATCH] Start new src tree to incrementally fix build files. Signed-off-by: vitsai --- .buildkite/pipeline.build.yml | 4 +- BUILD.bazel | 223 ++---------------- ci/ci.sh | 2 +- cpp/BUILD.bazel | 2 +- src/ray/util/BUILD | 45 ++++ src/ray/util/tests/BUILD | 147 ++++++++++++ .../util/{ => tests}/container_util_test.cc | 0 src/ray/util/{ => tests}/counter_test.cc | 0 src/ray/util/{ => tests}/event_test.cc | 2 - .../{ => tests}/exponential_backoff_test.cc | 0 src/ray/util/{ => tests}/filesystem_test.cc | 0 src/ray/util/{ => tests}/logging_test.cc | 0 src/ray/util/{ => tests}/sample_test.cc | 0 src/ray/util/{ => tests}/sequencer_test.cc | 0 src/ray/util/{ => tests}/signal_test.cc | 0 src/ray/util/{ => tests}/throttler_test.cc | 0 src/ray/util/{ => tests}/util_test.cc | 0 17 files changed, 216 insertions(+), 209 deletions(-) create mode 100644 src/ray/util/BUILD create mode 100644 src/ray/util/tests/BUILD rename src/ray/util/{ => tests}/container_util_test.cc (100%) rename src/ray/util/{ => tests}/counter_test.cc (100%) rename src/ray/util/{ => tests}/event_test.cc (99%) rename src/ray/util/{ => tests}/exponential_backoff_test.cc (100%) rename src/ray/util/{ => tests}/filesystem_test.cc (100%) rename src/ray/util/{ => tests}/logging_test.cc (100%) rename src/ray/util/{ => tests}/sample_test.cc (100%) rename src/ray/util/{ => tests}/sequencer_test.cc (100%) rename src/ray/util/{ => tests}/signal_test.cc (100%) rename src/ray/util/{ => tests}/throttler_test.cc (100%) rename src/ray/util/{ => tests}/util_test.cc (100%) diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml index b012e8631045..88e7c0dba19d 100644 --- a/.buildkite/pipeline.build.yml +++ b/.buildkite/pipeline.build.yml @@ -136,7 +136,7 @@ - unset CC CXX && bazel test --config=ci --config=ubsan $(./ci/run/bazel_export_options) --build_tests_only --jobs=2 - -- //:all -//:core_worker_test -//:logging_test -//:ray_syncer_test + -- //:all -//:core_worker_test -//src/ray/util/tests:logging_test -//:ray_syncer_test - label: ":cpp: Tests (TSAN)" conditions: [ "RAY_CI_CORE_CPP_AFFECTED" ] @@ -146,7 +146,7 @@ - bazel test --config=ci --config=tsan-clang $(./ci/run/bazel_export_options) --build_tests_only --jobs=2 - -- //:all -//:core_worker_test -//:event_test -//:gcs_actor_manager_test + -- //:all -//:core_worker_test -//src/ray/util/tests:event_test -//:gcs_actor_manager_test -//:gcs_placement_group_manager_test -//:gcs_placement_group_scheduler_test -//:gcs_server_rpc_test -//:gcs_client_test -//:metric_exporter_client_test -//:stats_test -//:worker_pool_test -//:ray_syncer_test diff --git a/BUILD.bazel b/BUILD.bazel index bb3a8ff433b7..87293486ee2c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,6 +1,11 @@ # Bazel build # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html +# ************************** IMPORTANT *********************** +# If a target or file is not found here, then please try searching in /src +# We are in the middle of transitioning to a tree of build files. +# If you would like to help with the move in your PR, please use `git mv` so that the history of the file is retained. + load("@rules_proto//proto:defs.bzl", "proto_library") load("@rules_python//python:defs.bzl", "py_library") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test") @@ -376,7 +381,7 @@ cc_library( deps = [ ":plasma_fbs", ":ray_common", - ":ray_util", + "//src/ray/util:util", "//src/ray/protobuf:common_cc_proto", "@msgpack", ], @@ -481,7 +486,7 @@ cc_library( deps = [ ":node_manager_fbs", ":ray_syncer_cc_grpc", - ":ray_util", + "//src/ray/util:util", ":stats_metric", "//src/ray/protobuf:common_cc_proto", "//src/ray/protobuf:gcs_cc_proto", @@ -505,7 +510,7 @@ cc_binary( copts = COPTS, visibility = ["//java:__subpackages__"], deps = [ - ":ray_util", + "//src/ray/util:util", ":raylet_lib", "@com_github_gflags_gflags//:gflags", ], @@ -619,7 +624,7 @@ cc_library( strip_include_prefix = "src", visibility = ["//visibility:public"], deps = [ - ":ray_util", + "//src/ray/util:util", "@com_github_jupp0r_prometheus_cpp//pull", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", @@ -691,7 +696,7 @@ cc_library( deps = [ ":gcs_client_lib", ":ray_common", - ":ray_util", + "//src/ray/util:util", ":stats_lib", "//src/ray/protobuf:common_cc_proto", "@boost//:asio", @@ -751,7 +756,7 @@ cc_library( ":plasma_client", ":pubsub_lib", ":ray_common", - ":ray_util", + "//src/ray/util:util", ":runtime_env_rpc", ":scheduler", ":stats_lib", @@ -794,7 +799,7 @@ cc_library( ":node_manager_fbs", ":node_manager_rpc", ":ray_common", - ":ray_util", + "//src/ray/util:util", "//src/ray/protobuf:gcs_cc_proto", "@boost//:asio", ], @@ -828,7 +833,7 @@ cc_library( ":gcs_client_lib", ":plasma_client", ":ray_common", - ":ray_util", + "//src/ray/util:util", ":raylet_client_lib", ":stats_lib", ":worker_rpc", @@ -1498,128 +1503,6 @@ cc_test( ], ) -cc_test( - name = "logging_test", - size = "small", - srcs = ["src/ray/util/logging_test.cc"], - args = [ - "--gtest_filter=PrintLogTest*", - # Disable so we can test terminate handler. - "--gtest_catch_exceptions=0", - ], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_util", - "@boost//:asio", - "@com_google_absl//absl/strings:str_format", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "event_test", - size = "small", - srcs = ["src/ray/util/event_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_util", - "@boost//:range", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "counter_test", - size = "small", - srcs = ["src/ray/util/counter_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_util", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "filesystem_test", - size = "small", - srcs = ["src/ray/util/filesystem_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_common", - ":ray_util", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "container_util_test", - size = "small", - srcs = ["src/ray/util/container_util_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_util", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "util_test", - size = "small", - srcs = ["src/ray/util/util_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_util", - "@boost//:asio", - "@boost//:process", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "throttler_test", - size = "small", - srcs = ["src/ray/util/throttler_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_util", - "@com_google_absl//absl/time", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "exponential_backoff_test", - size = "small", - srcs = ["src/ray/util/exponential_backoff_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_util", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "sample_test", - size = "small", - srcs = ["src/ray/util/sample_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_common", - "@com_google_googletest//:gtest_main", - ], -) - cc_test( name = "dependency_manager_test", size = "small", @@ -1776,30 +1659,6 @@ cc_test( ], ) -cc_test( - name = "signal_test", - size = "small", - srcs = ["src/ray/util/signal_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":raylet_lib", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "sequencer_test", - size = "small", - srcs = ["src/ray/util/sequencer_test.cc"], - copts = COPTS, - tags = ["team:core"], - deps = [ - ":ray_util", - "@com_google_googletest//:gtest_main", - ], -) - cc_test( name = "stats_test", size = "small", @@ -2403,7 +2262,7 @@ cc_library( ":object_manager_rpc", ":plasma_store_server_lib", ":ray_common", - ":ray_util", + "//src/ray/util:util", "@boost//:asio", ], ) @@ -2432,48 +2291,6 @@ cc_library( visibility = ["//visibility:public"], ) -cc_library( - name = "ray_util", - srcs = glob( - [ - "src/ray/util/*.cc", - ], - exclude = [ - "src/ray/util/*_test.cc", - ], - ), - hdrs = glob([ - "src/ray/util/*.h", - ]), - copts = COPTS, - linkopts = select({ - "@bazel_tools//src/conditions:windows": [ - ], - "//conditions:default": [ - "-lpthread", - ], - }), - strip_include_prefix = "src", - visibility = ["//visibility:public"], - deps = [ - ":aligned_alloc", - ":sha256", - "//src/ray/protobuf:event_cc_proto", - "@boost//:asio", - "@com_github_spdlog//:spdlog", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/debugging:failure_signal_handler", - "@com_google_absl//absl/debugging:stacktrace", - "@com_google_absl//absl/debugging:symbolize", - "@com_google_absl//absl/random", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/time", - "@com_google_googletest//:gtest_main", - "@nlohmann_json", - ], -) - filegroup( name = "extra_actions_base_proto", srcs = [ @@ -2563,7 +2380,7 @@ cc_library( deps = [ ":hiredis", ":ray_common", - ":ray_util", + "//src/ray/util:util", ":stats_lib", "@boost//:asio", ], @@ -2602,7 +2419,7 @@ cc_library( strip_include_prefix = "src", deps = [ ":ray_common", - ":ray_util", + "//src/ray/util:util", ], ) @@ -2620,7 +2437,7 @@ cc_library( strip_include_prefix = "src", deps = [ ":ray_common", - ":ray_util", + "//src/ray/util:util", ], ) @@ -2730,7 +2547,7 @@ cc_library( ":node_manager_fbs", ":node_manager_rpc", ":ray_common", - ":ray_util", + "//src/ray/util:util", ":stats_lib", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/protobuf:gcs_service_cc_proto", @@ -2812,7 +2629,7 @@ cc_test( tags = ["team:core"], deps = [ ":gcs", - ":ray_util", + "//src/ray/util:util", "@com_google_googletest//:gtest_main", ], ) @@ -2855,13 +2672,13 @@ pyx_library( deps = [ "//:core_worker_lib", "//:global_state_accessor_lib", - "//:ray_util", "//:raylet_lib", "//:redis_client", "//:src/ray/ray_exported_symbols.lds", "//:src/ray/ray_version_script.lds", "//:stats_lib", "//src/ray/protobuf:serialization_cc_proto", + "//src/ray/util:util", ], ) diff --git a/ci/ci.sh b/ci/ci.sh index b05d89c9f5ef..7700729374a9 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -123,7 +123,7 @@ test_core() { msys) args+=( -//:core_worker_test - -//:event_test + -//src/ray/util/tests:event_test -//:gcs_server_rpc_test -//:ray_syncer_test # TODO (iycheng): it's flaky on windows. Add it back once we figure out the cause -//:gcs_health_check_manager_test diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index d21601878e47..f70bd9f56fbb 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -59,7 +59,7 @@ cc_library( "//:core_worker_lib", "//:global_state_accessor_lib", "//:ray_common", - "//:ray_util", + "//src/ray/util:util", "@boost//:callable_traits", "@boost//:dll", "@com_google_absl//absl/flags:flag", diff --git a/src/ray/util/BUILD b/src/ray/util/BUILD new file mode 100644 index 000000000000..881ade2e2910 --- /dev/null +++ b/src/ray/util/BUILD @@ -0,0 +1,45 @@ +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test") +load("//bazel:ray.bzl", "COPTS") + +cc_library( + name = "util", + srcs = glob( + [ + "*.cc", + ], + exclude = [ + "*_test.cc", + ], + ), + hdrs = glob([ + "*.h", + ]), + copts = COPTS, + linkopts = select({ + "@bazel_tools//src/conditions:windows": [ + ], + "//conditions:default": [ + "-lpthread", + ], + }), + strip_include_prefix = "//src", + visibility = ["//visibility:public"], + deps = [ + "//:aligned_alloc", + "//:sha256", + "//src/ray/protobuf:event_cc_proto", + "@boost//:asio", + "@com_github_spdlog//:spdlog", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/debugging:failure_signal_handler", + "@com_google_absl//absl/debugging:stacktrace", + "@com_google_absl//absl/debugging:symbolize", + "@com_google_absl//absl/random", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "@com_google_googletest//:gtest_main", + "@nlohmann_json", + ], +) + diff --git a/src/ray/util/tests/BUILD b/src/ray/util/tests/BUILD new file mode 100644 index 000000000000..1e7cfcb3fc8e --- /dev/null +++ b/src/ray/util/tests/BUILD @@ -0,0 +1,147 @@ +load("@rules_cc//cc:defs.bzl", "cc_test") +load("//bazel:ray.bzl", "COPTS") + +cc_test( + name = "container_util_test", + size = "small", + srcs = ["container_util_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//src/ray/util:util", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "counter_test", + size = "small", + srcs = ["counter_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//src/ray/util:util", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "event_test", + size = "small", + srcs = ["event_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//src/ray/util:util", + "@boost//:range", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "exponential_backoff_test", + size = "small", + srcs = ["exponential_backoff_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//src/ray/util:util", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "filesystem_test", + size = "small", + srcs = ["filesystem_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//:ray_common", + "//src/ray/util:util", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "logging_test", + size = "small", + srcs = ["logging_test.cc"], + args = [ + "--gtest_filter=PrintLogTest*", + # Disable so we can test terminate handler. + "--gtest_catch_exceptions=0", + ], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//src/ray/util:util", + "@boost//:asio", + "@com_google_absl//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "sample_test", + size = "small", + srcs = ["sample_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//:ray_common", + "@com_google_googletest//:gtest_main", + ], +) +cc_test( + name = "sequencer_test", + size = "small", + srcs = ["sequencer_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//src/ray/util:util", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "signal_test", + size = "small", + srcs = ["signal_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//:raylet_lib", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "throttler_test", + size = "small", + srcs = ["throttler_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//src/ray/util:util", + "@com_google_absl//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "util_test", + size = "small", + srcs = ["util_test.cc"], + copts = COPTS, + tags = ["team:core"], + deps = [ + "//src/ray/util:util", + "@boost//:asio", + "@boost//:process", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/util/container_util_test.cc b/src/ray/util/tests/container_util_test.cc similarity index 100% rename from src/ray/util/container_util_test.cc rename to src/ray/util/tests/container_util_test.cc diff --git a/src/ray/util/counter_test.cc b/src/ray/util/tests/counter_test.cc similarity index 100% rename from src/ray/util/counter_test.cc rename to src/ray/util/tests/counter_test.cc diff --git a/src/ray/util/event_test.cc b/src/ray/util/tests/event_test.cc similarity index 99% rename from src/ray/util/event_test.cc rename to src/ray/util/tests/event_test.cc index e544741a4a57..c73531ea5cbf 100644 --- a/src/ray/util/event_test.cc +++ b/src/ray/util/tests/event_test.cc @@ -651,11 +651,9 @@ TEST_F(EventTest, VerifyOnlyNthOccurenceEventLogged) { EventManager::Instance().AddReporter(std::make_shared()); const std::string kLogStr = "this is a test log"; auto start_time = std::chrono::steady_clock::now().time_since_epoch(); - size_t num_iterations = 0; // printed 30, 60, 90, 120ms 4 times. while (std::chrono::steady_clock::now().time_since_epoch() - start_time < std::chrono::milliseconds(100)) { - num_iterations++; RAY_EVENT_EVERY_MS(INFO, "label", 30) << kLogStr; } diff --git a/src/ray/util/exponential_backoff_test.cc b/src/ray/util/tests/exponential_backoff_test.cc similarity index 100% rename from src/ray/util/exponential_backoff_test.cc rename to src/ray/util/tests/exponential_backoff_test.cc diff --git a/src/ray/util/filesystem_test.cc b/src/ray/util/tests/filesystem_test.cc similarity index 100% rename from src/ray/util/filesystem_test.cc rename to src/ray/util/tests/filesystem_test.cc diff --git a/src/ray/util/logging_test.cc b/src/ray/util/tests/logging_test.cc similarity index 100% rename from src/ray/util/logging_test.cc rename to src/ray/util/tests/logging_test.cc diff --git a/src/ray/util/sample_test.cc b/src/ray/util/tests/sample_test.cc similarity index 100% rename from src/ray/util/sample_test.cc rename to src/ray/util/tests/sample_test.cc diff --git a/src/ray/util/sequencer_test.cc b/src/ray/util/tests/sequencer_test.cc similarity index 100% rename from src/ray/util/sequencer_test.cc rename to src/ray/util/tests/sequencer_test.cc diff --git a/src/ray/util/signal_test.cc b/src/ray/util/tests/signal_test.cc similarity index 100% rename from src/ray/util/signal_test.cc rename to src/ray/util/tests/signal_test.cc diff --git a/src/ray/util/throttler_test.cc b/src/ray/util/tests/throttler_test.cc similarity index 100% rename from src/ray/util/throttler_test.cc rename to src/ray/util/tests/throttler_test.cc diff --git a/src/ray/util/util_test.cc b/src/ray/util/tests/util_test.cc similarity index 100% rename from src/ray/util/util_test.cc rename to src/ray/util/tests/util_test.cc