From c753e34e5a469bfbac203f89a6526fd32c26efb8 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:05:40 -0700 Subject: [PATCH 01/11] c/archival: serde_fields for archival_metadata_stm structs - archival_metadata_stm::segment - archival_metadata_stm::start_offset_with_delta - archival_metadata_stm::truncate_archive_commit_cmd Signed-off-by: Oren Leiman --- src/v/cluster/archival/archival_metadata_stm.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/v/cluster/archival/archival_metadata_stm.cc b/src/v/cluster/archival/archival_metadata_stm.cc index b9fc3418dc642..654f287c7e422 100644 --- a/src/v/cluster/archival/archival_metadata_stm.cc +++ b/src/v/cluster/archival/archival_metadata_stm.cc @@ -84,12 +84,17 @@ struct archival_metadata_stm::segment // Segment meta value generated by old redpanda versions will have the // default value of 'no'. segment_validated is_validated{segment_validated::no}; + + auto serde_fields() { + return std::tie(ntp_revision_deprecated, name, meta, is_validated); + } }; struct archival_metadata_stm::start_offset : public serde:: envelope, serde::compat_version<0>> { model::offset start_offset; + auto serde_fields() { return std::tie(start_offset); } }; struct archival_metadata_stm::start_offset_with_delta @@ -99,6 +104,7 @@ struct archival_metadata_stm::start_offset_with_delta serde::compat_version<0>> { model::offset start_offset; model::offset_delta delta; + auto serde_fields() { return std::tie(start_offset, delta); } }; struct archival_metadata_stm::add_segment_cmd { @@ -142,6 +148,7 @@ struct archival_metadata_stm::truncate_archive_commit_cmd { : serde::envelope, serde::compat_version<0>> { model::offset start_offset; uint64_t bytes_removed; + auto serde_fields() { return std::tie(start_offset, bytes_removed); } }; }; @@ -164,6 +171,8 @@ struct archival_metadata_stm::spillover_cmd static constexpr cmd_key key{9}; cloud_storage::segment_meta manifest_meta; + + auto serde_fields() { return std::tie(manifest_meta); } }; struct archival_metadata_stm::replace_manifest_cmd { @@ -208,6 +217,8 @@ struct archival_metadata_stm::read_write_fence_cmd static constexpr cmd_key key{14}; model::offset last_applied_offset; + + auto serde_fields() { return std::tie(last_applied_offset); } }; // Serde format description From d364da7d9b6ca22434f3f1a20ea2cd7b47b88472 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:06:59 -0700 Subject: [PATCH 02/11] c: serde_fields for client_quota_serde structs - entity_value - alter_quotas_request Signed-off-by: Oren Leiman --- src/v/cluster/client_quota_serde.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/v/cluster/client_quota_serde.h b/src/v/cluster/client_quota_serde.h index e605729cc53c1..747c47654b7e2 100644 --- a/src/v/cluster/client_quota_serde.h +++ b/src/v/cluster/client_quota_serde.h @@ -65,6 +65,8 @@ struct entity_key const client_id_default_match&, const client_id_default_match&) = default; + auto serde_fields() { return std::tie(); } + friend std::ostream& operator<<(std::ostream&, const client_id_default_match&); @@ -96,6 +98,8 @@ struct entity_key } ss::sstring value; + + auto serde_fields() { return std::tie(value); } }; /// client_id_prefix_match is the quota entity type corresponding to the @@ -122,8 +126,11 @@ struct entity_key } ss::sstring value; + auto serde_fields() { return std::tie(value); } }; + auto serde_fields() { return std::tie(part); } + serde::variant< client_id_default_match, client_id_match, @@ -166,6 +173,11 @@ struct entity_value std::optional producer_byte_rate; std::optional consumer_byte_rate; std::optional controller_mutation_rate; + + auto serde_fields() { + return std::tie( + producer_byte_rate, consumer_byte_rate, controller_mutation_rate); + } }; /// entity_value_diff describes the quotas diff for an entity_key @@ -237,6 +249,8 @@ struct alter_delta_cmd_data std::vector ops; + auto serde_fields() { return std::tie(ops); } + friend bool operator==(const alter_delta_cmd_data&, const alter_delta_cmd_data&) = default; @@ -286,6 +300,8 @@ struct alter_quotas_request alter_delta_cmd_data cmd_data; model::timeout_clock::duration timeout{}; + auto serde_fields() { return std::tie(cmd_data, timeout); } + friend bool operator==(const alter_quotas_request&, const alter_quotas_request&) = default; @@ -299,6 +315,7 @@ struct alter_quotas_response using rpc_adl_exempt = std::true_type; cluster::errc ec; + auto serde_fields() { return std::tie(ec); } friend bool operator==(const alter_quotas_response&, const alter_quotas_response&) From 905c9aec090798a9fd307440af9d5bc7cb2e7f79 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:07:38 -0700 Subject: [PATCH 03/11] c/node: serde_fields for local_state Signed-off-by: Oren Leiman --- src/v/cluster/node/types.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/v/cluster/node/types.h b/src/v/cluster/node/types.h index 324fcb71c989c..6acddf0ddde30 100644 --- a/src/v/cluster/node/types.h +++ b/src/v/cluster/node/types.h @@ -107,6 +107,10 @@ struct local_state uint64_t data_target_size{0}; uint64_t data_current_size{0}; uint64_t data_reclaimable_size{0}; + auto serde_fields() { + return std::tie( + data_target_size, data_current_size, data_reclaimable_size); + } friend bool operator==(const log_data_state&, const log_data_state&) = default; friend std::ostream& operator<<(std::ostream&, const log_data_state&); From 7cea4894b99e6fbbfd651139a921682f3915f981 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:07:59 -0700 Subject: [PATCH 04/11] c: serde_fields for prefix_truncate_record Signed-off-by: Oren Leiman --- src/v/cluster/prefix_truncate_record.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/v/cluster/prefix_truncate_record.h b/src/v/cluster/prefix_truncate_record.h index 2ae9c7b13929c..728bfc3a5f4d5 100644 --- a/src/v/cluster/prefix_truncate_record.h +++ b/src/v/cluster/prefix_truncate_record.h @@ -35,6 +35,10 @@ struct prefix_truncate_record // May not be empty. kafka::offset kafka_start_offset{}; + + auto serde_fields() { + return std::tie(rp_start_offset, kafka_start_offset); + } }; } // namespace cluster From 47a24b36ef0c91b3aa9b2985078b2c3503f4b967 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:08:15 -0700 Subject: [PATCH 05/11] c: serde_fields for self_test_rpc_types - empty_request - get_status_response - netcheck_request - netcheck_response Signed-off-by: Oren Leiman --- src/v/cluster/self_test_rpc_types.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/v/cluster/self_test_rpc_types.h b/src/v/cluster/self_test_rpc_types.h index e2841763e9ec7..e50976cdd88a9 100644 --- a/src/v/cluster/self_test_rpc_types.h +++ b/src/v/cluster/self_test_rpc_types.h @@ -335,6 +335,8 @@ struct empty_request : serde:: envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; + + auto serde_fields() { return std::tie(); } }; struct start_test_request @@ -386,6 +388,8 @@ struct get_status_response std::vector results; self_test_stage stage{}; + auto serde_fields() { return std::tie(id, status, results, stage); } + friend std::ostream& operator<<(std::ostream& o, const get_status_response& r) { fmt::print( @@ -405,6 +409,7 @@ struct netcheck_request using rpc_adl_exempt = std::true_type; model::node_id source; iobuf buf; + auto serde_fields() { return std::tie(source, buf); } friend std::ostream& operator<<(std::ostream& o, const netcheck_request& r) { fmt::print(o, "{{source: {} buf: {}}}", r.source, r.buf.size_bytes()); @@ -417,6 +422,9 @@ struct netcheck_response envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; size_t bytes_read{0}; + + auto serde_fields() { return std::tie(bytes_read); } + friend std::ostream& operator<<(std::ostream& o, const netcheck_response& r) { fmt::print(o, "{{bytes_read: {}}}", r.bytes_read); From ef7187b90ef39a4ba60a603fba15784f7e39402e Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:47:48 -0700 Subject: [PATCH 06/11] c: serde_fields for data_migration_types - copy_target Signed-off-by: Oren Leiman --- src/v/cluster/data_migration_types.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v/cluster/data_migration_types.h b/src/v/cluster/data_migration_types.h index 2932dc9bc9ee6..986687df39aa6 100644 --- a/src/v/cluster/data_migration_types.h +++ b/src/v/cluster/data_migration_types.h @@ -201,6 +201,8 @@ struct copy_target : serde::envelope, serde::compat_version<0>> { ss::sstring bucket; + auto serde_fields() { return std::tie(bucket); } + friend bool operator==(const copy_target&, const copy_target&) = default; friend std::ostream& operator<<(std::ostream&, const copy_target&); }; From ba76e4f67c52d1eb525e7419e666617becb0943e Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:48:49 -0700 Subject: [PATCH 07/11] c: serde_fields for node_status_rpc_types - node_status_metadata - node_status_request Signed-off-by: Oren Leiman --- src/v/cluster/node_status_rpc_types.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/v/cluster/node_status_rpc_types.h b/src/v/cluster/node_status_rpc_types.h index ce51c461e3dcd..11a7b58eaf63e 100644 --- a/src/v/cluster/node_status_rpc_types.h +++ b/src/v/cluster/node_status_rpc_types.h @@ -27,6 +27,8 @@ struct node_status_metadata model::node_id node_id; + auto serde_fields() { return std::tie(node_id); } + friend std::ostream& operator<<(std::ostream& o, const node_status_metadata& nsm) { fmt::print(o, "{{node_id:{}}}", nsm.node_id); @@ -43,6 +45,8 @@ struct node_status_request node_status_metadata sender_metadata; + auto serde_fields() { return std::tie(sender_metadata); } + friend std::ostream& operator<<(std::ostream& o, const node_status_request& r) { fmt::print(o, "{{sender_metadata: {}}}", r.sender_metadata); @@ -57,6 +61,8 @@ struct node_status_reply node_status_metadata replier_metadata; + auto serde_fields() { return std::tie(replier_metadata); } + friend std::ostream& operator<<(std::ostream& o, const node_status_reply& r) { fmt::print(o, "{{replier_metadata: {}}}", r.replier_metadata); From fd90f048fb6494bdebd5ff84c471b61ebc102c0e Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:51:39 -0700 Subject: [PATCH 08/11] c/recovery: serde_fields for status_request Signed-off-by: Oren Leiman --- src/v/cluster/topic_recovery_status_types.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v/cluster/topic_recovery_status_types.h b/src/v/cluster/topic_recovery_status_types.h index dc457e97e3386..9643a0f39fb52 100644 --- a/src/v/cluster/topic_recovery_status_types.h +++ b/src/v/cluster/topic_recovery_status_types.h @@ -22,6 +22,8 @@ struct status_request : serde:: envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; + + auto serde_fields() { return std::tie(); } }; struct topic_downloads From 3fd8e4223033b795959b6e3be0e1a7370f9c0260 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:52:11 -0700 Subject: [PATCH 09/11] c/types: serde_fields for controller_committed_offset_request Signed-off-by: Oren Leiman --- src/v/cluster/types.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v/cluster/types.h b/src/v/cluster/types.h index 391d83994f8e0..32eff61a9ff2f 100644 --- a/src/v/cluster/types.h +++ b/src/v/cluster/types.h @@ -3133,6 +3133,8 @@ struct controller_committed_offset_request serde::version<0>, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; + + auto serde_fields() { return std::tie(); } }; struct controller_committed_offset_reply From 84b086951575792196e865e80a0c5248ef8ba72f Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 13:50:04 -0700 Subject: [PATCH 10/11] c/test: serde_fields for commands_serialization_test Signed-off-by: Oren Leiman --- src/v/cluster/tests/commands_serialization_test.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/v/cluster/tests/commands_serialization_test.cc b/src/v/cluster/tests/commands_serialization_test.cc index 616445e52447f..c68f39ec42247 100644 --- a/src/v/cluster/tests/commands_serialization_test.cc +++ b/src/v/cluster/tests/commands_serialization_test.cc @@ -46,6 +46,8 @@ struct fake_serde_only_key using rpc_adl_exempt = std::true_type; ss::sstring str; + + auto serde_fields() { return std::tie(str); } }; struct fake_serde_only_val : serde::envelope< @@ -55,6 +57,8 @@ struct fake_serde_only_val using rpc_adl_exempt = std::true_type; ss::sstring str; + + auto serde_fields() { return std::tie(str); } }; using fake_serde_only_cmd = cluster::controller_command< fake_serde_only_key, From 9f93962754e97eee718a06ca07d4bb1ae122d7a7 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 16:05:58 -0700 Subject: [PATCH 11/11] c/test: serde_fields for archival_metadata_stm_test Signed-off-by: Oren Leiman --- src/v/cluster/archival/tests/archival_metadata_stm_test.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/v/cluster/archival/tests/archival_metadata_stm_test.cc b/src/v/cluster/archival/tests/archival_metadata_stm_test.cc index 76914eace0493..45cc7da9855d0 100644 --- a/src/v/cluster/archival/tests/archival_metadata_stm_test.cc +++ b/src/v/cluster/archival/tests/archival_metadata_stm_test.cc @@ -566,6 +566,10 @@ struct segment model::initial_revision_id ntp_revision_deprecated; cloud_storage::segment_name name; cloud_storage::partition_manifest::segment_meta meta; + + auto serde_fields() { + return std::tie(ntp_revision_deprecated, name, meta); + } }; struct snapshot @@ -573,6 +577,8 @@ struct snapshot envelope, serde::compat_version<0>> { /// List of segments std::vector segments; + + auto serde_fields() { return std::tie(segments); } }; } // namespace old