From e1500c1ad2e7a0582c3fe0254d11ca43076b7b5a Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:02:37 -0700 Subject: [PATCH 01/29] cs: serde_fields for lifecycle_marker Signed-off-by: Oren Leiman --- src/v/cloud_storage/lifecycle_marker.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v/cloud_storage/lifecycle_marker.h b/src/v/cloud_storage/lifecycle_marker.h index 1b35b78211c4a..dba1382e7fc20 100644 --- a/src/v/cloud_storage/lifecycle_marker.h +++ b/src/v/cloud_storage/lifecycle_marker.h @@ -94,6 +94,8 @@ struct remote_nt_lifecycle_marker lifecycle_status status; + auto serde_fields() { return std::tie(cluster_id, topic, status); } + cloud_storage_clients::object_key get_key(const remote_path_provider& path_provider) { return cloud_storage_clients::object_key{ From 15d216291ad0e0b2d91ec9fe3083dc2337d9d6f6 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:03:07 -0700 Subject: [PATCH 02/29] cs: serde_fields for partition_manifest_serde Signed-off-by: Oren Leiman --- src/v/cloud_storage/partition_manifest.cc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/v/cloud_storage/partition_manifest.cc b/src/v/cloud_storage/partition_manifest.cc index ccdc3e2cd8b05..da56c7d76c0a2 100644 --- a/src/v/cloud_storage/partition_manifest.cc +++ b/src/v/cloud_storage/partition_manifest.cc @@ -2523,6 +2523,29 @@ struct partition_manifest_serde std::optional _last_scrubbed_offset; model::producer_id _highest_producer_id; model::offset _applied_offset; + + auto serde_fields() { + return std::tie( + _ntp, + _rev, + _segments_serialized, + _replaced, + _last_offset, + _start_offset, + _last_uploaded_compacted_offset, + _insync_offset, + _cloud_log_size_bytes, + _archive_start_offset, + _archive_start_offset_delta, + _archive_clean_offset, + _start_kafka_offset, + archive_size_bytes, + _spillover_manifests_serialized, + _last_partition_scrub, + _last_scrubbed_offset, + _highest_producer_id, + _applied_offset); + } }; static_assert( From 69606c39cd6e3c70a5665d2cdedb44fb437fe868 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:03:30 -0700 Subject: [PATCH 03/29] cs: serde_fields for partition_manifest::lw_segment_meta Signed-off-by: Oren Leiman --- src/v/cloud_storage/partition_manifest.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/v/cloud_storage/partition_manifest.h b/src/v/cloud_storage/partition_manifest.h index ca13fa993b9b5..1976c07bb1037 100644 --- a/src/v/cloud_storage/partition_manifest.h +++ b/src/v/cloud_storage/partition_manifest.h @@ -90,6 +90,17 @@ class partition_manifest : public base_manifest { segment_name_format sname_format{segment_name_format::v1}; + auto serde_fields() { + return std::tie( + ntp_revision, + base_offset, + committed_offset, + archiver_term, + segment_term, + size_bytes, + sname_format); + } + auto operator<=>(const lw_segment_meta&) const = default; static lw_segment_meta convert(const segment_meta& m); From c82c0f807503c13dad021ce1cb28d67459e7b86c Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:04:12 -0700 Subject: [PATCH 04/29] cs: serde_fields for offset_index_header Signed-off-by: Oren Leiman --- src/v/cloud_storage/remote_segment_index.cc | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/v/cloud_storage/remote_segment_index.cc b/src/v/cloud_storage/remote_segment_index.cc index 2705c4ab3cd61..ea1b7c565181a 100644 --- a/src/v/cloud_storage/remote_segment_index.cc +++ b/src/v/cloud_storage/remote_segment_index.cc @@ -321,6 +321,28 @@ struct offset_index_header int64_t last_time{model::timestamp::missing().value()}; std::vector time_write_buf; iobuf time_index; + + auto serde_fields() { + return std::tie( + min_file_pos_step, + num_elements, + base_rp, + last_rp, + base_kaf, + last_kaf, + base_file, + last_file, + rp_write_buf, + kaf_write_buf, + file_write_buf, + rp_index, + kaf_index, + file_index, + base_time, + last_time, + time_write_buf, + time_index); + } }; iobuf offset_index::to_iobuf() { From 88aa996b97d15fc08494c6d5bba9c1dde8a2df83 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:05:40 -0700 Subject: [PATCH 05/29] c/archival: serde_fields for archival_metadata_stm structs - archival_metadata_stm::segment - archival_metadata_stm::start_offset_with_delta - archival_metadata_stm::truncate_archive_commit_cmd Signed-off-by: Oren Leiman --- src/v/cluster/archival/archival_metadata_stm.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/v/cluster/archival/archival_metadata_stm.cc b/src/v/cluster/archival/archival_metadata_stm.cc index b313d9eed200f..66db918912104 100644 --- a/src/v/cluster/archival/archival_metadata_stm.cc +++ b/src/v/cluster/archival/archival_metadata_stm.cc @@ -84,12 +84,17 @@ struct archival_metadata_stm::segment // Segment meta value generated by old redpanda versions will have the // default value of 'no'. segment_validated is_validated{segment_validated::no}; + + auto serde_fields() { + return std::tie(ntp_revision_deprecated, name, meta, is_validated); + } }; struct archival_metadata_stm::start_offset : public serde:: envelope, serde::compat_version<0>> { model::offset start_offset; + auto serde_fields() { return std::tie(start_offset); } }; struct archival_metadata_stm::start_offset_with_delta @@ -99,6 +104,7 @@ struct archival_metadata_stm::start_offset_with_delta serde::compat_version<0>> { model::offset start_offset; model::offset_delta delta; + auto serde_fields() { return std::tie(start_offset, delta); } }; struct archival_metadata_stm::add_segment_cmd { @@ -142,6 +148,7 @@ struct archival_metadata_stm::truncate_archive_commit_cmd { : serde::envelope, serde::compat_version<0>> { model::offset start_offset; uint64_t bytes_removed; + auto serde_fields() { return std::tie(start_offset, bytes_removed); } }; }; @@ -164,6 +171,8 @@ struct archival_metadata_stm::spillover_cmd static constexpr cmd_key key{9}; cloud_storage::segment_meta manifest_meta; + + auto serde_fields() { return std::tie(manifest_meta); } }; struct archival_metadata_stm::replace_manifest_cmd { @@ -208,6 +217,8 @@ struct archival_metadata_stm::read_write_fence_cmd static constexpr cmd_key key{14}; model::offset last_applied_offset; + + auto serde_fields() { return std::tie(last_applied_offset); } }; // Serde format description From 26a9b1d0946a1f970812e0e1c1e791ea49e87e27 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:06:59 -0700 Subject: [PATCH 06/29] c: serde_fields for client_quota_serde structs - entity_value - alter_quotas_request Signed-off-by: Oren Leiman --- src/v/cluster/client_quota_serde.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/v/cluster/client_quota_serde.h b/src/v/cluster/client_quota_serde.h index e605729cc53c1..747c47654b7e2 100644 --- a/src/v/cluster/client_quota_serde.h +++ b/src/v/cluster/client_quota_serde.h @@ -65,6 +65,8 @@ struct entity_key const client_id_default_match&, const client_id_default_match&) = default; + auto serde_fields() { return std::tie(); } + friend std::ostream& operator<<(std::ostream&, const client_id_default_match&); @@ -96,6 +98,8 @@ struct entity_key } ss::sstring value; + + auto serde_fields() { return std::tie(value); } }; /// client_id_prefix_match is the quota entity type corresponding to the @@ -122,8 +126,11 @@ struct entity_key } ss::sstring value; + auto serde_fields() { return std::tie(value); } }; + auto serde_fields() { return std::tie(part); } + serde::variant< client_id_default_match, client_id_match, @@ -166,6 +173,11 @@ struct entity_value std::optional producer_byte_rate; std::optional consumer_byte_rate; std::optional controller_mutation_rate; + + auto serde_fields() { + return std::tie( + producer_byte_rate, consumer_byte_rate, controller_mutation_rate); + } }; /// entity_value_diff describes the quotas diff for an entity_key @@ -237,6 +249,8 @@ struct alter_delta_cmd_data std::vector ops; + auto serde_fields() { return std::tie(ops); } + friend bool operator==(const alter_delta_cmd_data&, const alter_delta_cmd_data&) = default; @@ -286,6 +300,8 @@ struct alter_quotas_request alter_delta_cmd_data cmd_data; model::timeout_clock::duration timeout{}; + auto serde_fields() { return std::tie(cmd_data, timeout); } + friend bool operator==(const alter_quotas_request&, const alter_quotas_request&) = default; @@ -299,6 +315,7 @@ struct alter_quotas_response using rpc_adl_exempt = std::true_type; cluster::errc ec; + auto serde_fields() { return std::tie(ec); } friend bool operator==(const alter_quotas_response&, const alter_quotas_response&) From de2cad62beee5692b48763340bca38a187788f0f Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:07:38 -0700 Subject: [PATCH 07/29] c/node: serde_fields for local_state Signed-off-by: Oren Leiman --- src/v/cluster/node/types.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/v/cluster/node/types.h b/src/v/cluster/node/types.h index 324fcb71c989c..6acddf0ddde30 100644 --- a/src/v/cluster/node/types.h +++ b/src/v/cluster/node/types.h @@ -107,6 +107,10 @@ struct local_state uint64_t data_target_size{0}; uint64_t data_current_size{0}; uint64_t data_reclaimable_size{0}; + auto serde_fields() { + return std::tie( + data_target_size, data_current_size, data_reclaimable_size); + } friend bool operator==(const log_data_state&, const log_data_state&) = default; friend std::ostream& operator<<(std::ostream&, const log_data_state&); From d64ee9aa8c677afdd44150523648407b4c591c20 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:07:59 -0700 Subject: [PATCH 08/29] c: serde_fields for prefix_truncate_record Signed-off-by: Oren Leiman --- src/v/cluster/prefix_truncate_record.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/v/cluster/prefix_truncate_record.h b/src/v/cluster/prefix_truncate_record.h index 2ae9c7b13929c..728bfc3a5f4d5 100644 --- a/src/v/cluster/prefix_truncate_record.h +++ b/src/v/cluster/prefix_truncate_record.h @@ -35,6 +35,10 @@ struct prefix_truncate_record // May not be empty. kafka::offset kafka_start_offset{}; + + auto serde_fields() { + return std::tie(rp_start_offset, kafka_start_offset); + } }; } // namespace cluster From c85e87e6e3252c6e954ae9cbe79931ab3a05dbfd Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:08:15 -0700 Subject: [PATCH 09/29] c: serde_fields for self_test_rpc_types - empty_request - get_status_response - netcheck_request - netcheck_response Signed-off-by: Oren Leiman --- src/v/cluster/self_test_rpc_types.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/v/cluster/self_test_rpc_types.h b/src/v/cluster/self_test_rpc_types.h index e2841763e9ec7..e50976cdd88a9 100644 --- a/src/v/cluster/self_test_rpc_types.h +++ b/src/v/cluster/self_test_rpc_types.h @@ -335,6 +335,8 @@ struct empty_request : serde:: envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; + + auto serde_fields() { return std::tie(); } }; struct start_test_request @@ -386,6 +388,8 @@ struct get_status_response std::vector results; self_test_stage stage{}; + auto serde_fields() { return std::tie(id, status, results, stage); } + friend std::ostream& operator<<(std::ostream& o, const get_status_response& r) { fmt::print( @@ -405,6 +409,7 @@ struct netcheck_request using rpc_adl_exempt = std::true_type; model::node_id source; iobuf buf; + auto serde_fields() { return std::tie(source, buf); } friend std::ostream& operator<<(std::ostream& o, const netcheck_request& r) { fmt::print(o, "{{source: {} buf: {}}}", r.source, r.buf.size_bytes()); @@ -417,6 +422,9 @@ struct netcheck_response envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; size_t bytes_read{0}; + + auto serde_fields() { return std::tie(bytes_read); } + friend std::ostream& operator<<(std::ostream& o, const netcheck_response& r) { fmt::print(o, "{{bytes_read: {}}}", r.bytes_read); From 4d114dc19b379b821fe9590ed3702b1641e5eeaf Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:47:48 -0700 Subject: [PATCH 10/29] c: serde_fields for data_migration_types - cloud_storage_location - copy_target Signed-off-by: Oren Leiman --- src/v/cluster/data_migration_types.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/v/cluster/data_migration_types.h b/src/v/cluster/data_migration_types.h index 2932dc9bc9ee6..b94df302c400a 100644 --- a/src/v/cluster/data_migration_types.h +++ b/src/v/cluster/data_migration_types.h @@ -122,6 +122,8 @@ struct cloud_storage_location friend std::ostream& operator<<(std::ostream&, const cloud_storage_location&); + auto serde_fields() { return std::tie(); } + friend bool operator==(const cloud_storage_location&, const cloud_storage_location&) = default; @@ -201,6 +203,8 @@ struct copy_target : serde::envelope, serde::compat_version<0>> { ss::sstring bucket; + auto serde_fields() { return std::tie(bucket); } + friend bool operator==(const copy_target&, const copy_target&) = default; friend std::ostream& operator<<(std::ostream&, const copy_target&); }; From be74ea92c121e5e006e84df2c072d1633b2e837a Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:48:49 -0700 Subject: [PATCH 11/29] c: serde_fields for node_status_rpc_types - node_status_metadata - node_status_request Signed-off-by: Oren Leiman --- src/v/cluster/node_status_rpc_types.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/v/cluster/node_status_rpc_types.h b/src/v/cluster/node_status_rpc_types.h index ce51c461e3dcd..11a7b58eaf63e 100644 --- a/src/v/cluster/node_status_rpc_types.h +++ b/src/v/cluster/node_status_rpc_types.h @@ -27,6 +27,8 @@ struct node_status_metadata model::node_id node_id; + auto serde_fields() { return std::tie(node_id); } + friend std::ostream& operator<<(std::ostream& o, const node_status_metadata& nsm) { fmt::print(o, "{{node_id:{}}}", nsm.node_id); @@ -43,6 +45,8 @@ struct node_status_request node_status_metadata sender_metadata; + auto serde_fields() { return std::tie(sender_metadata); } + friend std::ostream& operator<<(std::ostream& o, const node_status_request& r) { fmt::print(o, "{{sender_metadata: {}}}", r.sender_metadata); @@ -57,6 +61,8 @@ struct node_status_reply node_status_metadata replier_metadata; + auto serde_fields() { return std::tie(replier_metadata); } + friend std::ostream& operator<<(std::ostream& o, const node_status_reply& r) { fmt::print(o, "{{replier_metadata: {}}}", r.replier_metadata); From 80498baec3a9514cd1e0ce26057e30636658bc03 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:51:39 -0700 Subject: [PATCH 12/29] c/recovery: serde_fields for status_request Signed-off-by: Oren Leiman --- src/v/cluster/topic_recovery_status_types.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v/cluster/topic_recovery_status_types.h b/src/v/cluster/topic_recovery_status_types.h index dc457e97e3386..9643a0f39fb52 100644 --- a/src/v/cluster/topic_recovery_status_types.h +++ b/src/v/cluster/topic_recovery_status_types.h @@ -22,6 +22,8 @@ struct status_request : serde:: envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; + + auto serde_fields() { return std::tie(); } }; struct topic_downloads From ba0e881a5a7bd7a6fb8e3cbba0d5712542a9477c Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:52:11 -0700 Subject: [PATCH 13/29] c/types: serde_fields for controller_committed_offset_request Signed-off-by: Oren Leiman --- src/v/cluster/types.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v/cluster/types.h b/src/v/cluster/types.h index 391d83994f8e0..32eff61a9ff2f 100644 --- a/src/v/cluster/types.h +++ b/src/v/cluster/types.h @@ -3133,6 +3133,8 @@ struct controller_committed_offset_request serde::version<0>, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; + + auto serde_fields() { return std::tie(); } }; struct controller_committed_offset_reply From 3ea3a741e9e1a23b25bee0be71dc23682309120a Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 13:50:04 -0700 Subject: [PATCH 14/29] c/test: serde_fields for commands_serialization_test Signed-off-by: Oren Leiman --- src/v/cluster/tests/commands_serialization_test.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/v/cluster/tests/commands_serialization_test.cc b/src/v/cluster/tests/commands_serialization_test.cc index 616445e52447f..c68f39ec42247 100644 --- a/src/v/cluster/tests/commands_serialization_test.cc +++ b/src/v/cluster/tests/commands_serialization_test.cc @@ -46,6 +46,8 @@ struct fake_serde_only_key using rpc_adl_exempt = std::true_type; ss::sstring str; + + auto serde_fields() { return std::tie(str); } }; struct fake_serde_only_val : serde::envelope< @@ -55,6 +57,8 @@ struct fake_serde_only_val using rpc_adl_exempt = std::true_type; ss::sstring str; + + auto serde_fields() { return std::tie(str); } }; using fake_serde_only_cmd = cluster::controller_command< fake_serde_only_key, From e6465bf2e7919ce6bc4b57efc02f55ba8aeeb1cd Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:09:00 -0700 Subject: [PATCH 15/29] rp: serde_fields for application::crash_tracker_metadata Signed-off-by: Oren Leiman --- src/v/redpanda/application.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/v/redpanda/application.h b/src/v/redpanda/application.h index 17b24f7cb953c..4682e9fe1316d 100644 --- a/src/v/redpanda/application.h +++ b/src/v/redpanda/application.h @@ -204,6 +204,10 @@ class application { uint32_t _crash_count{0}; uint64_t _config_checksum{0}; model::timestamp _last_start_ts; + + auto serde_fields() { + return std::tie(_crash_count, _config_checksum, _last_start_ts); + } }; // Constructs and starts the services required to provide cryptographic From 8f39ee3c3ae2d5a3533291f7d35deccaf6003989 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:10:31 -0700 Subject: [PATCH 16/29] storage: serde_fields for persisted_batches_map Signed-off-by: Oren Leiman --- src/v/storage/offset_translator_state.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/v/storage/offset_translator_state.cc b/src/v/storage/offset_translator_state.cc index f81282bf4a8e8..4deb67ad4acbc 100644 --- a/src/v/storage/offset_translator_state.cc +++ b/src/v/storage/offset_translator_state.cc @@ -354,6 +354,7 @@ struct persisted_batches_map serde::compat_version<0>> { int64_t start_delta = 0; chunked_vector batches; + auto serde_fields() { return std::tie(start_delta, batches); } }; } // namespace From 4d13a423642605c93c81ce92a7e9c98f5347a147 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:55:43 -0700 Subject: [PATCH 17/29] storage: serde_fields for clean_segment_value Signed-off-by: Oren Leiman --- src/v/storage/segment_utils.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/v/storage/segment_utils.h b/src/v/storage/segment_utils.h index 7da49e459de5e..365a69037fb15 100644 --- a/src/v/storage/segment_utils.h +++ b/src/v/storage/segment_utils.h @@ -209,6 +209,7 @@ struct clean_segment_value serde::version<0>, serde::compat_version<0>> { ss::sstring segment_name; + auto serde_fields() { return std::tie(segment_name); } }; inline bool From 73008ace7351d3a14c0f10fb3b23df15e6b311bd Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:53:08 -0700 Subject: [PATCH 18/29] feat: serde_fields for feature_table::version_fence Signed-off-by: Oren Leiman --- src/v/features/feature_table.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/v/features/feature_table.h b/src/v/features/feature_table.h index 6d87ce7eee36b..0489ed0e3ac61 100644 --- a/src/v/features/feature_table.h +++ b/src/v/features/feature_table.h @@ -550,6 +550,7 @@ class feature_table { : serde:: envelope, serde::compat_version<0>> { cluster::cluster_version active_version; + auto serde_fields() { return std::tie(active_version); } }; static constexpr std::string_view version_fence_batch_key = "state"; From f246b22088343ae80b0a6d9f93cfbc49db109d8a Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 11:54:23 -0700 Subject: [PATCH 19/29] model/x: serde_fields for transform_offset_options Signed-off-by: Oren Leiman --- src/v/model/transform.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/v/model/transform.h b/src/v/model/transform.h index 4eff8e17ac963..66b3ae6c91732 100644 --- a/src/v/model/transform.h +++ b/src/v/model/transform.h @@ -160,6 +160,7 @@ struct transform_offset_options : serde:: envelope, serde::compat_version<0>> { bool operator==(const latest_offset&) const = default; + auto serde_fields() { return std::tie(); } }; // A transform can either start at the latest offset, at a timestamp, or at // some delta from the start or end of an input partition. From 84471dd861dc555c58fee57a0cce8d6aac513d8b Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 21 Aug 2024 13:50:57 -0700 Subject: [PATCH 20/29] m/x/tests: serde_fields for legacy_transform_offset_options_2 Signed-off-by: Oren Leiman --- src/v/model/tests/transform_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/v/model/tests/transform_test.cc b/src/v/model/tests/transform_test.cc index 90967f49b349b..0aaef82bed390 100644 --- a/src/v/model/tests/transform_test.cc +++ b/src/v/model/tests/transform_test.cc @@ -209,6 +209,7 @@ struct legacy_transform_offset_options_2 : serde:: envelope, serde::compat_version<0>> { bool operator==(const latest_offset&) const = default; + auto serde_fields() { return std::tie(); } }; serde::variant position; bool operator==(const legacy_transform_offset_options_2&) const = default; From 3eab8ee8ff86da0811e7f104eaa3ba6467c592ef Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:09:51 -0700 Subject: [PATCH 21/29] serde/test: serde_fields for various structs Signed-off-by: Oren Leiman --- src/v/serde/test/serde_test.cc | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/v/serde/test/serde_test.cc b/src/v/serde/test/serde_test.cc index 8050dc4ad9eae..c5ff11fa41ad4 100644 --- a/src/v/serde/test/serde_test.cc +++ b/src/v/serde/test/serde_test.cc @@ -67,6 +67,7 @@ struct test_msg0 bool operator==(const test_msg0&) const = default; char _i, _j; + auto serde_fields() { return std::tie(_i, _j); } }; struct test_msg1 @@ -76,6 +77,8 @@ struct test_msg1 int _a; test_msg0 _m; int _b, _c; + + auto serde_fields() { return std::tie(_a, _m, _b, _c); } }; struct test_msg1_imcompatible @@ -83,6 +86,8 @@ struct test_msg1_imcompatible bool operator==(const test_msg1_imcompatible&) const = default; test_msg0 _m; + + auto serde_fields() { return std::tie(_m); } }; struct test_msg1_new @@ -93,6 +98,7 @@ struct test_msg1_new int _a; test_msg0 _m; int _b, _c; + auto serde_fields() { return std::tie(_a, _m, _b, _c); } }; struct test_msg1_new_manual { @@ -157,6 +163,7 @@ SEASTAR_THREAD_TEST_CASE(envelope_too_big_test) { bool operator==(const big&) const = default; std::vector data_; + auto serde_fields() { return std::tie(data_); } }; auto too_big = std::make_unique(); @@ -171,6 +178,8 @@ SEASTAR_THREAD_TEST_CASE(simple_envelope_test) { bool operator==(const msg&) const = default; int32_t _i, _j; + + auto serde_fields() { return std::tie(_i, _j); } }; auto b = iobuf(); @@ -264,6 +273,8 @@ struct inner_differing_sizes bool operator==(const inner_differing_sizes&) const = default; std::vector _ints; + + auto serde_fields() { return std::tie(_ints); } }; struct complex_msg @@ -272,6 +283,8 @@ struct complex_msg bool operator==(const complex_msg&) const = default; int32_t _x; + + auto serde_fields() { return std::tie(_vec, _x); } }; static_assert(serde::is_envelope); @@ -508,6 +521,8 @@ struct small bool operator==(const small&) const = default; int a, b, c; + + auto serde_fields() { return std::tie(a, b, c); } }; struct big @@ -515,6 +530,8 @@ struct big bool operator==(const big&) const = default; int a, b, c, d{0x1234}; + + auto serde_fields() { return std::tie(a, b, c, d); } }; SEASTAR_THREAD_TEST_CASE(compat_test_added_field) { @@ -573,6 +590,7 @@ SEASTAR_THREAD_TEST_CASE(compat_test_half_field_1) { int a, b; short c; + auto serde_fields() { return std::tie(a, b, c); } }; auto b = serde::to_iobuf(half_field_1{.a = 1, .b = 2, .c = 3}); @@ -588,6 +606,7 @@ SEASTAR_THREAD_TEST_CASE(compat_test_half_field_2) { int a, b, c; short d; + auto serde_fields() { return std::tie(a, b, c, d); } }; auto b = serde::to_iobuf(half_field_2{.a = 1, .b = 2, .c = 3, .d = 0x1234}); @@ -611,6 +630,7 @@ SEASTAR_THREAD_TEST_CASE(serde_checksum_envelope_test) { bool operator==(const checksummed&) const = default; std::vector data_; + auto serde_fields() { return std::tie(data_); } }; auto const obj = checksummed{ @@ -632,6 +652,7 @@ struct old_no_cs bool operator==(const old_no_cs&) const = default; std::vector data_; + auto serde_fields() { return std::tie(data_); } }; struct new_cs : public serde:: @@ -657,6 +678,7 @@ struct new_cs bool operator==(const new_cs&) const = default; std::vector data_; + auto serde_fields() { return std::tie(data_); } }; SEASTAR_THREAD_TEST_CASE(serde_checksum_update) { @@ -693,6 +715,7 @@ struct old_cs bool operator==(const old_cs&) const = default; std::vector data_; + auto serde_fields() { return std::tie(data_); } }; struct new_no_cs : public serde:: @@ -703,6 +726,8 @@ struct new_no_cs serde::checksum_t unchecked_dummy_checksum_{0U}; std::vector data_; + + auto serde_fields() { return std::tie(unchecked_dummy_checksum_, data_); } }; SEASTAR_THREAD_TEST_CASE(serde_checksum_update_1) { From 7019be2277b74d723ae7ea6958338cfd3416214e Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 11:17:42 -0700 Subject: [PATCH 22/29] serde/test: serde_fields for serde_rpbench structs Signed-off-by: Oren Leiman --- src/v/serde/test/bench.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/v/serde/test/bench.cc b/src/v/serde/test/bench.cc index ffb91bdd49484..a2f25692bc65d 100644 --- a/src/v/serde/test/bench.cc +++ b/src/v/serde/test/bench.cc @@ -26,6 +26,8 @@ struct small_t int16_t b = 2; int32_t c = 3; int64_t d = 4; + + auto serde_fields() { return std::tie(a, b, c, d); } }; static_assert(sizeof(small_t) == 16, "one more byte for padding"); @@ -47,6 +49,7 @@ struct big_t : public serde::envelope, serde::compat_version<2>> { small_t s; iobuf data; + auto serde_fields() { return std::tie(s, data); } }; inline big_t gen_big(size_t data_size, size_t chunk_size) { From e3b8d0b36ed2b2b27cf83cb84251ba11509c7ed5 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 10:11:09 -0700 Subject: [PATCH 23/29] serde/test: update jinja to generate serde_fields for gen'ed structs Signed-off-by: Oren Leiman --- src/v/serde/test/struct_gen.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/v/serde/test/struct_gen.py b/src/v/serde/test/struct_gen.py index 70a836c4d1d2e..89102d5458d85 100755 --- a/src/v/serde/test/struct_gen.py +++ b/src/v/serde/test/struct_gen.py @@ -125,6 +125,17 @@ def __str__(self): {{ field }} {%- endfor %} {%- endfor %} + + auto serde_fields() { + return std::tie( +{% for generation in s._field_generations %} + {%- for field in generation %} + {{ field._name}}{{ ", " if not loop.last else "" }} + {%- endfor %} + {{ ", " if not loop.last else "" }} +{%- endfor %} + ); + } }; """).render(s=self) From 96a251de950915f81285e915400566778782c026 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 16:05:28 -0700 Subject: [PATCH 24/29] cs/test: serde_fields for various structs in partition_manifest_test Signed-off-by: Oren Leiman --- .../tests/partition_manifest_test.cc | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/v/cloud_storage/tests/partition_manifest_test.cc b/src/v/cloud_storage/tests/partition_manifest_test.cc index 279cc371d1b0a..3a591fceaef86 100644 --- a/src/v/cloud_storage/tests/partition_manifest_test.cc +++ b/src/v/cloud_storage/tests/partition_manifest_test.cc @@ -1537,6 +1537,8 @@ struct metadata_stm_segment cloud_storage::segment_name name; cloud_storage::partition_manifest::segment_meta meta; + auto serde_fields() { return std::tie(name, meta); } + bool operator==(const metadata_stm_segment&) const = default; }; @@ -1554,6 +1556,17 @@ struct segment_meta_v0 { model::timestamp max_timestamp; model::offset delta_offset; + auto serde_fields() { + return std::tie( + is_compacted, + size_bytes, + base_offset, + committed_offset, + base_timestamp, + max_timestamp, + delta_offset); + } + auto operator<=>(const segment_meta_v0&) const = default; }; struct segment_meta_v1 { @@ -1572,6 +1585,19 @@ struct segment_meta_v1 { model::initial_revision_id ntp_revision; model::term_id archiver_term; + auto serde_fields() { + return std::tie( + is_compacted, + size_bytes, + base_offset, + committed_offset, + base_timestamp, + max_timestamp, + delta_offset, + ntp_revision, + archiver_term); + } + auto operator<=>(const segment_meta_v1&) const = default; }; @@ -1584,6 +1610,8 @@ struct metadata_stm_segment cloud_storage::segment_name name; segment_meta_t meta; + auto serde_fields() { return std::tie(name, meta); } + bool operator==(const metadata_stm_segment&) const = default; }; From 2a2413bb86e31a7dd359c60240d97d77a2da105b Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 16:05:58 -0700 Subject: [PATCH 25/29] c/test: serde_fields for archival_metadata_stm_test Signed-off-by: Oren Leiman --- src/v/cluster/archival/tests/archival_metadata_stm_test.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/v/cluster/archival/tests/archival_metadata_stm_test.cc b/src/v/cluster/archival/tests/archival_metadata_stm_test.cc index 76914eace0493..45cc7da9855d0 100644 --- a/src/v/cluster/archival/tests/archival_metadata_stm_test.cc +++ b/src/v/cluster/archival/tests/archival_metadata_stm_test.cc @@ -566,6 +566,10 @@ struct segment model::initial_revision_id ntp_revision_deprecated; cloud_storage::segment_name name; cloud_storage::partition_manifest::segment_meta meta; + + auto serde_fields() { + return std::tie(ntp_revision_deprecated, name, meta); + } }; struct snapshot @@ -573,6 +577,8 @@ struct snapshot envelope, serde::compat_version<0>> { /// List of segments std::vector segments; + + auto serde_fields() { return std::tie(segments); } }; } // namespace old From 4f45da58dc705897387ff3dbea2ba59251d31671 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 16:06:38 -0700 Subject: [PATCH 26/29] rpc/test: serde_fields for various rpc_gen_types Signed-off-by: Oren Leiman --- src/v/rpc/test/rpc_gen_types.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/v/rpc/test/rpc_gen_types.h b/src/v/rpc/test/rpc_gen_types.h index 7ca9debccd716..6437fef4c73e0 100644 --- a/src/v/rpc/test/rpc_gen_types.h +++ b/src/v/rpc/test/rpc_gen_types.h @@ -45,18 +45,21 @@ struct echo_req : serde::envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; ss::sstring str; + auto serde_fields() { return std::tie(str); } }; struct echo_resp : serde::envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; ss::sstring str; + auto serde_fields() { return std::tie(str); } }; struct cnt_req : serde::envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; uint64_t expected; + auto serde_fields() { return std::tie(expected); } }; struct cnt_resp @@ -64,18 +67,21 @@ struct cnt_resp using rpc_adl_exempt = std::true_type; uint64_t expected; uint64_t current; + auto serde_fields() { return std::tie(expected, current); } }; struct sleep_req : serde::envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; uint64_t secs; + auto serde_fields() { return std::tie(secs); } }; struct sleep_resp : serde::envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; ss::sstring str; + auto serde_fields() { return std::tie(str); } }; enum class failure_type { throw_exception, exceptional_future, none }; @@ -84,12 +90,15 @@ struct throw_req : serde::envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; failure_type type; + auto serde_fields() { return std::tie(type); } }; struct throw_resp : serde::envelope, serde::compat_version<0>> { using rpc_adl_exempt = std::true_type; ss::sstring reply; + + auto serde_fields() { return std::tie(reply); } }; struct echo_req_serde_only From 315be733af8f2bb7ef24fbc222afb3f60cc58173 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 16:07:03 -0700 Subject: [PATCH 27/29] utils/test: serde_fields for uuid_test::uuid_struct Signed-off-by: Oren Leiman --- src/v/utils/tests/uuid_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v/utils/tests/uuid_test.cc b/src/v/utils/tests/uuid_test.cc index 4b5f80d018df2..0308353e9b289 100644 --- a/src/v/utils/tests/uuid_test.cc +++ b/src/v/utils/tests/uuid_test.cc @@ -66,6 +66,8 @@ struct uuid_struct std::optional opt2; std::vector vec; std::vector> opt_vec; + + auto serde_fields() { return std::tie(single, opt1, opt2, vec, opt_vec); } }; template From 9cb7bceea7633b9b9493267d50c316b0e10d4afb Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 16:07:21 -0700 Subject: [PATCH 28/29] s/mvlog: serde_fields for record_batch_entry_body Signed-off-by: Oren Leiman --- src/v/storage/mvlog/entry.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/v/storage/mvlog/entry.h b/src/v/storage/mvlog/entry.h index 53c791f25b9ab..3f427a5d7ba5e 100644 --- a/src/v/storage/mvlog/entry.h +++ b/src/v/storage/mvlog/entry.h @@ -43,6 +43,8 @@ struct record_batch_entry_body // The term of the record batch. model::term_id term; + + auto serde_fields() { return std::tie(record_batch_header, records, term); } }; // Container for the deserialized bytes from an entry. Note that this isn't an From 1c0ee0f015c0e8116b4ad468cc3c3ff4faf72341 Mon Sep 17 00:00:00 2001 From: Oren Leiman Date: Wed, 7 Aug 2024 11:21:21 -0700 Subject: [PATCH 29/29] serde: Require T::serde_fields for all serde structs With the exception of those that implement BOTH a custom reader and a custom writer. In that case, the fields tuple is not needed, so envelope_to_tuple will not participate in overload resolution. Signed-off-by: Oren Leiman --- src/v/serde/envelope_for_each_field.h | 171 -------------------------- 1 file changed, 171 deletions(-) diff --git a/src/v/serde/envelope_for_each_field.h b/src/v/serde/envelope_for_each_field.h index 935a2d8ab7907..a784b37b96923 100644 --- a/src/v/serde/envelope_for_each_field.h +++ b/src/v/serde/envelope_for_each_field.h @@ -29,177 +29,6 @@ constexpr inline auto envelope_to_tuple(T&& t) { return t.serde_fields(); } -template -requires(!detail::has_serde_fields) -constexpr inline auto envelope_to_tuple(T& t) { - static_assert(std::is_aggregate_v); - static_assert(std::is_standard_layout_v); - static_assert(!std::is_polymorphic_v); - - constexpr auto const a = reflection::arity() - 1; - if constexpr (a == 0) { - return std::tie(); - } else if constexpr (a == 1) { - auto& [p1] = t; - return std::tie(p1); - } else if constexpr (a == 2) { - auto& [p1, p2] = t; - return std::tie(p1, p2); - } else if constexpr (a == 3) { - auto& [p1, p2, p3] = t; - return std::tie(p1, p2, p3); - } else if constexpr (a == 4) { - auto& [p1, p2, p3, p4] = t; - return std::tie(p1, p2, p3, p4); - } else if constexpr (a == 5) { - auto& [p1, p2, p3, p4, p5] = t; - return std::tie(p1, p2, p3, p4, p5); - } else if constexpr (a == 6) { - auto& [p1, p2, p3, p4, p5, p6] = t; - return std::tie(p1, p2, p3, p4, p5, p6); - } else if constexpr (a == 7) { - auto& [p1, p2, p3, p4, p5, p6, p7] = t; - return std::tie(p1, p2, p3, p4, p5, p6, p7); - } else if constexpr (a == 8) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8] = t; - return std::tie(p1, p2, p3, p4, p5, p6, p7, p8); - } else if constexpr (a == 9) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9] = t; - return std::tie(p1, p2, p3, p4, p5, p6, p7, p8, p9); - } else if constexpr (a == 10) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10] = t; - return std::tie(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10); - } else if constexpr (a == 11) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11] = t; - return std::tie(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11); - } else if constexpr (a == 12) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12] = t; - return std::tie(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12); - } else if constexpr (a == 13) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13] = t; - return std::tie(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13); - } else if constexpr (a == 14) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14] = t; - return std::tie( - p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14); - } else if constexpr (a == 15) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15] - = t; - return std::tie( - p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15); - } else if constexpr (a == 16) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16] - = t; - return std::tie( - p1, - p2, - p3, - p4, - p5, - p6, - p7, - p8, - p9, - p10, - p11, - p12, - p13, - p14, - p15, - p16); - } else if constexpr (a == 17) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17] - = t; - return std::tie( - p1, - p2, - p3, - p4, - p5, - p6, - p7, - p8, - p9, - p10, - p11, - p12, - p13, - p14, - p15, - p16, - p17); - } else if constexpr (a == 18) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18] - = t; - return std::tie( - p1, - p2, - p3, - p4, - p5, - p6, - p7, - p8, - p9, - p10, - p11, - p12, - p13, - p14, - p15, - p16, - p17, - p18); - } else if constexpr (a == 19) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19] - = t; - return std::tie( - p1, - p2, - p3, - p4, - p5, - p6, - p7, - p8, - p9, - p10, - p11, - p12, - p13, - p14, - p15, - p16, - p17, - p18, - p19); - } else if constexpr (a == 20) { - auto& [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20] - = t; - return std::tie( - p1, - p2, - p3, - p4, - p5, - p6, - p7, - p8, - p9, - p10, - p11, - p12, - p13, - p14, - p15, - p16, - p17, - p18, - p19, - p20); - } -} - template concept check_for_more_fn = requires(Fn&& fn, int& f) { { fn(f) } -> std::convertible_to;