From d7f290e67e7e276f8b9a6a93b0575e7af672c7e0 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jan 2023 16:39:48 +0100 Subject: [PATCH 1/6] Add snapshot list message --- mithril-common/src/messages/mod.rs | 2 + mithril-common/src/messages/snapshot_list.rs | 97 ++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 mithril-common/src/messages/snapshot_list.rs diff --git a/mithril-common/src/messages/mod.rs b/mithril-common/src/messages/mod.rs index cf98f091e26..8aba3af1bc5 100644 --- a/mithril-common/src/messages/mod.rs +++ b/mithril-common/src/messages/mod.rs @@ -6,6 +6,7 @@ mod epoch_settings; mod register_signature; mod register_signer; mod snapshot; +mod snapshot_list; pub use certificate::CertificateMessage; pub use certificate_pending::{CertificatePendingMessage, SignerMessage}; @@ -13,3 +14,4 @@ pub use epoch_settings::EpochSettingsMessage; pub use register_signature::RegisterSignatureMessage; pub use register_signer::RegisterSignerMessage; pub use snapshot::SnapshotMessage; +pub use snapshot_list::{SnapshotListItemMessage, SnapshotListMessage}; diff --git a/mithril-common/src/messages/snapshot_list.rs b/mithril-common/src/messages/snapshot_list.rs new file mode 100644 index 00000000000..2362014b0d6 --- /dev/null +++ b/mithril-common/src/messages/snapshot_list.rs @@ -0,0 +1,97 @@ +use serde::{Deserialize, Serialize}; + +// TODO: We should probably not rely on entities when defining the message +use crate::entities::Beacon; + +// TODO: We should probably not rely on entities when defining the message +#[cfg(any(test, feature = "test_only"))] +use crate::entities::Epoch; +/// Message structure of a snapshot list +pub type SnapshotListMessage = Vec; + +/// Message structure of a snapshot list item +#[derive(Clone, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct SnapshotListItemMessage { + /// Digest that is signed by the signer participants + pub digest: String, + + /// Mithril beacon on the Cardano chain + pub beacon: Beacon, + + /// Hash of the associated certificate + pub certificate_hash: String, + + /// Size of the snapshot file in Bytes + pub size: u64, + + /// Date and time at which the snapshot was created + pub created_at: String, + + /// Locations where the binary content of the snapshot can be retrieved + pub locations: Vec, +} + +impl SnapshotListItemMessage { + #[cfg(any(test, feature = "test_only"))] + /// Return a dummy test entity (test-only). + pub fn dummy() -> Self { + Self { + digest: "0b9f5ad7f33cc523775c82249294eb8a1541d54f08eb3107cafc5638403ec7c6".to_string(), + beacon: Beacon { + network: "preview".to_string(), + epoch: Epoch(86), + immutable_file_number: 1728, + }, + certificate_hash: "d5daf6c03ace4a9c074e951844075b9b373bafc4e039160e3e2af01823e9abfb" + .to_string(), + size: 807803196, + created_at: "2023-01-19T13:43:05.618857482Z".to_string(), + locations: vec!["https://host/certificate.tar.gz".to_string()], + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn golden_message() -> SnapshotListMessage { + vec![SnapshotListItemMessage { + digest: "0b9f5ad7f33cc523775c82249294eb8a1541d54f08eb3107cafc5638403ec7c6".to_string(), + beacon: Beacon { + network: "preview".to_string(), + epoch: Epoch(86), + immutable_file_number: 1728, + }, + certificate_hash: "d5daf6c03ace4a9c074e951844075b9b373bafc4e039160e3e2af01823e9abfb" + .to_string(), + size: 807803196, + created_at: "2023-01-19T13:43:05.618857482Z".to_string(), + locations: vec!["https://host/certificate.tar.gz".to_string()], + }] + } + + // Test the retro compatibility with possible future upgrades. + #[test] + fn test_v1() { + let json = r#"[{ +"digest": "0b9f5ad7f33cc523775c82249294eb8a1541d54f08eb3107cafc5638403ec7c6", +"beacon": { + "network": "preview", + "epoch": 86, + "immutable_file_number": 1728 +}, +"certificate_hash": "d5daf6c03ace4a9c074e951844075b9b373bafc4e039160e3e2af01823e9abfb", +"size": 807803196, +"created_at": "2023-01-19T13:43:05.618857482Z", +"locations": [ + "https://host/certificate.tar.gz" +] +}]"#; + let message: SnapshotListMessage = serde_json::from_str(json).expect( + "This JSON is expected to be succesfully parsed into a SnapshotListMessage instance.", + ); + + assert_eq!(golden_message(), message); + } +} From d6262646a765f50ca3cf50e0a4c57f942a56ec4a Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jan 2023 17:19:21 +0100 Subject: [PATCH 2/6] Add snapshot list message adapter to client --- mithril-client/src/aggregator.rs | 10 +++-- mithril-client/src/lib.rs | 4 +- .../from_snapshot_list_message.rs | 37 +++++++++++++++++++ mithril-client/src/message_adapters/mod.rs | 2 + 4 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 mithril-client/src/message_adapters/from_snapshot_list_message.rs diff --git a/mithril-client/src/aggregator.rs b/mithril-client/src/aggregator.rs index 03ed80cc83e..2095794ad2a 100644 --- a/mithril-client/src/aggregator.rs +++ b/mithril-client/src/aggregator.rs @@ -16,11 +16,13 @@ use thiserror::Error; use mithril_common::{ certificate_chain::{CertificateRetriever, CertificateRetrieverError}, entities::{Certificate, Snapshot}, - messages::{CertificateMessage, SnapshotMessage}, + messages::{CertificateMessage, SnapshotListMessage, SnapshotMessage}, MITHRIL_API_VERSION, }; -use crate::{FromCertificateMessageAdapter, FromSnapshotMessageAdapter}; +use crate::{ + FromCertificateMessageAdapter, FromSnapshotListMessageAdapter, FromSnapshotMessageAdapter, +}; /// [AggregatorHandler] related errors. #[derive(Error, Debug)] @@ -173,8 +175,8 @@ impl AggregatorHandler for AggregatorHTTPClient { match response { Ok(response) => match response.status() { - StatusCode::OK => match response.json::>().await { - Ok(snapshots) => Ok(snapshots), + StatusCode::OK => match response.json::().await { + Ok(snapshots) => Ok(FromSnapshotListMessageAdapter::adapt(snapshots)), Err(err) => Err(AggregatorHandlerError::JsonParseFailed(err.to_string())), }, StatusCode::PRECONDITION_FAILED => Err(self.handle_api_error(&response)), diff --git a/mithril-client/src/lib.rs b/mithril-client/src/lib.rs index 94f73a06858..a3362f3b3b6 100644 --- a/mithril-client/src/lib.rs +++ b/mithril-client/src/lib.rs @@ -19,7 +19,9 @@ mod runtime; pub use aggregator::{AggregatorHTTPClient, AggregatorHandler, AggregatorHandlerError}; pub use entities::Config; -pub use message_adapters::{FromCertificateMessageAdapter, FromSnapshotMessageAdapter}; +pub use message_adapters::{ + FromCertificateMessageAdapter, FromSnapshotListMessageAdapter, FromSnapshotMessageAdapter, +}; pub use runtime::{Runtime, RuntimeError}; pub use runtime::convert_to_field_items; diff --git a/mithril-client/src/message_adapters/from_snapshot_list_message.rs b/mithril-client/src/message_adapters/from_snapshot_list_message.rs new file mode 100644 index 00000000000..ee2209a34a1 --- /dev/null +++ b/mithril-client/src/message_adapters/from_snapshot_list_message.rs @@ -0,0 +1,37 @@ +use mithril_common::entities::Snapshot; +use mithril_common::messages::SnapshotListMessage; + +/// Adapter to convert [SnapshotListMessage] to [SnapshotList] instances +pub struct FromSnapshotListMessageAdapter; + +impl FromSnapshotListMessageAdapter { + /// Method to trigger the conversion + pub fn adapt(snapshot_list_message: SnapshotListMessage) -> Vec { + snapshot_list_message + .into_iter() + .map(|snapshot_list_item_message| Snapshot { + digest: snapshot_list_item_message.digest, + beacon: snapshot_list_item_message.beacon, + certificate_hash: snapshot_list_item_message.certificate_hash, + size: snapshot_list_item_message.size, + created_at: snapshot_list_item_message.created_at, + locations: snapshot_list_item_message.locations, + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use mithril_common::messages::SnapshotListItemMessage; + + use super::*; + + #[test] + fn adapt_ok() { + let snapshot_list_message: SnapshotListMessage = vec![SnapshotListItemMessage::dummy()]; + let snapshot_list = FromSnapshotListMessageAdapter::adapt(snapshot_list_message.clone()); + + assert_eq!(snapshot_list_message[0].digest, snapshot_list[0].digest); + } +} diff --git a/mithril-client/src/message_adapters/mod.rs b/mithril-client/src/message_adapters/mod.rs index 0847784a244..2575d06db02 100644 --- a/mithril-client/src/message_adapters/mod.rs +++ b/mithril-client/src/message_adapters/mod.rs @@ -1,5 +1,7 @@ mod from_certificate_message_adapter; +mod from_snapshot_list_message; mod from_snapshot_message; pub use from_certificate_message_adapter::FromCertificateMessageAdapter; +pub use from_snapshot_list_message::FromSnapshotListMessageAdapter; pub use from_snapshot_message::FromSnapshotMessageAdapter; From 1dbd1a426182664e1e2bf6a818c1ef64d644146e Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jan 2023 17:46:45 +0100 Subject: [PATCH 3/6] Add snapshot list message adapter to aggregator --- .../src/http_server/routes/snapshot_routes.rs | 7 +++- .../src/message_adapters/mod.rs | 2 + .../to_snapshot_list_message.rs | 38 +++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 mithril-aggregator/src/message_adapters/to_snapshot_list_message.rs diff --git a/mithril-aggregator/src/http_server/routes/snapshot_routes.rs b/mithril-aggregator/src/http_server/routes/snapshot_routes.rs index 5abc71f9e97..795d3ab0aef 100644 --- a/mithril-aggregator/src/http_server/routes/snapshot_routes.rs +++ b/mithril-aggregator/src/http_server/routes/snapshot_routes.rs @@ -57,7 +57,7 @@ fn snapshot_digest( mod handlers { use crate::http_server::routes::reply; use crate::http_server::SERVER_BASE_PATH; - use crate::message_adapters::ToSnapshotMessageAdapter; + use crate::message_adapters::{ToSnapshotListMessageAdapter, ToSnapshotMessageAdapter}; use crate::{Configuration, SnapshotStore}; use slog_scope::{debug, warn}; use std::convert::Infallible; @@ -72,7 +72,10 @@ mod handlers { debug!("⇄ HTTP SERVER: snapshots"); match snapshot_store.list_snapshots().await { - Ok(snapshots) => Ok(reply::json(&snapshots, StatusCode::OK)), + Ok(snapshots) => Ok(reply::json( + &ToSnapshotListMessageAdapter::adapt(snapshots), + StatusCode::OK, + )), Err(err) => { warn!("snapshots::error"; "error" => ?err); Ok(reply::internal_server_error(err.to_string())) diff --git a/mithril-aggregator/src/message_adapters/mod.rs b/mithril-aggregator/src/message_adapters/mod.rs index c6737ec949b..530e7f5ffc0 100644 --- a/mithril-aggregator/src/message_adapters/mod.rs +++ b/mithril-aggregator/src/message_adapters/mod.rs @@ -3,6 +3,7 @@ mod from_register_signer; mod to_certificate_message; mod to_certificate_pending_message; mod to_epoch_settings_message; +mod to_snapshot_list_message; mod to_snasphot_message; pub use from_register_signature::FromRegisterSingleSignatureAdapter; @@ -10,4 +11,5 @@ pub use from_register_signer::FromRegisterSignerAdapter; pub use to_certificate_message::ToCertificateMessageAdapter; pub use to_certificate_pending_message::ToCertificatePendingMessageAdapter; pub use to_epoch_settings_message::ToEpochSettingsMessageAdapter; +pub use to_snapshot_list_message::ToSnapshotListMessageAdapter; pub use to_snasphot_message::ToSnapshotMessageAdapter; diff --git a/mithril-aggregator/src/message_adapters/to_snapshot_list_message.rs b/mithril-aggregator/src/message_adapters/to_snapshot_list_message.rs new file mode 100644 index 00000000000..15277caae50 --- /dev/null +++ b/mithril-aggregator/src/message_adapters/to_snapshot_list_message.rs @@ -0,0 +1,38 @@ +use mithril_common::entities::Snapshot; +use mithril_common::messages::{SnapshotListItemMessage, SnapshotListMessage}; + +/// Adapter to convert a list of [Snapshot] to [SnapshotListMessage] instances +pub struct ToSnapshotListMessageAdapter; + +impl ToSnapshotListMessageAdapter { + /// Method to trigger the conversion + pub fn adapt(snapshots: Vec) -> SnapshotListMessage { + snapshots + .into_iter() + .map(|snapshot| SnapshotListItemMessage { + digest: snapshot.digest, + beacon: snapshot.beacon, + certificate_hash: snapshot.certificate_hash, + size: snapshot.size, + created_at: snapshot.created_at, + locations: snapshot.locations, + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use mithril_common::test_utils::fake_data; + + use super::*; + + #[test] + fn adapt_ok() { + let mut snapshot = fake_data::snapshots(1)[0].to_owned(); + snapshot.digest = "digest123".to_string(); + let snapshot_list_message = ToSnapshotListMessageAdapter::adapt(vec![snapshot]); + + assert_eq!("digest123".to_string(), snapshot_list_message[0].digest); + } +} From ea9a069f678f7b66870cee48e80e7d490918bc87 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 19 Jan 2023 17:53:51 +0100 Subject: [PATCH 4/6] Update OpenAPI specs --- openapi.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index f6fdbaa33aa..c09fa8fe080 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -112,9 +112,7 @@ paths: content: application/json: schema: - type: array - items: - $ref: "#/components/schemas/Snapshot" + $ref: "#/components/schemas/SnapshotListMessage" "412": description: API version mismatch default: @@ -613,6 +611,12 @@ components: "genesis_signature": "", } + SnapshotListMessage: + description: SnapshotListMessage represents a list of snapshots + type: array + items: + $ref: "#/components/schemas/Snapshot" + Snapshot: description: Snapshot represents a snapshot file and its metadata type: object From 7465349ba3244e9fb56d30fdb8a465c61496d226 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 23 Jan 2023 10:56:36 +0100 Subject: [PATCH 5/6] Remove TODO usage entities in message --- mithril-common/src/messages/certificate.rs | 2 -- mithril-common/src/messages/snapshot.rs | 2 -- mithril-common/src/messages/snapshot_list.rs | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/mithril-common/src/messages/certificate.rs b/mithril-common/src/messages/certificate.rs index b05d1e85484..011fb0e365b 100644 --- a/mithril-common/src/messages/certificate.rs +++ b/mithril-common/src/messages/certificate.rs @@ -1,10 +1,8 @@ use serde::{Deserialize, Serialize}; -// TODO: We should probably not rely on entities when defining the message use crate::entities::{Beacon, CertificateMetadata, ProtocolMessage}; #[cfg(any(test, feature = "test_only"))] -// TODO: We should probably not rely on entities when defining the message use crate::entities::{ProtocolMessagePartKey, ProtocolParameters, SignerWithStake}; /// Message structure of a certificate diff --git a/mithril-common/src/messages/snapshot.rs b/mithril-common/src/messages/snapshot.rs index b9191a14219..1efc619abc6 100644 --- a/mithril-common/src/messages/snapshot.rs +++ b/mithril-common/src/messages/snapshot.rs @@ -1,9 +1,7 @@ use serde::{Deserialize, Serialize}; -// TODO: We should probably not rely on entities when defining the message use crate::entities::Beacon; -// TODO: We should probably not rely on entities when defining the message #[cfg(any(test, feature = "test_only"))] use crate::entities::Epoch; /// Message structure of a snapshot diff --git a/mithril-common/src/messages/snapshot_list.rs b/mithril-common/src/messages/snapshot_list.rs index 2362014b0d6..a32857aea12 100644 --- a/mithril-common/src/messages/snapshot_list.rs +++ b/mithril-common/src/messages/snapshot_list.rs @@ -1,11 +1,10 @@ use serde::{Deserialize, Serialize}; -// TODO: We should probably not rely on entities when defining the message use crate::entities::Beacon; -// TODO: We should probably not rely on entities when defining the message #[cfg(any(test, feature = "test_only"))] use crate::entities::Epoch; + /// Message structure of a snapshot list pub type SnapshotListMessage = Vec; From 954d19de77907c4002a030727971fc7a6ade2745 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 23 Jan 2023 15:07:28 +0100 Subject: [PATCH 6/6] Update crates version --- Cargo.lock | 8 ++++---- mithril-aggregator/Cargo.toml | 2 +- mithril-client/Cargo.toml | 2 +- mithril-common/Cargo.toml | 2 +- mithril-signer/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fcbd59527ee..4ac31642539 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2001,7 +2001,7 @@ dependencies = [ [[package]] name = "mithril-aggregator" -version = "0.2.7" +version = "0.2.8" dependencies = [ "async-trait", "chrono", @@ -2034,7 +2034,7 @@ dependencies = [ [[package]] name = "mithril-client" -version = "0.2.4" +version = "0.2.5" dependencies = [ "async-trait", "clap 4.1.1", @@ -2061,7 +2061,7 @@ dependencies = [ [[package]] name = "mithril-common" -version = "0.2.7" +version = "0.2.8" dependencies = [ "async-trait", "bech32", @@ -2125,7 +2125,7 @@ dependencies = [ [[package]] name = "mithril-signer" -version = "0.2.5" +version = "0.2.6" dependencies = [ "async-trait", "clap 4.1.1", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index c75979288f7..ef853421c2d 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-aggregator" -version = "0.2.7" +version = "0.2.8" description = "A Mithril Aggregator server" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-client/Cargo.toml b/mithril-client/Cargo.toml index 1eed22fa525..ed3d92d731b 100644 --- a/mithril-client/Cargo.toml +++ b/mithril-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-client" -version = "0.2.4" +version = "0.2.5" description = "A Mithril Client" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml index 37814353b66..df636f8bba2 100644 --- a/mithril-common/Cargo.toml +++ b/mithril-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-common" -version = "0.2.7" +version = "0.2.8" authors = { workspace = true } edition = { workspace = true } documentation = { workspace = true } diff --git a/mithril-signer/Cargo.toml b/mithril-signer/Cargo.toml index 19400adfa46..deaa1343840 100644 --- a/mithril-signer/Cargo.toml +++ b/mithril-signer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-signer" -version = "0.2.5" +version = "0.2.6" description = "A Mithril Signer" authors = { workspace = true } edition = { workspace = true }