From b2ac4cef8044b75bffa5ed8b6545aba0e9130e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 20 Jun 2024 10:25:49 -0600 Subject: [PATCH 01/50] feat(wit): Add bindings for hermes-ipfs --- wasm/.cspell.json | 1 + wasm/wasi/wit/deps/hermes-ipfs/api.wit | 39 ++++++++++++++++++++++++ wasm/wasi/wit/deps/hermes-ipfs/event.wit | 19 ++++++++++++ wasm/wasi/wit/deps/hermes-ipfs/world.wit | 6 ++++ wasm/wasi/wit/hermes.wit | 1 + 5 files changed, 66 insertions(+) create mode 100644 wasm/wasi/wit/deps/hermes-ipfs/api.wit create mode 100644 wasm/wasi/wit/deps/hermes-ipfs/event.wit create mode 100644 wasm/wasi/wit/deps/hermes-ipfs/world.wit diff --git a/wasm/.cspell.json b/wasm/.cspell.json index 52560cc36..c2e1db6d0 100644 --- a/wasm/.cspell.json +++ b/wasm/.cspell.json @@ -151,6 +151,7 @@ "inval", "Iovec", "iovs", + "ipfs", "IPPROTO", "ISDIR", "isdir", diff --git a/wasm/wasi/wit/deps/hermes-ipfs/api.wit b/wasm/wasi/wit/deps/hermes-ipfs/api.wit new file mode 100644 index 000000000..d728b3d69 --- /dev/null +++ b/wasm/wasi/wit/deps/hermes-ipfs/api.wit @@ -0,0 +1,39 @@ +/// Interface to local `IPFS` instance. +interface api { + /// The binary contents of an IPFS file. + type ipfs-content = list; + /// A path to an IPFS file. + type ipfs-path = string; + /// A DHT key. + type dht-key = list; + /// A DHT value. + type dht-value = list; + /// A PubSub topic. + type pubsub-topic = string; + /// A PubSub message from a topic subscription. + type pubsub-message = string; + + /// Errors that occur in IPFS networking. + enum errno { + file-add-error, + file-get-error, + dht-get-error, + } + + /// Uploads a file to IPFS. + file-add: func(contents: ipfs-content) -> result; + /// Retrieves a file from IPFS. + file-get: func(path: ipfs-path) -> result; + /// Pins a file by path to IPFS. + file-pin: func(path: ipfs-path) -> bool; + /// Puts a DHT key-value into IPFS. + dht-put: func(key: dht-key, contents: ipfs-content) -> bool; + /// Gets a DHT key-value from IPFS. + dht-get: func(key: dht-key) -> result; + /// Subscribes to a PubSub topic. + pubsub-subscribe: func(topic: pubsub-topic) -> bool; +} + +world ipfs-api { + export api; +} diff --git a/wasm/wasi/wit/deps/hermes-ipfs/event.wit b/wasm/wasi/wit/deps/hermes-ipfs/event.wit new file mode 100644 index 000000000..b6160c48d --- /dev/null +++ b/wasm/wasi/wit/deps/hermes-ipfs/event.wit @@ -0,0 +1,19 @@ +/// # IPFS API +/// +/// Event triggered on receiving a message on a PubSub topic. +/// +/// ## Permissions +/// +/// This API is ALWAYS available. + +/// IPFS API Interface - Export ONLY +interface event { + use api.{pubsub-message}; + + /// Triggers when a message is received on a topic. + on-topic: func(message: pubsub-message) -> bool; +} + +world ipfs-event { + export event; +} diff --git a/wasm/wasi/wit/deps/hermes-ipfs/world.wit b/wasm/wasi/wit/deps/hermes-ipfs/world.wit new file mode 100644 index 000000000..b3be4e3e9 --- /dev/null +++ b/wasm/wasi/wit/deps/hermes-ipfs/world.wit @@ -0,0 +1,6 @@ +package hermes:ipfs; + +world all { + import api; + export event; +} diff --git a/wasm/wasi/wit/hermes.wit b/wasm/wasi/wit/hermes.wit index c7ffc63d3..61afb27c0 100644 --- a/wasm/wasi/wit/hermes.wit +++ b/wasm/wasi/wit/hermes.wit @@ -18,6 +18,7 @@ world hermes { include hermes:crypto/all; include hermes:hash/all; include hermes:init/all; + include hermes:ipfs/all; include hermes:json/all; include hermes:kv-store/all; include hermes:localtime/all; From de585f1c08fd21ab4de5a6eec3d17fe083a887ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 20 Jun 2024 19:37:25 -0600 Subject: [PATCH 02/50] feat(wit): add exports_hermes_ipfs_event_on_topic to stub-module.c --- wasm/stub-module/stub-module.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wasm/stub-module/stub-module.c b/wasm/stub-module/stub-module.c index 89c47c0a5..020318b1f 100644 --- a/wasm/stub-module/stub-module.c +++ b/wasm/stub-module/stub-module.c @@ -30,6 +30,11 @@ bool exports_hermes_init_event_init(void) { return false; } +// Exported Functions from `hermes:cron/event` +bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { + return false; +} + // Exported Functions from `hermes:kv-store/event` void exports_hermes_kv_store_event_kv_update(hermes_string_t *key, exports_hermes_kv_store_event_kv_values_t *value) { From 8f1642954d74d299f5d8c55fe270678632272abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 20 Jun 2024 21:52:29 -0600 Subject: [PATCH 03/50] feat(wip): add stubbed ipfs::api::Host implementation and on-topic event --- .../runtime_extensions/hermes/ipfs/event.rs | 28 +++++++++++++++ .../runtime_extensions/hermes/ipfs/host.rs | 34 +++++++++++++++++++ .../src/runtime_extensions/hermes/ipfs/mod.rs | 6 ++++ .../bin/src/runtime_extensions/hermes/mod.rs | 1 + 4 files changed, 69 insertions(+) create mode 100644 hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs create mode 100644 hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs create mode 100644 hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs new file mode 100644 index 000000000..634f88836 --- /dev/null +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs @@ -0,0 +1,28 @@ +//! Hermes IPFS runtime extension event handler implementation. +use crate::{ + event::HermesEventPayload, + runtime_extensions::bindings::hermes::ipfs::api::{PubsubMessage, PubsubTopic}, +}; + +/// On cron event +#[derive(Clone, Debug)] +pub(crate) struct OnTopicEvent { + /// + pub(crate) topic: PubsubTopic, + pub(crate) message: PubsubMessage, +} + +impl HermesEventPayload for OnTopicEvent { + fn event_name(&self) -> &str { + "on-topic" + } + + fn execute(&self, module: &mut crate::wasm::module::ModuleInstance) -> anyhow::Result<()> { + let _res: bool = module + .instance + .hermes_ipfs_event() + .call_on_topic(&mut module.store, &self.message)?; + // TODO(@saibatizoku): WIP: add message handling + Ok(()) + } +} diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs new file mode 100644 index 000000000..82430012b --- /dev/null +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -0,0 +1,34 @@ +//! IPFS host implementation for WASM runtime. + +use crate::{ + runtime_context::HermesRuntimeContext, + runtime_extensions::bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PubsubTopic, + }, +}; + +impl Host for HermesRuntimeContext { + fn file_add(&mut self, _contents: IpfsContent) -> wasmtime::Result> { + todo!(); + } + + fn file_get(&mut self, _path: IpfsPath) -> wasmtime::Result> { + todo!(); + } + + fn file_pin(&mut self, _path: IpfsPath) -> wasmtime::Result { + todo!(); + } + + fn dht_put(&mut self, _key: DhtKey, _contents: IpfsContent) -> wasmtime::Result { + todo!(); + } + + fn dht_get(&mut self, _key: DhtKey) -> wasmtime::Result> { + todo!(); + } + + fn pubsub_subscribe(&mut self, _topic: PubsubTopic) -> wasmtime::Result { + todo!(); + } +} diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs new file mode 100644 index 000000000..dc4d81750 --- /dev/null +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs @@ -0,0 +1,6 @@ +//! Hermes IPFS runtime extension. +mod event; +mod host; + +/// Advise Runtime Extensions of a new context +pub(crate) fn new_context(_ctx: &crate::runtime_context::HermesRuntimeContext) {} diff --git a/hermes/bin/src/runtime_extensions/hermes/mod.rs b/hermes/bin/src/runtime_extensions/hermes/mod.rs index 3ed26a294..c32436779 100644 --- a/hermes/bin/src/runtime_extensions/hermes/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod crypto; pub(crate) mod hash; pub(crate) mod init; pub mod integration_test; +pub(crate) mod ipfs; pub(crate) mod json; pub(crate) mod kv_store; pub(crate) mod localtime; From 6dced7711e96fdc0304cfc238facff3b7e274199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 20 Jun 2024 22:03:50 -0600 Subject: [PATCH 04/50] fix: update integration test modules --- wasm/integration-test/clocks/clocks.c | 5 +++++ wasm/integration-test/cron/cron.c | 5 +++++ wasm/integration-test/crypto/crypto.c | 5 +++++ wasm/integration-test/hashing/hashing.c | 5 +++++ wasm/integration-test/localtime/localtime.c | 5 +++++ wasm/integration-test/logger/logger.c | 5 +++++ wasm/integration-test/smoke-test/smoke-test.c | 5 +++++ wasm/stub-module/stub-module.c | 2 +- 8 files changed, 36 insertions(+), 1 deletion(-) diff --git a/wasm/integration-test/clocks/clocks.c b/wasm/integration-test/clocks/clocks.c index 0c1d8eda6..d0fdfd1a1 100644 --- a/wasm/integration-test/clocks/clocks.c +++ b/wasm/integration-test/clocks/clocks.c @@ -33,6 +33,11 @@ bool exports_hermes_init_event_init(void) return false; } +// Exported Functions from `hermes:ipfs/event` +bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { + return false; +} + // Exported Functions from `hermes:kv-store/event` void exports_hermes_kv_store_event_kv_update(hermes_string_t *key, exports_hermes_kv_store_event_kv_values_t *value) { diff --git a/wasm/integration-test/cron/cron.c b/wasm/integration-test/cron/cron.c index 42b008021..5e3848573 100644 --- a/wasm/integration-test/cron/cron.c +++ b/wasm/integration-test/cron/cron.c @@ -33,6 +33,11 @@ bool exports_hermes_init_event_init(void) return false; } +// Exported Functions from `hermes:ipfs/event` +bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { + return false; +} + // Exported Functions from `hermes:kv-store/event` void exports_hermes_kv_store_event_kv_update(hermes_string_t *key, exports_hermes_kv_store_event_kv_values_t *value) { diff --git a/wasm/integration-test/crypto/crypto.c b/wasm/integration-test/crypto/crypto.c index 51e19aca4..2fa24852a 100644 --- a/wasm/integration-test/crypto/crypto.c +++ b/wasm/integration-test/crypto/crypto.c @@ -33,6 +33,11 @@ bool exports_hermes_init_event_init(void) return false; } +// Exported Functions from `hermes:ipfs/event` +bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { + return false; +} + // Exported Functions from `hermes:kv-store/event` void exports_hermes_kv_store_event_kv_update(hermes_string_t *key, exports_hermes_kv_store_event_kv_values_t *value) { diff --git a/wasm/integration-test/hashing/hashing.c b/wasm/integration-test/hashing/hashing.c index 6f614d620..c7c8bb968 100644 --- a/wasm/integration-test/hashing/hashing.c +++ b/wasm/integration-test/hashing/hashing.c @@ -33,6 +33,11 @@ bool exports_hermes_init_event_init(void) return false; } +// Exported Functions from `hermes:ipfs/event` +bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { + return false; +} + // Exported Functions from `hermes:kv-store/event` void exports_hermes_kv_store_event_kv_update(hermes_string_t *key, exports_hermes_kv_store_event_kv_values_t *value) { diff --git a/wasm/integration-test/localtime/localtime.c b/wasm/integration-test/localtime/localtime.c index 1355851b6..a31e32a36 100644 --- a/wasm/integration-test/localtime/localtime.c +++ b/wasm/integration-test/localtime/localtime.c @@ -33,6 +33,11 @@ bool exports_hermes_init_event_init(void) return false; } +// Exported Functions from `hermes:ipfs/event` +bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { + return false; +} + // Exported Functions from `hermes:kv-store/event` void exports_hermes_kv_store_event_kv_update(hermes_string_t *key, exports_hermes_kv_store_event_kv_values_t *value) { diff --git a/wasm/integration-test/logger/logger.c b/wasm/integration-test/logger/logger.c index 257c8223b..6184508fc 100644 --- a/wasm/integration-test/logger/logger.c +++ b/wasm/integration-test/logger/logger.c @@ -33,6 +33,11 @@ bool exports_hermes_init_event_init(void) return false; } +// Exported Functions from `hermes:ipfs/event` +bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { + return false; +} + // Exported Functions from `hermes:kv-store/event` void exports_hermes_kv_store_event_kv_update(hermes_string_t *key, exports_hermes_kv_store_event_kv_values_t *value) { diff --git a/wasm/integration-test/smoke-test/smoke-test.c b/wasm/integration-test/smoke-test/smoke-test.c index 2db8326e9..d15be7c71 100644 --- a/wasm/integration-test/smoke-test/smoke-test.c +++ b/wasm/integration-test/smoke-test/smoke-test.c @@ -32,6 +32,11 @@ bool exports_hermes_init_event_init(void) return false; } +// Exported Functions from `hermes:ipfs/event` +bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { + return false; +} + // Exported Functions from `hermes:kv-store/event` void exports_hermes_kv_store_event_kv_update(hermes_string_t *key, exports_hermes_kv_store_event_kv_values_t *value) { diff --git a/wasm/stub-module/stub-module.c b/wasm/stub-module/stub-module.c index 020318b1f..685f0b78a 100644 --- a/wasm/stub-module/stub-module.c +++ b/wasm/stub-module/stub-module.c @@ -30,7 +30,7 @@ bool exports_hermes_init_event_init(void) { return false; } -// Exported Functions from `hermes:cron/event` +// Exported Functions from `hermes:ipfs/event` bool exports_hermes_ipfs_event_on_topic(exports_hermes_ipfs_event_pubsub_message_t *message) { return false; } From f3e813d1badb85c974c2f034266ceee9e50c2c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 20 Jun 2024 22:05:41 -0600 Subject: [PATCH 05/50] feat: add ipfs runtime context --- hermes/bin/src/runtime_extensions/hermes/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/hermes/bin/src/runtime_extensions/hermes/mod.rs b/hermes/bin/src/runtime_extensions/hermes/mod.rs index c32436779..e8c5ae87e 100644 --- a/hermes/bin/src/runtime_extensions/hermes/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/mod.rs @@ -26,6 +26,7 @@ pub(crate) fn new_context(ctx: &HermesRuntimeContext) { crypto::new_context(ctx); hash::new_context(ctx); init::new_context(ctx); + ipfs::new_context(ctx); json::new_context(ctx); kv_store::new_context(ctx); localtime::new_context(ctx); From 75ebb5beabc8848963f4b8303cd7674653b16a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 20 Jun 2024 22:11:52 -0600 Subject: [PATCH 06/50] fix: lints --- hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs index 634f88836..9ef8f3665 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs @@ -4,11 +4,12 @@ use crate::{ runtime_extensions::bindings::hermes::ipfs::api::{PubsubMessage, PubsubTopic}, }; -/// On cron event -#[derive(Clone, Debug)] +/// Event handler for the `on-topic` event. +#[allow(dead_code)] pub(crate) struct OnTopicEvent { - /// + /// Topic pub(crate) topic: PubsubTopic, + /// Message pub(crate) message: PubsubMessage, } From 4cd6153cf616c6f554f14689220faac5d5f970e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 20 Jun 2024 22:31:01 -0600 Subject: [PATCH 07/50] fix: update rust integration test modules --- wasm/integration-test/cardano/src/lib.rs | 7 +++++++ wasm/integration-test/sqlite/src/lib.rs | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/wasm/integration-test/cardano/src/lib.rs b/wasm/integration-test/cardano/src/lib.rs index 2b6abe428..08eb9e694 100644 --- a/wasm/integration-test/cardano/src/lib.rs +++ b/wasm/integration-test/cardano/src/lib.rs @@ -10,6 +10,7 @@ use hermes::{ api::{BlockSrc, CardanoBlock, CardanoBlockchainId, CardanoTxn, Slot}, }, cron::api::CronTagged, + ipfs::api::PubsubMessage, kv_store::api::KvValues, }, wasi::http::types::{IncomingRequest, ResponseOutparam}, @@ -89,6 +90,12 @@ impl hermes::exports::hermes::init::event::Guest for TestComponent { } } +impl hermes::exports::hermes::ipfs::event::Guest for TestComponent { + fn on_topic(message: PubsubMessage) -> bool { + false + } +} + impl hermes::exports::hermes::kv_store::event::Guest for TestComponent { fn kv_update(_key: String, _value: KvValues) {} } diff --git a/wasm/integration-test/sqlite/src/lib.rs b/wasm/integration-test/sqlite/src/lib.rs index f4ffef919..9ad8f4bcb 100644 --- a/wasm/integration-test/sqlite/src/lib.rs +++ b/wasm/integration-test/sqlite/src/lib.rs @@ -11,6 +11,7 @@ use hermes::{ hermes::{ cardano::api::{BlockSrc, CardanoBlock, CardanoBlockchainId, CardanoTxn}, cron::api::CronTagged, + ipfs::api::PubsubMessage, kv_store::api::KvValues, sqlite, }, @@ -79,6 +80,12 @@ impl hermes::exports::hermes::init::event::Guest for TestComponent { } } +impl hermes::exports::hermes::ipfs::event::Guest for TestComponent { + fn on_topic(message: PubsubMessage) -> bool { + false + } +} + impl hermes::exports::hermes::kv_store::event::Guest for TestComponent { fn kv_update(_key: String, _value: KvValues) {} } From 34631bd48ef1fa4a1e46ef60b64766ef591d747a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 20 Jun 2024 23:01:50 -0600 Subject: [PATCH 08/50] chore: update project dictionary --- .config/dictionaries/project.dic | 1 + 1 file changed, 1 insertion(+) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index bffc5ffdd..dfb688b44 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -85,6 +85,7 @@ IFMT Intellij ioerr iohk +ipfs jetbrains jsonschema jorm From 435a96088c7fc9078e0624585999ccb6827aba2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Fri, 21 Jun 2024 07:44:06 -0600 Subject: [PATCH 09/50] fix: pubsub-message includes topic --- wasm/wasi/wit/deps/hermes-ipfs/api.wit | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wasm/wasi/wit/deps/hermes-ipfs/api.wit b/wasm/wasi/wit/deps/hermes-ipfs/api.wit index d728b3d69..15631c5ce 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/api.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/api.wit @@ -11,7 +11,12 @@ interface api { /// A PubSub topic. type pubsub-topic = string; /// A PubSub message from a topic subscription. - type pubsub-message = string; + record pubsub-message { + /// The topic that the message was received on. + topic: pubsub-topic, + /// The contents of the message. + message: string, + } /// Errors that occur in IPFS networking. enum errno { From ef69e0fe02e93203a9d3f682cb69894ddf4741ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Sun, 30 Jun 2024 20:51:03 -0600 Subject: [PATCH 10/50] fix: update IPFS WIT bindings --- .../src/runtime_extensions/hermes/ipfs/host.rs | 6 +++--- wasm/wasi/wit/deps/hermes-ipfs/api.wit | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index 82430012b..b5f4536df 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -16,11 +16,11 @@ impl Host for HermesRuntimeContext { todo!(); } - fn file_pin(&mut self, _path: IpfsPath) -> wasmtime::Result { + fn file_pin(&mut self, ipfs_path: IpfsPath) -> wasmtime::Result> { todo!(); } - fn dht_put(&mut self, _key: DhtKey, _contents: IpfsContent) -> wasmtime::Result { + fn dht_put(&mut self, _key: DhtKey, _contents: IpfsContent) -> wasmtime::Result> { todo!(); } @@ -28,7 +28,7 @@ impl Host for HermesRuntimeContext { todo!(); } - fn pubsub_subscribe(&mut self, _topic: PubsubTopic) -> wasmtime::Result { + fn pubsub_subscribe(&mut self, _topic: PubsubTopic) -> wasmtime::Result> { todo!(); } } diff --git a/wasm/wasi/wit/deps/hermes-ipfs/api.wit b/wasm/wasi/wit/deps/hermes-ipfs/api.wit index 15631c5ce..15fffb1dd 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/api.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/api.wit @@ -20,9 +20,18 @@ interface api { /// Errors that occur in IPFS networking. enum errno { + /// Unable to publish file to IPFS. file-add-error, + /// Unable to get file from IPFS. file-get-error, + /// Unable to pin file. + file-pin-error, + /// Unable to get DHT value, dht-get-error, + /// Unable to parse a valid IPFS path. + invalid-ipfs-path, + /// Invalid CID. + invalid-cid, } /// Uploads a file to IPFS. @@ -30,13 +39,13 @@ interface api { /// Retrieves a file from IPFS. file-get: func(path: ipfs-path) -> result; /// Pins a file by path to IPFS. - file-pin: func(path: ipfs-path) -> bool; + file-pin: func(path: ipfs-path) -> result; /// Puts a DHT key-value into IPFS. - dht-put: func(key: dht-key, contents: ipfs-content) -> bool; + dht-put: func(key: dht-key, contents: ipfs-content) -> result; /// Gets a DHT key-value from IPFS. dht-get: func(key: dht-key) -> result; /// Subscribes to a PubSub topic. - pubsub-subscribe: func(topic: pubsub-topic) -> bool; + pubsub-subscribe: func(topic: pubsub-topic) -> result; } world ipfs-api { From 4f0672270b0ac2baa05824c19b919110fb0f9585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 13:04:35 -0600 Subject: [PATCH 11/50] fix: add errors and types, peer-evict function --- wasm/wasi/wit/deps/hermes-ipfs/api.wit | 28 +++++++++++++++++++----- wasm/wasi/wit/deps/hermes-ipfs/event.wit | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/wasm/wasi/wit/deps/hermes-ipfs/api.wit b/wasm/wasi/wit/deps/hermes-ipfs/api.wit index 15fffb1dd..0f7ad0e3c 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/api.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/api.wit @@ -1,13 +1,15 @@ /// Interface to local `IPFS` instance. interface api { - /// The binary contents of an IPFS file. - type ipfs-content = list; - /// A path to an IPFS file. - type ipfs-path = string; /// A DHT key. type dht-key = list; /// A DHT value. type dht-value = list; + /// The binary contents of an IPFS file. + type ipfs-content = list; + /// A path to an IPFS file. + type ipfs-path = string; + /// The ID of a peer. + type peer-id = string; /// A PubSub topic. type pubsub-topic = string; /// A PubSub message from a topic subscription. @@ -16,22 +18,34 @@ interface api { topic: pubsub-topic, /// The contents of the message. message: string, + /// Peer ID that sent the message. + peer: peer-id, } /// Errors that occur in IPFS networking. enum errno { + /// Unable to get DHT value. + dht-get-error, + /// Unable to put DHT value. + dht-put-error, /// Unable to publish file to IPFS. file-add-error, /// Unable to get file from IPFS. file-get-error, /// Unable to pin file. file-pin-error, - /// Unable to get DHT value, - dht-get-error, /// Unable to parse a valid IPFS path. invalid-ipfs-path, /// Invalid CID. invalid-cid, + /// Invalid Peer ID. + invalid-peer-id, + /// Unable to evict peer. + peer-eviction-error, + /// Unable to publish to IPFS topic. + pubsub-publish-error, + /// Unable to subscripe to IPFS topic. + pubsub-subscribe-error, } /// Uploads a file to IPFS. @@ -46,6 +60,8 @@ interface api { dht-get: func(key: dht-key) -> result; /// Subscribes to a PubSub topic. pubsub-subscribe: func(topic: pubsub-topic) -> result; + /// Evict peer from network. + peer-evict: func(peer: peer-id) -> result; } world ipfs-api { diff --git a/wasm/wasi/wit/deps/hermes-ipfs/event.wit b/wasm/wasi/wit/deps/hermes-ipfs/event.wit index b6160c48d..11d8f77be 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/event.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/event.wit @@ -8,7 +8,7 @@ /// IPFS API Interface - Export ONLY interface event { - use api.{pubsub-message}; + use api.{pubsub-message, peer-id}; /// Triggers when a message is received on a topic. on-topic: func(message: pubsub-message) -> bool; From 0a3935f5e43c6662bf9bc8f069e22877546447c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 13:19:16 -0600 Subject: [PATCH 12/50] fix: typo --- wasm/wasi/wit/deps/hermes-ipfs/api.wit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/wasi/wit/deps/hermes-ipfs/api.wit b/wasm/wasi/wit/deps/hermes-ipfs/api.wit index 0f7ad0e3c..e49c1db79 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/api.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/api.wit @@ -44,7 +44,7 @@ interface api { peer-eviction-error, /// Unable to publish to IPFS topic. pubsub-publish-error, - /// Unable to subscripe to IPFS topic. + /// Unable to subscribe to IPFS topic. pubsub-subscribe-error, } From 5ce22fee3cd22e59c4108b13ddd43109fc24fd8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 13:28:37 -0600 Subject: [PATCH 13/50] chore: format code --- hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index b5f4536df..4771a2782 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -20,7 +20,9 @@ impl Host for HermesRuntimeContext { todo!(); } - fn dht_put(&mut self, _key: DhtKey, _contents: IpfsContent) -> wasmtime::Result> { + fn dht_put( + &mut self, _key: DhtKey, _contents: IpfsContent, + ) -> wasmtime::Result> { todo!(); } From 52b11bc213abd74f96e7b5219c67f473b78fb174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 07:04:16 -0600 Subject: [PATCH 14/50] chore: add hermes-ipfs to deps --- hermes/Cargo.toml | 2 ++ hermes/bin/Cargo.toml | 1 + hermes/crates/hermes-ipfs/Cargo.toml | 1 + 3 files changed, 4 insertions(+) diff --git a/hermes/Cargo.toml b/hermes/Cargo.toml index 6230012b8..464a36ec3 100644 --- a/hermes/Cargo.toml +++ b/hermes/Cargo.toml @@ -58,6 +58,8 @@ pallas-hardano = { git = "https://github.com/input-output-hk/catalyst-pallas.git cardano-chain-follower = { path = "crates/cardano-chain-follower", version = "0.0.1" } +hermes-ipfs = { path = "crates/hermes-ipfs", version = "0.0.1" } + wasmtime = "20.0.2" rusty_ulid = "2.0.0" anyhow = "1.0.71" diff --git a/hermes/bin/Cargo.toml b/hermes/bin/Cargo.toml index 738ad77fe..0a4f3cc69 100644 --- a/hermes/bin/Cargo.toml +++ b/hermes/bin/Cargo.toml @@ -69,6 +69,7 @@ sha2 = { workspace = true } ed25519-dalek = { workspace = true, features = ["pem"] } x509-cert = { workspace = true, features = ["pem"] } coset = { workspace = true } +hermes-ipfs = { workspace = true } [build-dependencies] build-info-build = { workspace = true } diff --git a/hermes/crates/hermes-ipfs/Cargo.toml b/hermes/crates/hermes-ipfs/Cargo.toml index 0595c860d..e5874cf46 100644 --- a/hermes/crates/hermes-ipfs/Cargo.toml +++ b/hermes/crates/hermes-ipfs/Cargo.toml @@ -14,6 +14,7 @@ workspace = true anyhow.workspace = true libipld.workspace = true rust-ipfs.workspace = true +tracing.workspace = true [dev-dependencies] # Dependencies used by examples From 3ae88b19aa6118837bdb0ac043d27bf355accdca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 07:05:55 -0600 Subject: [PATCH 15/50] chore: add docs to show how to start ipfs node --- hermes/crates/hermes-ipfs/src/lib.rs | 35 +++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index eafd8a759..d1054ddd0 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -1,6 +1,37 @@ //! Hermes IPFS //! //! Provides support for storage, and `PubSub` functionality. +//! +//! ```no_run +//! use hermes_ipfs::HermesIpfs; +//! +//! #[tokio::main] +//! #[allow(clippy::println_empty_string)] +//! async fn main() -> anyhow::Result<()> { +//! // Start with default options +//! let node = HermesIpfs::start().await?; +//! node.stop().await; +//! Ok(()) +//! } +//! ``` +//! +//! ```no_run +//! use hermes_ipfs::{HermesIpfs, IpfsBuilder}; +//! +//! #[tokio::main] +//! #[allow(clippy::println_empty_string)] +//! async fn main() -> anyhow::Result<()> { +//! // Start with default options +//! let node: HermesIpfs = IpfsBuilder::new() +//! .with_default() +//! .set_default_listener() +//! .start() +//! .await? +//! .into(); +//! node.stop().await; +//! Ok(()) +//! } +//! ``` use std::str::FromStr; @@ -20,13 +51,15 @@ pub use rust_ipfs::Ipfs; pub use rust_ipfs::Multiaddr; /// Peer ID type. pub use rust_ipfs::PeerId; +/// Stream for `PubSub` Topic Subscriptions. +pub use rust_ipfs::SubscriptionStream; /// Builder type for IPFS Node configuration. pub use rust_ipfs::UninitializedIpfsNoop as IpfsBuilder; use rust_ipfs::{ dag::ResolveError, libp2p::{futures::stream::BoxStream, kad::Record}, unixfs::AddOpt, - MessageId, PubsubEvent, Quorum, SubscriptionStream, + MessageId, PubsubEvent, Quorum, }; /// Hermes IPFS From a7ae63cc14c1d98b5340e6b103c0ace7989f061b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 07:11:08 -0600 Subject: [PATCH 16/50] feat: add methods to publish/get/pin files --- .../runtime_extensions/hermes/ipfs/host.rs | 19 +- .../src/runtime_extensions/hermes/ipfs/mod.rs | 1 + .../runtime_extensions/hermes/ipfs/state.rs | 174 ++++++++++++++++++ 3 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index 4771a2782..d98d4d150 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -2,22 +2,27 @@ use crate::{ runtime_context::HermesRuntimeContext, - runtime_extensions::bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PubsubTopic, + runtime_extensions::{ + bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PubsubTopic, + }, + hermes::ipfs::state::{hermes_ipfs_add_file, hermes_ipfs_get_file, hermes_ipfs_pin_file}, }, }; impl Host for HermesRuntimeContext { - fn file_add(&mut self, _contents: IpfsContent) -> wasmtime::Result> { - todo!(); + fn file_add(&mut self, contents: IpfsContent) -> wasmtime::Result> { + let path: IpfsPath = hermes_ipfs_add_file(contents)?.to_string(); + Ok(Ok(path)) } - fn file_get(&mut self, _path: IpfsPath) -> wasmtime::Result> { - todo!(); + fn file_get(&mut self, path: IpfsPath) -> wasmtime::Result> { + let contents = hermes_ipfs_get_file(path)?; + Ok(Ok(contents)) } fn file_pin(&mut self, ipfs_path: IpfsPath) -> wasmtime::Result> { - todo!(); + Ok(hermes_ipfs_pin_file(ipfs_path)?) } fn dht_put( diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs index dc4d81750..983bdc450 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs @@ -1,6 +1,7 @@ //! Hermes IPFS runtime extension. mod event; mod host; +mod state; /// Advise Runtime Extensions of a new context pub(crate) fn new_context(_ctx: &crate::runtime_context::HermesRuntimeContext) {} diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs new file mode 100644 index 000000000..a3ad3ff67 --- /dev/null +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs @@ -0,0 +1,174 @@ +//! Hermes IPFS Internal State + +use std::str::FromStr; +use dashmap::DashMap; +use hermes_ipfs::{AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, SubscriptionStream}; +use once_cell::sync::Lazy; +use tokio::{ + runtime::Builder, + sync::{mpsc, oneshot}, +}; + +use crate::{ + app::HermesAppName, + runtime_extensions::bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, IpfsContent, IpfsPath, PubsubTopic, + }, +}; + +/// Hermes IPFS Internal State +static HERMES_IPFS_STATE: Lazy = Lazy::new(|| { + let sender = if let Ok(runtime) = Builder::new_current_thread().enable_all().build() { + let (sender, receiver) = mpsc::channel(1); + let _handle = std::thread::spawn(move || { + runtime.block_on(async move { + let h = tokio::spawn(ipfs_task(receiver)); + drop(tokio::join!(h)); + }); + std::process::exit(0); + }); + Some(sender) + } else { + // Failed to start the IPFS task + tracing::error!("Failed to start the IPFS task"); + None + }; + HermesIpfsState { + sender, + } +}); + +/// Add File to IPFS +pub(crate) fn hermes_ipfs_add_file(contents: IpfsContent) -> Result { + HERMES_IPFS_STATE.file_add(contents) +} + +/// Get File from Ipfs +pub(crate) fn hermes_ipfs_get_file(path: IpfsPath) -> Result { + HERMES_IPFS_STATE.file_get(path) +} + +/// Pin IPFS File +pub(crate) fn hermes_ipfs_pin_file(cid: Cid) -> Result { + HERMES_IPFS_STATE.file_pin(cid.to_string()) +} + +/// Get DHT Value +pub(crate) fn hermes_ipfs_get_dht_value(key: DhtKey) -> Result { + todo!(); +} + +/// Put DHT Value +pub(crate) fn hermes_ipfs_put_dht_value(key: DhtKey, value: DhtValue) -> bool { + todo!(); +} + +/// Subscribe to a topic +pub(crate) fn hermes_ipfs_subscribe(topic: PubsubTopic) -> bool { + todo!(); +} + +/// Hermes IPFS Internal State +struct HermesIpfsState { + /// Send events to the IPFS node. + sender: Option>, +} + +impl HermesIpfsState { + /// Add file + fn file_add(&self, contents: IpfsContent) -> Result { + let (cmd_tx, cmd_rx) = oneshot::channel(); + self + .sender + .as_ref() + .ok_or(Errno::FileAddError)? + .blocking_send(IpfsCommand::AddFile(AddIpfsFile::Stream((None, contents)), cmd_tx)) + .map_err(|_| Errno::FileAddError)?; + cmd_rx.blocking_recv().map_err(|_| Errno::FileAddError)? + } + + /// Get file + #[allow(clippy::needless_pass_by_value)] + fn file_get(&self, ipfs_path: IpfsPath) -> Result { + let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; + let (cmd_tx, cmd_rx) = oneshot::channel(); + self + .sender + .as_ref() + .ok_or(Errno::FileGetError)? + .blocking_send(IpfsCommand::GetFile(ipfs_path, cmd_tx)) + .map_err(|_| Errno::FileGetError)?; + cmd_rx.blocking_recv().map_err(|_| Errno::FileGetError)? + } + + fn file_pin(&self, ipfs_path: IpfsPath) -> Result { + let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; + let cid = ipfs_path.root().cid().ok_or(Errno::InvalidCid)?; + let (cmd_tx, cmd_rx) = oneshot::channel(); + self + .sender + .as_ref() + .ok_or(Errno::FilePinError)? + .blocking_send(IpfsCommand::PinFile(*cid, cmd_tx)) + .map_err(|_| Errno::FilePinError)?; + cmd_rx.blocking_recv().map_err(|_| Errno::FilePinError) + } + + fn dht_put(&self, _key: DhtKey, _contents: IpfsContent) -> anyhow::Result { + todo!(); + } + + fn dht_get(&self, _key: DhtKey) -> Result { + todo!(); + } + + fn pubsub_subscribe(&self, _topic: PubsubTopic) -> anyhow::Result { + todo!(); + } +} + +/// IPFS Command +enum IpfsCommand { + /// Add a new IPFS file + AddFile(AddIpfsFile, oneshot::Sender>), + /// Get a file from IPFS + GetFile(PathIpfsFile, oneshot::Sender, Errno>>), + /// Pin a file + PinFile(Cid, oneshot::Sender), +} + +#[allow(dead_code)] +/// IPFS +async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result<()> { + let hermes_node = HermesIpfs::start().await?; + if let Some(ipfs_command) = queue_rx.recv().await { + match ipfs_command { + IpfsCommand::AddFile(ipfs_file, tx) => { + let ipfs_path = hermes_node.add_ipfs_file(ipfs_file).await?; + if let Err(_err) = tx.send(Ok(ipfs_path.to_string())) { + tracing::error!("Failed to send IPFS path"); + } + } + IpfsCommand::GetFile(ipfs_path, tx) => { + let contents = hermes_node.get_ipfs_file(ipfs_path.into()).await?; + if let Err(_err) = tx.send(Ok(contents)) { + tracing::error!("Failed to get IPFS contents"); + } + } + IpfsCommand::PinFile(cid, tx) => { + let status = match hermes_node.insert_pin(&cid).await { + Ok(_) => true, + Err(err) => { + tracing::error!("Failed to pin block {}: {}", cid, err); + false + } + }; + if let Err(_err) = tx.send(status) { + tracing::error!("Failed to pin IPFS file"); + } + } + } + } + hermes_node.stop().await; + Ok(()) +} From 2afb295d1b92031ff868411880885e9e29550f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 12:31:13 -0600 Subject: [PATCH 17/50] feat: add unsubscribe method, return friendlier types --- .../examples/provide-content-dht.rs | 8 ++-- hermes/crates/hermes-ipfs/examples/pubsub.rs | 6 +-- .../hermes-ipfs/examples/put-get-dht.rs | 16 ++------ hermes/crates/hermes-ipfs/src/lib.rs | 39 +++++++++++++++---- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/hermes/crates/hermes-ipfs/examples/provide-content-dht.rs b/hermes/crates/hermes-ipfs/examples/provide-content-dht.rs index c90d5f645..63cf97475 100644 --- a/hermes/crates/hermes-ipfs/examples/provide-content-dht.rs +++ b/hermes/crates/hermes-ipfs/examples/provide-content-dht.rs @@ -11,8 +11,7 @@ async fn connect_node_a_upload_and_provide( println!("***************************************"); println!("* Hermes IPFS node A has started."); println!(""); - let peer_info = hermes_ipfs.identity(None).await?; - let peer_id_a = peer_info.peer_id; + let peer_id_a = hermes_ipfs.identity(None).await?; let addresses = hermes_ipfs.listening_addresses().await?; println!("* Peer ID: {peer_id_a}"); for addr in addresses { @@ -47,8 +46,7 @@ async fn connect_node_b_to_node_a(node_a: &HermesIpfs) -> anyhow::Result anyhow::Result anyhow::Result<(HermesIpfs, HermesIpfs)> let hermes_a = HermesIpfs::start().await?; println!("***************************************"); println!("* Hermes IPFS node A has started."); - let peer_info = hermes_a.identity(None).await?; - let peer_id_a = peer_info.peer_id; + let peer_id_a = hermes_a.identity(None).await?; println!(" Peer ID: {peer_id_a}"); let addresses = hermes_a.listening_addresses().await?; let a_address = addresses[0].clone(); @@ -37,8 +36,7 @@ async fn start_bootstrapped_nodes() -> anyhow::Result<(HermesIpfs, HermesIpfs)> println!("***************************************"); println!("* Hermes IPFS node B has started."); let hermes_b = HermesIpfs::start().await?; - let peer_info = hermes_b.identity(None).await?; - let peer_id_b = peer_info.peer_id; + let peer_id_b = hermes_b.identity(None).await?; println!(" Peer ID: {peer_id_b}"); let addresses = hermes_b.listening_addresses().await?; let b_address = addresses[0].clone(); diff --git a/hermes/crates/hermes-ipfs/examples/put-get-dht.rs b/hermes/crates/hermes-ipfs/examples/put-get-dht.rs index 0366b26bf..3cfacc9f5 100644 --- a/hermes/crates/hermes-ipfs/examples/put-get-dht.rs +++ b/hermes/crates/hermes-ipfs/examples/put-get-dht.rs @@ -2,7 +2,6 @@ #![allow(clippy::println_empty_string)] use hermes_ipfs::HermesIpfs; -use rust_ipfs::libp2p::futures::{pin_mut, StreamExt}; #[allow(clippy::indexing_slicing)] /// Connect Node A, upload file and provide CID by adding to DHT @@ -10,8 +9,7 @@ async fn start_bootstrapped_nodes() -> anyhow::Result<(HermesIpfs, HermesIpfs)> let hermes_a = HermesIpfs::start().await?; println!("***************************************"); println!("* Hermes IPFS node A has started."); - let peer_info = hermes_a.identity(None).await?; - let peer_id_a = peer_info.peer_id; + let peer_id_a = hermes_a.identity(None).await?; println!(" Peer ID: {peer_id_a}"); let addresses = hermes_a.listening_addresses().await?; let a_address = addresses[0].clone(); @@ -20,8 +18,7 @@ async fn start_bootstrapped_nodes() -> anyhow::Result<(HermesIpfs, HermesIpfs)> println!("***************************************"); println!("* Hermes IPFS node B has started."); let hermes_b = HermesIpfs::start().await?; - let peer_info = hermes_b.identity(None).await?; - let peer_id_b = peer_info.peer_id; + let peer_id_b = hermes_b.identity(None).await?; println!(" Peer ID: {peer_id_b}"); let addresses = hermes_b.listening_addresses().await?; let b_address = addresses[0].clone(); @@ -51,13 +48,8 @@ async fn main() -> anyhow::Result<()> { println!("* Hermes IPFS node A is publishing 'my_key' to DHT."); hermes_ipfs_a.dht_put(b"my_key", ipfs_file).await?; println!("* Hermes IPFS node B is getting 'my_key' from DHT."); - let records = hermes_ipfs_b.dht_get(b"my_key").await?; - pin_mut!(records); - let data_retrieved = records - .next() - .await - .ok_or_else(|| anyhow::anyhow!("Unable to fetch content from DHT"))?; - let data = String::from_utf8(data_retrieved.value)?; + let data_retrieved = hermes_ipfs_b.dht_get(b"my_key").await?; + let data = String::from_utf8(data_retrieved)?; println!(" Got data: {data:?}"); // Stop the nodes and exit. hermes_ipfs_a.stop().await; diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index d1054ddd0..c92018764 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -39,6 +39,8 @@ use std::str::FromStr; pub use libipld::Ipld; /// IPFS Content Identifier. pub use libipld::{self as ipld, Cid}; +/// libp2p re-export. +pub use rust_ipfs::libp2p; /// Peer Info type. pub use rust_ipfs::p2p::PeerInfo; /// Enum for specifying paths in IPFS. @@ -57,7 +59,7 @@ pub use rust_ipfs::SubscriptionStream; pub use rust_ipfs::UninitializedIpfsNoop as IpfsBuilder; use rust_ipfs::{ dag::ResolveError, - libp2p::{futures::stream::BoxStream, kad::Record}, + libp2p::futures::{pin_mut, stream::BoxStream, StreamExt}, unixfs::AddOpt, MessageId, PubsubEvent, Quorum, }; @@ -193,8 +195,8 @@ impl HermesIpfs { /// ## Errors /// /// Returns error if peer info cannot be retrieved. - pub async fn identity(&self, peer_id: Option) -> anyhow::Result { - self.node.identity(peer_id).await + pub async fn identity(&self, peer_id: Option) -> anyhow::Result { + self.node.identity(peer_id).await.map(|p| p.peer_id) } /// Add peer to address book. @@ -303,15 +305,19 @@ impl HermesIpfs { /// /// ## Returns /// - /// * `Result>` + /// * `Result>` /// /// ## Errors /// /// Returns error if unable to get content from DHT - pub async fn dht_get( - &self, key: impl AsRef<[u8]>, - ) -> anyhow::Result> { - self.node.dht_get(key).await + pub async fn dht_get(&self, key: impl AsRef<[u8]>) -> anyhow::Result> { + let record_stream = self.node.dht_get(key).await?; + pin_mut!(record_stream); + let record = record_stream + .next() + .await + .ok_or(anyhow::anyhow!("No record found"))?; + Ok(record.value) } /// Add address to bootstrap nodes. @@ -382,6 +388,23 @@ impl HermesIpfs { self.node.pubsub_subscribe(topic).await } + /// Unsubscribes from a pubsub topic. + /// + /// ## Parameters + /// + /// * `topic` - `impl Into` + /// + /// ## Returns + /// + /// * `bool` + /// + /// ## Errors + /// + /// Returns error if unable to unsubscribe from pubsub topic. + pub async fn pubsub_unsubscribe(&self, topic: impl Into) -> anyhow::Result { + self.node.pubsub_unsubscribe(topic).await + } + /// Publishes a message to a pubsub topic. /// /// ## Parameters From 78bc21b49d2a0b42ffbef3d77d53ee4b71601c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 12:36:14 -0600 Subject: [PATCH 18/50] feat(wip): implement ipfs methods --- .../runtime_extensions/hermes/ipfs/event.rs | 5 +- .../runtime_extensions/hermes/ipfs/host.rs | 26 ++- .../runtime_extensions/hermes/ipfs/state.rs | 180 ++++++++++++++---- hermes/crates/hermes-ipfs/src/lib.rs | 17 ++ 4 files changed, 178 insertions(+), 50 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs index 9ef8f3665..3eac4d7ca 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs @@ -1,14 +1,11 @@ //! Hermes IPFS runtime extension event handler implementation. use crate::{ - event::HermesEventPayload, - runtime_extensions::bindings::hermes::ipfs::api::{PubsubMessage, PubsubTopic}, + event::HermesEventPayload, runtime_extensions::bindings::hermes::ipfs::api::PubsubMessage, }; /// Event handler for the `on-topic` event. #[allow(dead_code)] pub(crate) struct OnTopicEvent { - /// Topic - pub(crate) topic: PubsubTopic, /// Message pub(crate) message: PubsubMessage, } diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index d98d4d150..9c892554e 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -4,9 +4,13 @@ use crate::{ runtime_context::HermesRuntimeContext, runtime_extensions::{ bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PubsubTopic, + DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PeerId, PubsubTopic, + }, + hermes::ipfs::state::{ + hermes_ipfs_add_file, hermes_ipfs_evict_peer, hermes_ipfs_get_dht_value, + hermes_ipfs_get_file, hermes_ipfs_pin_file, hermes_ipfs_put_dht_value, + hermes_ipfs_subscribe, }, - hermes::ipfs::state::{hermes_ipfs_add_file, hermes_ipfs_get_file, hermes_ipfs_pin_file}, }, }; @@ -22,20 +26,24 @@ impl Host for HermesRuntimeContext { } fn file_pin(&mut self, ipfs_path: IpfsPath) -> wasmtime::Result> { - Ok(hermes_ipfs_pin_file(ipfs_path)?) + Ok(hermes_ipfs_pin_file(ipfs_path)) } fn dht_put( - &mut self, _key: DhtKey, _contents: IpfsContent, + &mut self, key: DhtKey, contents: IpfsContent, ) -> wasmtime::Result> { - todo!(); + Ok(hermes_ipfs_put_dht_value(key, contents)) + } + + fn dht_get(&mut self, key: DhtKey) -> wasmtime::Result> { + Ok(hermes_ipfs_get_dht_value(key)) } - fn dht_get(&mut self, _key: DhtKey) -> wasmtime::Result> { - todo!(); + fn pubsub_subscribe(&mut self, topic: PubsubTopic) -> wasmtime::Result> { + Ok(hermes_ipfs_subscribe(topic)) } - fn pubsub_subscribe(&mut self, _topic: PubsubTopic) -> wasmtime::Result> { - todo!(); + fn peer_evict(&mut self, peer: PeerId) -> wasmtime::Result> { + Ok(hermes_ipfs_evict_peer(peer)) } } diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs index a3ad3ff67..bba477ed6 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs @@ -1,19 +1,19 @@ //! Hermes IPFS Internal State use std::str::FromStr; -use dashmap::DashMap; -use hermes_ipfs::{AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, SubscriptionStream}; + +use hermes_ipfs::{ + AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, + SubscriptionStream, +}; use once_cell::sync::Lazy; use tokio::{ runtime::Builder, sync::{mpsc, oneshot}, }; -use crate::{ - app::HermesAppName, - runtime_extensions::bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, IpfsContent, IpfsPath, PubsubTopic, - }, +use crate::runtime_extensions::bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, IpfsContent, IpfsPath, PeerId, PubsubTopic, }; /// Hermes IPFS Internal State @@ -33,9 +33,7 @@ static HERMES_IPFS_STATE: Lazy = Lazy::new(|| { tracing::error!("Failed to start the IPFS task"); None }; - HermesIpfsState { - sender, - } + HermesIpfsState::new(sender) }); /// Add File to IPFS @@ -49,40 +47,56 @@ pub(crate) fn hermes_ipfs_get_file(path: IpfsPath) -> Result } /// Pin IPFS File -pub(crate) fn hermes_ipfs_pin_file(cid: Cid) -> Result { - HERMES_IPFS_STATE.file_pin(cid.to_string()) +pub(crate) fn hermes_ipfs_pin_file(path: IpfsPath) -> Result { + HERMES_IPFS_STATE.file_pin(path) } /// Get DHT Value pub(crate) fn hermes_ipfs_get_dht_value(key: DhtKey) -> Result { - todo!(); + HERMES_IPFS_STATE.dht_get(key) } /// Put DHT Value -pub(crate) fn hermes_ipfs_put_dht_value(key: DhtKey, value: DhtValue) -> bool { - todo!(); +pub(crate) fn hermes_ipfs_put_dht_value(key: DhtKey, value: DhtValue) -> Result { + HERMES_IPFS_STATE.dht_put(key, value) } /// Subscribe to a topic -pub(crate) fn hermes_ipfs_subscribe(topic: PubsubTopic) -> bool { - todo!(); +pub(crate) fn hermes_ipfs_subscribe(topic: PubsubTopic) -> Result { + let _stream = HERMES_IPFS_STATE.pubsub_subscribe(topic)?; + Ok(true) +} + +/// Evict Peer from node +pub(crate) fn hermes_ipfs_evict_peer(peer: PeerId) -> Result { + HERMES_IPFS_STATE.peer_evict(peer) } /// Hermes IPFS Internal State struct HermesIpfsState { - /// Send events to the IPFS node. - sender: Option>, + /// State related to `HermesAppName` + apps: AppIpfsState, } impl HermesIpfsState { + /// Create a new `HermesIpfsState` + fn new(sender: Option>) -> Self { + Self { + apps: AppIpfsState { sender }, + } + } + /// Add file fn file_add(&self, contents: IpfsContent) -> Result { let (cmd_tx, cmd_rx) = oneshot::channel(); - self + self.apps .sender .as_ref() .ok_or(Errno::FileAddError)? - .blocking_send(IpfsCommand::AddFile(AddIpfsFile::Stream((None, contents)), cmd_tx)) + .blocking_send(IpfsCommand::AddFile( + AddIpfsFile::Stream((None, contents)), + cmd_tx, + )) .map_err(|_| Errno::FileAddError)?; cmd_rx.blocking_recv().map_err(|_| Errno::FileAddError)? } @@ -92,7 +106,7 @@ impl HermesIpfsState { fn file_get(&self, ipfs_path: IpfsPath) -> Result { let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let (cmd_tx, cmd_rx) = oneshot::channel(); - self + self.apps .sender .as_ref() .ok_or(Errno::FileGetError)? @@ -101,32 +115,82 @@ impl HermesIpfsState { cmd_rx.blocking_recv().map_err(|_| Errno::FileGetError)? } + /// Pin file + #[allow(clippy::needless_pass_by_value)] fn file_pin(&self, ipfs_path: IpfsPath) -> Result { let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let cid = ipfs_path.root().cid().ok_or(Errno::InvalidCid)?; let (cmd_tx, cmd_rx) = oneshot::channel(); - self + self.apps .sender .as_ref() .ok_or(Errno::FilePinError)? .blocking_send(IpfsCommand::PinFile(*cid, cmd_tx)) .map_err(|_| Errno::FilePinError)?; - cmd_rx.blocking_recv().map_err(|_| Errno::FilePinError) + cmd_rx.blocking_recv().map_err(|_| Errno::FilePinError)? } - fn dht_put(&self, _key: DhtKey, _contents: IpfsContent) -> anyhow::Result { - todo!(); + /// Put DHT Key-Value + fn dht_put(&self, key: DhtKey, contents: IpfsContent) -> Result { + let (cmd_tx, cmd_rx) = oneshot::channel(); + self.apps + .sender + .as_ref() + .ok_or(Errno::DhtPutError)? + .blocking_send(IpfsCommand::PutDhtValue(key, contents, cmd_tx)) + .map_err(|_| Errno::DhtPutError)?; + cmd_rx.blocking_recv().map_err(|_| Errno::DhtPutError)? } - fn dht_get(&self, _key: DhtKey) -> Result { - todo!(); + /// Get DHT Value by Key + fn dht_get(&self, key: DhtKey) -> Result { + let (cmd_tx, cmd_rx) = oneshot::channel(); + self.apps + .sender + .as_ref() + .ok_or(Errno::DhtGetError)? + .blocking_send(IpfsCommand::GetDhtValue(key, cmd_tx)) + .map_err(|_| Errno::DhtGetError)?; + cmd_rx.blocking_recv().map_err(|_| Errno::DhtGetError)? } - fn pubsub_subscribe(&self, _topic: PubsubTopic) -> anyhow::Result { - todo!(); + #[allow(clippy::needless_pass_by_value)] + /// Subscribe to a `PubSub` topic + fn pubsub_subscribe(&self, topic: PubsubTopic) -> Result { + let (cmd_tx, cmd_rx) = oneshot::channel(); + self.apps + .sender + .as_ref() + .ok_or(Errno::PubsubSubscribeError)? + .blocking_send(IpfsCommand::Subscribe(topic, cmd_tx)) + .map_err(|_| Errno::PubsubSubscribeError)?; + cmd_rx + .blocking_recv() + .map_err(|_| Errno::PubsubSubscribeError)? + } + + #[allow(clippy::needless_pass_by_value)] + /// Evict peer + fn peer_evict(&self, peer: PeerId) -> Result { + let (cmd_tx, cmd_rx) = oneshot::channel(); + self.apps + .sender + .as_ref() + .ok_or(Errno::PeerEvictionError)? + .blocking_send(IpfsCommand::EvictPeer(peer, cmd_tx)) + .map_err(|_| Errno::PeerEvictionError)?; + cmd_rx + .blocking_recv() + .map_err(|_| Errno::PeerEvictionError)? } } +/// IPFS app state +struct AppIpfsState { + /// Send events to the IPFS node. + sender: Option>, +} + /// IPFS Command enum IpfsCommand { /// Add a new IPFS file @@ -134,7 +198,18 @@ enum IpfsCommand { /// Get a file from IPFS GetFile(PathIpfsFile, oneshot::Sender, Errno>>), /// Pin a file - PinFile(Cid, oneshot::Sender), + PinFile(Cid, oneshot::Sender>), + /// Get DHT value + GetDhtValue(DhtKey, oneshot::Sender>), + /// Put DHT value + PutDhtValue(DhtKey, DhtValue, oneshot::Sender>), + /// Subscribe to a topic + Subscribe( + PubsubTopic, + oneshot::Sender>, + ), + /// Evict Peer from node + EvictPeer(PeerId, oneshot::Sender>), } #[allow(dead_code)] @@ -148,25 +223,56 @@ async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result< if let Err(_err) = tx.send(Ok(ipfs_path.to_string())) { tracing::error!("Failed to send IPFS path"); } - } + }, IpfsCommand::GetFile(ipfs_path, tx) => { let contents = hermes_node.get_ipfs_file(ipfs_path.into()).await?; if let Err(_err) = tx.send(Ok(contents)) { tracing::error!("Failed to get IPFS contents"); } - } + }, IpfsCommand::PinFile(cid, tx) => { let status = match hermes_node.insert_pin(&cid).await { - Ok(_) => true, + Ok(()) => true, Err(err) => { tracing::error!("Failed to pin block {}: {}", cid, err); false - } + }, }; + if let Err(err) = tx.send(Ok(status)) { + tracing::error!("sending response of pin IPFS file should not fail: {err:?}"); + } + }, + IpfsCommand::GetDhtValue(key, tx) => { + let response = hermes_node + .dht_get(key) + .await + .map_err(|_| Errno::DhtGetError); + if let Err(err) = tx.send(response) { + tracing::error!("sending DHT value should not fail: {err:?}"); + } + }, + IpfsCommand::PutDhtValue(key, value, tx) => { + let status = hermes_node.dht_put(key, value).await.is_ok(); + if let Err(err) = tx.send(Ok(status)) { + tracing::error!("sending status of DHT put should not fail: {err:?}"); + } + }, + IpfsCommand::Subscribe(topic, tx) => { + let status = hermes_node + .pubsub_subscribe(topic) + .await + .map_err(|_| Errno::PubsubSubscribeError); if let Err(_err) = tx.send(status) { - tracing::error!("Failed to pin IPFS file"); + tracing::error!("Failed to subscribe to topic"); + } + }, + IpfsCommand::EvictPeer(peer, tx) => { + let peer_id = TargetPeerId::from_str(&peer).map_err(|_| Errno::InvalidPeerId)?; + let status = hermes_node.ban_peer(peer_id).await.is_ok(); + if let Err(err) = tx.send(Ok(status)) { + tracing::error!("sending status of peer eviction should not fail: {err:?}"); } - } + }, } } hermes_node.stop().await; diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index c92018764..b06053691 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -424,6 +424,23 @@ impl HermesIpfs { ) -> anyhow::Result { self.node.pubsub_publish(topic, message).await } + + /// Ban peer from node. + /// + /// ## Parameters + /// + /// * `peer` - `PeerId` + /// + /// ## Returns + /// + /// * `Result<()>` + /// + /// ## Errors + /// + /// Returns error if unable to ban peer. + pub async fn ban_peer(&self, peer: PeerId) -> anyhow::Result<()> { + self.node.ban_peer(peer).await + } } impl From for HermesIpfs { From 8a9f8314282a46853db41c0568a85a8c57078adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 13:27:31 -0600 Subject: [PATCH 19/50] chore: format code and remove unused dep --- hermes/crates/hermes-ipfs/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/hermes/crates/hermes-ipfs/Cargo.toml b/hermes/crates/hermes-ipfs/Cargo.toml index e5874cf46..0595c860d 100644 --- a/hermes/crates/hermes-ipfs/Cargo.toml +++ b/hermes/crates/hermes-ipfs/Cargo.toml @@ -14,7 +14,6 @@ workspace = true anyhow.workspace = true libipld.workspace = true rust-ipfs.workspace = true -tracing.workspace = true [dev-dependencies] # Dependencies used by examples From 9815a59b76b16b8faa2adb2dca95971eb70ddddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 14:38:04 -0600 Subject: [PATCH 20/50] fix: add peer_evict method to Host implementation --- hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index 4771a2782..d43fbec45 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -3,7 +3,7 @@ use crate::{ runtime_context::HermesRuntimeContext, runtime_extensions::bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PubsubTopic, + DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PeerId, PubsubTopic, }, }; @@ -16,7 +16,7 @@ impl Host for HermesRuntimeContext { todo!(); } - fn file_pin(&mut self, ipfs_path: IpfsPath) -> wasmtime::Result> { + fn file_pin(&mut self, _ipfs_path: IpfsPath) -> wasmtime::Result> { todo!(); } @@ -33,4 +33,8 @@ impl Host for HermesRuntimeContext { fn pubsub_subscribe(&mut self, _topic: PubsubTopic) -> wasmtime::Result> { todo!(); } + + fn peer_evict(&mut self, _peer: PeerId) -> wasmtime::Result> { + todo!(); + } } From 0502d011f6224e114d52212196bd82de254ae8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 15:17:22 -0600 Subject: [PATCH 21/50] chore: format code --- .../runtime_extensions/hermes/ipfs/state.rs | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs index bba477ed6..ff1c69ad3 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs @@ -36,42 +36,6 @@ static HERMES_IPFS_STATE: Lazy = Lazy::new(|| { HermesIpfsState::new(sender) }); -/// Add File to IPFS -pub(crate) fn hermes_ipfs_add_file(contents: IpfsContent) -> Result { - HERMES_IPFS_STATE.file_add(contents) -} - -/// Get File from Ipfs -pub(crate) fn hermes_ipfs_get_file(path: IpfsPath) -> Result { - HERMES_IPFS_STATE.file_get(path) -} - -/// Pin IPFS File -pub(crate) fn hermes_ipfs_pin_file(path: IpfsPath) -> Result { - HERMES_IPFS_STATE.file_pin(path) -} - -/// Get DHT Value -pub(crate) fn hermes_ipfs_get_dht_value(key: DhtKey) -> Result { - HERMES_IPFS_STATE.dht_get(key) -} - -/// Put DHT Value -pub(crate) fn hermes_ipfs_put_dht_value(key: DhtKey, value: DhtValue) -> Result { - HERMES_IPFS_STATE.dht_put(key, value) -} - -/// Subscribe to a topic -pub(crate) fn hermes_ipfs_subscribe(topic: PubsubTopic) -> Result { - let _stream = HERMES_IPFS_STATE.pubsub_subscribe(topic)?; - Ok(true) -} - -/// Evict Peer from node -pub(crate) fn hermes_ipfs_evict_peer(peer: PeerId) -> Result { - HERMES_IPFS_STATE.peer_evict(peer) -} - /// Hermes IPFS Internal State struct HermesIpfsState { /// State related to `HermesAppName` @@ -278,3 +242,39 @@ async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result< hermes_node.stop().await; Ok(()) } + +/// Add File to IPFS +pub(crate) fn hermes_ipfs_add_file(contents: IpfsContent) -> Result { + HERMES_IPFS_STATE.file_add(contents) +} + +/// Get File from Ipfs +pub(crate) fn hermes_ipfs_get_file(path: IpfsPath) -> Result { + HERMES_IPFS_STATE.file_get(path) +} + +/// Pin IPFS File +pub(crate) fn hermes_ipfs_pin_file(path: IpfsPath) -> Result { + HERMES_IPFS_STATE.file_pin(path) +} + +/// Get DHT Value +pub(crate) fn hermes_ipfs_get_dht_value(key: DhtKey) -> Result { + HERMES_IPFS_STATE.dht_get(key) +} + +/// Put DHT Value +pub(crate) fn hermes_ipfs_put_dht_value(key: DhtKey, value: DhtValue) -> Result { + HERMES_IPFS_STATE.dht_put(key, value) +} + +/// Subscribe to a topic +pub(crate) fn hermes_ipfs_subscribe(topic: PubsubTopic) -> Result { + let _stream = HERMES_IPFS_STATE.pubsub_subscribe(topic)?; + Ok(true) +} + +/// Evict Peer from node +pub(crate) fn hermes_ipfs_evict_peer(peer: PeerId) -> Result { + HERMES_IPFS_STATE.peer_evict(peer) +} From ed82e8109f8bde793d4a4ff10f58c3d8f7e42a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 15:26:25 -0600 Subject: [PATCH 22/50] fix: remove dup function --- hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index 2068d7f83..9c892554e 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -46,8 +46,4 @@ impl Host for HermesRuntimeContext { fn peer_evict(&mut self, peer: PeerId) -> wasmtime::Result> { Ok(hermes_ipfs_evict_peer(peer)) } - - fn peer_evict(&mut self, _peer: PeerId) -> wasmtime::Result> { - todo!(); - } } From 5f997b1af309ac7c74ae401a9f91be71eb224491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Mon, 1 Jul 2024 20:43:14 -0600 Subject: [PATCH 23/50] feat: keep track of ipfs files belonging to apps * send app name to hermes ipfs functions --- .../runtime_extensions/hermes/ipfs/host.rs | 14 ++-- .../runtime_extensions/hermes/ipfs/state.rs | 74 +++++++++++++++---- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index 9c892554e..bbb11f557 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -16,34 +16,34 @@ use crate::{ impl Host for HermesRuntimeContext { fn file_add(&mut self, contents: IpfsContent) -> wasmtime::Result> { - let path: IpfsPath = hermes_ipfs_add_file(contents)?.to_string(); + let path: IpfsPath = hermes_ipfs_add_file(self.app_name(), contents)?.to_string(); Ok(Ok(path)) } fn file_get(&mut self, path: IpfsPath) -> wasmtime::Result> { - let contents = hermes_ipfs_get_file(path)?; + let contents = hermes_ipfs_get_file(self.app_name(), path)?; Ok(Ok(contents)) } fn file_pin(&mut self, ipfs_path: IpfsPath) -> wasmtime::Result> { - Ok(hermes_ipfs_pin_file(ipfs_path)) + Ok(hermes_ipfs_pin_file(self.app_name(), ipfs_path)) } fn dht_put( &mut self, key: DhtKey, contents: IpfsContent, ) -> wasmtime::Result> { - Ok(hermes_ipfs_put_dht_value(key, contents)) + Ok(hermes_ipfs_put_dht_value(self.app_name(), key, contents)) } fn dht_get(&mut self, key: DhtKey) -> wasmtime::Result> { - Ok(hermes_ipfs_get_dht_value(key)) + Ok(hermes_ipfs_get_dht_value(self.app_name(), key)) } fn pubsub_subscribe(&mut self, topic: PubsubTopic) -> wasmtime::Result> { - Ok(hermes_ipfs_subscribe(topic)) + Ok(hermes_ipfs_subscribe(self.app_name(), topic)) } fn peer_evict(&mut self, peer: PeerId) -> wasmtime::Result> { - Ok(hermes_ipfs_evict_peer(peer)) + Ok(hermes_ipfs_evict_peer(self.app_name(), peer)) } } diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs index ff1c69ad3..e98833e73 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs @@ -2,6 +2,7 @@ use std::str::FromStr; +use dashmap::{DashMap, DashSet}; use hermes_ipfs::{ AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, SubscriptionStream, @@ -12,8 +13,11 @@ use tokio::{ sync::{mpsc, oneshot}, }; -use crate::runtime_extensions::bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, IpfsContent, IpfsPath, PeerId, PubsubTopic, +use crate::{ + app::HermesAppName, + runtime_extensions::bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, IpfsContent, IpfsPath, PeerId, PubsubTopic, + }, }; /// Hermes IPFS Internal State @@ -46,7 +50,7 @@ impl HermesIpfsState { /// Create a new `HermesIpfsState` fn new(sender: Option>) -> Self { Self { - apps: AppIpfsState { sender }, + apps: AppIpfsState::new(sender), } } @@ -65,8 +69,8 @@ impl HermesIpfsState { cmd_rx.blocking_recv().map_err(|_| Errno::FileAddError)? } - /// Get file #[allow(clippy::needless_pass_by_value)] + /// Get file fn file_get(&self, ipfs_path: IpfsPath) -> Result { let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let (cmd_tx, cmd_rx) = oneshot::channel(); @@ -79,8 +83,8 @@ impl HermesIpfsState { cmd_rx.blocking_recv().map_err(|_| Errno::FileGetError)? } - /// Pin file #[allow(clippy::needless_pass_by_value)] + /// Pin file fn file_pin(&self, ipfs_path: IpfsPath) -> Result { let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let cid = ipfs_path.root().cid().ok_or(Errno::InvalidCid)?; @@ -153,6 +157,32 @@ impl HermesIpfsState { struct AppIpfsState { /// Send events to the IPFS node. sender: Option>, + /// List of uploaded files for each app. + files: DashMap>, +} + +impl AppIpfsState { + /// Create new `AppIpfsState` + fn new(sender: Option>) -> Self { + Self { + sender, + files: DashMap::default(), + } + } + + /// Add `ipfs_path` from file added by an app. + fn added_file(&self, app_name: HermesAppName, ipfs_path: IpfsPath) { + self.files + .entry(app_name) + .and_modify(|paths| { + paths.insert(ipfs_path.clone()); + }) + .or_insert_with(|| { + let paths = DashSet::new(); + paths.insert(ipfs_path); + paths + }); + } } /// IPFS Command @@ -244,37 +274,55 @@ async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result< } /// Add File to IPFS -pub(crate) fn hermes_ipfs_add_file(contents: IpfsContent) -> Result { - HERMES_IPFS_STATE.file_add(contents) +pub(crate) fn hermes_ipfs_add_file( + app_name: &HermesAppName, contents: IpfsContent, +) -> Result { + let ipfs_path = HERMES_IPFS_STATE.file_add(contents)?; + HERMES_IPFS_STATE + .apps + .added_file(app_name.clone(), ipfs_path.clone()); + Ok(ipfs_path) } /// Get File from Ipfs -pub(crate) fn hermes_ipfs_get_file(path: IpfsPath) -> Result { +pub(crate) fn hermes_ipfs_get_file( + _app_name: &HermesAppName, path: IpfsPath, +) -> Result { HERMES_IPFS_STATE.file_get(path) } /// Pin IPFS File -pub(crate) fn hermes_ipfs_pin_file(path: IpfsPath) -> Result { +pub(crate) fn hermes_ipfs_pin_file( + _app_name: &HermesAppName, path: IpfsPath, +) -> Result { HERMES_IPFS_STATE.file_pin(path) } /// Get DHT Value -pub(crate) fn hermes_ipfs_get_dht_value(key: DhtKey) -> Result { +pub(crate) fn hermes_ipfs_get_dht_value( + _app_name: &HermesAppName, key: DhtKey, +) -> Result { HERMES_IPFS_STATE.dht_get(key) } /// Put DHT Value -pub(crate) fn hermes_ipfs_put_dht_value(key: DhtKey, value: DhtValue) -> Result { +pub(crate) fn hermes_ipfs_put_dht_value( + _app_name: &HermesAppName, key: DhtKey, value: DhtValue, +) -> Result { HERMES_IPFS_STATE.dht_put(key, value) } /// Subscribe to a topic -pub(crate) fn hermes_ipfs_subscribe(topic: PubsubTopic) -> Result { +pub(crate) fn hermes_ipfs_subscribe( + _app_name: &HermesAppName, topic: PubsubTopic, +) -> Result { let _stream = HERMES_IPFS_STATE.pubsub_subscribe(topic)?; Ok(true) } /// Evict Peer from node -pub(crate) fn hermes_ipfs_evict_peer(peer: PeerId) -> Result { +pub(crate) fn hermes_ipfs_evict_peer( + _app_name: &HermesAppName, peer: PeerId, +) -> Result { HERMES_IPFS_STATE.peer_evict(peer) } From 3b64188ded90950d19411f85d82ff830715ec507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 2 Jul 2024 07:51:01 -0600 Subject: [PATCH 24/50] fix: remove unused import, update event.wit --- wasm/wasi/wit/deps/hermes-ipfs/event.wit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/wasi/wit/deps/hermes-ipfs/event.wit b/wasm/wasi/wit/deps/hermes-ipfs/event.wit index 11d8f77be..250dafb16 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/event.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/event.wit @@ -8,7 +8,7 @@ /// IPFS API Interface - Export ONLY interface event { - use api.{pubsub-message, peer-id}; + use api.{pubsub-messaged}; /// Triggers when a message is received on a topic. on-topic: func(message: pubsub-message) -> bool; From 044959725c9a445b5b860c527f8ba8c21ae72219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 2 Jul 2024 07:51:01 -0600 Subject: [PATCH 25/50] fix: remove unused import, update event.wit --- wasm/wasi/wit/deps/hermes-ipfs/event.wit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/wasi/wit/deps/hermes-ipfs/event.wit b/wasm/wasi/wit/deps/hermes-ipfs/event.wit index 11d8f77be..b6160c48d 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/event.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/event.wit @@ -8,7 +8,7 @@ /// IPFS API Interface - Export ONLY interface event { - use api.{pubsub-message, peer-id}; + use api.{pubsub-message}; /// Triggers when a message is received on a topic. on-topic: func(message: pubsub-message) -> bool; From 9d75ead282863a4ba0b155cc1c4fe5e654fbdcac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 3 Jul 2024 20:46:08 -0600 Subject: [PATCH 26/50] fix: update IPFS WIT bindings --- .../runtime_extensions/hermes/ipfs/event.rs | 6 +-- .../runtime_extensions/hermes/ipfs/host.rs | 26 +++++++----- wasm/wasi/wit/deps/hermes-ipfs/api.wit | 40 +++++++++++++------ 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs index 9ef8f3665..3d09ce7bb 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs @@ -1,14 +1,12 @@ //! Hermes IPFS runtime extension event handler implementation. use crate::{ - event::HermesEventPayload, - runtime_extensions::bindings::hermes::ipfs::api::{PubsubMessage, PubsubTopic}, + event::HermesEventPayload, runtime_extensions::bindings::hermes::ipfs::api::PubsubMessage, }; /// Event handler for the `on-topic` event. #[allow(dead_code)] +#[derive(Debug, Clone)] pub(crate) struct OnTopicEvent { - /// Topic - pub(crate) topic: PubsubTopic, /// Message pub(crate) message: PubsubMessage, } diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index bbb11f557..4bf61f631 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -4,24 +4,24 @@ use crate::{ runtime_context::HermesRuntimeContext, runtime_extensions::{ bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PeerId, PubsubTopic, + DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsFile, IpfsPath, PeerId, PubsubTopic, }, hermes::ipfs::state::{ - hermes_ipfs_add_file, hermes_ipfs_evict_peer, hermes_ipfs_get_dht_value, - hermes_ipfs_get_file, hermes_ipfs_pin_file, hermes_ipfs_put_dht_value, - hermes_ipfs_subscribe, + hermes_ipfs_add_file, hermes_ipfs_content_validate, hermes_ipfs_evict_peer, + hermes_ipfs_get_dht_value, hermes_ipfs_get_file, hermes_ipfs_pin_file, + hermes_ipfs_put_dht_value, hermes_ipfs_subscribe, }, }, }; impl Host for HermesRuntimeContext { - fn file_add(&mut self, contents: IpfsContent) -> wasmtime::Result> { + fn file_add(&mut self, contents: IpfsFile) -> wasmtime::Result> { let path: IpfsPath = hermes_ipfs_add_file(self.app_name(), contents)?.to_string(); Ok(Ok(path)) } - fn file_get(&mut self, path: IpfsPath) -> wasmtime::Result> { - let contents = hermes_ipfs_get_file(self.app_name(), path)?; + fn file_get(&mut self, path: IpfsPath) -> wasmtime::Result> { + let contents = hermes_ipfs_get_file(self.app_name(), &path)?; Ok(Ok(contents)) } @@ -29,10 +29,8 @@ impl Host for HermesRuntimeContext { Ok(hermes_ipfs_pin_file(self.app_name(), ipfs_path)) } - fn dht_put( - &mut self, key: DhtKey, contents: IpfsContent, - ) -> wasmtime::Result> { - Ok(hermes_ipfs_put_dht_value(self.app_name(), key, contents)) + fn dht_put(&mut self, key: DhtKey, value: DhtValue) -> wasmtime::Result> { + Ok(hermes_ipfs_put_dht_value(self.app_name(), key, value)) } fn dht_get(&mut self, key: DhtKey) -> wasmtime::Result> { @@ -43,6 +41,12 @@ impl Host for HermesRuntimeContext { Ok(hermes_ipfs_subscribe(self.app_name(), topic)) } + fn ipfs_content_validate( + &mut self, content: IpfsContent, + ) -> wasmtime::Result> { + Ok(Ok(hermes_ipfs_content_validate(self.app_name(), &content))) + } + fn peer_evict(&mut self, peer: PeerId) -> wasmtime::Result> { Ok(hermes_ipfs_evict_peer(self.app_name(), peer)) } diff --git a/wasm/wasi/wit/deps/hermes-ipfs/api.wit b/wasm/wasi/wit/deps/hermes-ipfs/api.wit index e49c1db79..33b4381c1 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/api.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/api.wit @@ -4,8 +4,15 @@ interface api { type dht-key = list; /// A DHT value. type dht-value = list; + /// IPFS Content. + /// This is content that can be validated. + variant ipfs-content { + /// DHT value + dht(tuple), + pubsub(pubsub-message), + } /// The binary contents of an IPFS file. - type ipfs-content = list; + type ipfs-file = list; /// A path to an IPFS file. type ipfs-path = string; /// The ID of a peer. @@ -19,9 +26,8 @@ interface api { /// The contents of the message. message: string, /// Peer ID that sent the message. - peer: peer-id, + peer: option, } - /// Errors that occur in IPFS networking. enum errno { /// Unable to get DHT value. @@ -34,12 +40,18 @@ interface api { file-get-error, /// Unable to pin file. file-pin-error, - /// Unable to parse a valid IPFS path. - invalid-ipfs-path, /// Invalid CID. invalid-cid, + /// Invalid DHT key. + invalid-dht-key, + /// Invalid DHT value. + invalid-dht-value, + /// Unable to parse a valid IPFS path. + invalid-ipfs-path, /// Invalid Peer ID. invalid-peer-id, + /// Invalid PubSub message. + invalid-pubsub-message, /// Unable to evict peer. peer-eviction-error, /// Unable to publish to IPFS topic. @@ -48,20 +60,22 @@ interface api { pubsub-subscribe-error, } + /// Puts a DHT key-value into IPFS. + dht-put: func(key: dht-key, value: dht-value) -> result; + /// Gets a DHT key-value from IPFS. + dht-get: func(key: dht-key) -> result; + /// Validates IPFS content from DHT or PubSub. + ipfs-content-validate: func(content: ipfs-content) -> result; /// Uploads a file to IPFS. - file-add: func(contents: ipfs-content) -> result; + file-add: func(contents: ipfs-file) -> result; /// Retrieves a file from IPFS. - file-get: func(path: ipfs-path) -> result; + file-get: func(path: ipfs-path) -> result; /// Pins a file by path to IPFS. file-pin: func(path: ipfs-path) -> result; - /// Puts a DHT key-value into IPFS. - dht-put: func(key: dht-key, contents: ipfs-content) -> result; - /// Gets a DHT key-value from IPFS. - dht-get: func(key: dht-key) -> result; - /// Subscribes to a PubSub topic. - pubsub-subscribe: func(topic: pubsub-topic) -> result; /// Evict peer from network. peer-evict: func(peer: peer-id) -> result; + /// Subscribes to a PubSub topic. + pubsub-subscribe: func(topic: pubsub-topic) -> result; } world ipfs-api { From 8117b1e846f2fd95de71b6930779ee134bea0331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 3 Jul 2024 20:47:27 -0600 Subject: [PATCH 27/50] feat: implement IPFS api --- .../runtime_extensions/hermes/ipfs/state.rs | 247 +++++++++++++++--- 1 file changed, 204 insertions(+), 43 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs index e98833e73..45a612902 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs @@ -1,22 +1,28 @@ //! Hermes IPFS Internal State -use std::str::FromStr; +use std::{collections::HashSet, str::FromStr}; use dashmap::{DashMap, DashSet}; use hermes_ipfs::{ + libp2p::futures::{pin_mut, StreamExt}, AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, - SubscriptionStream, }; use once_cell::sync::Lazy; use tokio::{ runtime::Builder, sync::{mpsc, oneshot}, + task::JoinHandle, }; use crate::{ app::HermesAppName, - runtime_extensions::bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, IpfsContent, IpfsPath, PeerId, PubsubTopic, + event::{queue::send, HermesEvent}, + runtime_extensions::{ + bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, PeerId, PubsubMessage, + PubsubTopic, + }, + hermes::ipfs::event::OnTopicEvent, }, }; @@ -55,7 +61,7 @@ impl HermesIpfsState { } /// Add file - fn file_add(&self, contents: IpfsContent) -> Result { + fn file_add(&self, contents: IpfsFile) -> Result { let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps .sender @@ -71,7 +77,7 @@ impl HermesIpfsState { #[allow(clippy::needless_pass_by_value)] /// Get file - fn file_get(&self, ipfs_path: IpfsPath) -> Result { + fn file_get(&self, ipfs_path: IpfsPath) -> Result { let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps @@ -99,13 +105,13 @@ impl HermesIpfsState { } /// Put DHT Key-Value - fn dht_put(&self, key: DhtKey, contents: IpfsContent) -> Result { + fn dht_put(&self, key: DhtKey, value: DhtValue) -> Result { let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps .sender .as_ref() .ok_or(Errno::DhtPutError)? - .blocking_send(IpfsCommand::PutDhtValue(key, contents, cmd_tx)) + .blocking_send(IpfsCommand::PutDhtValue(key, value, cmd_tx)) .map_err(|_| Errno::DhtPutError)?; cmd_rx.blocking_recv().map_err(|_| Errno::DhtPutError)? } @@ -124,7 +130,7 @@ impl HermesIpfsState { #[allow(clippy::needless_pass_by_value)] /// Subscribe to a `PubSub` topic - fn pubsub_subscribe(&self, topic: PubsubTopic) -> Result { + fn pubsub_subscribe(&self, topic: PubsubTopic) -> Result, Errno> { let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps .sender @@ -157,8 +163,18 @@ impl HermesIpfsState { struct AppIpfsState { /// Send events to the IPFS node. sender: Option>, - /// List of uploaded files for each app. - files: DashMap>, + /// List of uploaded files per app. + published_files: DashMap>, + /// List of pinned files per app. + pinned_files: DashMap>, + /// List of DHT values per app. + dht_keys: DashMap>, + /// List of subscriptions per app. + topic_subscriptions: DashMap>, + /// Collection of stream join handles per topic subscription. + subscriptions_streams: DashMap>, + /// List of evicted peers per app. + evicted_peers: DashMap>, } impl AppIpfsState { @@ -166,22 +182,75 @@ impl AppIpfsState { fn new(sender: Option>) -> Self { Self { sender, - files: DashMap::default(), + published_files: DashMap::default(), + pinned_files: DashMap::default(), + dht_keys: DashMap::default(), + topic_subscriptions: DashMap::default(), + subscriptions_streams: DashMap::default(), + evicted_peers: DashMap::default(), } } - /// Add `ipfs_path` from file added by an app. + /// Keep track of `ipfs_path` from file added by an app. fn added_file(&self, app_name: HermesAppName, ipfs_path: IpfsPath) { - self.files + self.published_files .entry(app_name) - .and_modify(|paths| { - paths.insert(ipfs_path.clone()); - }) - .or_insert_with(|| { - let paths = DashSet::new(); - paths.insert(ipfs_path); - paths - }); + .or_default() + .value_mut() + .insert(ipfs_path); + } + + /// Keep track of `ipfs_path` of file pinned by an app. + fn pinned_file(&self, app_name: HermesAppName, ipfs_path: IpfsPath) { + self.pinned_files + .entry(app_name) + .or_default() + .value_mut() + .insert(ipfs_path); + } + + /// Keep track of `dht_key` of DHT value added by an app. + fn added_dht_key(&self, app_name: HermesAppName, dht_key: DhtKey) { + self.dht_keys + .entry(app_name) + .or_default() + .value_mut() + .insert(dht_key); + } + + /// Keep track of `topic` subscription added by an app. + fn added_app_topic_subscription(&self, app_name: HermesAppName, topic: PubsubTopic) { + self.topic_subscriptions + .entry(topic) + .or_default() + .value_mut() + .insert(app_name); + } + + /// Keep track of `topic` stream handle. + fn added_topic_stream(&self, topic: PubsubTopic, handle: JoinHandle<()>) { + self.subscriptions_streams.entry(topic).insert(handle); + } + + /// Check if a topic subscription already exists. + fn topic_subscriptions_contains(&self, topic: &PubsubTopic) -> bool { + self.topic_subscriptions.contains_key(topic) + } + + /// Returns a list of apps subscribed to a topic. + fn subscribed_apps(&self, topic: &PubsubTopic) -> Vec { + self.topic_subscriptions + .get(topic) + .map_or(vec![], |apps| apps.value().iter().cloned().collect()) + } + + /// Add `peer_id` of evicted peer by an app. + fn evicted_peer(&self, app_name: HermesAppName, peer_id: PeerId) { + self.evicted_peers + .entry(app_name) + .or_default() + .value_mut() + .insert(peer_id); } } @@ -198,14 +267,30 @@ enum IpfsCommand { /// Put DHT value PutDhtValue(DhtKey, DhtValue, oneshot::Sender>), /// Subscribe to a topic - Subscribe( - PubsubTopic, - oneshot::Sender>, - ), + Subscribe(PubsubTopic, oneshot::Sender, Errno>>), /// Evict Peer from node EvictPeer(PeerId, oneshot::Sender>), } +/// A valid DHT Value. +struct ValidDhtValue {} + +impl ValidDhtValue { + /// Checks for `DhtValue` validity. + fn is_valid(value: &DhtValue) -> bool { + !value.is_empty() + } +} +/// A valid `PubsubMessage` +struct ValidPubsubMessage {} + +impl ValidPubsubMessage { + /// Checks for `PubsubMessage` validity. + fn is_valid(message: &PubsubMessage) -> bool { + !message.message.is_empty() + } +} + #[allow(dead_code)] /// IPFS async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result<()> { @@ -252,11 +337,32 @@ async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result< } }, IpfsCommand::Subscribe(topic, tx) => { - let status = hermes_node + let stream = hermes_node .pubsub_subscribe(topic) .await - .map_err(|_| Errno::PubsubSubscribeError); - if let Err(_err) = tx.send(status) { + .map_err(|_| Errno::PubsubSubscribeError)?; + let handle = tokio::spawn(async move { + pin_mut!(stream); + while let Some(msg) = stream.next().await { + let msg_topic = msg.topic.into_string(); + let on_topic_event = OnTopicEvent { + message: PubsubMessage { + topic: msg_topic.clone(), + message: String::from_utf8_lossy(&msg.data).to_string(), + peer: msg.source.map(|p| p.to_string()), + }, + }; + let app_names = HERMES_IPFS_STATE.apps.subscribed_apps(&msg_topic); + if let Err(err) = send(HermesEvent::new( + on_topic_event.clone(), + crate::event::TargetApp::List(app_names), + crate::event::TargetModule::All, + )) { + tracing::error!(on_topic_event = ?on_topic_event, "failed to send on_topic_event {err:?}"); + } + } + }); + if let Err(_err) = tx.send(Ok(handle)) { tracing::error!("Failed to subscribe to topic"); } }, @@ -275,54 +381,109 @@ async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result< /// Add File to IPFS pub(crate) fn hermes_ipfs_add_file( - app_name: &HermesAppName, contents: IpfsContent, + app_name: &HermesAppName, contents: IpfsFile, ) -> Result { + tracing::debug!(app_name = %app_name, "adding IPFS file"); let ipfs_path = HERMES_IPFS_STATE.file_add(contents)?; + tracing::debug!(app_name = %app_name, path = %ipfs_path, "added IPFS file"); HERMES_IPFS_STATE .apps .added_file(app_name.clone(), ipfs_path.clone()); Ok(ipfs_path) } +/// Validate IPFS Content from DHT or `PubSub` +pub(crate) fn hermes_ipfs_content_validate( + app_name: &HermesAppName, content: &IpfsContent, +) -> bool { + match content { + IpfsContent::Dht((key, v)) => { + // TODO(@saibatizoku): Implement types and validation + let key_str = format!("{key:x?}"); + let is_valid = ValidDhtValue::is_valid(v); + tracing::debug!(app_name = %app_name, dht_key = %key_str, is_valid = %is_valid, "DHT value validation"); + is_valid + }, + IpfsContent::Pubsub(m) => { + // TODO(@saibatizoku): Implement types and validation + let is_valid = ValidPubsubMessage::is_valid(m); + tracing::debug!(app_name = %app_name, topic = %m.topic, is_valid = %is_valid, "PubSub message validation"); + is_valid + }, + } +} + /// Get File from Ipfs pub(crate) fn hermes_ipfs_get_file( - _app_name: &HermesAppName, path: IpfsPath, -) -> Result { - HERMES_IPFS_STATE.file_get(path) + app_name: &HermesAppName, path: &IpfsPath, +) -> Result { + tracing::debug!(app_name = %app_name, path = %path, "get IPFS file"); + let content = HERMES_IPFS_STATE.file_get(path.to_string())?; + tracing::debug!(app_name = %app_name, path = %path, "got IPFS file"); + Ok(content) } /// Pin IPFS File pub(crate) fn hermes_ipfs_pin_file( - _app_name: &HermesAppName, path: IpfsPath, + app_name: &HermesAppName, path: IpfsPath, ) -> Result { - HERMES_IPFS_STATE.file_pin(path) + tracing::debug!(app_name = %app_name, path = %path, "pin IPFS file"); + let status = HERMES_IPFS_STATE.file_pin(path.clone())?; + tracing::debug!(app_name = %app_name, path = %path, "pinned IPFS file"); + HERMES_IPFS_STATE.apps.pinned_file(app_name.clone(), path); + Ok(status) } /// Get DHT Value pub(crate) fn hermes_ipfs_get_dht_value( - _app_name: &HermesAppName, key: DhtKey, + app_name: &HermesAppName, key: DhtKey, ) -> Result { - HERMES_IPFS_STATE.dht_get(key) + let key_str = format!("{key:x?}"); + tracing::debug!(app_name = %app_name, dht_key = %key_str, "get DHT value"); + let value = HERMES_IPFS_STATE.dht_get(key)?; + tracing::debug!(app_name = %app_name, dht_key = %key_str, "got DHT value"); + Ok(value) } /// Put DHT Value pub(crate) fn hermes_ipfs_put_dht_value( - _app_name: &HermesAppName, key: DhtKey, value: DhtValue, + app_name: &HermesAppName, key: DhtKey, value: DhtValue, ) -> Result { - HERMES_IPFS_STATE.dht_put(key, value) + let key_str = format!("{key:x?}"); + tracing::debug!(app_name = %app_name, dht_key = %key_str, "putting DHT value"); + let status = HERMES_IPFS_STATE.dht_put(key.clone(), value)?; + tracing::debug!(app_name = %app_name, dht_key = %key_str, "have put DHT value"); + HERMES_IPFS_STATE.apps.added_dht_key(app_name.clone(), key); + Ok(status) } /// Subscribe to a topic pub(crate) fn hermes_ipfs_subscribe( - _app_name: &HermesAppName, topic: PubsubTopic, + app_name: &HermesAppName, topic: PubsubTopic, ) -> Result { - let _stream = HERMES_IPFS_STATE.pubsub_subscribe(topic)?; + tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "subscribing to PubSub topic"); + if HERMES_IPFS_STATE.apps.topic_subscriptions_contains(&topic) { + tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "topic subscription stream already exists"); + } else { + let handle = HERMES_IPFS_STATE.pubsub_subscribe(topic.to_string())?; + HERMES_IPFS_STATE + .apps + .added_topic_stream(topic.clone(), handle); + tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "added subscription topic stream"); + } + HERMES_IPFS_STATE + .apps + .added_app_topic_subscription(app_name.clone(), topic); Ok(true) } /// Evict Peer from node pub(crate) fn hermes_ipfs_evict_peer( - _app_name: &HermesAppName, peer: PeerId, + app_name: &HermesAppName, peer: PeerId, ) -> Result { - HERMES_IPFS_STATE.peer_evict(peer) + tracing::debug!(app_name = %app_name, peer_id = %peer, "evicting peer"); + let status = HERMES_IPFS_STATE.peer_evict(peer.to_string())?; + tracing::debug!(app_name = %app_name, peer_id = %peer, "evicted peer"); + HERMES_IPFS_STATE.apps.evicted_peer(app_name.clone(), peer); + Ok(status) } From 0a409400707c5eb59b7ce0b2ddb75c199d74db93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 9 Jul 2024 13:32:40 -0600 Subject: [PATCH 28/50] feat: add pubsub-publish to IPFS wasi definitions * fix: cleanup WIT type definitions --- .../bin/src/runtime_extensions/hermes/ipfs/host.rs | 9 ++++++++- wasm/wasi/wit/deps/hermes-ipfs/api.wit | 12 ++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index 4bf61f631..f84197e6f 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -4,7 +4,8 @@ use crate::{ runtime_context::HermesRuntimeContext, runtime_extensions::{ bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsFile, IpfsPath, PeerId, PubsubTopic, + DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsFile, IpfsPath, MessageData, PeerId, + PubsubTopic, }, hermes::ipfs::state::{ hermes_ipfs_add_file, hermes_ipfs_content_validate, hermes_ipfs_evict_peer, @@ -37,6 +38,12 @@ impl Host for HermesRuntimeContext { Ok(hermes_ipfs_get_dht_value(self.app_name(), key)) } + fn pubsub_publish( + &mut self, _topic: PubsubTopic, _message: MessageData, + ) -> wasmtime::Result> { + Ok(Ok(true)) + } + fn pubsub_subscribe(&mut self, topic: PubsubTopic) -> wasmtime::Result> { Ok(hermes_ipfs_subscribe(self.app_name(), topic)) } diff --git a/wasm/wasi/wit/deps/hermes-ipfs/api.wit b/wasm/wasi/wit/deps/hermes-ipfs/api.wit index 33b4381c1..63ef5a7d6 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/api.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/api.wit @@ -9,12 +9,14 @@ interface api { variant ipfs-content { /// DHT value dht(tuple), - pubsub(pubsub-message), + pubsub(tuple), } /// The binary contents of an IPFS file. type ipfs-file = list; /// A path to an IPFS file. type ipfs-path = string; + /// PubSub Message Data + type message-data = list; /// The ID of a peer. type peer-id = string; /// A PubSub topic. @@ -24,9 +26,9 @@ interface api { /// The topic that the message was received on. topic: pubsub-topic, /// The contents of the message. - message: string, - /// Peer ID that sent the message. - peer: option, + message: message-data, + /// Optional Peer ID that published the message. + publisher: option, } /// Errors that occur in IPFS networking. enum errno { @@ -74,6 +76,8 @@ interface api { file-pin: func(path: ipfs-path) -> result; /// Evict peer from network. peer-evict: func(peer: peer-id) -> result; + /// Publish a message to a topic. + pubsub-publish: func(topic: pubsub-topic, message: message-data) -> result; /// Subscribes to a PubSub topic. pubsub-subscribe: func(topic: pubsub-topic) -> result; } From fb9d5cd02229cb374dcaf30521d76e1fb0ed7d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 9 Jul 2024 13:43:36 -0600 Subject: [PATCH 29/50] feat: add integration testing for IPFS runtime extension --- wasm/integration-test/ipfs/.gitignore | 2 + wasm/integration-test/ipfs/Cargo.toml | 10 + wasm/integration-test/ipfs/Earthfile | 32 +++ .../integration-test/ipfs/rust-toolchain.toml | 3 + wasm/integration-test/ipfs/src/lib.rs | 223 ++++++++++++++++++ 5 files changed, 270 insertions(+) create mode 100644 wasm/integration-test/ipfs/.gitignore create mode 100644 wasm/integration-test/ipfs/Cargo.toml create mode 100644 wasm/integration-test/ipfs/Earthfile create mode 100644 wasm/integration-test/ipfs/rust-toolchain.toml create mode 100644 wasm/integration-test/ipfs/src/lib.rs diff --git a/wasm/integration-test/ipfs/.gitignore b/wasm/integration-test/ipfs/.gitignore new file mode 100644 index 000000000..de51b958f --- /dev/null +++ b/wasm/integration-test/ipfs/.gitignore @@ -0,0 +1,2 @@ +target/ +src/hermes.rs diff --git a/wasm/integration-test/ipfs/Cargo.toml b/wasm/integration-test/ipfs/Cargo.toml new file mode 100644 index 000000000..2b0812a83 --- /dev/null +++ b/wasm/integration-test/ipfs/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ipfs-test-component" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = "0.24.0" diff --git a/wasm/integration-test/ipfs/Earthfile b/wasm/integration-test/ipfs/Earthfile new file mode 100644 index 000000000..96a89323c --- /dev/null +++ b/wasm/integration-test/ipfs/Earthfile @@ -0,0 +1,32 @@ +VERSION 0.8 + +IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.1.8 AS rust-ci +# Use when debugging cat-ci locally. +# IMPORT ../../catalyst-ci/earthly/rust AS rust-ci + +IMPORT ../../wasi AS wasi +IMPORT ../../wasi-hermes-component-adapter AS wasi-hermes-component-adapter + +# gen-bindings - generates `hermes.rs` bindings to work with. +gen-bindings: + FROM wasi+build-rust-bindings + + SAVE ARTIFACT hermes.rs AS LOCAL src/hermes.rs + +# build - builds the IPFS integration test wasm component +build: + DO rust-ci+SETUP + + COPY --dir src . + COPY Cargo.toml . + COPY wasi+build-rust-bindings/hermes.rs src/hermes.rs + + DO rust-ci+CARGO \ + --args "build --target wasm32-unknown-unknown --release" \ + --output="wasm32-unknown-unknown/release/ipfs_test_component.wasm" + + COPY wasi-hermes-component-adapter+build/wasi-hermes-component-adapter.wasm . + + RUN wasm-tools component new -o ipfs.wasm target/wasm32-unknown-unknown/release/ipfs_test_component.wasm --adapt wasi-hermes-component-adapter.wasm + + SAVE ARTIFACT ipfs.wasm diff --git a/wasm/integration-test/ipfs/rust-toolchain.toml b/wasm/integration-test/ipfs/rust-toolchain.toml new file mode 100644 index 000000000..dcb74252a --- /dev/null +++ b/wasm/integration-test/ipfs/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.78" +targets = ["wasm32-unknown-unknown"] diff --git a/wasm/integration-test/ipfs/src/lib.rs b/wasm/integration-test/ipfs/src/lib.rs new file mode 100644 index 000000000..e2688bb1d --- /dev/null +++ b/wasm/integration-test/ipfs/src/lib.rs @@ -0,0 +1,223 @@ +//! Hermes SQLite module integration test with WASM runtime. +//! Generate `hermes.rs` with `earthly +gen-bindings` before writing the test. + +// Allow everything since this is generated code. +#![allow(clippy::all, unused)] +mod hermes; + +use hermes::{ + exports::hermes::integration_test::event::TestResult, + hermes::{ + cardano::api::{BlockSrc, CardanoBlock, CardanoBlockchainId, CardanoTxn}, + cron::api::CronTagged, + ipfs::api::{self as ipfs_api, IpfsContent, PeerId, PubsubMessage}, + kv_store::api::KvValues, + }, + wasi::http::types::{IncomingRequest, ResponseOutparam}, +}; + +const IPFS_DEMO_FILE: &[u8] = b"ipfs file uploaded from wasm"; +struct TestComponent; + +fn test_file_add_and_get_and_pin(run: bool) -> Option { + let status = if run { + if let Ok(ipfs_path) = ipfs_api::file_add(&IPFS_DEMO_FILE.to_vec()) { + let contents_match = if let Ok(ipfs_file) = ipfs_api::file_get(&ipfs_path) { + ipfs_file == IPFS_DEMO_FILE + } else { + false + }; + let expected_status_is_true = if let Ok(status) = ipfs_api::file_pin(&ipfs_path) { + status + } else { + false + }; + contents_match && expected_status_is_true + } else { + false + } + } else { true }; + + Some(TestResult { + name: "IPFS File Add/Get".to_string(), + status, + }) +} +fn test_dht_put_and_get(run: bool) -> Option { + let key = b"my-key".to_vec(); + let value = b"demo dht value".to_vec(); + let status = if run { + if let Ok(dht_value) = ipfs_api::dht_put(&key, &value) { + if let Ok(dht_value) = ipfs_api::dht_get(&key) { + dht_value == value + } else { + false + } + } else { + false + } + } else { true }; + Some(TestResult { + name: "IPFS DHT Put/Get".to_string(), + status, + }) +} + +fn test_pubsub_topic_subscribe(run: bool) -> Option { + let topic = "demo-topic".to_string(); + let status = if run { + ipfs_api::pubsub_subscribe(&topic).is_ok() + } else { + true + }; + Some(TestResult { + name: "IPFS Pubsub Subscribe To Topic".to_string(), + status, + }) +} + +fn test_pubsub_topic_publish(run: bool) -> Option { + let topic = "demo-topic".to_string(); + let message = b"demo message".to_vec(); + let status = if run { + ipfs_api::pubsub_publish(&topic, &message).is_ok() + } else { + false + }; + Some(TestResult { + name: "IPFS Pubsub Publish To Topic".to_string(), + status, + }) +} + +fn test_peer_evict(run: bool) -> Option { + let peer = "12D3KooWMisUYkyVLdVsMcJukAVhPHjGKaJNZG8BKNwh5WGnGk8P".to_string(); + let status = if run { + if let Ok(status) = ipfs_api::peer_evict(&peer) { + status + } else { + false + } + } else { + false + }; + Some(TestResult { + name: "IPFS Ban Peer".to_string(), + status, + }) +} + +fn test_validate_dht_value(run: bool) -> Option { + let key = b"valid-value".to_vec(); + let value = b"demo dht value".to_vec(); + let content = IpfsContent::Dht((key, value)); + let status_a = if run { + if let Ok(is_valid) = ipfs_api::ipfs_content_validate(&content) { + is_valid + } else { + false + } + } else { + false + }; + let key = b"invalid-value".to_vec(); + let value = b"".to_vec(); + let content = IpfsContent::Dht((key, value)); + let status_b = if run { + if let Ok(is_valid) = ipfs_api::ipfs_content_validate(&content) { + !is_valid + } else { + false + } + } else { + false + }; + let status = status_a && status_b; + Some(TestResult { + name: "IPFS Validate DHT Value".to_string(), + status, + }) +} +impl hermes::exports::hermes::integration_test::event::Guest for TestComponent { + fn test(test: u32, run: bool) -> Option { + match test { + 0 => { + // + test_file_add_and_get_and_pin(run) + } + 1 => { + // + test_dht_put_and_get(run) + } + 2 => { + // Test IPFS Pubsub + test_pubsub_topic_subscribe(run) + } + 3 => { + // Test IPFS Pubsub + test_pubsub_topic_publish(run) + } + 4 => { + // Test IPFS Peer Evict + test_peer_evict(run) + } + 5 => { + // Test IPFS Validate DHT Value + test_validate_dht_value(run) + } + + _ => None, + } + } + + fn bench(_test: u32, _run: bool) -> Option { + None + } +} + +impl hermes::exports::hermes::cardano::event_on_block::Guest for TestComponent { + fn on_cardano_block(_blockchain: CardanoBlockchainId, _block: CardanoBlock, _source: BlockSrc) { + } +} + +impl hermes::exports::hermes::cardano::event_on_rollback::Guest for TestComponent { + fn on_cardano_rollback(_blockchain: CardanoBlockchainId, _slot: u64) {} +} + +impl hermes::exports::hermes::cardano::event_on_txn::Guest for TestComponent { + fn on_cardano_txn( + _blockchain: CardanoBlockchainId, + _slot: u64, + _txn_index: u32, + _txn: CardanoTxn, + ) { + } +} + +impl hermes::exports::hermes::cron::event::Guest for TestComponent { + fn on_cron(_event: CronTagged, _last: bool) -> bool { + false + } +} + +impl hermes::exports::hermes::init::event::Guest for TestComponent { + fn init() -> bool { + true + } +} + +impl hermes::exports::hermes::ipfs::event::Guest for TestComponent { + fn on_topic(_message: PubsubMessage) -> bool { + false + } +} + +impl hermes::exports::hermes::kv_store::event::Guest for TestComponent { + fn kv_update(_key: String, _value: KvValues) {} +} + +impl hermes::exports::wasi::http::incoming_handler::Guest for TestComponent { + fn handle(_request: IncomingRequest, _response_out: ResponseOutparam) {} +} + +hermes::export!(TestComponent with_types_in hermes); From c171e377480ab32beb5ac235e01ae32ec578a606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 9 Jul 2024 14:08:32 -0600 Subject: [PATCH 30/50] fix: update rte and add ipfs tests to earthly target --- hermes/Earthfile | 1 + .../runtime_extensions/hermes/ipfs/event.rs | 2 +- .../runtime_extensions/hermes/ipfs/state.rs | 51 +++++++++---------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/hermes/Earthfile b/hermes/Earthfile index ba82aeaaa..6e46fe10c 100644 --- a/hermes/Earthfile +++ b/hermes/Earthfile @@ -80,6 +80,7 @@ test-wasm-integration: COPY ../wasm/integration-test/crypto+build/crypto.wasm ../wasm/test-components/ COPY ../wasm/integration-test/cardano+build/cardano.wasm ../wasm/test-components/ COPY ../wasm/integration-test/hashing+build/hashing.wasm ../wasm/test-components/ + COPY ../wasm/integration-test/ipfs+build/ipfs.wasm ../wasm/test-components/ COPY ../wasm/integration-test/localtime+build/localtime.wasm ../wasm/test-components/ COPY ../wasm/integration-test/logger+build/logger.wasm ../wasm/test-components/ COPY ../wasm/integration-test/sqlite+build/sqlite.wasm ../wasm/test-components/ diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs index 3d09ce7bb..182806c09 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs @@ -7,7 +7,7 @@ use crate::{ #[allow(dead_code)] #[derive(Debug, Clone)] pub(crate) struct OnTopicEvent { - /// Message + /// Topic message received. pub(crate) message: PubsubMessage, } diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs index 45a612902..c6b9d1c1c 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs @@ -19,8 +19,8 @@ use crate::{ event::{queue::send, HermesEvent}, runtime_extensions::{ bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, PeerId, PubsubMessage, - PubsubTopic, + DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, MessageData, PeerId, + PubsubMessage, PubsubTopic, }, hermes::ipfs::event::OnTopicEvent, }, @@ -272,23 +272,14 @@ enum IpfsCommand { EvictPeer(PeerId, oneshot::Sender>), } -/// A valid DHT Value. -struct ValidDhtValue {} - -impl ValidDhtValue { - /// Checks for `DhtValue` validity. - fn is_valid(value: &DhtValue) -> bool { - !value.is_empty() - } +/// Checks for `DhtKey`, and `DhtValue` validity. +fn is_valid_dht_content(_key: &DhtKey, value: &DhtValue) -> bool { + !value.is_empty() } -/// A valid `PubsubMessage` -struct ValidPubsubMessage {} -impl ValidPubsubMessage { - /// Checks for `PubsubMessage` validity. - fn is_valid(message: &PubsubMessage) -> bool { - !message.message.is_empty() - } +/// Checks for `PubsubTopic`, and `MessageData` validity. +fn is_valid_pubsub_content(_topic: &PubsubTopic, message: &MessageData) -> bool { + !message.is_empty() } #[allow(dead_code)] @@ -311,13 +302,17 @@ async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result< }, IpfsCommand::PinFile(cid, tx) => { let status = match hermes_node.insert_pin(&cid).await { - Ok(()) => true, + Ok(()) => Ok(true), + Err(err) if err.to_string().contains("already pinned recursively") => { + tracing::debug!(cid = %cid, "file already pinned"); + Ok(true) + }, Err(err) => { tracing::error!("Failed to pin block {}: {}", cid, err); - false + Err(Errno::FilePinError) }, }; - if let Err(err) = tx.send(Ok(status)) { + if let Err(err) = tx.send(status) { tracing::error!("sending response of pin IPFS file should not fail: {err:?}"); } }, @@ -348,8 +343,8 @@ async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result< let on_topic_event = OnTopicEvent { message: PubsubMessage { topic: msg_topic.clone(), - message: String::from_utf8_lossy(&msg.data).to_string(), - peer: msg.source.map(|p| p.to_string()), + message: msg.data, + publisher: msg.source.map(|p| p.to_string()), }, }; let app_names = HERMES_IPFS_STATE.apps.subscribed_apps(&msg_topic); @@ -397,17 +392,17 @@ pub(crate) fn hermes_ipfs_content_validate( app_name: &HermesAppName, content: &IpfsContent, ) -> bool { match content { - IpfsContent::Dht((key, v)) => { + IpfsContent::Dht((k, v)) => { // TODO(@saibatizoku): Implement types and validation - let key_str = format!("{key:x?}"); - let is_valid = ValidDhtValue::is_valid(v); + let key_str = format!("{k:x?}"); + let is_valid = is_valid_dht_content(k, v); tracing::debug!(app_name = %app_name, dht_key = %key_str, is_valid = %is_valid, "DHT value validation"); is_valid }, - IpfsContent::Pubsub(m) => { + IpfsContent::Pubsub((topic, message)) => { // TODO(@saibatizoku): Implement types and validation - let is_valid = ValidPubsubMessage::is_valid(m); - tracing::debug!(app_name = %app_name, topic = %m.topic, is_valid = %is_valid, "PubSub message validation"); + let is_valid = is_valid_pubsub_content(topic, message); + tracing::debug!(app_name = %app_name, topic = %topic, is_valid = %is_valid, "PubSub message validation"); is_valid }, } From 9697b45faf518dd258b2a162e33b0c88f3f85c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 9 Jul 2024 14:09:38 -0600 Subject: [PATCH 31/50] fix: update hermes-ipfs example to handle content already pinned --- .../hermes-ipfs/examples/add-file-with-pinning.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/hermes/crates/hermes-ipfs/examples/add-file-with-pinning.rs b/hermes/crates/hermes-ipfs/examples/add-file-with-pinning.rs index a9346857c..53a924b3e 100644 --- a/hermes/crates/hermes-ipfs/examples/add-file-with-pinning.rs +++ b/hermes/crates/hermes-ipfs/examples/add-file-with-pinning.rs @@ -35,12 +35,25 @@ async fn main() -> anyhow::Result<()> { println!("***************************************"); println!("* CID Pinning:"); println!(""); + if let Err(e) = hermes_ipfs.insert_pin(cid).await { + if e.to_string().contains("already pinned recursively") { + println!("{cid} is already pinned"); + } else { + println!("AN ERROR OCCURRED: {e}"); + } + }; println!("* Removing pin."); hermes_ipfs.remove_pin(cid).await?; print_cid_pinned(&hermes_ipfs, cid).await?; println!(""); println!("* Re-pinning CID:."); - hermes_ipfs.insert_pin(cid).await?; + if let Err(e) = hermes_ipfs.insert_pin(cid).await { + if e.to_string().contains("already pinned recursively") { + println!("{cid} is already pinned"); + } else { + println!("AN ERROR OCCURRED: {e}"); + } + }; print_cid_pinned(&hermes_ipfs, cid).await?; println!("***************************************"); println!(""); From 58f7f3ae870864388b2baa3a145799dcffd72cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 9 Jul 2024 14:32:56 -0600 Subject: [PATCH 32/50] fix: remove re-exports --- .../bin/src/runtime_extensions/hermes/ipfs/state.rs | 4 ++-- hermes/crates/hermes-ipfs/examples/pubsub.rs | 7 ++----- hermes/crates/hermes-ipfs/src/lib.rs | 13 ++++--------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs index c6b9d1c1c..92ceb621a 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs @@ -4,8 +4,8 @@ use std::{collections::HashSet, str::FromStr}; use dashmap::{DashMap, DashSet}; use hermes_ipfs::{ - libp2p::futures::{pin_mut, StreamExt}, - AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, + pin_mut, AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, + StreamExt, }; use once_cell::sync::Lazy; use tokio::{ diff --git a/hermes/crates/hermes-ipfs/examples/pubsub.rs b/hermes/crates/hermes-ipfs/examples/pubsub.rs index aa65272c7..2e3696ef8 100644 --- a/hermes/crates/hermes-ipfs/examples/pubsub.rs +++ b/hermes/crates/hermes-ipfs/examples/pubsub.rs @@ -14,11 +14,8 @@ //! * The task that reads lines from stdin and publishes them as either node. use std::io::Write; -use hermes_ipfs::HermesIpfs; -use rust_ipfs::{ - libp2p::futures::{pin_mut, FutureExt, StreamExt}, - PubsubEvent, -}; +use hermes_ipfs::{pin_mut, FutureExt, HermesIpfs, StreamExt}; +use rust_ipfs::PubsubEvent; use rustyline_async::Readline; #[allow(clippy::indexing_slicing)] diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index b06053691..283168c90 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -35,12 +35,12 @@ use std::str::FromStr; +/// IPFS Content Identifier. +pub use libipld::Cid; /// IPLD pub use libipld::Ipld; -/// IPFS Content Identifier. -pub use libipld::{self as ipld, Cid}; /// libp2p re-export. -pub use rust_ipfs::libp2p; +pub use rust_ipfs::libp2p::futures::{pin_mut, stream::BoxStream, FutureExt, StreamExt}; /// Peer Info type. pub use rust_ipfs::p2p::PeerInfo; /// Enum for specifying paths in IPFS. @@ -57,12 +57,7 @@ pub use rust_ipfs::PeerId; pub use rust_ipfs::SubscriptionStream; /// Builder type for IPFS Node configuration. pub use rust_ipfs::UninitializedIpfsNoop as IpfsBuilder; -use rust_ipfs::{ - dag::ResolveError, - libp2p::futures::{pin_mut, stream::BoxStream, StreamExt}, - unixfs::AddOpt, - MessageId, PubsubEvent, Quorum, -}; +use rust_ipfs::{dag::ResolveError, unixfs::AddOpt, MessageId, PubsubEvent, Quorum}; /// Hermes IPFS #[allow(dead_code)] From 71cf59be7736f63d2a4b5e237f23dbff7d36200b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 9 Jul 2024 14:38:39 -0600 Subject: [PATCH 33/50] fix: type_lenght_limit to build doctests --- hermes/bin/src/lib.rs | 1 + hermes/crates/hermes-ipfs/src/lib.rs | 31 ---------------------------- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/hermes/bin/src/lib.rs b/hermes/bin/src/lib.rs index 6dfd436e0..b1002a8e5 100644 --- a/hermes/bin/src/lib.rs +++ b/hermes/bin/src/lib.rs @@ -1,5 +1,6 @@ //! Intentionally empty //! This file exists, so that doc tests can be used inside binary crates. +#![type_length_limit = "45079293105"] pub mod app; #[allow(dead_code)] diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index 283168c90..f5242d97f 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -1,37 +1,6 @@ //! Hermes IPFS //! //! Provides support for storage, and `PubSub` functionality. -//! -//! ```no_run -//! use hermes_ipfs::HermesIpfs; -//! -//! #[tokio::main] -//! #[allow(clippy::println_empty_string)] -//! async fn main() -> anyhow::Result<()> { -//! // Start with default options -//! let node = HermesIpfs::start().await?; -//! node.stop().await; -//! Ok(()) -//! } -//! ``` -//! -//! ```no_run -//! use hermes_ipfs::{HermesIpfs, IpfsBuilder}; -//! -//! #[tokio::main] -//! #[allow(clippy::println_empty_string)] -//! async fn main() -> anyhow::Result<()> { -//! // Start with default options -//! let node: HermesIpfs = IpfsBuilder::new() -//! .with_default() -//! .set_default_listener() -//! .start() -//! .await? -//! .into(); -//! node.stop().await; -//! Ok(()) -//! } -//! ``` use std::str::FromStr; From 2c5b38341a586512c57601a6467444f3649a3c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 9 Jul 2024 14:53:26 -0600 Subject: [PATCH 34/50] chore: refactor 'mod task' out of state --- .../hermes/ipfs/{state.rs => state/mod.rs} | 126 +---------------- .../hermes/ipfs/state/task.rs | 132 ++++++++++++++++++ 2 files changed, 137 insertions(+), 121 deletions(-) rename hermes/bin/src/runtime_extensions/hermes/ipfs/{state.rs => state/mod.rs} (70%) create mode 100644 hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs similarity index 70% rename from hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs rename to hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs index 92ceb621a..7bb8a05ef 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs @@ -1,13 +1,12 @@ //! Hermes IPFS Internal State +mod task; use std::{collections::HashSet, str::FromStr}; use dashmap::{DashMap, DashSet}; -use hermes_ipfs::{ - pin_mut, AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, - StreamExt, -}; +use hermes_ipfs::{AddIpfsFile, IpfsPath as PathIpfsFile}; use once_cell::sync::Lazy; +use task::{ipfs_task, IpfsCommand}; use tokio::{ runtime::Builder, sync::{mpsc, oneshot}, @@ -16,13 +15,8 @@ use tokio::{ use crate::{ app::HermesAppName, - event::{queue::send, HermesEvent}, - runtime_extensions::{ - bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, MessageData, PeerId, - PubsubMessage, PubsubTopic, - }, - hermes::ipfs::event::OnTopicEvent, + runtime_extensions::bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, MessageData, PeerId, PubsubTopic, }, }; @@ -254,24 +248,6 @@ impl AppIpfsState { } } -/// IPFS Command -enum IpfsCommand { - /// Add a new IPFS file - AddFile(AddIpfsFile, oneshot::Sender>), - /// Get a file from IPFS - GetFile(PathIpfsFile, oneshot::Sender, Errno>>), - /// Pin a file - PinFile(Cid, oneshot::Sender>), - /// Get DHT value - GetDhtValue(DhtKey, oneshot::Sender>), - /// Put DHT value - PutDhtValue(DhtKey, DhtValue, oneshot::Sender>), - /// Subscribe to a topic - Subscribe(PubsubTopic, oneshot::Sender, Errno>>), - /// Evict Peer from node - EvictPeer(PeerId, oneshot::Sender>), -} - /// Checks for `DhtKey`, and `DhtValue` validity. fn is_valid_dht_content(_key: &DhtKey, value: &DhtValue) -> bool { !value.is_empty() @@ -282,98 +258,6 @@ fn is_valid_pubsub_content(_topic: &PubsubTopic, message: &MessageData) -> bool !message.is_empty() } -#[allow(dead_code)] -/// IPFS -async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result<()> { - let hermes_node = HermesIpfs::start().await?; - if let Some(ipfs_command) = queue_rx.recv().await { - match ipfs_command { - IpfsCommand::AddFile(ipfs_file, tx) => { - let ipfs_path = hermes_node.add_ipfs_file(ipfs_file).await?; - if let Err(_err) = tx.send(Ok(ipfs_path.to_string())) { - tracing::error!("Failed to send IPFS path"); - } - }, - IpfsCommand::GetFile(ipfs_path, tx) => { - let contents = hermes_node.get_ipfs_file(ipfs_path.into()).await?; - if let Err(_err) = tx.send(Ok(contents)) { - tracing::error!("Failed to get IPFS contents"); - } - }, - IpfsCommand::PinFile(cid, tx) => { - let status = match hermes_node.insert_pin(&cid).await { - Ok(()) => Ok(true), - Err(err) if err.to_string().contains("already pinned recursively") => { - tracing::debug!(cid = %cid, "file already pinned"); - Ok(true) - }, - Err(err) => { - tracing::error!("Failed to pin block {}: {}", cid, err); - Err(Errno::FilePinError) - }, - }; - if let Err(err) = tx.send(status) { - tracing::error!("sending response of pin IPFS file should not fail: {err:?}"); - } - }, - IpfsCommand::GetDhtValue(key, tx) => { - let response = hermes_node - .dht_get(key) - .await - .map_err(|_| Errno::DhtGetError); - if let Err(err) = tx.send(response) { - tracing::error!("sending DHT value should not fail: {err:?}"); - } - }, - IpfsCommand::PutDhtValue(key, value, tx) => { - let status = hermes_node.dht_put(key, value).await.is_ok(); - if let Err(err) = tx.send(Ok(status)) { - tracing::error!("sending status of DHT put should not fail: {err:?}"); - } - }, - IpfsCommand::Subscribe(topic, tx) => { - let stream = hermes_node - .pubsub_subscribe(topic) - .await - .map_err(|_| Errno::PubsubSubscribeError)?; - let handle = tokio::spawn(async move { - pin_mut!(stream); - while let Some(msg) = stream.next().await { - let msg_topic = msg.topic.into_string(); - let on_topic_event = OnTopicEvent { - message: PubsubMessage { - topic: msg_topic.clone(), - message: msg.data, - publisher: msg.source.map(|p| p.to_string()), - }, - }; - let app_names = HERMES_IPFS_STATE.apps.subscribed_apps(&msg_topic); - if let Err(err) = send(HermesEvent::new( - on_topic_event.clone(), - crate::event::TargetApp::List(app_names), - crate::event::TargetModule::All, - )) { - tracing::error!(on_topic_event = ?on_topic_event, "failed to send on_topic_event {err:?}"); - } - } - }); - if let Err(_err) = tx.send(Ok(handle)) { - tracing::error!("Failed to subscribe to topic"); - } - }, - IpfsCommand::EvictPeer(peer, tx) => { - let peer_id = TargetPeerId::from_str(&peer).map_err(|_| Errno::InvalidPeerId)?; - let status = hermes_node.ban_peer(peer_id).await.is_ok(); - if let Err(err) = tx.send(Ok(status)) { - tracing::error!("sending status of peer eviction should not fail: {err:?}"); - } - }, - } - } - hermes_node.stop().await; - Ok(()) -} - /// Add File to IPFS pub(crate) fn hermes_ipfs_add_file( app_name: &HermesAppName, contents: IpfsFile, diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs new file mode 100644 index 000000000..d58687474 --- /dev/null +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs @@ -0,0 +1,132 @@ +//! IPFS Task +use std::str::FromStr; + +use hermes_ipfs::{ + pin_mut, AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, + StreamExt, +}; +use tokio::{ + sync::{mpsc, oneshot}, + task::JoinHandle, +}; + +use super::HERMES_IPFS_STATE; +use crate::{ + event::{queue::send, HermesEvent}, + runtime_extensions::{ + bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, IpfsPath, PeerId, PubsubMessage, PubsubTopic, + }, + hermes::ipfs::event::OnTopicEvent, + }, +}; + +/// IPFS Command +pub(crate) enum IpfsCommand { + /// Add a new IPFS file + AddFile(AddIpfsFile, oneshot::Sender>), + /// Get a file from IPFS + GetFile(PathIpfsFile, oneshot::Sender, Errno>>), + /// Pin a file + PinFile(Cid, oneshot::Sender>), + /// Get DHT value + GetDhtValue(DhtKey, oneshot::Sender>), + /// Put DHT value + PutDhtValue(DhtKey, DhtValue, oneshot::Sender>), + /// Subscribe to a topic + Subscribe(PubsubTopic, oneshot::Sender, Errno>>), + /// Evict Peer from node + EvictPeer(PeerId, oneshot::Sender>), +} + +#[allow(dead_code)] +/// IPFS +pub(crate) async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result<()> { + let hermes_node = HermesIpfs::start().await?; + if let Some(ipfs_command) = queue_rx.recv().await { + match ipfs_command { + IpfsCommand::AddFile(ipfs_file, tx) => { + let ipfs_path = hermes_node.add_ipfs_file(ipfs_file).await?; + if let Err(_err) = tx.send(Ok(ipfs_path.to_string())) { + tracing::error!("Failed to send IPFS path"); + } + }, + IpfsCommand::GetFile(ipfs_path, tx) => { + let contents = hermes_node.get_ipfs_file(ipfs_path.into()).await?; + if let Err(_err) = tx.send(Ok(contents)) { + tracing::error!("Failed to get IPFS contents"); + } + }, + IpfsCommand::PinFile(cid, tx) => { + let status = match hermes_node.insert_pin(&cid).await { + Ok(()) => Ok(true), + Err(err) if err.to_string().contains("already pinned recursively") => { + tracing::debug!(cid = %cid, "file already pinned"); + Ok(true) + }, + Err(err) => { + tracing::error!("Failed to pin block {}: {}", cid, err); + Err(Errno::FilePinError) + }, + }; + if let Err(err) = tx.send(status) { + tracing::error!("sending response of pin IPFS file should not fail: {err:?}"); + } + }, + IpfsCommand::GetDhtValue(key, tx) => { + let response = hermes_node + .dht_get(key) + .await + .map_err(|_| Errno::DhtGetError); + if let Err(err) = tx.send(response) { + tracing::error!("sending DHT value should not fail: {err:?}"); + } + }, + IpfsCommand::PutDhtValue(key, value, tx) => { + let status = hermes_node.dht_put(key, value).await.is_ok(); + if let Err(err) = tx.send(Ok(status)) { + tracing::error!("sending status of DHT put should not fail: {err:?}"); + } + }, + IpfsCommand::Subscribe(topic, tx) => { + let stream = hermes_node + .pubsub_subscribe(topic) + .await + .map_err(|_| Errno::PubsubSubscribeError)?; + let handle = tokio::spawn(async move { + pin_mut!(stream); + while let Some(msg) = stream.next().await { + let msg_topic = msg.topic.into_string(); + let on_topic_event = OnTopicEvent { + message: PubsubMessage { + topic: msg_topic.clone(), + message: msg.data, + publisher: msg.source.map(|p| p.to_string()), + }, + }; + let app_names = HERMES_IPFS_STATE.apps.subscribed_apps(&msg_topic); + if let Err(err) = send(HermesEvent::new( + on_topic_event.clone(), + crate::event::TargetApp::List(app_names), + crate::event::TargetModule::All, + )) { + tracing::error!(on_topic_event = ?on_topic_event, "failed to send on_topic_event {err:?}"); + } + } + }); + if let Err(_err) = tx.send(Ok(handle)) { + tracing::error!("Failed to subscribe to topic"); + } + }, + IpfsCommand::EvictPeer(peer, tx) => { + let peer_id = TargetPeerId::from_str(&peer).map_err(|_| Errno::InvalidPeerId)?; + let status = hermes_node.ban_peer(peer_id).await.is_ok(); + if let Err(err) = tx.send(Ok(status)) { + tracing::error!("sending status of peer eviction should not fail: {err:?}"); + } + }, + } + } + hermes_node.stop().await; + Ok(()) +} From 9e42e4e0c0244cc9bc97dbee578fe840c5326aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 10:20:44 -0600 Subject: [PATCH 35/50] fix: types for pubsub-publishing --- hermes/crates/hermes-ipfs/src/lib.rs | 6 ++++-- wasm/wasi/wit/deps/hermes-ipfs/api.wit | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index f5242d97f..540c9b773 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -18,6 +18,8 @@ pub use rust_ipfs::path::IpfsPath; pub use rust_ipfs::DhtMode; /// Server, Client, or Auto mode pub use rust_ipfs::Ipfs; +/// `PubSub` Message ID type. +pub use rust_ipfs::MessageId as PubsubMessageId; /// Multiaddr type. pub use rust_ipfs::Multiaddr; /// Peer ID type. @@ -26,7 +28,7 @@ pub use rust_ipfs::PeerId; pub use rust_ipfs::SubscriptionStream; /// Builder type for IPFS Node configuration. pub use rust_ipfs::UninitializedIpfsNoop as IpfsBuilder; -use rust_ipfs::{dag::ResolveError, unixfs::AddOpt, MessageId, PubsubEvent, Quorum}; +use rust_ipfs::{dag::ResolveError, unixfs::AddOpt, PubsubEvent, Quorum}; /// Hermes IPFS #[allow(dead_code)] @@ -385,7 +387,7 @@ impl HermesIpfs { /// Returns error if unable to publish to a pubsub topic. pub async fn pubsub_publish( &self, topic: impl Into, message: Vec, - ) -> anyhow::Result { + ) -> anyhow::Result { self.node.pubsub_publish(topic, message).await } diff --git a/wasm/wasi/wit/deps/hermes-ipfs/api.wit b/wasm/wasi/wit/deps/hermes-ipfs/api.wit index 63ef5a7d6..275192c32 100644 --- a/wasm/wasi/wit/deps/hermes-ipfs/api.wit +++ b/wasm/wasi/wit/deps/hermes-ipfs/api.wit @@ -17,6 +17,8 @@ interface api { type ipfs-path = string; /// PubSub Message Data type message-data = list; + /// PubSub Message ID + type message-id = list; /// The ID of a peer. type peer-id = string; /// A PubSub topic. @@ -77,7 +79,7 @@ interface api { /// Evict peer from network. peer-evict: func(peer: peer-id) -> result; /// Publish a message to a topic. - pubsub-publish: func(topic: pubsub-topic, message: message-data) -> result; + pubsub-publish: func(topic: pubsub-topic, message: message-data) -> result; /// Subscribes to a PubSub topic. pubsub-subscribe: func(topic: pubsub-topic) -> result; } From 3c8bb57a4cf0ba2fbf2349608382c569f13c80a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 10:22:06 -0600 Subject: [PATCH 36/50] fix: add publishing implementation --- .../runtime_extensions/hermes/ipfs/host.rs | 11 +++---- .../hermes/ipfs/state/mod.rs | 29 +++++++++++++++++-- .../hermes/ipfs/state/task.rs | 13 +++++++-- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index f84197e6f..59e510ac0 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -1,11 +1,12 @@ //! IPFS host implementation for WASM runtime. +use super::state::hermes_ipfs_publish; use crate::{ runtime_context::HermesRuntimeContext, runtime_extensions::{ bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsFile, IpfsPath, MessageData, PeerId, - PubsubTopic, + DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsFile, IpfsPath, MessageData, MessageId, + PeerId, PubsubTopic, }, hermes::ipfs::state::{ hermes_ipfs_add_file, hermes_ipfs_content_validate, hermes_ipfs_evict_peer, @@ -39,9 +40,9 @@ impl Host for HermesRuntimeContext { } fn pubsub_publish( - &mut self, _topic: PubsubTopic, _message: MessageData, - ) -> wasmtime::Result> { - Ok(Ok(true)) + &mut self, topic: PubsubTopic, message: MessageData, + ) -> wasmtime::Result> { + Ok(hermes_ipfs_publish(self.app_name(), &topic, message)) } fn pubsub_subscribe(&mut self, topic: PubsubTopic) -> wasmtime::Result> { diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs index 7bb8a05ef..c72c2ca9e 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs @@ -4,7 +4,7 @@ mod task; use std::{collections::HashSet, str::FromStr}; use dashmap::{DashMap, DashSet}; -use hermes_ipfs::{AddIpfsFile, IpfsPath as PathIpfsFile}; +use hermes_ipfs::{AddIpfsFile, IpfsPath as PathIpfsFile, PubsubMessageId}; use once_cell::sync::Lazy; use task::{ipfs_task, IpfsCommand}; use tokio::{ @@ -16,7 +16,8 @@ use tokio::{ use crate::{ app::HermesAppName, runtime_extensions::bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, MessageData, PeerId, PubsubTopic, + DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, MessageData, MessageId, PeerId, + PubsubTopic, }, }; @@ -122,6 +123,22 @@ impl HermesIpfsState { cmd_rx.blocking_recv().map_err(|_| Errno::DhtGetError)? } + /// Publish message to a `PubSub` topic + fn pubsub_publish( + &self, topic: PubsubTopic, message: MessageData, + ) -> Result { + let (cmd_tx, cmd_rx) = oneshot::channel(); + self.apps + .sender + .as_ref() + .ok_or(Errno::PubsubPublishError)? + .blocking_send(IpfsCommand::Publish(topic, message, cmd_tx)) + .map_err(|_| Errno::PubsubPublishError)?; + cmd_rx + .blocking_recv() + .map_err(|_| Errno::PubsubPublishError)? + } + #[allow(clippy::needless_pass_by_value)] /// Subscribe to a `PubSub` topic fn pubsub_subscribe(&self, topic: PubsubTopic) -> Result, Errno> { @@ -356,6 +373,14 @@ pub(crate) fn hermes_ipfs_subscribe( Ok(true) } +/// Publish message to a topic +pub(crate) fn hermes_ipfs_publish( + _app_name: &HermesAppName, topic: &PubsubTopic, message: MessageData, +) -> Result { + let message_id = HERMES_IPFS_STATE.pubsub_publish(topic.to_string(), message)?; + Ok(message_id.0) +} + /// Evict Peer from node pub(crate) fn hermes_ipfs_evict_peer( app_name: &HermesAppName, peer: PeerId, diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs index d58687474..4d18a4fca 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use hermes_ipfs::{ pin_mut, AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, - StreamExt, + PubsubMessageId, StreamExt, }; use tokio::{ sync::{mpsc, oneshot}, @@ -15,7 +15,7 @@ use crate::{ event::{queue::send, HermesEvent}, runtime_extensions::{ bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, IpfsPath, PeerId, PubsubMessage, PubsubTopic, + DhtKey, DhtValue, Errno, IpfsPath, MessageData, PeerId, PubsubMessage, PubsubTopic, }, hermes::ipfs::event::OnTopicEvent, }, @@ -88,6 +88,15 @@ pub(crate) async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyh tracing::error!("sending status of DHT put should not fail: {err:?}"); } }, + IpfsCommand::Publish(topic, message, tx) => { + let message_id = hermes_node + .pubsub_publish(topic, message) + .await + .map_err(|_| Errno::PubsubPublishError)?; + if let Err(err) = tx.send(Ok(message_id)) { + tracing::error!("sending message id should not fail: {err:?}"); + } + }, IpfsCommand::Subscribe(topic, tx) => { let stream = hermes_node .pubsub_subscribe(topic) From 52ec9cae759926d7b7d6c9faee4575cee03f5a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 10:26:54 -0600 Subject: [PATCH 37/50] chore: update docs --- .../hermes/ipfs/state/mod.rs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs index c72c2ca9e..d74eedd16 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs @@ -22,6 +22,16 @@ use crate::{ }; /// Hermes IPFS Internal State +/// +/// This is a wrapper around `HermesIpfsState` which provides a singleton instance of the +/// IPFS state. +/// +/// This is done to ensure the IPFS state is initialized only once when the +/// `HermesIpfsState` is first used. This is done to avoid any issues that may arise if +/// the IPFS state is initialized multiple times. +/// +/// The IPFS state is initialized in a separate thread and the sender channel is stored in +/// the `HermesIpfsState`. static HERMES_IPFS_STATE: Lazy = Lazy::new(|| { let sender = if let Ok(runtime) = Builder::new_current_thread().enable_all().build() { let (sender, receiver) = mpsc::channel(1); @@ -56,6 +66,14 @@ impl HermesIpfsState { } /// Add file + /// + /// Returns the IPFS path of the added file + /// + /// ## Parameters + /// - `contents`: The content to add + /// + /// ## Errors + /// - `Errno::FileAddError`: Failed to add the content fn file_add(&self, contents: IpfsFile) -> Result { let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps @@ -72,6 +90,15 @@ impl HermesIpfsState { #[allow(clippy::needless_pass_by_value)] /// Get file + /// + /// Returns the content of the file + /// + /// ## Parameters + /// - `ipfs_path`: The IPFS path of the file + /// + /// ## Errors + /// - `Errno::InvalidIpfsPath`: Invalid IPFS path + /// - `Errno::FileGetError`: Failed to get the file fn file_get(&self, ipfs_path: IpfsPath) -> Result { let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let (cmd_tx, cmd_rx) = oneshot::channel(); @@ -86,6 +113,14 @@ impl HermesIpfsState { #[allow(clippy::needless_pass_by_value)] /// Pin file + /// + /// ## Parameters + /// - `ipfs_path`: The IPFS path of the file + /// + /// ## Errors + /// - `Errno::InvalidCid`: Invalid CID + /// - `Errno::InvalidIpfsPath`: Invalid IPFS path + /// - `Errno::FilePinError`: Failed to pin the file fn file_pin(&self, ipfs_path: IpfsPath) -> Result { let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let cid = ipfs_path.root().cid().ok_or(Errno::InvalidCid)?; @@ -267,11 +302,13 @@ impl AppIpfsState { /// Checks for `DhtKey`, and `DhtValue` validity. fn is_valid_dht_content(_key: &DhtKey, value: &DhtValue) -> bool { + // TODO(@anyone): Implement DHT content validation !value.is_empty() } /// Checks for `PubsubTopic`, and `MessageData` validity. fn is_valid_pubsub_content(_topic: &PubsubTopic, message: &MessageData) -> bool { + // TODO(@anyone): Implement PubSub content validation !message.is_empty() } From 030426157d1d200e03e7a9c1c86ab634de3e324c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 10:27:53 -0600 Subject: [PATCH 38/50] wip: publish --- hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs index 4d18a4fca..607321559 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs @@ -33,6 +33,12 @@ pub(crate) enum IpfsCommand { GetDhtValue(DhtKey, oneshot::Sender>), /// Put DHT value PutDhtValue(DhtKey, DhtValue, oneshot::Sender>), + /// Publish to a topic + Publish( + PubsubTopic, + MessageData, + oneshot::Sender>, + ), /// Subscribe to a topic Subscribe(PubsubTopic, oneshot::Sender, Errno>>), /// Evict Peer from node From 98f88ba8464692ce6ef6457244a1471593248471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 10:28:11 -0600 Subject: [PATCH 39/50] fix: cleanup code * remove lint attributes * DRY code * improve docs --- .../runtime_extensions/hermes/ipfs/event.rs | 1 - .../hermes/ipfs/state/mod.rs | 30 +++++----- .../hermes/ipfs/state/task.rs | 58 +++++++++---------- 3 files changed, 41 insertions(+), 48 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs index 182806c09..62533e155 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs @@ -4,7 +4,6 @@ use crate::{ }; /// Event handler for the `on-topic` event. -#[allow(dead_code)] #[derive(Debug, Clone)] pub(crate) struct OnTopicEvent { /// Topic message received. diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs index d74eedd16..af2f440b0 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs @@ -88,7 +88,6 @@ impl HermesIpfsState { cmd_rx.blocking_recv().map_err(|_| Errno::FileAddError)? } - #[allow(clippy::needless_pass_by_value)] /// Get file /// /// Returns the content of the file @@ -99,19 +98,18 @@ impl HermesIpfsState { /// ## Errors /// - `Errno::InvalidIpfsPath`: Invalid IPFS path /// - `Errno::FileGetError`: Failed to get the file - fn file_get(&self, ipfs_path: IpfsPath) -> Result { - let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; + fn file_get(&self, ipfs_path: &IpfsPath) -> Result { + let ipfs_path = PathIpfsFile::from_str(ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps .sender .as_ref() .ok_or(Errno::FileGetError)? - .blocking_send(IpfsCommand::GetFile(ipfs_path, cmd_tx)) + .blocking_send(IpfsCommand::GetFile(ipfs_path.clone(), cmd_tx)) .map_err(|_| Errno::FileGetError)?; cmd_rx.blocking_recv().map_err(|_| Errno::FileGetError)? } - #[allow(clippy::needless_pass_by_value)] /// Pin file /// /// ## Parameters @@ -121,8 +119,8 @@ impl HermesIpfsState { /// - `Errno::InvalidCid`: Invalid CID /// - `Errno::InvalidIpfsPath`: Invalid IPFS path /// - `Errno::FilePinError`: Failed to pin the file - fn file_pin(&self, ipfs_path: IpfsPath) -> Result { - let ipfs_path = PathIpfsFile::from_str(&ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; + fn file_pin(&self, ipfs_path: &IpfsPath) -> Result { + let ipfs_path = PathIpfsFile::from_str(ipfs_path).map_err(|_| Errno::InvalidIpfsPath)?; let cid = ipfs_path.root().cid().ok_or(Errno::InvalidCid)?; let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps @@ -174,30 +172,28 @@ impl HermesIpfsState { .map_err(|_| Errno::PubsubPublishError)? } - #[allow(clippy::needless_pass_by_value)] /// Subscribe to a `PubSub` topic - fn pubsub_subscribe(&self, topic: PubsubTopic) -> Result, Errno> { + fn pubsub_subscribe(&self, topic: &PubsubTopic) -> Result, Errno> { let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps .sender .as_ref() .ok_or(Errno::PubsubSubscribeError)? - .blocking_send(IpfsCommand::Subscribe(topic, cmd_tx)) + .blocking_send(IpfsCommand::Subscribe(topic.clone(), cmd_tx)) .map_err(|_| Errno::PubsubSubscribeError)?; cmd_rx .blocking_recv() .map_err(|_| Errno::PubsubSubscribeError)? } - #[allow(clippy::needless_pass_by_value)] /// Evict peer - fn peer_evict(&self, peer: PeerId) -> Result { + fn peer_evict(&self, peer: &PeerId) -> Result { let (cmd_tx, cmd_rx) = oneshot::channel(); self.apps .sender .as_ref() .ok_or(Errno::PeerEvictionError)? - .blocking_send(IpfsCommand::EvictPeer(peer, cmd_tx)) + .blocking_send(IpfsCommand::EvictPeer(peer.clone(), cmd_tx)) .map_err(|_| Errno::PeerEvictionError)?; cmd_rx .blocking_recv() @@ -351,7 +347,7 @@ pub(crate) fn hermes_ipfs_get_file( app_name: &HermesAppName, path: &IpfsPath, ) -> Result { tracing::debug!(app_name = %app_name, path = %path, "get IPFS file"); - let content = HERMES_IPFS_STATE.file_get(path.to_string())?; + let content = HERMES_IPFS_STATE.file_get(path)?; tracing::debug!(app_name = %app_name, path = %path, "got IPFS file"); Ok(content) } @@ -361,7 +357,7 @@ pub(crate) fn hermes_ipfs_pin_file( app_name: &HermesAppName, path: IpfsPath, ) -> Result { tracing::debug!(app_name = %app_name, path = %path, "pin IPFS file"); - let status = HERMES_IPFS_STATE.file_pin(path.clone())?; + let status = HERMES_IPFS_STATE.file_pin(&path)?; tracing::debug!(app_name = %app_name, path = %path, "pinned IPFS file"); HERMES_IPFS_STATE.apps.pinned_file(app_name.clone(), path); Ok(status) @@ -398,7 +394,7 @@ pub(crate) fn hermes_ipfs_subscribe( if HERMES_IPFS_STATE.apps.topic_subscriptions_contains(&topic) { tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "topic subscription stream already exists"); } else { - let handle = HERMES_IPFS_STATE.pubsub_subscribe(topic.to_string())?; + let handle = HERMES_IPFS_STATE.pubsub_subscribe(&topic)?; HERMES_IPFS_STATE .apps .added_topic_stream(topic.clone(), handle); @@ -423,7 +419,7 @@ pub(crate) fn hermes_ipfs_evict_peer( app_name: &HermesAppName, peer: PeerId, ) -> Result { tracing::debug!(app_name = %app_name, peer_id = %peer, "evicting peer"); - let status = HERMES_IPFS_STATE.peer_evict(peer.to_string())?; + let status = HERMES_IPFS_STATE.peer_evict(&peer.to_string())?; tracing::debug!(app_name = %app_name, peer_id = %peer, "evicted peer"); HERMES_IPFS_STATE.apps.evicted_peer(app_name.clone(), peer); Ok(status) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs index 607321559..bc2a8993a 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs @@ -45,26 +45,28 @@ pub(crate) enum IpfsCommand { EvictPeer(PeerId, oneshot::Sender>), } -#[allow(dead_code)] -/// IPFS +/// IPFS asynchronous task pub(crate) async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyhow::Result<()> { let hermes_node = HermesIpfs::start().await?; if let Some(ipfs_command) = queue_rx.recv().await { match ipfs_command { IpfsCommand::AddFile(ipfs_file, tx) => { - let ipfs_path = hermes_node.add_ipfs_file(ipfs_file).await?; - if let Err(_err) = tx.send(Ok(ipfs_path.to_string())) { - tracing::error!("Failed to send IPFS path"); - } + let response = hermes_node + .add_ipfs_file(ipfs_file) + .await + .map(|ipfs_path| ipfs_path.to_string()) + .map_err(|_| Errno::FileAddError); + send_response(response, tx); }, IpfsCommand::GetFile(ipfs_path, tx) => { - let contents = hermes_node.get_ipfs_file(ipfs_path.into()).await?; - if let Err(_err) = tx.send(Ok(contents)) { - tracing::error!("Failed to get IPFS contents"); - } + let response = hermes_node + .get_ipfs_file(ipfs_path.into()) + .await + .map_err(|_| Errno::FileGetError); + send_response(response, tx); }, IpfsCommand::PinFile(cid, tx) => { - let status = match hermes_node.insert_pin(&cid).await { + let response = match hermes_node.insert_pin(&cid).await { Ok(()) => Ok(true), Err(err) if err.to_string().contains("already pinned recursively") => { tracing::debug!(cid = %cid, "file already pinned"); @@ -75,33 +77,25 @@ pub(crate) async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyh Err(Errno::FilePinError) }, }; - if let Err(err) = tx.send(status) { - tracing::error!("sending response of pin IPFS file should not fail: {err:?}"); - } + send_response(response, tx); }, IpfsCommand::GetDhtValue(key, tx) => { let response = hermes_node .dht_get(key) .await .map_err(|_| Errno::DhtGetError); - if let Err(err) = tx.send(response) { - tracing::error!("sending DHT value should not fail: {err:?}"); - } + send_response(response, tx); }, IpfsCommand::PutDhtValue(key, value, tx) => { - let status = hermes_node.dht_put(key, value).await.is_ok(); - if let Err(err) = tx.send(Ok(status)) { - tracing::error!("sending status of DHT put should not fail: {err:?}"); - } + let response = hermes_node.dht_put(key, value).await.is_ok(); + send_response(Ok(response), tx); }, IpfsCommand::Publish(topic, message, tx) => { let message_id = hermes_node .pubsub_publish(topic, message) .await .map_err(|_| Errno::PubsubPublishError)?; - if let Err(err) = tx.send(Ok(message_id)) { - tracing::error!("sending message id should not fail: {err:?}"); - } + send_response(Ok(message_id), tx); }, IpfsCommand::Subscribe(topic, tx) => { let stream = hermes_node @@ -120,6 +114,7 @@ pub(crate) async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyh }, }; let app_names = HERMES_IPFS_STATE.apps.subscribed_apps(&msg_topic); + // Dispatch Hermes Event if let Err(err) = send(HermesEvent::new( on_topic_event.clone(), crate::event::TargetApp::List(app_names), @@ -129,19 +124,22 @@ pub(crate) async fn ipfs_task(mut queue_rx: mpsc::Receiver) -> anyh } } }); - if let Err(_err) = tx.send(Ok(handle)) { - tracing::error!("Failed to subscribe to topic"); - } + send_response(Ok(handle), tx); }, IpfsCommand::EvictPeer(peer, tx) => { let peer_id = TargetPeerId::from_str(&peer).map_err(|_| Errno::InvalidPeerId)?; let status = hermes_node.ban_peer(peer_id).await.is_ok(); - if let Err(err) = tx.send(Ok(status)) { - tracing::error!("sending status of peer eviction should not fail: {err:?}"); - } + send_response(Ok(status), tx); }, } } hermes_node.stop().await; Ok(()) } + +/// Send the response of the IPFS command +fn send_response(response: T, tx: oneshot::Sender) { + if tx.send(response).is_err() { + tracing::error!("sending IPFS command response should not fail"); + } +} From 05f24b8c0e37877fe5b0e162e9ba256bd24268d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 11:18:36 -0600 Subject: [PATCH 40/50] fix: refactor ipfs api into its own module --- .../runtime_extensions/hermes/ipfs/host.rs | 19 ++- .../hermes/ipfs/state/api.rs | 126 ++++++++++++++++++ .../hermes/ipfs/state/mod.rs | 126 +----------------- 3 files changed, 141 insertions(+), 130 deletions(-) create mode 100644 hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs index 59e510ac0..c4c3bd1b8 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs @@ -1,18 +1,15 @@ //! IPFS host implementation for WASM runtime. -use super::state::hermes_ipfs_publish; +use super::state::{ + hermes_ipfs_add_file, hermes_ipfs_content_validate, hermes_ipfs_evict_peer, + hermes_ipfs_get_dht_value, hermes_ipfs_get_file, hermes_ipfs_pin_file, hermes_ipfs_publish, + hermes_ipfs_put_dht_value, hermes_ipfs_subscribe, +}; use crate::{ runtime_context::HermesRuntimeContext, - runtime_extensions::{ - bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsFile, IpfsPath, MessageData, MessageId, - PeerId, PubsubTopic, - }, - hermes::ipfs::state::{ - hermes_ipfs_add_file, hermes_ipfs_content_validate, hermes_ipfs_evict_peer, - hermes_ipfs_get_dht_value, hermes_ipfs_get_file, hermes_ipfs_pin_file, - hermes_ipfs_put_dht_value, hermes_ipfs_subscribe, - }, + runtime_extensions::bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsFile, IpfsPath, MessageData, MessageId, + PeerId, PubsubTopic, }, }; diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs new file mode 100644 index 000000000..9c657ae86 --- /dev/null +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs @@ -0,0 +1,126 @@ +//! Hermes IPFS State API +use super::{is_valid_dht_content, is_valid_pubsub_content, HERMES_IPFS_STATE}; +use crate::{ + app::HermesAppName, + runtime_extensions::bindings::hermes::ipfs::api::{ + DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, MessageData, MessageId, PeerId, + PubsubTopic, + }, +}; + +/// Add File to IPFS +pub(crate) fn hermes_ipfs_add_file( + app_name: &HermesAppName, contents: IpfsFile, +) -> Result { + tracing::debug!(app_name = %app_name, "adding IPFS file"); + let ipfs_path = HERMES_IPFS_STATE.file_add(contents)?; + tracing::debug!(app_name = %app_name, path = %ipfs_path, "added IPFS file"); + HERMES_IPFS_STATE + .apps + .added_file(app_name.clone(), ipfs_path.clone()); + Ok(ipfs_path) +} + +/// Validate IPFS Content from DHT or `PubSub` +pub(crate) fn hermes_ipfs_content_validate( + app_name: &HermesAppName, content: &IpfsContent, +) -> bool { + match content { + IpfsContent::Dht((k, v)) => { + // TODO(@saibatizoku): Implement types and validation + let key_str = format!("{k:x?}"); + let is_valid = is_valid_dht_content(k, v); + tracing::debug!(app_name = %app_name, dht_key = %key_str, is_valid = %is_valid, "DHT value validation"); + is_valid + }, + IpfsContent::Pubsub((topic, message)) => { + // TODO(@saibatizoku): Implement types and validation + let is_valid = is_valid_pubsub_content(topic, message); + tracing::debug!(app_name = %app_name, topic = %topic, is_valid = %is_valid, "PubSub message validation"); + is_valid + }, + } +} + +/// Get File from Ipfs +pub(crate) fn hermes_ipfs_get_file( + app_name: &HermesAppName, path: &IpfsPath, +) -> Result { + tracing::debug!(app_name = %app_name, path = %path, "get IPFS file"); + let content = HERMES_IPFS_STATE.file_get(path)?; + tracing::debug!(app_name = %app_name, path = %path, "got IPFS file"); + Ok(content) +} + +/// Pin IPFS File +pub(crate) fn hermes_ipfs_pin_file( + app_name: &HermesAppName, path: IpfsPath, +) -> Result { + tracing::debug!(app_name = %app_name, path = %path, "pin IPFS file"); + let status = HERMES_IPFS_STATE.file_pin(&path)?; + tracing::debug!(app_name = %app_name, path = %path, "pinned IPFS file"); + HERMES_IPFS_STATE.apps.pinned_file(app_name.clone(), path); + Ok(status) +} + +/// Get DHT Value +pub(crate) fn hermes_ipfs_get_dht_value( + app_name: &HermesAppName, key: DhtKey, +) -> Result { + let key_str = format!("{key:x?}"); + tracing::debug!(app_name = %app_name, dht_key = %key_str, "get DHT value"); + let value = HERMES_IPFS_STATE.dht_get(key)?; + tracing::debug!(app_name = %app_name, dht_key = %key_str, "got DHT value"); + Ok(value) +} + +/// Put DHT Value +pub(crate) fn hermes_ipfs_put_dht_value( + app_name: &HermesAppName, key: DhtKey, value: DhtValue, +) -> Result { + let key_str = format!("{key:x?}"); + tracing::debug!(app_name = %app_name, dht_key = %key_str, "putting DHT value"); + let status = HERMES_IPFS_STATE.dht_put(key.clone(), value)?; + tracing::debug!(app_name = %app_name, dht_key = %key_str, "have put DHT value"); + HERMES_IPFS_STATE.apps.added_dht_key(app_name.clone(), key); + Ok(status) +} + +/// Subscribe to a topic +pub(crate) fn hermes_ipfs_subscribe( + app_name: &HermesAppName, topic: PubsubTopic, +) -> Result { + tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "subscribing to PubSub topic"); + if HERMES_IPFS_STATE.apps.topic_subscriptions_contains(&topic) { + tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "topic subscription stream already exists"); + } else { + let handle = HERMES_IPFS_STATE.pubsub_subscribe(&topic)?; + HERMES_IPFS_STATE + .apps + .added_topic_stream(topic.clone(), handle); + tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "added subscription topic stream"); + } + HERMES_IPFS_STATE + .apps + .added_app_topic_subscription(app_name.clone(), topic); + Ok(true) +} + +/// Publish message to a topic +pub(crate) fn hermes_ipfs_publish( + _app_name: &HermesAppName, topic: &PubsubTopic, message: MessageData, +) -> Result { + let message_id = HERMES_IPFS_STATE.pubsub_publish(topic.to_string(), message)?; + Ok(message_id.0) +} + +/// Evict Peer from node +pub(crate) fn hermes_ipfs_evict_peer( + app_name: &HermesAppName, peer: PeerId, +) -> Result { + tracing::debug!(app_name = %app_name, peer_id = %peer, "evicting peer"); + let status = HERMES_IPFS_STATE.peer_evict(&peer.to_string())?; + tracing::debug!(app_name = %app_name, peer_id = %peer, "evicted peer"); + HERMES_IPFS_STATE.apps.evicted_peer(app_name.clone(), peer); + Ok(status) +} diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs index af2f440b0..ed8092713 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs @@ -1,8 +1,14 @@ //! Hermes IPFS Internal State +mod api; mod task; use std::{collections::HashSet, str::FromStr}; +pub(crate) use api::{ + hermes_ipfs_add_file, hermes_ipfs_content_validate, hermes_ipfs_evict_peer, + hermes_ipfs_get_dht_value, hermes_ipfs_get_file, hermes_ipfs_pin_file, hermes_ipfs_publish, + hermes_ipfs_put_dht_value, hermes_ipfs_subscribe, +}; use dashmap::{DashMap, DashSet}; use hermes_ipfs::{AddIpfsFile, IpfsPath as PathIpfsFile, PubsubMessageId}; use once_cell::sync::Lazy; @@ -16,8 +22,7 @@ use tokio::{ use crate::{ app::HermesAppName, runtime_extensions::bindings::hermes::ipfs::api::{ - DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, MessageData, MessageId, PeerId, - PubsubTopic, + DhtKey, DhtValue, Errno, IpfsFile, IpfsPath, MessageData, PeerId, PubsubTopic, }, }; @@ -307,120 +312,3 @@ fn is_valid_pubsub_content(_topic: &PubsubTopic, message: &MessageData) -> bool // TODO(@anyone): Implement PubSub content validation !message.is_empty() } - -/// Add File to IPFS -pub(crate) fn hermes_ipfs_add_file( - app_name: &HermesAppName, contents: IpfsFile, -) -> Result { - tracing::debug!(app_name = %app_name, "adding IPFS file"); - let ipfs_path = HERMES_IPFS_STATE.file_add(contents)?; - tracing::debug!(app_name = %app_name, path = %ipfs_path, "added IPFS file"); - HERMES_IPFS_STATE - .apps - .added_file(app_name.clone(), ipfs_path.clone()); - Ok(ipfs_path) -} - -/// Validate IPFS Content from DHT or `PubSub` -pub(crate) fn hermes_ipfs_content_validate( - app_name: &HermesAppName, content: &IpfsContent, -) -> bool { - match content { - IpfsContent::Dht((k, v)) => { - // TODO(@saibatizoku): Implement types and validation - let key_str = format!("{k:x?}"); - let is_valid = is_valid_dht_content(k, v); - tracing::debug!(app_name = %app_name, dht_key = %key_str, is_valid = %is_valid, "DHT value validation"); - is_valid - }, - IpfsContent::Pubsub((topic, message)) => { - // TODO(@saibatizoku): Implement types and validation - let is_valid = is_valid_pubsub_content(topic, message); - tracing::debug!(app_name = %app_name, topic = %topic, is_valid = %is_valid, "PubSub message validation"); - is_valid - }, - } -} - -/// Get File from Ipfs -pub(crate) fn hermes_ipfs_get_file( - app_name: &HermesAppName, path: &IpfsPath, -) -> Result { - tracing::debug!(app_name = %app_name, path = %path, "get IPFS file"); - let content = HERMES_IPFS_STATE.file_get(path)?; - tracing::debug!(app_name = %app_name, path = %path, "got IPFS file"); - Ok(content) -} - -/// Pin IPFS File -pub(crate) fn hermes_ipfs_pin_file( - app_name: &HermesAppName, path: IpfsPath, -) -> Result { - tracing::debug!(app_name = %app_name, path = %path, "pin IPFS file"); - let status = HERMES_IPFS_STATE.file_pin(&path)?; - tracing::debug!(app_name = %app_name, path = %path, "pinned IPFS file"); - HERMES_IPFS_STATE.apps.pinned_file(app_name.clone(), path); - Ok(status) -} - -/// Get DHT Value -pub(crate) fn hermes_ipfs_get_dht_value( - app_name: &HermesAppName, key: DhtKey, -) -> Result { - let key_str = format!("{key:x?}"); - tracing::debug!(app_name = %app_name, dht_key = %key_str, "get DHT value"); - let value = HERMES_IPFS_STATE.dht_get(key)?; - tracing::debug!(app_name = %app_name, dht_key = %key_str, "got DHT value"); - Ok(value) -} - -/// Put DHT Value -pub(crate) fn hermes_ipfs_put_dht_value( - app_name: &HermesAppName, key: DhtKey, value: DhtValue, -) -> Result { - let key_str = format!("{key:x?}"); - tracing::debug!(app_name = %app_name, dht_key = %key_str, "putting DHT value"); - let status = HERMES_IPFS_STATE.dht_put(key.clone(), value)?; - tracing::debug!(app_name = %app_name, dht_key = %key_str, "have put DHT value"); - HERMES_IPFS_STATE.apps.added_dht_key(app_name.clone(), key); - Ok(status) -} - -/// Subscribe to a topic -pub(crate) fn hermes_ipfs_subscribe( - app_name: &HermesAppName, topic: PubsubTopic, -) -> Result { - tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "subscribing to PubSub topic"); - if HERMES_IPFS_STATE.apps.topic_subscriptions_contains(&topic) { - tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "topic subscription stream already exists"); - } else { - let handle = HERMES_IPFS_STATE.pubsub_subscribe(&topic)?; - HERMES_IPFS_STATE - .apps - .added_topic_stream(topic.clone(), handle); - tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "added subscription topic stream"); - } - HERMES_IPFS_STATE - .apps - .added_app_topic_subscription(app_name.clone(), topic); - Ok(true) -} - -/// Publish message to a topic -pub(crate) fn hermes_ipfs_publish( - _app_name: &HermesAppName, topic: &PubsubTopic, message: MessageData, -) -> Result { - let message_id = HERMES_IPFS_STATE.pubsub_publish(topic.to_string(), message)?; - Ok(message_id.0) -} - -/// Evict Peer from node -pub(crate) fn hermes_ipfs_evict_peer( - app_name: &HermesAppName, peer: PeerId, -) -> Result { - tracing::debug!(app_name = %app_name, peer_id = %peer, "evicting peer"); - let status = HERMES_IPFS_STATE.peer_evict(&peer.to_string())?; - tracing::debug!(app_name = %app_name, peer_id = %peer, "evicted peer"); - HERMES_IPFS_STATE.apps.evicted_peer(app_name.clone(), peer); - Ok(status) -} From 6344863378789632e5f50cb9624d0e76b76af881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 11:34:06 -0600 Subject: [PATCH 41/50] fix: simplify type usage --- hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs | 2 +- hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs | 4 ++-- hermes/crates/hermes-ipfs/src/lib.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs index ed8092713..f68e60f73 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs @@ -10,7 +10,7 @@ pub(crate) use api::{ hermes_ipfs_put_dht_value, hermes_ipfs_subscribe, }; use dashmap::{DashMap, DashSet}; -use hermes_ipfs::{AddIpfsFile, IpfsPath as PathIpfsFile, PubsubMessageId}; +use hermes_ipfs::{AddIpfsFile, IpfsPath as PathIpfsFile, MessageId as PubsubMessageId}; use once_cell::sync::Lazy; use task::{ipfs_task, IpfsCommand}; use tokio::{ diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs index bc2a8993a..78e445ded 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/task.rs @@ -2,8 +2,8 @@ use std::str::FromStr; use hermes_ipfs::{ - pin_mut, AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, PeerId as TargetPeerId, - PubsubMessageId, StreamExt, + pin_mut, AddIpfsFile, Cid, HermesIpfs, IpfsPath as PathIpfsFile, MessageId as PubsubMessageId, + PeerId as TargetPeerId, StreamExt, }; use tokio::{ sync::{mpsc, oneshot}, diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index 540c9b773..8e8be3b4d 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -19,7 +19,7 @@ pub use rust_ipfs::DhtMode; /// Server, Client, or Auto mode pub use rust_ipfs::Ipfs; /// `PubSub` Message ID type. -pub use rust_ipfs::MessageId as PubsubMessageId; +pub use rust_ipfs::MessageId; /// Multiaddr type. pub use rust_ipfs::Multiaddr; /// Peer ID type. @@ -387,7 +387,7 @@ impl HermesIpfs { /// Returns error if unable to publish to a pubsub topic. pub async fn pubsub_publish( &self, topic: impl Into, message: Vec, - ) -> anyhow::Result { + ) -> anyhow::Result { self.node.pubsub_publish(topic, message).await } From 4edddeaa16e4d91eb4e6ea4d72b52ba8ac52aaf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 18:38:50 -0600 Subject: [PATCH 42/50] fix: use new type for gossip message id --- hermes/Cargo.toml | 1 + .../src/runtime_extensions/hermes/ipfs/state/api.rs | 4 +++- hermes/crates/hermes-ipfs/Cargo.toml | 2 ++ hermes/crates/hermes-ipfs/src/lib.rs | 13 ++++++++++--- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hermes/Cargo.toml b/hermes/Cargo.toml index 464a36ec3..833531c37 100644 --- a/hermes/Cargo.toml +++ b/hermes/Cargo.toml @@ -105,5 +105,6 @@ ed25519-dalek = "2.1.1" x509-cert = "0.2.5" coset = "0.3.7" libipld = "0.16.0" +libp2p = "0.53.2" rust-ipfs = "0.11.19" rustyline-async = "0.4.2" diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs index 9c657ae86..c336995ef 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs @@ -110,7 +110,9 @@ pub(crate) fn hermes_ipfs_subscribe( pub(crate) fn hermes_ipfs_publish( _app_name: &HermesAppName, topic: &PubsubTopic, message: MessageData, ) -> Result { - let message_id = HERMES_IPFS_STATE.pubsub_publish(topic.to_string(), message)?; + let message_id = HERMES_IPFS_STATE + .pubsub_publish(topic.to_string(), message) + .map(|m| m.0)?; Ok(message_id.0) } diff --git a/hermes/crates/hermes-ipfs/Cargo.toml b/hermes/crates/hermes-ipfs/Cargo.toml index 0595c860d..d7a076c4e 100644 --- a/hermes/crates/hermes-ipfs/Cargo.toml +++ b/hermes/crates/hermes-ipfs/Cargo.toml @@ -12,7 +12,9 @@ workspace = true [dependencies] anyhow.workspace = true +derive_more.workspace = true libipld.workspace = true +libp2p.workspace = true rust-ipfs.workspace = true [dev-dependencies] diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index 8e8be3b4d..b98ce585d 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -4,10 +4,12 @@ use std::str::FromStr; +use derive_more::{Display, From, Into}; /// IPFS Content Identifier. pub use libipld::Cid; /// IPLD pub use libipld::Ipld; +use libp2p::gossipsub::MessageId as PubsubMesssageId; /// libp2p re-export. pub use rust_ipfs::libp2p::futures::{pin_mut, stream::BoxStream, FutureExt, StreamExt}; /// Peer Info type. @@ -18,8 +20,6 @@ pub use rust_ipfs::path::IpfsPath; pub use rust_ipfs::DhtMode; /// Server, Client, or Auto mode pub use rust_ipfs::Ipfs; -/// `PubSub` Message ID type. -pub use rust_ipfs::MessageId; /// Multiaddr type. pub use rust_ipfs::Multiaddr; /// Peer ID type. @@ -30,6 +30,10 @@ pub use rust_ipfs::SubscriptionStream; pub use rust_ipfs::UninitializedIpfsNoop as IpfsBuilder; use rust_ipfs::{dag::ResolveError, unixfs::AddOpt, PubsubEvent, Quorum}; +#[derive(Debug, Display, From, Into)] +/// `PubSub` Message ID. +pub struct MessageId(pub PubsubMesssageId); + /// Hermes IPFS #[allow(dead_code)] pub struct HermesIpfs { @@ -388,7 +392,10 @@ impl HermesIpfs { pub async fn pubsub_publish( &self, topic: impl Into, message: Vec, ) -> anyhow::Result { - self.node.pubsub_publish(topic, message).await + self.node + .pubsub_publish(topic, message) + .await + .map(std::convert::Into::into) } /// Ban peer from node. From 94ed7562c52115accd0a0bbea449ceb54525bc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 18:42:07 -0600 Subject: [PATCH 43/50] fix: spelling --- .config/dictionaries/project.dic | 1 + hermes/crates/hermes-ipfs/src/lib.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index dfb688b44..3ddb7a79c 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -72,6 +72,7 @@ genhtml GETFL getres gmtime +gossipsub happ hardano hasher diff --git a/hermes/crates/hermes-ipfs/src/lib.rs b/hermes/crates/hermes-ipfs/src/lib.rs index b98ce585d..15a2f7bb5 100644 --- a/hermes/crates/hermes-ipfs/src/lib.rs +++ b/hermes/crates/hermes-ipfs/src/lib.rs @@ -9,7 +9,7 @@ use derive_more::{Display, From, Into}; pub use libipld::Cid; /// IPLD pub use libipld::Ipld; -use libp2p::gossipsub::MessageId as PubsubMesssageId; +use libp2p::gossipsub::MessageId as PubsubMessageId; /// libp2p re-export. pub use rust_ipfs::libp2p::futures::{pin_mut, stream::BoxStream, FutureExt, StreamExt}; /// Peer Info type. @@ -32,7 +32,7 @@ use rust_ipfs::{dag::ResolveError, unixfs::AddOpt, PubsubEvent, Quorum}; #[derive(Debug, Display, From, Into)] /// `PubSub` Message ID. -pub struct MessageId(pub PubsubMesssageId); +pub struct MessageId(pub PubsubMessageId); /// Hermes IPFS #[allow(dead_code)] From 8be50f1f5a9e875726f0ca60eacb503c6518a96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 18:58:14 -0600 Subject: [PATCH 44/50] fix: use HashSet instead of DashSet --- .../src/runtime_extensions/hermes/ipfs/state/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs index f68e60f73..2279f60d9 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs @@ -9,7 +9,7 @@ pub(crate) use api::{ hermes_ipfs_get_dht_value, hermes_ipfs_get_file, hermes_ipfs_pin_file, hermes_ipfs_publish, hermes_ipfs_put_dht_value, hermes_ipfs_subscribe, }; -use dashmap::{DashMap, DashSet}; +use dashmap::DashMap; use hermes_ipfs::{AddIpfsFile, IpfsPath as PathIpfsFile, MessageId as PubsubMessageId}; use once_cell::sync::Lazy; use task::{ipfs_task, IpfsCommand}; @@ -211,17 +211,17 @@ struct AppIpfsState { /// Send events to the IPFS node. sender: Option>, /// List of uploaded files per app. - published_files: DashMap>, + published_files: DashMap>, /// List of pinned files per app. - pinned_files: DashMap>, + pinned_files: DashMap>, /// List of DHT values per app. - dht_keys: DashMap>, + dht_keys: DashMap>, /// List of subscriptions per app. topic_subscriptions: DashMap>, /// Collection of stream join handles per topic subscription. subscriptions_streams: DashMap>, /// List of evicted peers per app. - evicted_peers: DashMap>, + evicted_peers: DashMap>, } impl AppIpfsState { From c6c9f92e6d7df90fd9f1c99c945297df29c0dc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Wed, 10 Jul 2024 20:35:09 -0600 Subject: [PATCH 45/50] fix: broken integration tests --- wasm/integration-test/cardano/src/lib.rs | 13 +++++++++++-- wasm/integration-test/ipfs/src/lib.rs | 12 ++++++++++-- wasm/integration-test/sqlite/src/lib.rs | 13 +++++++++++-- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/wasm/integration-test/cardano/src/lib.rs b/wasm/integration-test/cardano/src/lib.rs index 08eb9e694..6656498a9 100644 --- a/wasm/integration-test/cardano/src/lib.rs +++ b/wasm/integration-test/cardano/src/lib.rs @@ -3,7 +3,10 @@ mod hermes; use hermes::{ - exports::hermes::integration_test::event::TestResult, + exports::hermes::{ + integration_test::event::TestResult, + http_gateway::event::{Bstr, Headers, HttpResponse} + }, hermes::{ cardano::{ self, @@ -91,7 +94,7 @@ impl hermes::exports::hermes::init::event::Guest for TestComponent { } impl hermes::exports::hermes::ipfs::event::Guest for TestComponent { - fn on_topic(message: PubsubMessage) -> bool { + fn on_topic(_message: PubsubMessage) -> bool { false } } @@ -100,6 +103,12 @@ impl hermes::exports::hermes::kv_store::event::Guest for TestComponent { fn kv_update(_key: String, _value: KvValues) {} } +impl hermes::exports::hermes::http_gateway::event::Guest for TestComponent { + fn reply(_body: Bstr, _headers: Headers, _path: String, method: String,) -> Option { + None + } +} + impl hermes::exports::wasi::http::incoming_handler::Guest for TestComponent { fn handle(_request: IncomingRequest, _response_out: ResponseOutparam) {} } diff --git a/wasm/integration-test/ipfs/src/lib.rs b/wasm/integration-test/ipfs/src/lib.rs index e2688bb1d..051dea93f 100644 --- a/wasm/integration-test/ipfs/src/lib.rs +++ b/wasm/integration-test/ipfs/src/lib.rs @@ -4,9 +4,11 @@ // Allow everything since this is generated code. #![allow(clippy::all, unused)] mod hermes; - use hermes::{ - exports::hermes::integration_test::event::TestResult, + exports::hermes::{ + integration_test::event::TestResult, + http_gateway::event::{Bstr, Headers, HttpResponse} + }, hermes::{ cardano::api::{BlockSrc, CardanoBlock, CardanoBlockchainId, CardanoTxn}, cron::api::CronTagged, @@ -216,6 +218,12 @@ impl hermes::exports::hermes::kv_store::event::Guest for TestComponent { fn kv_update(_key: String, _value: KvValues) {} } +impl hermes::exports::hermes::http_gateway::event::Guest for TestComponent { + fn reply(_body: Bstr, _headers: Headers, _path: String, method: String,) -> Option { + None + } +} + impl hermes::exports::wasi::http::incoming_handler::Guest for TestComponent { fn handle(_request: IncomingRequest, _response_out: ResponseOutparam) {} } diff --git a/wasm/integration-test/sqlite/src/lib.rs b/wasm/integration-test/sqlite/src/lib.rs index 9ad8f4bcb..6f4ff3248 100644 --- a/wasm/integration-test/sqlite/src/lib.rs +++ b/wasm/integration-test/sqlite/src/lib.rs @@ -7,7 +7,10 @@ mod hermes; mod test; use hermes::{ - exports::hermes::integration_test::event::TestResult, + exports::hermes::{ + integration_test::event::TestResult, + http_gateway::event::{Bstr, Headers, HttpResponse} + }, hermes::{ cardano::api::{BlockSrc, CardanoBlock, CardanoBlockchainId, CardanoTxn}, cron::api::CronTagged, @@ -81,7 +84,7 @@ impl hermes::exports::hermes::init::event::Guest for TestComponent { } impl hermes::exports::hermes::ipfs::event::Guest for TestComponent { - fn on_topic(message: PubsubMessage) -> bool { + fn on_topic(_message: PubsubMessage) -> bool { false } } @@ -90,6 +93,12 @@ impl hermes::exports::hermes::kv_store::event::Guest for TestComponent { fn kv_update(_key: String, _value: KvValues) {} } +impl hermes::exports::hermes::http_gateway::event::Guest for TestComponent { + fn reply(_body: Bstr, _headers: Headers, _path: String, method: String,) -> Option { + None + } +} + impl hermes::exports::wasi::http::incoming_handler::Guest for TestComponent { fn handle(_request: IncomingRequest, _response_out: ResponseOutparam) {} } From 89b9bc2077c9cb9f4936037a670fa299f0eec23a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 11 Jul 2024 07:24:27 -0600 Subject: [PATCH 46/50] fix: Update hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs Code cleanup Co-authored-by: Apisit Ritreungroj <38898766+apskhem@users.noreply.github.com> --- hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs index c336995ef..9ea4361f8 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs @@ -110,10 +110,9 @@ pub(crate) fn hermes_ipfs_subscribe( pub(crate) fn hermes_ipfs_publish( _app_name: &HermesAppName, topic: &PubsubTopic, message: MessageData, ) -> Result { - let message_id = HERMES_IPFS_STATE + HERMES_IPFS_STATE .pubsub_publish(topic.to_string(), message) - .map(|m| m.0)?; - Ok(message_id.0) + .map(|m| m.0 .0) } /// Evict Peer from node From 26da0d8451ad309a5d60cb12ec70c6b82e5bb687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 11 Jul 2024 07:39:00 -0600 Subject: [PATCH 47/50] chore: update TODO comments with issue url --- hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs | 2 -- hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs index 9ea4361f8..b29d02be3 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs @@ -27,14 +27,12 @@ pub(crate) fn hermes_ipfs_content_validate( ) -> bool { match content { IpfsContent::Dht((k, v)) => { - // TODO(@saibatizoku): Implement types and validation let key_str = format!("{k:x?}"); let is_valid = is_valid_dht_content(k, v); tracing::debug!(app_name = %app_name, dht_key = %key_str, is_valid = %is_valid, "DHT value validation"); is_valid }, IpfsContent::Pubsub((topic, message)) => { - // TODO(@saibatizoku): Implement types and validation let is_valid = is_valid_pubsub_content(topic, message); tracing::debug!(app_name = %app_name, topic = %topic, is_valid = %is_valid, "PubSub message validation"); is_valid diff --git a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs index 2279f60d9..1285d3e34 100644 --- a/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs +++ b/hermes/bin/src/runtime_extensions/hermes/ipfs/state/mod.rs @@ -303,12 +303,12 @@ impl AppIpfsState { /// Checks for `DhtKey`, and `DhtValue` validity. fn is_valid_dht_content(_key: &DhtKey, value: &DhtValue) -> bool { - // TODO(@anyone): Implement DHT content validation + // TODO(anyone): https://github.com/input-output-hk/hermes/issues/288 !value.is_empty() } /// Checks for `PubsubTopic`, and `MessageData` validity. fn is_valid_pubsub_content(_topic: &PubsubTopic, message: &MessageData) -> bool { - // TODO(@anyone): Implement PubSub content validation + // TODO(anyone): https://github.com/input-output-hk/hermes/issues/288 !message.is_empty() } From 7ddf77800413fc6e32f35be2f04fb972d1949970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 11 Jul 2024 07:52:58 -0600 Subject: [PATCH 48/50] fix: Update wasm/integration-test/ipfs/src/lib.rs Cleaner code Co-authored-by: Apisit Ritreungroj <38898766+apskhem@users.noreply.github.com> --- wasm/integration-test/ipfs/src/lib.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/wasm/integration-test/ipfs/src/lib.rs b/wasm/integration-test/ipfs/src/lib.rs index 051dea93f..61db68eba 100644 --- a/wasm/integration-test/ipfs/src/lib.rs +++ b/wasm/integration-test/ipfs/src/lib.rs @@ -24,16 +24,10 @@ struct TestComponent; fn test_file_add_and_get_and_pin(run: bool) -> Option { let status = if run { if let Ok(ipfs_path) = ipfs_api::file_add(&IPFS_DEMO_FILE.to_vec()) { - let contents_match = if let Ok(ipfs_file) = ipfs_api::file_get(&ipfs_path) { - ipfs_file == IPFS_DEMO_FILE - } else { - false - }; - let expected_status_is_true = if let Ok(status) = ipfs_api::file_pin(&ipfs_path) { - status - } else { - false - }; + let contents_match = ipfs_api::file_get(&ipfs_path) + .map_or(false, |ipfs_file| ipfs_file == IPFS_DEMO_FILE); + let expected_status_is_true = ipfs_api::file_pin(&ipfs_path) + .map_or(false, |status| status); contents_match && expected_status_is_true } else { false From 683706105d218fc082df3aeec1d32badcea9b4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Thu, 11 Jul 2024 17:02:04 -0600 Subject: [PATCH 49/50] chore: minor fix to trigger ci --- wasm/integration-test/ipfs/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/integration-test/ipfs/src/lib.rs b/wasm/integration-test/ipfs/src/lib.rs index 61db68eba..9c6a84231 100644 --- a/wasm/integration-test/ipfs/src/lib.rs +++ b/wasm/integration-test/ipfs/src/lib.rs @@ -18,7 +18,7 @@ use hermes::{ wasi::http::types::{IncomingRequest, ResponseOutparam}, }; -const IPFS_DEMO_FILE: &[u8] = b"ipfs file uploaded from wasm"; +const IPFS_DEMO_FILE: &[u8] = b"ipfs file uploaded from wasm!"; struct TestComponent; fn test_file_add_and_get_and_pin(run: bool) -> Option { From c9b846a8d9dc7e17fbefbe5107233a7ca1977ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Fri, 12 Jul 2024 11:15:25 -0600 Subject: [PATCH 50/50] fix: earthly integration tests in Rust use --keep-ts flag --- wasm/integration-test/cardano/Earthfile | 2 +- wasm/integration-test/ipfs/Earthfile | 4 +++- wasm/integration-test/ipfs/src/lib.rs | 2 +- wasm/integration-test/sqlite/Earthfile | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/wasm/integration-test/cardano/Earthfile b/wasm/integration-test/cardano/Earthfile index 8f094624e..d7ae66ad4 100644 --- a/wasm/integration-test/cardano/Earthfile +++ b/wasm/integration-test/cardano/Earthfile @@ -11,7 +11,7 @@ IMPORT ../../wasi-hermes-component-adapter AS wasi-hermes-component-adapter build: DO rust-ci+SETUP - COPY --dir src . + COPY --keep-ts --dir src . COPY Cargo.toml . COPY wasi+build-rust-bindings/hermes.rs src/hermes.rs diff --git a/wasm/integration-test/ipfs/Earthfile b/wasm/integration-test/ipfs/Earthfile index 96a89323c..61f3359c5 100644 --- a/wasm/integration-test/ipfs/Earthfile +++ b/wasm/integration-test/ipfs/Earthfile @@ -17,7 +17,7 @@ gen-bindings: build: DO rust-ci+SETUP - COPY --dir src . + COPY --keep-ts --dir src . COPY Cargo.toml . COPY wasi+build-rust-bindings/hermes.rs src/hermes.rs @@ -25,8 +25,10 @@ build: --args "build --target wasm32-unknown-unknown --release" \ --output="wasm32-unknown-unknown/release/ipfs_test_component.wasm" + COPY wasi-hermes-component-adapter+build/wasi-hermes-component-adapter.wasm . RUN wasm-tools component new -o ipfs.wasm target/wasm32-unknown-unknown/release/ipfs_test_component.wasm --adapt wasi-hermes-component-adapter.wasm + RUN sha256sum ipfs.wasm SAVE ARTIFACT ipfs.wasm diff --git a/wasm/integration-test/ipfs/src/lib.rs b/wasm/integration-test/ipfs/src/lib.rs index 9c6a84231..61db68eba 100644 --- a/wasm/integration-test/ipfs/src/lib.rs +++ b/wasm/integration-test/ipfs/src/lib.rs @@ -18,7 +18,7 @@ use hermes::{ wasi::http::types::{IncomingRequest, ResponseOutparam}, }; -const IPFS_DEMO_FILE: &[u8] = b"ipfs file uploaded from wasm!"; +const IPFS_DEMO_FILE: &[u8] = b"ipfs file uploaded from wasm"; struct TestComponent; fn test_file_add_and_get_and_pin(run: bool) -> Option { diff --git a/wasm/integration-test/sqlite/Earthfile b/wasm/integration-test/sqlite/Earthfile index 86e52b07c..4906b9bb6 100644 --- a/wasm/integration-test/sqlite/Earthfile +++ b/wasm/integration-test/sqlite/Earthfile @@ -17,7 +17,7 @@ gen-bindings: build: DO rust-ci+SETUP - COPY --dir src . + COPY --keep-ts --dir src . COPY Cargo.toml . COPY wasi+build-rust-bindings/hermes.rs src/hermes.rs