feat: introduce the dev mode in which alow all effectors (#2127)

* introduce the dev mode in which allow all effectors
* chore(ci): disable registry check

---------

Co-authored-by: Alexey Proshutinskiy <alexey.prosh@fluence.one>

.github/workflows/e2e.yml

@@ -84,7 +84,6 @@ jobs:
       - nox-snapshot
     uses: fluencelabs/cli/.github/workflows/tests.yml@main
     with:
-      ref: up-spell-and-fixes
       nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}"
 
   js-client:
@@ -102,11 +101,11 @@ jobs:
     with:
       nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}"
 
-  registry:
-    needs:
-      - nox-snapshot
-    uses: fluencelabs/registry/.github/workflows/tests.yml@main
-    with:
-      nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}"
-      if-no-artifacts-found: warn
+  # registry:
+  #   needs:
+  #     - nox-snapshot
+  #   uses: fluencelabs/registry/.github/workflows/tests.yml@main
+  #   with:
+  #     nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}"
+  #     if-no-artifacts-found: warn
 

crates/created-swarm/src/swarm.rs

@@ -404,8 +404,6 @@ pub async fn create_swarm_with_runtime<RT: AquaRuntime>(
         resolved.node_config.aquavm_pool_size = config.pool_size.unwrap_or(1);
         resolved.node_config.particle_execution_timeout = EXECUTION_TIMEOUT;
 
-        resolved.node_config.allowed_binaries = config.allowed_binaries.clone();
-
         let allowed_effectors = config.allowed_effectors.iter().map(|(cid, binaries)| {
             (Hash::from_string(cid).unwrap(), binaries.clone())
         }).collect::<_>();

crates/nox-tests/tests/modules.rs

@@ -116,7 +116,7 @@ async fn test_add_module_effectors_forbidden() {
 
     let response = client.execute_particle(script, data).await.unwrap();
     if let Some(result) = response[0].as_str() {
-        let expected = "Local service error, ret_code is 1, error message is '\"Error: Config error: requested module effector tetraplets with CID bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe is forbidden on this host\\nForbiddenEffector { module_name: \\\"tetraplets\\\", forbidden_cid: Hash(Cid(bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe)) }\"'";
+        let expected = "Local service error, ret_code is 1, error message is '\"Error: Config error: requested module effector tetraplets with CID bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe is forbidden on this host\\nForbiddenEffector { module_name: \\\"tetraplets\\\", forbidden_cid: \\\"bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe\\\" }\"'";
         assert_eq!(expected, result);
     } else {
         panic!("can't receive response from node");

crates/server-config/src/args.rs

@@ -62,11 +62,11 @@ pub struct RootKeyPairArgs {
     #[arg(
         short('g'),
         long("gen-keypair"),
-        value_parser = clap::value_parser!(bool),
+        value_parser = clap::value_parser ! (bool),
         id = "ROOT_KEY_PAIR_GENERATE",
         help_heading = "Node keypair",
         display_order = 13,
-        action =  clap::ArgAction::SetTrue
+        action = clap::ArgAction::SetTrue
     )]
     generate_on_absence: Option<bool>,
     #[arg(
@@ -165,6 +165,7 @@ pub enum EnabledSystemServices {
 // - "service1,service2" to EnabledSystemServices::Some(vec!["service1", "service2"])
 #[derive(Debug, Clone)]
 struct EnabledSystemServicesValueParser;
+
 impl clap::builder::TypedValueParser for EnabledSystemServicesValueParser {
     type Value = EnabledSystemServices;
     fn parse_ref(
@@ -192,11 +193,11 @@ impl clap::builder::TypedValueParser for EnabledSystemServicesValueParser {
 pub(crate) struct SystemServicesArgs {
     // TODO: how to provide the list of available system services automatically
     #[arg(
-    long,
-    id = "SERVICES",
-    help = "List of enabled system services. Can be: all, none or comma-separated list of services (serivce1,service2)",
-    help_heading = "System services configuration",
-    value_parser = EnabledSystemServicesValueParser
+        long,
+        id = "SERVICES",
+        help = "List of enabled system services. Can be: all, none or comma-separated list of services (serivce1,service2)",
+        help_heading = "System services configuration",
+        value_parser = EnabledSystemServicesValueParser
     )]
     enable_system_services: Option<EnabledSystemServices>,
 
@@ -305,7 +306,7 @@ pub(crate) struct DerivedArgs {
         value_name = "MULTIADDR",
         help_heading = "Networking",
         display_order = 5,
-        action =  clap::ArgAction::Append,
+        action = clap::ArgAction::Append,
         num_args = 1..
     )]
     external_multiaddresses: Option<Vec<String>>,
@@ -316,7 +317,7 @@ pub(crate) struct DerivedArgs {
         help = "allow private IP addresses from other nodes",
         help_heading = "Networking",
         display_order = 6,
-        action =  clap::ArgAction::SetTrue
+        action = clap::ArgAction::SetTrue
     )]
     allow_local_addresses: Option<bool>,
     #[arg(
@@ -328,7 +329,7 @@ pub(crate) struct DerivedArgs {
         help_heading = "Networking",
         display_order = 7,
         conflicts_with = "LOCAL",
-        action =  clap::ArgAction::Append,
+        action = clap::ArgAction::Append,
         num_args = 1..
     )]
     bootstrap_nodes: Option<Vec<String>>,
@@ -346,11 +347,11 @@ pub(crate) struct DerivedArgs {
         short('l'),
         long,
         id = "LOCAL",
-        value_parser = clap::value_parser!(bool),
+        value_parser = clap::value_parser ! (bool),
         help = "if passed, bootstrap nodes aren't used",
         help_heading = "Networking",
         display_order = 9,
-        action =  clap::ArgAction::SetTrue
+        action = clap::ArgAction::SetTrue
     )]
     local: Option<bool>,
 
@@ -364,14 +365,13 @@ pub(crate) struct DerivedArgs {
         help_heading = "Node configuration",
         help = "TOML configuration file",
         long_help = "TOML configuration file. If not specified, the default configuration is used. \
-        If specified, the default configuration is merged with the specified one. \
-        The argument can by used multiple times. \
-        The last configuration overrides the previous ones.",
+            If specified, the default configuration is merged with the specified one. \
+            The argument can by used multiple times. \
+            The last configuration overrides the previous ones.",
         value_name = "PATH",
         num_args(1..),
         value_delimiter(','),
         display_order = 15,
-
     )]
     pub(crate) configs: Option<Vec<PathBuf>>,
     #[arg(
@@ -426,7 +426,7 @@ pub(crate) struct DerivedArgs {
     aquavm_pool_size: Option<usize>,
     #[arg(
         long,
-        value_parser = clap::value_parser!(bool),
+        value_parser = clap::value_parser ! (bool),
         id = "PRINT_CONFIG",
         help = "Print applied config",
         help_heading = "Node configuration",
@@ -436,7 +436,7 @@ pub(crate) struct DerivedArgs {
     pub(crate) print_config: Option<bool>,
     #[arg(
         long,
-        value_parser = clap::value_parser!(bool),
+        value_parser = clap::value_parser ! (bool),
         id = "NO_BANNER",
         help = "Disable banner",
         help_heading = "Node configuration",
@@ -453,6 +453,9 @@ pub(crate) struct DerivedArgs {
 
     #[command(flatten)]
     tracing: Option<TracingArgs>,
+
+    #[command(flatten)]
+    dev_mode: Option<DevModeArgs>,
 }
 
 impl Source for DerivedArgs {
@@ -468,3 +471,26 @@ impl Source for DerivedArgs {
         Ok(result)
     }
 }
+
+#[derive(Parser, Debug, Clone)]
+pub struct DevModeArgs {
+    #[arg(
+        long("dev-mode"),
+        value_parser = clap::value_parser!(bool),
+        id = "ENABLE_DEV_MODE",
+        help = "The private wallet key for signing transactions for joining deals",
+        help_heading = "System services configuration"
+    )]
+    enable: bool,
+}
+
+impl Serialize for DevModeArgs {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut struct_serializer = serializer.serialize_struct("DevModeConfig", 5)?;
+        struct_serializer.serialize_field("enable", &self.enable)?;
+        struct_serializer.end()
+    }
+}

crates/server-config/src/defaults.rs

@@ -264,7 +264,14 @@ pub fn default_decider_network_id() -> u64 {
 pub fn default_effectors() -> HashMap<String, (String, HashMap<String, String>)> {
     hashmap! {
         "curl".to_string() => ("bafkreids22lgia5bqs63uigw4mqwhsoxvtnkpfqxqy5uwyyerrldsr32ce".to_string(), hashmap! {
-            "curl".to_string() => "/usr/bin/curl".to_string()
+            "curl".to_string() => default_curl_binary_path(),
         })
     }
 }
+
+pub fn default_binaries_mapping() -> HashMap<String, String> {
+    hashmap! {
+        "curl".to_string() => default_curl_binary_path(),
+        "ipfs".to_string() => default_ipfs_binary_path(),
+    }
+}

crates/server-config/src/node_config.rs

@@ -144,6 +144,9 @@ pub struct UnresolvedNodeConfig {
     pub chain_config: Option<ChainConfig>,
 
     pub chain_listener_config: Option<ChainListenerConfig>,
+
+    #[serde(default = "default_dev_mode_config")]
+    pub dev_mode: DevModeConfig,
 }
 
 impl UnresolvedNodeConfig {
@@ -165,10 +168,6 @@ impl UnresolvedNodeConfig {
             .unwrap_or_default()
             .get_keypair(default_builtins_keypair_path(persistent_base_dir))?;
 
-        let mut allowed_binaries = self.allowed_binaries;
-        allowed_binaries.push(self.system_services.aqua_ipfs.ipfs_binary_path.clone());
-        allowed_binaries.push(self.system_services.connector.curl_binary_path.clone());
-
         let allowed_effectors = self
             .effectors
             .0
@@ -210,8 +209,8 @@ impl UnresolvedNodeConfig {
             management_peer_id: self.management_peer_id,
             transport_config: self.transport_config,
             listen_config: self.listen_config,
-            allowed_binaries,
             allowed_effectors,
+            dev_mode_config: self.dev_mode,
             system_services: self.system_services,
             http_config: self.http_config,
             chain_config: self.chain_config,
@@ -380,10 +379,10 @@ pub struct NodeConfig {
 
     pub management_peer_id: PeerId,
 
-    pub allowed_binaries: Vec<String>,
-
     pub allowed_effectors: HashMap<Hash, HashMap<String, String>>,
 
+    pub dev_mode_config: DevModeConfig,
+
     pub system_services: SystemServicesConfig,
 
     pub http_config: Option<HttpConfig>,
@@ -603,3 +602,20 @@ fn default_effectors_config() -> EffectorsConfig {
         .collect::<_>();
     EffectorsConfig(config)
 }
+
+#[derive(Clone, Deserialize, Serialize, Derivative)]
+#[derivative(Debug)]
+pub struct DevModeConfig {
+    #[serde(default)]
+    pub enable: bool,
+    /// Mounted binaries mapping: binary name (used in the effector modules) to binary path
+    #[serde(default = "default_binaries_mapping")]
+    pub binaries: HashMap<String, String>,
+}
+
+fn default_dev_mode_config() -> DevModeConfig {
+    DevModeConfig {
+        enable: false,
+        binaries: default_binaries_mapping(),
+    }
+}

crates/server-config/src/services_config.rs

@@ -50,6 +50,10 @@ pub struct ServicesConfig {
     pub default_service_memory_limit: Option<ByteSize>,
     /// List of allowed effector modules by CID
     pub allowed_effectors: HashMap<Hash, HashMap<String, PathBuf>>,
+    /// Mapping of binary names to their paths for mounted binaries used in developer mode
+    pub mounted_binaries_mapping: HashMap<String, PathBuf>,
+    /// Is in the developer mode
+    pub is_dev_mode: bool,
 }
 
 impl ServicesConfig {
@@ -64,6 +68,8 @@ impl ServicesConfig {
         builtins_management_peer_id: PeerId,
         default_service_memory_limit: Option<ByteSize>,
         allowed_effectors: HashMap<Hash, HashMap<String, String>>,
+        mounted_binaries_mapping: HashMap<String, String>,
+        is_dev_mode: bool,
     ) -> Result<Self, std::io::Error> {
         let persistent_dir = to_abs_path(persistent_dir);
         let ephemeral_dir = to_abs_path(ephemeral_dir);
@@ -76,8 +82,12 @@ impl ServicesConfig {
                     .map(|(name, path_str)| {
                         let path = Path::new(&path_str);
                         match path.try_exists() {
-                            Err(err) => log::warn!("cannot check effector `{path_str}`: {err}"),
-                            Ok(false) => log::warn!("effector `{path_str}` does not exist"),
+                            Err(err) => log::warn!(
+                                "cannot check binary `{path_str}` for effector `{cid}`: {err}"
+                            ),
+                            Ok(false) => log::warn!(
+                                "binary `{path_str}` for effector `{cid}` does not exist"
+                            ),
                             _ => {}
                         };
                         (name, path.to_path_buf())
@@ -87,6 +97,23 @@ impl ServicesConfig {
             })
             .collect::<_>();
 
+        let mounted_binaries_mapping = if !is_dev_mode {
+            HashMap::new()
+        } else {
+            mounted_binaries_mapping
+                .into_iter()
+                .map(|(name, path_str)| {
+                    let path = Path::new(&path_str);
+                    match path.try_exists() {
+                        Err(err) => log::warn!("cannot check binary `{path_str}`: {err}"),
+                        Ok(false) => log::warn!("binary `{path_str}` does not exist"),
+                        _ => {}
+                    };
+                    (name, path.to_path_buf())
+                })
+                .collect::<_>()
+        };
+
         let this = Self {
             local_peer_id,
             blueprint_dir: config_utils::blueprint_dir(&persistent_dir),
@@ -100,6 +127,8 @@ impl ServicesConfig {
             builtins_management_peer_id,
             default_service_memory_limit,
             allowed_effectors,
+            mounted_binaries_mapping,
+            is_dev_mode,
         };
 
         create_dirs(&[

crates/spell-service-api/src/lib.rs

@@ -343,6 +343,8 @@ mod tests {
             root_key_pair.get_peer_id(),
             Some(service_memory_limit),
             Default::default(),
+            Default::default(),
+            true,
         )
         .unwrap();
 

nox/src/node.rs

@@ -159,6 +159,8 @@ impl<RT: AquaRuntime> Node<RT> {
             builtins_peer_id,
             config.node_config.default_service_memory_limit,
             config.node_config.allowed_effectors.clone(),
+            config.node_config.dev_mode_config.binaries.clone(),
+            config.node_config.dev_mode_config.enable,
         )
         .expect("create services config");
 

particle-builtins/src/builtins.rs

@@ -39,7 +39,7 @@ use now_millis::{now_ms, now_sec};
 use particle_args::{from_base58, Args, ArgsError, JError};
 use particle_execution::{FunctionOutcome, ParticleParams, ServiceFunction};
 use particle_modules::{
-    AddBlueprint, ModuleConfig, ModuleRepository, NamedModuleConfig, WASIConfig,
+    AddBlueprint, EffectorsMode, ModuleConfig, ModuleRepository, NamedModuleConfig, WASIConfig,
 };
 use particle_protocol::Contact;
 use particle_services::{ParticleAppServices, PeerScope, ServiceInfo, ServiceType};
@@ -108,8 +108,16 @@ where
     ) -> Self {
         let modules_dir = &config.modules_dir;
         let blueprint_dir = &config.blueprint_dir;
-        let modules =
-            ModuleRepository::new(modules_dir, blueprint_dir, config.allowed_effectors.clone());
+        let effectors_mode = if config.is_dev_mode {
+            EffectorsMode::AllEffectors {
+                binaries: config.mounted_binaries_mapping.clone(),
+            }
+        } else {
+            EffectorsMode::RestrictedEffectors {
+                effectors: config.allowed_effectors.clone(),
+            }
+        };
+        let modules = ModuleRepository::new(modules_dir, blueprint_dir, effectors_mode);
         let services = ParticleAppServices::new(
             config,
             modules.clone(),