move fetch metadata to a separate crate subxt_utils_fetchmetadata (…

…#1829)

* macros: feature-gate jsonrpsee/fetch metadata url

* make CI happy

* Update codegen/src/error.rs

* extract `fetch-metdata` to separate crate

* add missing license headers

* introduce subxt-utils crate

* add missing files

* codegen: remove unused hex crate

* fix test build

* move subxt_utils -> subxt_utils_fetchmetadata

* cargo fmt

* runtime-path -> runtime-metadata-path

* Update utils/fetch-metadata/src/lib.rs

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "signer",
     "subxt",
     "scripts/artifacts",
+    "utils/fetch-metadata",
 ]
 
 # We exclude any crates that would depend on non mutually
@@ -140,6 +141,7 @@ polkadot-sdk = { version = "0.7", default-features = false }
 # Subxt workspace crates:
 subxt = { version = "0.37.0", path = "subxt", default-features = false }
 subxt-core = { version = "0.37.0", path = "core", default-features = false }
+subxt-utils-fetchmetadata = { version = "0.37.0", path = "utils/fetch-metadata", default-features = false }
 subxt-macro = { version = "0.37.0", path = "macro" }
 subxt-metadata = { version = "0.37.0", path = "metadata", default-features = false }
 subxt-codegen = { version = "0.37.0", path = "codegen" }

cli/Cargo.toml

@@ -26,9 +26,10 @@ workspace = true
 chain-spec-pruning = ["smoldot"]
 
 [dependencies]
-subxt-codegen = { workspace = true, features = ["fetch-metadata"] }
+subxt-codegen = { workspace = true }
+subxt-utils-fetchmetadata = { workspace = true, features = ["url"] }
 subxt-metadata = { workspace = true }
-subxt = { workspace = true, features = ["native", "jsonrpsee"] }
+subxt = { workspace = true, features = ["default"] }
 clap = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 color-eyre = { workspace = true }

cli/src/commands/chain_spec/fetch.rs

@@ -9,7 +9,7 @@ use jsonrpsee::{
     http_client::HttpClientBuilder,
 };
 use std::time::Duration;
-use subxt_codegen::fetch_metadata::Url;
+use subxt_utils_fetchmetadata::Url;
 
 /// Returns the node's chainSpec from the provided URL.
 pub async fn fetch_chain_spec(url: Url) -> Result<serde_json::Value, FetchSpecError> {

cli/src/commands/chain_spec/mod.rs

@@ -6,7 +6,7 @@ use clap::Parser as ClapParser;
 #[cfg(feature = "chain-spec-pruning")]
 use serde_json::Value;
 use std::{io::Write, path::PathBuf};
-use subxt_codegen::fetch_metadata::Url;
+use subxt_utils_fetchmetadata::Url;
 
 mod fetch;
 

cli/src/commands/compatibility.rs

@@ -8,8 +8,8 @@ use color_eyre::eyre::WrapErr;
 use jsonrpsee::client_transport::ws::Url;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use subxt_codegen::fetch_metadata::MetadataVersion;
 use subxt_metadata::Metadata;
+use subxt_utils_fetchmetadata::MetadataVersion;
 
 use crate::utils::validate_url_security;
 
@@ -137,7 +137,7 @@ async fn fetch_runtime_metadata(
     url: Url,
     version: MetadataVersion,
 ) -> color_eyre::Result<Metadata> {
-    let bytes = subxt_codegen::fetch_metadata::fetch_metadata_from_url(url, version).await?;
+    let bytes = subxt_utils_fetchmetadata::from_url(url, version).await?;
     let metadata = Metadata::decode(&mut &bytes[..])?;
     Ok(metadata)
 }

cli/src/utils.rs

@@ -14,7 +14,7 @@ use std::{fs, io::Read, path::PathBuf};
 use subxt::{OnlineClient, PolkadotConfig};
 
 use scale_value::Value;
-use subxt_codegen::fetch_metadata::{fetch_metadata_from_url, MetadataVersion, Url};
+use subxt_utils_fetchmetadata::{self as fetch_metadata, MetadataVersion, Url};
 
 /// The source of the metadata.
 #[derive(Debug, Args, Clone)]
@@ -117,12 +117,12 @@ impl FileOrUrl {
             }
             // Fetch from --url
             (None, Some(uri), version) => {
-                Ok(fetch_metadata_from_url(uri.clone(), version.unwrap_or_default()).await?)
+                Ok(fetch_metadata::from_url(uri.clone(), version.unwrap_or_default()).await?)
             }
             // Default if neither is provided; fetch from local url
             (None, None, version) => {
                 let url = Url::parse("ws://localhost:9944").expect("Valid URL; qed");
-                Ok(fetch_metadata_from_url(url, version.unwrap_or_default()).await?)
+                Ok(fetch_metadata::from_url(url, version.unwrap_or_default()).await?)
             }
         }
     }

codegen/Cargo.toml

@@ -13,8 +13,7 @@ description = "Generate an API for interacting with a substrate node from FRAME
 
 [features]
 default = []
-fetch-metadata = ["dep:jsonrpsee", "dep:tokio", "dep:frame-metadata"]
-web = ["jsonrpsee?/async-wasm-client", "jsonrpsee?/client-web-transport", "getrandom/js"]
+web = ["getrandom/js"]
 
 [dependencies]
 codec = { package = "parity-scale-codec", workspace = true, features = ["derive"] }
@@ -25,9 +24,6 @@ quote = { workspace = true }
 syn = { workspace = true }
 scale-info = { workspace = true }
 subxt-metadata = { workspace = true }
-jsonrpsee = { workspace = true, features = ["async-client", "client-ws-transport-tls", "http-client"], optional = true }
-hex = { workspace = true, features = ["std"] }
-tokio = { workspace = true, features = ["rt-multi-thread"], optional = true }
 thiserror = { workspace = true }
 scale-typegen = { workspace = true }
 
@@ -36,9 +32,10 @@ getrandom = { workspace = true, optional = true }
 
 [dev-dependencies]
 scale-info = { workspace = true, features = ["bit-vec"] }
+frame-metadata = { workspace = true }
 
 [package.metadata.docs.rs]
-features = ["fetch-metadata"]
+features = ["default"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [package.metadata.playground]

codegen/src/error.rs

@@ -11,9 +11,6 @@ use scale_typegen::TypegenError;
 #[derive(Debug, thiserror::Error)]
 #[non_exhaustive]
 pub enum CodegenError {
-    /// Cannot fetch the metadata bytes.
-    #[error("Failed to fetch metadata, make sure that you're pointing at a node which is providing substrate-based metadata: {0}")]
-    Fetch(#[from] FetchMetadataError),
     /// Cannot decode the metadata bytes.
     #[error("Could not decode metadata, only V14 and V15 metadata are supported: {0}")]
     Decode(#[from] codec::Error),
@@ -60,6 +57,9 @@ pub enum CodegenError {
     /// Error when generating metadata from Wasm-runtime
     #[error("Failed to generate metadata from wasm file. reason: {0}")]
     Wasm(String),
+    /// Other error.
+    #[error("Other error: {0}")]
+    Other(String),
 }
 
 impl CodegenError {
@@ -81,28 +81,3 @@ impl CodegenError {
         syn::Error::new(span, msg).into_compile_error()
     }
 }
-
-/// Error attempting to load metadata.
-#[derive(Debug, thiserror::Error)]
-#[non_exhaustive]
-pub enum FetchMetadataError {
-    /// Error decoding from a hex value.
-    #[error("Cannot decode hex value: {0}")]
-    DecodeError(#[from] hex::FromHexError),
-    /// Some SCALE codec error.
-    #[error("Cannot scale encode/decode value: {0}")]
-    CodecError(#[from] codec::Error),
-    /// JSON-RPC error fetching metadata.
-    #[cfg(feature = "fetch-metadata")]
-    #[error("Request error: {0}")]
-    RequestError(#[from] jsonrpsee::core::ClientError),
-    /// Failed IO when fetching from a file.
-    #[error("Failed IO for {0}, make sure that you are providing the correct file path for metadata: {1}")]
-    Io(String, std::io::Error),
-    /// URL scheme is not http, https, ws or wss.
-    #[error("'{0}' not supported, supported URI schemes are http, https, ws or wss.")]
-    InvalidScheme(String),
-    /// Some other error.
-    #[error("Other error: {0}")]
-    Other(String),
-}

codegen/src/lib.rs

@@ -6,20 +6,13 @@
 //! This is used by the `#[subxt]` macro and `subxt codegen` CLI command, but can also
 //! be used directly if preferable.
 
-#![deny(unused_crate_dependencies, missing_docs)]
+#![deny(missing_docs)]
 #![cfg_attr(docsrs, feature(doc_cfg))]
 
 mod api;
 pub mod error;
 mod ir;
 
-// These should probably be in a separate crate; they are used by the
-// macro and CLI tool, so they only live here because this is a common
-// crate that both depend on.
-#[cfg(feature = "fetch-metadata")]
-#[cfg_attr(docsrs, doc(cfg(feature = "fetch-metadata")))]
-pub mod fetch_metadata;
-
 #[cfg(feature = "web")]
 use getrandom as _;
 

macro/Cargo.toml

@@ -15,7 +15,8 @@ description = "Generate types and helpers for interacting with Substrate runtime
 
 [features]
 web = ["subxt-codegen/web"]
-runtime-path = ["polkadot-sdk"]
+runtime-metadata-path = ["polkadot-sdk"]
+runtime-metadata-insecure-url = ["subxt-utils-fetchmetadata/url"]
 
 [lib]
 proc-macro = true
@@ -26,7 +27,8 @@ darling = { workspace = true }
 proc-macro-error2 = { workspace = true }
 syn = { workspace = true }
 quote = { workspace = true }
-subxt-codegen = { workspace = true, features = ["fetch-metadata"] }
+subxt-codegen = { workspace = true }
+subxt-utils-fetchmetadata = { workspace = true }
 scale-typegen = { workspace = true }
 polkadot-sdk = { workspace = true, optional = true, features = ["sp-io", "sc-executor-common", "sp-state-machine", "sp-maybe-compressed-blob", "sc-executor"] }
 

macro/src/lib.rs

@@ -13,15 +13,10 @@ use scale_typegen::typegen::{
     settings::substitutes::path_segments,
     validation::{registry_contains_type_path, similar_type_paths_in_registry},
 };
-use subxt_codegen::{
-    fetch_metadata::{
-        fetch_metadata_from_file_blocking, fetch_metadata_from_url_blocking, MetadataVersion, Url,
-    },
-    CodegenBuilder, CodegenError, Metadata,
-};
+use subxt_codegen::{CodegenBuilder, CodegenError, Metadata};
 use syn::{parse_macro_input, punctuated::Punctuated};
 
-#[cfg(feature = "runtime-path")]
+#[cfg(feature = "runtime-metadata-path")]
 mod wasm_loader;
 
 #[derive(Clone, Debug)]
@@ -61,7 +56,7 @@ struct RuntimeMetadataArgs {
     no_default_substitutions: bool,
     #[darling(default)]
     unstable_metadata: darling::util::Flag,
-    #[cfg(feature = "runtime-path")]
+    #[cfg(feature = "runtime-metadata-path")]
     #[darling(default)]
     runtime_path: Option<String>,
 }
@@ -211,7 +206,7 @@ fn fetch_metadata(args: &RuntimeMetadataArgs) -> Result<subxt_codegen::Metadata,
     // Do we want to fetch unstable metadata? This only works if fetching from a URL.
     let unstable_metadata = args.unstable_metadata.is_present();
 
-    #[cfg(feature = "runtime-path")]
+    #[cfg(feature = "runtime-metadata-path")]
     if let Some(path) = &args.runtime_path {
         if args.runtime_metadata_insecure_url.is_some() || args.runtime_metadata_path.is_some() {
             abort_call_site!(
@@ -240,11 +235,14 @@ fn fetch_metadata(args: &RuntimeMetadataArgs) -> Result<subxt_codegen::Metadata,
             let root = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
             let root_path = std::path::Path::new(&root);
             let path = root_path.join(rest_of_path);
-            fetch_metadata_from_file_blocking(&path)
+            subxt_utils_fetchmetadata::from_file_blocking(&path)
                 .and_then(|b| subxt_codegen::Metadata::decode(&mut &*b).map_err(Into::into))
-                .map_err(|e| CodegenError::from(e).into_compile_error())?
+                .map_err(|e| CodegenError::Other(e.to_string()).into_compile_error())?
         }
+        #[cfg(feature = "runtime-metadata-insecure-url")]
         (None, Some(url_string)) => {
+            use subxt_utils_fetchmetadata::{from_url_blocking, MetadataVersion, Url};
+
             let url = Url::parse(url_string).unwrap_or_else(|_| {
                 abort_call_site!("Cannot download metadata; invalid url: {}", url_string)
             });
@@ -254,30 +252,36 @@ fn fetch_metadata(args: &RuntimeMetadataArgs) -> Result<subxt_codegen::Metadata,
                 false => MetadataVersion::Latest,
             };
 
-            fetch_metadata_from_url_blocking(url, version)
-                .map_err(CodegenError::from)
+            from_url_blocking(url, version)
+                .map_err(|e| CodegenError::Other(e.to_string()))
                 .and_then(|b| subxt_codegen::Metadata::decode(&mut &*b).map_err(Into::into))
                 .map_err(|e| e.into_compile_error())?
         }
-        #[cfg(feature = "runtime-path")]
+        #[cfg(not(feature = "runtime-metadata-insecure-url"))]
+        (None, Some(_)) => {
+            abort_call_site!(
+                "'runtime_metadata_insecure_url' requires the 'runtime-metadata-insecure-url' feature to be enabled"
+            )
+        }
+        #[cfg(feature = "runtime-metadata-path")]
         (None, None) => {
             abort_call_site!(
                     "At least one of 'runtime_metadata_path', 'runtime_metadata_insecure_url'  or  'runtime_path` can be provided"
                 )
         }
-        #[cfg(not(feature = "runtime-path"))]
+        #[cfg(not(feature = "runtime-metadata-path"))]
         (None, None) => {
             abort_call_site!(
                 "At least one of 'runtime_metadata_path', 'runtime_metadata_insecure_url' can be provided"
             )
         }
-        #[cfg(feature = "runtime-path")]
+        #[cfg(feature = "runtime-metadata-path")]
         _ => {
             abort_call_site!(
                     "Only one of 'runtime_metadata_path', 'runtime_metadata_insecure_url'  or  'runtime_path` can be provided"
                 )
         }
-        #[cfg(not(feature = "runtime-path"))]
+        #[cfg(not(feature = "runtime-metadata-path"))]
         _ => {
             abort_call_site!(
                 "Only one of 'runtime_metadata_path' or 'runtime_metadata_insecure_url' can be provided"

macro/src/wasm_loader.rs

@@ -12,15 +12,15 @@ use polkadot_sdk::{
     sp_maybe_compressed_blob::{self, CODE_BLOB_BOMB_LIMIT},
     sp_state_machine,
 };
-use subxt_codegen::{fetch_metadata::fetch_metadata_from_file_blocking, CodegenError, Metadata};
+use subxt_codegen::{CodegenError, Metadata};
 
 /// Result type shorthand
 pub type WasmMetadataResult<A> = Result<A, CodegenError>;
 
 /// Uses wasm artifact produced by compiling the runtime to generate metadata
 pub fn from_wasm_file(wasm_file_path: &Path) -> WasmMetadataResult<Metadata> {
-    let wasm_file = fetch_metadata_from_file_blocking(wasm_file_path)
-        .map_err(Into::<CodegenError>::into)
+    let wasm_file = subxt_utils_fetchmetadata::from_file_blocking(wasm_file_path)
+        .map_err(|e| CodegenError::Other(e.to_string()))
         .and_then(maybe_decompress)?;
     call_and_decode(wasm_file)
 }

subxt/Cargo.toml

@@ -60,7 +60,7 @@ reconnecting-rpc-client = ["dep:finito", "jsonrpsee"]
 # Enable this to use jsonrpsee (allowing for example `OnlineClient::from_url`).
 jsonrpsee = [
     "dep:jsonrpsee",
-    "runtime",
+    "runtime"
 ]
 
 # Enable this to pull in extra Substrate dependencies which make it possible to
@@ -79,7 +79,7 @@ unstable-metadata = []
 unstable-light-client = ["subxt-lightclient"]
 
 # Activate this to expose the ability to generate metadata from Wasm runtime files.
-runtime-path = ["subxt-macro/runtime-path"]
+runtime-metadata-path = ["subxt-macro/runtime-metadata-path"]
 
 [dependencies]
 async-trait = { workspace = true }