Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

feat: add WebRTC transport #12529

Open
wants to merge 79 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
79 commits
Select commit Hold shift + click to select a range
81bdbdf
upgrade libp2p to 0.50.0
melekes Nov 18, 2022
21cf90c
on_swarm_event and on_connection_handler_event
melekes Nov 21, 2022
8a227e4
replace `Swarm::new` with `Swarm::with_threadpool_executor`
melekes Nov 21, 2022
8a97a52
on_swarm_event and on_connection_handler_event part 2
melekes Nov 21, 2022
e9b731b
on_swarm_event and on_connection_handler_event part 3
melekes Nov 21, 2022
d27d73a
on_swarm_event and on_connection_handler_event part 4
melekes Nov 21, 2022
672cac2
update libp2p
melekes Nov 21, 2022
41ab633
Merge branch 'master' into anton/upgrade-libp2p-to-0.50.0
melekes Nov 28, 2022
2fa1b2b
libp2p 0.50.0
melekes Nov 28, 2022
b3c33b5
rename OutboundQueryCompleted to OutboundQueryProgressed
melekes Nov 28, 2022
20437d7
remove unused var
melekes Nov 28, 2022
dbb7bec
accumulate outbound_query_records until query is finished
melekes Nov 28, 2022
f793dad
client: add WebRTC transport
melekes Jun 30, 2022
1e6f631
put webrtc certificate within base, not network config directory
melekes Nov 22, 2022
27ef639
Merge branch 'master' into anton/webrtc-transport
melekes Jan 10, 2023
a955481
fixes after the merge
melekes Jan 10, 2023
82fd633
fix docs
melekes Jan 10, 2023
c6eb8c8
better vars names in tests
melekes Jan 11, 2023
f964453
Merge branch 'master' into anton/webrtc-transport
melekes Jan 11, 2023
0db37d0
always give preference to tcp transport
melekes Jan 11, 2023
b7070e0
Merge branch 'master' into anton/webrtc-transport
melekes Jan 17, 2023
05238ab
Merge branch 'master' into anton/webrtc-transport
melekes Jan 26, 2023
2f404a4
use udp protocol, not tcp for webrtc
melekes Jan 26, 2023
05fb813
copy warp-sync zombienet test
melekes Jan 26, 2023
ce05840
rename test files
melekes Jan 26, 2023
f929b72
update test name
melekes Jan 26, 2023
7781776
remove generate-warp-sync-database from webrtc test
melekes Jan 26, 2023
1e153d2
change listen addr in zombienet tests
melekes Jan 27, 2023
dcd00f4
Apply suggestions from code review
melekes Jan 30, 2023
1c7a375
bump zombienet version
melekes Jan 30, 2023
727b487
Merge branch 'master' into anton/webrtc-transport
melekes Feb 6, 2023
b549d2b
Update zombienet version.
pepoviola Feb 7, 2023
35b971a
Merge branch 'master' into anton/webrtc-transport
melekes Feb 8, 2023
6e9c7ea
Update zombienet/0004-webrtc/test-webrtc.toml
pepoviola Feb 10, 2023
4516910
Update zombienet/0004-webrtc/test-webrtc.toml
pepoviola Feb 10, 2023
41fc433
Update zombienet/0004-webrtc/test-webrtc.toml
pepoviola Feb 10, 2023
722ccad
Update .gitlab-ci.yml
pepoviola Feb 10, 2023
be72387
bump version of zombienet and update snaps links (#13359)
pepoviola Feb 10, 2023
22c1a15
Merge branch 'master' into anton/webrtc-transport
melekes Feb 12, 2023
c596bb4
Merge branch 'master' into anton/webrtc-transport
melekes Mar 8, 2023
2d35bec
bump zombienet version
melekes Mar 8, 2023
a63e5d6
remove unneeded lines from zombienet test
melekes Mar 8, 2023
407ce5a
add missing features
melekes Mar 8, 2023
a26fabb
Merge branch 'master' into anton/webrtc-transport
melekes Mar 17, 2023
f2ca191
enable logging for bob
melekes Mar 17, 2023
52be5fd
add `--webrtc-certificate-raw` flag for zombienet
melekes Mar 20, 2023
b10a5cb
Merge branch 'master' into anton/webrtc-transport
melekes Mar 24, 2023
b4b92bf
zombienet: set p2p_cert_hash
melekes Mar 24, 2023
a9f2bd7
set p2p_cert_hash for alice
melekes Mar 24, 2023
0764cc9
Merge branch 'master' into anton/webrtc-transport
melekes Mar 27, 2023
1282e57
try to fix unsupported address error
melekes Mar 27, 2023
45c3d64
Merge branch 'master' into anton/webrtc-transport
melekes May 30, 2023
e4003ef
remove leftovers from merge
melekes May 30, 2023
342be2a
add missing dev dep
melekes May 30, 2023
2b37e04
rename /webrtc to /webrtc-direct
melekes May 31, 2023
8347be8
Merge branch 'master' into anton/webrtc-transport
melekes May 31, 2023
62a4658
Revert "rename /webrtc to /webrtc-direct"
melekes Jun 5, 2023
a518c81
Revert "Revert "rename /webrtc to /webrtc-direct""
melekes Jun 5, 2023
35ef2e8
disable webrtc for non-validators
melekes Jun 5, 2023
ae89a63
add logging to all nodes
melekes Jun 7, 2023
ef632a1
Merge branch 'master' into anton/webrtc-transport
melekes Jun 19, 2023
656faf7
trace webrtc package
melekes Jun 20, 2023
baf5b8f
Merge branch 'master' into anton/webrtc-transport
melekes Jul 25, 2023
424e319
fixes after merge
melekes Jul 25, 2023
e24baa9
fix protocol name
melekes Jul 25, 2023
6025ea4
use libp2p-webrtc dep in CLI
melekes Jul 25, 2023
09efa75
Merge branch 'master' into anton/webrtc-transport
melekes Jul 25, 2023
f0f7075
remove webrtc_sctp trace from dave's log
melekes Jul 26, 2023
142b884
bump regex to resolve polkadot companion CI job
melekes Jul 26, 2023
01ba6d2
more logging
melekes Jul 26, 2023
7d4e8e2
set mode to server by default
melekes Jul 26, 2023
ff958f8
Revert "bump regex to resolve polkadot companion CI job"
melekes Jul 27, 2023
27ef6c9
Revert "set mode to server by default"
melekes Jul 27, 2023
634ed48
Merge branch 'master' into anton/webrtc-transport
melekes Jul 27, 2023
2647ae6
reduce logging
melekes Jul 27, 2023
fb2caf8
Merge branch 'master' into anton/webrtc-transport
melekes Jul 28, 2023
c650ae8
Merge branch 'master' into anton/webrtc-transport
melekes Jul 31, 2023
20d33ad
Merge branch 'master' into anton/webrtc-transport
melekes Aug 2, 2023
7d8623a
more logging for dave
melekes Aug 2, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion bin/node/cli/benches/block_production.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use sc_consensus::{
use sc_service::{
config::{
BlocksPruning, DatabaseSource, KeystoreConfig, NetworkConfiguration, OffchainWorkerConfig,
PruningMode, WasmExecutionMethod, WasmtimeInstantiationStrategy,
PruningMode, WasmExecutionMethod, WasmtimeInstantiationStrategy, WebRTCConfig,
},
BasePath, Configuration, Role,
};
Expand All @@ -52,6 +52,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
Sr25519Keyring::Alice.to_seed(),
"network/test/0.1",
Default::default(),
WebRTCConfig::Ephemeral,
None,
);

Expand Down
3 changes: 2 additions & 1 deletion bin/node/cli/benches/transaction_pool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ use sc_client_api::execution_extensions::ExecutionStrategies;
use sc_service::{
config::{
BlocksPruning, DatabaseSource, KeystoreConfig, NetworkConfiguration, OffchainWorkerConfig,
PruningMode, TransactionPoolOptions, WasmExecutionMethod,
PruningMode, TransactionPoolOptions, WasmExecutionMethod, WebRTCConfig,
},
BasePath, Configuration, Role,
};
Expand All @@ -46,6 +46,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
Sr25519Keyring::Alice.to_seed(),
"network/test/0.1",
Default::default(),
WebRTCConfig::Ephemeral,
None,
);

Expand Down
24 changes: 22 additions & 2 deletions client/cli/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
use crate::{
arg_enums::Database, error::Result, DatabaseParams, ImportParams, KeystoreParams,
NetworkParams, NodeKeyParams, OffchainWorkerParams, PruningParams, SharedParams, SubstrateCli,
WebRTCCertificateParams,
};
use log::warn;
use names::{Generator, Name};
Expand All @@ -29,7 +30,7 @@ use sc_service::{
config::{
BasePath, Configuration, DatabaseSource, KeystoreConfig, NetworkConfiguration,
NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods,
TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod,
TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod, WebRTCConfig,
},
BlocksPruning, ChainSpec, TracingReceiver,
};
Expand Down Expand Up @@ -116,6 +117,11 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
self.network_params().map(|x| &x.node_key_params)
}

/// Get the [`WebRTCCertificateParams`] for this object.
fn webrtc_certificate_params(&self) -> Option<&WebRTCCertificateParams> {
self.network_params().map(|x| &x.webrtc_certificate_params)
}

/// Get the DatabaseParams for this object
fn database_params(&self) -> Option<&DatabaseParams> {
self.import_params().map(|x| &x.database_params)
Expand Down Expand Up @@ -163,6 +169,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
client_id: &str,
node_name: &str,
node_key: NodeKeyConfig,
webrtc: WebRTCConfig,
default_listen_port: u16,
) -> Result<NetworkConfiguration> {
Ok(if let Some(network_params) = self.network_params() {
Expand All @@ -174,10 +181,11 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
client_id,
node_name,
node_key,
webrtc,
default_listen_port,
)
} else {
NetworkConfiguration::new(node_name, client_id, node_key, Some(net_config_dir))
NetworkConfiguration::new(node_name, client_id, node_key, webrtc, Some(net_config_dir))
})
}

Expand Down Expand Up @@ -454,6 +462,16 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
.unwrap_or_else(|| Ok(Default::default()))
}

/// Get the WebRTC config from the current object.
///
/// By default this is retrieved from [`WebRTCCertificateParams`] if it is available. Otherwise
/// its [`WebRTCConfig::Ephemeral`].
fn webrtc(&self, config_dir: &PathBuf) -> Result<WebRTCConfig> {
self.webrtc_certificate_params()
.map(|x| x.webrtc_certificate(config_dir))
.unwrap_or_else(|| Ok(WebRTCConfig::Ephemeral))
}

/// Get maximum runtime instances
///
/// By default this is `None`.
Expand Down Expand Up @@ -502,6 +520,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
},
);
let node_key = self.node_key(&net_config_dir)?;
let webrtc = self.webrtc(&config_dir)?;
let role = self.role(is_dev)?;
let max_runtime_instances = self.max_runtime_instances()?.unwrap_or(8);
let is_validator = role.is_authority();
Expand All @@ -522,6 +541,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
client_id.as_str(),
self.node_name()?.as_str(),
node_key,
webrtc,
DCV::p2p_listen_port(),
)?,
keystore_remote,
Expand Down
3 changes: 2 additions & 1 deletion client/cli/src/params/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ mod offchain_worker_params;
mod pruning_params;
mod shared_params;
mod transaction_pool_params;
mod webrtc_certificate_params;

use crate::arg_enums::{CryptoScheme, OutputType};
use clap::Args;
Expand All @@ -37,7 +38,7 @@ use std::{fmt::Debug, str::FromStr};
pub use crate::params::{
database_params::*, import_params::*, keystore_params::*, network_params::*,
node_key_params::*, offchain_worker_params::*, pruning_params::*, shared_params::*,
transaction_pool_params::*,
transaction_pool_params::*, webrtc_certificate_params::*,
};

/// Parse Ss58AddressFormat
Expand Down
33 changes: 30 additions & 3 deletions client/cli/src/params/network_params.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

use crate::{arg_enums::SyncMode, params::node_key_params::NodeKeyParams};
use crate::{
arg_enums::SyncMode,
params::{node_key_params::NodeKeyParams, webrtc_certificate_params::WebRTCCertificateParams},
};
use clap::Args;
use sc_network::{
config::{NetworkConfiguration, NodeKeyConfig},
config::{NetworkConfiguration, NodeKeyConfig, WebRTCConfig},
multiaddr::Protocol,
};
use sc_network_common::config::{NonReservedPeerMode, SetConfig, TransportConfig};
Expand Down Expand Up @@ -111,6 +114,10 @@ pub struct NetworkParams {
#[clap(flatten)]
pub node_key_params: NodeKeyParams,

#[allow(missing_docs)]
#[clap(flatten)]
pub webrtc_certificate_params: WebRTCCertificateParams,

/// Enable peer discovery on local networks.
///
/// By default this option is `true` for `--dev` or when the chain type is
Expand Down Expand Up @@ -158,12 +165,13 @@ impl NetworkParams {
client_id: &str,
node_name: &str,
node_key: NodeKeyConfig,
webrtc: WebRTCConfig,
default_listen_port: u16,
) -> NetworkConfiguration {
let port = self.port.unwrap_or(default_listen_port);

let listen_addresses = if self.listen_addr.is_empty() {
if is_validator || is_dev {
let mut addrs = if is_validator || is_dev {
vec![
Multiaddr::empty()
.with(Protocol::Ip6([0, 0, 0, 0, 0, 0, 0, 0].into()))
Expand All @@ -183,7 +191,25 @@ impl NetworkParams {
.with(Protocol::Tcp(port))
.with(Protocol::Ws(Cow::Borrowed("/"))),
]
};

// Enable WebRTC for non-validators and developers by default.
if !is_validator || is_dev {
addrs.push(
Multiaddr::empty()
.with(Protocol::Ip6([0, 0, 0, 0, 0, 0, 0, 0].into()))
.with(Protocol::Udp(port))
.with(Protocol::WebRTC),
);
addrs.push(
Multiaddr::empty()
.with(Protocol::Ip4([0, 0, 0, 0].into()))
.with(Protocol::Tcp(port))
.with(Protocol::WebRTC),
);
}

addrs
} else {
self.listen_addr.clone()
};
Expand Down Expand Up @@ -227,6 +253,7 @@ impl NetworkParams {
extra_sets: Vec::new(),
request_response_protocols: Vec::new(),
node_key,
webrtc,
node_name: node_name.to_string(),
client_version: client_id.to_string(),
transport: TransportConfig::Normal {
Expand Down
121 changes: 121 additions & 0 deletions client/cli/src/params/webrtc_certificate_params.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// This file is part of Substrate.

// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

use clap::Args;
use sc_network::config::WebRTCConfig;
use std::path::PathBuf;

use crate::error;

/// The file name of the WebRTC's certificate inside the chain-specific
/// network config directory.
const WEBRTC_CERTIFICATE_FILENAME: &str = "webrtc_certificate";

/// Parameters used to create the `WebRTCConfig`, which determines the certificate used
/// for libp2p WebRTC transport.
#[derive(Debug, Clone, Args)]
pub struct WebRTCCertificateParams {
/// The file from which to read the node's WebRTC certificate to use for libp2p networking.
///
/// The contents of the file are parsed as follows:
///
/// The file must contain an ASCII PEM encoded
/// `webrtc::peer_connection::certificate::RTCCertificate`.
///
/// If the file does not exist, it is created with a newly generated certificate.
#[clap(long, value_name = "FILE")]
pub webrtc_certificate_file: Option<PathBuf>,

/// When set to `true`, a new WebRTC certificate will be created each time you start a node.
///
/// The certificate won't be stored on disk. Use this option only if you DON'T want to preserve
/// node's WebRTC identity between (re)starts.
///
/// This option takes precedence over `--webrtc-certificate-file` option.
#[arg(long, value_name = "EPHEMERAL")]
pub webrtc_certificate_ephemeral: Option<bool>,
}

impl WebRTCCertificateParams {
/// Create a `WebRTCConfig` from the given [`WebRTCCertificateParams`] in the context
/// of an optional base config storage directory.
pub fn webrtc_certificate(&self, config_dir: &PathBuf) -> error::Result<WebRTCConfig> {
if let Some(true) = self.webrtc_certificate_ephemeral {
return Ok(WebRTCConfig::Ephemeral)
}
melekes marked this conversation as resolved.
Show resolved Hide resolved

let filename = self
.webrtc_certificate_file
.clone()
.unwrap_or_else(|| config_dir.join(WEBRTC_CERTIFICATE_FILENAME));

Ok(WebRTCConfig::File(filename))
}
}

#[cfg(test)]
mod tests {
use super::*;
use libp2p::webrtc::tokio::Certificate as WebRTCCertificate;
use rand::thread_rng;
use std::fs;

#[test]
fn test_webrtc_certificate_file() {
fn load_cert_and_assert_eq(file: PathBuf, cert: WebRTCCertificate) {
let params = WebRTCCertificateParams {
webrtc_certificate_file: Some(file),
webrtc_certificate_ephemeral: None,
};

let loaded_cert = params
.webrtc_certificate(&PathBuf::from("not-used"))
.expect("Creates certificate config")
.into_certificate()
.expect("Creates certificate");

assert_eq!(cert, loaded_cert, "expected the same certificate");
}

let tmp = tempfile::Builder::new().prefix("alice").tempdir().expect("Creates tempfile");
let file = tmp.path().join("mycertificate").to_path_buf();

let cert = WebRTCCertificate::generate(&mut thread_rng()).expect("Generates certificate");

fs::write(&file, cert.serialize_pem().as_bytes()).expect("Writes certificate");
load_cert_and_assert_eq(file.clone(), cert);
}

#[test]
fn test_webrtc_certificate_ephemeral() {
let filepath = PathBuf::from("not-used");
let params = WebRTCCertificateParams {
webrtc_certificate_file: Some(filepath.clone()),
webrtc_certificate_ephemeral: Some(true),
};

let _loaded_cert = params
.webrtc_certificate(&filepath)
melekes marked this conversation as resolved.
Show resolved Hide resolved
.expect("Creates certificate config")
.into_certificate()
.expect("Creates certificate");

assert!(!filepath.exists(), "Does not create a file");
assert!(!filepath.join(WEBRTC_CERTIFICATE_FILENAME).exists(), "Does not create a file");
}
}
2 changes: 1 addition & 1 deletion client/network/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ fnv = "1.0.6"
futures = "0.3.21"
futures-timer = "3.0.2"
ip_network = "0.4.1"
libp2p = { version = "0.50.0", features = ["dns", "identify", "kad", "macros", "mdns", "mplex", "noise", "ping", "tcp", "tokio", "yamux", "websocket"] }
libp2p = { version = "0.50.0", features = ["dns", "identify", "kad", "macros", "mdns", "mplex", "noise", "ping", "tcp", "tokio", "yamux", "webrtc", "websocket"] }
log = "0.4.17"
lru = "0.8.1"
parking_lot = "0.12.1"
Expand Down
Loading