fix: Remove circular dependencies across workspace (#3023)

Circular dependencies are problematic in several ways:

- They result in cognitive overhead for developers, in trying to figure out what depends on what.
- They present `cargo` with limits in what order the crates can be compiled in.
- They invalidate build caches unnecessarily thus forcing `cargo` to rebuild certain crates.
- They cause problems with tooling such as `release-please`.

To actually break the circular dependencies, this patch inlines the uses of `development_transport` in the examples and tests for all sub-crates. This is only meant to be a short-term fix until #3111 and #2888 are fixed.

To ensure we don't accidentally reintroduce this dependency, we add a basic CI that queries `cargo metadata` using `jq`.

Resolves #3053.
Fixes #3223.
Related: #2918 (comment)
Related: googleapis/release-please#1662

.github/workflows/ci.yml

@@ -75,6 +75,11 @@ jobs:
           cargo install cargo-semver-checks --locked
           cargo semver-checks check-release -p ${{ matrix.crate }}
 
+      - name: Enforce no dependency on meta crate
+        run: |
+          cargo metadata --format-version=1 --no-deps | \
+          jq -e -r '.packages[] | select(.name == "${{ matrix.crate }}") | .dependencies | all(.name != "libp2p")'
+
   cross:
     name: Compile on ${{ matrix.target }}
     strategy:

Cargo.toml

@@ -61,7 +61,6 @@ identify = ["dep:libp2p-identify", "libp2p-metrics?/identify"]
 kad = ["dep:libp2p-kad", "libp2p-metrics?/kad"]
 macros = ["libp2p-swarm/macros"]
 mdns = ["dep:libp2p-mdns"]
-tls = ["dep:libp2p-tls"]
 metrics = ["dep:libp2p-metrics"]
 mplex = ["dep:libp2p-mplex"]
 noise = ["dep:libp2p-noise"]
@@ -76,6 +75,7 @@ rsa = ["libp2p-core/rsa"]
 secp256k1 = ["libp2p-core/secp256k1"]
 serde = ["libp2p-core/serde", "libp2p-kad?/serde", "libp2p-gossipsub?/serde"]
 tcp = ["dep:libp2p-tcp"]
+tls = ["dep:libp2p-tls"]
 tokio = ["libp2p-swarm/tokio", "libp2p-mdns?/tokio", "libp2p-tcp?/tokio", "libp2p-dns?/tokio", "libp2p-quic?/tokio", "libp2p-webrtc?/tokio"]
 uds = ["dep:libp2p-uds"]
 wasm-bindgen = ["futures-timer/wasm-bindgen", "instant/wasm-bindgen", "getrandom/js", "libp2p-swarm/wasm-bindgen"]

core/Cargo.toml

@@ -47,7 +47,8 @@ ring = { version = "0.16.9", features = ["alloc", "std"], default-features = fal
 async-std = { version = "1.6.2", features = ["attributes"] }
 base64 = "0.13.0"
 criterion = "0.4"
-libp2p = { path = "..", features = ["full"] }
+libp2p-mplex = { path = "../muxers/mplex" }
+libp2p-noise = { path = "../transports/noise" }
 multihash = { version = "0.16", default-features = false, features = ["arb"] }
 quickcheck = { package = "quickcheck-ext", path = "../misc/quickcheck-ext" }
 rmp-serde = "1.0"

core/tests/transport_upgrade.rs

@@ -19,11 +19,11 @@
 // DEALINGS IN THE SOFTWARE.
 
 use futures::prelude::*;
-use libp2p::core::identity;
-use libp2p::core::transport::{MemoryTransport, Transport};
-use libp2p::core::upgrade::{self, InboundUpgrade, OutboundUpgrade, UpgradeInfo};
-use libp2p::mplex::MplexConfig;
-use libp2p::noise;
+use libp2p_core::identity;
+use libp2p_core::transport::{MemoryTransport, Transport};
+use libp2p_core::upgrade::{self, InboundUpgrade, OutboundUpgrade, UpgradeInfo};
+use libp2p_mplex::MplexConfig;
+use libp2p_noise as noise;
 use multiaddr::{Multiaddr, Protocol};
 use rand::random;
 use std::{io, pin::Pin};

misc/metrics/Cargo.toml

@@ -32,11 +32,15 @@ prometheus-client = "0.18.0"
 libp2p-gossipsub =  { version = "0.44.0", path = "../../protocols/gossipsub", optional = true }
 
 [dev-dependencies]
-log = "0.4.0"
 env_logger = "0.10.0"
 futures = "0.3.1"
-libp2p = { path = "../..", features = ["full"] }
 hyper = { version="0.14", features = ["server", "tcp", "http1"] }
+libp2p-noise = { path = "../../transports/noise" }
+libp2p-ping = { path = "../../protocols/ping" }
+libp2p-swarm = { path = "../../swarm", features = ["macros"] }
+libp2p-tcp = { path = "../../transports/tcp", features = ["async-io"] }
+libp2p-yamux = { path = "../../muxers/yamux" }
+log = "0.4.0"
 tokio = { version = "1", features = ["rt-multi-thread"] }
 
 # Passing arguments to the docsrs builder in order to properly document cfg's.
@@ -45,3 +49,7 @@ tokio = { version = "1", features = ["rt-multi-thread"] }
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 rustc-args = ["--cfg", "docsrs"]
+
+[[example]]
+name = "metrics"
+required-features = ["ping"]

misc/metrics/examples/metrics/main.rs

@@ -51,11 +51,17 @@
 use env_logger::Env;
 use futures::executor::block_on;
 use futures::stream::StreamExt;
-use libp2p::core::Multiaddr;
-use libp2p::metrics::{Metrics, Recorder};
-use libp2p::swarm::{NetworkBehaviour, SwarmEvent};
-use libp2p::{identify, identity, ping, PeerId, Swarm};
+use libp2p_core::{identity, upgrade::Version, Multiaddr, PeerId, Transport};
+use libp2p_identify as identify;
+use libp2p_metrics::{Metrics, Recorder};
+use libp2p_noise as noise;
+use libp2p_ping as ping;
 use libp2p_swarm::keep_alive;
+use libp2p_swarm::NetworkBehaviour;
+use libp2p_swarm::Swarm;
+use libp2p_swarm::SwarmEvent;
+use libp2p_tcp as tcp;
+use libp2p_yamux as yamux;
 use log::info;
 use prometheus_client::registry::Registry;
 use std::error::Error;
@@ -69,10 +75,14 @@ fn main() -> Result<(), Box<dyn Error>> {
     let local_key = identity::Keypair::generate_ed25519();
     let local_peer_id = PeerId::from(local_key.public());
     let local_pub_key = local_key.public();
-    info!("Local peer id: {:?}", local_peer_id);
+    info!("Local peer id: {local_peer_id:?}");
 
     let mut swarm = Swarm::without_executor(
-        block_on(libp2p::development_transport(local_key))?,
+        tcp::async_io::Transport::default()
+            .upgrade(Version::V1)
+            .authenticate(noise::NoiseAuthenticated::xx(&local_key)?)
+            .multiplex(yamux::YamuxConfig::default())
+            .boxed(),
         Behaviour::new(local_pub_key),
         local_peer_id,
     );
@@ -115,14 +125,15 @@ fn main() -> Result<(), Box<dyn Error>> {
 /// For illustrative purposes, this includes the [`keep_alive::Behaviour`]) behaviour so the ping actually happen
 /// and can be observed via the metrics.
 #[derive(NetworkBehaviour)]
+#[behaviour(prelude = "libp2p_swarm::derive_prelude")]
 struct Behaviour {
     identify: identify::Behaviour,
     keep_alive: keep_alive::Behaviour,
     ping: ping::Behaviour,
 }
 
 impl Behaviour {
-    fn new(local_pub_key: libp2p::identity::PublicKey) -> Self {
+    fn new(local_pub_key: identity::PublicKey) -> Self {
         Self {
             ping: ping::Behaviour::default(),
             identify: identify::Behaviour::new(identify::Config::new(

misc/multistream-select/Cargo.toml

@@ -21,7 +21,10 @@ unsigned-varint = "0.7"
 [dev-dependencies]
 async-std = "1.6.2"
 env_logger = "0.10"
-libp2p = { path = "../..", features = ["full"] }
+libp2p-core = { path = "../../core" }
+libp2p-mplex = { path = "../../muxers/mplex" }
+libp2p-plaintext = { path = "../../transports/plaintext" }
+libp2p-swarm = { path = "../../swarm", features = ["async-std"] }
 quickcheck = { package = "quickcheck-ext", path = "../../misc/quickcheck-ext" }
 rand = "0.8"
 rw-stream-sink = { version = "0.3.0", path = "../../misc/rw-stream-sink" }

misc/multistream-select/tests/transport.rs

@@ -19,16 +19,16 @@
 // DEALINGS IN THE SOFTWARE.
 
 use futures::{channel::oneshot, prelude::*, ready};
-use libp2p::core::{
+use libp2p_core::{
     identity,
     multiaddr::Protocol,
     muxing::StreamMuxerBox,
     transport::{self, MemoryTransport},
     upgrade, Multiaddr, PeerId, Transport,
 };
-use libp2p::mplex::MplexConfig;
-use libp2p::plaintext::PlainText2Config;
-use libp2p::swarm::{dummy, Swarm, SwarmEvent};
+use libp2p_mplex::MplexConfig;
+use libp2p_plaintext::PlainText2Config;
+use libp2p_swarm::{dummy, Swarm, SwarmEvent};
 use rand::random;
 use std::task::Poll;
 

muxers/mplex/Cargo.toml

@@ -27,8 +27,9 @@ async-std = { version = "1.7.0", features = ["attributes"] }
 criterion = "0.4"
 env_logger = "0.10"
 futures = "0.3"
-libp2p = { path = "../..", features = ["full"] }
 libp2p-muxer-test-harness = { path = "../test-harness" }
+libp2p-plaintext = { path = "../../transports/plaintext" }
+libp2p-tcp = { path = "../../transports/tcp", features = ["async-io"] }
 quickcheck = { package = "quickcheck-ext", path = "../../misc/quickcheck-ext" }
 
 [[bench]]

muxers/mplex/benches/split_send_size.rs

@@ -26,12 +26,12 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughpu
 use futures::future::poll_fn;
 use futures::prelude::*;
 use futures::{channel::oneshot, future::join};
-use libp2p::core::muxing::StreamMuxerExt;
-use libp2p::core::{
+use libp2p_core::muxing::StreamMuxerExt;
+use libp2p_core::{
     identity, multiaddr::multiaddr, muxing, transport, upgrade, Multiaddr, PeerId, Transport,
 };
-use libp2p::plaintext::PlainText2Config;
-use libp2p::{mplex, tcp};
+use libp2p_mplex as mplex;
+use libp2p_plaintext::PlainText2Config;
 use std::pin::Pin;
 use std::time::Duration;
 
@@ -169,7 +169,7 @@ fn tcp_transport(split_send_size: usize) -> BenchTransport {
     let mut mplex = mplex::MplexConfig::default();
     mplex.set_split_send_size(split_send_size);
 
-    tcp::async_io::Transport::new(tcp::Config::default().nodelay(true))
+    libp2p_tcp::async_io::Transport::new(libp2p_tcp::Config::default().nodelay(true))
         .upgrade(upgrade::Version::V1)
         .authenticate(PlainText2Config { local_public_key })
         .multiplex(mplex)

protocols/autonat/Cargo.toml

@@ -27,9 +27,13 @@ prost = "0.11"
 
 [dev-dependencies]
 async-std = { version = "1.10", features = ["attributes"] }
-env_logger = "0.10"
 clap = { version = "4.0.13", features = ["derive"] }
-libp2p = { path = "../..", features = ["full"] }
+env_logger = "0.10"
+libp2p-identify = { path = "../identify" }
+libp2p-noise = { path = "../../transports/noise" }
+libp2p-swarm = { path = "../../swarm", features = ["async-std", "macros"] }
+libp2p-tcp = { path = "../../transports/tcp", features = ["async-io"] }
+libp2p-yamux = { path = "../../muxers/yamux" }
 
 # Passing arguments to the docsrs builder in order to properly document cfg's.
 # More information: https://docs.rs/about/builds#cross-compiling

protocols/autonat/examples/autonat_client.rs

@@ -31,11 +31,14 @@
 
 use clap::Parser;
 use futures::prelude::*;
-use libp2p::autonat;
-use libp2p::identify;
-use libp2p::multiaddr::Protocol;
-use libp2p::swarm::{NetworkBehaviour, Swarm, SwarmEvent};
-use libp2p::{identity, Multiaddr, PeerId};
+use libp2p_autonat as autonat;
+use libp2p_core::multiaddr::Protocol;
+use libp2p_core::{identity, upgrade::Version, Multiaddr, PeerId, Transport};
+use libp2p_identify as identify;
+use libp2p_noise as noise;
+use libp2p_swarm::{NetworkBehaviour, Swarm, SwarmEvent};
+use libp2p_tcp as tcp;
+use libp2p_yamux as yamux;
 use std::error::Error;
 use std::net::Ipv4Addr;
 use std::time::Duration;
@@ -63,7 +66,11 @@ async fn main() -> Result<(), Box<dyn Error>> {
     let local_peer_id = PeerId::from(local_key.public());
     println!("Local peer id: {:?}", local_peer_id);
 
-    let transport = libp2p::development_transport(local_key.clone()).await?;
+    let transport = tcp::async_io::Transport::default()
+        .upgrade(Version::V1)
+        .authenticate(noise::NoiseAuthenticated::xx(&local_key)?)
+        .multiplex(yamux::YamuxConfig::default())
+        .boxed();
 
     let behaviour = Behaviour::new(local_key.public());
 
@@ -89,7 +96,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 }
 
 #[derive(NetworkBehaviour)]
-#[behaviour(out_event = "Event")]
+#[behaviour(out_event = "Event", prelude = "libp2p_swarm::derive_prelude")]
 struct Behaviour {
     identify: identify::Behaviour,
     auto_nat: autonat::Behaviour,

protocols/autonat/examples/autonat_server.rs

@@ -28,11 +28,13 @@
 
 use clap::Parser;
 use futures::prelude::*;
-use libp2p::autonat;
-use libp2p::identify;
-use libp2p::multiaddr::Protocol;
-use libp2p::swarm::{NetworkBehaviour, Swarm, SwarmEvent};
-use libp2p::{identity, Multiaddr, PeerId};
+use libp2p_autonat as autonat;
+use libp2p_core::{identity, multiaddr::Protocol, upgrade::Version, Multiaddr, PeerId, Transport};
+use libp2p_identify as identify;
+use libp2p_noise as noise;
+use libp2p_swarm::{NetworkBehaviour, Swarm, SwarmEvent};
+use libp2p_tcp as tcp;
+use libp2p_yamux as yamux;
 use std::error::Error;
 use std::net::Ipv4Addr;
 
@@ -53,7 +55,11 @@ async fn main() -> Result<(), Box<dyn Error>> {
     let local_peer_id = PeerId::from(local_key.public());
     println!("Local peer id: {:?}", local_peer_id);
 
-    let transport = libp2p::development_transport(local_key.clone()).await?;
+    let transport = tcp::async_io::Transport::default()
+        .upgrade(Version::V1)
+        .authenticate(noise::NoiseAuthenticated::xx(&local_key)?)
+        .multiplex(yamux::YamuxConfig::default())
+        .boxed();
 
     let behaviour = Behaviour::new(local_key.public());
 
@@ -74,7 +80,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 }
 
 #[derive(NetworkBehaviour)]
-#[behaviour(out_event = "Event")]
+#[behaviour(out_event = "Event", prelude = "libp2p_swarm::derive_prelude")]
 struct Behaviour {
     identify: identify::Behaviour,
     auto_nat: autonat::Behaviour,

protocols/autonat/tests/test_client.rs

@@ -20,15 +20,14 @@
 
 use futures::{channel::oneshot, Future, FutureExt, StreamExt};
 use futures_timer::Delay;
-use libp2p::{
-    development_transport,
-    identity::Keypair,
-    swarm::{AddressScore, Swarm, SwarmEvent},
-    Multiaddr, PeerId,
-};
 use libp2p_autonat::{
     Behaviour, Config, Event, NatStatus, OutboundProbeError, OutboundProbeEvent, ResponseError,
 };
+use libp2p_core::{identity::Keypair, upgrade::Version, Multiaddr, PeerId, Transport};
+use libp2p_noise as noise;
+use libp2p_swarm::{AddressScore, Swarm, SwarmEvent};
+use libp2p_tcp as tcp;
+use libp2p_yamux as yamux;
 use std::time::Duration;
 
 const MAX_CONFIDENCE: usize = 3;
@@ -38,7 +37,11 @@ const TEST_REFRESH_INTERVAL: Duration = Duration::from_secs(2);
 async fn init_swarm(config: Config) -> Swarm<Behaviour> {
     let keypair = Keypair::generate_ed25519();
     let local_id = PeerId::from_public_key(&keypair.public());
-    let transport = development_transport(keypair).await.unwrap();
+    let transport = tcp::async_io::Transport::default()
+        .upgrade(Version::V1)
+        .authenticate(noise::NoiseAuthenticated::xx(&keypair).unwrap())
+        .multiplex(yamux::YamuxConfig::default())
+        .boxed();
     let behaviour = Behaviour::new(local_id, config);
     Swarm::with_async_std_executor(transport, behaviour, local_id)
 }

protocols/autonat/tests/test_server.rs

@@ -20,24 +20,28 @@
 
 use futures::{channel::oneshot, Future, FutureExt, StreamExt};
 use futures_timer::Delay;
-use libp2p::core::{ConnectedPoint, Endpoint};
-use libp2p::swarm::DialError;
-use libp2p::{
-    development_transport,
-    identity::Keypair,
-    multiaddr::Protocol,
-    swarm::{AddressScore, Swarm, SwarmEvent},
-    Multiaddr, PeerId,
-};
 use libp2p_autonat::{
     Behaviour, Config, Event, InboundProbeError, InboundProbeEvent, ResponseError,
 };
+use libp2p_core::{
+    identity::Keypair, multiaddr::Protocol, upgrade::Version, ConnectedPoint, Endpoint, Multiaddr,
+    PeerId, Transport,
+};
+use libp2p_noise as noise;
+use libp2p_swarm::DialError;
+use libp2p_swarm::{AddressScore, Swarm, SwarmEvent};
+use libp2p_tcp as tcp;
+use libp2p_yamux as yamux;
 use std::{num::NonZeroU32, time::Duration};
 
 async fn init_swarm(config: Config) -> Swarm<Behaviour> {
     let keypair = Keypair::generate_ed25519();
     let local_id = PeerId::from_public_key(&keypair.public());
-    let transport = development_transport(keypair).await.unwrap();
+    let transport = tcp::async_io::Transport::default()
+        .upgrade(Version::V1)
+        .authenticate(noise::NoiseAuthenticated::xx(&keypair).unwrap())
+        .multiplex(yamux::YamuxConfig::default())
+        .boxed();
     let behaviour = Behaviour::new(local_id, config);
     Swarm::with_async_std_executor(transport, behaviour, local_id)
 }

protocols/dcutr/Cargo.toml

@@ -29,10 +29,18 @@ void = "1"
 prost-build = "0.11"
 
 [dev-dependencies]
+clap = { version = "4.0.13", features = ["derive"] }
 env_logger = "0.10.0"
-libp2p = { path = "../..", features = ["full"] }
+libp2p-dns = { path = "../../transports/dns", features = ["async-std"] }
+libp2p-identify = { path = "../../protocols/identify" }
+libp2p-noise = { path = "../../transports/noise" }
+libp2p-ping = { path = "../../protocols/ping" }
+libp2p-plaintext = { path = "../../transports/plaintext" }
+libp2p-relay = { path = "../relay" }
+libp2p-swarm = { path = "../../swarm", features = ["macros"] }
+libp2p-tcp = { path = "../../transports/tcp", features = ["async-io"] }
+libp2p-yamux = { path = "../../muxers/yamux" }
 rand = "0.8"
-clap = { version = "4.0.13", features = ["derive"] }
 
 # Passing arguments to the docsrs builder in order to properly document cfg's.
 # More information: https://docs.rs/about/builds#cross-compiling