From bd1912b344089f633c3514602f41876ee8579282 Mon Sep 17 00:00:00 2001 From: Miguel Guarniz Date: Thu, 2 Mar 2023 05:45:07 -0500 Subject: [PATCH] feat: migrate to quick-protobuf Instead of relying on `protoc` and buildscripts, we generate the bindings using `pb-rs` and version them within our codebase. This makes for a better IDE integration, a faster build and an easier use of `rust-libp2p` because we don't force the `protoc` dependency onto them. Resolves #3024. Pull-Request: #3312. --- CHANGELOG.md | 3 + Cargo.toml | 5 +- build.rs | 23 --- src/generated/mod.rs | 2 + src/{ => generated}/structs.proto | 0 src/generated/structs.rs | 242 ++++++++++++++++++++++++++++++ src/lib.rs | 6 +- src/protocol.rs | 146 +++++++++--------- 8 files changed, 326 insertions(+), 101 deletions(-) delete mode 100644 build.rs create mode 100644 src/generated/mod.rs rename src/{ => generated}/structs.proto (100%) create mode 100644 src/generated/structs.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d37d10b092a..f90a7a69b32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - Update to `libp2p-swarm` `v0.42.0`. +- Migrate from `prost` to `quick-protobuf`. This removes `protoc` dependency. See [PR 3312]. + +[PR 3312]: https://github.com/libp2p/rust-libp2p/pull/3312 [PR 3153]: https://github.com/libp2p/rust-libp2p/pull/3153 # 0.9.1 diff --git a/Cargo.toml b/Cargo.toml index fbd713d9253..71a8cd89e73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,9 +10,6 @@ repository = "https://github.com/libp2p/rust-libp2p" keywords = ["peer-to-peer", "libp2p", "networking"] categories = ["network-programming", "asynchronous"] -[build-dependencies] -prost-build = "0.11" - [dependencies] async-trait = "0.1" futures = "0.3" @@ -23,7 +20,7 @@ libp2p-swarm = { version = "0.42.0", path = "../../swarm" } libp2p-request-response = { version = "0.24.0", path = "../request-response" } log = "0.4" rand = "0.8" -prost = "0.11" +quick-protobuf = "0.8" [dev-dependencies] async-std = { version = "1.10", features = ["attributes"] } diff --git a/build.rs b/build.rs deleted file mode 100644 index d3714fdec14..00000000000 --- a/build.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2021 Protocol Labs. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -fn main() { - prost_build::compile_protos(&["src/structs.proto"], &["src"]).unwrap(); -} diff --git a/src/generated/mod.rs b/src/generated/mod.rs new file mode 100644 index 00000000000..e52c5a80bc0 --- /dev/null +++ b/src/generated/mod.rs @@ -0,0 +1,2 @@ +// Automatically generated mod.rs +pub mod structs; diff --git a/src/structs.proto b/src/generated/structs.proto similarity index 100% rename from src/structs.proto rename to src/generated/structs.proto diff --git a/src/generated/structs.rs b/src/generated/structs.rs new file mode 100644 index 00000000000..3a6d416b2b1 --- /dev/null +++ b/src/generated/structs.rs @@ -0,0 +1,242 @@ +// Automatically generated rust module for 'structs.proto' file + +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(unused_imports)] +#![allow(unknown_lints)] +#![allow(clippy::all)] +#![cfg_attr(rustfmt, rustfmt_skip)] + + +use quick_protobuf::{MessageInfo, MessageRead, MessageWrite, BytesReader, Writer, WriterBackend, Result}; +use quick_protobuf::sizeofs::*; +use super::*; + +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Debug, Default, PartialEq, Clone)] +pub struct Message { + pub type_pb: Option, + pub dial: Option, + pub dialResponse: Option, +} + +impl<'a> MessageRead<'a> for Message { + fn from_reader(r: &mut BytesReader, bytes: &'a [u8]) -> Result { + let mut msg = Self::default(); + while !r.is_eof() { + match r.next_tag(bytes) { + Ok(8) => msg.type_pb = Some(r.read_enum(bytes)?), + Ok(18) => msg.dial = Some(r.read_message::(bytes)?), + Ok(26) => msg.dialResponse = Some(r.read_message::(bytes)?), + Ok(t) => { r.read_unknown(bytes, t)?; } + Err(e) => return Err(e), + } + } + Ok(msg) + } +} + +impl MessageWrite for Message { + fn get_size(&self) -> usize { + 0 + + self.type_pb.as_ref().map_or(0, |m| 1 + sizeof_varint(*(m) as u64)) + + self.dial.as_ref().map_or(0, |m| 1 + sizeof_len((m).get_size())) + + self.dialResponse.as_ref().map_or(0, |m| 1 + sizeof_len((m).get_size())) + } + + fn write_message(&self, w: &mut Writer) -> Result<()> { + if let Some(ref s) = self.type_pb { w.write_with_tag(8, |w| w.write_enum(*s as i32))?; } + if let Some(ref s) = self.dial { w.write_with_tag(18, |w| w.write_message(s))?; } + if let Some(ref s) = self.dialResponse { w.write_with_tag(26, |w| w.write_message(s))?; } + Ok(()) + } +} + +pub mod mod_Message { + +use super::*; + +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Debug, Default, PartialEq, Clone)] +pub struct PeerInfo { + pub id: Option>, + pub addrs: Vec>, +} + +impl<'a> MessageRead<'a> for PeerInfo { + fn from_reader(r: &mut BytesReader, bytes: &'a [u8]) -> Result { + let mut msg = Self::default(); + while !r.is_eof() { + match r.next_tag(bytes) { + Ok(10) => msg.id = Some(r.read_bytes(bytes)?.to_owned()), + Ok(18) => msg.addrs.push(r.read_bytes(bytes)?.to_owned()), + Ok(t) => { r.read_unknown(bytes, t)?; } + Err(e) => return Err(e), + } + } + Ok(msg) + } +} + +impl MessageWrite for PeerInfo { + fn get_size(&self) -> usize { + 0 + + self.id.as_ref().map_or(0, |m| 1 + sizeof_len((m).len())) + + self.addrs.iter().map(|s| 1 + sizeof_len((s).len())).sum::() + } + + fn write_message(&self, w: &mut Writer) -> Result<()> { + if let Some(ref s) = self.id { w.write_with_tag(10, |w| w.write_bytes(&**s))?; } + for s in &self.addrs { w.write_with_tag(18, |w| w.write_bytes(&**s))?; } + Ok(()) + } +} + +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Debug, Default, PartialEq, Clone)] +pub struct Dial { + pub peer: Option, +} + +impl<'a> MessageRead<'a> for Dial { + fn from_reader(r: &mut BytesReader, bytes: &'a [u8]) -> Result { + let mut msg = Self::default(); + while !r.is_eof() { + match r.next_tag(bytes) { + Ok(10) => msg.peer = Some(r.read_message::(bytes)?), + Ok(t) => { r.read_unknown(bytes, t)?; } + Err(e) => return Err(e), + } + } + Ok(msg) + } +} + +impl MessageWrite for Dial { + fn get_size(&self) -> usize { + 0 + + self.peer.as_ref().map_or(0, |m| 1 + sizeof_len((m).get_size())) + } + + fn write_message(&self, w: &mut Writer) -> Result<()> { + if let Some(ref s) = self.peer { w.write_with_tag(10, |w| w.write_message(s))?; } + Ok(()) + } +} + +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Debug, Default, PartialEq, Clone)] +pub struct DialResponse { + pub status: Option, + pub statusText: Option, + pub addr: Option>, +} + +impl<'a> MessageRead<'a> for DialResponse { + fn from_reader(r: &mut BytesReader, bytes: &'a [u8]) -> Result { + let mut msg = Self::default(); + while !r.is_eof() { + match r.next_tag(bytes) { + Ok(8) => msg.status = Some(r.read_enum(bytes)?), + Ok(18) => msg.statusText = Some(r.read_string(bytes)?.to_owned()), + Ok(26) => msg.addr = Some(r.read_bytes(bytes)?.to_owned()), + Ok(t) => { r.read_unknown(bytes, t)?; } + Err(e) => return Err(e), + } + } + Ok(msg) + } +} + +impl MessageWrite for DialResponse { + fn get_size(&self) -> usize { + 0 + + self.status.as_ref().map_or(0, |m| 1 + sizeof_varint(*(m) as u64)) + + self.statusText.as_ref().map_or(0, |m| 1 + sizeof_len((m).len())) + + self.addr.as_ref().map_or(0, |m| 1 + sizeof_len((m).len())) + } + + fn write_message(&self, w: &mut Writer) -> Result<()> { + if let Some(ref s) = self.status { w.write_with_tag(8, |w| w.write_enum(*s as i32))?; } + if let Some(ref s) = self.statusText { w.write_with_tag(18, |w| w.write_string(&**s))?; } + if let Some(ref s) = self.addr { w.write_with_tag(26, |w| w.write_bytes(&**s))?; } + Ok(()) + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum MessageType { + DIAL = 0, + DIAL_RESPONSE = 1, +} + +impl Default for MessageType { + fn default() -> Self { + MessageType::DIAL + } +} + +impl From for MessageType { + fn from(i: i32) -> Self { + match i { + 0 => MessageType::DIAL, + 1 => MessageType::DIAL_RESPONSE, + _ => Self::default(), + } + } +} + +impl<'a> From<&'a str> for MessageType { + fn from(s: &'a str) -> Self { + match s { + "DIAL" => MessageType::DIAL, + "DIAL_RESPONSE" => MessageType::DIAL_RESPONSE, + _ => Self::default(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum ResponseStatus { + OK = 0, + E_DIAL_ERROR = 100, + E_DIAL_REFUSED = 101, + E_BAD_REQUEST = 200, + E_INTERNAL_ERROR = 300, +} + +impl Default for ResponseStatus { + fn default() -> Self { + ResponseStatus::OK + } +} + +impl From for ResponseStatus { + fn from(i: i32) -> Self { + match i { + 0 => ResponseStatus::OK, + 100 => ResponseStatus::E_DIAL_ERROR, + 101 => ResponseStatus::E_DIAL_REFUSED, + 200 => ResponseStatus::E_BAD_REQUEST, + 300 => ResponseStatus::E_INTERNAL_ERROR, + _ => Self::default(), + } + } +} + +impl<'a> From<&'a str> for ResponseStatus { + fn from(s: &'a str) -> Self { + match s { + "OK" => ResponseStatus::OK, + "E_DIAL_ERROR" => ResponseStatus::E_DIAL_ERROR, + "E_DIAL_REFUSED" => ResponseStatus::E_DIAL_REFUSED, + "E_BAD_REQUEST" => ResponseStatus::E_BAD_REQUEST, + "E_INTERNAL_ERROR" => ResponseStatus::E_INTERNAL_ERROR, + _ => Self::default(), + } + } +} + +} + diff --git a/src/lib.rs b/src/lib.rs index 07771fe0615..e0fc3e9bc81 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,7 +34,7 @@ pub use self::{ }; pub use libp2p_request_response::{InboundFailure, OutboundFailure}; -#[allow(clippy::derive_partial_eq_without_eq)] -mod structs_proto { - include!(concat!(env!("OUT_DIR"), "/structs.rs")); +mod proto { + include!("generated/mod.rs"); + pub use self::structs::{mod_Message::*, Message}; } diff --git a/src/protocol.rs b/src/protocol.rs index 9da7bb5a309..2bee14200d1 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -18,12 +18,12 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -use crate::structs_proto; +use crate::proto; use async_trait::async_trait; use futures::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use libp2p_core::{upgrade, Multiaddr, PeerId}; use libp2p_request_response::{self as request_response, ProtocolName}; -use prost::Message; +use quick_protobuf::{BytesReader, Writer}; use std::{convert::TryFrom, io}; #[derive(Clone, Debug)] @@ -108,14 +108,17 @@ pub struct DialRequest { impl DialRequest { pub fn from_bytes(bytes: &[u8]) -> Result { - let msg = structs_proto::Message::decode(bytes) + use quick_protobuf::MessageRead; + + let mut reader = BytesReader::from_bytes(bytes); + let msg = proto::Message::from_reader(&mut reader, bytes) .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - if msg.r#type != Some(structs_proto::message::MessageType::Dial as _) { + if msg.type_pb != Some(proto::MessageType::DIAL) { return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid type")); } - let (peer_id, addrs) = if let Some(structs_proto::message::Dial { + let (peer_id, addrs) = if let Some(proto::Dial { peer: - Some(structs_proto::message::PeerInfo { + Some(proto::PeerInfo { id: Some(peer_id), addrs, }), @@ -131,12 +134,13 @@ impl DialRequest { }; let peer_id = { - PeerId::try_from(peer_id) + PeerId::try_from(peer_id.to_vec()) .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid peer id"))? }; + let addrs = addrs .into_iter() - .filter_map(|a| match Multiaddr::try_from(a) { + .filter_map(|a| match Multiaddr::try_from(a.to_vec()) { Ok(a) => Some(a), Err(e) => { log::debug!("Unable to parse multiaddr: {e}"); @@ -151,6 +155,8 @@ impl DialRequest { } pub fn into_bytes(self) -> Vec { + use quick_protobuf::MessageWrite; + let peer_id = self.peer_id.to_bytes(); let addrs = self .addresses @@ -158,21 +164,21 @@ impl DialRequest { .map(|addr| addr.to_vec()) .collect(); - let msg = structs_proto::Message { - r#type: Some(structs_proto::message::MessageType::Dial as _), - dial: Some(structs_proto::message::Dial { - peer: Some(structs_proto::message::PeerInfo { - id: Some(peer_id), + let msg = proto::Message { + type_pb: Some(proto::MessageType::DIAL), + dial: Some(proto::Dial { + peer: Some(proto::PeerInfo { + id: Some(peer_id.to_vec()), addrs, }), }), - dial_response: None, + dialResponse: None, }; - let mut bytes = Vec::with_capacity(msg.encoded_len()); - msg.encode(&mut bytes) - .expect("Vec provides capacity as needed"); - bytes + let mut buf = Vec::with_capacity(msg.get_size()); + let mut writer = Writer::new(&mut buf); + msg.write_message(&mut writer).expect("Encoding to succeed"); + buf } } @@ -184,29 +190,27 @@ pub enum ResponseError { InternalError, } -impl From for i32 { +impl From for proto::ResponseStatus { fn from(t: ResponseError) -> Self { match t { - ResponseError::DialError => 100, - ResponseError::DialRefused => 101, - ResponseError::BadRequest => 200, - ResponseError::InternalError => 300, + ResponseError::DialError => proto::ResponseStatus::E_DIAL_ERROR, + ResponseError::DialRefused => proto::ResponseStatus::E_DIAL_REFUSED, + ResponseError::BadRequest => proto::ResponseStatus::E_BAD_REQUEST, + ResponseError::InternalError => proto::ResponseStatus::E_INTERNAL_ERROR, } } } -impl TryFrom for ResponseError { +impl TryFrom for ResponseError { type Error = io::Error; - fn try_from(value: structs_proto::message::ResponseStatus) -> Result { + fn try_from(value: proto::ResponseStatus) -> Result { match value { - structs_proto::message::ResponseStatus::EDialError => Ok(ResponseError::DialError), - structs_proto::message::ResponseStatus::EDialRefused => Ok(ResponseError::DialRefused), - structs_proto::message::ResponseStatus::EBadRequest => Ok(ResponseError::BadRequest), - structs_proto::message::ResponseStatus::EInternalError => { - Ok(ResponseError::InternalError) - } - structs_proto::message::ResponseStatus::Ok => { + proto::ResponseStatus::E_DIAL_ERROR => Ok(ResponseError::DialError), + proto::ResponseStatus::E_DIAL_REFUSED => Ok(ResponseError::DialRefused), + proto::ResponseStatus::E_BAD_REQUEST => Ok(ResponseError::BadRequest), + proto::ResponseStatus::E_INTERNAL_ERROR => Ok(ResponseError::InternalError), + proto::ResponseStatus::OK => { log::debug!("Received response with status code OK but expected error."); Err(io::Error::new( io::ErrorKind::InvalidData, @@ -225,38 +229,35 @@ pub struct DialResponse { impl DialResponse { pub fn from_bytes(bytes: &[u8]) -> Result { - let msg = structs_proto::Message::decode(bytes) + use quick_protobuf::MessageRead; + + let mut reader = BytesReader::from_bytes(bytes); + let msg = proto::Message::from_reader(&mut reader, bytes) .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - if msg.r#type != Some(structs_proto::message::MessageType::DialResponse as _) { + if msg.type_pb != Some(proto::MessageType::DIAL_RESPONSE) { return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid type")); } - Ok(match msg.dial_response { - Some(structs_proto::message::DialResponse { - status: Some(status), - status_text, + Ok(match msg.dialResponse { + Some(proto::DialResponse { + status: Some(proto::ResponseStatus::OK), + statusText, addr: Some(addr), - }) if structs_proto::message::ResponseStatus::from_i32(status) - == Some(structs_proto::message::ResponseStatus::Ok) => - { - let addr = Multiaddr::try_from(addr) + }) => { + let addr = Multiaddr::try_from(addr.to_vec()) .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; Self { - status_text, + status_text: statusText, result: Ok(addr), } } - Some(structs_proto::message::DialResponse { + Some(proto::DialResponse { status: Some(status), - status_text, + statusText, addr: None, }) => Self { - status_text, - result: Err(ResponseError::try_from( - structs_proto::message::ResponseStatus::from_i32(status).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "invalid response status code") - })?, - )?), + status_text: statusText, + result: Err(ResponseError::try_from(status)?), }, _ => { log::debug!("Received malformed response message."); @@ -269,35 +270,38 @@ impl DialResponse { } pub fn into_bytes(self) -> Vec { + use quick_protobuf::MessageWrite; + let dial_response = match self.result { - Ok(addr) => structs_proto::message::DialResponse { - status: Some(0), - status_text: self.status_text, + Ok(addr) => proto::DialResponse { + status: Some(proto::ResponseStatus::OK), + statusText: self.status_text, addr: Some(addr.to_vec()), }, - Err(error) => structs_proto::message::DialResponse { + Err(error) => proto::DialResponse { status: Some(error.into()), - status_text: self.status_text, + statusText: self.status_text, addr: None, }, }; - let msg = structs_proto::Message { - r#type: Some(structs_proto::message::MessageType::DialResponse as _), + let msg = proto::Message { + type_pb: Some(proto::MessageType::DIAL_RESPONSE), dial: None, - dial_response: Some(dial_response), + dialResponse: Some(dial_response), }; - let mut bytes = Vec::with_capacity(msg.encoded_len()); - msg.encode(&mut bytes) - .expect("Vec provides capacity as needed"); - bytes + let mut buf = Vec::with_capacity(msg.get_size()); + let mut writer = Writer::new(&mut buf); + msg.write_message(&mut writer).expect("Encoding to succeed"); + buf } } #[cfg(test)] mod tests { use super::*; + use quick_protobuf::MessageWrite; #[test] fn test_request_encode_decode() { @@ -346,20 +350,20 @@ mod tests { a }; - let msg = structs_proto::Message { - r#type: Some(structs_proto::message::MessageType::Dial.into()), - dial: Some(structs_proto::message::Dial { - peer: Some(structs_proto::message::PeerInfo { + let msg = proto::Message { + type_pb: Some(proto::MessageType::DIAL), + dial: Some(proto::Dial { + peer: Some(proto::PeerInfo { id: Some(PeerId::random().to_bytes()), addrs: vec![valid_multiaddr_bytes, invalid_multiaddr], }), }), - dial_response: None, + dialResponse: None, }; - let mut bytes = Vec::with_capacity(msg.encoded_len()); - msg.encode(&mut bytes) - .expect("Vec provides capacity as needed"); + let mut bytes = Vec::with_capacity(msg.get_size()); + let mut writer = Writer::new(&mut bytes); + msg.write_message(&mut writer).expect("Encoding to succeed"); let request = DialRequest::from_bytes(&bytes).expect("not to fail");