Replace amino with protobuf types (#527)

* Ripped out amino types and replaced them with protobuf types.

* Vote and proposal serialization fix

* fmt fix

* As close as test_validator_set gets without issue #506

* sad fmt noises

* Domain types for protobuf structs (#537)

* Domain types
* DomainType derive macro
* DomainType added to all amino_types

* Minor fixes and cleanup

* Fixed voting and proposals

* Updated CHANGELOG

* Documentation update

CHANGELOG.md

@@ -3,10 +3,14 @@
 - Add spec for the light client attack evidence handling ([#526])
 - Return RFC6962 hash for empty merkle tree ([#498])
 - The `tendermint`, `tendermint-rpc`, and `tendermint-light-client` crates now compile to WASM on the `wasm32-unknown-unknown` and `wasm32-wasi` targets ([#463])
+- Implement protobuf encoding/decoding of Tendermint Proto types ([#504])
+- Separate protobuf types from Rust domain types using the DomainType trait ([#535])
 
 [#526]: https://github.com/informalsystems/tendermint-rs/issues/526
 [#498]: https://github.com/informalsystems/tendermint-rs/issues/498
 [#463]: https://github.com/informalsystems/tendermint-rs/issues/463
+[#504]: https://github.com/informalsystems/tendermint-rs/issues/504
+[#535]: https://github.com/informalsystems/tendermint-rs/issues/535
 
 ## v0.16.0
 

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "light-client",
     "light-node",
     "proto",
+    "proto-derive",
     "rpc",
     "tendermint",
     "testgen"

light-client/src/operations/voting_power.rs

@@ -115,7 +115,7 @@ impl VotingPowerCalculator for ProdVotingPowerCalculator {
 
         // Get non-absent votes from the signatures
         let non_absent_votes = signatures.iter().enumerate().flat_map(|(idx, signature)| {
-            if let Some(vote) = non_absent_vote(signature, idx as u64, &signed_header.commit) {
+            if let Some(vote) = non_absent_vote(signature, idx as u16, &signed_header.commit) {
                 Some((signature, vote))
             } else {
                 None
@@ -179,7 +179,7 @@ impl VotingPowerCalculator for ProdVotingPowerCalculator {
     }
 }
 
-fn non_absent_vote(commit_sig: &CommitSig, validator_index: u64, commit: &Commit) -> Option<Vote> {
+fn non_absent_vote(commit_sig: &CommitSig, validator_index: u16, commit: &Commit) -> Option<Vote> {
     let (validator_address, timestamp, signature, block_id) = match commit_sig {
         CommitSig::BlockIDFlagAbsent { .. } => return None,
         CommitSig::BlockIDFlagCommit {

proto-derive/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "tendermint-proto-derive"
+version = "0.1.0"
+authors = ["Greg Szabo <greg@philosobear.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1.0"
+quote = "1.0"
+syn = "1.0"

proto-derive/src/lib.rs

@@ -0,0 +1,95 @@
+//! The DomainType derive macro implements the tendermint_proto::DomainType trait.
+//! This implementation uses the Prost library to convert between Raw types and byte streams.
+//!
+//! Read more about how to use this macro in the DomainType trait definition.
+
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::spanned::Spanned;
+
+#[proc_macro_derive(DomainType, attributes(rawtype))]
+pub fn domaintype(input: TokenStream) -> TokenStream {
+    let input = syn::parse_macro_input!(input as syn::DeriveInput);
+    expand_domaintype(&input)
+}
+
+fn expand_domaintype(input: &syn::DeriveInput) -> TokenStream {
+    let ident = &input.ident;
+
+    // Todo: Make this function more robust and easier to read.
+
+    let rawtype_attributes = &input
+        .attrs
+        .iter()
+        .filter(|&attr| attr.path.is_ident("rawtype"))
+        .collect::<Vec<&syn::Attribute>>();
+    if rawtype_attributes.len() != 1 {
+        return syn::Error::new(
+            rawtype_attributes.first().span(),
+            "exactly one #[rawtype(RawType)] expected",
+        )
+        .to_compile_error()
+        .into();
+    }
+
+    let rawtype_tokens = rawtype_attributes[0]
+        .tokens
+        .clone()
+        .into_iter()
+        .collect::<Vec<quote::__private::TokenTree>>();
+    if rawtype_tokens.len() != 1 {
+        return syn::Error::new(rawtype_attributes[0].span(), "#[rawtype(RawType)] expected")
+            .to_compile_error()
+            .into();
+    }
+
+    let rawtype = match &rawtype_tokens[0] {
+        proc_macro2::TokenTree::Group(group) => group.stream(),
+        _ => {
+            return syn::Error::new(
+                rawtype_tokens[0].span(),
+                "#[rawtype(RawType)] group expected",
+            )
+            .to_compile_error()
+            .into()
+        }
+    };
+
+    let gen = quote! {
+        impl ::tendermint_proto::DomainType<#rawtype> for #ident {
+
+            fn encode<B: ::tendermint_proto::bytes::BufMut>(self, buf: &mut B) -> ::std::result::Result<(), ::tendermint_proto::Error> {
+                use ::tendermint_proto::prost::Message;
+                #rawtype::from(self).encode(buf).map_err(|e| ::tendermint_proto::Kind::EncodeMessage.context(e).into())
+            }
+
+            fn encode_length_delimited<B: ::tendermint_proto::bytes::BufMut>(self, buf: &mut B) -> ::std::result::Result<(), ::tendermint_proto::Error> {
+                use ::tendermint_proto::prost::Message;
+                #rawtype::from(self).encode_length_delimited(buf).map_err(|e| ::tendermint_proto::Kind::EncodeMessage.context(e).into())
+            }
+
+            fn decode<B: ::tendermint_proto::bytes::Buf>(buf: B) -> Result<Self, ::tendermint_proto::Error> {
+                use ::tendermint_proto::prost::Message;
+                #rawtype::decode(buf).map_or_else(
+                    |e| ::std::result::Result::Err(::tendermint_proto::Kind::DecodeMessage.context(e).into()),
+                    |t| Self::try_from(t).map_err(|e| ::tendermint_proto::Kind::TryIntoDomainType.context(e).into())
+                )
+            }
+
+            fn decode_length_delimited<B: ::tendermint_proto::bytes::Buf>(buf: B) -> Result<Self, ::tendermint_proto::Error> {
+                use ::tendermint_proto::prost::Message;
+                #rawtype::decode_length_delimited(buf).map_or_else(
+                    |e| ::std::result::Result::Err(::tendermint_proto::Kind::DecodeMessage.context(e).into()),
+                    |t| Self::try_from(t).map_err(|e| ::tendermint_proto::Kind::TryIntoDomainType.context(e).into())
+                )
+            }
+
+            fn encoded_len(self) -> usize {
+                use ::tendermint_proto::prost::Message;
+                #rawtype::from(self).encoded_len()
+            }
+
+        }
+    };
+    gen.into()
+}

proto/Cargo.toml

@@ -16,6 +16,13 @@ description = """
 [package.metadata.docs.rs]
 all-features = true
 
+[features]
+default = ["tendermint-proto-derive"]
+
 [dependencies]
 prost = { version = "0.6" }
 prost-types = { version = "0.6" }
+tendermint-proto-derive = { path = "../proto-derive", optional = true }
+bytes = "0.5"
+anomaly = "0.2"
+thiserror = "1.0"

proto/src/domaintype.rs

@@ -0,0 +1,79 @@
+//! DomainType trait
+//!
+//! The DomainType trait allows separation of the data sent on the wire (currently encoded using
+//! protobuf) from the structures used in Rust. The structures used to encode/decode from/to the wire
+//! are called "Raw" types (they mirror the definitions in the specifications) and the Rust types
+//! we use internally are called the "Domain" types. These Domain types can implement additional
+//! checks and conversions to consume the incoming data easier for a Rust developer.
+//!
+//! The benefits include decoding the wire into a struct that is inherently valid as well as hiding
+//! the encoding and decoding details from the developer. This latter is important if/when we decide
+//! to exchange the underlying Prost library with something else. (Another protobuf implementation
+//! or a completely different encoding.) Encoding is not the core product of Tendermint it's a
+//! necessary dependency.
+//!
+//!
+//! Decode: bytestream -> Raw -> Domain
+//! The `decode` function takes two steps to decode from a bytestream to a DomainType:
+//!
+//! 1. Decode the bytestream into a Raw type using the Prost library,
+//! 2. Transform that Raw type into a Domain type using the TryFrom trait of the DomainType.
+//!
+//!
+//! Encode: Domain -> Raw -> bytestream
+//! The `encode` function takes two steps to encode a DomainType into a bytestream:
+//!
+//! 1. Transform the Domain type into a Raw type using the From trait of the DomainType,
+//! 2. Encode the Raw type into a bytestream using the Prost library.
+//!
+//!
+//! Note that in the case of encode, the transformation to Raw type is infallible:
+//! Rust structs should always be ready to be encoded to the wire.
+//!
+//! Note that the Prost library and the TryFrom method have their own set of errors. These are
+//! merged into a custom Error type defined in this crate for easier handling.
+//!
+//!
+//! How to implement a DomainType struct:
+//! 1. Implement your struct based on your expectations for the developer
+//! 2. Add the derive macro `#[derive(DomainType)]` on top of it
+//! 3. Add the Raw type as a parameter of the DomainType trait (`[rawtype(MyRawType)]`)
+//! 4. Implement the `TryFrom<MyRawType> for MyDomainType` trait
+//! 5. Implement the `From<MyDomainType> for MyRawType` trait
+//!
+//! Note: the `[rawtype()]` parameter is similar to how `serde` implements serialization through a
+//! `[serde(with="")]` interim type.
+//!
+
+use crate::Error;
+use bytes::{Buf, BufMut};
+use prost::Message;
+
+/// DomainType trait allows protobuf encoding and decoding for domain types
+pub trait DomainType<T: Message + From<Self>>: Sized {
+    /// Encodes the DomainType into a buffer.
+    ///
+    /// The DomainType will be consumed.
+    fn encode<B: BufMut>(self, buf: &mut B) -> Result<(), Error>;
+
+    /// Encodes the DomainType with a length-delimiter to a buffer.
+    ///
+    /// The DomainType will be consumed.
+    /// An error will be returned if the buffer does not have sufficient capacity.
+    fn encode_length_delimited<B: BufMut>(self, buf: &mut B) -> Result<(), Error>;
+
+    /// Decodes an instance of the message from a buffer and then converts it into DomainType.
+    ///
+    /// The entire buffer will be consumed.
+    fn decode<B: Buf>(buf: B) -> Result<Self, Error>;
+
+    /// Decodes a length-delimited instance of the message from the buffer.
+    ///
+    /// The entire buffer will be consumed.
+    fn decode_length_delimited<B: Buf>(buf: B) -> Result<Self, Error>;
+
+    /// Returns the encoded length of the message without a length delimiter.
+    ///
+    /// The DomainType will be consumed.
+    fn encoded_len(self) -> usize;
+}

proto/src/error.rs

@@ -0,0 +1,37 @@
+//! This module defines the various errors that be raised during DomainType conversions.
+
+use anomaly::{BoxError, Context};
+use thiserror::Error;
+
+/// An error that can be raised by the DomainType conversions.
+pub type Error = anomaly::Error<Kind>;
+
+/// Various kinds of errors that can be raised.
+#[derive(Clone, Debug, Error)]
+pub enum Kind {
+    /// TryFrom Prost Message failed during decoding
+    #[error("error converting message type into domain type")]
+    TryIntoDomainType,
+
+    /// encoding prost Message into buffer failed
+    #[error("error encoding message into buffer")]
+    EncodeMessage,
+
+    /// decoding buffer into prost Message failed
+    #[error("error decoding buffer into message")]
+    DecodeMessage,
+}
+
+impl Kind {
+    /// Add a given source error as context for this error kind
+    ///
+    /// This is typically use with `map_err` as follows:
+    ///
+    /// ```ignore
+    /// let x = self.something.do_stuff()
+    ///     .map_err(|e| error::Kind::Config.context(e))?;
+    /// ```
+    pub fn context(self, source: impl Into<BoxError>) -> Context<Self> {
+        Context::new(self, Some(source.into()))
+    }
+}

proto/src/lib.rs

@@ -86,3 +86,20 @@ mod tendermint {
 }
 
 pub use tendermint::*;
+
+mod domaintype;
+pub use domaintype::DomainType;
+
+mod error;
+pub use error::{Error, Kind};
+
+// Re-export the bytes and prost crates for use within derived code.
+#[doc(hidden)]
+pub use bytes;
+#[doc(hidden)]
+pub use prost;
+
+// Re-export the DomainType derive macro #[derive(DomainType)]
+#[cfg(feature = "tendermint-proto-derive")]
+#[doc(hidden)]
+pub use tendermint_proto_derive::DomainType;

proto/tests/unit.rs

@@ -1,10 +1,79 @@
+use std::convert::TryFrom;
+use tendermint_proto::types::BlockId as RawBlockId;
+use tendermint_proto::types::PartSetHeader as RawPartSetHeader;
+use tendermint_proto::DomainType;
+
+// Example implementation of a protobuf struct using DomainType.
+#[derive(DomainType, Clone)]
+#[rawtype(RawBlockId)]
+pub struct BlockId {
+    hash: String,
+    part_set_header_exists: bool,
+}
+
+// DomainTypes MUST have the TryFrom trait to convert from RawTypes.
+impl TryFrom<RawBlockId> for BlockId {
+    type Error = &'static str;
+
+    fn try_from(value: RawBlockId) -> Result<Self, Self::Error> {
+        Ok(BlockId {
+            hash: String::from_utf8(value.hash)
+                .map_err(|_| "Could not convert vector to string")?,
+            part_set_header_exists: value.part_set_header != None,
+        })
+    }
+}
+
+// DomainTypes MUST be able to convert to RawTypes without errors using the From trait.
+impl From<BlockId> for RawBlockId {
+    fn from(value: BlockId) -> Self {
+        RawBlockId {
+            hash: value.hash.into_bytes(),
+            part_set_header: match value.part_set_header_exists {
+                true => Some(RawPartSetHeader {
+                    total: 0,
+                    hash: vec![],
+                }),
+                false => None,
+            },
+        }
+    }
+}
+
+#[test]
+pub fn domaintype_struct_example() {
+    let my_domain_type = BlockId {
+        hash: "Hello world!".to_string(),
+        part_set_header_exists: false,
+    };
+
+    let mut wire = vec![];
+    my_domain_type.clone().encode(&mut wire).unwrap();
+    assert_eq!(
+        wire,
+        vec![10, 12, 72, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 33]
+    );
+    let new_domain_type = BlockId::decode(wire.as_ref()).unwrap();
+    assert_eq!(new_domain_type.hash, "Hello world!".to_string());
+    assert_eq!(new_domain_type.part_set_header_exists, false);
+    assert_eq!(my_domain_type.encoded_len(), 14);
+}
+
 #[test]
-pub fn import_evidence_info() {
-    use tendermint_proto::evidence::Info;
-    let x = Info {
-        committed: true,
-        priority: 0,
-        evidence: None,
+pub fn domaintype_struct_length_delimited_example() {
+    let my_domain_type = BlockId {
+        hash: "Hello world!".to_string(),
+        part_set_header_exists: false,
     };
-    assert_eq!(x.committed, true);
+
+    let mut wire = vec![];
+    my_domain_type.encode_length_delimited(&mut wire).unwrap();
+    assert_eq!(
+        wire,
+        vec![14, 10, 12, 72, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 33]
+    );
+
+    let new_domain_type = BlockId::decode_length_delimited(wire.as_ref()).unwrap();
+    assert_eq!(new_domain_type.hash, "Hello world!".to_string());
+    assert_eq!(new_domain_type.part_set_header_exists, false);
 }