From a2c9b708ccef399ec57e17e1ec7c60235f2fd383 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Fri, 16 Oct 2020 09:59:17 +0200 Subject: [PATCH 1/7] Consolidate proto-compiler logic into main.rs --- proto-compiler/Cargo.toml | 16 ++--- proto-compiler/build.rs | 50 ---------------- proto-compiler/src/main.rs | 119 +++++++++++++++++++++++++++++++++---- 3 files changed, 111 insertions(+), 74 deletions(-) delete mode 100644 proto-compiler/build.rs diff --git a/proto-compiler/Cargo.toml b/proto-compiler/Cargo.toml index 49d4563651..a3572744a2 100644 --- a/proto-compiler/Cargo.toml +++ b/proto-compiler/Cargo.toml @@ -1,18 +1,12 @@ [package] -name = "ibc-proto-compiler" +name = "ibc-proto-compiler" version = "0.1.0" authors = ["Greg Szabo "] edition = "2018" publish = false -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] -walkdir = { version = "2.3" } - -[build-dependencies] -prost-build = { version = "0.6" } -walkdir = { version = "2.3" } -git2 = { version = "0.13" } - -[workspace] +git2 = "0.13" +prost-build = "0.6" +tempdir = "0.3.7" +walkdir = "2.3" diff --git a/proto-compiler/build.rs b/proto-compiler/build.rs deleted file mode 100644 index ee60d5e25f..0000000000 --- a/proto-compiler/build.rs +++ /dev/null @@ -1,50 +0,0 @@ -use git2::Repository; -use std::env::var; -use std::path::{Path, PathBuf}; -use walkdir::WalkDir; - -fn main() { - let sdk_dir = var("SDK_DIR").unwrap_or_else(|_| "target/cosmos-sdk".to_string()); - if !Path::new(&sdk_dir).exists() { - let url = "https://github.com/cosmos/cosmos-sdk"; - Repository::clone(url, &sdk_dir).unwrap(); - } - - // Paths - let proto_paths = [ - "../proto/definitions/mock".to_string(), - format!("{}/proto/ibc", sdk_dir), - format!("{}/proto/cosmos/tx", sdk_dir), - format!("{}/proto/cosmos/base", sdk_dir), - ]; - let proto_includes_paths = [ - "../proto/definitions".to_string(), - format!("{}/proto", sdk_dir), - format!("{}/third_party/proto", sdk_dir), - ]; - - // List available proto files - let mut protos: Vec = vec![]; - for proto_path in &proto_paths { - protos.append( - &mut WalkDir::new(proto_path) - .into_iter() - .filter_map(|e| e.ok()) - .filter(|e| { - e.file_type().is_file() - && e.path().extension().is_some() - && e.path().extension().unwrap() == "proto" - }) - .map(|e| e.into_path()) - .collect(), - ); - } - - // List available paths for dependencies - let includes: Vec = proto_includes_paths.iter().map(PathBuf::from).collect(); - - // Compile all proto files - let mut pb = prost_build::Config::new(); - pb.extern_path(".tendermint", "::tendermint_proto"); - pb.compile_protos(&protos, &includes).unwrap(); -} diff --git a/proto-compiler/src/main.rs b/proto-compiler/src/main.rs index 2254e040f9..715460661f 100644 --- a/proto-compiler/src/main.rs +++ b/proto-compiler/src/main.rs @@ -1,36 +1,129 @@ +use std::env::var; use std::fs::remove_dir_all; use std::fs::{copy, create_dir_all}; +use std::path::{Path, PathBuf}; + +use git2::Repository; +use tempdir::TempDir; use walkdir::WalkDir; -pub(crate) fn main() { - let ibc_proto_path = "../proto/src/prost"; +fn main() { + let target_dir = "../proto/src/prost"; + + let out_dir = var("OUT_DIR") + .map(PathBuf::from) + .or_else(|_| TempDir::new("ibc_proto_out").map(|d| d.into_path())) + .unwrap(); + + let sdk_dir = clone_cosmos_sdk(); + compile_protos(&sdk_dir, &out_dir); + copy_generated_files(&out_dir, &target_dir); +} + +fn clone_cosmos_sdk() -> PathBuf { + let sdk_dir = var("SDK_DIR").unwrap_or_else(|_| "target/cosmos-sdk".to_string()); + + if Path::new(&sdk_dir).exists() { + println!("[info ] Found Cosmos SDK source at '{}'", sdk_dir); + } else { + println!("[info ] Cloning cosmos/cosmos-sdk repository..."); + + let url = "https://github.com/cosmos/cosmos-sdk"; + Repository::clone(url, &sdk_dir).unwrap(); + + println!("[info ] => Cloned at '{}'", sdk_dir); + } + + PathBuf::from(sdk_dir) +} + +fn compile_protos(sdk_dir: impl AsRef, out_dir: impl AsRef) { + println!( + "[info ] Compiling .proto files to Rust into '{}'...", + out_dir.as_ref().display() + ); + + let root = env!("CARGO_MANIFEST_DIR"); + let sdk_dir = sdk_dir.as_ref().display(); + + // Paths + let proto_paths = [ + format!("{}/../proto/definitions/mock", root), + format!("{}/proto/ibc", sdk_dir), + format!("{}/proto/cosmos/tx", sdk_dir), + format!("{}/proto/cosmos/base", sdk_dir), + ]; + + let proto_includes_paths = [ + format!("{}/../proto", root), + format!("{}/proto", sdk_dir), + format!("{}/third_party/proto", sdk_dir), + ]; + + // List available proto files + let mut protos: Vec = vec![]; + for proto_path in &proto_paths { + protos.append( + &mut WalkDir::new(proto_path) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_type().is_file() + && e.path().extension().is_some() + && e.path().extension().unwrap() == "proto" + }) + .map(|e| e.into_path()) + .collect(), + ); + } + + // List available paths for dependencies + let includes: Vec = proto_includes_paths.iter().map(PathBuf::from).collect(); + + // Compile all proto files + let mut config = prost_build::Config::default(); + config.out_dir(out_dir.as_ref()); + config.extern_path(".tendermint", "::tendermint_proto"); + config.compile_protos(&protos, &includes).unwrap(); + + println!("[info ] => Done!"); +} + +fn copy_generated_files(from_dir: impl AsRef, to_dir: impl AsRef) { + println!( + "[info ] Copying generated files into '{}'...", + to_dir.as_ref().display() + ); // Remove old compiled files - remove_dir_all(ibc_proto_path).unwrap_or_default(); - create_dir_all(ibc_proto_path).unwrap(); + remove_dir_all(&to_dir).unwrap_or_default(); + create_dir_all(&to_dir).unwrap(); // Copy new compiled files (prost does not use folder structures) - let err: Vec = WalkDir::new(env!("OUT_DIR")) + let errors = WalkDir::new(from_dir) .into_iter() .filter_map(|e| e.ok()) .filter(|e| e.file_type().is_file()) .map(|e| { copy( e.path(), - std::path::Path::new(&format!( + format!( "{}/{}", - ibc_proto_path, + to_dir.as_ref().display(), &e.file_name().to_os_string().to_str().unwrap() - )), + ), ) }) .filter_map(|e| e.err()) - .collect(); + .collect::>(); - if !err.is_empty() { - for e in err { - dbg!(e); + if !errors.is_empty() { + for e in errors { + println!("[error] Error while copying compiled file: {}", e); } - panic!("error while copying compiled files") + + panic!("[error] Aborted."); } + + println!("[info ] => Done!"); } From 22d2e4b1e60bc8446b34ab6b43f6de787fc50f20 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Fri, 16 Oct 2020 16:52:43 +0200 Subject: [PATCH 2/7] Remove dependency on tempdir --- proto-compiler/Cargo.toml | 1 - proto-compiler/src/main.rs | 23 +++++++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/proto-compiler/Cargo.toml b/proto-compiler/Cargo.toml index a3572744a2..0ed553fad8 100644 --- a/proto-compiler/Cargo.toml +++ b/proto-compiler/Cargo.toml @@ -8,5 +8,4 @@ publish = false [dependencies] git2 = "0.13" prost-build = "0.6" -tempdir = "0.3.7" walkdir = "2.3" diff --git a/proto-compiler/src/main.rs b/proto-compiler/src/main.rs index 715460661f..3904f4303e 100644 --- a/proto-compiler/src/main.rs +++ b/proto-compiler/src/main.rs @@ -4,16 +4,11 @@ use std::fs::{copy, create_dir_all}; use std::path::{Path, PathBuf}; use git2::Repository; -use tempdir::TempDir; use walkdir::WalkDir; fn main() { - let target_dir = "../proto/src/prost"; - - let out_dir = var("OUT_DIR") - .map(PathBuf::from) - .or_else(|_| TempDir::new("ibc_proto_out").map(|d| d.into_path())) - .unwrap(); + let target_dir = PathBuf::from("../proto/src/prost"); + let out_dir = PathBuf::from(var("OUT_DIR").unwrap_or_else(|_| "target/proto-rust".to_string())); let sdk_dir = clone_cosmos_sdk(); compile_protos(&sdk_dir, &out_dir); @@ -21,20 +16,20 @@ fn main() { } fn clone_cosmos_sdk() -> PathBuf { - let sdk_dir = var("SDK_DIR").unwrap_or_else(|_| "target/cosmos-sdk".to_string()); + let sdk_dir = PathBuf::from(var("SDK_DIR").unwrap_or_else(|_| "target/cosmos-sdk".to_string())); - if Path::new(&sdk_dir).exists() { - println!("[info ] Found Cosmos SDK source at '{}'", sdk_dir); + if sdk_dir.exists() { + println!("[info ] Found Cosmos SDK source at '{}'", sdk_dir.display()); } else { println!("[info ] Cloning cosmos/cosmos-sdk repository..."); let url = "https://github.com/cosmos/cosmos-sdk"; Repository::clone(url, &sdk_dir).unwrap(); - println!("[info ] => Cloned at '{}'", sdk_dir); + println!("[info ] => Cloned at '{}'", sdk_dir.display()); } - PathBuf::from(sdk_dir) + sdk_dir } fn compile_protos(sdk_dir: impl AsRef, out_dir: impl AsRef) { @@ -43,6 +38,10 @@ fn compile_protos(sdk_dir: impl AsRef, out_dir: impl AsRef) { out_dir.as_ref().display() ); + if !out_dir.as_ref().exists() { + create_dir_all(&out_dir).unwrap(); + } + let root = env!("CARGO_MANIFEST_DIR"); let sdk_dir = sdk_dir.as_ref().display(); From 0e0846279bc7ec038d40b9ae3192102636e99371 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Fri, 16 Oct 2020 17:09:16 +0200 Subject: [PATCH 3/7] Add README --- proto-compiler/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 proto-compiler/README.md diff --git a/proto-compiler/README.md b/proto-compiler/README.md new file mode 100644 index 0000000000..780c28e252 --- /dev/null +++ b/proto-compiler/README.md @@ -0,0 +1,23 @@ +# ibc-proto-compiler + +The `ibc-proto-compiler` is a simple command-line tool to automate the compilation of [Protocol Buffers](https://developers.google.com/protocol-buffers) message definitions from the [Cosmos SDK](https://github.com/cosmos/cosmos-sdk) to Rust source code with [Prost](https://lib.rs/crates/prost), for use in the [`ibc-proto` crate](https://lib.rs/crates/ibc-proto) in the [`ibc-rs` project](https://github.com/informalsystems/ibc-rs/). + +## Usage + +From within the `ibc-proto-compiler` directory, run the following command to clone the Cosmos SDK repository, generate the Rust sources from the Protobuf definitions, and copy them to the `ibc-proto` crate within the `ibc-rs` project: + +```bash +$ cargo run +``` + +The path to the Cosmos SDK checkout can be specified with the `SDK_DIR` environment variable: (default: `target/cosmos-sdk/`) + +```bash +$ SDK_DIR=$HOME/Code/cosmos-sdk cargo run +``` + +The directory to which the Rust sources should be generated in before being copied to the appropriate location can be specified with the `OUT_DIR` environment variable: (default: `target/proto-rust/`) + +```bash +$ OUT_DIR=/tmp/rust cargo run +``` From eeac77a49cd8efd335493834068db4dd54bbd7ea Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Fri, 16 Oct 2020 17:09:38 +0200 Subject: [PATCH 4/7] Use path API instead of string formatting to compose paths --- proto-compiler/src/main.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/proto-compiler/src/main.rs b/proto-compiler/src/main.rs index 3904f4303e..215d85f32c 100644 --- a/proto-compiler/src/main.rs +++ b/proto-compiler/src/main.rs @@ -7,8 +7,11 @@ use git2::Repository; use walkdir::WalkDir; fn main() { - let target_dir = PathBuf::from("../proto/src/prost"); - let out_dir = PathBuf::from(var("OUT_DIR").unwrap_or_else(|_| "target/proto-rust".to_string())); + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let target_dir = root.join("../proto/src/prost"); + let out_dir = var("OUT_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| root.join("target/proto-rust")); let sdk_dir = clone_cosmos_sdk(); compile_protos(&sdk_dir, &out_dir); @@ -42,21 +45,21 @@ fn compile_protos(sdk_dir: impl AsRef, out_dir: impl AsRef) { create_dir_all(&out_dir).unwrap(); } - let root = env!("CARGO_MANIFEST_DIR"); - let sdk_dir = sdk_dir.as_ref().display(); + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let sdk_dir = sdk_dir.as_ref().to_owned(); // Paths let proto_paths = [ - format!("{}/../proto/definitions/mock", root), - format!("{}/proto/ibc", sdk_dir), - format!("{}/proto/cosmos/tx", sdk_dir), - format!("{}/proto/cosmos/base", sdk_dir), + root.join("../proto/definitions/mock"), + sdk_dir.join("proto/ibc"), + sdk_dir.join("proto/cosmos/tx"), + sdk_dir.join("proto/cosmos/base"), ]; let proto_includes_paths = [ - format!("{}/../proto", root), - format!("{}/proto", sdk_dir), - format!("{}/third_party/proto", sdk_dir), + root.join("../proto"), + sdk_dir.join("proto"), + sdk_dir.join("third_party/proto"), ]; // List available proto files From 2709789faa29129471e8d3d2c2e181b2771a7aa5 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Fri, 16 Oct 2020 17:48:32 +0200 Subject: [PATCH 5/7] Revert "Remove dependency on tempdir" This reverts commit 22d2e4b1e60bc8446b34ab6b43f6de787fc50f20. --- proto-compiler/Cargo.toml | 1 + proto-compiler/src/main.rs | 20 +++++++++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/proto-compiler/Cargo.toml b/proto-compiler/Cargo.toml index 0ed553fad8..a3572744a2 100644 --- a/proto-compiler/Cargo.toml +++ b/proto-compiler/Cargo.toml @@ -8,4 +8,5 @@ publish = false [dependencies] git2 = "0.13" prost-build = "0.6" +tempdir = "0.3.7" walkdir = "2.3" diff --git a/proto-compiler/src/main.rs b/proto-compiler/src/main.rs index 215d85f32c..4cf9bf07eb 100644 --- a/proto-compiler/src/main.rs +++ b/proto-compiler/src/main.rs @@ -4,6 +4,7 @@ use std::fs::{copy, create_dir_all}; use std::path::{Path, PathBuf}; use git2::Repository; +use tempdir::TempDir; use walkdir::WalkDir; fn main() { @@ -11,7 +12,8 @@ fn main() { let target_dir = root.join("../proto/src/prost"); let out_dir = var("OUT_DIR") .map(PathBuf::from) - .unwrap_or_else(|_| root.join("target/proto-rust")); + .or_else(|_| TempDir::new("ibc_proto_out").map(|d| d.into_path())) + .unwrap(); let sdk_dir = clone_cosmos_sdk(); compile_protos(&sdk_dir, &out_dir); @@ -19,20 +21,20 @@ fn main() { } fn clone_cosmos_sdk() -> PathBuf { - let sdk_dir = PathBuf::from(var("SDK_DIR").unwrap_or_else(|_| "target/cosmos-sdk".to_string())); + let sdk_dir = var("SDK_DIR").unwrap_or_else(|_| "target/cosmos-sdk".to_string()); - if sdk_dir.exists() { - println!("[info ] Found Cosmos SDK source at '{}'", sdk_dir.display()); + if Path::new(&sdk_dir).exists() { + println!("[info ] Found Cosmos SDK source at '{}'", sdk_dir); } else { println!("[info ] Cloning cosmos/cosmos-sdk repository..."); let url = "https://github.com/cosmos/cosmos-sdk"; Repository::clone(url, &sdk_dir).unwrap(); - println!("[info ] => Cloned at '{}'", sdk_dir.display()); + println!("[info ] => Cloned at '{}'", sdk_dir); } - sdk_dir + PathBuf::from(sdk_dir) } fn compile_protos(sdk_dir: impl AsRef, out_dir: impl AsRef) { @@ -41,10 +43,6 @@ fn compile_protos(sdk_dir: impl AsRef, out_dir: impl AsRef) { out_dir.as_ref().display() ); - if !out_dir.as_ref().exists() { - create_dir_all(&out_dir).unwrap(); - } - let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let sdk_dir = sdk_dir.as_ref().to_owned(); @@ -93,7 +91,7 @@ fn compile_protos(sdk_dir: impl AsRef, out_dir: impl AsRef) { fn copy_generated_files(from_dir: impl AsRef, to_dir: impl AsRef) { println!( - "[info ] Copying generated files into '{}'...", + "[info ] Copying generated Rust sources into '{}'...", to_dir.as_ref().display() ); From 6026582814db52f218c6dd643722afbfaad2272e Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Fri, 16 Oct 2020 17:49:47 +0200 Subject: [PATCH 6/7] Update READMEc --- proto-compiler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto-compiler/README.md b/proto-compiler/README.md index 780c28e252..3cddc8fe04 100644 --- a/proto-compiler/README.md +++ b/proto-compiler/README.md @@ -16,7 +16,7 @@ The path to the Cosmos SDK checkout can be specified with the `SDK_DIR` environm $ SDK_DIR=$HOME/Code/cosmos-sdk cargo run ``` -The directory to which the Rust sources should be generated in before being copied to the appropriate location can be specified with the `OUT_DIR` environment variable: (default: `target/proto-rust/`) +The directory to which the Rust sources should be generated in before being copied to the appropriate location can be specified with the `OUT_DIR` environment variable: (default: temporary directory via the `tempdir` crate) ```bash $ OUT_DIR=/tmp/rust cargo run From f9059c1e75d1a714a5382ce927e345d44e10992e Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Mon, 19 Oct 2020 10:46:23 +0200 Subject: [PATCH 7/7] Update proto-compiler/README.md Co-authored-by: Greg Szabo <16846635+greg-szabo@users.noreply.github.com> --- proto-compiler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto-compiler/README.md b/proto-compiler/README.md index 3cddc8fe04..cce1e9dffa 100644 --- a/proto-compiler/README.md +++ b/proto-compiler/README.md @@ -4,7 +4,7 @@ The `ibc-proto-compiler` is a simple command-line tool to automate the compilati ## Usage -From within the `ibc-proto-compiler` directory, run the following command to clone the Cosmos SDK repository, generate the Rust sources from the Protobuf definitions, and copy them to the `ibc-proto` crate within the `ibc-rs` project: +From within the `proto-compiler` directory, run the following command to clone the Cosmos SDK repository, generate the Rust sources from the Protobuf definitions, and copy them to the `ibc-proto` crate within the `ibc-rs` project: ```bash $ cargo run