remove cargo env var and improve code gen

.github/workflows/test.yml

@@ -1,5 +1,6 @@
 name: Tests
-
+env:
+  CI: true
 on:
   workflow_dispatch:
   push:
@@ -27,7 +28,7 @@ jobs:
         uses: helm/kind-action@v1.5.0
         with:
           version: v0.18.0
-          config: test_environment/cluster-config.yml
+          config: sequencer-relayer-test/kubernetes/cluster-config.yml
           kubectl_version: v1.27.1
       - name: install just
         uses: taiki-e/install-action@just

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
   "sequencer-relayer",
+  "sequencer-relayer-proto",
   "sequencer-relayer-test",
 ]

justfile

@@ -2,7 +2,7 @@ default:
   @just --list
 
 create-cluster:
-  kind create cluster --config ./test_environment/cluster-config.yml
+  kind create cluster --config ./sequencer-relayer-test/kubernetes/cluster-config.yml
 
 delete-cluster:
   kind delete cluster --name test-cluster
@@ -11,15 +11,15 @@ deploy-ingress-controller:
   kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
 
 perform-prepull:
-  kubectl apply -f ./test_environment/prepull-daemon-set.yml
+  kubectl apply -f ./sequencer-relayer-test/kubernetes/prepull-daemon-set.yml
 
 prepare-test-environment: create-cluster deploy-ingress-controller perform-prepull
 
 create-namespace:
   kubectl create namespace test
 
 deploy-test-environment:
-  kubectl apply -n test -k ./test_environment/
+  kubectl apply -n test -k ./sequencer-relayer-test/kubernetes
 
 query-sequencer:
   curl http://test.localdev.me/sequencer/cosmos/base/tendermint/v1beta1/blocks/latest
@@ -34,7 +34,7 @@ wait-for-test-environment:
   kubectl wait -n test --for=condition=available deployment.apps/sequencer-relayer-environment-deployment --timeout=600s
 
 kustomize:
-  kubectl kustomize ./test_environment -o ./test_environment/test-environment.yml
+  kubectl kustomize ./sequencer-relayer-test/kubernetes -o ./sequencer-relayer-test/kubernetes/test-environment.yml
 
 create-ingress-rule:
-  kubectl apply -n test -f ./test_environment/ingress.yml
+  kubectl apply -n test -f ./sequencer-relayer-test/kubernetes/ingress.yml

sequencer-relayer-proto/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "sequencer-relayer-proto"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+prost = "0.11"
+prost-types = "0.11.9"
+
+[build-dependencies]
+prost-build = "0.11"

sequencer-relayer-proto/build.rs

@@ -0,0 +1,5 @@
+fn main() {
+    const PROTO_ROOT: &str = "./proto/";
+    println!("cargo:rerun-if-changed={PROTO_ROOT}");
+    prost_build::compile_protos(&["proto/msg.proto", "proto/tx.proto"], &[PROTO_ROOT]).unwrap();
+}

sequencer-relayer-proto/src/lib.rs

@@ -0,0 +1 @@
+include!(concat!(env!("OUT_DIR"), "/_.rs"));

sequencer-relayer-test/Cargo.toml

@@ -18,3 +18,7 @@ tokio = { version = "1.24", features = [ "sync", "time" ] }
 serde_yaml = "0.9.21"
 minijinja = "0.32.1"
 reqwest = { version = "0.11.17", default-features = false, features = [ "rustls-tls" ] }
+
+[dev-dependencies]
+tempfile = "3.5.0"
+which = "4.4.0"

sequencer-relayer-test/src/lib.rs

@@ -13,13 +13,12 @@ use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
 const TEST_ENVIRONMENT_YAML: &str = include_str!(concat!(
-    env!("CARGO_WORKSPACE_DIR"),
-    "/test_environment/test-environment.yml"
+    env!("CARGO_MANIFEST_DIR"),
+    "/kubernetes/test-environment.yml"
 ));
-
 const TEST_INGRESS_TEMPLATE: &str = include_str!(concat!(
-    env!("CARGO_WORKSPACE_DIR"),
-    "/test_environment/ingress.yml.j2"
+    env!("CARGO_MANIFEST_DIR"),
+    "/kubernetes/ingress.yml.j2"
 ));
 
 static STOP_POD_TX: Lazy<UnboundedSender<String>> = Lazy::new(|| {

sequencer-relayer-test/tests/kustomize_build.rs

@@ -0,0 +1,81 @@
+use std::{
+    env, fs,
+    path::{Path, PathBuf},
+    process::{exit, Command},
+};
+
+const KUSTOMIZE_DIR: &str = "kubernetes/";
+const TEST_ENV_KUBE_YAML: &str = "kubernetes/test-environment.yml";
+
+fn kubectl_from_env() -> PathBuf {
+    let os_specific_hint = if cfg!(target_os = "macos") {
+        "You could try running `brew install kubectl` or follow the official guide at https://kubernetes.io/docs/tasks/tools/install-kubectl-macos/"
+    } else if cfg!(target_os = "linux") {
+        "If you're on Arch Linux you could running `sudo pacman -S kubectl`. or follow the official guide at https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/"
+    } else {
+        "You could try installing it by following the official guide at https://kubernetes.io/docs/tasks/tools/#kubectl"
+    };
+    let error_msg =
+        "Could not find `kubectl` installation and this build crate cannot proceed without
+    this knowledge. If `kubectl` is installed and this crate had trouble finding
+    it, you can set the `KUBECTL` environment variable with the specific path to your
+    installed `kubectl` binary.";
+    let msg = format!("{error_msg} {os_specific_hint}");
+
+    env::var_os("KUBECTL")
+        .map(PathBuf::from)
+        .or_else(|| which::which("kubectl").ok())
+        .expect(&msg)
+}
+
+#[test]
+fn run_kustomize() {
+    let kubectl = kubectl_from_env();
+    let out_file = tempfile::NamedTempFile::new()
+        .expect("failed creating a temporary file to store the output of kubectl kustomize");
+    let mut cmd = Command::new(kubectl.clone());
+    cmd.current_dir(env!("CARGO_MANIFEST_DIR"))
+        .arg("kustomize")
+        .arg(KUSTOMIZE_DIR)
+        .arg("-o")
+        .arg(out_file.path());
+
+    match cmd.output() {
+        Err(e) => {
+            eprintln!("failed to invoke kubectl (path: {kubectl:?}): {e:?}");
+            exit(e.raw_os_error().unwrap_or(-1));
+        }
+        Ok(output) if !output.status.success() => {
+            eprintln!(
+                "kubectl kustomize failed: {}",
+                String::from_utf8_lossy(&output.stderr)
+            );
+            exit(output.status.code().unwrap_or(-1));
+        }
+        Ok(_) => {}
+    }
+
+    let before = read_content(TEST_ENV_KUBE_YAML);
+    let after = read_content(out_file.path());
+
+    ensure_files_are_same(before, after);
+}
+
+fn read_content(path: impl AsRef<Path>) -> String {
+    fs::read_to_string(path).expect("cannot read from existing generated kube yml file")
+}
+
+fn ensure_files_are_same(before: String, after: String) {
+    if before == after {
+        return;
+    }
+
+    if env::var("CI").is_ok() {
+        panic!("generated kube yaml file has changed but it's a CI environment; please rerun this test locally and commit the changes");
+    }
+
+    fs::write(TEST_ENV_KUBE_YAML, after)
+        .expect("cannot write generated kube yaml file to its target; if this is happening in a CI environment rerun the test locally and commit the changes");
+
+    panic!("generated file has changed; commit the changed files and rerun the test");
+}

sequencer-relayer/Cargo.toml

@@ -13,7 +13,6 @@ ed25519-dalek = "1.0"
 eyre = "0.6"
 hex = "0.4"
 prost = "0.11"
-prost-types = "0.11"
 rand = { version = "0.7", features = [ "getrandom" ] }
 reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] }
 serde = { version = "1.0", features = ["derive"] }
@@ -27,14 +26,12 @@ axum = "0.6.16"
 http = "0.2.9"
 tokio = { version = "1.24", features = [ "macros", "rt-multi-thread" ] }
 
+sequencer-relayer-proto = { path = "../sequencer-relayer-proto" }
+
 [dependencies.rs-cnc]
 git = "https://github.com/astriaorg/rs-cnc.git"
 default-features = false
 features = ["rustls"]
 
-[build-dependencies]
-const_format = "0.2.30"
-prost-build = "0.11"
-
 [dev-dependencies]
 sequencer-relayer-test = { path = "../sequencer-relayer-test"}

sequencer-relayer/src/lib.rs

@@ -9,7 +9,3 @@ pub mod sequencer_block;
 pub mod tests;
 pub mod transaction;
 pub mod types;
-
-pub mod proto {
-    include!(concat!(env!("OUT_DIR"), "/_.rs"));
-}

sequencer-relayer/src/sequencer_block.rs

@@ -7,9 +7,9 @@ use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use tracing::debug;
 
+use sequencer_relayer_proto::{SequencerMsg, TxBody, TxRaw};
+
 use crate::base64_string::Base64String;
-use crate::proto::SequencerMsg;
-use crate::proto::{TxBody, TxRaw};
 use crate::transaction::txs_to_data_hash;
 use crate::types::{Block, Header};