chore: Use cargo workspaces (#102)

Signed-off-by: Sreekanth <prsreekanth920@gmail.com>

.dockerignore

@@ -1,2 +1 @@
 target/
-**/target/

.github/workflows/ci.yaml

@@ -27,17 +27,6 @@ jobs:
         with:
           cache-workspaces: |
             . -> target
-            ./examples/batchmap-cat -> target
-            ./examples/batchmap-flatmap -> target
-            ./examples/map-cat -> target
-            ./examples/map-tickgen-serde -> target
-            ./examples/mapt-event-time-filter -> target
-            ./examples/reduce-counter -> target
-            ./examples/sideinput -> target
-            ./examples/sideinput/udf -> target
-            ./examples/simple-source -> target
-            ./examples/sink-log -> target
-            ./examples/source-transformer-now -> target
 
       - name: Configure sccache
         run: |
@@ -65,5 +54,6 @@ jobs:
       - name: Documentation generation
         run: RUSTFLAGS="-D warnings" cargo doc
 
+      # To view the list of files included in the package: cargo package -p numaflow --list
       - name: Dry run Cargo Publish
-        run: cargo publish --dry-run
+        run: cargo publish -p numaflow --dry-run

.github/workflows/release.yaml

@@ -42,5 +42,6 @@ jobs:
       - name: Documentation generation
         run: RUSTFLAGS="-D warnings" cargo doc
 
+      # To view the list of files included in the package: cargo package -p numaflow --list
       - name: Dry run Cargo Publish
-        run: CARGO_REGISTRY_TOKEN=${{ secrets.CARGO_PUBLISH }} cargo publish
+        run: CARGO_REGISTRY_TOKEN=${{ secrets.CARGO_PUBLISH }} cargo publish -p numaflow

Cargo.toml

@@ -1,48 +1,8 @@
-[package]
-name = "numaflow"
-version = "0.2.0"
-edition = "2021"
-rust-version = "1.80"
-description = "Rust SDK for Numaflow"
-authors = ["Numaflow Developers"]
-license = "Apache-2.0"
-readme = "README.md"
-homepage = "https://numaproj.github.io/numaflow/"
-repository = "https://github.com/numaproj/numaflow-rs"
-keywords = ["numaflow", "streaming", "messaging", "event-driven"]
-categories = ["network-programming", "api-bindings"]
-exclude = [
-    ".github/*",
-    ".gitignore",
-    ".dockerignore",
-    "hack/*",
-    "Makefile",
-]
-
-
-[lib]
-name = "numaflow"
-path = "src/lib.rs"
-
-[dependencies]
-tonic = "0.12.3"
-prost = "0.13.3"
-prost-types = "0.13.3"
-tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "signal"] }
-tokio-util = "0.7.12"
-tokio-stream = { version = "0.1.16", features = ["net"] }
-serde = { version = "1.0.210", features = ["derive"] }
-chrono = "0.4.38"
-serde_json = "1.0.128"
-futures-util = "0.3.30"
-tracing = "0.1.40"
-uuid = { version = "1.10.0", features = ["v4"] }
-thiserror = "1.0"
-hyper-util = "0.1.7"
-
-[build-dependencies]
-tonic-build = "0.12.3"
-
-[dev-dependencies]
-tempfile = "3.9.0"
-tower = { version = "0.5.0", features = ["util"] }
+[workspace]
+members = ["numaflow", "examples/*"]
+# Only check / build main crate by default (check all with `--workspace`)
+default-members = ["numaflow"]
+resolver = "2"
+
+[workspace.package]
+rust-version = "1.82"

Makefile

@@ -1,33 +1,18 @@
-# Description: Makefile for Rust projects
+.PHONY: lint fmt test-fmt clippy test codegen
 
-# perform a cargo fmt on all directories containing a Cargo.toml file
-.PHONY: lint
-# find all directories containing Cargo.toml files
-DIRS := $(shell find . -type f -name Cargo.toml -not -path "./target/*" -exec dirname {} \; | sort -u)
-$(info Included directories: $(DIRS))
 fmt:
-	@for dir in $(DIRS); do \
-		echo "Formatting code in $$dir"; \
-		cargo fmt --all --manifest-path "$$dir/Cargo.toml"; \
-	done
+	cargo fmt --all
 
-# Check if all files are formatted and run clippy on all directories containing a Cargo.toml file
 .PHONY: lint
 lint: test-fmt clippy
 
 .PHONY: test-fmt
 test-fmt:
-	@for dir in $(DIRS); do \
-		echo "Checking if code is formatted in directory: $$dir"; \
-		cargo fmt --all --check --manifest-path "$$dir/Cargo.toml" || { echo "Code is not formatted in $$dir"; exit 1; }; \
-	done
+	cargo fmt --all --check
 
 .PHONY: clippy
 clippy:
-	@for dir in $(DIRS); do \
-		echo "Running clippy in directory: $$dir"; \
-		cargo clippy --workspace --manifest-path "$$dir/Cargo.toml" -- -D warnings || { echo "Clippy warnings/errors found in $$dir"; exit 1; }; \
-	done
+	cargo clippy --workspace -- -D warnings
 
 # run cargo test on the repository root
 .PHONY: test
@@ -37,5 +22,4 @@ test:
 .PHONY: codegen
 codegen:
 	# Change timestamps so that tonic_build code generation will always be triggered.
-	touch proto/*
-	PROTO_CODE_GEN=1 cargo build
+	cd numaflow && mkdir -p src/servers && touch proto/* && PROTO_CODE_GEN=1 cargo build

README.md

@@ -0,0 +1 @@
+numaflow/README.md

examples/batchmap-cat/Cargo.toml

@@ -3,12 +3,7 @@ name = "batchmap-cat"
 version = "0.1.0"
 edition = "2021"
 
-
-[[bin]]
-name = "server"
-path = "src/main.rs"
-
 [dependencies]
 tonic = "0.12.0"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
-numaflow = { path = "../../" }
+numaflow = { path = "../../numaflow" }

examples/batchmap-cat/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.80-bookworm AS build
+FROM rust:1.82-bullseye AS build
 
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
@@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/batchmap-cat
 RUN cargo build --release
 
 # our final base
-FROM debian:bookworm AS batchmap-cat
+FROM debian:bullseye AS batchmap-cat
 
 # copy the build artifact from the build stage
-COPY --from=build /numaflow-rs/examples/batchmap-cat/target/release/server .
+COPY --from=build /numaflow-rs/target/release/batchmap-cat .
 
 # set the startup command to run your binary
-CMD ["./server"]
+CMD ["./batchmap-cat"]

examples/batchmap-cat/Makefile

@@ -14,7 +14,3 @@ image: update
 	-f ${DOCKER_FILE_PATH}  \
 	-t ${IMAGE_REGISTRY} .
 	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi
-
-.PHONY: clean
-clean:
-	-rm -rf target

examples/batchmap-flatmap/Cargo.toml

@@ -11,4 +11,4 @@ path = "src/main.rs"
 [dependencies]
 tonic = "0.12.0"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
-numaflow = { path = "../../" }
+numaflow = { path = "../../numaflow" }

examples/batchmap-flatmap/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.80-bookworm AS build
+FROM rust:1.82-bullseye AS build
 
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
@@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/batchmap-flatmap
 RUN cargo build --release
 
 # our final base
-FROM debian:bookworm AS batchmap-flatmap
+FROM debian:bullseye AS batchmap-flatmap
 
 # copy the build artifact from the build stage
-COPY --from=build /numaflow-rs/examples/batchmap-flatmap/target/release/server .
+COPY --from=build /numaflow-rs/target/release/batchmap-flatmap .
 
 # set the startup command to run your binary
-CMD ["./server"]
+CMD ["./batchmap-flatmap"]

examples/batchmap-flatmap/Makefile

@@ -14,7 +14,3 @@ image: update
 	-f ${DOCKER_FILE_PATH}  \
 	-t ${IMAGE_REGISTRY} .
 	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi
-
-.PHONY: clean
-clean:
-	-rm -rf target

examples/map-cat/Cargo.toml

@@ -3,12 +3,7 @@ name = "map-cat"
 version = "0.1.0"
 edition = "2021"
 
-
-[[bin]]
-name = "server"
-path = "src/main.rs"
-
 [dependencies]
 tonic = "0.12.0"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
-numaflow = { path = "../../" }
+numaflow = { path = "../../numaflow" }

examples/map-cat/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.80-bookworm AS build
+FROM rust:1.82-bullseye AS build
 
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
@@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/map-cat
 RUN cargo build --release
 
 # our final base
-FROM debian:bookworm AS map-cat
+FROM debian:bullseye AS map-cat
 
 # copy the build artifact from the build stage
-COPY --from=build /numaflow-rs/examples/map-cat/target/release/server .
+COPY --from=build /numaflow-rs/target/release/map-cat .
 
 # set the startup command to run your binary
-CMD ["./server"]
+CMD ["./map-cat"]

examples/map-tickgen-serde/Cargo.toml

@@ -3,14 +3,10 @@ name = "map-tickgen-serde"
 version = "0.1.0"
 edition = "2021"
 
-[[bin]]
-name = "server"
-path = "src/main.rs"
-
 [dependencies]
 tonic = "0.12.0"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
 serde = { version = "1.0.103", features = ["derive"] }
 serde_json = "1.0.103"
 chrono = "0.4.26"
-numaflow = { path = "../../" }
+numaflow = { path = "../../numaflow" }

examples/map-tickgen-serde/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.80-bookworm AS build
+FROM rust:1.82-bullseye AS build
 
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
@@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/map-tickgen-serde
 RUN cargo build --release
 
 # our final base
-FROM debian:bookworm AS map-tickgen-serde
+FROM debian:bullseye AS map-tickgen-serde
 
 # copy the build artifact from the build stage
-COPY --from=build /numaflow-rs/examples/map-tickgen-serde/target/release/server .
+COPY --from=build /numaflow-rs/target/release/map-tickgen-serde .
 
 # set the startup command to run your binary
-CMD ["./server"]
+CMD ["./map-tickgen-serde"]

examples/map-tickgen-serde/Makefile

@@ -14,7 +14,3 @@ image: update
 	-f ${DOCKER_FILE_PATH} \
 	-t ${IMAGE_REGISTRY} .
 	@if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi
-
-.PHONY: clean
-clean:
-	-rm -rf target

examples/mapt-event-time-filter/Cargo.toml

@@ -3,12 +3,8 @@ name = "source-transformer-event-time-filter"
 version = "0.1.0"
 edition = "2021"
 
-[[bin]]
-name = "server"
-path = "src/main.rs"
-
 [dependencies]
-numaflow = { path = "../../" }
+numaflow = { path = "../../numaflow" }
 tokio = "1.38.0"
 chrono = "0.4.38"
-tonic = "0.12.0"
+tonic = "0.12.0"

examples/mapt-event-time-filter/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.80-bookworm AS build
+FROM rust:1.82-bullseye AS build
 
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
@@ -11,10 +11,10 @@ WORKDIR /numaflow-rs/examples/mapt-event-time-filter
 RUN cargo build --release
 
 # our final base
-FROM debian:bookworm AS mapt-event-time-filter
+FROM debian:bullseye AS mapt-event-time-filter
 
 # copy the build artifact from the build stage
-COPY --from=build /numaflow-rs/examples/mapt-event-time-filter/target/release/server .
+COPY --from=build /numaflow-rs/target/release/mapt-event-time-filter .
 
 # set the startup command to run your binary
-CMD ["./server"]
+CMD ["./mapt-event-time-filter"]

examples/mapt-event-time-filter/src/main.rs

@@ -43,7 +43,7 @@ mod filter_impl {
 mod tests {
     use crate::filter_impl::filter_event_time;
     use chrono::{TimeZone, Utc};
-    use numaflow::sourcetransform::{Message, SourceTransformRequest};
+    use numaflow::sourcetransform::SourceTransformRequest;
     /// Tests that events from 2022 are tagged as within the year 2022.
     #[test]
     fn test_filter_event_time_should_return_within_year_2022() {

examples/reduce-counter/Cargo.toml

@@ -3,12 +3,7 @@ name = "reduce-counter"
 version = "0.1.0"
 edition = "2021"
 
-[[bin]]
-name = "server"
-path = "src/main.rs"
-
 [dependencies]
 tonic = "0.12.0"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
-numaflow = { path = "../../" }
-
+numaflow = { path = "../../numaflow" }