From 0a465e55dd39c76bc1aefb020460a639d792fe87 Mon Sep 17 00:00:00 2001 From: Shadaj Laddad Date: Thu, 22 Aug 2024 11:15:34 -0700 Subject: [PATCH] refactor(hydro_deploy)!: rename integration crates to drop CLI references (#1413) --- .github/workflows/release.yml | 4 +- .idea/hydroflow.iml | 4 +- Cargo.lock | 51 ++++++------- Cargo.toml | 4 +- RELEASING.md | 4 +- docs/docs/deploy/your-first-deploy.md | 12 ++-- .../hydroflow_plus/quickstart/clusters.mdx | 2 +- hydro_deploy/core/Cargo.toml | 2 +- hydro_deploy/core/src/custom_service.rs | 2 +- hydro_deploy/core/src/hydroflow_crate/mod.rs | 2 +- .../core/src/hydroflow_crate/ports.rs | 2 +- .../core/src/hydroflow_crate/service.rs | 6 +- hydro_deploy/core/src/lib.rs | 8 +-- .../core/src/localhost/launched_binary.rs | 12 ++-- hydro_deploy/core/src/localhost/mod.rs | 2 +- hydro_deploy/core/src/ssh.rs | 14 ++-- hydro_deploy/core/src/util.rs | 14 ++-- hydro_deploy/hydro_cli/Cargo.toml | 2 +- hydro_deploy/hydro_cli/src/lib.rs | 4 +- hydro_deploy/hydro_cli_examples/Cargo.toml | 2 +- .../examples/dedalus_2pc_coordinator/main.rs | 4 +- .../examples/dedalus_2pc_participant/main.rs | 4 +- .../examples/dedalus_receiver/main.rs | 6 +- .../examples/dedalus_sender/main.rs | 6 +- .../examples/dedalus_vote_leader/main.rs | 4 +- .../examples/dedalus_vote_participant/main.rs | 4 +- .../examples/empty_program/main.rs | 2 +- .../examples/panic_program/main.rs | 2 +- .../examples/stdout_receiver/main.rs | 6 +- .../examples/tagged_stdout_receiver/main.rs | 6 +- .../examples/ws_chat_server/main.rs | 10 +-- .../CHANGELOG.md | 0 .../Cargo.toml | 6 +- .../src/lib.rs | 0 .../CHANGELOG.md | 71 +++++++++++++++++-- .../Cargo.toml | 6 +- .../build.rs | 0 .../src/deploy.rs | 0 .../src/deploy_runtime.rs | 16 ++--- .../src/lib.rs | 0 .../src/runtime.rs | 6 +- .../src/trybuild.rs | 6 +- hydroflow/Cargo.toml | 4 +- hydroflow/src/util/{cli.rs => deploy.rs} | 18 ++--- hydroflow/src/util/mod.rs | 4 +- hydroflow_plus/Cargo.toml | 5 +- hydroflow_plus/src/profiler.rs | 1 - hydroflow_plus_test/Cargo.toml | 5 +- hydroflow_plus_test/examples/compute_pi.rs | 2 +- .../examples/first_ten_distributed.rs | 2 +- hydroflow_plus_test/examples/map_reduce.rs | 2 +- .../examples/perf_compute_pi.rs | 2 +- .../examples/simple_cluster.rs | 2 +- hydroflow_plus_test/src/cluster/compute_pi.rs | 3 +- .../src/cluster/many_to_many.rs | 2 +- hydroflow_plus_test/src/cluster/map_reduce.rs | 3 +- .../src/cluster/simple_cluster.rs | 2 +- ..._tests__compute_pi_ir@surface_graph_0.snap | 2 +- ..._tests__compute_pi_ir@surface_graph_1.snap | 2 +- ..._tests__map_reduce_ir@surface_graph_0.snap | 4 +- ..._tests__map_reduce_ir@surface_graph_1.snap | 4 +- .../src/distributed/first_ten.rs | 2 +- hydroflow_plus_test_local/Cargo.toml | 7 +- hydroflow_plus_test_local_macro/Cargo.toml | 3 - scripts/build_dist_release.sh | 2 +- template/hydroflow_plus/Cargo.toml | 4 +- .../examples/first_ten_distributed.rs | 2 +- .../examples/first_ten_distributed_gcp.rs | 2 +- .../src/first_ten_distributed.rs | 2 +- topolotree/Cargo.toml | 2 +- topolotree/src/latency_measure.rs | 4 +- topolotree/src/main.rs | 6 +- topolotree/src/pn.rs | 6 +- topolotree/src/pn_delta.rs | 6 +- 74 files changed, 237 insertions(+), 198 deletions(-) rename hydro_deploy/{hydroflow_cli_integration => hydroflow_deploy_integration}/CHANGELOG.md (100%) rename hydro_deploy/{hydroflow_cli_integration => hydroflow_deploy_integration}/Cargo.toml (82%) rename hydro_deploy/{hydroflow_cli_integration => hydroflow_deploy_integration}/src/lib.rs (100%) rename hydro_deploy/{hydroflow_plus_cli_integration => hydroflow_plus_deploy}/CHANGELOG.md (80%) rename hydro_deploy/{hydroflow_plus_cli_integration => hydroflow_plus_deploy}/Cargo.toml (88%) rename hydro_deploy/{hydroflow_plus_cli_integration => hydroflow_plus_deploy}/build.rs (100%) rename hydro_deploy/{hydroflow_plus_cli_integration => hydroflow_plus_deploy}/src/deploy.rs (100%) rename hydro_deploy/{hydroflow_plus_cli_integration => hydroflow_plus_deploy}/src/deploy_runtime.rs (87%) rename hydro_deploy/{hydroflow_plus_cli_integration => hydroflow_plus_deploy}/src/lib.rs (100%) rename hydro_deploy/{hydroflow_plus_cli_integration => hydroflow_plus_deploy}/src/runtime.rs (98%) rename hydro_deploy/{hydroflow_plus_cli_integration => hydroflow_plus_deploy}/src/trybuild.rs (94%) rename hydroflow/src/util/{cli.rs => deploy.rs} (85%) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 24ca5f8a5d9..630a3276f0d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -88,8 +88,8 @@ jobs: ${{ inputs.execute && '--execute' || '--no-publish' }} hydroflow hydroflow_lang hydroflow_macro hydroflow_plus hydroflow_datalog hydroflow_datalog_core - hydro_deploy hydro_cli hydroflow_cli_integration - hydroflow_plus_cli_integration + hydro_deploy hydro_cli hydroflow_deploy_integration + hydroflow_plus_deploy stageleft stageleft_macro stageleft_tool multiplatform_test env: diff --git a/.idea/hydroflow.iml b/.idea/hydroflow.iml index aa287283f22..fc353ac331a 100644 --- a/.idea/hydroflow.iml +++ b/.idea/hydroflow.iml @@ -7,8 +7,8 @@ - - + + diff --git a/Cargo.lock b/Cargo.lock index 3e74cce13c5..a7a0fd20af0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1417,7 +1417,7 @@ dependencies = [ "clap", "futures", "hydro_deploy", - "hydroflow_cli_integration", + "hydroflow_deploy_integration", "pyo3", "pyo3-asyncio", "pythonize", @@ -1456,7 +1456,7 @@ dependencies = [ "dyn-clone", "futures", "futures-core", - "hydroflow_cli_integration", + "hydroflow_deploy_integration", "indicatif", "inferno", "memo-map", @@ -1487,8 +1487,8 @@ dependencies = [ "futures", "getrandom 0.2.11", "hdrhistogram", - "hydroflow_cli_integration", "hydroflow_datalog", + "hydroflow_deploy_integration", "hydroflow_lang", "hydroflow_macro", "insta", @@ -1522,22 +1522,6 @@ dependencies = [ "zipf", ] -[[package]] -name = "hydroflow_cli_integration" -version = "0.5.2" -dependencies = [ - "async-recursion", - "async-trait", - "bytes", - "futures", - "pin-project", - "serde", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util", -] - [[package]] name = "hydroflow_datalog" version = "0.8.0" @@ -1566,6 +1550,22 @@ dependencies = [ "tempfile", ] +[[package]] +name = "hydroflow_deploy_integration" +version = "0.5.2" +dependencies = [ + "async-recursion", + "async-trait", + "bytes", + "futures", + "pin-project", + "serde", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", +] + [[package]] name = "hydroflow_lang" version = "0.8.0" @@ -1602,7 +1602,6 @@ name = "hydroflow_plus" version = "0.8.0" dependencies = [ "bincode", - "dyn-clone", "hydroflow", "hydroflow_lang", "insta", @@ -1616,7 +1615,7 @@ dependencies = [ ] [[package]] -name = "hydroflow_plus_cli_integration" +name = "hydroflow_plus_deploy" version = "0.8.0" dependencies = [ "hydro_deploy", @@ -1638,9 +1637,8 @@ version = "0.0.0" dependencies = [ "futures", "hydro_deploy", - "hydroflow", "hydroflow_plus", - "hydroflow_plus_cli_integration", + "hydroflow_plus_deploy", "insta", "rand 0.8.5", "serde", @@ -1654,17 +1652,13 @@ name = "hydroflow_plus_test_local" version = "0.0.0" dependencies = [ "futures", - "hydro_deploy", "hydroflow", "hydroflow_plus", - "hydroflow_plus_cli_integration", "hydroflow_plus_test_local_macro", "insta", "rand 0.8.5", - "serde", "stageleft", "stageleft_tool", - "tokio", ] [[package]] @@ -1672,12 +1666,9 @@ name = "hydroflow_plus_test_local_macro" version = "0.0.0" dependencies = [ "hydroflow_plus", - "hydroflow_plus_cli_integration", "rand 0.8.5", - "serde", "stageleft", "stageleft_tool", - "tokio", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2e0b8a88e60..53af7d2e836 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,8 @@ members = [ "hydro_deploy/core", "hydro_deploy/hydro_cli", "hydro_deploy/hydro_cli_examples", - "hydro_deploy/hydroflow_cli_integration", - "hydro_deploy/hydroflow_plus_cli_integration", + "hydro_deploy/hydroflow_deploy_integration", + "hydro_deploy/hydroflow_plus_deploy", "hydroflow", "hydroflow_datalog", "hydroflow_datalog_core", diff --git a/RELEASING.md b/RELEASING.md index 97875c5e5ac..3fdfa932f21 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -41,8 +41,8 @@ showing that all the changelogs can be modified. Make sure the version bumps loo ```log [INFO ] Updating crates-io index -[WARN ] Refused to publish 'hydroflow_cli_integration' as as it didn't change. -[INFO ] Will not publish or alter 3 dependent crates: unchanged = 'hydroflow_cli_integration', 'variadics', 'pusherator' +[WARN ] Refused to publish 'hydroflow_deploy_integration' as as it didn't change. +[INFO ] Will not publish or alter 3 dependent crates: unchanged = 'hydroflow_deploy_integration', 'variadics', 'pusherator' [INFO ] WOULD auto-bump dependent package 'hydroflow_lang' from 0.4.0 to 0.5.0 for publishing [INFO ] WOULD auto-bump dependent package 'hydroflow_datalog_core' from 0.4.0 to 0.5.0 for publishing, for SAFETY due to breaking package 'hydroflow_lang' [INFO ] WOULD auto-bump dependent package 'hydroflow_datalog' from 0.4.0 to 0.5.0 for publishing, for SAFETY due to breaking package 'hydroflow_datalog_core' diff --git a/docs/docs/deploy/your-first-deploy.md b/docs/docs/deploy/your-first-deploy.md index 06084f58a9f..1c727172341 100644 --- a/docs/docs/deploy/your-first-deploy.md +++ b/docs/docs/deploy/your-first-deploy.md @@ -26,12 +26,12 @@ rustup update cargo test ``` -We'll need to add an additional dependency for `hydroflow_cli_integration` to our `Cargo.toml`: +We'll need to add an additional dependency for `hydroflow_deploy_integration` to our `Cargo.toml`: ```toml [dependencies] # ... -hydroflow_cli_integration = "0.1.1" +hydroflow_deploy_integration = "0.1.1" ``` Let's open up `src/main.rs` in the generated project and write a new `main` function that initializes Hydro Deploy: @@ -39,7 +39,7 @@ Let's open up `src/main.rs` in the generated project and write a new `main` func ```rust #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init().await; + let ports = hydroflow::util::deploy::init().await; } ``` @@ -77,12 +77,12 @@ Now, we need to wire up the ports. Hydro Deploy uses _named ports_, which can th Returning briefly to our Hydroflow code, we can then load these ports and use them to send and receive packets: ```rust -use hydroflow_cli_integration::ConnectedDirect; +use hydroflow_deploy_integration::ConnectedDirect; use hydroflow::hydroflow_syntax; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init().await; + let ports = hydroflow::util::deploy::init().await; let input_recv = ports .port("input") @@ -97,7 +97,7 @@ async fn main() { .await .into_sink(); - hydroflow::util::cli::launch_flow(hydroflow_syntax! { + hydroflow::util::deploy::launch_flow(hydroflow_syntax! { source_iter(["hello".to_string()]) -> dest_sink(output_send); input = source_stream(input_recv) -> tee(); input -> dest_sink(output_send); diff --git a/docs/docs/hydroflow_plus/quickstart/clusters.mdx b/docs/docs/hydroflow_plus/quickstart/clusters.mdx index e045da0f7df..889b92ef2b9 100644 --- a/docs/docs/hydroflow_plus/quickstart/clusters.mdx +++ b/docs/docs/hydroflow_plus/quickstart/clusters.mdx @@ -59,7 +59,7 @@ To deploy this application, we must set up the Hydro Deploy configuration as bef use std::cell::RefCell; use hydro_deploy::{Deployment, HydroflowCrate}; -use hydroflow_plus_cli_integration::TrybuildHost; +use hydroflow_plus_deploy::TrybuildHost; #[tokio::main] async fn main() { diff --git a/hydro_deploy/core/Cargo.toml b/hydro_deploy/core/Cargo.toml index 3622845bef1..4ff7ae5612a 100644 --- a/hydro_deploy/core/Cargo.toml +++ b/hydro_deploy/core/Cargo.toml @@ -20,7 +20,7 @@ dunce = "1.0.4" dyn-clone = "1" futures = "0.3.26" futures-core = "0.3.26" -hydroflow_cli_integration = { path = "../hydroflow_cli_integration", version = "^0.5.2" } +hydroflow_deploy_integration = { path = "../hydroflow_deploy_integration", version = "^0.5.2" } indicatif = "0.17.8" inferno = "0.11.20" memo-map = "0.3.2" diff --git a/hydro_deploy/core/src/custom_service.rs b/hydro_deploy/core/src/custom_service.rs index f0eb4db5a3d..947b2e8410f 100644 --- a/hydro_deploy/core/src/custom_service.rs +++ b/hydro_deploy/core/src/custom_service.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, OnceLock, Weak}; use anyhow::{bail, Result}; use async_trait::async_trait; -use hydroflow_cli_integration::{ConnectedDirect, ServerPort}; +use hydroflow_deploy_integration::{ConnectedDirect, ServerPort}; use tokio::sync::RwLock; use super::hydroflow_crate::ports::{ diff --git a/hydro_deploy/core/src/hydroflow_crate/mod.rs b/hydro_deploy/core/src/hydroflow_crate/mod.rs index a528c26a298..668d0ab0a4b 100644 --- a/hydro_deploy/core/src/hydroflow_crate/mod.rs +++ b/hydro_deploy/core/src/hydroflow_crate/mod.rs @@ -21,7 +21,7 @@ pub enum CrateTarget { Example(String), } -/// Specifies a crate that uses `hydroflow_cli_integration` to be +/// Specifies a crate that uses `hydroflow_deploy_integration` to be /// deployed as a service. #[derive(Clone)] pub struct HydroflowCrate { diff --git a/hydro_deploy/core/src/hydroflow_crate/ports.rs b/hydro_deploy/core/src/hydroflow_crate/ports.rs index b4fe7248fb1..2d346ba61b4 100644 --- a/hydro_deploy/core/src/hydroflow_crate/ports.rs +++ b/hydro_deploy/core/src/hydroflow_crate/ports.rs @@ -7,7 +7,7 @@ use anyhow::Result; use async_recursion::async_recursion; use async_trait::async_trait; use dyn_clone::DynClone; -use hydroflow_cli_integration::ServerPort; +use hydroflow_deploy_integration::ServerPort; use tokio::sync::RwLock; use super::HydroflowCrateService; diff --git a/hydro_deploy/core/src/hydroflow_crate/service.rs b/hydro_deploy/core/src/hydroflow_crate/service.rs index fa1e2e0c59f..372a1cddcdd 100644 --- a/hydro_deploy/core/src/hydroflow_crate/service.rs +++ b/hydro_deploy/core/src/hydroflow_crate/service.rs @@ -6,7 +6,7 @@ use std::time::Duration; use anyhow::{bail, Result}; use async_trait::async_trait; use futures_core::Future; -use hydroflow_cli_integration::{InitConfig, ServerPort}; +use hydroflow_deploy_integration::{InitConfig, ServerPort}; use serde::Serialize; use tokio::sync::{mpsc, RwLock}; @@ -254,7 +254,7 @@ impl Service for HydroflowCrateService { serde_json::to_string::(&(bind_config, self.meta.clone())).unwrap(); // request stdout before sending config so we don't miss the "ready" response - let stdout_receiver = binary.cli_stdout(); + let stdout_receiver = binary.deploy_stdout(); binary.stdin().send(format!("{formatted_bind_config}\n"))?; @@ -290,7 +290,7 @@ impl Service for HydroflowCrateService { let formatted_defns = serde_json::to_string(&sink_ports).unwrap(); - let stdout_receiver = self.launched_binary.as_ref().unwrap().cli_stdout(); + let stdout_receiver = self.launched_binary.as_ref().unwrap().deploy_stdout(); self.launched_binary .as_ref() diff --git a/hydro_deploy/core/src/lib.rs b/hydro_deploy/core/src/lib.rs index 4f147deb604..e91f65bc287 100644 --- a/hydro_deploy/core/src/lib.rs +++ b/hydro_deploy/core/src/lib.rs @@ -4,8 +4,8 @@ use std::sync::Arc; use anyhow::Result; use async_trait::async_trait; -use hydroflow_cli_integration::ServerBindConfig; use hydroflow_crate::perf_options::PerfOptions; +use hydroflow_deploy_integration::ServerBindConfig; pub mod deployment; pub use deployment::Deployment; @@ -74,11 +74,11 @@ pub struct ResourceResult { pub trait LaunchedBinary: Send + Sync { fn stdin(&self) -> mpsc::UnboundedSender; - /// Provides a oneshot channel for the CLI to handshake with the binary, - /// with the guarantee that as long as the CLI is holding on + /// Provides a oneshot channel to handshake with the binary, + /// with the guarantee that as long as deploy is holding on /// to a handle, none of the messages will also be broadcast /// to the user-facing [`LaunchedBinary::stdout`] channel. - fn cli_stdout(&self) -> oneshot::Receiver; + fn deploy_stdout(&self) -> oneshot::Receiver; fn stdout(&self) -> mpsc::UnboundedReceiver; fn stderr(&self) -> mpsc::UnboundedReceiver; diff --git a/hydro_deploy/core/src/localhost/launched_binary.rs b/hydro_deploy/core/src/localhost/launched_binary.rs index 86c81595009..04ea5215dec 100644 --- a/hydro_deploy/core/src/localhost/launched_binary.rs +++ b/hydro_deploy/core/src/localhost/launched_binary.rs @@ -16,7 +16,7 @@ use crate::LaunchedBinary; pub struct LaunchedLocalhostBinary { child: Mutex, stdin_sender: mpsc::UnboundedSender, - stdout_cli_receivers: Arc>>>, + stdout_deploy_receivers: Arc>>>, stdout_receivers: Arc>>>, stderr_receivers: Arc>>>, } @@ -55,7 +55,7 @@ impl LaunchedLocalhostBinary { }); let id_clone = id.clone(); - let (stdout_cli_receivers, stdout_receivers) = prioritized_broadcast( + let (stdout_deploy_receivers, stdout_receivers) = prioritized_broadcast( BufReader::new(child.stdout.take().unwrap()).lines(), move |s| ProgressTracker::println(format!("[{id_clone}] {s}")), ); @@ -67,7 +67,7 @@ impl LaunchedLocalhostBinary { Self { child: Mutex::new(child), stdin_sender, - stdout_cli_receivers, + stdout_deploy_receivers, stdout_receivers, stderr_receivers, } @@ -80,11 +80,11 @@ impl LaunchedBinary for LaunchedLocalhostBinary { self.stdin_sender.clone() } - fn cli_stdout(&self) -> oneshot::Receiver { - let mut receivers = self.stdout_cli_receivers.lock().unwrap(); + fn deploy_stdout(&self) -> oneshot::Receiver { + let mut receivers = self.stdout_deploy_receivers.lock().unwrap(); if receivers.is_some() { - panic!("Only one CLI stdout receiver is allowed at a time"); + panic!("Only one deploy stdout receiver is allowed at a time"); } let (sender, receiver) = oneshot::channel::(); diff --git a/hydro_deploy/core/src/localhost/mod.rs b/hydro_deploy/core/src/localhost/mod.rs index 0f355665cf0..9a7d7dcc1aa 100644 --- a/hydro_deploy/core/src/localhost/mod.rs +++ b/hydro_deploy/core/src/localhost/mod.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use anyhow::{Context, Result}; use async_process::{Command, Stdio}; use async_trait::async_trait; -use hydroflow_cli_integration::ServerBindConfig; +use hydroflow_deploy_integration::ServerBindConfig; use super::{ ClientStrategy, Host, HostTargetType, LaunchedBinary, LaunchedHost, ResourceBatch, diff --git a/hydro_deploy/core/src/ssh.rs b/hydro_deploy/core/src/ssh.rs index 6e4fcd5f822..26eb7bbe780 100644 --- a/hydro_deploy/core/src/ssh.rs +++ b/hydro_deploy/core/src/ssh.rs @@ -14,7 +14,7 @@ use async_trait::async_trait; use futures::io::BufReader as FuturesBufReader; use futures::stream::FuturesUnordered; use futures::{AsyncBufReadExt, AsyncWriteExt}; -use hydroflow_cli_integration::ServerBindConfig; +use hydroflow_deploy_integration::ServerBindConfig; use inferno::collapse::perf::Folder; use inferno::collapse::Collapse; use nanoid::nanoid; @@ -40,7 +40,7 @@ struct LaunchedSshBinary { channel: AsyncChannel, stdin_sender: mpsc::UnboundedSender, stdout_receivers: Arc>>>, - stdout_cli_receivers: Arc>>>, + stdout_deploy_receivers: Arc>>>, stderr_receivers: Arc>>>, perf: Option, } @@ -51,11 +51,11 @@ impl LaunchedBinary for LaunchedSshBinary { self.stdin_sender.clone() } - fn cli_stdout(&self) -> oneshot::Receiver { - let mut receivers = self.stdout_cli_receivers.lock().unwrap(); + fn deploy_stdout(&self) -> oneshot::Receiver { + let mut receivers = self.stdout_deploy_receivers.lock().unwrap(); if receivers.is_some() { - panic!("Only one CLI stdout receiver is allowed at a time"); + panic!("Only one deploy stdout receiver is allowed at a time"); } let (sender, receiver) = oneshot::channel::(); @@ -469,7 +469,7 @@ impl LaunchedHost for T { }); let id_clone = id.clone(); - let (stdout_cli_receivers, stdout_receivers) = + let (stdout_deploy_receivers, stdout_receivers) = prioritized_broadcast(FuturesBufReader::new(channel.stream(0)).lines(), move |s| { ProgressTracker::println(format!("[{id_clone}] {s}")); }); @@ -483,7 +483,7 @@ impl LaunchedHost for T { session: Some(session), channel, stdin_sender, - stdout_cli_receivers, + stdout_deploy_receivers, stdout_receivers, stderr_receivers, perf, diff --git a/hydro_deploy/core/src/util.rs b/hydro_deploy/core/src/util.rs index ba47ffc464a..b49ecfc0314 100644 --- a/hydro_deploy/core/src/util.rs +++ b/hydro_deploy/core/src/util.rs @@ -41,15 +41,15 @@ pub fn prioritized_broadcast> + Send + Unpin tokio::spawn(async move { while let Some(Result::Ok(line)) = lines.next().await { - if let Some(cli_receivers) = weak_priority_receivers.upgrade() { - let mut cli_receivers = cli_receivers.lock().unwrap(); + if let Some(deploy_receivers) = weak_priority_receivers.upgrade() { + let mut deploy_receivers = deploy_receivers.lock().unwrap(); - let successful_send = if let Some(r) = cli_receivers.take() { + let successful_send = if let Some(r) = deploy_receivers.take() { r.send(line.clone()).is_ok() } else { false }; - drop(cli_receivers); + drop(deploy_receivers); if successful_send { continue; @@ -72,9 +72,9 @@ pub fn prioritized_broadcast> + Send + Unpin } } - if let Some(cli_receivers) = weak_priority_receivers.upgrade() { - let mut cli_receivers = cli_receivers.lock().unwrap(); - drop(cli_receivers.take()); + if let Some(deploy_receivers) = weak_priority_receivers.upgrade() { + let mut deploy_receivers = deploy_receivers.lock().unwrap(); + drop(deploy_receivers.take()); } if let Some(receivers) = weak_receivers.upgrade() { diff --git a/hydro_deploy/hydro_cli/Cargo.toml b/hydro_deploy/hydro_cli/Cargo.toml index 8114a6a787e..f4055eeca46 100644 --- a/hydro_deploy/hydro_cli/Cargo.toml +++ b/hydro_deploy/hydro_cli/Cargo.toml @@ -22,7 +22,7 @@ pyo3-asyncio = { version = "0.20", features = ["attributes", "tokio-runtime"] } pythonize = "0.20" futures = "0.3.26" bytes = "1.1.0" -hydroflow_cli_integration = { path = "../hydroflow_cli_integration", version = "^0.5.2" } +hydroflow_deploy_integration = { path = "../hydroflow_deploy_integration", version = "^0.5.2" } # request vendored openssl async-ssh2-lite = { version = "0.5.0", features = [ "vendored-openssl" ] } diff --git a/hydro_deploy/hydro_cli/src/lib.rs b/hydro_deploy/hydro_cli/src/lib.rs index 4d158b626bf..f441aacfed1 100644 --- a/hydro_deploy/hydro_cli/src/lib.rs +++ b/hydro_deploy/hydro_cli/src/lib.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, OnceLock}; use bytes::Bytes; use futures::{Future, SinkExt, StreamExt}; -use hydroflow_cli_integration::{ +use hydroflow_deploy_integration::{ ConnectedDirect, ConnectedSink, ConnectedSource, DynSink, DynStream, ServerOrBound, }; use pyo3::exceptions::{PyException, PyStopAsyncIteration}; @@ -710,7 +710,7 @@ fn null(py: Python<'_>) -> PyResult> { #[pyclass] struct ServerPort { - underlying: hydroflow_cli_integration::ServerPort, + underlying: hydroflow_deploy_integration::ServerPort, } fn with_tokio_runtime(f: impl Fn() -> T) -> T { diff --git a/hydro_deploy/hydro_cli_examples/Cargo.toml b/hydro_deploy/hydro_cli_examples/Cargo.toml index 0e15058ef8a..97588ffb05b 100644 --- a/hydro_deploy/hydro_cli_examples/Cargo.toml +++ b/hydro_deploy/hydro_cli_examples/Cargo.toml @@ -35,7 +35,7 @@ name = "dedalus_2pc_participant" name = "ws_chat_server" [dev-dependencies] -hydroflow = { path = "../../hydroflow", features = [ "cli_integration" ] } +hydroflow = { path = "../../hydroflow", features = [ "deploy_integration" ] } hydroflow_datalog = { path = "../../hydroflow_datalog" } tokio = { version = "1.16", features = [ "full" ] } diff --git a/hydro_deploy/hydro_cli_examples/examples/dedalus_2pc_coordinator/main.rs b/hydro_deploy/hydro_cli_examples/examples/dedalus_2pc_coordinator/main.rs index dabacafa93c..57eff93ca75 100644 --- a/hydro_deploy/hydro_cli_examples/examples/dedalus_2pc_coordinator/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/dedalus_2pc_coordinator/main.rs @@ -1,10 +1,10 @@ -use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; +use hydroflow::util::deploy::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; use hydroflow_datalog::datalog; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let vote_to_participant_port = ports .port("vote_to_participant") .connect::>() diff --git a/hydro_deploy/hydro_cli_examples/examples/dedalus_2pc_participant/main.rs b/hydro_deploy/hydro_cli_examples/examples/dedalus_2pc_participant/main.rs index e69226c3fe6..97a806db7c9 100644 --- a/hydro_deploy/hydro_cli_examples/examples/dedalus_2pc_participant/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/dedalus_2pc_participant/main.rs @@ -1,10 +1,10 @@ -use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; +use hydroflow::util::deploy::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; use hydroflow_datalog::datalog; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let vote_to_participant_source = ports .port("vote_to_participant") .connect::() diff --git a/hydro_deploy/hydro_cli_examples/examples/dedalus_receiver/main.rs b/hydro_deploy/hydro_cli_examples/examples/dedalus_receiver/main.rs index a1fb8f98d82..ae37b42e8b7 100644 --- a/hydro_deploy/hydro_cli_examples/examples/dedalus_receiver/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/dedalus_receiver/main.rs @@ -1,10 +1,10 @@ -use hydroflow::util::cli::{ConnectedDirect, ConnectedSource}; +use hydroflow::util::deploy::{ConnectedDirect, ConnectedSource}; use hydroflow::util::deserialize_from_bytes; use hydroflow_datalog::datalog; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let broadcast_recv = ports .port("broadcast") .connect::() @@ -20,5 +20,5 @@ async fn main() { "# ); - hydroflow::util::cli::launch_flow(df).await; + hydroflow::util::deploy::launch_flow(df).await; } diff --git a/hydro_deploy/hydro_cli_examples/examples/dedalus_sender/main.rs b/hydro_deploy/hydro_cli_examples/examples/dedalus_sender/main.rs index 59b620ef529..b9e2c9d43f2 100644 --- a/hydro_deploy/hydro_cli_examples/examples/dedalus_sender/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/dedalus_sender/main.rs @@ -1,11 +1,11 @@ use hydroflow::tokio_stream::wrappers::IntervalStream; -use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink}; +use hydroflow::util::deploy::{ConnectedDemux, ConnectedDirect, ConnectedSink}; use hydroflow::util::serialize_to_bytes; use hydroflow_datalog::datalog; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let broadcast_port = ports .port("broadcast") .connect::>() @@ -32,5 +32,5 @@ async fn main() { "# ); - hydroflow::util::cli::launch_flow(df).await; + hydroflow::util::deploy::launch_flow(df).await; } diff --git a/hydro_deploy/hydro_cli_examples/examples/dedalus_vote_leader/main.rs b/hydro_deploy/hydro_cli_examples/examples/dedalus_vote_leader/main.rs index 978fc489487..7e21656bba5 100644 --- a/hydro_deploy/hydro_cli_examples/examples/dedalus_vote_leader/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/dedalus_vote_leader/main.rs @@ -1,10 +1,10 @@ -use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; +use hydroflow::util::deploy::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; use hydroflow_datalog::datalog; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let to_replica_port = ports .port("to_replica") .connect::>() diff --git a/hydro_deploy/hydro_cli_examples/examples/dedalus_vote_participant/main.rs b/hydro_deploy/hydro_cli_examples/examples/dedalus_vote_participant/main.rs index 9c07df16e9c..eaa142a3994 100644 --- a/hydro_deploy/hydro_cli_examples/examples/dedalus_vote_participant/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/dedalus_vote_participant/main.rs @@ -1,10 +1,10 @@ -use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; +use hydroflow::util::deploy::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; use hydroflow_datalog::datalog; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let to_replica_source = ports .port("to_replica") .connect::() diff --git a/hydro_deploy/hydro_cli_examples/examples/empty_program/main.rs b/hydro_deploy/hydro_cli_examples/examples/empty_program/main.rs index 0794176236f..19f3e1f285a 100644 --- a/hydro_deploy/hydro_cli_examples/examples/empty_program/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/empty_program/main.rs @@ -1,6 +1,6 @@ #[hydroflow::main] async fn main() { - let _ = hydroflow::util::cli::init::<()>().await; + let _ = hydroflow::util::deploy::init::<()>().await; loop { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } diff --git a/hydro_deploy/hydro_cli_examples/examples/panic_program/main.rs b/hydro_deploy/hydro_cli_examples/examples/panic_program/main.rs index 08d5f1914bf..3b57b068093 100644 --- a/hydro_deploy/hydro_cli_examples/examples/panic_program/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/panic_program/main.rs @@ -2,7 +2,7 @@ use std::io::Write; #[hydroflow::main] async fn main() { - let _ = hydroflow::util::cli::init::<()>().await; + let _ = hydroflow::util::deploy::init::<()>().await; println!("hello!"); std::io::stdout().flush().unwrap(); diff --git a/hydro_deploy/hydro_cli_examples/examples/stdout_receiver/main.rs b/hydro_deploy/hydro_cli_examples/examples/stdout_receiver/main.rs index 3a575a8854c..e92794c4d5b 100644 --- a/hydro_deploy/hydro_cli_examples/examples/stdout_receiver/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/stdout_receiver/main.rs @@ -1,9 +1,9 @@ use hydroflow::hydroflow_syntax; -use hydroflow::util::cli::{ConnectedDirect, ConnectedSource}; +use hydroflow::util::deploy::{ConnectedDirect, ConnectedSource}; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let echo_recv = ports .port("echo") .connect::() @@ -16,5 +16,5 @@ async fn main() { for_each(|x| println!("echo {:?}", x)); }; - hydroflow::util::cli::launch_flow(df).await; + hydroflow::util::deploy::launch_flow(df).await; } diff --git a/hydro_deploy/hydro_cli_examples/examples/tagged_stdout_receiver/main.rs b/hydro_deploy/hydro_cli_examples/examples/tagged_stdout_receiver/main.rs index 4859c59e458..36c0dc19154 100644 --- a/hydro_deploy/hydro_cli_examples/examples/tagged_stdout_receiver/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/tagged_stdout_receiver/main.rs @@ -1,9 +1,9 @@ use hydroflow::hydroflow_syntax; -use hydroflow::util::cli::{ConnectedDirect, ConnectedSource, ConnectedTagged}; +use hydroflow::util::deploy::{ConnectedDirect, ConnectedSource, ConnectedTagged}; #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let echo_recv = ports .port("echo") .connect::>() @@ -19,5 +19,5 @@ async fn main() { for_each(|x| println!("echo {:?}", x)); }; - hydroflow::util::cli::launch_flow(df).await; + hydroflow::util::deploy::launch_flow(df).await; } diff --git a/hydro_deploy/hydro_cli_examples/examples/ws_chat_server/main.rs b/hydro_deploy/hydro_cli_examples/examples/ws_chat_server/main.rs index bf09b95596a..3525c81f142 100644 --- a/hydro_deploy/hydro_cli_examples/examples/ws_chat_server/main.rs +++ b/hydro_deploy/hydro_cli_examples/examples/ws_chat_server/main.rs @@ -1,6 +1,6 @@ use hydroflow::compiled::pull::HalfMultisetJoinState; use hydroflow::hydroflow_syntax; -use hydroflow::util::cli::{ConnectedSink, ConnectedSource}; +use hydroflow::util::deploy::{ConnectedSink, ConnectedSource}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; use serde::{Deserialize, Serialize}; use tokio::net::TcpListener; @@ -29,17 +29,17 @@ struct ChatMessage { #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let from_peer = ports .port("from_peer") - .connect::() + .connect::() .await .into_source(); let to_peer = ports .port("to_peer") - .connect::>() + .connect::>() .await .into_sink(); @@ -116,5 +116,5 @@ async fn main() { all_messages -> [1]broadcast_clients; }; - hydroflow::util::cli::launch_flow(df).await; + hydroflow::util::deploy::launch_flow(df).await; } diff --git a/hydro_deploy/hydroflow_cli_integration/CHANGELOG.md b/hydro_deploy/hydroflow_deploy_integration/CHANGELOG.md similarity index 100% rename from hydro_deploy/hydroflow_cli_integration/CHANGELOG.md rename to hydro_deploy/hydroflow_deploy_integration/CHANGELOG.md diff --git a/hydro_deploy/hydroflow_cli_integration/Cargo.toml b/hydro_deploy/hydroflow_deploy_integration/Cargo.toml similarity index 82% rename from hydro_deploy/hydroflow_cli_integration/Cargo.toml rename to hydro_deploy/hydroflow_deploy_integration/Cargo.toml index 0f157adf83f..5808d895cbb 100644 --- a/hydro_deploy/hydroflow_cli_integration/Cargo.toml +++ b/hydro_deploy/hydroflow_deploy_integration/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "hydroflow_cli_integration" +name = "hydroflow_deploy_integration" publish = true version = "0.5.2" edition = "2021" license = "Apache-2.0" -documentation = "https://docs.rs/hydroflow_cli_integration/" -description = "`hydro_cli` integration for Hydroflow" +documentation = "https://docs.rs/hydroflow_deploy_integration/" +description = "`hydro_deploy` integration for Hydroflow" [dependencies] async-recursion = "1" diff --git a/hydro_deploy/hydroflow_cli_integration/src/lib.rs b/hydro_deploy/hydroflow_deploy_integration/src/lib.rs similarity index 100% rename from hydro_deploy/hydroflow_cli_integration/src/lib.rs rename to hydro_deploy/hydroflow_deploy_integration/src/lib.rs diff --git a/hydro_deploy/hydroflow_plus_cli_integration/CHANGELOG.md b/hydro_deploy/hydroflow_plus_deploy/CHANGELOG.md similarity index 80% rename from hydro_deploy/hydroflow_plus_cli_integration/CHANGELOG.md rename to hydro_deploy/hydroflow_plus_deploy/CHANGELOG.md index 43305a4a595..20d183e7cef 100644 --- a/hydro_deploy/hydroflow_plus_cli_integration/CHANGELOG.md +++ b/hydro_deploy/hydroflow_plus_deploy/CHANGELOG.md @@ -1,7 +1,64 @@ +## Unreleased + +### New Features + + - use trybuild to compile subgraph binaries + +### Refactor (BREAKING) + + - disentangle instantiated nodes from locations + - simplify process/cluster specs + --- + [//]: # (BEGIN SAPLING FOOTER) + Stack created with [Sapling](https://sapling-scm.com). Best reviewed + with + [ReviewStack](https://reviewstack.dev/hydro-project/hydroflow/pull/1394). + * #1395 + * __->__ #1394 + - defer network instantiation until after finalizing IR + --- + [//]: # (BEGIN SAPLING FOOTER) + Stack created with [Sapling](https://sapling-scm.com). Best reviewed + with + [ReviewStack](https://reviewstack.dev/hydro-project/hydroflow/pull/1377). + * #1395 + * #1394 + * __->__ #1377 + +### Pre-Move Commit Statistics + + + + - 4 commits contributed to the release over the course of 7 calendar days. + - 30 days passed between releases. + - 4 commits were understood as [conventional](https://www.conventionalcommits.org). + - 4 unique issues were worked on: [#1377](https://github.com/hydro-project/hydroflow/issues/1377), [#1394](https://github.com/hydro-project/hydroflow/issues/1394), [#1395](https://github.com/hydro-project/hydroflow/issues/1395), [#1398](https://github.com/hydro-project/hydroflow/issues/1398) + +### Pre-Move Commit Details + + + +
view details + + * **[#1377](https://github.com/hydro-project/hydroflow/issues/1377)** + - Defer network instantiation until after finalizing IR ([`0eba702`](https://github.com/hydro-project/hydroflow/commit/0eba702f62e7a6816cf931b01a2ea5643bd7321d)) + * **[#1394](https://github.com/hydro-project/hydroflow/issues/1394)** + - Simplify process/cluster specs ([`128aaec`](https://github.com/hydro-project/hydroflow/commit/128aaecd40edce57dc254afdcd61ecd5b9948d71)) + * **[#1395](https://github.com/hydro-project/hydroflow/issues/1395)** + - Disentangle instantiated nodes from locations ([`5f2789a`](https://github.com/hydro-project/hydroflow/commit/5f2789a13d1602f170e678fe9bbc59caf69db4b5)) + * **[#1398](https://github.com/hydro-project/hydroflow/issues/1398)** + - Use trybuild to compile subgraph binaries ([`46a8a2c`](https://github.com/hydro-project/hydroflow/commit/46a8a2cb08732bb21096e824bc4542d208c68fb2)) +
+ ## v0.8.0 (2024-07-23) + + + + + ### Refactor (BREAKING) - make `Host` trait use `&self` interior mutability to remove `RwLock` wrappings #430 @@ -18,7 +75,7 @@ - - 4 commits contributed to the release over the course of 5 calendar days. + - 5 commits contributed to the release over the course of 4 calendar days. - 59 days passed between releases. - 4 commits were understood as [conventional](https://www.conventionalcommits.org). - 4 unique issues were worked on: [#1339](https://github.com/hydro-project/hydroflow/issues/1339), [#1346](https://github.com/hydro-project/hydroflow/issues/1346), [#1347](https://github.com/hydro-project/hydroflow/issues/1347), [#1356](https://github.com/hydro-project/hydroflow/issues/1356) @@ -37,6 +94,8 @@ - Make `Host` trait use `&self` interior mutability to remove `RwLock` wrappings #430 ([`c5a8de2`](https://github.com/hydro-project/hydroflow/commit/c5a8de28e7844b3c29d58116d8340967f2e6bcc4)) * **[#1356](https://github.com/hydro-project/hydroflow/issues/1356)** - Replace `async-channel` with `tokio::sync::mpsc::unbounded_channel` ([`6039078`](https://github.com/hydro-project/hydroflow/commit/60390782dd7dcec18d193c800af716843a944dba)) + * **Uncategorized** + - Release hydroflow_lang v0.8.0, hydroflow_datalog_core v0.8.0, hydroflow_datalog v0.8.0, hydroflow_macro v0.8.0, lattices_macro v0.5.5, lattices v0.5.6, variadics v0.0.5, pusherator v0.0.7, hydroflow v0.8.0, hydroflow_plus v0.8.0, hydro_deploy v0.8.0, hydro_cli v0.8.0, hydroflow_plus_cli_integration v0.8.0, safety bump 7 crates ([`ca6c16b`](https://github.com/hydro-project/hydroflow/commit/ca6c16b4a7ce35e155fe7fc6c7d1676c37c9e4de)) ## v0.7.0 (2024-05-24) @@ -81,11 +140,11 @@ - simplify Location trait to remove lifetimes - simplify lifetime bounds for processes and clusters - feat(hydroflow_plus): simplify lifetime bounds for processes and - clusters - - This allows `extract` to move the flow builder, which is a prerequisite - for having developers run the optimizer during deployment as well in + feat(hydroflow_plus): simplify lifetime bounds for processes and + clusters + + This allows `extract` to move the flow builder, which is a prerequisite + for having developers run the optimizer during deployment as well in case it changes the network topology. ### Commit Statistics diff --git a/hydro_deploy/hydroflow_plus_cli_integration/Cargo.toml b/hydro_deploy/hydroflow_plus_deploy/Cargo.toml similarity index 88% rename from hydro_deploy/hydroflow_plus_cli_integration/Cargo.toml rename to hydro_deploy/hydroflow_plus_deploy/Cargo.toml index 2d026d95462..6d748a2b16a 100644 --- a/hydro_deploy/hydroflow_plus_cli_integration/Cargo.toml +++ b/hydro_deploy/hydroflow_plus_deploy/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "hydroflow_plus_cli_integration" +name = "hydroflow_plus_deploy" publish = true version = "0.8.0" edition = "2021" license = "Apache-2.0" -documentation = "https://docs.rs/hydroflow_plus_cli_integration/" +documentation = "https://docs.rs/hydroflow_plus_deploy/" description = "Library for working with hydro_deploy and hydroflow_plus" [features] @@ -13,7 +13,7 @@ stageleft_devel = [] [dependencies] stageleft = { path = "../../stageleft", version = "^0.3.0" } -hydroflow_plus = { path = "../../hydroflow_plus", version = "^0.8.0", features = [ "cli_integration" ] } +hydroflow_plus = { path = "../../hydroflow_plus", version = "^0.8.0", features = [ "deploy_integration" ] } syn = { version = "2.0.0", features = [ "parsing", "extra-traits" ] } tokio = { version = "1.16", features = [ "full" ] } serde = { version = "1", features = [ "derive" ] } diff --git a/hydro_deploy/hydroflow_plus_cli_integration/build.rs b/hydro_deploy/hydroflow_plus_deploy/build.rs similarity index 100% rename from hydro_deploy/hydroflow_plus_cli_integration/build.rs rename to hydro_deploy/hydroflow_plus_deploy/build.rs diff --git a/hydro_deploy/hydroflow_plus_cli_integration/src/deploy.rs b/hydro_deploy/hydroflow_plus_deploy/src/deploy.rs similarity index 100% rename from hydro_deploy/hydroflow_plus_cli_integration/src/deploy.rs rename to hydro_deploy/hydroflow_plus_deploy/src/deploy.rs diff --git a/hydro_deploy/hydroflow_plus_cli_integration/src/deploy_runtime.rs b/hydro_deploy/hydroflow_plus_deploy/src/deploy_runtime.rs similarity index 87% rename from hydro_deploy/hydroflow_plus_cli_integration/src/deploy_runtime.rs rename to hydro_deploy/hydroflow_plus_deploy/src/deploy_runtime.rs index 6050768805b..e98d3e30ca6 100644 --- a/hydro_deploy/hydroflow_plus_cli_integration/src/deploy_runtime.rs +++ b/hydro_deploy/hydroflow_plus_deploy/src/deploy_runtime.rs @@ -1,18 +1,18 @@ -use hydroflow_plus::util::cli::{ - ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, HydroCLI, +use hydroflow_plus::util::deploy::{ + ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, DeployPorts, }; use stageleft::{q, Quoted, RuntimeData}; use crate::HydroflowPlusMeta; pub fn cluster_members<'a>(of_cluster: usize) -> impl Quoted<'a, &'a Vec> + Copy + 'a { - let cli: RuntimeData<&HydroCLI> = + let cli: RuntimeData<&DeployPorts> = RuntimeData::new("__hydroflow_plus_trybuild_cli"); q!(cli.meta.clusters.get(&of_cluster).unwrap()) } pub fn cluster_self_id<'a>() -> impl Quoted<'a, u32> + Copy + 'a { - let cli: RuntimeData<&HydroCLI> = + let cli: RuntimeData<&DeployPorts> = RuntimeData::new("__hydroflow_plus_trybuild_cli"); q!(cli .meta @@ -21,7 +21,7 @@ pub fn cluster_self_id<'a>() -> impl Quoted<'a, u32> + Copy + 'a { } pub fn deploy_o2o(p1_port: &str, p2_port: &str) -> (syn::Expr, syn::Expr) { - let env: RuntimeData<&HydroCLI> = + let env: RuntimeData<&DeployPorts> = RuntimeData::new("__hydroflow_plus_trybuild_cli"); ( { @@ -44,7 +44,7 @@ pub fn deploy_o2o(p1_port: &str, p2_port: &str) -> (syn::Expr, syn::Expr) { } pub fn deploy_o2m(p1_port: &str, c2_port: &str) -> (syn::Expr, syn::Expr) { - let env: RuntimeData<&HydroCLI> = + let env: RuntimeData<&DeployPorts> = RuntimeData::new("__hydroflow_plus_trybuild_cli"); ( { @@ -67,7 +67,7 @@ pub fn deploy_o2m(p1_port: &str, c2_port: &str) -> (syn::Expr, syn::Expr) { } pub fn deploy_m2o(c1_port: &str, p2_port: &str) -> (syn::Expr, syn::Expr) { - let env: RuntimeData<&HydroCLI> = + let env: RuntimeData<&DeployPorts> = RuntimeData::new("__hydroflow_plus_trybuild_cli"); ( { @@ -90,7 +90,7 @@ pub fn deploy_m2o(c1_port: &str, p2_port: &str) -> (syn::Expr, syn::Expr) { } pub fn deploy_m2m(c1_port: &str, c2_port: &str) -> (syn::Expr, syn::Expr) { - let env: RuntimeData<&HydroCLI> = + let env: RuntimeData<&DeployPorts> = RuntimeData::new("__hydroflow_plus_trybuild_cli"); ( { diff --git a/hydro_deploy/hydroflow_plus_cli_integration/src/lib.rs b/hydro_deploy/hydroflow_plus_deploy/src/lib.rs similarity index 100% rename from hydro_deploy/hydroflow_plus_cli_integration/src/lib.rs rename to hydro_deploy/hydroflow_plus_deploy/src/lib.rs diff --git a/hydro_deploy/hydroflow_plus_cli_integration/src/runtime.rs b/hydro_deploy/hydroflow_plus_deploy/src/runtime.rs similarity index 98% rename from hydro_deploy/hydroflow_plus_cli_integration/src/runtime.rs rename to hydro_deploy/hydroflow_plus_deploy/src/runtime.rs index 6b74f010518..ea9fcbc6335 100644 --- a/hydro_deploy/hydroflow_plus_cli_integration/src/runtime.rs +++ b/hydro_deploy/hydroflow_plus_deploy/src/runtime.rs @@ -3,8 +3,8 @@ use std::rc::Rc; use hydroflow_plus::deploy::{ClusterSpec, Deploy, Node, ProcessSpec}; use hydroflow_plus::lang::graph::HydroflowGraph; -use hydroflow_plus::util::cli::{ - ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, HydroCLI, +use hydroflow_plus::util::deploy::{ + ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, DeployPorts, }; use stageleft::{q, Quoted, RuntimeData}; @@ -14,7 +14,7 @@ pub struct CLIRuntime {} impl<'a> Deploy<'a> for CLIRuntime { type InstantiateEnv = (); - type CompileEnv = RuntimeData<&'a HydroCLI>; + type CompileEnv = RuntimeData<&'a DeployPorts>; type Process = CLIRuntimeNode; type Cluster = CLIRuntimeCluster; type Meta = (); diff --git a/hydro_deploy/hydroflow_plus_cli_integration/src/trybuild.rs b/hydro_deploy/hydroflow_plus_deploy/src/trybuild.rs similarity index 94% rename from hydro_deploy/hydroflow_plus_cli_integration/src/trybuild.rs rename to hydro_deploy/hydroflow_plus_deploy/src/trybuild.rs index 5d4e20b37fd..096dd1eae6a 100644 --- a/hydro_deploy/hydroflow_plus_cli_integration/src/trybuild.rs +++ b/hydro_deploy/hydroflow_plus_deploy/src/trybuild.rs @@ -27,17 +27,17 @@ pub fn compile_graph_trybuild(graph: HydroflowGraph, extra_stmts: Vec #![allow(unused_crate_dependencies, missing_docs)] #[allow(unused)] - fn __hfplus_runtime<'a>(__hydroflow_plus_trybuild_cli: &'a hydroflow_plus::util::cli::HydroCLI) -> hydroflow_plus::Hydroflow<'a> { + fn __hfplus_runtime<'a>(__hydroflow_plus_trybuild_cli: &'a hydroflow_plus::util::deploy::DeployPorts) -> hydroflow_plus::Hydroflow<'a> { #(#extra_stmts)* #tokens } #[tokio::main] async fn main() { - let ports = hydroflow_plus::util::cli::init_no_ack_start().await; + let ports = hydroflow_plus::util::deploy::init_no_ack_start().await; let flow = __hfplus_runtime(&ports); println!("ack start"); - hydroflow_plus::util::cli::launch_flow(flow).await; + hydroflow_plus::util::deploy::launch_flow(flow).await; } }; source_ast diff --git a/hydroflow/Cargo.toml b/hydroflow/Cargo.toml index 96cc8318225..810eb8a4cf4 100644 --- a/hydroflow/Cargo.toml +++ b/hydroflow/Cargo.toml @@ -14,7 +14,7 @@ nightly = [ "hydroflow_macro", "hydroflow_macro/diagnostics" ] macros = [ "hydroflow_macro", "hydroflow_datalog" ] hydroflow_macro = [ "dep:hydroflow_macro" ] hydroflow_datalog = [ "dep:hydroflow_datalog" ] -cli_integration = [ "dep:hydroflow_cli_integration" ] +deploy_integration = [ "dep:hydroflow_deploy_integration" ] python = [ "dep:pyo3" ] debugging = [ "hydroflow_lang/debugging" ] @@ -31,7 +31,7 @@ bincode = "1.3" byteorder = "1.4.3" bytes = "1.1.0" futures = "0.3" -hydroflow_cli_integration = { optional = true, path = "../hydro_deploy/hydroflow_cli_integration", version = "^0.5.2" } +hydroflow_deploy_integration = { optional = true, path = "../hydro_deploy/hydroflow_deploy_integration", version = "^0.5.2" } hydroflow_datalog = { optional = true, path = "../hydroflow_datalog", version = "^0.8.0" } hydroflow_lang = { path = "../hydroflow_lang", version = "^0.8.0" } hydroflow_macro = { optional = true, path = "../hydroflow_macro", version = "^0.8.0" } diff --git a/hydroflow/src/util/cli.rs b/hydroflow/src/util/deploy.rs similarity index 85% rename from hydroflow/src/util/cli.rs rename to hydroflow/src/util/deploy.rs index c1576d68425..522550ec768 100644 --- a/hydroflow/src/util/cli.rs +++ b/hydroflow/src/util/deploy.rs @@ -3,7 +3,7 @@ use std::cell::RefCell; use std::collections::HashMap; -pub use hydroflow_cli_integration::*; +pub use hydroflow_deploy_integration::*; use serde::de::DeserializeOwned; use crate::scheduled::graph::Hydroflow; @@ -12,12 +12,12 @@ use crate::scheduled::graph::Hydroflow; macro_rules! launch { ($f:expr) => { async { - let ports = $crate::util::cli::init_no_ack_start().await; + let ports = $crate::util::deploy::init_no_ack_start().await; let flow = $f(&ports); println!("ack start"); - $crate::util::cli::launch_flow(flow).await + $crate::util::deploy::launch_flow(flow).await } }; } @@ -45,12 +45,14 @@ pub async fn launch_flow(mut flow: Hydroflow<'_>) { } } -pub struct HydroCLI> { +/// Contains runtime information passed by Hydro Deploy to a program, +/// describing how to connect to other services and metadata about them. +pub struct DeployPorts> { ports: RefCell>, pub meta: T, } -impl HydroCLI { +impl DeployPorts { pub fn port(&self, name: &str) -> ServerOrBound { self.ports .try_borrow_mut() @@ -60,7 +62,7 @@ impl HydroCLI { } } -pub async fn init_no_ack_start() -> HydroCLI { +pub async fn init_no_ack_start() -> DeployPorts { let mut input = String::new(); std::io::stdin().read_line(&mut input).unwrap(); let trimmed = input.trim(); @@ -99,7 +101,7 @@ pub async fn init_no_ack_start() -> HydroCLI { all_connected.insert(name, ServerOrBound::Bound(defn)); } - HydroCLI { + DeployPorts { ports: RefCell::new(all_connected), meta: bind_config .1 @@ -108,7 +110,7 @@ pub async fn init_no_ack_start() -> HydroCLI { } } -pub async fn init() -> HydroCLI { +pub async fn init() -> DeployPorts { let ret = init_no_ack_start::().await; println!("ack start"); diff --git a/hydroflow/src/util/mod.rs b/hydroflow/src/util/mod.rs index 76a1d58076c..1357e6cf7ce 100644 --- a/hydroflow/src/util/mod.rs +++ b/hydroflow/src/util/mod.rs @@ -25,8 +25,8 @@ mod socket; #[cfg(unix)] pub use socket::*; -#[cfg(feature = "cli_integration")] -pub mod cli; +#[cfg(feature = "deploy_integration")] +pub mod deploy; use std::io::Read; use std::net::SocketAddr; diff --git a/hydroflow_plus/Cargo.toml b/hydroflow_plus/Cargo.toml index d1884eb8ad6..58448648f25 100644 --- a/hydroflow_plus/Cargo.toml +++ b/hydroflow_plus/Cargo.toml @@ -12,7 +12,7 @@ path = "src/lib.rs" [features] diagnostics = [ "hydroflow_lang/diagnostics" ] -cli_integration = [ "hydroflow/cli_integration" ] +deploy_integration = [ "hydroflow/deploy_integration" ] stageleft_devel = [] [dependencies] @@ -22,10 +22,9 @@ proc-macro2 = "1.0.57" proc-macro-crate = "1.1.0" hydroflow = { path = "../hydroflow", version = "^0.8.0", default-features = false } hydroflow_lang = { path = "../hydroflow_lang", version = "^0.8.0" } -serde = { version = "1", features = [ "derive" ] } +serde = "1" bincode = "1.3" stageleft = { path = "../stageleft", version = "^0.3.0" } -dyn-clone = "1.0.17" # added to workaround `cargo smart-release` https://github.com/Byron/cargo-smart-release/issues/16 stageleft_tool = { path = "../stageleft_tool", version = "^0.2.0", optional = true } diff --git a/hydroflow_plus/src/profiler.rs b/hydroflow_plus/src/profiler.rs index 9bb9575f586..853fbf5560e 100644 --- a/hydroflow_plus/src/profiler.rs +++ b/hydroflow_plus/src/profiler.rs @@ -75,7 +75,6 @@ pub fn profiling<'a>( .collect() } -#[stageleft::runtime] #[cfg(test)] mod tests { use stageleft::*; diff --git a/hydroflow_plus_test/Cargo.toml b/hydroflow_plus_test/Cargo.toml index fb98a188389..271cb2d719d 100644 --- a/hydroflow_plus_test/Cargo.toml +++ b/hydroflow_plus_test/Cargo.toml @@ -9,11 +9,10 @@ default = ["stageleft_devel"] stageleft_devel = [] [dependencies] -hydroflow = { path = "../hydroflow", version = "^0.8.0", features = ["debugging"] } hydroflow_plus = { path = "../hydroflow_plus", version = "^0.8.0" } tokio = { version = "1.16", features = [ "full" ] } stageleft = { path = "../stageleft", version = "^0.3.0" } -hydroflow_plus_cli_integration = { path = "../hydro_deploy/hydroflow_plus_cli_integration", version = "^0.8.0" } +hydroflow_plus_deploy = { path = "../hydro_deploy/hydroflow_plus_deploy", version = "^0.8.0" } rand = "0.8.5" serde = { version = "1", features = [ "derive" ] } @@ -23,5 +22,5 @@ stageleft_tool = { path = "../stageleft_tool", version = "^0.2.0" } [dev-dependencies] insta = "1.7.1" hydro_deploy = { path = "../hydro_deploy/core", version = "^0.8.0" } -hydroflow_plus_cli_integration = { path = "../hydro_deploy/hydroflow_plus_cli_integration", version = "^0.8.0", features = [ "deploy" ] } +hydroflow_plus_deploy = { path = "../hydro_deploy/hydroflow_plus_deploy", version = "^0.8.0", features = [ "deploy" ] } futures = "0.3" diff --git a/hydroflow_plus_test/examples/compute_pi.rs b/hydroflow_plus_test/examples/compute_pi.rs index fee3bbe5aba..4ff6e008744 100644 --- a/hydroflow_plus_test/examples/compute_pi.rs +++ b/hydroflow_plus_test/examples/compute_pi.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_cli_integration::TrybuildHost; +use hydroflow_plus_deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/examples/first_ten_distributed.rs b/hydroflow_plus_test/examples/first_ten_distributed.rs index 1d6903c2ae7..0c909a1a900 100644 --- a/hydroflow_plus_test/examples/first_ten_distributed.rs +++ b/hydroflow_plus_test/examples/first_ten_distributed.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_cli_integration::TrybuildHost; +use hydroflow_plus_deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/examples/map_reduce.rs b/hydroflow_plus_test/examples/map_reduce.rs index 9294ad56fb8..ce9a39c3419 100644 --- a/hydroflow_plus_test/examples/map_reduce.rs +++ b/hydroflow_plus_test/examples/map_reduce.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_cli_integration::TrybuildHost; +use hydroflow_plus_deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/examples/perf_compute_pi.rs b/hydroflow_plus_test/examples/perf_compute_pi.rs index 2e4ac310b3a..434b5e7f3cc 100644 --- a/hydroflow_plus_test/examples/perf_compute_pi.rs +++ b/hydroflow_plus_test/examples/perf_compute_pi.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::hydroflow_crate::perf_options::PerfOptions; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_cli_integration::TrybuildHost; +use hydroflow_plus_deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/examples/simple_cluster.rs b/hydroflow_plus_test/examples/simple_cluster.rs index 2a6d87d9f08..64ebad93641 100644 --- a/hydroflow_plus_test/examples/simple_cluster.rs +++ b/hydroflow_plus_test/examples/simple_cluster.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_cli_integration::TrybuildHost; +use hydroflow_plus_deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/src/cluster/compute_pi.rs b/hydroflow_plus_test/src/cluster/compute_pi.rs index af262688370..cec4d047897 100644 --- a/hydroflow_plus_test/src/cluster/compute_pi.rs +++ b/hydroflow_plus_test/src/cluster/compute_pi.rs @@ -44,10 +44,9 @@ pub fn compute_pi(flow: &FlowBuilder, batch_size: usize) -> (Cluster, Pr (cluster, process) } -#[stageleft::runtime] #[cfg(test)] mod tests { - use hydroflow_plus_cli_integration::CLIRuntime; + use hydroflow_plus_deploy::CLIRuntime; use stageleft::RuntimeData; #[test] diff --git a/hydroflow_plus_test/src/cluster/many_to_many.rs b/hydroflow_plus_test/src/cluster/many_to_many.rs index 0f5e29e6812..f0e34e09820 100644 --- a/hydroflow_plus_test/src/cluster/many_to_many.rs +++ b/hydroflow_plus_test/src/cluster/many_to_many.rs @@ -13,7 +13,7 @@ pub fn many_to_many(flow: &FlowBuilder) -> Cluster<()> { #[cfg(test)] mod tests { use hydro_deploy::Deployment; - use hydroflow_plus_cli_integration::{DeployCrateWrapper, TrybuildHost}; + use hydroflow_plus_deploy::{DeployCrateWrapper, TrybuildHost}; #[tokio::test] async fn many_to_many() { diff --git a/hydroflow_plus_test/src/cluster/map_reduce.rs b/hydroflow_plus_test/src/cluster/map_reduce.rs index 3f4b9ca2992..8280ea3bbef 100644 --- a/hydroflow_plus_test/src/cluster/map_reduce.rs +++ b/hydroflow_plus_test/src/cluster/map_reduce.rs @@ -34,10 +34,9 @@ pub fn map_reduce(flow: &FlowBuilder) -> (Process, Cluster) { (process, cluster) } -#[stageleft::runtime] #[cfg(test)] mod tests { - use hydroflow_plus_cli_integration::CLIRuntime; + use hydroflow_plus_deploy::CLIRuntime; use stageleft::RuntimeData; #[test] diff --git a/hydroflow_plus_test/src/cluster/simple_cluster.rs b/hydroflow_plus_test/src/cluster/simple_cluster.rs index 983494ed629..1d70a39ac67 100644 --- a/hydroflow_plus_test/src/cluster/simple_cluster.rs +++ b/hydroflow_plus_test/src/cluster/simple_cluster.rs @@ -28,7 +28,7 @@ pub fn simple_cluster(flow: &FlowBuilder) -> (Process<()>, Cluster<()>) { #[cfg(test)] mod tests { use hydro_deploy::Deployment; - use hydroflow_plus_cli_integration::{DeployCrateWrapper, TrybuildHost}; + use hydroflow_plus_deploy::{DeployCrateWrapper, TrybuildHost}; #[tokio::test] async fn simple_cluster() { diff --git a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_0.snap b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_0.snap index a73c92f3bc1..6d4451bde43 100644 --- a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_0.snap +++ b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_0.snap @@ -9,7 +9,7 @@ expression: ir.surface_syntax_string() 5v1 = map ({ use crate :: __staged :: cluster :: compute_pi :: * ; | (x , y) | x * x + y * y < 1.0 }); 6v1 = fold :: < 'tick > ({ use crate :: __staged :: cluster :: compute_pi :: * ; | | (0u64 , 0u64) } , { use crate :: __staged :: cluster :: compute_pi :: * ; | (inside , total) , sample_inside | { if sample_inside { * inside += 1 ; } * total += 1 ; } }); 7v1 = map (| data | { hydroflow_plus :: runtime_support :: bincode :: serialize :: < (u64 , u64) > (& data) . unwrap () . into () }); -8v1 = dest_sink ({ use hydroflow_plus_cli_integration :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_0" ; { env . port (port) . connect_local_blocking :: < ConnectedDirect > () . into_sink () } }); +8v1 = dest_sink ({ use hydroflow_plus_deploy :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_0" ; { env . port (port) . connect_local_blocking :: < ConnectedDirect > () . into_sink () } }); 1v1 -> 2v1; 2v1 -> 3v1; diff --git a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_1.snap b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_1.snap index a6965bf77b3..4ed471302ae 100644 --- a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_1.snap +++ b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_1.snap @@ -2,7 +2,7 @@ source: hydroflow_plus_test/src/cluster/compute_pi.rs expression: ir.surface_syntax_string() --- -1v1 = source_stream ({ use hydroflow_plus_cli_integration :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_0" ; { env . port (port) . connect_local_blocking :: < ConnectedTagged < ConnectedDirect > > () . into_source () } }); +1v1 = source_stream ({ use hydroflow_plus_deploy :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_0" ; { env . port (port) . connect_local_blocking :: < ConnectedTagged < ConnectedDirect > > () . into_source () } }); 2v1 = map (| res | { let (id , b) = res . unwrap () ; (id , hydroflow_plus :: runtime_support :: bincode :: deserialize :: < (u64 , u64) > (& b) . unwrap ()) }); 3v1 = map ({ use hydroflow_plus :: __staged :: stream :: * ; | (_ , b) | b }); 4v1 = reduce :: < 'static > ({ use crate :: __staged :: cluster :: compute_pi :: * ; | (inside , total) , (inside_batch , total_batch) | { * inside += inside_batch ; * total += total_batch ; } }); diff --git a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_0.snap b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_0.snap index 8099e7ec3da..83814e14d99 100644 --- a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_0.snap +++ b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_0.snap @@ -7,8 +7,8 @@ expression: ir.surface_syntax_string() 3v1 = enumerate (); 4v1 = map ({ use crate :: __staged :: cluster :: map_reduce :: * ; let all_ids_vec = __hydroflow_plus_cluster_ids_1 ; | (i , w) | ((i % all_ids_vec . len ()) as u32 , w) }); 5v1 = map (| (id , data) | { (id , hydroflow_plus :: runtime_support :: bincode :: serialize :: < std :: string :: String > (& data) . unwrap () . into ()) }); -6v1 = dest_sink ({ use hydroflow_plus_cli_integration :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_0" ; { env . port (port) . connect_local_blocking :: < ConnectedDemux < ConnectedDirect > > () . into_sink () } }); -7v1 = source_stream ({ use hydroflow_plus_cli_integration :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_1" ; { env . port (port) . connect_local_blocking :: < ConnectedTagged < ConnectedDirect > > () . into_source () } }); +6v1 = dest_sink ({ use hydroflow_plus_deploy :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_0" ; { env . port (port) . connect_local_blocking :: < ConnectedDemux < ConnectedDirect > > () . into_sink () } }); +7v1 = source_stream ({ use hydroflow_plus_deploy :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_1" ; { env . port (port) . connect_local_blocking :: < ConnectedTagged < ConnectedDirect > > () . into_source () } }); 8v1 = map (| res | { let (id , b) = res . unwrap () ; (id , hydroflow_plus :: runtime_support :: bincode :: deserialize :: < (std :: string :: String , i32) > (& b) . unwrap ()) }); 9v1 = map ({ use hydroflow_plus :: __staged :: stream :: * ; | (_ , b) | b }); 10v1 = reduce_keyed :: < 'static > ({ use crate :: __staged :: cluster :: map_reduce :: * ; | total , count | * total += count }); diff --git a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_1.snap b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_1.snap index 97ddaac14c7..29fbe06e91b 100644 --- a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_1.snap +++ b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_1.snap @@ -2,13 +2,13 @@ source: hydroflow_plus_test/src/cluster/map_reduce.rs expression: ir.surface_syntax_string() --- -1v1 = source_stream ({ use hydroflow_plus_cli_integration :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_0" ; { env . port (port) . connect_local_blocking :: < ConnectedDirect > () . into_source () } }); +1v1 = source_stream ({ use hydroflow_plus_deploy :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_0" ; { env . port (port) . connect_local_blocking :: < ConnectedDirect > () . into_source () } }); 2v1 = map (| res | { hydroflow_plus :: runtime_support :: bincode :: deserialize :: < std :: string :: String > (& res . unwrap ()) . unwrap () }); 3v1 = map ({ use crate :: __staged :: cluster :: map_reduce :: * ; | string | (string , ()) }); 4v1 = fold_keyed :: < 'tick > ({ use crate :: __staged :: cluster :: map_reduce :: * ; | | 0 } , { use crate :: __staged :: cluster :: map_reduce :: * ; | count , _ | * count += 1 }); 5v1 = inspect ({ use crate :: __staged :: cluster :: map_reduce :: * ; | (string , count) | println ! ("partition count: {} - {}" , string , count) }); 6v1 = map (| data | { hydroflow_plus :: runtime_support :: bincode :: serialize :: < (std :: string :: String , i32) > (& data) . unwrap () . into () }); -7v1 = dest_sink ({ use hydroflow_plus_cli_integration :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_1" ; { env . port (port) . connect_local_blocking :: < ConnectedDirect > () . into_sink () } }); +7v1 = dest_sink ({ use hydroflow_plus_deploy :: __staged :: runtime :: * ; let env = FAKE ; let port = "port_1" ; { env . port (port) . connect_local_blocking :: < ConnectedDirect > () . into_sink () } }); 1v1 -> 2v1; 2v1 -> 3v1; diff --git a/hydroflow_plus_test/src/distributed/first_ten.rs b/hydroflow_plus_test/src/distributed/first_ten.rs index 103aedb494b..5c3cb6a3733 100644 --- a/hydroflow_plus_test/src/distributed/first_ten.rs +++ b/hydroflow_plus_test/src/distributed/first_ten.rs @@ -26,7 +26,7 @@ pub fn first_ten_distributed(flow: &FlowBuilder) -> (Process, Process) { #[cfg(test)] mod tests { use hydro_deploy::Deployment; - use hydroflow_plus_cli_integration::{DeployCrateWrapper, TrybuildHost}; + use hydroflow_plus_deploy::{DeployCrateWrapper, TrybuildHost}; #[tokio::test] async fn first_ten_distributed() { diff --git a/hydroflow_plus_test_local/Cargo.toml b/hydroflow_plus_test_local/Cargo.toml index 9ca69e7c4db..25fde7396a9 100644 --- a/hydroflow_plus_test_local/Cargo.toml +++ b/hydroflow_plus_test_local/Cargo.toml @@ -8,13 +8,10 @@ edition = "2021" stageleft_devel = [] [dependencies] -hydroflow = { path = "../hydroflow", version = "^0.8.0", features = ["debugging"] } +hydroflow = { path = "../hydroflow", version = "^0.8.0", default-features = false, features = ["debugging"] } hydroflow_plus = { path = "../hydroflow_plus", version = "^0.8.0" } -tokio = { version = "1.16", features = [ "full" ] } stageleft = { path = "../stageleft", version = "^0.3.0" } -hydroflow_plus_cli_integration = { path = "../hydro_deploy/hydroflow_plus_cli_integration", version = "^0.8.0" } rand = "0.8.5" -serde = { version = "1", features = [ "derive" ] } hydroflow_plus_test_local_macro = { path = "../hydroflow_plus_test_local_macro" } @@ -23,6 +20,4 @@ stageleft_tool = { path = "../stageleft_tool", version = "^0.2.0" } [dev-dependencies] insta = "1.7.1" -hydro_deploy = { path = "../hydro_deploy/core", version = "^0.8.0" } -hydroflow_plus_cli_integration = { path = "../hydro_deploy/hydroflow_plus_cli_integration", version = "^0.8.0", features = [ "deploy" ] } futures = "0.3" diff --git a/hydroflow_plus_test_local_macro/Cargo.toml b/hydroflow_plus_test_local_macro/Cargo.toml index 1925ec93c51..fcaa24df7c9 100644 --- a/hydroflow_plus_test_local_macro/Cargo.toml +++ b/hydroflow_plus_test_local_macro/Cargo.toml @@ -10,11 +10,8 @@ path = "../hydroflow_plus_test_local/src/lib.rs" [dependencies] hydroflow_plus = { path = "../hydroflow_plus", version = "^0.8.0" } -tokio = { version = "1.16", features = [ "full" ] } stageleft = { path = "../stageleft", version = "^0.3.0" } -hydroflow_plus_cli_integration = { path = "../hydro_deploy/hydroflow_plus_cli_integration", version = "^0.8.0" } rand = "0.8.5" -serde = { version = "1", features = [ "derive" ] } [build-dependencies] stageleft_tool = { path = "../stageleft_tool", version = "^0.2.0" } diff --git a/scripts/build_dist_release.sh b/scripts/build_dist_release.sh index e935800416d..43ea621bf0f 100755 --- a/scripts/build_dist_release.sh +++ b/scripts/build_dist_release.sh @@ -52,4 +52,4 @@ fi # The CARGO_NET_GIT_FETCH_WITH_CLI="true" environment variable is a Workaround to an issue similar # to the one encountered by pytorch in https://github.com/pytorch/pytorch/issues/82174 -CARGO_NET_GIT_FETCH_WITH_CLI="true" cargo build --release --all-targets --workspace --exclude hydro_deploy --exclude hydro_cli --exclude hydroflow_plus_cli_integration --exclude hydroflow_plus_test --exclude hydroflow_plus_test_macro --target ${RUST_TARGET} +CARGO_NET_GIT_FETCH_WITH_CLI="true" cargo build --release --all-targets --workspace --exclude hydro_deploy --exclude hydro_cli --exclude hydroflow_plus_deploy --exclude hydroflow_plus_test --exclude hydroflow_plus_test_macro --target ${RUST_TARGET} diff --git a/template/hydroflow_plus/Cargo.toml b/template/hydroflow_plus/Cargo.toml index c2eefc6d420..ac0a82b9358 100644 --- a/template/hydroflow_plus/Cargo.toml +++ b/template/hydroflow_plus/Cargo.toml @@ -10,7 +10,7 @@ stageleft_devel = [] [dependencies] hydroflow_plus = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } -hydroflow_plus_cli_integration = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } +hydroflow_plus_deploy = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } stageleft = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } tokio = { version = "1.16", features = ["full"] } @@ -20,7 +20,7 @@ stageleft_tool = { git = "{{ hydroflow_git | default: 'https://github.com/hydro- [dev-dependencies] async-ssh2-lite = { version = "0.5.0", features = ["vendored-openssl"] } hydro_deploy = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } -hydroflow_plus_cli_integration = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}", features = [ +hydroflow_plus_deploy = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}", features = [ "deploy", ] } tokio-stream = "0.1.15" diff --git a/template/hydroflow_plus/examples/first_ten_distributed.rs b/template/hydroflow_plus/examples/first_ten_distributed.rs index f70b377ac3f..19c87c82087 100644 --- a/template/hydroflow_plus/examples/first_ten_distributed.rs +++ b/template/hydroflow_plus/examples/first_ten_distributed.rs @@ -1,5 +1,5 @@ use hydro_deploy::Deployment; -use hydroflow_plus_cli_integration::TrybuildHost; +use hydroflow_plus_deploy::TrybuildHost; #[tokio::main] async fn main() { diff --git a/template/hydroflow_plus/examples/first_ten_distributed_gcp.rs b/template/hydroflow_plus/examples/first_ten_distributed_gcp.rs index 91828bd951a..9ce50f956fa 100644 --- a/template/hydroflow_plus/examples/first_ten_distributed_gcp.rs +++ b/template/hydroflow_plus/examples/first_ten_distributed_gcp.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::Deployment; -use hydroflow_plus_cli_integration::TrybuildHost; +use hydroflow_plus_deploy::TrybuildHost; use tokio::sync::RwLock; static RELEASE_RUSTFLAGS: &str = diff --git a/template/hydroflow_plus/src/first_ten_distributed.rs b/template/hydroflow_plus/src/first_ten_distributed.rs index 0fab310309f..4afaf7d1d0c 100644 --- a/template/hydroflow_plus/src/first_ten_distributed.rs +++ b/template/hydroflow_plus/src/first_ten_distributed.rs @@ -20,7 +20,7 @@ pub fn first_ten_distributed(flow: &FlowBuilder) -> (Process, Process) { mod tests { use hydro_deploy::Deployment; use hydroflow_plus::futures::StreamExt; - use hydroflow_plus_cli_integration::{DeployCrateWrapper, TrybuildHost}; + use hydroflow_plus_deploy::{DeployCrateWrapper, TrybuildHost}; use tokio_stream::wrappers::UnboundedReceiverStream; #[tokio::test] diff --git a/topolotree/Cargo.toml b/topolotree/Cargo.toml index 07d545d7abe..068f17b7f49 100644 --- a/topolotree/Cargo.toml +++ b/topolotree/Cargo.toml @@ -21,7 +21,7 @@ name = "latency_measure" path = "src/latency_measure.rs" [dependencies] -hydroflow = { path = "../hydroflow", features = [ "cli_integration" ] } +hydroflow = { path = "../hydroflow", features = [ "deploy_integration" ] } hydroflow_datalog = { path = "../hydroflow_datalog" } tokio = { version = "1.16", features = [ "full" ] } diff --git a/topolotree/src/latency_measure.rs b/topolotree/src/latency_measure.rs index ef1eec10a89..724cc349193 100644 --- a/topolotree/src/latency_measure.rs +++ b/topolotree/src/latency_measure.rs @@ -7,7 +7,7 @@ use std::time::Instant; use futures::{SinkExt, StreamExt}; use hydroflow::bytes::Bytes; -use hydroflow::util::cli::{ConnectedDirect, ConnectedSink, ConnectedSource}; +use hydroflow::util::deploy::{ConnectedDirect, ConnectedSink, ConnectedSource}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; mod protocol; @@ -15,7 +15,7 @@ use protocol::*; #[tokio::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let mut start_node = ports .port("increment_start_node") .connect::() diff --git a/topolotree/src/main.rs b/topolotree/src/main.rs index 8f1282d442a..74a2fdfb30d 100644 --- a/topolotree/src/main.rs +++ b/topolotree/src/main.rs @@ -12,7 +12,7 @@ use futures::{SinkExt, Stream}; use hydroflow::bytes::{Bytes, BytesMut}; use hydroflow::hydroflow_syntax; use hydroflow::scheduled::graph::Hydroflow; -use hydroflow::util::cli::{ +use hydroflow::util::deploy::{ ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, }; @@ -217,7 +217,7 @@ async fn main() { let _self_id: u32 = args.next().unwrap().parse().unwrap(); let neighbors: Vec = args.map(|x| x.parse().unwrap()).collect(); - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let input_recv = ports .port("from_peer") @@ -289,6 +289,6 @@ async fn main() { } let f1_handle = tokio::spawn(f1); - hydroflow::util::cli::launch_flow(flow).await; + hydroflow::util::deploy::launch_flow(flow).await; f1_handle.abort(); } diff --git a/topolotree/src/pn.rs b/topolotree/src/pn.rs index 7723a112a7f..6d28dc9cf08 100644 --- a/topolotree/src/pn.rs +++ b/topolotree/src/pn.rs @@ -6,7 +6,7 @@ use std::rc::Rc; use hydroflow::hydroflow_syntax; use hydroflow::scheduled::ticks::TickInstant; use hydroflow::serde::{Deserialize, Serialize}; -use hydroflow::util::cli::{ +use hydroflow::util::deploy::{ ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, }; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; @@ -24,7 +24,7 @@ enum GossipOrIncrement { #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let my_id: Vec = serde_json::from_str(&std::env::args().nth(1).unwrap()).unwrap(); let my_id = my_id[0]; @@ -162,6 +162,6 @@ async fn main() { } let f1_handle = tokio::spawn(f1); - hydroflow::util::cli::launch_flow(df).await; + hydroflow::util::deploy::launch_flow(df).await; f1_handle.abort(); } diff --git a/topolotree/src/pn_delta.rs b/topolotree/src/pn_delta.rs index f2e6a60e0e1..23ddeee5e73 100644 --- a/topolotree/src/pn_delta.rs +++ b/topolotree/src/pn_delta.rs @@ -6,7 +6,7 @@ use std::rc::Rc; use hydroflow::hydroflow_syntax; use hydroflow::scheduled::ticks::TickInstant; use hydroflow::serde::{Deserialize, Serialize}; -use hydroflow::util::cli::{ +use hydroflow::util::deploy::{ ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, }; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; @@ -24,7 +24,7 @@ type NextStateType = (u64, bool, Rc, Vec)>>); #[hydroflow::main] async fn main() { - let ports = hydroflow::util::cli::init::<()>().await; + let ports = hydroflow::util::deploy::init::<()>().await; let my_id: Vec = serde_json::from_str(&std::env::args().nth(1).unwrap()).unwrap(); let my_id = my_id[0]; @@ -163,6 +163,6 @@ async fn main() { } let f1_handle = tokio::spawn(f1); - hydroflow::util::cli::launch_flow(df).await; + hydroflow::util::deploy::launch_flow(df).await; f1_handle.abort(); }