Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: merge shuttle-ecs-common to main #1753

Merged
merged 30 commits into from
Apr 25, 2024
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
f60106c
common: add control backend type
iulianbarbu Mar 26, 2024
222b613
backends: reinstate http otel exporter
iulianbarbu Mar 26, 2024
01b69a2
Merge remote-tracking branch 'upstream/main' into feat/shuttle-ecs-co…
oddgrd Apr 2, 2024
e40002b
feat: merge runtime updates in main ecs branch (#1709)
oddgrd Apr 2, 2024
45e5d4e
feat: add resource request struct to backends (#1718)
oddgrd Apr 5, 2024
f732be0
Merge branch 'main' into feat/shuttle-ecs-common
jonaro00 Apr 10, 2024
72839f0
feat(cargo-shuttle): beta flag, remove project list pagination logic …
jonaro00 Apr 11, 2024
d09d803
feat: add models and header for new ecs provisioner (#1730)
oddgrd Apr 11, 2024
2286c04
fix: runtime panic (#1735)
oddgrd Apr 11, 2024
cd7b71c
fix(cargo-shuttle): deploy http method
jonaro00 Apr 11, 2024
d3f6c6b
cargo-shuttle: update deploy cmd for beta platform (#1734)
iulianbarbu Apr 16, 2024
ae3c5b0
feat: add services client constructor for default headers (#1737)
oddgrd Apr 16, 2024
81ecaae
feat(cargo-shuttle): beta deploy zip file, package name (#1742)
jonaro00 Apr 17, 2024
6231811
cargo-shuttle: added beta deployment status (#1740)
iulianbarbu Apr 18, 2024
8767db4
Add log streaming in beta platform (#1743)
Kazy Apr 18, 2024
82876bd
feat(cargo-shuttle beta): enable resource list (#1744)
oddgrd Apr 19, 2024
a44d095
cargo-shuttle: added beta deployment list (#1741)
iulianbarbu Apr 19, 2024
fe9f49c
feat(c-s beta): enable resource delete (#1745)
oddgrd Apr 19, 2024
b84ea8f
feat: support secrets on beta platform (#1748)
oddgrd Apr 23, 2024
90bd111
feat(common): add building EcsState (#1752)
Kazy Apr 23, 2024
ceed492
Merge remote-tracking branch 'upstream/main' into feat/shuttle-ecs-co…
jonaro00 Apr 23, 2024
f1f4b4e
feat(cargo-shuttle): nits
jonaro00 Apr 23, 2024
b52786b
fix(cargo-shuttle): skip org project listing on beta
jonaro00 Apr 23, 2024
85a8d54
fix(backends): switch to http otel collector for compatibility with beta
jonaro00 Apr 23, 2024
3fc73ea
fix(runtime): make compatible with alpha & beta
jonaro00 Apr 23, 2024
a45addb
fix: make services compatible with beta runtime
jonaro00 Apr 23, 2024
a0e1bf1
Merge remote-tracking branch 'upstream/main' into feat/shuttle-ecs-co…
jonaro00 Apr 23, 2024
3757bfe
feat(runtime): version endpoint for runner version check
jonaro00 Apr 24, 2024
cd2c7a8
fix: review comments on ecs-common
jonaro00 Apr 24, 2024
95c155b
nit: update timeout comment
jonaro00 Apr 24, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
816 changes: 451 additions & 365 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ once_cell = "1.16.0"
opentelemetry = "0.21.0"
opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio", "logs"] }
opentelemetry-http = "0.10.0"
opentelemetry-otlp = { version = "0.14.0", features = ["logs", "grpc-tonic"] }
opentelemetry-otlp = { version = "0.14.0", features = ["logs", "grpc-tonic", "http-proto"] }
opentelemetry-proto = "0.4.0"
opentelemetry-contrib = { version = "0.4.0", features = ["datadog"] }
opentelemetry-appender-tracing = "0.2.0"
Expand Down
7 changes: 7 additions & 0 deletions backends/src/client/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,13 @@ impl ServicesApiClient {
}
}

pub fn new_with_default_headers(base: Uri, headers: HeaderMap) -> Self {
Self {
client: Self::builder().default_headers(headers).build().unwrap(),
base,
}
}

pub async fn get<T: DeserializeOwned>(
&self,
path: &str,
Expand Down
65 changes: 65 additions & 0 deletions backends/src/headers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,71 @@ impl Header for XShuttleAdminSecret {
}
}

pub static X_SHUTTLE_PROJECT_SECRET: HeaderName =
HeaderName::from_static("x-shuttle-project-secret");

/// Typed header for sending admin secrets to Shuttle components
pub struct XShuttleProjectSecret(pub String);

impl Header for XShuttleProjectSecret {
fn name() -> &'static HeaderName {
&X_SHUTTLE_PROJECT_SECRET
}

fn decode<'i, I>(values: &mut I) -> Result<Self, headers::Error>
where
Self: Sized,
I: Iterator<Item = &'i http::HeaderValue>,
{
let value = values
.next()
.ok_or_else(headers::Error::invalid)?
.to_str()
.map_err(|_| headers::Error::invalid())?
.to_string();

Ok(Self(value))
}

fn encode<E: Extend<http::HeaderValue>>(&self, values: &mut E) {
if let Ok(value) = HeaderValue::from_str(&self.0) {
values.extend(std::iter::once(value));
}
}
}

/// Used to ensure requests originate from the control service.
pub static X_SHUTTLE_CTL_SECRET: HeaderName = HeaderName::from_static("x-shuttle-ctl-secret");

pub struct XShuttleCtlSecret(pub String);

impl Header for XShuttleCtlSecret {
fn name() -> &'static HeaderName {
&X_SHUTTLE_CTL_SECRET
}

fn decode<'i, I>(values: &mut I) -> Result<Self, headers::Error>
where
Self: Sized,
I: Iterator<Item = &'i HeaderValue>,
{
let value = values
.next()
.ok_or_else(headers::Error::invalid)?
.to_str()
.map_err(|_| headers::Error::invalid())?
.to_string();

Ok(Self(value))
}

fn encode<E: Extend<HeaderValue>>(&self, values: &mut E) {
if let Ok(value) = HeaderValue::from_str(self.0.as_str()) {
values.extend(std::iter::once(value));
}
}
}

/// Used by deployers <=0.38.0. Can be removed when those are no longer supported
pub static X_SHUTTLE_PROJECT: HeaderName = HeaderName::from_static("x-shuttle-project");

Expand Down
1 change: 1 addition & 0 deletions backends/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ pub mod headers;
pub mod metrics;
mod otlp_tracing_bridge;
pub mod project_name;
pub mod resource;
pub mod trace;

#[cfg(any(test, feature = "test-utils"))]
Expand Down
16 changes: 16 additions & 0 deletions backends/src/resource.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
use serde::{Deserialize, Serialize};

/// Used by the runner service to send requests to control plane, where the requested resources
/// will be provisioned.
#[derive(Serialize, Deserialize)]
pub struct ResourceRequest {
/// The resource input returned from the runtime::load call.
pub resources: Vec<Vec<u8>>,
}

/// Used to request the provisioning or deletion of a shared DB from the provisioner service.
#[derive(Deserialize, Serialize)]
pub struct SharedDbRequest {
pub db_name: String,
pub role_name: String,
}
4 changes: 2 additions & 2 deletions backends/src/trace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ where
.tracing()
.with_exporter(
opentelemetry_otlp::new_exporter()
.tonic()
.http()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we making this change?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Making it compatible with an ALB on beta platform IIRC

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, I had some trouble making gRPC work in front of the load balancer. I can try again though, I'll let you know how it goes.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep can confirm, it's a PITA. We have an ALB in front of OTEL, which requires an HTTPS listener when targeting a gRPC target group. The issue is that to have an HTTPS listener, you need to have a valid ACM certificate attached to it, and to get a valid ACM certificate, you need to be able to verify it, which isn't possible for internal facing address.

I think we might be able to use ACM PCA (Private Certificate Authority), but that's not my priority right now.

.with_endpoint(otlp_address.clone()),
)
.with_trace_config(trace::config().with_resource(Resource::new(resources.clone())))
Expand All @@ -57,7 +57,7 @@ where
.with_log_config(Config::default().with_resource(Resource::new(resources.clone())))
.with_exporter(
opentelemetry_otlp::new_exporter()
.tonic()
.http()
jonaro00 marked this conversation as resolved.
Show resolved Hide resolved
.with_endpoint(otlp_address),
)
.install_batch(Tokio)
Expand Down
7 changes: 4 additions & 3 deletions cargo-shuttle/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ version = "0.44.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
description = "A cargo command for the shuttle platform (https://www.shuttle.rs/)"
description = "A cargo command for the Shuttle platform (https://www.shuttle.rs/)"
homepage = "https://www.shuttle.rs"

[dependencies]
Expand Down Expand Up @@ -32,10 +32,10 @@ gix = { version = "0.62.0", default-features = false, features = [
"worktree-mutation",
] }
globset = "0.4.13"
home = { workspace = true }
headers = { workspace = true }
indicatif = "0.17.3"
home = { workspace = true }
ignore = "0.4.20"
indicatif = "0.17.3"
indoc = "2.0.1"
percent-encoding = { workspace = true }
portpicker = { workspace = true }
Expand Down Expand Up @@ -65,6 +65,7 @@ url = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
walkdir = "2.3.3"
webbrowser = "0.8.2"
zip = "0.6.6"

[dev-dependencies]
assert_cmd = "2.0.6"
Expand Down
11 changes: 7 additions & 4 deletions cargo-shuttle/src/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ pub struct ShuttleArgs {
/// Turn on tracing output for cargo-shuttle and shuttle libraries.
#[arg(long, env = "SHUTTLE_DEBUG")]
pub debug: bool,
/// Target Shuttle's development environment
#[arg(long, env = "SHUTTLE_BETA", hide = true)]
pub beta: bool,

#[command(subcommand)]
pub cmd: Command,
Expand Down Expand Up @@ -158,13 +161,13 @@ pub enum DeploymentCommand {
limit: u32,

#[arg(long, default_value_t = false)]
/// Output table in `raw` format
/// Output table without borders
raw: bool,
},
/// View status of a deployment
Status {
/// ID of deployment to get status for
id: Uuid,
id: String,
},
}

Expand All @@ -173,7 +176,7 @@ pub enum ResourceCommand {
/// List all the resources for a project
List {
#[arg(long, default_value_t = false)]
/// Output table in `raw` format
/// Output table without borders
raw: bool,

#[arg(
Expand Down Expand Up @@ -219,7 +222,7 @@ pub enum ProjectCommand {
limit: u32,

#[arg(long, default_value_t = false)]
/// Output table in `raw` format
/// Output table without borders
raw: bool,
},
/// Delete a project and all linked data
Expand Down
Loading