From ba471a3dbd41e2dfe33d29d848ee00796be55ac1 Mon Sep 17 00:00:00 2001 From: oddgrd <29732646+oddgrd@users.noreply.github.com> Date: Tue, 11 Apr 2023 11:31:20 +0200 Subject: [PATCH 1/4] fix: secrets not archived in workspace crates --- cargo-shuttle/src/lib.rs | 14 ++++++++++++-- cargo-shuttle/src/logger.rs | 1 - deployer/src/deployment/queue.rs | 13 ++++++++++--- 3 files changed, 22 insertions(+), 6 deletions(-) delete mode 100644 cargo-shuttle/src/logger.rs diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 2772ea22f..7daec0cc0 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -854,8 +854,18 @@ impl Shuttle { { let dir_entry = dir_entry.context("get directory entry")?; - // It's not possible to add a directory to an archive if dir_entry.file_type().context("get file type")?.is_dir() { + let secrets_path = dir_entry.path().join("Secrets.toml"); + + // Make sure to add any `Secrets.toml` files in the subdirectories. + if secrets_path.exists() { + let path = secrets_path + .strip_prefix(base_directory) + .context("strip the base of the archive entry")?; + tar.append_path_with_name(secrets_path.clone(), path)?; + } + + // It's not possible to add a directory to an archive continue; } @@ -868,7 +878,7 @@ impl Shuttle { .context("archive entry")?; } - // Make sure to add any `Secrets.toml` files + // Make sure to add any `Secrets.toml` files in the root of the workspace. let secrets_path = self.ctx.working_directory().join("Secrets.toml"); if secrets_path.exists() { tar.append_path_with_name(secrets_path, Path::new("shuttle").join("Secrets.toml"))?; diff --git a/cargo-shuttle/src/logger.rs b/cargo-shuttle/src/logger.rs deleted file mode 100644 index 8b1378917..000000000 --- a/cargo-shuttle/src/logger.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/deployer/src/deployment/queue.rs b/deployer/src/deployment/queue.rs index 67a2192dd..bdc251739 100644 --- a/deployer/src/deployment/queue.rs +++ b/deployer/src/deployment/queue.rs @@ -172,9 +172,6 @@ impl Queued { extract_tar_gz_data(self.data.as_slice(), &project_path).await?; - let secrets = get_secrets(&project_path).await?; - set_secrets(secrets, &self.service_id, secret_recorder).await?; - info!("Building deployment"); let (tx, rx): (crossbeam_channel::Sender, _) = crossbeam_channel::bounded(0); @@ -213,8 +210,18 @@ impl Queued { }); let project_path = project_path.canonicalize()?; + + // Currently returns the first found shuttle service in a given workspace. let runtime = build_deployment(&project_path, tx.clone()).await?; + // Get the Secrets.toml from the shuttle service in the workspace. + let secrets = get_secrets(&runtime.working_directory).await?; + + // Set the secrets from the service, ignoring any Secrets.toml if it is in the root of the workspace. + // TODO: refactor this when we support starting multiple services. Do we want to set secrets in the + // workspace root? + set_secrets(secrets, &self.service_id, secret_recorder).await?; + if self.will_run_tests { info!( build_line = "Running tests before starting up", From 5c16db65dcd20961c194b253d214581805942682 Mon Sep 17 00:00:00 2001 From: oddgrd <29732646+oddgrd@users.noreply.github.com> Date: Tue, 11 Apr 2023 12:23:57 +0200 Subject: [PATCH 2/4] fix: tmp fix to generate docker-compose.rendered --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 539fc3795..119cfe8fc 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ MONGO_INITDB_ROOT_USERNAME?=mongodb MONGO_INITDB_ROOT_PASSWORD?=password ifeq ($(PROD),true) -DOCKER_COMPOSE_FILES=docker-compose.yml +DOCKER_COMPOSE_FILES=-f docker-compose.yml STACK=shuttle-prod APPS_FQDN=shuttleapp.rs DB_FQDN=db.shuttle.rs @@ -54,7 +54,7 @@ USE_TLS=enable CARGO_PROFILE=release RUST_LOG=debug else -DOCKER_COMPOSE_FILES=docker-compose.yml docker-compose.dev.yml +DOCKER_COMPOSE_FILES=-f docker-compose.yml -f docker-compose.dev.yml STACK?=shuttle-dev APPS_FQDN=unstable.shuttleapp.rs DB_FQDN=db.unstable.shuttle.rs @@ -146,6 +146,9 @@ deploy: docker-compose.yml test: cd e2e; POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) APPS_FQDN=$(APPS_FQDN) cargo test $(CARGO_TEST_FLAGS) -- --nocapture +docker-compose.rendered.yml: docker-compose.yml docker-compose.dev.yml + $(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) $(DOCKER_COMPOSE_FILES) $(DOCKER_COMPOSE_CONFIG_FLAGS) -p $(STACK) config > $@ + # Start the containers locally. This does not start panamax by default, # to start panamax locally run this command with an override for the profiles: # `make COMPOSE_PROFILES=panamax up` From 77ccaa8d37baf55e465e658a19fd98116d092ec9 Mon Sep 17 00:00:00 2001 From: oddgrd <29732646+oddgrd@users.noreply.github.com> Date: Tue, 11 Apr 2023 13:41:55 +0200 Subject: [PATCH 3/4] fix: restore docker-compose.rendered.yml --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 119cfe8fc..193c96537 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ MONGO_INITDB_ROOT_USERNAME?=mongodb MONGO_INITDB_ROOT_PASSWORD?=password ifeq ($(PROD),true) -DOCKER_COMPOSE_FILES=-f docker-compose.yml +DOCKER_COMPOSE_FILES=docker-compose.yml STACK=shuttle-prod APPS_FQDN=shuttleapp.rs DB_FQDN=db.shuttle.rs @@ -54,7 +54,7 @@ USE_TLS=enable CARGO_PROFILE=release RUST_LOG=debug else -DOCKER_COMPOSE_FILES=-f docker-compose.yml -f docker-compose.dev.yml +DOCKER_COMPOSE_FILES=docker-compose.yml docker-compose.dev.yml STACK?=shuttle-dev APPS_FQDN=unstable.shuttleapp.rs DB_FQDN=db.unstable.shuttle.rs @@ -147,7 +147,7 @@ test: cd e2e; POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) APPS_FQDN=$(APPS_FQDN) cargo test $(CARGO_TEST_FLAGS) -- --nocapture docker-compose.rendered.yml: docker-compose.yml docker-compose.dev.yml - $(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) $(DOCKER_COMPOSE_FILES) $(DOCKER_COMPOSE_CONFIG_FLAGS) -p $(STACK) config > $@ + $(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml $(DOCKER_COMPOSE_CONFIG_FLAGS) -p $(STACK) config > $@ # Start the containers locally. This does not start panamax by default, # to start panamax locally run this command with an override for the profiles: From 0a87b4fb68ec375103ac909fb31ec76c8ea728fa Mon Sep 17 00:00:00 2001 From: oddgrd <29732646+oddgrd@users.noreply.github.com> Date: Tue, 11 Apr 2023 13:53:40 +0200 Subject: [PATCH 4/4] docs: update contributing --- CONTRIBUTING.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb499d898..8962b7785 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,6 +148,10 @@ cargo run --manifest-path ../../../Cargo.toml --bin cargo-shuttle -- logs The steps outlined above starts all the services used by shuttle locally (ie. both `gateway` and `deployer`). However, sometimes you will want to quickly test changes to `deployer` only. To do this replace `make up` with the following: ```bash +# first generate the local docker-compose file +make docker-compose.rendered.yml + +# then run it docker compose -f docker-compose.rendered.yml up provisioner ``` @@ -178,13 +182,14 @@ config (which will be a file named `config.toml` in a directory named `shuttle` echo "api_key = ''" > ~/.config/shuttle/config.toml ``` -> Note: if you have [`jq`](https://github.com/stedolan/jq/wiki/Installation) installed you can combine the two ->above commands into the following: ->```bash ->curl -s -H "Authorization: Bearer test-key" localhost:8008/auth/key \ -> | jq -r '.token' \ -> | read token; echo "api_key='$token'" > ~/.config/shuttle/config.toml ->``` +> Note: The JWT will expire in 15 minutes, at which point you need to run the commands again. +> If you have [`jq`](https://github.com/stedolan/jq/wiki/Installation) installed you can combine +> the two above commands into the following: +> ```bash +> curl -s -H "Authorization: Bearer test-key" localhost:8008/auth/key \ +> | jq -r '.token' \ +> | read token; echo "api_key='$token'" > ~/.config/shuttle/config.toml +> ``` Finally we need to comment out the admin layer in the deployer handlers. So in `deployer/handlers/mod.rs`, in the `make_router` function comment out this line: `.layer(AdminSecretLayer::new(admin_secret))`.