fix: secrets not archived in workspace crates (#785)

* fix: secrets not archived in workspace crates

* fix: tmp fix to generate docker-compose.rendered

* fix: restore docker-compose.rendered.yml

* docs: update contributing

CONTRIBUTING.md

@@ -148,6 +148,10 @@ cargo run --manifest-path ../../../Cargo.toml --bin cargo-shuttle -- logs
 The steps outlined above starts all the services used by shuttle locally (ie. both `gateway` and `deployer`). However, sometimes you will want to quickly test changes to `deployer` only. To do this replace `make up` with the following:
 
 ```bash
+# first generate the local docker-compose file
+make docker-compose.rendered.yml
+
+# then run it
 docker compose -f docker-compose.rendered.yml up provisioner
 ```
 
@@ -178,13 +182,14 @@ config (which will be a file named `config.toml` in a directory named `shuttle`
 echo "api_key = '<jwt>'" > ~/.config/shuttle/config.toml
 ```
 
-> Note: if you have [`jq`](https://github.com/stedolan/jq/wiki/Installation) installed you can combine the two
->above commands into the following:
->```bash
->curl -s -H "Authorization: Bearer test-key" localhost:8008/auth/key \
->    | jq -r '.token' \
->    | read token; echo "api_key='$token'" > ~/.config/shuttle/config.toml
->```
+> Note: The JWT will expire in 15 minutes, at which point you need to run the commands again.
+> If you have [`jq`](https://github.com/stedolan/jq/wiki/Installation) installed you can combine
+> the two above commands into the following:
+> ```bash
+> curl -s -H "Authorization: Bearer test-key" localhost:8008/auth/key \
+>     | jq -r '.token' \
+>     | read token; echo "api_key='$token'" > ~/.config/shuttle/config.toml
+> ```
 
 Finally we need to comment out the admin layer in the deployer handlers. So in `deployer/handlers/mod.rs`,
 in the `make_router` function comment out this line: `.layer(AdminSecretLayer::new(admin_secret))`.

Makefile

@@ -146,6 +146,9 @@ deploy: docker-compose.yml
 test:
 	cd e2e; POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) APPS_FQDN=$(APPS_FQDN) cargo test $(CARGO_TEST_FLAGS) -- --nocapture
 
+docker-compose.rendered.yml: docker-compose.yml docker-compose.dev.yml
+	$(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml $(DOCKER_COMPOSE_CONFIG_FLAGS) -p $(STACK) config > $@
+
 # Start the containers locally. This does not start panamax by default,
 # to start panamax locally run this command with an override for the profiles:
 # `make COMPOSE_PROFILES=panamax up`

cargo-shuttle/src/lib.rs

@@ -853,8 +853,18 @@ impl Shuttle {
         {
             let dir_entry = dir_entry.context("get directory entry")?;
 
-            // It's not possible to add a directory to an archive
             if dir_entry.file_type().context("get file type")?.is_dir() {
+                let secrets_path = dir_entry.path().join("Secrets.toml");
+
+                // Make sure to add any `Secrets.toml` files in the subdirectories.
+                if secrets_path.exists() {
+                    let path = secrets_path
+                        .strip_prefix(base_directory)
+                        .context("strip the base of the archive entry")?;
+                    tar.append_path_with_name(secrets_path.clone(), path)?;
+                }
+
+                // It's not possible to add a directory to an archive
                 continue;
             }
 
@@ -867,7 +877,7 @@ impl Shuttle {
                 .context("archive entry")?;
         }
 
-        // Make sure to add any `Secrets.toml` files
+        // Make sure to add any `Secrets.toml` files in the root of the workspace.
         let secrets_path = self.ctx.working_directory().join("Secrets.toml");
         if secrets_path.exists() {
             tar.append_path_with_name(secrets_path, Path::new("shuttle").join("Secrets.toml"))?;

deployer/src/deployment/queue.rs

@@ -172,9 +172,6 @@ impl Queued {
 
         extract_tar_gz_data(self.data.as_slice(), &project_path).await?;
 
-        let secrets = get_secrets(&project_path).await?;
-        set_secrets(secrets, &self.service_id, secret_recorder).await?;
-
         info!("Building deployment");
 
         let (tx, rx): (crossbeam_channel::Sender<Message>, _) = crossbeam_channel::bounded(0);
@@ -213,8 +210,18 @@ impl Queued {
         });
 
         let project_path = project_path.canonicalize()?;
+
+        // Currently returns the first found shuttle service in a given workspace.
         let runtime = build_deployment(&project_path, tx.clone()).await?;
 
+        // Get the Secrets.toml from the shuttle service in the workspace.
+        let secrets = get_secrets(&runtime.working_directory).await?;
+
+        // Set the secrets from the service, ignoring any Secrets.toml if it is in the root of the workspace.
+        // TODO: refactor this when we support starting multiple services. Do we want to set secrets in the
+        // workspace root?
+        set_secrets(secrets, &self.service_id, secret_recorder).await?;
+
         if self.will_run_tests {
             info!(
                 build_line = "Running tests before starting up",