diff --git a/.circleci/config.yml b/.circleci/config.yml index 01a31a151..3e9650676 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,6 +118,10 @@ commands: type: string default: "" steps: + - run: + name: Set git tag in the environment" + command: | + echo TAG=$(git describe --tags) >> $BASH_ENV - run: name: Make artifact command: | @@ -125,12 +129,16 @@ commands: mv target/<< parameters.target >>/release/cargo-shuttle<< parameters.suffix >> shuttle/cargo-shuttle<< parameters.suffix >> mv LICENSE shuttle/ mv README.md shuttle/ - mkdir artifacts - tar -cvzf artifacts/cargo-shuttle-${CIRCLE_TAG}-<< parameters.target >>.tar.gz shuttle + mkdir -p artifacts/<< parameters.target >> + cp $BASH_ENV artifacts/bash.env + tar -cvzf artifacts/<< parameters.target >>/cargo-shuttle-$TAG-<< parameters.target >>.tar.gz shuttle + # Persist the bash environment to the workspace as well, we need it for the release job. + # https://discuss.circleci.com/t/share-environment-variable-between-different-job/45647/4 - persist_to_workspace: root: artifacts paths: - - cargo-shuttle-${CIRCLE_TAG}-<< parameters.target >>.tar.gz + - << parameters.target >>/* + - bash.env jobs: workspace-fmt: @@ -354,98 +362,94 @@ jobs: steps: - attach_workspace: at: artifacts + - run: + name: "Set tag in environment" + command: | + cat artifacts/bash.env >> "$BASH_ENV" + rm artifacts/bash.env - run: name: "Publish Release on GitHub" - environment: - GITHUB_TOKEN: $GITHUB_TOKEN + # Since each binary is in a sub directory named after its target, we flatten + # the artifacts directory before passing it to ghr command: | + find artifacts -mindepth 2 -type f -exec mv -t artifacts {} + go install github.com/tcnksm/ghr@v0.16.0 - ghr -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete -draft ${CIRCLE_TAG} artifacts/ + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete -draft ${TAG} ./artifacts/ workflows: version: 2 ci: jobs: - - workspace-fmt - - workspace-clippy: - name: workspace-clippy-<< matrix.framework >> - requires: - - workspace-fmt - matrix: - parameters: - framework: ["web-actix-web", "web-axum", "web-rocket", "web-poem", "web-thruster", "web-tide", "web-tower","web-warp", "web-salvo", "bot-serenity"] - - check-standalone: - matrix: - parameters: - path: - - resources/aws-rds - - resources/persist - - resources/secrets - - resources/shared-db - - resources/static-folder - - service-test: - requires: - - workspace-clippy - - platform-test: - requires: - - workspace-clippy - matrix: - parameters: - crate: ["shuttle-deployer", "cargo-shuttle", "shuttle-codegen", "shuttle-common", "shuttle-proto", "shuttle-provisioner"] - - e2e-test: - requires: - - service-test - - platform-test - - check-standalone - filters: - branches: - only: production - - build-and-push: - requires: - - e2e-test + - workspace-fmt + - workspace-clippy: + name: workspace-clippy-<< matrix.framework >> + requires: + - workspace-fmt + matrix: + parameters: + framework: ["web-actix-web", "web-axum", "web-rocket", "web-poem", "web-thruster", "web-tide", "web-tower","web-warp", "web-salvo", "bot-serenity", "bot-poise"] + - check-standalone: + matrix: + parameters: + path: + - resources/aws-rds + - resources/persist + - resources/secrets + - resources/shared-db + - resources/static-folder + - service-test: + requires: + - workspace-clippy + - platform-test: + requires: + - workspace-clippy + matrix: + parameters: + crate: ["shuttle-deployer", "cargo-shuttle", "shuttle-codegen", "shuttle-common", "shuttle-proto", "shuttle-provisioner"] + - e2e-test: + requires: + - service-test + - platform-test + - check-standalone + filters: + branches: + only: production + - build-and-push: + requires: + - e2e-test + filters: + branches: + only: production + - build-binaries-linux: + name: build-binaries-x86_64 + image: ubuntu-2204:2022.04.1 + target: x86_64-unknown-linux-musl + resource_class: medium + filters: + branches: + only: production + - build-binaries-linux: + name: build-binaries-aarch64 + image: ubuntu-2004:202101-01 + target: aarch64-unknown-linux-musl + resource_class: arm.medium + filters: + branches: + only: production + - build-binaries-windows: filters: branches: only: production - - build-binaries-linux: - name: build-binaries-x86_64 - image: ubuntu-2204:2022.04.1 - target: x86_64-unknown-linux-musl - resource_class: medium + - build-binaries-mac: filters: - tags: - only: /^v.*/ branches: - only: production - - build-binaries-linux: - name: build-binaries-aarch64 - image: ubuntu-2004:202101-01 - target: aarch64-unknown-linux-musl - resource_class: arm.medium - filters: - tags: - only: /^v.*/ - branches: - only: production - - build-binaries-windows: - filters: - tags: - only: /^v.*/ - branches: - only: production - - build-binaries-mac: - filters: - tags: - only: /^v.*/ - branches: - only: production - - publish-github-release: + only: production + - publish-github-release: requires: - build-binaries-x86_64 - build-binaries-aarch64 - build-binaries-windows - build-binaries-mac filters: - tags: - only: /^v.*/ branches: - only: production + only: production diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 145a72150..f4842765a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ Build the required images with: make images ``` -> Note: The current [Makefile](https://github.com/shuttle-hq/shuttle/blob/main/Makefile) does not work on Windows systems, if you want to build the local environment on Windows you could use [Windows Subsystem for Linux](https://learn.microsoft.com/en-us/windows/wsl/install). +> Note: The current [Makefile](https://github.com/shuttle-hq/shuttle/blob/main/Makefile) does not work on Windows systems by itself - if you want to build the local environment on Windows you could use [Windows Subsystem for Linux](https://learn.microsoft.com/en-us/windows/wsl/install). Additional Windows considerations are listed at the bottom of this page. The images get built with [cargo-chef](https://github.com/LukeMathWalker/cargo-chef) and therefore support incremental builds (most of the time). So they will be much faster to re-build after an incremental change in your code - should you wish to deploy it locally straight away. @@ -109,7 +109,7 @@ cargo run --manifest-path ../../../Cargo.toml --bin cargo-shuttle -- logs The steps outlined above starts all the services used by shuttle locally (ie. both `gateway` and `deployer`). However, sometimes you will want to quickly test changes to `deployer` only. To do this replace `make up` with the following: ```bash -docker-compose -f docker-compose.rendered.yml up provisioner +docker compose -f docker-compose.rendered.yml up provisioner ``` This prevents `gateway` from starting up. Now you can start deployer only using: @@ -224,3 +224,22 @@ The rest are the following libraries: - `e2e` just contains tests which starts up the `deployer` in a container and then deploys services to it using `cargo-shuttle`. Lastly, the `user service` is not a folder in this repository, but is the user service that will be deployed by `deployer`. + +## Windows Considerations +Currently, if you try to use 'make images' on Windows, you may find that the shell files cannot be read by Bash/WSL. This is due to the fact that Windows may have pulled the files in CRLF format rather than LF[^1], which causes problems with Bash as to run the commands, Linux needs the file in LF format. + +Thankfully, we can fix this problem by simply using the `git config core.autocrlf` command to change how Git handles line endings. It takes a single argument: + +``` +git config --global core.autocrlf input +``` + +This should allow you to run `make images` and other Make commands with no issues. + +If you need to change it back for whatever reason, you can just change the last argument from 'input' to 'true' like so: +``` +git config --global core.autocrlf true +``` +After you run this command, you should be able to checkout projects that are maintained using CRLF (Windows) again. + +[^1]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Configuration#_core_autocrlf diff --git a/Cargo.lock b/Cargo.lock index 01d04921a..ff2cff0bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1332,9 +1332,9 @@ dependencies = [ [[package]] name = "cap-fs-ext" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ed685fe2949d035b080fbe7536b944efffb648af1d34630aa887ca2b132d2b" +checksum = "0f8079425cfd20227020f2bff1320710ca68d6eddb4f64aba8e2741b2b4d8133" dependencies = [ "cap-primitives", "cap-std", @@ -1344,9 +1344,9 @@ dependencies = [ [[package]] name = "cap-primitives" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0315442c0232cb9a1c2be55ee289a0e9bf5fd0b0f162be8e7f16673e095f5e09" +checksum = "84bf8faa0b6397a4e26082818be03641a40e3aba1afc4ec44cbd6228c73c3a61" dependencies = [ "ambient-authority", "fs-set-times", @@ -1361,9 +1361,9 @@ dependencies = [ [[package]] name = "cap-rand" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b78d30c0b3c656f6193bef0697cff6bd903d9b2b1437c7af3d35a6a9d1a7af2e" +checksum = "53df044ddcb88611e19b712211b342ab106105cf658406f5ed4ee09ab10ed727" dependencies = [ "ambient-authority", "rand 0.8.5", @@ -1371,9 +1371,9 @@ dependencies = [ [[package]] name = "cap-std" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9256648eae510b29aae4d52ed71877239a61f9a2494d23ddad7fb6f50e5de8" +checksum = "e4ad2b9e262a5c3b67ee92e4b9607ace704384c50c32aa6017a9282ddf15df20" dependencies = [ "cap-primitives", "io-extras", @@ -1384,9 +1384,9 @@ dependencies = [ [[package]] name = "cap-time-ext" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384a81c0fb05dbd361f157fd2c822e3d16709540e49300d26a27d3d57f02e8cb" +checksum = "6dcbdbcced5c88b20f27c637faaed5dd283898cbefea48d2d8f3dcfaf048e5cc" dependencies = [ "cap-primitives", "once_cell", @@ -2387,6 +2387,7 @@ dependencies = [ "hashbrown", "lock_api", "parking_lot_core 0.9.3", + "serde", ] [[package]] @@ -3623,9 +3624,9 @@ dependencies = [ [[package]] name = "io-extras" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad797ac2cd70ff82f6d9246d36762b41c1db15b439fd48bcb70914269642354" +checksum = "b87bc110777311d7832025f38c4ab0f089f764644009edef3b5cbadfedee8c40" dependencies = [ "io-lifetimes", "windows-sys 0.42.0", @@ -3703,9 +3704,9 @@ checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "ittapi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "663fe0550070071ff59e981864a9cd3ee1c869ed0a088140d9ac4dc05ea6b1a1" +checksum = "e8c4f6ff06169ce7048dac5150b1501c7e3716a929721aeb06b87e51a43e42f4" dependencies = [ "anyhow", "ittapi-sys", @@ -3714,9 +3715,9 @@ dependencies = [ [[package]] name = "ittapi-sys" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e21911b7183f38c71d75ab478a527f314e28db51027037ece2e5511ed9410703" +checksum = "87e078cce01485f418bae3beb34dd604aaedf2065502853c7da17fbce8e64eda" dependencies = [ "cc", ] @@ -4752,6 +4753,37 @@ dependencies = [ "syn 1.0.104", ] +[[package]] +name = "poise" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca787e4e516076de1995a83ee05fbdfed71d072ea0da3df018318db42a87803" +dependencies = [ + "async-trait", + "derivative", + "futures-core", + "futures-util", + "log", + "once_cell", + "parking_lot 0.12.1", + "poise_macros", + "regex", + "serenity", + "tokio", +] + +[[package]] +name = "poise_macros" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b80c1f4e04114527f9d41ed6bb31707a095276f51bb0aef3ca11f062b25a67c4" +dependencies = [ + "darling 0.14.1", + "proc-macro2 1.0.47", + "quote 1.0.21", + "syn 1.0.104", +] + [[package]] name = "polling" version = "2.2.0" @@ -6055,12 +6087,16 @@ dependencies = [ "bitflags", "bytes 1.3.0", "cfg-if 1.0.0", + "chrono", + "dashmap", "flate2", "futures", "mime", "mime_guess", + "parking_lot 0.12.1", "percent-encoding", "reqwest", + "rustversion", "serde", "serde-value", "serde_json", @@ -6404,6 +6440,7 @@ dependencies = [ "num_cpus", "pipe", "poem", + "poise", "portpicker", "rocket", "salvo", @@ -8729,9 +8766,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.1+zstd.1.5.2" +version = "2.0.4+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +checksum = "4fa202f2ef00074143e219d15b62ffc317d17cc33909feac471c044087cad7b0" dependencies = [ "cc", "libc", diff --git a/Containerfile b/Containerfile index b866f55b4..9c382250a 100644 --- a/Containerfile +++ b/Containerfile @@ -1,5 +1,6 @@ #syntax=docker/dockerfile-upstream:1.4.0-rc1 -FROM rust:1.65.0-buster as shuttle-build +ARG RUSTUP_TOOLCHAIN +FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-build RUN apt-get update &&\ apt-get install -y curl # download protoc binary and unzip it in usr/bin @@ -26,7 +27,8 @@ COPY --from=cache /build . ARG folder RUN cargo build --bin shuttle-${folder} -FROM rust:1.65.0-buster as shuttle-common +ARG RUSTUP_TOOLCHAIN +FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common RUN apt-get update &&\ apt-get install -y curl RUN rustup component add rust-src @@ -37,4 +39,6 @@ ARG folder COPY ${folder}/prepare.sh /prepare.sh RUN /prepare.sh COPY --from=builder /build/target/debug/shuttle-${folder} /usr/local/bin/service +ARG RUSTUP_TOOLCHAIN +ENV RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN} ENTRYPOINT ["/usr/local/bin/service"] diff --git a/Makefile b/Makefile index ee94af8f2..e57ba12b4 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,10 @@ endif BUILDX_FLAGS=$(BUILDX_OP) $(PLATFORM_FLAGS) $(CACHE_FLAGS) +# the rust version used by our containers, and as an override for our deployers +# ensuring all user crates are compiled with the same rustc toolchain +RUSTUP_TOOLCHAIN=1.65.0 + TAG?=$(shell git describe --tags) BACKEND_TAG?=$(TAG) DEPLOYER_TAG?=$(TAG) @@ -107,6 +111,7 @@ down: docker-compose.rendered.yml shuttle-%: ${SRC} Cargo.lock docker buildx build \ --build-arg folder=$(*) \ + --build-arg RUSTUP_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) \ --tag $(CONTAINER_REGISTRY)/$(*):$(COMMIT_SHA) \ --tag $(CONTAINER_REGISTRY)/$(*):$(TAG) \ --tag $(CONTAINER_REGISTRY)/$(*):latest \ diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index d2a37e792..865cd4e67 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -63,7 +63,7 @@ path = "../resources/secrets" [dependencies.shuttle-service] workspace = true -features = ["loader", "codegen"] +features = ["loader"] [features] vendored-openssl = ["openssl/vendored"] diff --git a/cargo-shuttle/README.md b/cargo-shuttle/README.md index 964a2a478..8a4d3ff2c 100644 --- a/cargo-shuttle/README.md +++ b/cargo-shuttle/README.md @@ -77,10 +77,15 @@ To initialize a shuttle project with boilerplates, run `cargo shuttle init [OPTI Currently, `cargo shuttle init` supports the following frameworks: - `--axum`: for [axum](https://github.com/tokio-rs/axum) framework +- `--actix-web`: for [actix web](https://actix.rs/) framework - `--poem`: for [poem](https://github.com/poem-web/poem) framework - `--rocket`: for [rocket](https://rocket.rs/) framework +- `--salvo`: for [salvo](https://salvo.rs/) framework +- `--serenity`: for [serenity](https://serenity.rs/) discord bot framework +- `--thruster`: for [thruster](https://github.com/thruster-rs/Thruster) framework - `--tide`: for [tide](https://github.com/http-rs/tide) framework - `--tower`: for [tower](https://github.com/tower-rs/tower) library +- `--warp`: for [warp](https://github.com/seanmonstar/warp) framework For example, running the following command will initialize a project for [rocket](https://rocket.rs/): @@ -177,14 +182,6 @@ Check the logs of your deployed shuttle project with: $ cargo shuttle logs ``` -### Subcommand: `auth` - -Run the following to create user credentials for shuttle platform: - -```sh -$ cargo shuttle auth your-desired-username -``` - ### Subcommand: `delete` Once you are done with a deployment, you can delete it by running: diff --git a/cargo-shuttle/src/args.rs b/cargo-shuttle/src/args.rs index 9976d3f6d..e7f0d8d59 100644 --- a/cargo-shuttle/src/args.rs +++ b/cargo-shuttle/src/args.rs @@ -85,8 +85,6 @@ pub enum Command { Delete, /// manage secrets for this shuttle service Secrets, - /// create user credentials for the shuttle platform - Auth(AuthArgs), /// login to the shuttle platform Login(LoginArgs), /// run a shuttle service locally @@ -111,6 +109,8 @@ pub enum DeploymentCommand { pub enum ProjectCommand { /// create an environment for this project on shuttle New, + /// list all projects belonging to the calling account + List, /// remove this project environment from shuttle Rm, /// show the status of this project's environment on shuttle @@ -128,13 +128,6 @@ pub struct LoginArgs { pub api_key: Option, } -#[derive(Parser)] -pub struct AuthArgs { - /// the desired username for the shuttle platform - #[clap()] - pub username: String, -} - #[derive(Parser)] pub struct DeployArgs { /// allow dirty working directories to be packaged @@ -150,42 +143,48 @@ pub struct RunArgs { /// port to start service on #[clap(long, default_value = "8000")] pub port: u16, + /// use 0.0.0.0 instead of localhost (for usage with local external devices) + #[clap(long)] + pub external: bool, } #[derive(Parser, Debug)] pub struct InitArgs { /// Initialize with actix-web framework - #[clap(long="actix-web", conflicts_with_all = &["axum", "rocket", "tide", "tower", "poem", "serenity", "warp", "salvo", "thruster", "no-framework"])] + #[clap(long="actix-web", conflicts_with_all = &["axum", "rocket", "tide", "tower", "poem", "serenity", "poise", "warp", "salvo", "thruster", "no-framework"])] pub actix_web: bool, /// Initialize with axum framework - #[clap(long, conflicts_with_all = &["actix-web","rocket", "tide", "tower", "poem", "serenity", "warp", "salvo", "thruster", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","rocket", "tide", "tower", "poem", "serenity", "poise", "warp", "salvo", "thruster", "no-framework"])] pub axum: bool, /// Initialize with rocket framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "tide", "tower", "poem", "serenity", "warp", "salvo", "thruster", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "tide", "tower", "poem", "serenity", "poise", "warp", "salvo", "thruster", "no-framework"])] pub rocket: bool, /// Initialize with tide framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tower", "poem", "serenity", "warp", "salvo", "thruster", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tower", "poem", "serenity", "poise", "warp", "salvo", "thruster", "no-framework"])] pub tide: bool, /// Initialize with tower framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "poem", "serenity", "warp", "salvo", "thruster", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "poem", "serenity", "poise", "warp", "salvo", "thruster", "no-framework"])] pub tower: bool, /// Initialize with poem framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "serenity", "warp", "salvo", "thruster", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "serenity", "poise", "warp", "salvo", "thruster", "no-framework"])] pub poem: bool, /// Initialize with salvo framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "serenity", "thruster", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "serenity", "poise", "thruster", "no-framework"])] pub salvo: bool, /// Initialize with serenity framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "salvo", "thruster", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "poise", "salvo", "thruster", "no-framework"])] pub serenity: bool, + /// Initialize with poise framework + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "serenity", "salvo", "thruster", "no-framework"])] + pub poise: bool, /// Initialize with warp framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "serenity", "salvo", "thruster", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "serenity", "poise", "salvo", "thruster", "no-framework"])] pub warp: bool, /// Initialize with thruster framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "salvo", "serenity", "no-framework"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "salvo", "serenity", "poise", "no-framework"])] pub thruster: bool, /// Initialize without a framework - #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "salvo", "serenity", "thruster"])] + #[clap(long, conflicts_with_all = &["actix-web","axum", "rocket", "tide", "tower", "poem", "warp", "salvo", "serenity", "poise", "thruster"])] pub no_framework: bool, /// Whether to create the environment for this project on Shuttle #[clap(long)] @@ -216,6 +215,8 @@ impl InitArgs { Some(Framework::Poem) } else if self.salvo { Some(Framework::Salvo) + } else if self.poise { + Some(Framework::Poise) } else if self.serenity { Some(Framework::Serenity) } else if self.warp { @@ -264,6 +265,7 @@ mod tests { poem: false, salvo: false, serenity: false, + poise: false, warp: false, thruster: false, no_framework: false, @@ -281,6 +283,7 @@ mod tests { "poem" => init_args.poem = true, "salvo" => init_args.salvo = true, "serenity" => init_args.serenity = true, + "poise" => init_args.poise = true, "warp" => init_args.warp = true, "thruster" => init_args.thruster = true, "none" => init_args.no_framework = true, diff --git a/cargo-shuttle/src/client.rs b/cargo-shuttle/src/client.rs index 4f3c4c778..3a7e98263 100644 --- a/cargo-shuttle/src/client.rs +++ b/cargo-shuttle/src/client.rs @@ -7,7 +7,7 @@ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, RequestBuilder}; use reqwest_retry::policies::ExponentialBackoff; use reqwest_retry::RetryTransientMiddleware; use serde::Deserialize; -use shuttle_common::models::{deployment, project, secret, service, user, ToJson}; +use shuttle_common::models::{deployment, project, secret, service, ToJson}; use shuttle_common::project::ProjectName; use shuttle_common::{ApiKey, ApiUrl, LogItem}; use tokio::net::TcpStream; @@ -33,16 +33,6 @@ impl Client { self.api_key = Some(api_key); } - pub async fn auth(&self, username: String) -> Result { - let path = format!("/users/{}", username); - - self.post(path, Option::::None) - .await - .context("failed to get API key from Shuttle server")? - .to_json() - .await - } - pub async fn deploy( &self, data: Vec, @@ -122,6 +112,12 @@ impl Client { self.get(path).await } + pub async fn get_projects_list(&self) -> Result> { + let path = "/projects".to_string(); + + self.get(path).await + } + pub async fn delete_project(&self, project: &ProjectName) -> Result { let path = format!("/projects/{}", project.as_str()); diff --git a/cargo-shuttle/src/init.rs b/cargo-shuttle/src/init.rs index ea09f305a..b36c1c52c 100644 --- a/cargo-shuttle/src/init.rs +++ b/cargo-shuttle/src/init.rs @@ -20,6 +20,7 @@ pub enum Framework { Poem, Salvo, Serenity, + Poise, Warp, Thruster, None, @@ -39,6 +40,7 @@ impl Framework { Framework::Poem => Box::new(ShuttleInitPoem), Framework::Salvo => Box::new(ShuttleInitSalvo), Framework::Serenity => Box::new(ShuttleInitSerenity), + Framework::Poise => Box::new(ShuttleInitPoise), Framework::Warp => Box::new(ShuttleInitWarp), Framework::Thruster => Box::new(ShuttleInitThruster), Framework::None => Box::new(ShuttleInitNoOp), @@ -446,6 +448,106 @@ impl ShuttleInit for ShuttleInitSerenity { } } +pub struct ShuttleInitPoise; + +impl ShuttleInit for ShuttleInitPoise { + fn set_cargo_dependencies( + &self, + dependencies: &mut Table, + manifest_path: &Path, + url: &Url, + get_dependency_version_fn: GetDependencyVersionFn, + ) { + set_inline_table_dependency_features( + "shuttle-service", + dependencies, + vec!["bot-poise".to_string()], + ); + + set_key_value_dependency_version( + "anyhow", + dependencies, + manifest_path, + url, + false, + get_dependency_version_fn, + ); + + set_key_value_dependency_version( + "poise", + dependencies, + manifest_path, + url, + false, + get_dependency_version_fn, + ); + + set_key_value_dependency_version( + "shuttle-secrets", + dependencies, + manifest_path, + url, + false, + get_dependency_version_fn, + ); + + set_key_value_dependency_version( + "tracing", + dependencies, + manifest_path, + url, + false, + get_dependency_version_fn, + ); + } + + fn get_boilerplate_code_for_framework(&self) -> &'static str { + indoc! {r#" + use anyhow::Context as _; + use poise::serenity_prelude as serenity; + use shuttle_secrets::SecretStore; + use shuttle_service::ShuttlePoise; + + struct Data {} // User data, which is stored and accessible in all command invocations + type Error = Box; + type Context<'a> = poise::Context<'a, Data, Error>; + + /// Responds with "world!" + #[poise::command(slash_command)] + async fn hello(ctx: Context<'_>) -> Result<(), Error> { + ctx.say("world!").await?; + Ok(()) + } + + #[shuttle_service::main] + async fn poise(#[shuttle_secrets::Secrets] secret_store: SecretStore) -> ShuttlePoise { + // Get the discord token set in `Secrets.toml` + let discord_token = secret_store + .get("DISCORD_TOKEN") + .context("'DISCORD_TOKEN' was not found")?; + + let framework = poise::Framework::builder() + .options(poise::FrameworkOptions { + commands: vec![hello()], + ..Default::default() + }) + .token(discord_token) + .intents(serenity::GatewayIntents::non_privileged()) + .setup(|ctx, _ready, framework| { + Box::pin(async move { + poise::builtins::register_globally(ctx, &framework.options().commands).await?; + Ok(Data {}) + }) + }) + .build() + .await + .map_err(shuttle_service::error::CustomError::new)?; + + Ok(framework) + }"#} + } +} + pub struct ShuttleInitTower; impl ShuttleInit for ShuttleInitTower { @@ -1125,6 +1227,41 @@ mod shuttle_init_tests { assert_eq!(cargo_toml.to_string(), expected); } + #[test] + fn test_set_cargo_dependencies_poise() { + let mut cargo_toml = cargo_toml_factory(); + let dependencies = cargo_toml["dependencies"].as_table_mut().unwrap(); + let manifest_path = PathBuf::new(); + let url = Url::parse("https://shuttle.rs").unwrap(); + + set_inline_table_dependency_version( + "shuttle-service", + dependencies, + &manifest_path, + &url, + false, + mock_get_latest_dependency_version, + ); + + ShuttleInitPoise.set_cargo_dependencies( + dependencies, + &manifest_path, + &url, + mock_get_latest_dependency_version, + ); + + let expected = indoc! {r#" + [dependencies] + shuttle-service = { version = "1.0", features = ["bot-poise"] } + anyhow = "1.0" + poise = "1.0" + shuttle-secrets = "1.0" + tracing = "1.0" + "#}; + + assert_eq!(cargo_toml.to_string(), expected); + } + #[test] fn test_set_cargo_dependencies_warp() { let mut cargo_toml = cargo_toml_factory(); diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 3709bffff..56e4edea9 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -14,7 +14,6 @@ use std::net::{Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use anyhow::{anyhow, bail, Context, Result}; -use args::AuthArgs; pub use args::{Args, Command, DeployArgs, InitArgs, LoginArgs, ProjectArgs, RunArgs}; use cargo_metadata::Message; use clap::CommandFactory; @@ -95,11 +94,11 @@ impl Shuttle { Command::Delete => self.delete(&client).await, Command::Clean => self.clean(&client).await, Command::Secrets => self.secrets(&client).await, - Command::Auth(auth_args) => self.auth(auth_args, &client).await, Command::Project(ProjectCommand::New) => self.project_create(&client).await, Command::Project(ProjectCommand::Status { follow }) => { self.project_status(&client, follow).await } + Command::Project(ProjectCommand::List) => self.projects_list(&client).await, Command::Project(ProjectCommand::Rm) => self.project_delete(&client).await, _ => { unreachable!("commands that don't need a client have already been matched") @@ -250,17 +249,6 @@ impl Shuttle { Ok(()) } - async fn auth(&mut self, auth_args: AuthArgs, client: &Client) -> Result<()> { - let user = client.auth(auth_args.username).await?; - - self.ctx.set_api_key(user.key)?; - - println!("User authorized!!!"); - println!("Run `cargo shuttle init --help` next"); - - Ok(()) - } - async fn delete(&self, client: &Client) -> Result<()> { let service = client.delete_service(self.ctx.project_name()).await?; @@ -493,7 +481,13 @@ impl Shuttle { trace!(response = ?response, "client response: "); - let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), run_args.port); + let addr = if run_args.external { + std::net::IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)) + } else { + Ipv4Addr::LOCALHOST.into() + }; + + let addr = SocketAddr::new(addr, run_args.port); println!( "\n{:>12} {} on http://{}", @@ -581,6 +575,15 @@ impl Shuttle { Ok(()) } + async fn projects_list(&self, client: &Client) -> Result<()> { + let projects = client.get_projects_list().await?; + let projects_table = project::get_table(&projects); + + println!("{projects_table}"); + + Ok(()) + } + async fn project_status(&self, client: &Client, follow: bool) -> Result<()> { match follow { true => { diff --git a/cargo-shuttle/tests/integration/run.rs b/cargo-shuttle/tests/integration/run.rs index 318145ffb..3207d019c 100644 --- a/cargo-shuttle/tests/integration/run.rs +++ b/cargo-shuttle/tests/integration/run.rs @@ -5,10 +5,18 @@ use std::{fs::canonicalize, process::exit, time::Duration}; use tokio::time::sleep; /// creates a `cargo-shuttle` run instance with some reasonable defaults set. -async fn cargo_shuttle_run(working_directory: &str) -> u16 { +async fn cargo_shuttle_run(working_directory: &str, external: bool) -> String { let working_directory = canonicalize(working_directory).unwrap(); + let port = pick_unused_port().unwrap(); - let run_args = RunArgs { port }; + + let url = if !external { + format!("http://localhost:{port}") + } else { + format!("http://0.0.0.0:{port}") + }; + + let run_args = RunArgs { port, external }; let runner = Shuttle::new().unwrap().run(Args { api_url: Some("http://shuttle.invalid:80".to_string()), @@ -34,12 +42,7 @@ async fn cargo_shuttle_run(working_directory: &str) -> u16 { tokio::spawn(runner); // Wait for service to be responsive - while (reqwest::Client::new() - .get(format!("http://localhost:{port}")) - .send() - .await) - .is_err() - { + while (reqwest::Client::new().get(url.clone()).send().await).is_err() { println!( "waiting for '{}' to start up...", working_directory.display() @@ -47,15 +50,15 @@ async fn cargo_shuttle_run(working_directory: &str) -> u16 { sleep(Duration::from_millis(350)).await; } - port + url } #[tokio::test(flavor = "multi_thread")] async fn rocket_hello_world() { - let port = cargo_shuttle_run("../examples/rocket/hello-world").await; + let url = cargo_shuttle_run("../examples/rocket/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -68,10 +71,10 @@ async fn rocket_hello_world() { #[tokio::test(flavor = "multi_thread")] async fn rocket_secrets() { - let port = cargo_shuttle_run("../examples/rocket/secrets").await; + let url = cargo_shuttle_run("../examples/rocket/secrets", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/secret")) + .get(format!("{url}/secret")) .send() .await .unwrap() @@ -85,11 +88,11 @@ async fn rocket_secrets() { // This example uses a shared Postgres. Thus local runs should create a docker container for it. #[tokio::test(flavor = "multi_thread")] async fn rocket_postgres() { - let port = cargo_shuttle_run("../examples/rocket/postgres").await; + let url = cargo_shuttle_run("../examples/rocket/postgres", false).await; let client = reqwest::Client::new(); let post_text = client - .post(format!("http://localhost:{port}/todo")) + .post(format!("{url}/todo")) .body("{\"note\": \"Deploy to shuttle\"}") .send() .await @@ -101,7 +104,7 @@ async fn rocket_postgres() { assert_eq!(post_text, "{\"id\":1,\"note\":\"Deploy to shuttle\"}"); let request_text = client - .get(format!("http://localhost:{port}/todo/1")) + .get(format!("{url}/todo/1")) .send() .await .unwrap() @@ -114,11 +117,11 @@ async fn rocket_postgres() { #[tokio::test(flavor = "multi_thread")] async fn rocket_authentication() { - let port = cargo_shuttle_run("../examples/rocket/authentication").await; + let url = cargo_shuttle_run("../examples/rocket/authentication", false).await; let client = reqwest::Client::new(); let public_text = client - .get(format!("http://localhost:{port}/public")) + .get(format!("{url}/public")) .send() .await .unwrap() @@ -132,7 +135,7 @@ async fn rocket_authentication() { ); let private_status = client - .get(format!("http://localhost:{port}/private")) + .get(format!("{url}/private")) .send() .await .unwrap() @@ -141,7 +144,7 @@ async fn rocket_authentication() { assert_eq!(private_status, StatusCode::FORBIDDEN); let body = client - .post(format!("http://localhost:{port}/login")) + .post(format!("{url}/login")) .body("{\"username\": \"username\", \"password\": \"password\"}") .send() .await @@ -153,7 +156,7 @@ async fn rocket_authentication() { let token = format!("Bearer {}", json["token"].as_str().unwrap()); let private_text = client - .get(format!("http://localhost:{port}/private")) + .get(format!("{url}/private")) .header("Authorization", token) .send() .await @@ -170,10 +173,10 @@ async fn rocket_authentication() { #[tokio::test(flavor = "multi_thread")] async fn actix_web_hello_world() { - let port = cargo_shuttle_run("../examples/actix-web/hello-world").await; + let url = cargo_shuttle_run("../examples/actix-web/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -186,10 +189,10 @@ async fn actix_web_hello_world() { #[tokio::test(flavor = "multi_thread")] async fn axum_hello_world() { - let port = cargo_shuttle_run("../examples/axum/hello-world").await; + let url = cargo_shuttle_run("../examples/axum/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -202,10 +205,10 @@ async fn axum_hello_world() { #[tokio::test(flavor = "multi_thread")] async fn tide_hello_world() { - let port = cargo_shuttle_run("../examples/tide/hello-world").await; + let url = cargo_shuttle_run("../examples/tide/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -218,10 +221,10 @@ async fn tide_hello_world() { #[tokio::test(flavor = "multi_thread")] async fn tower_hello_world() { - let port = cargo_shuttle_run("../examples/tower/hello-world").await; + let url = cargo_shuttle_run("../examples/tower/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -234,10 +237,10 @@ async fn tower_hello_world() { #[tokio::test(flavor = "multi_thread")] async fn warp_hello_world() { - let port = cargo_shuttle_run("../examples/warp/hello-world").await; + let url = cargo_shuttle_run("../examples/warp/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -250,10 +253,10 @@ async fn warp_hello_world() { #[tokio::test(flavor = "multi_thread")] async fn poem_hello_world() { - let port = cargo_shuttle_run("../examples/poem/hello-world").await; + let url = cargo_shuttle_run("../examples/poem/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -267,11 +270,11 @@ async fn poem_hello_world() { // This example uses a shared Postgres. Thus local runs should create a docker container for it. #[tokio::test(flavor = "multi_thread")] async fn poem_postgres() { - let port = cargo_shuttle_run("../examples/poem/postgres").await; + let url = cargo_shuttle_run("../examples/poem/postgres", false).await; let client = reqwest::Client::new(); let post_text = client - .post(format!("http://localhost:{port}/todo")) + .post(format!("{url}/todo")) .body("{\"note\": \"Deploy to shuttle\"}") .header("content-type", "application/json") .send() @@ -284,7 +287,7 @@ async fn poem_postgres() { assert_eq!(post_text, "{\"id\":1,\"note\":\"Deploy to shuttle\"}"); let request_text = client - .get(format!("http://localhost:{port}/todo/1")) + .get(format!("{url}/todo/1")) .send() .await .unwrap() @@ -298,12 +301,12 @@ async fn poem_postgres() { // This example uses a shared MongoDb. Thus local runs should create a docker container for it. #[tokio::test(flavor = "multi_thread")] async fn poem_mongodb() { - let port = cargo_shuttle_run("../examples/poem/mongodb").await; + let url = cargo_shuttle_run("../examples/poem/mongodb", false).await; let client = reqwest::Client::new(); // Post a todo note and get the persisted todo objectId let post_text = client - .post(format!("http://localhost:{port}/todo")) + .post(format!("{url}/todo")) .body("{\"note\": \"Deploy to shuttle\"}") .header("content-type", "application/json") .send() @@ -317,7 +320,7 @@ async fn poem_mongodb() { assert_eq!(post_text.len(), 24); let request_text = client - .get(format!("http://localhost:{port}/todo/{post_text}")) + .get(format!("{url}/todo/{post_text}")) .send() .await .unwrap() @@ -330,10 +333,10 @@ async fn poem_mongodb() { #[tokio::test(flavor = "multi_thread")] async fn salvo_hello_world() { - let port = cargo_shuttle_run("../examples/salvo/hello-world").await; + let url = cargo_shuttle_run("../examples/salvo/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -346,10 +349,10 @@ async fn salvo_hello_world() { #[tokio::test(flavor = "multi_thread")] async fn thruster_hello_world() { - let port = cargo_shuttle_run("../examples/thruster/hello-world").await; + let url = cargo_shuttle_run("../examples/thruster/hello-world", false).await; let request_text = reqwest::Client::new() - .get(format!("http://localhost:{port}/hello")) + .get(format!("{url}/hello")) .send() .await .unwrap() @@ -359,3 +362,19 @@ async fn thruster_hello_world() { assert_eq!(request_text, "Hello, World!"); } + +#[tokio::test(flavor = "multi_thread")] +async fn rocket_hello_world_with_router_ip() { + let url = cargo_shuttle_run("../examples/rocket/hello-world", true).await; + + let request_text = reqwest::Client::new() + .get(format!("{url}/hello")) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!(request_text, "Hello, world!"); +} diff --git a/common/src/models/project.rs b/common/src/models/project.rs index 171c4e46a..82ba3f927 100644 --- a/common/src/models/project.rs +++ b/common/src/models/project.rs @@ -1,4 +1,7 @@ -use comfy_table::Color; +use comfy_table::{ + modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL, Cell, CellAlignment, Color, + ContentArrangement, Table, +}; use crossterm::style::Stylize; use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; @@ -15,6 +18,7 @@ pub struct Response { #[strum(serialize_all = "lowercase")] pub enum State { Creating, + Attaching, Starting, Started, Ready, @@ -39,10 +43,10 @@ impl Display for Response { impl State { pub fn get_color(&self) -> Color { match self { - State::Creating | State::Starting | State::Started => Color::Cyan, - State::Ready => Color::Green, - State::Stopped | State::Stopping | State::Destroying | State::Destroyed => Color::Blue, - State::Errored => Color::Red, + Self::Creating | Self::Attaching | Self::Starting | Self::Started => Color::Cyan, + Self::Ready => Color::Green, + Self::Stopped | Self::Stopping | Self::Destroying | Self::Destroyed => Color::Blue, + Self::Errored => Color::Red, } } } @@ -52,3 +56,39 @@ pub struct AdminResponse { pub project_name: String, pub account_name: String, } + +pub fn get_table(projects: &Vec) -> String { + if projects.is_empty() { + format!( + "{}\n", + "No projects are linked to this account".yellow().bold() + ) + } else { + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .apply_modifier(UTF8_ROUND_CORNERS) + .set_content_arrangement(ContentArrangement::DynamicFullWidth) + .set_header(vec![ + Cell::new("Project Name").set_alignment(CellAlignment::Center), + Cell::new("Status").set_alignment(CellAlignment::Center), + ]); + + for project in projects.iter() { + table.add_row(vec![ + Cell::new(&project.name), + Cell::new(&project.state) + .fg(project.state.get_color()) + .set_alignment(CellAlignment::Center), + ]); + } + + format!( + r#" +These projects are linked to this account +{} +"#, + table, + ) + } +} diff --git a/deployer/Cargo.toml b/deployer/Cargo.toml index fa10591f3..71e9fa3b4 100644 --- a/deployer/Cargo.toml +++ b/deployer/Cargo.toml @@ -53,7 +53,7 @@ workspace = true [dependencies.shuttle-service] workspace = true -features = ["loader", "codegen"] +features = ["loader"] [dev-dependencies] ctor = "0.1.26" diff --git a/deployer/src/deployment/storage_manager.rs b/deployer/src/deployment/storage_manager.rs new file mode 100644 index 000000000..5a5fa1300 --- /dev/null +++ b/deployer/src/deployment/storage_manager.rs @@ -0,0 +1,69 @@ +use std::{fs, io, path::PathBuf}; + +use uuid::Uuid; + +/// Manager to take care of directories for storing project, services and deployment files +#[derive(Clone)] +pub struct StorageManager { + artifacts_path: PathBuf, +} + +impl StorageManager { + pub fn new(artifacts_path: PathBuf) -> Self { + Self { artifacts_path } + } + + /// Path of the directory that contains extracted service Cargo projects. + pub fn builds_path(&self) -> Result { + let builds_path = self.artifacts_path.join("shuttle-builds"); + fs::create_dir_all(&builds_path)?; + + Ok(builds_path) + } + + /// Path for a specific service + pub fn service_build_path>(&self, service_name: S) -> Result { + let builds_path = self.builds_path()?.join(service_name.as_ref()); + fs::create_dir_all(&builds_path)?; + + Ok(builds_path) + } + + /// The directory in which compiled '.so' files are stored. + pub fn libraries_path(&self) -> Result { + let libs_path = self.artifacts_path.join("shuttle-libs"); + fs::create_dir_all(&libs_path)?; + + Ok(libs_path) + } + + /// Path to `.so` for a service + pub fn deployment_library_path(&self, deployment_id: &Uuid) -> Result { + let library_path = self.libraries_path()?.join(deployment_id.to_string()); + + Ok(library_path) + } + + /// Path of the directory to store user files + pub fn storage_path(&self) -> Result { + let storage_path = self.artifacts_path.join("shuttle-storage"); + fs::create_dir_all(&storage_path)?; + + Ok(storage_path) + } + + /// Path to folder for storing deployment files + pub fn deployment_storage_path>( + &self, + service_name: S, + deployment_id: &Uuid, + ) -> Result { + let storage_path = self + .storage_path()? + .join(service_name.as_ref()) + .join(deployment_id.to_string()); + fs::create_dir_all(&storage_path)?; + + Ok(storage_path) + } +} diff --git a/deployer/src/lib.rs b/deployer/src/lib.rs index 901b818e2..cb9b3bf0d 100644 --- a/deployer/src/lib.rs +++ b/deployer/src/lib.rs @@ -34,6 +34,8 @@ pub async fn start(persistence: Persistence, runtime_client: RuntimeClient Result<()> { + sqlx::query("UPDATE deployments SET state = ? WHERE state IN(?, ?, ?, ?)") + .bind(State::Stopped) + .bind(State::Queued) + .bind(State::Built) + .bind(State::Building) + .bind(State::Loading) + .execute(&self.pool) + .await?; + + Ok(()) + } + pub async fn get_or_create_service(&self, name: &str) -> Result { if let Some(service) = self.get_service_by_name(name).await? { Ok(service) @@ -554,6 +568,108 @@ mod tests { ); } + // Test that we are correctly cleaning up any stale / unexpected states for a deployment + // The reason this does not clean up two (or more) running states for a single deployment is because + // it should theoretically be impossible for a service to have two deployments in the running state. + // And even if a service where to have this, then the start ups of these deployments (more specifically + // the last deployment that is starting up) will stop all the deployments correctly. + #[tokio::test(flavor = "multi_thread")] + async fn cleanup_invalid_states() { + let (p, _) = Persistence::new_in_memory().await; + + let service_id = add_service(&p.pool).await.unwrap(); + + let queued_id = Uuid::new_v4(); + let building_id = Uuid::new_v4(); + let built_id = Uuid::new_v4(); + let loading_id = Uuid::new_v4(); + + let deployment_crashed = Deployment { + id: Uuid::new_v4(), + service_id, + state: State::Crashed, + last_update: Utc::now(), + address: None, + }; + let deployment_stopped = Deployment { + id: Uuid::new_v4(), + service_id, + state: State::Stopped, + last_update: Utc::now(), + address: None, + }; + let deployment_running = Deployment { + id: Uuid::new_v4(), + service_id, + state: State::Running, + last_update: Utc::now(), + address: Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 9876)), + }; + let deployment_queued = Deployment { + id: queued_id, + service_id, + state: State::Queued, + last_update: Utc::now(), + address: None, + }; + let deployment_building = Deployment { + id: building_id, + service_id, + state: State::Building, + last_update: Utc::now(), + address: None, + }; + let deployment_built = Deployment { + id: built_id, + service_id, + state: State::Built, + last_update: Utc::now(), + address: None, + }; + let deployment_loading = Deployment { + id: loading_id, + service_id, + state: State::Loading, + last_update: Utc::now(), + address: None, + }; + + for deployment in [ + &deployment_crashed, + &deployment_stopped, + &deployment_running, + &deployment_queued, + &deployment_built, + &deployment_building, + &deployment_loading, + ] { + p.insert_deployment(deployment.clone()).await.unwrap(); + } + + p.cleanup_invalid_states().await.unwrap(); + + let actual: Vec<_> = p + .get_deployments(&service_id) + .await + .unwrap() + .into_iter() + .map(|deployment| (deployment.id, deployment.state)) + .collect(); + let expected = vec![ + (deployment_crashed.id, State::Crashed), + (deployment_stopped.id, State::Stopped), + (deployment_running.id, State::Running), + (queued_id, State::Stopped), + (built_id, State::Stopped), + (building_id, State::Stopped), + (loading_id, State::Stopped), + ]; + + assert_eq!( + actual, expected, + "invalid states should be moved to the stopped state" + ); + } #[tokio::test(flavor = "multi_thread")] async fn fetching_runnable_deployments() { let (p, _) = Persistence::new_in_memory().await; diff --git a/gateway/src/api/latest.rs b/gateway/src/api/latest.rs index 9b7093f1a..b9210cc97 100644 --- a/gateway/src/api/latest.rs +++ b/gateway/src/api/latest.rs @@ -104,6 +104,23 @@ async fn get_project( Ok(AxumJson(response)) } +async fn get_projects_list( + State(RouterState { service, .. }): State, + User { name, .. }: User, +) -> Result>, Error> { + let projects = service + .iter_user_projects_detailed(name.clone()) + .await? + .into_iter() + .map(|project| project::Response { + name: project.0.to_string(), + state: project.1.into(), + }) + .collect(); + + Ok(AxumJson(projects)) +} + #[instrument(skip_all, fields(%project))] async fn post_project( State(RouterState { @@ -457,6 +474,7 @@ impl ApiBuilder { self.router = self .router .route("/", get(get_status)) + .route("/projects", get(get_projects_list)) .route( "/projects/:project_name", get(get_project).delete(delete_project).post(post_project), diff --git a/gateway/src/project.rs b/gateway/src/project.rs index e77f4a62e..d352d9f14 100644 --- a/gateway/src/project.rs +++ b/gateway/src/project.rs @@ -8,6 +8,8 @@ use bollard::container::{ }; use bollard::errors::Error as DockerError; use bollard::models::{ContainerInspectResponse, ContainerStateStatusEnum}; +use bollard::network::{ConnectNetworkOptions, DisconnectNetworkOptions}; +use bollard::service::EndpointSettings; use bollard::system::EventsOptions; use fqdn::FQDN; use futures::prelude::*; @@ -19,7 +21,7 @@ use once_cell::sync::Lazy; use rand::distributions::{Alphanumeric, DistString}; use serde::{Deserialize, Serialize}; use tokio::time::{self, timeout}; -use tracing::{debug, error, instrument}; +use tracing::{debug, error, info, instrument}; use crate::{ ContainerSettings, DockerContext, EndState, Error, ErrorKind, IntoTryState, ProjectName, @@ -146,6 +148,7 @@ impl From for Error { #[serde(rename_all = "lowercase")] pub enum Project { Creating(ProjectCreating), + Attaching(ProjectAttaching), Starting(ProjectStarting), Started(ProjectStarted), Ready(ProjectReady), @@ -158,6 +161,7 @@ pub enum Project { impl_from_variant!(Project: ProjectCreating => Creating, + ProjectAttaching => Attaching, ProjectStarting => Starting, ProjectStarted => Started, ProjectReady => Ready, @@ -220,6 +224,7 @@ impl Project { Self::Starting(_) => "starting", Self::Stopping(_) => "stopping", Self::Creating(_) => "creating", + Self::Attaching(_) => "attaching", Self::Destroying(_) => "destroying", Self::Destroyed(_) => "destroyed", Self::Errored(_) => "error", @@ -230,6 +235,7 @@ impl Project { match self { Self::Starting(ProjectStarting { container, .. }) | Self::Started(ProjectStarted { container, .. }) + | Self::Attaching(ProjectAttaching { container, .. }) | Self::Ready(ProjectReady { container, .. }) | Self::Stopping(ProjectStopping { container }) | Self::Stopped(ProjectStopped { container }) @@ -256,6 +262,7 @@ impl From for shuttle_common::models::project::State { fn from(project: Project) -> Self { match project { Project::Creating(_) => Self::Creating, + Project::Attaching(_) => Self::Attaching, Project::Starting(_) => Self::Starting, Project::Started(_) => Self::Started, Project::Ready(_) => Self::Ready, @@ -283,6 +290,17 @@ where let mut new = match self { Self::Creating(creating) => creating.next(ctx).await.into_try_state(), + Self::Attaching(attaching) => match attaching.next(ctx).await { + Err(ProjectError { + kind: ProjectErrorKind::NoNetwork, + ctx, + .. + }) => { + // Restart the container to try and connect to the network again + Ok(ctx.unwrap().stop().unwrap()) + } + attaching => attaching.into_try_state(), + }, Self::Starting(ready) => ready.next(ctx).await.into_try_state(), Self::Started(started) => match started.next(ctx).await { Ok(ProjectReadying::Ready(ready)) => Ok(ready.into()), @@ -350,6 +368,7 @@ where async fn refresh(self, ctx: &Ctx) -> Result { let refreshed = match self { Self::Creating(creating) => Self::Creating(creating), + Self::Attaching(attaching) => Self::Attaching(attaching), Self::Starting(ProjectStarting { container }) | Self::Started(ProjectStarted { container, .. }) | Self::Ready(ProjectReady { container, .. }) @@ -463,8 +482,6 @@ impl ProjectCreating { image: default_image, prefix, provisioner_host, - network_name, - network_id, fqdn: public, .. } = ctx.container_settings(); @@ -501,9 +518,7 @@ impl ProjectCreating { "--api-address", format!("0.0.0.0:{RUNTIME_API_PORT}"), "--provisioner-address", - provisioner_host, - "--provisioner-port", - "8000", + format!("http://{provisioner_host}:8000"), "--proxy-address", "0.0.0.0:8000", "--proxy-fqdn", @@ -521,14 +536,6 @@ impl ProjectCreating { let mut config = Config::::from(container_config); - config.networking_config = deserialize_json!({ - "EndpointsConfig": { - network_name: { - "NetworkID": network_id - } - } - }); - config.host_config = deserialize_json!({ "Mounts": [{ "Target": "/opt/shuttle", @@ -559,7 +566,7 @@ impl State for ProjectCreating where Ctx: DockerContext, { - type Next = ProjectStarting; + type Next = ProjectAttaching; type Error = ProjectError; #[instrument(skip_all)] @@ -582,6 +589,85 @@ where } }) .await?; + Ok(ProjectAttaching { container }) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ProjectAttaching { + container: ContainerInspectResponse, +} + +#[async_trait] +impl State for ProjectAttaching +where + Ctx: DockerContext, +{ + type Next = ProjectStarting; + type Error = ProjectError; + + #[instrument(skip_all)] + async fn next(self, ctx: &Ctx) -> Result { + let Self { container } = self; + + let container_id = container.id.as_ref().unwrap(); + let ContainerSettings { + network_name, + network_id, + .. + } = ctx.container_settings(); + + // Disconnect the bridge network before trying to start up + // For docker bug https://github.com/docker/cli/issues/1891 + // + // Also disconnecting from all network because docker just losses track of their IDs sometimes when restarting + for network in safe_unwrap!(container.network_settings.networks).keys() { + ctx.docker().disconnect_network(network, DisconnectNetworkOptions{ + container: container_id, + force: true, + }) + .await + .or_else(|err| { + if matches!(err, DockerError::DockerResponseServerError { status_code, .. } if status_code == 500) { + info!("already disconnected from the {network} network"); + Ok(()) + } else { + Err(err) + } + })?; + } + + // Make sure the container is connected to the user network + let network_config = ConnectNetworkOptions { + container: container_id, + endpoint_config: EndpointSettings { + network_id: Some(network_id.to_string()), + ..Default::default() + }, + }; + ctx.docker() + .connect_network(network_name, network_config) + .await + .or_else(|err| { + if matches!( + err, + DockerError::DockerResponseServerError { status_code, .. } if status_code == 409 + ) { + info!("already connected to the shuttle network"); + Ok(()) + } else { + error!( + error = &err as &dyn std::error::Error, + "failed to connect to shuttle network" + ); + Err(ProjectError::no_network( + "failed to connect to shuttle network", + )) + } + })?; + + let container = container.refresh(ctx).await?; + Ok(ProjectStarting { container }) } } @@ -602,6 +688,7 @@ where #[instrument(skip_all)] async fn next(self, ctx: &Ctx) -> Result { let container_id = self.container.id.as_ref().unwrap(); + ctx.docker() .start_container::(container_id, None) .await @@ -916,6 +1003,7 @@ where #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum ProjectErrorKind { Internal, + NoNetwork, } /// A runtime error coming from inside a project @@ -934,6 +1022,14 @@ impl ProjectError { ctx: None, } } + + pub fn no_network>(message: S) -> Self { + Self { + kind: ProjectErrorKind::NoNetwork, + message: message.as_ref().to_string(), + ctx: None, + } + } } impl std::fmt::Display for ProjectError { @@ -1032,22 +1128,47 @@ pub mod exec { .inspect_container(safe_unwrap!(container.id), None) .await { - if let Some(ContainerState { - status: Some(ContainerStateStatusEnum::EXITED), - .. - }) = container.state - { - debug!("{} will be revived", project_name.clone()); - _ = gateway - .new_task() - .project(project_name) - .and_then(task::run(|ctx| async move { - TaskResult::Done(Project::Stopped(ProjectStopped { - container: ctx.state.container().unwrap(), + match container.state { + Some(ContainerState { + status: Some(ContainerStateStatusEnum::EXITED), + .. + }) => { + debug!("{} will be revived", project_name.clone()); + _ = gateway + .new_task() + .project(project_name) + .and_then(task::run(|ctx| async move { + TaskResult::Done(Project::Stopped(ProjectStopped { + container: ctx.state.container().unwrap(), + })) + })) + .send(&sender) + .await; + } + Some(ContainerState { + status: Some(ContainerStateStatusEnum::RUNNING), + .. + }) + | Some(ContainerState { + status: Some(ContainerStateStatusEnum::CREATED), + .. + }) => { + debug!( + "{} is errored but ready according to docker. So restarting it", + project_name.clone() + ); + _ = gateway + .new_task() + .project(project_name) + .and_then(task::run(|ctx| async move { + TaskResult::Done(Project::Stopping(ProjectStopping { + container: ctx.state.container().unwrap(), + })) })) - })) - .send(&sender) - .await; + .send(&sender) + .await; + } + _ => {} } } } @@ -1062,6 +1183,7 @@ pub mod exec { pub mod tests { use bollard::models::ContainerState; + use bollard::service::NetworkSettings; use futures::prelude::*; use hyper::{Body, Request, StatusCode}; @@ -1084,7 +1206,21 @@ pub mod tests { image: None, from: None, }), - #[assertion = "Container created, assigned an `id`"] + #[assertion = "Container created, attach network"] + Ok(Project::Attaching(ProjectAttaching { + container: ContainerInspectResponse { + state: Some(ContainerState { + status: Some(ContainerStateStatusEnum::CREATED), + .. + }), + network_settings: Some(NetworkSettings { + networks: Some(networks), + .. + }), + .. + } + })) if networks.keys().collect::>() == vec!["bridge"], + #[assertion = "Container attached, assigned an `id`"] Ok(Project::Starting(ProjectStarting { container: ContainerInspectResponse { id: Some(container_id), @@ -1092,9 +1228,13 @@ pub mod tests { status: Some(ContainerStateStatusEnum::CREATED), .. }), + network_settings: Some(NetworkSettings { + networks: Some(networks), + .. + }), .. } - })), + })) if networks.keys().collect::>() == vec![&ctx.container_settings.network_name], #[assertion = "Container started, in a running state"] Ok(Project::Started(ProjectStarted { container: ContainerInspectResponse { diff --git a/gateway/src/service.rs b/gateway/src/service.rs index 20ca92a05..16d6bfcc9 100644 --- a/gateway/src/service.rs +++ b/gateway/src/service.rs @@ -276,6 +276,25 @@ impl GatewayService { .ok_or_else(|| Error::from_kind(ErrorKind::ProjectNotFound)) } + pub async fn iter_user_projects_detailed( + &self, + account_name: AccountName, + ) -> Result, Error> { + let iter = + query("SELECT project_name, project_state FROM projects WHERE account_name = ?1") + .bind(account_name) + .fetch_all(&self.db) + .await? + .into_iter() + .map(|row| { + ( + row.get("project_name"), + row.get::, _>("project_state").0, + ) + }); + Ok(iter) + } + pub async fn update_project( &self, project_name: &ProjectName, @@ -687,6 +706,14 @@ pub mod tests { account_name: neo.clone(), } ); + assert_eq!( + svc.iter_user_projects_detailed(neo.clone()) + .await + .unwrap() + .map(|item| item.0) + .collect::>(), + vec![matrix.clone()] + ); let mut work = svc .new_task() @@ -814,11 +841,22 @@ pub mod tests { assert_eq!(custom_domain.certificate, certificate); assert_eq!(custom_domain.private_key, private_key); - assert_err_kind!( - svc.create_custom_domain(project_name.clone(), &domain, certificate, private_key) - .await, - ErrorKind::CustomDomainAlreadyExists - ); + // Should auto replace the domain details + let certificate = "dummy certificate update"; + let private_key = "dummy private key update"; + + svc.create_custom_domain(project_name.clone(), &domain, certificate, private_key) + .await + .unwrap(); + + let custom_domain = svc + .project_details_for_custom_domain(&domain) + .await + .unwrap(); + + assert_eq!(custom_domain.project_name, project_name); + assert_eq!(custom_domain.certificate, certificate); + assert_eq!(custom_domain.private_key, private_key); Ok(()) } diff --git a/resources/aws-rds/Cargo.toml b/resources/aws-rds/Cargo.toml index 2403eeaad..bcf5652c1 100644 --- a/resources/aws-rds/Cargo.toml +++ b/resources/aws-rds/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "rds"] [dependencies] async-trait = "0.1.56" paste = "1.0.7" -shuttle-service = { path = "../../service", version = "0.8.0" } +shuttle-service = { path = "../../service", version = "0.8.0", default-features = false } sqlx = { version = "0.6.2", features = ["runtime-tokio-native-tls"] } tokio = { version = "1.19.2", features = ["rt"] } diff --git a/resources/persist/Cargo.toml b/resources/persist/Cargo.toml index 837caf372..7686a44c1 100644 --- a/resources/persist/Cargo.toml +++ b/resources/persist/Cargo.toml @@ -11,7 +11,7 @@ keywords = ["shuttle-service", "persistence"] async-trait = "0.1.56" bincode = "1.2.1" serde = { version = "1.0.0", features = ["derive"] } -shuttle-common = { path = "../../common", version = "0.8.0" } -shuttle-service = { path = "../../service", version = "0.8.0" } +shuttle-common = { path = "../../common", version = "0.8.0", default-features = false } +shuttle-service = { path = "../../service", version = "0.8.0", default-features = false } thiserror = "1.0.32" tokio = { version = "1.19.2", features = ["rt"] } diff --git a/resources/secrets/Cargo.toml b/resources/secrets/Cargo.toml index f5b1b713e..13455707e 100644 --- a/resources/secrets/Cargo.toml +++ b/resources/secrets/Cargo.toml @@ -9,5 +9,5 @@ keywords = ["shuttle-service", "secrets"] [dependencies] async-trait = "0.1.56" -shuttle-service = { path = "../../service", version = "0.8.0" } +shuttle-service = { path = "../../service", version = "0.8.0", default-features = false } tokio = { version = "1.19.2", features = ["rt"] } diff --git a/resources/shared-db/Cargo.toml b/resources/shared-db/Cargo.toml index 6e205e6aa..611d5126a 100644 --- a/resources/shared-db/Cargo.toml +++ b/resources/shared-db/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "database"] [dependencies] async-trait = "0.1.56" mongodb = { version = "2.3.0", optional = true } -shuttle-service = { path = "../../service", version = "0.8.0" } +shuttle-service = { path = "../../service", version = "0.8.0", default-features = false } sqlx = { version = "0.6.2", features = ["runtime-tokio-native-tls"], optional = true } tokio = { version = "1.19.2", features = ["rt"] } diff --git a/resources/static-folder/Cargo.toml b/resources/static-folder/Cargo.toml index 4ac0f8671..bd44daee7 100644 --- a/resources/static-folder/Cargo.toml +++ b/resources/static-folder/Cargo.toml @@ -9,7 +9,7 @@ keywords = ["shuttle-service", "static-folder"] [dependencies] async-trait = "0.1.56" -shuttle-service = { path = "../../service", version = "0.8.0" } +shuttle-service = { path = "../../service", version = "0.8.0", default-features = false } tokio = { version = "1.19.2", features = ["rt"] } [dev-dependencies] diff --git a/service/Cargo.toml b/service/Cargo.toml index 6c0baa322..b829619c9 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -30,6 +30,7 @@ rocket = { version = "0.5.0-rc.2", optional = true } salvo = { version = "0.37.5", optional = true } serde_json = { workspace = true } serenity = { version = "0.11.5", default-features = false, features = ["client", "gateway", "rustls_backend", "model"], optional = true } +poise = { version = "0.5.2", optional = true } sync_wrapper = { version = "0.1.1", optional = true } thiserror = { workspace = true } thruster = { version = "1.3.0", optional = true } @@ -78,4 +79,5 @@ web-poem = ["poem"] web-salvo = ["salvo"] bot-serenity = ["serenity"] +bot-poise = ["poise"] web-warp = ["warp"] diff --git a/service/src/lib.rs b/service/src/lib.rs index aafe1f887..66e673e05 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -264,8 +264,8 @@ extern crate shuttle_codegen; /// | `ShuttlePoem` | web-poem | [poem](https://docs.rs/poem/1.3.35) | 1.3.35 | [GitHub](https://github.com/shuttle-hq/examples/tree/main/poem/hello-world) | /// | `Result` | web-tower | [tower](https://docs.rs/tower/0.4.12) | 0.14.12 | [GitHub](https://github.com/shuttle-hq/examples/tree/main/tower/hello-world) | /// | `ShuttleSerenity` | bot-serenity | [serenity](https://docs.rs/serenity/0.11.5) | 0.11.5 | [GitHub](https://github.com/shuttle-hq/examples/tree/main/serenity/hello-world) | -/// | `ShuttleActixWeb` | web-actix-web| [actix-web](https://docs.rs/actix-web/4.2.1)| 4.2.1 | [GitHub](https://github.com/shuttle-hq/examples/tree/main/actix-web/hello-world) | - +/// | `ShuttlePoise` | bot-poise | [poise](https://docs.rs/poise/0.5.2) | 0.5.2 | [GitHub](https://github.com/shuttle-hq/examples/tree/main/poise/hello-world) | +/// | `ShuttleActixWeb` | web-actix-web| [actix-web](https://docs.rs/actix-web/4.2.1)| 4.2.1 | [GitHub](https://github.com/shuttle-hq/examples/tree/main/actix-web/hello-world) | /// /// # Getting shuttle managed resources /// Shuttle is able to manage resource dependencies for you. These resources are passed in as inputs to your `#[shuttle_service::main]` function and are configured using attributes: @@ -661,5 +661,22 @@ impl Service for serenity::Client { #[cfg(feature = "bot-serenity")] pub type ShuttleSerenity = Result; +#[cfg(feature = "bot-poise")] +#[async_trait] +impl Service for std::sync::Arc> +where + T: std::marker::Send + std::marker::Sync + 'static, + E: std::marker::Send + std::marker::Sync + 'static, +{ + async fn bind(mut self: Box, _addr: SocketAddr) -> Result<(), error::Error> { + self.start().await.map_err(error::CustomError::new)?; + + Ok(()) + } +} + +#[cfg(feature = "bot-poise")] +pub type ShuttlePoise = Result>, Error>; + pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const NAME: &str = env!("CARGO_PKG_NAME"); diff --git a/service/src/loader.rs b/service/src/loader.rs index fd4f93cf7..19d1114c3 100644 --- a/service/src/loader.rs +++ b/service/src/loader.rs @@ -320,7 +320,7 @@ fn check_version(summary: &Summary) -> anyhow::Result<()> { { shuttle.version_req() } else { - return Err(anyhow!("this crate does not use the shutte service")); + return Err(anyhow!("this crate does not use the shuttle service")); }; if version_req.matches(&valid_version) { diff --git a/service/tests/integration/helpers/loader.rs b/service/tests/integration/helpers/loader.rs index 9f3b7294a..2d09238c2 100644 --- a/service/tests/integration/helpers/loader.rs +++ b/service/tests/integration/helpers/loader.rs @@ -24,5 +24,5 @@ pub fn build_so_create_loader(resources: &str, crate_name: &str) -> Result