diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000000..9779e52426 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,52 @@ +name: cd + +on: + push: + tags: + - "[v]?[0-9]+.[0-9]+.[0-9]+*" + +env: + CONTAINER_REGISTRY: ghcr.io + +jobs: + build_and_publish_notary_server_image: + name: Build and publish notary server's image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Wait for test workflow to succeed + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.ref }} + # Have to be specify '(notary-server)', as we are using matrix for build_and_test job in ci.yml, else it will fail, more details [here](https://github.com/lewagon/wait-on-check-action#check-name) + check-name: 'Build and test (notary-server)' + repo-token: ${{ secrets.GITHUB_TOKEN }} + # How frequent (in seconds) this job will call GitHub API to check the status of the job specified at 'check-name' + wait-interval: 60 + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.CONTAINER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker image of notary server + id: meta-notary-server + uses: docker/metadata-action@v4 + with: + images: ${{ env.CONTAINER_REGISTRY }}/${{ github.repository }}/notary-server + + - name: Build and push Docker image of notary server + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: ${{ steps.meta-notary-server.outputs.tags }} + labels: ${{ steps.meta-notary-server.outputs.labels }} + file: ./notary-server/notary-server.Dockerfile diff --git a/.github/workflows/rust.yml b/.github/workflows/ci.yml similarity index 83% rename from .github/workflows/rust.yml rename to .github/workflows/ci.yml index 3fd9cacc83..85e026c131 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,14 @@ -name: Rust +name: ci on: push: - branches: [dev] + branches: + - dev + tags: + - "[v]?[0-9]+.[0-9]+.[0-9]+*" pull_request: - branches: [dev] + branches: + - dev env: CARGO_TERM_COLOR: always @@ -12,6 +16,7 @@ env: jobs: build_and_test: + name: Build and test if: ( ! github.event.pull_request.draft ) runs-on: ubuntu-latest strategy: @@ -28,6 +33,7 @@ jobs: - components/prf - components/tls - tlsn + - notary-server include: - package: components/integration-tests release: true @@ -37,9 +43,10 @@ jobs: run: working-directory: ${{ matrix.package }} steps: - - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v3 - - name: Nightly with rustfmt + - name: Install nightly rust toolchain with rustfmt uses: dtolnay/rust-toolchain@stable with: toolchain: nightly @@ -48,13 +55,17 @@ jobs: - name: "Check formatting" run: cargo +nightly fmt --check --all - - name: Stable + - name: Install stable rust toolchain uses: dtolnay/rust-toolchain@stable with: toolchain: stable components: clippy - - uses: Swatinem/rust-cache@v2.5.0 + - name: "Clippy" + run: cargo clippy --all-features -- -D warnings + + - name: Use caching + uses: Swatinem/rust-cache@v2.5.0 with: workspaces: ${{ matrix.package }} -> target @@ -79,6 +90,3 @@ jobs: - name: "Check that benches compile" run: cargo bench --no-run - - - name: "Clippy" - run: cargo clippy --all-features -- -D warnings diff --git a/.gitignore b/.gitignore index 4b9b054a4c..c169048c8c 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ Cargo.lock # env var *.env + +# logs +*.log diff --git a/README.md b/README.md index 9bb5d374ee..e127563920 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg [apache-badge]: https://img.shields.io/github/license/saltstack/salt -[actions-badge]: https://github.com/tlsnotary/tlsn/actions/workflows/rust.yml/badge.svg +[actions-badge]: https://github.com/tlsnotary/tlsn/actions/workflows/ci.yml/badge.svg [actions-url]: https://github.com/tlsnotary/tlsn/actions?query=workflow%3Arust+branch%3Adev [Website](https://tlsnotary.org) | diff --git a/notary-server/Cargo.toml b/notary-server/Cargo.toml new file mode 100644 index 0000000000..85cd9062ab --- /dev/null +++ b/notary-server/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "notary-server" +version = "0.1.0-alpha.2" +edition = "2021" + +[dependencies] +async-trait = "0.1.67" +async-tungstenite = { version = "0.22.2", features = ["tokio-native-tls"] } +axum = { version = "0.6.18", features = ["ws"] } +axum-core = "0.3.4" +axum-macros = "0.3.8" +base64 = "0.21.0" +eyre = "0.6.8" +futures = "0.3" +futures-util = "0.3.28" +http = "0.2.9" +hyper = { version = "0.14", features = ["client", "http1", "server", "tcp"] } +opentelemetry = { version = "0.19" } +p256 = "0.13" +rustls = { version = "0.21" } +rustls-pemfile = { version = "1.0.2" } +serde = { version = "1.0.147", features = ["derive"] } +serde_json = "1.0" +serde_yaml = "0.9.21" +sha1 = "0.10" +structopt = "0.3.26" +thiserror = "1" +tlsn-notary = { path = "../tlsn/tlsn-notary" } +tlsn-tls-core = { path = "../components/tls/tls-core" } +tokio = { version = "1", features = ["full"] } +tokio-rustls = { version = "0.24.1" } +tokio-util = { version = "0.7", features = ["compat"] } +tower = { version = "0.4.12", features = ["make"] } +tracing = "0.1" +tracing-opentelemetry = "0.19" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +uuid = { version = "1.4.1", features = ["v4", "fast-rng"] } +ws_stream_tungstenite = { version = "0.10.0", features = ["tokio_io"] } + +[dev-dependencies] +# specify vendored feature to use statically linked copy of OpenSSL +hyper-tls = { version = "0.5.0", features = ["vendored"] } +tls-server-fixture = { path = "../components/tls/tls-server-fixture" } +tlsn-prover = { path = "../tlsn/tlsn-prover" } +tokio-native-tls = { version = "0.3.1", features = ["vendored"] } diff --git a/notary-server/README.md b/notary-server/README.md new file mode 100644 index 0000000000..3b0bc8fee9 --- /dev/null +++ b/notary-server/README.md @@ -0,0 +1,106 @@ +# notary-server + +An implementation of the notary server in Rust. + +## ⚠️ Notice + +This crate is currently under active development and should not be used in production. Expect bugs and regular major breaking changes. + +--- +## Running the server +### Using Cargo +1. Configure the server setting in this config [file](./config/config.yaml) — refer [here](./src/config.rs) for more information on the definition of the setting parameters. +2. Start the server by running the following in a terminal at the root of this crate. +```bash +cargo run --release +``` +3. To use a config file from a different location, run the following command to override the default config file location. +```bash +cargo run --release -- --config-file +``` + +### Using Docker +There are two ways to obtain the notary server's Docker image: +- [GitHub](#obtaining-the-image-via-github) +- [Building from source](#building-from-source) + +#### GitHub +1. Obtain the latest image with: +```bash +docker pull ghcr.io/tlsnotary/tlsn/notary-server:latest +``` +2. Run the docker container with: +```bash +docker run --init -p 127.0.0.1:7047:7047 ghcr.io/tlsnotary/tlsn/notary-server:latest +``` +3. If you want to change the default configuration, create a `config` folder locally, that contains a `config.yaml`, whose content follows the format of the default config file [here](./config/config.yaml). +4. Instead of step 2, run the docker container with the following (remember to change the port mapping if you have changed that in the config): +```bash +docker run --init -p 127.0.0.1:7047:7047 -v :/root/.notary-server/config ghcr.io/tlsnotary/tlsn/notary-server:latest +``` + +#### Building from source +1. Configure the server setting in this config [file](./config/config.yaml). +2. Build the docker image by running the following in a terminal at the root of this *repository*. +```bash +docker build . -t notary-server:local -f notary-server/notary-server.Dockerfile +``` +3. Run the docker container and specify the port specified in the config file, e.g. for the default port 7047 +```bash +docker run --init -p 127.0.0.1:7047:7047 notary-server:local +``` + +### Using different key/cert for TLS or/and notarization with Docker +1. Instead of changing the key/cert file path(s) in the config file, create a folder containing your key/cert by following the folder structure [here](./fixture/). +2. When launching the docker container, mount your folder onto the docker container at the relevant path prefixed by `/root/.notary-server`. +- Example 1: Using different key/cert for both TLS and notarization: +```bash +docker run --init -p 127.0.0.1:7047:7047 -v :/root/.notary-server/fixture notary-server:local +``` +- Example 2: Using different key for notarization (your folder should only contain `notary.key`): +```bash +docker run --init -p 127.0.0.1:7047:7047 -v :/root/.notary-server/fixture/notary notary-server:local +``` +--- +## API +All APIs are TLS-protected, hence please use `https://` or `wss://`. +### HTTP APIs +Defined in the [OpenAPI specification](./openapi.yaml). + +### WebSocket APIs +#### /notarize +##### Description +To perform notarization using the session id (unique id returned upon calling the `/session` endpoint successfully) submitted as a custom header. + +##### Query Parameter +`sessionId` + +##### Query Parameter Type +String + +--- +## Architecture +### Objective +The main objective of a notary server is to perform notarization together with a prover. In this case, the prover can either be +1. TCP client — which has access and control over the transport layer, i.e. TCP +2. WebSocket client — which has no access over TCP and instead uses WebSocket for notarization + +### Design Choices +#### Web Framework +Axum is chosen as the framework to serve HTTP and WebSocket requests from the prover clients due to its rich and well supported features, e.g. native integration with Tokio/Hyper/Tower, customizable middleware, ability to support lower level integration of TLS ([example](https://github.com/tokio-rs/axum/blob/main/examples/low-level-rustls/src/main.rs)). To simplify the notary server setup, a single Axum router is used to support both HTTP and WebSocket connections, i.e. all requests can be made to the same port of the notary server. + +#### WebSocket +Axum's internal implementation of WebSocket uses [tokio_tungstenite](https://docs.rs/tokio-tungstenite/latest/tokio_tungstenite/), which provides a WebSocket struct that doesn't implement [AsyncRead](https://docs.rs/futures/latest/futures/io/trait.AsyncRead.html) and [AsyncWrite](https://docs.rs/futures/latest/futures/io/trait.AsyncWrite.html). Both these traits are required by TLSN core libraries for prover and notary. To overcome this, a [slight modification](./src/service/axum_websocket.rs) of Axum's implementation of WebSocket is used, where [async_tungstenite](https://docs.rs/async-tungstenite/latest/async_tungstenite/) is used instead so that [ws_stream_tungstenite](https://docs.rs/ws_stream_tungstenite/latest/ws_stream_tungstenite/index.html) can be used to wrap on top of the WebSocket struct to get AsyncRead and AsyncWrite implemented. + +#### Notarization Configuration +To perform notarization, some parameters need to be configured by the prover and notary server (more details in the [OpenAPI specification](./openapi.yaml)), i.e. +- maximum transcript size +- unique session id + +To streamline this process, a single HTTP endpoint (`/session`) is used by both TCP and WebSocket clients. + +#### Notarization +After calling the configuration endpoint above, prover can proceed to start notarization. For TCP client, that means calling the `/notarize` endpoint using HTTP (`https`), while WebSocket client should call the same endpoint but using WebSocket (`wss`). Example implementations of these clients can be found in the [integration test](./tests/integration_test.rs). + +#### Signatures +Currently, both the private key (and cert) used to establish TLS connection with prover, and the private key used by notary server to sign the notarized transcript, are hardcoded PEM keys stored in this repository. Though the paths of these keys can be changed in the config to use different keys instead. diff --git a/notary-server/config/config.yaml b/notary-server/config/config.yaml new file mode 100644 index 0000000000..4fc54a85f7 --- /dev/null +++ b/notary-server/config/config.yaml @@ -0,0 +1,17 @@ +server: + name: "notary-server" + host: "0.0.0.0" + port: 7047 + +notarization: + max-transcript-size: 16384 + +tls-signature: + private-key-pem-path: "./fixture/tls/notary.key" + certificate-pem-path: "./fixture/tls/notary.crt" + +notary-signature: + private-key-pem-path: "./fixture/notary/notary.key" + +tracing: + default-level: DEBUG diff --git a/notary-server/fixture/notary/notary.key b/notary-server/fixture/notary/notary.key new file mode 100644 index 0000000000..a88cf51f80 --- /dev/null +++ b/notary-server/fixture/notary/notary.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgEvBc/VMWn3E4PGfe +ETc/ekdTRmRwNN9J6eKDPxJ98ZmhRANCAAQG/foUjhkWzMlrQNAUnfBYJe9UsWtx +HMwbmRpN4cahLMO7pwWrHe4RZikUajoLQQ5SB/6YSBuS0utehy/nIfMq +-----END PRIVATE KEY----- diff --git a/notary-server/fixture/tls/notary.crt b/notary-server/fixture/tls/notary.crt new file mode 100644 index 0000000000..17c52c3558 --- /dev/null +++ b/notary-server/fixture/tls/notary.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDTzCCAjegAwIBAgIJALo+PtyTmxELMA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNV +BAoMCXRsc25vdGFyeTESMBAGA1UEAwwJdGxzbm90YXJ5MB4XDTIzMDYyNjE2MTI1 +N1oXDTI0MDYyNTE2MTI1N1owNDEYMBYGA1UECgwPdGxzbm90YXJ5c2VydmVyMRgw +FgYDVQQDDA90bHNub3RhcnlzZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQCqo+rOvL/l3ehVLrOBpzQrjWClV03rl+xiDIElEcVSz017gvoHX0ti ++etBHX+plJOhVRQrO+a3QeYv7NqDnQKIMozsStClkK4MagU1114JO+z4eArfQFDv +Czq2VYwDYBmLj4Lz0y54oQLyy/O8ON/ganYaW/3quGhufo+d774m8qCjSvhdTBnL +h1GxiZKfM8PFaRmBCMGa4mViTlpmnZq7eDzLumlh8WeOTFIWmbjNL4DkMJKu/gHK +5uOCtIUkFPezIN88Pq6wC88jRihXM7hrJUofPZZKzkDpwydGxol9fS0kiMANG6L8 +CUIeQhMDElCV/XiAXHi4MtH93XWjTR3VAgMBAAGjcDBuMEIGA1UdIwQ7MDmhLKQq +MCgxEjAQBgNVBAoMCXRsc25vdGFyeTESMBAGA1UEAwwJdGxzbm90YXJ5ggkAwxok +9FN4wLMwCQYDVR0TBAIwADAdBgNVHREEFjAUghJ0bHNub3RhcnlzZXJ2ZXIuaW8w +DQYJKoZIhvcNAQELBQADggEBAByvWsHE5qZYAJT1io1mwVQdXkDnlVjT/GAdu/Mx +EoUPJ9Pt/1XiS1dWXJMIZFbfOiZJBnX+sKxPpy/flaI4kbnXJY8nB5gFPkLWI7ok +V+r2iqEapsX3zrLx7x3AAM2kJbTieMLaGWe9g40wkGzmnpFJf5W8SgI2JEc4KlDo +joQJtsJa85PeOGtMsKLXnqUofDHbvDR0ab9obkh4Ngw+D1CGVXEGduCx1+SwB1jO +eDysCo+8ikyrrlzyDR1OyFJW28WVzLRJH0Z2bwldekM1RvCXqBYeLtAgNtS3Xb1w +RVP9VAx7KlmNF6kG52R2dQ1Z7J7i8JIZEkBcKjITEmpKrfE= +-----END CERTIFICATE----- diff --git a/notary-server/fixture/tls/notary.csr b/notary-server/fixture/tls/notary.csr new file mode 100644 index 0000000000..069d205b66 --- /dev/null +++ b/notary-server/fixture/tls/notary.csr @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICeTCCAWECAQAwNDEYMBYGA1UECgwPdGxzbm90YXJ5c2VydmVyMRgwFgYDVQQD +DA90bHNub3RhcnlzZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQCqo+rOvL/l3ehVLrOBpzQrjWClV03rl+xiDIElEcVSz017gvoHX0ti+etBHX+p +lJOhVRQrO+a3QeYv7NqDnQKIMozsStClkK4MagU1114JO+z4eArfQFDvCzq2VYwD +YBmLj4Lz0y54oQLyy/O8ON/ganYaW/3quGhufo+d774m8qCjSvhdTBnLh1GxiZKf +M8PFaRmBCMGa4mViTlpmnZq7eDzLumlh8WeOTFIWmbjNL4DkMJKu/gHK5uOCtIUk +FPezIN88Pq6wC88jRihXM7hrJUofPZZKzkDpwydGxol9fS0kiMANG6L8CUIeQhMD +ElCV/XiAXHi4MtH93XWjTR3VAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAqkWE +FyI9r3cY3tXt6j0/xGYZCX3X1AGje7vcEUeYzlED32putmH96Fkia+X2CMpEwcn7 +jaojJWvtAKGAk46p/cRpbPEOhLLebXn4znaeBVF5ph283WmeExRlhQml0e7kwTs9 +MwSniEFKBtvq4cSqO7BM1+NXDpjauVpaACl2+E9KTE8LcGG0BvH2eJOM/yW6wZmG +ykgyMeSg5UV/i5STWlryeaGBLCCmXx4jVfkBgaXw2Zq4ve1F/qU/eQFNUPk/iRSh +aQEQIfEC0hwqEe2Nc7X6PoVd7Py/x7Bke1JP9mRI7EPoN/IT0XHanJ08tusDCcYG +omGrHBGk9mELh39TXQ== +-----END CERTIFICATE REQUEST----- diff --git a/notary-server/fixture/tls/notary.ext b/notary-server/fixture/tls/notary.ext new file mode 100644 index 0000000000..b9a398df9b --- /dev/null +++ b/notary-server/fixture/tls/notary.ext @@ -0,0 +1,5 @@ +authorityKeyIdentifier=keyid,issuer +basicConstraints=CA:FALSE +subjectAltName = @alt_names +[alt_names] +DNS.1 = tlsnotaryserver.io diff --git a/notary-server/fixture/tls/notary.key b/notary-server/fixture/tls/notary.key new file mode 100644 index 0000000000..9e52bce222 --- /dev/null +++ b/notary-server/fixture/tls/notary.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCqo+rOvL/l3ehV +LrOBpzQrjWClV03rl+xiDIElEcVSz017gvoHX0ti+etBHX+plJOhVRQrO+a3QeYv +7NqDnQKIMozsStClkK4MagU1114JO+z4eArfQFDvCzq2VYwDYBmLj4Lz0y54oQLy +y/O8ON/ganYaW/3quGhufo+d774m8qCjSvhdTBnLh1GxiZKfM8PFaRmBCMGa4mVi +TlpmnZq7eDzLumlh8WeOTFIWmbjNL4DkMJKu/gHK5uOCtIUkFPezIN88Pq6wC88j +RihXM7hrJUofPZZKzkDpwydGxol9fS0kiMANG6L8CUIeQhMDElCV/XiAXHi4MtH9 +3XWjTR3VAgMBAAECggEBAIYDgk+nMVbIdsUfjl8PAAwMVpDEBjA2+rDufSat1Dj7 +EjEkZlUP5FbxTG+xSSfXxjH4bYSe4M2f9bZB4ENpNinc+YxCHadJ/0dEpJ7qa7H4 +3F0veepnyqhSO2Qjv3iPKsDOjtwLSP34BibFQsDaMgk/001UXhDPj0ToJMa3GLHg +pw1G2ri4WO4NxQA354y61jBNy0D4mjHHlcnofi4iLOFr2Kf538f0RgUyw8EkZ2sE +QyqL5HpHE93qIuLzl3/NxjQNHfO99dNNl6oWzPmXGi0nPGCMith3+8dMH7QiR/sS +r2bjdusIccV3tlZqCJUdWDC/RVgVQKDV3pWBx+i/1gECgYEA19UrQAwgxaimcs7E +NXISBzm2XgOOg9e5/W5EJvObu9zflqB3CBvrdZhhl1ZR+hTAbP5rHZQHWkMMAFbD +dT+VYIqTWUCIDkFpcB7vNa41A5eIbSdz1W+V4ZdAOUuwFcaG2xQA/F37S5DvH12V +JZ4ktJQklYUKmlUXSDTDgRUiApUCgYEAymWqTJjlSi1ADa3bAENaSKGaTazoeWBF +OesnwTLYCT+Aap+3aMjnG5+gxlSbdfJ1odrahXA3VSZwUL0IvCg8HcJsvwc0Bw3/ +LUpwCk2yLlq+OsOtpQSgsOVOzKzXJTEnjHBZxyInJsuTb+Kf5qn7/zvGQSNvbePT +h+YMAwGpHkECgYEAmg3DkzGU6sCYHfZLwkIrcBDXhH9RZ/XBAY2FA7B6Bjt/NApR +K+6RwBwF/HlWhgPt3V4zoqcYIGse09caKEQ8IO6Igfo3osU5txe9cjloCapNbGvu +l/fPqXfGFZ9ajhBoDVNX6MpEJgnLRD4NyQ358RKUkkyl5sa5mYZfzXECF4kCgYA+ +PcmDSLmqeAPssPxaNlw7XccQAA511Q804oYVOceKAIdDQt6qUK4RpqNQmpA8U1Wt +cpok0v+RJgMAMUHQaychl7rNfC+Zw8onaW7PHFmhO7Koa6ioyKWKANqcwsJe46Df +5WUWggA8Q/qRO8Ykrz2Zng431efciWVxs2MaQZZ6gQKBgA+QdsMadsrWoqh3tCZA +uruQ7hXCfALJfgexWFAtLIwlHujXI81+YVICCutCe6riktPZ/zTz/nbEAyt7kIiz +6BF7UYGT29qu1rC0MLzjfwK9ExuMSkvy9ZXGM0bCEgANIkZ0A/zTtTeRaFFGbx6l +F1Y+ihMVuZ4rOGQbVUfQxz1F +-----END PRIVATE KEY----- diff --git a/notary-server/fixture/tls/rootCA.crt b/notary-server/fixture/tls/rootCA.crt new file mode 100644 index 0000000000..d12936ff51 --- /dev/null +++ b/notary-server/fixture/tls/rootCA.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICzDCCAbQCCQDDGiT0U3jAszANBgkqhkiG9w0BAQsFADAoMRIwEAYDVQQKDAl0 +bHNub3RhcnkxEjAQBgNVBAMMCXRsc25vdGFyeTAeFw0yMzA2MjYxMTE2MTZaFw0y +ODA2MjQxMTE2MTZaMCgxEjAQBgNVBAoMCXRsc25vdGFyeTESMBAGA1UEAwwJdGxz +bm90YXJ5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7Vf+O9l4WNXE +Xh48MwjnvZ9wGN/Ls+jzzF1Q+J/QfXAYR/REQgJQmuk6sBgJyXUW7Dr5dKAY5tfL +rjfSaLhdMSxBH/tMepf5HVfEo6jvgk1bdR43DIZw7Z0hfuGUo6qOue8LZry2Nl+9 +VZpG64quRZ///4LdMBQyXcS2yeWKU10yVNBvstKW0i8krqQfbWOIG1nu5nDg5onB +paKUvbyrLyuHLz8gzKDFezxADTugq2KRXYKIZmyRucK+kmnJnZ/k46GZ84Vju15v +ktC0CvaR9IfvLfJMAo1Y0lUR4HjQkEAfjnDFYj5B18KFxXABraVD8UxjeMbAHTjf +i1lV0yp+qQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQABxRni6FZIFeK0KCS1Nrks +ONLVPfvDSNEKpImWFFoJbaSAAankTiQM1nKTY9SRIhqG2t+xJ6c8+qe905lFFvOy +r85LMb3z2ZWs4ez6Uy6IdpSdkTULk+1huE/Y9ZqRJ5aQy7PqiHTe+mNDFmHXGdcS +azHywd4hQeRQhCBXlAG7I18uZR9DPtGaJnvZlfbpD6Iq7x3ocfGhQiV9VJS1JaQ3 +Z7CJs2pa4da5FXQMAbKI2f7V5kbn3bjMp57yeYFo5wJMhEeSFqkrojR0oZDzfxW9 +b0W/PI4R4d2hUvX0fwrQyXbGo8HvYDFUhlMMSF60gUNcbpF6P93tXxR2FM/hnu+T +-----END CERTIFICATE----- diff --git a/notary-server/fixture/tls/rootCA.key b/notary-server/fixture/tls/rootCA.key new file mode 100644 index 0000000000..ca615a9799 --- /dev/null +++ b/notary-server/fixture/tls/rootCA.key @@ -0,0 +1,30 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIODItYjQ2oGICAggA +MB0GCWCGSAFlAwQBKgQQOVlpTszSmqOQ13RqJ0k9vQSCBNDOS9KT7QzKhXaSquKQ +vNylu9+hwkO2+SyVqkf82kHNezEr45r6DuxW0tQJhd93v6DMqGKS9LvFf0qshM2t +OhMD52PVvwHA4Fg4xMQORvvmHOHw7sxzrsQAWIjZ2cPpCRX2zcHYC5zwIaICBBdl +1qzUiO7n0nw77GRBUFoX0eSJPTkH42Nbc/nB+oa5t+n8mxY89Hqdh2wO5k569vkW +7ZC3FphzjPXIWk885qpZ7/O9eeR1OEhBf07PYqzar8RPDU02/lrUZ6Y+QMh69W/y +zNl7CGjy0IuDao77IFsu7rk6cBFq3uTXSyoyDoLmpkLbJEnS+ydjRyfbOjHJfG10 +Ca82mBA51IFs2Werf4+hzyM1EsvlGaGz2vhUffMK2lvJjoWocCi//E2XGQb+7hdY +HOP8sEWuDRqHGtZlMpIxp1JJs5jm+7RGCtgtO/tTb5hCdCB8msqNmCugzjUtg9ZJ +z3BygoRaBVgjBPKtDNo6NHf0Hfq1NzwImed3kTJAgNqkTxpCWYxvuJOWFROvps2t +jGkpGN0S17Ee9JAdsmsz3Mr92OXi9ncig3YTCI+WwN06xjjyIVPvtGDhvP30dPft +A6B1TgeS8g3yEFPkXjDqpKYsiZm38iN+GIcncgU1ng8r1gUnDzwZHwDFPAq/PwYC +sB4CA097eEaFLz49ttcO5ZXl7/pxMYKsBsUJQIG5MQcnyUBugQB2hVT+8WPjAjQE +ocScxHfMQAofMXn+Nwv+J5WlswZjoaXdqa/GG2AKHMPbrlMBdPDADMe0KGWxSKvt +HIbvhd/i+mXS2bwNDzHRCk3GEgdgEzOBrUbxzzJiN5EhRLqxuI2VnMUu5JcjxO/k +X2X6Ekpuc0D3PKezCGC98JbtwfVb5A1vBmVaD7ZMXSLCdBU+27qq96txsKP+WRzL +kcnp3EMfTscIgJOApslwkncGar6lsgfzBVD0bQy3luPEvEfhY+UOisApYsJEkuXy +HEbALLAvSibf1+1YCq00KTd7ERboCJ4N4e5ONE8wJBLoRMvRvxbi4zZsW4sApSjP +3r2P5FuB5x2VGlYo3BFd5yzAYzPQl+dFc7wg8yDohKNOVa9XAuNgDhXG+RS7imNL +PP3BIMFuj8+uH0rLRtse+pVhXKQ9pqvgvTpvGAKzqHjFHmH9GDKTnntL3B7kjRa4 +n/0DKT99iHEekvBAEN2qjIuYq0/xiSKSkVeGVksPUtZ/8V2SKdBRXnuDNlItaICW +bgZH/qCisNKx53jSv25yfoq0+rJtxLDNVuPFdKcQa97SsiUqElnxH/gwqSDLViYn +B3XsX+RKZJVNm1rqaoYf7yRFpddld4BgqUYQEj3rAyIelMGuTPSaiJxMysENnJl6 +MPdfug0NoUYJC+xle4YBeMjLj2qout7kq08414ZOEF4MdR6y1oqcM1IW14nSPAdG +O40Kma3V+aTJDIOY3cbS13f9yFY0n7KrD7dZ7faXDRT81LEhEVo9RZ01PBvES6lx +amL1CxQmVrhUD3YbeKvc17tvfOa67YU+6g0ELKFoWf2/5OJyj6wchACU98JyYonr +RGKSaL2zSDiFPTUKYscsFDMZuRtXs98okwoIbK8TujHTWZnI5DrBj2XLiVwPRYiu +dkFsSOjDwjmd11CglKpiBh5a7A== +-----END ENCRYPTED PRIVATE KEY----- diff --git a/notary-server/fixture/tls/rootCA.srl b/notary-server/fixture/tls/rootCA.srl new file mode 100644 index 0000000000..4c6854ba78 --- /dev/null +++ b/notary-server/fixture/tls/rootCA.srl @@ -0,0 +1 @@ +BA3E3EDC939B110B diff --git a/notary-server/notary-server.Dockerfile b/notary-server/notary-server.Dockerfile new file mode 100644 index 0000000000..964977a29c --- /dev/null +++ b/notary-server/notary-server.Dockerfile @@ -0,0 +1,40 @@ +# !!! To use this file, please run docker run at the root level of this repository +# +# Using rust:bookworm so that the builder image has OpenSSL 3.0 which is required by async-tungstenite, because +# +# (1) async-tungstenite dynamically links to the OS' OpenSSL by using openssl-sys crate (https://docs.rs/openssl/0.10.56/openssl/#automatic) +# +# (2) async-tungstenite does not utilise the "vendored" feature for its dependency crates, i.e. +# tokio-native-tls, tungstenite and native-tls. The "vendored" feature would have statically linked +# to a OpenSSL copy instead of dynamically link to the OS' OpenSSL (https://docs.rs/openssl/0.10.56/openssl/#vendored) +# — reported an issue here (https://github.com/sdroege/async-tungstenite/issues/119) +# +# (3) We want to use ubuntu:latest (22.04) as the runner image, which (only) has OpenSSL 3.0, because +# OpenSSL 1.1.1 is reaching EOL in Sept 2023 (https://www.openssl.org/blog/blog/2023/03/28/1.1.1-EOL/) +# +# (4) Therefore, we need the builder image to have the same OpenSSL version, else the built binary will +# try to dynamically link to a different (non-existing) version in the runner image +# +# (5) rust:latest is still using bullseye somehow which only has OpenSSL 1.1.1 +FROM rust:bookworm AS builder +WORKDIR /usr/src/tlsn +COPY . . +RUN cd notary-server; cargo install --path . + +FROM ubuntu:latest +WORKDIR /root/.notary-server +# Install pkg-config and libssl-dev for async-tungstenite to use (as explained above) +RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +# Copy default fixture folder for default usage +COPY --from=builder /usr/src/tlsn/notary-server/fixture ./fixture +# Copy default config folder for default usage +COPY --from=builder /usr/src/tlsn/notary-server/config ./config +COPY --from=builder /usr/local/cargo/bin/notary-server /usr/local/bin/notary-server +# Label to link this image with the repository in Github Container Registry (https://docs.github.com/en/packages/learn-github-packages/connecting-a-repository-to-a-package#connecting-a-repository-to-a-container-image-using-the-command-line) +LABEL org.opencontainers.image.source=https://github.com/tlsnotary/tlsn +LABEL org.opencontainers.image.description="An implementation of the notary server in Rust." +CMD [ "notary-server" ] diff --git a/notary-server/notary-server.Dockerfile.dockerignore b/notary-server/notary-server.Dockerfile.dockerignore new file mode 100644 index 0000000000..a18a55133f --- /dev/null +++ b/notary-server/notary-server.Dockerfile.dockerignore @@ -0,0 +1,12 @@ +# exclude everything +* + +# include notary-server +!/notary-server + +# include core library dependencies +!/tlsn +!/components + +# exclude any /target folders inside the included folders above +**/target* diff --git a/notary-server/openapi.yaml b/notary-server/openapi.yaml new file mode 100644 index 0000000000..21e58bb763 --- /dev/null +++ b/notary-server/openapi.yaml @@ -0,0 +1,123 @@ +openapi: 3.0.0 + +info: + title: Notary Server + description: Notary server written in Rust to provide notarization service. + version: 0.1.0 + +tags: + - name: Notarization + +paths: + /session: + post: + tags: + - Notarization + description: Initialize and configure notarization for both TCP and WebSocket clients + parameters: + - in: header + name: Content-Type + description: The value must be application/json + schema: + type: string + enum: + - "application/json" + required: true + requestBody: + description: Notarization session request to server + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NotarizationSessionRequest" + responses: + "200": + description: Notarization session response from server + content: + application/json: + schema: + $ref: "#/components/schemas/NotarizationSessionResponse" + "400": + description: Configuration parameters or headers provided by prover are invalid + content: + text/plain: + schema: + type: string + example: "Invalid request from prover: Failed to deserialize the JSON body into the target type" + "500": + description: There was some internal error when processing + content: + text/plain: + schema: + type: string + example: "Something is wrong" + /notarize: + get: + tags: + - Notarization + description: Start notarization for TCP client + parameters: + - in: header + name: Connection + description: The value should be 'Upgrade' + schema: + type: string + enum: + - "Upgrade" + required: true + - in: header + name: Upgrade + description: The value should be 'TCP' + schema: + type: string + enum: + - "TCP" + required: true + - in: query + name: sessionId + description: Unique ID returned from server upon calling POST /session + schema: + type: string + required: true + responses: + "101": + description: Switching protocol response + "400": + description: Headers provided by prover are invalid + content: + text/plain: + schema: + type: string + example: "Invalid request from prover: Upgrade header is not set for client" + "500": + description: There was some internal error when processing + content: + text/plain: + schema: + type: string + example: "Something is wrong" + +components: + schemas: + NotarizationSessionRequest: + type: object + properties: + clientType: + description: Types of client that the prover is using + type: string + enum: + - "Tcp" + - "Websocket" + maxTranscriptSize: + description: Maximum transcript size in bytes + type: integer + required: + - "clientType" + - "maxTranscriptSize" + NotarizationSessionResponse: + type: object + properties: + sessionId: + type: string + required: + - "sessionId" diff --git a/notary-server/src/config.rs b/notary-server/src/config.rs new file mode 100644 index 0000000000..77b6bfe8c3 --- /dev/null +++ b/notary-server/src/config.rs @@ -0,0 +1,52 @@ +use serde::Deserialize; + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct NotaryServerProperties { + /// Name and address of the notary server + pub server: ServerProperties, + /// Setting for notarization + pub notarization: NotarizationProperties, + /// File path of private key and certificate (in PEM format) used for establishing TLS with prover + pub tls_signature: TLSSignatureProperties, + /// File path of private key (in PEM format) used to sign the notarization + pub notary_signature: NotarySignatureProperties, + /// Setting for logging/tracing + pub tracing: TracingProperties, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct NotarizationProperties { + /// Global limit for maximum transcript size in bytes + pub max_transcript_size: usize, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ServerProperties { + /// Used for testing purpose + pub name: String, + pub host: String, + pub port: u16, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct TLSSignatureProperties { + pub private_key_pem_path: String, + pub certificate_pem_path: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct NotarySignatureProperties { + pub private_key_pem_path: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct TracingProperties { + /// The minimum logging level, must be either of + pub default_level: String, +} diff --git a/notary-server/src/domain.rs b/notary-server/src/domain.rs new file mode 100644 index 0000000000..421317ad75 --- /dev/null +++ b/notary-server/src/domain.rs @@ -0,0 +1,2 @@ +pub mod cli; +pub mod notary; diff --git a/notary-server/src/domain/cli.rs b/notary-server/src/domain/cli.rs new file mode 100644 index 0000000000..83a03e98d8 --- /dev/null +++ b/notary-server/src/domain/cli.rs @@ -0,0 +1,10 @@ +use structopt::StructOpt; + +/// Fields loaded from the command line when launching this server. +#[derive(Clone, Debug, StructOpt)] +#[structopt(name = "Notary Server")] +pub struct CliFields { + /// Configuration file location + #[structopt(long, default_value = "./config/config.yaml")] + pub config_file: String, +} diff --git a/notary-server/src/domain/notary.rs b/notary-server/src/domain/notary.rs new file mode 100644 index 0000000000..cbe9d5ca8a --- /dev/null +++ b/notary-server/src/domain/notary.rs @@ -0,0 +1,63 @@ +use std::{collections::HashMap, sync::Arc}; + +use p256::ecdsa::SigningKey; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; + +use crate::config::NotarizationProperties; + +/// Response object of the /session API +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NotarizationSessionResponse { + /// Unique session id that is generated by notary and shared to prover + pub session_id: String, +} + +/// Request object of the /session API +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NotarizationSessionRequest { + pub client_type: ClientType, + /// Maximum transcript size in bytes + pub max_transcript_size: Option, +} + +/// Request query of the /notarize API +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NotarizationRequestQuery { + /// Session id that is returned from /session API + pub session_id: String, +} + +/// Types of client that the prover is using +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum ClientType { + /// Client that has access to the transport layer + Tcp, + /// Client that cannot directly access transport layer, e.g. browser extension + Websocket, +} + +/// Global data that needs to be shared with the axum handlers +#[derive(Clone, Debug)] +pub struct NotaryGlobals { + pub notary_signing_key: SigningKey, + pub notarization_config: NotarizationProperties, + /// A temporary storage to store configuration data, mainly used for WebSocket client + pub store: Arc>>>, +} + +impl NotaryGlobals { + pub fn new( + notary_signing_key: SigningKey, + notarization_config: NotarizationProperties, + ) -> Self { + Self { + notary_signing_key, + notarization_config, + store: Default::default(), + } + } +} diff --git a/notary-server/src/error.rs b/notary-server/src/error.rs new file mode 100644 index 0000000000..c0455fb412 --- /dev/null +++ b/notary-server/src/error.rs @@ -0,0 +1,48 @@ +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; +use eyre::Report; +use std::error::Error; + +use tlsn_notary::{NotaryConfigBuilderError, NotaryError}; + +#[derive(Debug, thiserror::Error)] +pub enum NotaryServerError { + #[error(transparent)] + Unexpected(#[from] Report), + #[error("Failed to connect to prover: {0}")] + Connection(String), + #[error("Error occurred during notarization: {0}")] + Notarization(Box), + #[error("Invalid request from prover: {0}")] + BadProverRequest(String), +} + +impl From for NotaryServerError { + fn from(error: NotaryError) -> Self { + Self::Notarization(Box::new(error)) + } +} + +impl From for NotaryServerError { + fn from(error: NotaryConfigBuilderError) -> Self { + Self::Notarization(Box::new(error)) + } +} + +/// Trait implementation to convert this error into an axum http response +impl IntoResponse for NotaryServerError { + fn into_response(self) -> Response { + match self { + bad_request_error @ NotaryServerError::BadProverRequest(_) => { + (StatusCode::BAD_REQUEST, bad_request_error.to_string()).into_response() + } + _ => ( + StatusCode::INTERNAL_SERVER_ERROR, + "Something wrong happened.", + ) + .into_response(), + } + } +} diff --git a/notary-server/src/lib.rs b/notary-server/src/lib.rs new file mode 100644 index 0000000000..0010ab1517 --- /dev/null +++ b/notary-server/src/lib.rs @@ -0,0 +1,20 @@ +mod config; +mod domain; +mod error; +mod server; +mod server_tracing; +mod service; +mod util; + +pub use config::{ + NotarizationProperties, NotaryServerProperties, NotarySignatureProperties, ServerProperties, + TLSSignatureProperties, TracingProperties, +}; +pub use domain::{ + cli::CliFields, + notary::{ClientType, NotarizationSessionRequest, NotarizationSessionResponse}, +}; +pub use error::NotaryServerError; +pub use server::{read_pem_file, run_server}; +pub use server_tracing::init_tracing; +pub use util::parse_config_file; diff --git a/notary-server/src/main.rs b/notary-server/src/main.rs new file mode 100644 index 0000000000..58d8ed3a80 --- /dev/null +++ b/notary-server/src/main.rs @@ -0,0 +1,25 @@ +use eyre::{eyre, Result}; +use structopt::StructOpt; +use tracing::debug; + +use notary_server::{ + init_tracing, parse_config_file, run_server, CliFields, NotaryServerError, + NotaryServerProperties, +}; + +#[tokio::main] +async fn main() -> Result<(), NotaryServerError> { + // Load command line arguments which contains the config file location + let cli_fields: CliFields = CliFields::from_args(); + let config: NotaryServerProperties = parse_config_file(&cli_fields.config_file)?; + + // Set up tracing for logging + init_tracing(&config).map_err(|err| eyre!("Failed to set up tracing: {err}"))?; + + debug!(?config, "Server config loaded"); + + // Run the server + run_server(&config).await?; + + Ok(()) +} diff --git a/notary-server/src/server.rs b/notary-server/src/server.rs new file mode 100644 index 0000000000..e8dacce262 --- /dev/null +++ b/notary-server/src/server.rs @@ -0,0 +1,194 @@ +use axum::{ + http::{Request, StatusCode}, + response::IntoResponse, + routing::{get, post}, + Router, +}; +use eyre::{ensure, eyre, Result}; +use futures_util::future::poll_fn; +use hyper::server::{ + accept::Accept, + conn::{AddrIncoming, Http}, +}; +use p256::{ecdsa::SigningKey, pkcs8::DecodePrivateKey}; +use rustls::{Certificate, PrivateKey, ServerConfig}; +use std::{ + fs::File as StdFile, + io::BufReader, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::Arc, +}; + +use tokio::{fs::File, net::TcpListener}; +use tokio_rustls::TlsAcceptor; +use tower::MakeService; +use tracing::{debug, error, info}; + +use crate::{ + config::{NotaryServerProperties, NotarySignatureProperties, TLSSignatureProperties}, + domain::notary::NotaryGlobals, + error::NotaryServerError, + service::{initialize, upgrade_protocol}, +}; + +/// Start a TLS-secured TCP server to accept notarization request for both TCP and WebSocket clients +#[tracing::instrument(skip(config))] +pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotaryServerError> { + // Load the private key and cert needed for TLS connection from fixture folder — can be swapped out when we stop using static self signed cert + let (tls_private_key, tls_certificates) = load_tls_key_and_cert(&config.tls_signature).await?; + // Load the private key for notarized transcript signing from fixture folder — can be swapped out when we use proper ephemeral signing key + let notary_signing_key = load_notary_signing_key(&config.notary_signature).await?; + + // Build a TCP listener with TLS enabled + let mut server_config = ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(tls_certificates, tls_private_key) + .map_err(|err| eyre!("Failed to instantiate notary server tls config: {err}"))?; + + // Set the http protocols we support + server_config.alpn_protocols = vec![b"http/1.1".to_vec()]; + let tls_config = Arc::new(server_config); + + let notary_address = SocketAddr::new( + IpAddr::V4(config.server.host.parse().map_err(|err| { + eyre!("Failed to parse notary host address from server config: {err}") + })?), + config.server.port, + ); + + let acceptor = TlsAcceptor::from(tls_config); + let listener = TcpListener::bind(notary_address) + .await + .map_err(|err| eyre!("Failed to bind server address to tcp listener: {err}"))?; + let mut listener = AddrIncoming::from_listener(listener) + .map_err(|err| eyre!("Failed to build hyper tcp listener: {err}"))?; + + info!( + "Listening for TLS-secured TCP traffic at {}", + notary_address + ); + + let protocol = Arc::new(Http::new()); + let notary_globals = NotaryGlobals::new(notary_signing_key, config.notarization.clone()); + let router = Router::new() + .route( + "/healthcheck", + get(|| async move { (StatusCode::OK, "Ok").into_response() }), + ) + .route("/session", post(initialize)) + .route("/notarize", get(upgrade_protocol)) + .with_state(notary_globals); + let mut app = router.into_make_service(); + + loop { + // Poll and await for any incoming connection, ensure that all operations inside are infallible to prevent bringing down the server + let (prover_address, stream) = + match poll_fn(|cx| Pin::new(&mut listener).poll_accept(cx)).await { + Some(Ok(connection)) => (connection.remote_addr(), connection), + Some(Err(err)) => { + error!("{}", NotaryServerError::Connection(err.to_string())); + continue; + } + None => unreachable!("The poll_accept method should never return None"), + }; + debug!(?prover_address, "Received a prover's TCP connection"); + + let acceptor = acceptor.clone(); + let protocol = protocol.clone(); + let service = MakeService::<_, Request>::make_service(&mut app, &stream); + + // Spawn a new async task to handle the new connection + tokio::spawn(async move { + match acceptor.accept(stream).await { + Ok(stream) => { + info!( + ?prover_address, + "Accepted prover's TLS-secured TCP connection", + ); + // Serve different requests using the same hyper protocol and axum router + let _ = protocol + // Can unwrap because it's infallible + .serve_connection(stream, service.await.unwrap()) + // use with_upgrades to upgrade connection to websocket for websocket clients + // and to extract tcp connection for tcp clients + .with_upgrades() + .await; + } + Err(err) => { + error!( + ?prover_address, + "{}", + NotaryServerError::Connection(err.to_string()) + ); + } + } + }); + } +} + +/// Temporary function to load notary signing key from static file +async fn load_notary_signing_key(config: &NotarySignatureProperties) -> Result { + debug!("Loading notary server's signing key"); + + let notary_signing_key = SigningKey::read_pkcs8_pem_file(&config.private_key_pem_path) + .map_err(|err| eyre!("Failed to load notary signing key for notarization: {err}"))?; + + debug!("Successfully loaded notary server's signing key!"); + Ok(notary_signing_key) +} + +/// Read a PEM-formatted file and return its buffer reader +pub async fn read_pem_file(file_path: &str) -> Result> { + let key_file = File::open(file_path).await?.into_std().await; + Ok(BufReader::new(key_file)) +} + +/// Load notary tls private key and cert from static files +async fn load_tls_key_and_cert( + config: &TLSSignatureProperties, +) -> Result<(PrivateKey, Vec)> { + debug!("Loading notary server's tls private key and certificate"); + + let mut private_key_file_reader = read_pem_file(&config.private_key_pem_path).await?; + let mut private_keys = rustls_pemfile::pkcs8_private_keys(&mut private_key_file_reader)?; + ensure!( + private_keys.len() == 1, + "More than 1 key found in the tls private key pem file" + ); + let private_key = PrivateKey(private_keys.remove(0)); + + let mut certificate_file_reader = read_pem_file(&config.certificate_pem_path).await?; + let certificates = rustls_pemfile::certs(&mut certificate_file_reader)? + .into_iter() + .map(Certificate) + .collect(); + + debug!("Successfully loaded notary server's tls private key and certificate!"); + Ok((private_key, certificates)) +} + +#[cfg(test)] +mod test { + use super::*; + + #[tokio::test] + async fn test_load_notary_key_and_cert() { + let config = TLSSignatureProperties { + private_key_pem_path: "./fixture/tls/notary.key".to_string(), + certificate_pem_path: "./fixture/tls/notary.crt".to_string(), + }; + let result: Result<(PrivateKey, Vec)> = load_tls_key_and_cert(&config).await; + assert!(result.is_ok(), "Could not load tls private key and cert"); + } + + #[tokio::test] + async fn test_load_notary_signing_key() { + let config = NotarySignatureProperties { + private_key_pem_path: "./fixture/notary/notary.key".to_string(), + }; + let result: Result = load_notary_signing_key(&config).await; + assert!(result.is_ok(), "Could not load notary private key"); + } +} diff --git a/notary-server/src/server_tracing.rs b/notary-server/src/server_tracing.rs new file mode 100644 index 0000000000..65b5aa8c2a --- /dev/null +++ b/notary-server/src/server_tracing.rs @@ -0,0 +1,37 @@ +use eyre::Result; +use opentelemetry::{ + global, + sdk::{export::trace::stdout, propagation::TraceContextPropagator}, +}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry}; + +use crate::config::NotaryServerProperties; + +pub fn init_tracing(config: &NotaryServerProperties) -> Result<()> { + // Create a new OpenTelemetry pipeline + let tracer = stdout::new_pipeline().install_simple(); + + // Create a tracing layer with the configured tracer + let tracing_layer = tracing_opentelemetry::layer().with_tracer(tracer); + + // Set the log level + let env_filter_layer = EnvFilter::new(&config.tracing.default_level); + + // Format the log + let format_layer = tracing_subscriber::fmt::layer() + // Use a more compact, abbreviated log format + .compact() + .with_thread_ids(true) + .with_thread_names(true); + + // Set up context propagation + global::set_text_map_propagator(TraceContextPropagator::default()); + + Registry::default() + .with(tracing_layer) + .with(env_filter_layer) + .with(format_layer) + .try_init()?; + + Ok(()) +} diff --git a/notary-server/src/service.rs b/notary-server/src/service.rs new file mode 100644 index 0000000000..6cee179c34 --- /dev/null +++ b/notary-server/src/service.rs @@ -0,0 +1,173 @@ +pub mod axum_websocket; +pub mod tcp; +pub mod websocket; + +use async_trait::async_trait; +use axum::{ + extract::{rejection::JsonRejection, FromRequestParts, Query, State}, + http::{header, request::Parts, StatusCode}, + response::{IntoResponse, Json, Response}, +}; +use axum_macros::debug_handler; +use p256::ecdsa::{Signature, SigningKey}; +use tlsn_notary::{bind_notary, NotaryConfig}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio_util::compat::TokioAsyncReadCompatExt; +use tracing::{debug, error, info, trace}; +use uuid::Uuid; + +use crate::{ + domain::notary::{ + NotarizationRequestQuery, NotarizationSessionRequest, NotarizationSessionResponse, + NotaryGlobals, + }, + error::NotaryServerError, + service::{ + axum_websocket::{header_eq, WebSocketUpgrade}, + tcp::{tcp_notarize, TcpUpgrade}, + websocket::websocket_notarize, + }, +}; + +/// A wrapper enum to facilitate extracting TCP connection for either WebSocket or TCP clients, +/// so that we can use a single endpoint and handler for notarization for both types of clients +pub enum ProtocolUpgrade { + Tcp(TcpUpgrade), + Ws(WebSocketUpgrade), +} + +#[async_trait] +impl FromRequestParts for ProtocolUpgrade +where + S: Send + Sync, +{ + type Rejection = NotaryServerError; + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + // Extract tcp connection for websocket client + if header_eq(&parts.headers, header::UPGRADE, "websocket") { + let extractor = WebSocketUpgrade::from_request_parts(parts, state) + .await + .map_err(|err| NotaryServerError::BadProverRequest(err.to_string()))?; + return Ok(Self::Ws(extractor)); + // Extract tcp connection for tcp client + } else if header_eq(&parts.headers, header::UPGRADE, "tcp") { + let extractor = TcpUpgrade::from_request_parts(parts, state) + .await + .map_err(|err| NotaryServerError::BadProverRequest(err.to_string()))?; + return Ok(Self::Tcp(extractor)); + } else { + return Err(NotaryServerError::BadProverRequest( + "Upgrade header is not set for client".to_string(), + )); + } + } +} + +/// Handler to upgrade protocol from http to either websocket or underlying tcp depending on the type of client +/// the session_id parameter is also extracted here to fetch the configuration parameters +/// that have been submitted in the previous request to /session made by the same client +pub async fn upgrade_protocol( + protocol_upgrade: ProtocolUpgrade, + State(notary_globals): State, + Query(params): Query, +) -> Response { + info!("Received upgrade protocol request"); + let session_id = params.session_id; + // Fetch the configuration data from the store using the session_id + let max_transcript_size = match notary_globals.store.lock().await.get(&session_id) { + Some(max_transcript_size) => max_transcript_size.to_owned(), + None => { + let err_msg = format!("Session id {} does not exist", session_id); + error!(err_msg); + return NotaryServerError::BadProverRequest(err_msg).into_response(); + } + }; + // This completes the HTTP Upgrade request and returns a successful response to the client, meanwhile initiating the websocket or tcp connection + match protocol_upgrade { + ProtocolUpgrade::Ws(ws) => ws.on_upgrade(move |socket| { + websocket_notarize(socket, notary_globals, session_id, max_transcript_size) + }), + ProtocolUpgrade::Tcp(tcp) => tcp.on_upgrade(move |stream| { + tcp_notarize(stream, notary_globals, session_id, max_transcript_size) + }), + } +} + +/// Handler to initialize and configure notarization for both TCP and WebSocket clients +#[debug_handler(state = NotaryGlobals)] +pub async fn initialize( + State(notary_globals): State, + payload: Result, JsonRejection>, +) -> impl IntoResponse { + info!( + ?payload, + "Received request for initializing a notarization session" + ); + + // Parse the body payload + let payload = match payload { + Ok(payload) => payload, + Err(err) => { + error!("Malformed payload submitted for initializing notarization: {err}"); + return NotaryServerError::BadProverRequest(err.to_string()).into_response(); + } + }; + + // Ensure that the max_transcript_size submitted is not larger than the global max limit configured in notary server + if payload.max_transcript_size > Some(notary_globals.notarization_config.max_transcript_size) { + error!( + "Max transcript size requested {:?} exceeds the maximum threshold {:?}", + payload.max_transcript_size, notary_globals.notarization_config.max_transcript_size + ); + return NotaryServerError::BadProverRequest( + "Max transcript size requested exceeds the maximum threshold".to_string(), + ) + .into_response(); + } + + let prover_session_id = Uuid::new_v4().to_string(); + + // Store the configuration data in a temporary store + notary_globals + .store + .lock() + .await + .insert(prover_session_id.clone(), payload.max_transcript_size); + + trace!("Latest store state: {:?}", notary_globals.store); + + // Return the session id in the response to the client + ( + StatusCode::OK, + Json(NotarizationSessionResponse { + session_id: prover_session_id, + }), + ) + .into_response() +} + +/// Run the notarization +pub async fn notary_service( + socket: T, + signing_key: &SigningKey, + session_id: &str, + max_transcript_size: Option, +) -> Result<(), NotaryServerError> { + debug!(?session_id, "Starting notarization..."); + + let mut config_builder = NotaryConfig::builder(); + + config_builder.id(session_id); + + if let Some(max_transcript_size) = max_transcript_size { + config_builder.max_transcript_size(max_transcript_size); + } + + let config = config_builder.build()?; + + let (notary, notary_fut) = bind_notary(config, socket.compat())?; + + // Run the notary and background processes concurrently + tokio::try_join!(notary_fut, notary.notarize::(signing_key),).map(|_| Ok(()))? +} diff --git a/notary-server/src/service/axum_websocket.rs b/notary-server/src/service/axum_websocket.rs new file mode 100644 index 0000000000..eea235871e --- /dev/null +++ b/notary-server/src/service/axum_websocket.rs @@ -0,0 +1,914 @@ +//! The following code is adapted from https://github.com/tokio-rs/axum/blob/axum-v0.6.19/axum/src/extract/ws.rs +//! where we swapped out tokio_tungstenite (https://docs.rs/tokio-tungstenite/latest/tokio_tungstenite/) +//! with async_tungstenite (https://docs.rs/async-tungstenite/latest/async_tungstenite/) so that we can use +//! ws_stream_tungstenite (https://docs.rs/ws_stream_tungstenite/latest/ws_stream_tungstenite/index.html) +//! to get AsyncRead and AsyncWrite implemented for the WebSocket. Any other modification is commented with the prefix "NOTARY_MODIFICATION:" +//! +//! The code is under the following license: +//! +//! Copyright (c) 2019 Axum Contributors +//! +//! Permission is hereby granted, free of charge, to any +//! person obtaining a copy of this software and associated +//! documentation files (the "Software"), to deal in the +//! Software without restriction, including without +//! limitation the rights to use, copy, modify, merge, +//! publish, distribute, sublicense, and/or sell copies of +//! the Software, and to permit persons to whom the Software +//! is furnished to do so, subject to the following +//! conditions: +//! +//! The above copyright notice and this permission notice +//! shall be included in all copies or substantial portions +//! of the Software. +//! +//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +//! ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +//! TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +//! PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +//! SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +//! CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +//! OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +//! IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +//! DEALINGS IN THE SOFTWARE. +//! +//! +//! Handle WebSocket connections. +//! +//! # Example +//! +//! ``` +//! use axum::{ +//! extract::ws::{WebSocketUpgrade, WebSocket}, +//! routing::get, +//! response::{IntoResponse, Response}, +//! Router, +//! }; +//! +//! let app = Router::new().route("/ws", get(handler)); +//! +//! async fn handler(ws: WebSocketUpgrade) -> Response { +//! ws.on_upgrade(handle_socket) +//! } +//! +//! async fn handle_socket(mut socket: WebSocket) { +//! while let Some(msg) = socket.recv().await { +//! let msg = if let Ok(msg) = msg { +//! msg +//! } else { +//! // client disconnected +//! return; +//! }; +//! +//! if socket.send(msg).await.is_err() { +//! // client disconnected +//! return; +//! } +//! } +//! } +//! # async { +//! # axum::Server::bind(&"".parse().unwrap()).serve(app.into_make_service()).await.unwrap(); +//! # }; +//! ``` +//! +//! # Passing data and/or state to an `on_upgrade` callback +//! +//! ``` +//! use axum::{ +//! extract::{ws::{WebSocketUpgrade, WebSocket}, State}, +//! response::Response, +//! routing::get, +//! Router, +//! }; +//! +//! #[derive(Clone)] +//! struct AppState { +//! // ... +//! } +//! +//! async fn handler(ws: WebSocketUpgrade, State(state): State) -> Response { +//! ws.on_upgrade(|socket| handle_socket(socket, state)) +//! } +//! +//! async fn handle_socket(socket: WebSocket, state: AppState) { +//! // ... +//! } +//! +//! let app = Router::new() +//! .route("/ws", get(handler)) +//! .with_state(AppState { /* ... */ }); +//! # async { +//! # axum::Server::bind(&"".parse().unwrap()).serve(app.into_make_service()).await.unwrap(); +//! # }; +//! ``` +//! +//! # Read and write concurrently +//! +//! If you need to read and write concurrently from a [`WebSocket`] you can use +//! [`StreamExt::split`]: +//! +//! ```rust,no_run +//! use axum::{Error, extract::ws::{WebSocket, Message}}; +//! use futures_util::{sink::SinkExt, stream::{StreamExt, SplitSink, SplitStream}}; +//! +//! async fn handle_socket(mut socket: WebSocket) { +//! let (mut sender, mut receiver) = socket.split(); +//! +//! tokio::spawn(write(sender)); +//! tokio::spawn(read(receiver)); +//! } +//! +//! async fn read(receiver: SplitStream) { +//! // ... +//! } +//! +//! async fn write(sender: SplitSink) { +//! // ... +//! } +//! ``` +//! +//! [`StreamExt::split`]: https://docs.rs/futures/0.3.17/futures/stream/trait.StreamExt.html#method.split + +#![allow(unused)] + +use self::rejection::*; +use async_trait::async_trait; +use async_tungstenite::{ + tokio::TokioAdapter, + tungstenite::{ + self as ts, + protocol::{self, WebSocketConfig}, + }, + WebSocketStream, +}; +use axum::{ + body::{self, Bytes}, + extract::FromRequestParts, + response::Response, + Error, +}; + +use futures_util::{ + sink::{Sink, SinkExt}, + stream::{Stream, StreamExt}, +}; +use http::{ + header::{self, HeaderMap, HeaderName, HeaderValue}, + request::Parts, + Method, StatusCode, +}; +use hyper::upgrade::{OnUpgrade, Upgraded}; +use sha1::{Digest, Sha1}; +use std::{ + borrow::Cow, + future::Future, + pin::Pin, + task::{Context, Poll}, +}; +use tracing::error; + +/// Extractor for establishing WebSocket connections. +/// +/// Note: This extractor requires the request method to be `GET` so it should +/// always be used with [`get`](crate::routing::get). Requests with other methods will be +/// rejected. +/// +/// See the [module docs](self) for an example. +#[cfg_attr(docsrs, doc(cfg(feature = "ws")))] +pub struct WebSocketUpgrade { + config: WebSocketConfig, + /// The chosen protocol sent in the `Sec-WebSocket-Protocol` header of the response. + protocol: Option, + sec_websocket_key: HeaderValue, + on_upgrade: OnUpgrade, + on_failed_upgrade: F, + sec_websocket_protocol: Option, +} + +impl std::fmt::Debug for WebSocketUpgrade { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WebSocketUpgrade") + .field("config", &self.config) + .field("protocol", &self.protocol) + .field("sec_websocket_key", &self.sec_websocket_key) + .field("sec_websocket_protocol", &self.sec_websocket_protocol) + .finish_non_exhaustive() + } +} + +impl WebSocketUpgrade { + /// Set the size of the internal message send queue. + pub fn max_send_queue(mut self, max: usize) -> Self { + self.config.max_send_queue = Some(max); + self + } + + /// Set the maximum message size (defaults to 64 megabytes) + pub fn max_message_size(mut self, max: usize) -> Self { + self.config.max_message_size = Some(max); + self + } + + /// Set the maximum frame size (defaults to 16 megabytes) + pub fn max_frame_size(mut self, max: usize) -> Self { + self.config.max_frame_size = Some(max); + self + } + + /// Allow server to accept unmasked frames (defaults to false) + pub fn accept_unmasked_frames(mut self, accept: bool) -> Self { + self.config.accept_unmasked_frames = accept; + self + } + + /// Set the known protocols. + /// + /// If the protocol name specified by `Sec-WebSocket-Protocol` header + /// to match any of them, the upgrade response will include `Sec-WebSocket-Protocol` header and + /// return the protocol name. + /// + /// The protocols should be listed in decreasing order of preference: if the client offers + /// multiple protocols that the server could support, the server will pick the first one in + /// this list. + /// + /// # Examples + /// + /// ``` + /// use axum::{ + /// extract::ws::{WebSocketUpgrade, WebSocket}, + /// routing::get, + /// response::{IntoResponse, Response}, + /// Router, + /// }; + /// + /// let app = Router::new().route("/ws", get(handler)); + /// + /// async fn handler(ws: WebSocketUpgrade) -> Response { + /// ws.protocols(["graphql-ws", "graphql-transport-ws"]) + /// .on_upgrade(|socket| async { + /// // ... + /// }) + /// } + /// # async { + /// # axum::Server::bind(&"".parse().unwrap()).serve(app.into_make_service()).await.unwrap(); + /// # }; + /// ``` + pub fn protocols(mut self, protocols: I) -> Self + where + I: IntoIterator, + I::Item: Into>, + { + if let Some(req_protocols) = self + .sec_websocket_protocol + .as_ref() + .and_then(|p| p.to_str().ok()) + { + self.protocol = protocols + .into_iter() + // FIXME: This will often allocate a new `String` and so is less efficient than it + // could be. But that can't be fixed without breaking changes to the public API. + .map(Into::into) + .find(|protocol| { + req_protocols + .split(',') + .any(|req_protocol| req_protocol.trim() == protocol) + }) + .map(|protocol| match protocol { + Cow::Owned(s) => HeaderValue::from_str(&s).unwrap(), + Cow::Borrowed(s) => HeaderValue::from_static(s), + }); + } + + self + } + + /// Provide a callback to call if upgrading the connection fails. + /// + /// The connection upgrade is performed in a background task. If that fails this callback + /// will be called. + /// + /// By default any errors will be silently ignored. + /// + /// # Example + /// + /// ``` + /// use axum::{ + /// extract::{WebSocketUpgrade}, + /// response::Response, + /// }; + /// + /// async fn handler(ws: WebSocketUpgrade) -> Response { + /// ws.on_failed_upgrade(|error| { + /// report_error(error); + /// }) + /// .on_upgrade(|socket| async { /* ... */ }) + /// } + /// # + /// # fn report_error(_: axum::Error) {} + /// ``` + pub fn on_failed_upgrade(self, callback: C) -> WebSocketUpgrade + where + C: OnFailedUpdgrade, + { + WebSocketUpgrade { + config: self.config, + protocol: self.protocol, + sec_websocket_key: self.sec_websocket_key, + on_upgrade: self.on_upgrade, + on_failed_upgrade: callback, + sec_websocket_protocol: self.sec_websocket_protocol, + } + } + + /// Finalize upgrading the connection and call the provided callback with + /// the stream. + #[must_use = "to setup the WebSocket connection, this response must be returned"] + pub fn on_upgrade(self, callback: C) -> Response + where + C: FnOnce(WebSocket) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + F: OnFailedUpdgrade, + { + let on_upgrade = self.on_upgrade; + let config = self.config; + let on_failed_upgrade = self.on_failed_upgrade; + + let protocol = self.protocol.clone(); + + tokio::spawn(async move { + let upgraded = match on_upgrade.await { + Ok(upgraded) => upgraded, + Err(err) => { + error!("Something wrong with on_upgrade: {:?}", err); + on_failed_upgrade.call(Error::new(err)); + return; + } + }; + let socket = WebSocketStream::from_raw_socket( + // NOTARY_MODIFICATION: Need to use TokioAdapter to wrap Upgraded which doesn't implement futures crate's AsyncRead and AsyncWrite + TokioAdapter::new(upgraded), + protocol::Role::Server, + Some(config), + ) + .await; + let socket = WebSocket { + inner: socket, + protocol, + }; + callback(socket).await; + }); + + #[allow(clippy::declare_interior_mutable_const)] + const UPGRADE: HeaderValue = HeaderValue::from_static("upgrade"); + #[allow(clippy::declare_interior_mutable_const)] + const WEBSOCKET: HeaderValue = HeaderValue::from_static("websocket"); + + let mut builder = Response::builder() + .status(StatusCode::SWITCHING_PROTOCOLS) + .header(header::CONNECTION, UPGRADE) + .header(header::UPGRADE, WEBSOCKET) + .header( + header::SEC_WEBSOCKET_ACCEPT, + sign(self.sec_websocket_key.as_bytes()), + ); + + if let Some(protocol) = self.protocol { + builder = builder.header(header::SEC_WEBSOCKET_PROTOCOL, protocol); + } + + builder.body(body::boxed(body::Empty::new())).unwrap() + } +} + +/// What to do when a connection upgrade fails. +/// +/// See [`WebSocketUpgrade::on_failed_upgrade`] for more details. +pub trait OnFailedUpdgrade: Send + 'static { + /// Call the callback. + fn call(self, error: Error); +} + +impl OnFailedUpdgrade for F +where + F: FnOnce(Error) + Send + 'static, +{ + fn call(self, error: Error) { + self(error) + } +} + +/// The default `OnFailedUpdgrade` used by `WebSocketUpgrade`. +/// +/// It simply ignores the error. +#[non_exhaustive] +#[derive(Debug)] +pub struct DefaultOnFailedUpdgrade; + +impl OnFailedUpdgrade for DefaultOnFailedUpdgrade { + #[inline] + fn call(self, _error: Error) {} +} + +#[async_trait] +impl FromRequestParts for WebSocketUpgrade +where + S: Send + Sync, +{ + type Rejection = WebSocketUpgradeRejection; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + if parts.method != Method::GET { + return Err(MethodNotGet.into()); + } + + if !header_contains(&parts.headers, header::CONNECTION, "upgrade") { + return Err(InvalidConnectionHeader.into()); + } + + if !header_eq(&parts.headers, header::UPGRADE, "websocket") { + return Err(InvalidUpgradeHeader.into()); + } + + if !header_eq(&parts.headers, header::SEC_WEBSOCKET_VERSION, "13") { + return Err(InvalidWebSocketVersionHeader.into()); + } + + let sec_websocket_key = parts + .headers + .get(header::SEC_WEBSOCKET_KEY) + .ok_or(WebSocketKeyHeaderMissing)? + .clone(); + + let on_upgrade = parts + .extensions + .remove::() + .ok_or(ConnectionNotUpgradable)?; + + let sec_websocket_protocol = parts.headers.get(header::SEC_WEBSOCKET_PROTOCOL).cloned(); + + Ok(Self { + config: Default::default(), + protocol: None, + sec_websocket_key, + on_upgrade, + sec_websocket_protocol, + on_failed_upgrade: DefaultOnFailedUpdgrade, + }) + } +} + +pub fn header_eq(headers: &HeaderMap, key: HeaderName, value: &'static str) -> bool { + if let Some(header) = headers.get(&key) { + header.as_bytes().eq_ignore_ascii_case(value.as_bytes()) + } else { + false + } +} + +fn header_contains(headers: &HeaderMap, key: HeaderName, value: &'static str) -> bool { + let header = if let Some(header) = headers.get(&key) { + header + } else { + return false; + }; + + if let Ok(header) = std::str::from_utf8(header.as_bytes()) { + header.to_ascii_lowercase().contains(value) + } else { + false + } +} + +/// A stream of WebSocket messages. +/// +/// See [the module level documentation](self) for more details. +#[derive(Debug)] +pub struct WebSocket { + inner: WebSocketStream>, + protocol: Option, +} + +impl WebSocket { + /// Consume `self` and get the inner [`async_tungstenite::WebSocketStream`]. + pub fn into_inner(self) -> WebSocketStream> { + self.inner + } + + /// Receive another message. + /// + /// Returns `None` if the stream has closed. + pub async fn recv(&mut self) -> Option> { + self.next().await + } + + /// Send a message. + pub async fn send(&mut self, msg: Message) -> Result<(), Error> { + self.inner + .send(msg.into_tungstenite()) + .await + .map_err(Error::new) + } + + /// Gracefully close this WebSocket. + pub async fn close(mut self) -> Result<(), Error> { + self.inner.close(None).await.map_err(Error::new) + } + + /// Return the selected WebSocket subprotocol, if one has been chosen. + pub fn protocol(&self) -> Option<&HeaderValue> { + self.protocol.as_ref() + } +} + +impl Stream for WebSocket { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + loop { + match futures_util::ready!(self.inner.poll_next_unpin(cx)) { + Some(Ok(msg)) => { + if let Some(msg) = Message::from_tungstenite(msg) { + return Poll::Ready(Some(Ok(msg))); + } + } + Some(Err(err)) => return Poll::Ready(Some(Err(Error::new(err)))), + None => return Poll::Ready(None), + } + } + } +} + +impl Sink for WebSocket { + type Error = Error; + + fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_ready(cx).map_err(Error::new) + } + + fn start_send(mut self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> { + Pin::new(&mut self.inner) + .start_send(item.into_tungstenite()) + .map_err(Error::new) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx).map_err(Error::new) + } + + fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_close(cx).map_err(Error::new) + } +} + +/// Status code used to indicate why an endpoint is closing the WebSocket connection. +pub type CloseCode = u16; + +/// A struct representing the close command. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct CloseFrame<'t> { + /// The reason as a code. + pub code: CloseCode, + /// The reason as text string. + pub reason: Cow<'t, str>, +} + +/// A WebSocket message. +// +// This code comes from https://github.com/snapview/tungstenite-rs/blob/master/src/protocol/message.rs and is under following license: +// Copyright (c) 2017 Alexey Galakhov +// Copyright (c) 2016 Jason Housley +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +#[derive(Debug, Eq, PartialEq, Clone)] +pub enum Message { + /// A text WebSocket message + Text(String), + /// A binary WebSocket message + Binary(Vec), + /// A ping message with the specified payload + /// + /// The payload here must have a length less than 125 bytes. + /// + /// Ping messages will be automatically responded to by the server, so you do not have to worry + /// about dealing with them yourself. + Ping(Vec), + /// A pong message with the specified payload + /// + /// The payload here must have a length less than 125 bytes. + /// + /// Pong messages will be automatically sent to the client if a ping message is received, so + /// you do not have to worry about constructing them yourself unless you want to implement a + /// [unidirectional heartbeat](https://tools.ietf.org/html/rfc6455#section-5.5.3). + Pong(Vec), + /// A close message with the optional close frame. + Close(Option>), +} + +impl Message { + fn into_tungstenite(self) -> ts::Message { + match self { + Self::Text(text) => ts::Message::Text(text), + Self::Binary(binary) => ts::Message::Binary(binary), + Self::Ping(ping) => ts::Message::Ping(ping), + Self::Pong(pong) => ts::Message::Pong(pong), + Self::Close(Some(close)) => ts::Message::Close(Some(ts::protocol::CloseFrame { + code: ts::protocol::frame::coding::CloseCode::from(close.code), + reason: close.reason, + })), + Self::Close(None) => ts::Message::Close(None), + } + } + + fn from_tungstenite(message: ts::Message) -> Option { + match message { + ts::Message::Text(text) => Some(Self::Text(text)), + ts::Message::Binary(binary) => Some(Self::Binary(binary)), + ts::Message::Ping(ping) => Some(Self::Ping(ping)), + ts::Message::Pong(pong) => Some(Self::Pong(pong)), + ts::Message::Close(Some(close)) => Some(Self::Close(Some(CloseFrame { + code: close.code.into(), + reason: close.reason, + }))), + ts::Message::Close(None) => Some(Self::Close(None)), + // we can ignore `Frame` frames as recommended by the tungstenite maintainers + // https://github.com/snapview/tungstenite-rs/issues/268 + ts::Message::Frame(_) => None, + } + } + + /// Consume the WebSocket and return it as binary data. + pub fn into_data(self) -> Vec { + match self { + Self::Text(string) => string.into_bytes(), + Self::Binary(data) | Self::Ping(data) | Self::Pong(data) => data, + Self::Close(None) => Vec::new(), + Self::Close(Some(frame)) => frame.reason.into_owned().into_bytes(), + } + } + + /// Attempt to consume the WebSocket message and convert it to a String. + pub fn into_text(self) -> Result { + match self { + Self::Text(string) => Ok(string), + Self::Binary(data) | Self::Ping(data) | Self::Pong(data) => Ok(String::from_utf8(data) + .map_err(|err| err.utf8_error()) + .map_err(Error::new)?), + Self::Close(None) => Ok(String::new()), + Self::Close(Some(frame)) => Ok(frame.reason.into_owned()), + } + } + + /// Attempt to get a &str from the WebSocket message, + /// this will try to convert binary data to utf8. + pub fn to_text(&self) -> Result<&str, Error> { + match *self { + Self::Text(ref string) => Ok(string), + Self::Binary(ref data) | Self::Ping(ref data) | Self::Pong(ref data) => { + Ok(std::str::from_utf8(data).map_err(Error::new)?) + } + Self::Close(None) => Ok(""), + Self::Close(Some(ref frame)) => Ok(&frame.reason), + } + } +} + +impl From for Message { + fn from(string: String) -> Self { + Message::Text(string) + } +} + +impl<'s> From<&'s str> for Message { + fn from(string: &'s str) -> Self { + Message::Text(string.into()) + } +} + +impl<'b> From<&'b [u8]> for Message { + fn from(data: &'b [u8]) -> Self { + Message::Binary(data.into()) + } +} + +impl From> for Message { + fn from(data: Vec) -> Self { + Message::Binary(data) + } +} + +impl From for Vec { + fn from(msg: Message) -> Self { + msg.into_data() + } +} + +fn sign(key: &[u8]) -> HeaderValue { + use base64::engine::Engine as _; + + let mut sha1 = Sha1::default(); + sha1.update(key); + sha1.update(&b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"[..]); + let b64 = Bytes::from(base64::engine::general_purpose::STANDARD.encode(sha1.finalize())); + HeaderValue::from_maybe_shared(b64).expect("base64 is a valid value") +} + +pub mod rejection { + //! WebSocket specific rejections. + + use axum_core::{ + __composite_rejection as composite_rejection, __define_rejection as define_rejection, + }; + + define_rejection! { + #[status = METHOD_NOT_ALLOWED] + #[body = "Request method must be `GET`"] + /// Rejection type for [`WebSocketUpgrade`](super::WebSocketUpgrade). + pub struct MethodNotGet; + } + + define_rejection! { + #[status = BAD_REQUEST] + #[body = "Connection header did not include 'upgrade'"] + /// Rejection type for [`WebSocketUpgrade`](super::WebSocketUpgrade). + pub struct InvalidConnectionHeader; + } + + define_rejection! { + #[status = BAD_REQUEST] + #[body = "`Upgrade` header did not include 'websocket'"] + /// Rejection type for [`WebSocketUpgrade`](super::WebSocketUpgrade). + pub struct InvalidUpgradeHeader; + } + + define_rejection! { + #[status = BAD_REQUEST] + #[body = "`Sec-WebSocket-Version` header did not include '13'"] + /// Rejection type for [`WebSocketUpgrade`](super::WebSocketUpgrade). + pub struct InvalidWebSocketVersionHeader; + } + + define_rejection! { + #[status = BAD_REQUEST] + #[body = "`Sec-WebSocket-Key` header missing"] + /// Rejection type for [`WebSocketUpgrade`](super::WebSocketUpgrade). + pub struct WebSocketKeyHeaderMissing; + } + + define_rejection! { + #[status = UPGRADE_REQUIRED] + #[body = "WebSocket request couldn't be upgraded since no upgrade state was present"] + /// Rejection type for [`WebSocketUpgrade`](super::WebSocketUpgrade). + /// + /// This rejection is returned if the connection cannot be upgraded for example if the + /// request is HTTP/1.0. + /// + /// See [MDN] for more details about connection upgrades. + /// + /// [MDN]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Upgrade + pub struct ConnectionNotUpgradable; + } + + composite_rejection! { + /// Rejection used for [`WebSocketUpgrade`](super::WebSocketUpgrade). + /// + /// Contains one variant for each way the [`WebSocketUpgrade`](super::WebSocketUpgrade) + /// extractor can fail. + pub enum WebSocketUpgradeRejection { + MethodNotGet, + InvalidConnectionHeader, + InvalidUpgradeHeader, + InvalidWebSocketVersionHeader, + WebSocketKeyHeaderMissing, + ConnectionNotUpgradable, + } + } +} + +pub mod close_code { + //! Constants for [`CloseCode`]s. + //! + //! [`CloseCode`]: super::CloseCode + + /// Indicates a normal closure, meaning that the purpose for which the connection was + /// established has been fulfilled. + pub const NORMAL: u16 = 1000; + + /// Indicates that an endpoint is "going away", such as a server going down or a browser having + /// navigated away from a page. + pub const AWAY: u16 = 1001; + + /// Indicates that an endpoint is terminating the connection due to a protocol error. + pub const PROTOCOL: u16 = 1002; + + /// Indicates that an endpoint is terminating the connection because it has received a type of + /// data it cannot accept (e.g., an endpoint that understands only text data MAY send this if + /// it receives a binary message). + pub const UNSUPPORTED: u16 = 1003; + + /// Indicates that no status code was included in a closing frame. + pub const STATUS: u16 = 1005; + + /// Indicates an abnormal closure. + pub const ABNORMAL: u16 = 1006; + + /// Indicates that an endpoint is terminating the connection because it has received data + /// within a message that was not consistent with the type of the message (e.g., non-UTF-8 + /// RFC3629 data within a text message). + pub const INVALID: u16 = 1007; + + /// Indicates that an endpoint is terminating the connection because it has received a message + /// that violates its policy. This is a generic status code that can be returned when there is + /// no other more suitable status code (e.g., `UNSUPPORTED` or `SIZE`) or if there is a need to + /// hide specific details about the policy. + pub const POLICY: u16 = 1008; + + /// Indicates that an endpoint is terminating the connection because it has received a message + /// that is too big for it to process. + pub const SIZE: u16 = 1009; + + /// Indicates that an endpoint (client) is terminating the connection because it has expected + /// the server to negotiate one or more extension, but the server didn't return them in the + /// response message of the WebSocket handshake. The list of extensions that are needed should + /// be given as the reason for closing. Note that this status code is not used by the server, + /// because it can fail the WebSocket handshake instead. + pub const EXTENSION: u16 = 1010; + + /// Indicates that a server is terminating the connection because it encountered an unexpected + /// condition that prevented it from fulfilling the request. + pub const ERROR: u16 = 1011; + + /// Indicates that the server is restarting. + pub const RESTART: u16 = 1012; + + /// Indicates that the server is overloaded and the client should either connect to a different + /// IP (when multiple targets exist), or reconnect to the same IP when a user has performed an + /// action. + pub const AGAIN: u16 = 1013; +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::Body, routing::get, Router}; + use http::{Request, Version}; + use tower::ServiceExt; + + #[tokio::test] + async fn rejects_http_1_0_requests() { + let svc = get(|ws: Result| { + let rejection = ws.unwrap_err(); + assert!(matches!( + rejection, + WebSocketUpgradeRejection::ConnectionNotUpgradable(_) + )); + std::future::ready(()) + }); + + let req = Request::builder() + .version(Version::HTTP_10) + .method(Method::GET) + .header("upgrade", "websocket") + .header("connection", "Upgrade") + .header("sec-websocket-key", "6D69KGBOr4Re+Nj6zx9aQA==") + .header("sec-websocket-version", "13") + .body(Body::empty()) + .unwrap(); + + let res = svc.oneshot(req).await.unwrap(); + + assert_eq!(res.status(), StatusCode::OK); + } + + #[allow(dead_code)] + fn default_on_failed_upgrade() { + async fn handler(ws: WebSocketUpgrade) -> Response { + ws.on_upgrade(|_| async {}) + } + let _: Router = Router::new().route("/", get(handler)); + } + + #[allow(dead_code)] + fn on_failed_upgrade() { + async fn handler(ws: WebSocketUpgrade) -> Response { + ws.on_failed_upgrade(|_error: Error| println!("oops!")) + .on_upgrade(|_| async {}) + } + let _: Router = Router::new().route("/", get(handler)); + } +} diff --git a/notary-server/src/service/tcp.rs b/notary-server/src/service/tcp.rs new file mode 100644 index 0000000000..c401d9beff --- /dev/null +++ b/notary-server/src/service/tcp.rs @@ -0,0 +1,101 @@ +use async_trait::async_trait; +use axum::{ + body, + extract::FromRequestParts, + http::{header, request::Parts, HeaderValue, StatusCode}, + response::Response, +}; +use hyper::upgrade::{OnUpgrade, Upgraded}; +use std::future::Future; +use tracing::{debug, error, info}; + +use crate::{domain::notary::NotaryGlobals, service::notary_service, NotaryServerError}; + +/// Custom extractor used to extract underlying TCP connection for TCP client — using the same upgrade primitives used by +/// the WebSocket implementation where the underlying TCP connection (wrapped in an Upgraded object) only gets polled as an OnUpgrade future +/// after the ongoing HTTP request is finished (ref: https://github.com/tokio-rs/axum/blob/a6a849bb5b96a2f641fa077fe76f70ad4d20341c/axum/src/extract/ws.rs#L122) +/// +/// More info on the upgrade primitives: https://docs.rs/hyper/latest/hyper/upgrade/index.html +pub struct TcpUpgrade { + pub on_upgrade: OnUpgrade, +} + +#[async_trait] +impl FromRequestParts for TcpUpgrade +where + S: Send + Sync, +{ + type Rejection = NotaryServerError; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + let on_upgrade = + parts + .extensions + .remove::() + .ok_or(NotaryServerError::BadProverRequest( + "Upgrade header is not set for TCP client".to_string(), + ))?; + + Ok(Self { on_upgrade }) + } +} + +impl TcpUpgrade { + /// Utility function to complete the http upgrade protocol by + /// (1) Return 101 switching protocol response to client to indicate the switching to TCP + /// (2) Spawn a new thread to await on the OnUpgrade object to claim the underlying TCP connection + pub fn on_upgrade(self, callback: C) -> Response + where + C: FnOnce(Upgraded) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + let on_upgrade = self.on_upgrade; + tokio::spawn(async move { + let upgraded = match on_upgrade.await { + Ok(upgraded) => upgraded, + Err(err) => { + error!("Something wrong with upgrading HTTP: {:?}", err); + return; + } + }; + callback(upgraded).await; + }); + + #[allow(clippy::declare_interior_mutable_const)] + const UPGRADE: HeaderValue = HeaderValue::from_static("upgrade"); + #[allow(clippy::declare_interior_mutable_const)] + const TCP: HeaderValue = HeaderValue::from_static("tcp"); + + let builder = Response::builder() + .status(StatusCode::SWITCHING_PROTOCOLS) + .header(header::CONNECTION, UPGRADE) + .header(header::UPGRADE, TCP); + + builder.body(body::boxed(body::Empty::new())).unwrap() + } +} + +/// Perform notarization using the extracted tcp connection +pub async fn tcp_notarize( + stream: Upgraded, + notary_globals: NotaryGlobals, + session_id: String, + max_transcript_size: Option, +) { + debug!(?session_id, "Upgraded to tcp connection"); + match notary_service( + stream, + ¬ary_globals.notary_signing_key, + &session_id, + max_transcript_size, + ) + .await + { + Ok(_) => { + info!(?session_id, "Successful notarization using tcp!"); + } + Err(err) => { + error!(?session_id, "Failed notarization using tcp: {err}"); + } + } +} diff --git a/notary-server/src/service/websocket.rs b/notary-server/src/service/websocket.rs new file mode 100644 index 0000000000..1a06c0d5ca --- /dev/null +++ b/notary-server/src/service/websocket.rs @@ -0,0 +1,34 @@ +use tracing::{debug, error, info}; +use ws_stream_tungstenite::WsStream; + +use crate::{ + domain::notary::NotaryGlobals, + service::{axum_websocket::WebSocket, notary_service}, +}; + +/// Perform notarization using the established websocket connection +pub async fn websocket_notarize( + socket: WebSocket, + notary_globals: NotaryGlobals, + session_id: String, + max_transcript_size: Option, +) { + debug!(?session_id, "Upgraded to websocket connection"); + // Wrap the websocket in WsStream so that we have AsyncRead and AsyncWrite implemented + let stream = WsStream::new(socket.into_inner()); + match notary_service( + stream, + ¬ary_globals.notary_signing_key, + &session_id, + max_transcript_size, + ) + .await + { + Ok(_) => { + info!(?session_id, "Successful notarization using websocket!"); + } + Err(err) => { + error!(?session_id, "Failed notarization using websocket: {err}"); + } + } +} diff --git a/notary-server/src/util.rs b/notary-server/src/util.rs new file mode 100644 index 0000000000..55bf3d1487 --- /dev/null +++ b/notary-server/src/util.rs @@ -0,0 +1,27 @@ +use eyre::Result; +use serde::de::DeserializeOwned; + +/// Parse a yaml configuration file into a struct +pub fn parse_config_file(location: &str) -> Result { + let file = std::fs::File::open(location)?; + let config: T = serde_yaml::from_reader(file)?; + Ok(config) +} + +#[cfg(test)] +mod test { + + use crate::config::NotaryServerProperties; + + use super::{parse_config_file, Result}; + + #[test] + fn test_parse_config_file() { + let location = "./config/config.yaml"; + let config: Result = parse_config_file(location); + assert!( + config.is_ok(), + "Could not open file or read the file's values." + ); + } +} diff --git a/notary-server/tests/integration_test.rs b/notary-server/tests/integration_test.rs new file mode 100644 index 0000000000..4febb4922a --- /dev/null +++ b/notary-server/tests/integration_test.rs @@ -0,0 +1,431 @@ +use async_tungstenite::{ + tokio::connect_async_with_tls_connector_and_config, tungstenite::protocol::WebSocketConfig, +}; +use futures::AsyncWriteExt; +use hyper::{ + body::to_bytes, + client::{conn::Parts, HttpConnector}, + Body, Client, Request, StatusCode, +}; +use hyper_tls::HttpsConnector; +use rustls::{Certificate, ClientConfig, RootCertStore}; +use std::{ + net::{IpAddr, SocketAddr}, + sync::Arc, + time::Duration, +}; +use tls_server_fixture::{bind_test_server_hyper, CA_CERT_DER, SERVER_DOMAIN}; +use tlsn_prover::{Prover, ProverConfig}; +use tokio_rustls::TlsConnector; +use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt}; +use tracing::debug; +use ws_stream_tungstenite::WsStream; + +use notary_server::{ + read_pem_file, run_server, NotarizationProperties, NotarizationSessionRequest, + NotarizationSessionResponse, NotaryServerProperties, NotarySignatureProperties, + ServerProperties, TLSSignatureProperties, TracingProperties, +}; + +const NOTARY_CA_CERT_PATH: &str = "./fixture/tls/rootCA.crt"; +const NOTARY_CA_CERT_BYTES: &[u8] = include_bytes!("../fixture/tls/rootCA.crt"); + +async fn setup_config_and_server(sleep_ms: u64, port: u16) -> NotaryServerProperties { + let notary_config = NotaryServerProperties { + server: ServerProperties { + name: "tlsnotaryserver.io".to_string(), + host: "127.0.0.1".to_string(), + port, + }, + notarization: NotarizationProperties { + max_transcript_size: 1 << 14, + }, + tls_signature: TLSSignatureProperties { + private_key_pem_path: "./fixture/tls/notary.key".to_string(), + certificate_pem_path: "./fixture/tls/notary.crt".to_string(), + }, + notary_signature: NotarySignatureProperties { + private_key_pem_path: "./fixture/notary/notary.key".to_string(), + }, + tracing: TracingProperties { + default_level: "DEBUG".to_string(), + }, + }; + + let _ = tracing_subscriber::fmt::try_init(); + + let config = notary_config.clone(); + + // Run the notary server + tokio::spawn(async move { + run_server(&config).await.unwrap(); + }); + + // Sleep for a while to allow notary server to finish set up and start listening + tokio::time::sleep(Duration::from_millis(sleep_ms)).await; + + notary_config +} + +#[tokio::test] +async fn test_tcp_prover() { + // Notary server configuration setup + let notary_config = setup_config_and_server(100, 7048).await; + + // Connect to the Notary via TLS-TCP + let mut certificate_file_reader = read_pem_file(NOTARY_CA_CERT_PATH).await.unwrap(); + let mut certificates: Vec = rustls_pemfile::certs(&mut certificate_file_reader) + .unwrap() + .into_iter() + .map(Certificate) + .collect(); + let certificate = certificates.remove(0); + + let mut root_store = RootCertStore::empty(); + root_store.add(&certificate).unwrap(); + + let client_notary_config = ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(root_store) + .with_no_client_auth(); + let notary_connector = TlsConnector::from(Arc::new(client_notary_config)); + + let notary_host = notary_config.server.host.clone(); + let notary_port = notary_config.server.port; + let notary_socket = tokio::net::TcpStream::connect(SocketAddr::new( + IpAddr::V4(notary_host.parse().unwrap()), + notary_port, + )) + .await + .unwrap(); + + let notary_tls_socket = notary_connector + .connect( + notary_config.server.name.as_str().try_into().unwrap(), + notary_socket, + ) + .await + .unwrap(); + + // Attach the hyper HTTP client to the notary TLS connection to send request to the /session endpoint to configure notarization and obtain session id + let (mut request_sender, connection) = hyper::client::conn::handshake(notary_tls_socket) + .await + .unwrap(); + + // Spawn the HTTP task to be run concurrently + let connection_task = tokio::spawn(connection.without_shutdown()); + + // Build the HTTP request to configure notarization + let payload = serde_json::to_string(&NotarizationSessionRequest { + client_type: notary_server::ClientType::Tcp, + max_transcript_size: Some(notary_config.notarization.max_transcript_size), + }) + .unwrap(); + let request = Request::builder() + .uri(format!("https://{notary_host}:{notary_port}/session")) + .method("POST") + .header("Host", notary_host.clone()) + // Need to specify application/json for axum to parse it as json + .header("Content-Type", "application/json") + .body(Body::from(payload)) + .unwrap(); + + debug!("Sending configuration request"); + + let response = request_sender.send_request(request).await.unwrap(); + + debug!("Sent configuration request"); + + assert!(response.status() == StatusCode::OK); + + debug!("Response OK"); + + // Pretty printing :) + let payload = to_bytes(response.into_body()).await.unwrap().to_vec(); + let notarization_response = + serde_json::from_str::(&String::from_utf8_lossy(&payload)) + .unwrap(); + + debug!("Notarization response: {:?}", notarization_response,); + + // Send notarization request via HTTP, where the underlying TCP connection will be extracted later + let request = Request::builder() + // Need to specify the session_id so that notary server knows the right configuration to use + // as the configuration is set in the previous HTTP call + .uri(format!( + "https://{}:{}/notarize?sessionId={}", + notary_host, + notary_port, + notarization_response.session_id.clone() + )) + .method("GET") + .header("Host", notary_host) + .header("Connection", "Upgrade") + // Need to specify this upgrade header for server to extract tcp connection later + .header("Upgrade", "TCP") + .body(Body::empty()) + .unwrap(); + + debug!("Sending notarization request"); + + let response = request_sender.send_request(request).await.unwrap(); + + debug!("Sent notarization request"); + + assert!(response.status() == StatusCode::SWITCHING_PROTOCOLS); + + debug!("Switched protocol OK"); + + // Claim back the TCP socket after HTTP exchange is done so that client can use it for notarization + let Parts { + io: notary_tls_socket, + .. + } = connection_task.await.unwrap().unwrap(); + + // Connect to the Server + let (client_socket, server_socket) = tokio::io::duplex(2 << 16); + let server_task = tokio::spawn(bind_test_server_hyper(server_socket.compat())); + + let mut root_store = tls_core::anchors::RootCertStore::empty(); + root_store + .add(&tls_core::key::Certificate(CA_CERT_DER.to_vec())) + .unwrap(); + + // Basic default prover config — use the responded session id from notary server + let prover_config = ProverConfig::builder() + .id(notarization_response.session_id) + .server_dns(SERVER_DOMAIN) + .root_cert_store(root_store) + .build() + .unwrap(); + + // Bind the Prover to the sockets + let prover = Prover::new(prover_config) + .setup(notary_tls_socket.compat()) + .await + .unwrap(); + let (tls_connection, prover_fut) = prover.connect(client_socket.compat()).await.unwrap(); + + // Spawn the Prover task to be run concurrently + let prover_task = tokio::spawn(prover_fut); + + let (mut request_sender, connection) = hyper::client::conn::handshake(tls_connection.compat()) + .await + .unwrap(); + + let connection_task = tokio::spawn(connection.without_shutdown()); + + let request = Request::builder() + .uri(format!("https://{}/echo", SERVER_DOMAIN)) + .header("Host", SERVER_DOMAIN) + .header("Connection", "close") + .method("POST") + .body(Body::from("echo")) + .unwrap(); + + debug!("Sending request to server: {:?}", request); + + let response = request_sender.send_request(request).await.unwrap(); + + assert!(response.status() == StatusCode::OK); + + debug!( + "Received response from server: {:?}", + String::from_utf8_lossy(&to_bytes(response.into_body()).await.unwrap()) + ); + + let mut server_tls_conn = server_task.await.unwrap().unwrap(); + + // Make sure the server closes cleanly (sends close notify) + server_tls_conn.close().await.unwrap(); + + let mut client_socket = connection_task.await.unwrap().unwrap().io.into_inner(); + + client_socket.close().await.unwrap(); + + let mut prover = prover_task.await.unwrap().unwrap().start_notarize(); + + let sent_len = prover.sent_transcript().data().len(); + let recv_len = prover.recv_transcript().data().len(); + + let builder = prover.commitment_builder(); + + builder.commit_sent(0..sent_len).unwrap(); + builder.commit_recv(0..recv_len).unwrap(); + + _ = prover.finalize().await.unwrap(); + + debug!("Done notarization!"); +} + +#[tokio::test] +async fn test_websocket_prover() { + // Notary server configuration setup + let notary_config = setup_config_and_server(100, 7049).await; + let notary_host = notary_config.server.host.clone(); + let notary_port = notary_config.server.port; + + // Connect to the notary server via TLS-WebSocket + // Try to avoid dealing with transport layer directly to mimic the limitation of a browser extension that uses websocket + // + // Establish TLS setup for connections later + let certificate = + tokio_native_tls::native_tls::Certificate::from_pem(NOTARY_CA_CERT_BYTES).unwrap(); + let notary_tls_connector = tokio_native_tls::native_tls::TlsConnector::builder() + .add_root_certificate(certificate) + .use_sni(false) + .danger_accept_invalid_certs(true) + .build() + .unwrap(); + + // Call the /session HTTP API to configure notarization and obtain session id + let mut hyper_http_connector = HttpConnector::new(); + hyper_http_connector.enforce_http(false); + let mut hyper_tls_connector = + HttpsConnector::from((hyper_http_connector, notary_tls_connector.clone().into())); + hyper_tls_connector.https_only(true); + let https_client = Client::builder().build::<_, hyper::Body>(hyper_tls_connector); + + // Build the HTTP request to configure notarization + let payload = serde_json::to_string(&NotarizationSessionRequest { + client_type: notary_server::ClientType::Websocket, + max_transcript_size: Some(notary_config.notarization.max_transcript_size), + }) + .unwrap(); + + let request = Request::builder() + .uri(format!("https://{notary_host}:{notary_port}/session")) + .method("POST") + .header("Host", notary_host.clone()) + // Need to specify application/json for axum to parse it as json + .header("Content-Type", "application/json") + .body(Body::from(payload)) + .unwrap(); + + debug!("Sending request"); + + let response = https_client.request(request).await.unwrap(); + + debug!("Sent request"); + + assert!(response.status() == StatusCode::OK); + + debug!("Response OK"); + + // Pretty printing :) + let payload = to_bytes(response.into_body()).await.unwrap().to_vec(); + let notarization_response = + serde_json::from_str::(&String::from_utf8_lossy(&payload)) + .unwrap(); + + debug!("Notarization response: {:?}", notarization_response,); + + // Connect to the Notary via TLS-Websocket + // + // Note: This will establish a new TLS-TCP connection instead of reusing the previous TCP connection + // used in the previous HTTP POST request because we cannot claim back the tcp connection used in hyper + // client while using its high level request function — there does not seem to have a crate that can let you + // make a request without establishing TCP connection where you can claim the TCP connection later after making the request + let request = http::Request::builder() + // Need to specify the session_id so that notary server knows the right configuration to use + // as the configuration is set in the previous HTTP call + .uri(format!( + "wss://{}:{}/notarize?sessionId={}", + notary_host, + notary_port, + notarization_response.session_id.clone() + )) + .header("Host", notary_host.clone()) + .header("Sec-WebSocket-Key", uuid::Uuid::new_v4().to_string()) + .header("Sec-WebSocket-Version", "13") + .header("Connection", "Upgrade") + .header("Upgrade", "Websocket") + .body(()) + .unwrap(); + + let (notary_ws_stream, _) = connect_async_with_tls_connector_and_config( + request, + Some(notary_tls_connector.into()), + Some(WebSocketConfig::default()), + ) + .await + .unwrap(); + + // Wrap the socket with the adapter so that we get AsyncRead and AsyncWrite implemented + let notary_ws_socket = WsStream::new(notary_ws_stream); + + // Connect to the Server + let (client_socket, server_socket) = tokio::io::duplex(2 << 16); + let server_task = tokio::spawn(bind_test_server_hyper(server_socket.compat())); + + let mut root_store = tls_core::anchors::RootCertStore::empty(); + root_store + .add(&tls_core::key::Certificate(CA_CERT_DER.to_vec())) + .unwrap(); + + // Basic default prover config — use the responded session id from notary server + let prover_config = ProverConfig::builder() + .id(notarization_response.session_id) + .server_dns(SERVER_DOMAIN) + .root_cert_store(root_store) + .build() + .unwrap(); + + // Bind the Prover to the sockets + let prover = Prover::new(prover_config) + .setup(notary_ws_socket) + .await + .unwrap(); + let (tls_connection, prover_fut) = prover.connect(client_socket.compat()).await.unwrap(); + + // Spawn the Prover and Mux tasks to be run concurrently + let prover_task = tokio::spawn(prover_fut); + + let (mut request_sender, connection) = hyper::client::conn::handshake(tls_connection.compat()) + .await + .unwrap(); + + let connection_task = tokio::spawn(connection.without_shutdown()); + + let request = Request::builder() + .uri(format!("https://{}/echo", SERVER_DOMAIN)) + .header("Host", SERVER_DOMAIN) + .header("Connection", "close") + .method("POST") + .body(Body::from("echo")) + .unwrap(); + + debug!("Sending request to server: {:?}", request); + + let response = request_sender.send_request(request).await.unwrap(); + + assert!(response.status() == StatusCode::OK); + + debug!( + "Received response from server: {:?}", + String::from_utf8_lossy(&to_bytes(response.into_body()).await.unwrap()) + ); + + let mut server_tls_conn = server_task.await.unwrap().unwrap(); + + // Make sure the server closes cleanly (sends close notify) + server_tls_conn.close().await.unwrap(); + + let mut client_socket = connection_task.await.unwrap().unwrap().io.into_inner(); + + client_socket.close().await.unwrap(); + + let mut prover = prover_task.await.unwrap().unwrap().start_notarize(); + + let sent_len = prover.sent_transcript().data().len(); + let recv_len = prover.recv_transcript().data().len(); + + let builder = prover.commitment_builder(); + + builder.commit_sent(0..sent_len).unwrap(); + builder.commit_recv(0..recv_len).unwrap(); + + _ = prover.finalize().await.unwrap(); + + debug!("Done notarization!"); +} diff --git a/tlsn/examples/Cargo.toml b/tlsn/examples/Cargo.toml index e296daead5..66a2392231 100644 --- a/tlsn/examples/Cargo.toml +++ b/tlsn/examples/Cargo.toml @@ -10,7 +10,7 @@ tlsn-notary.workspace = true tlsn-core.workspace = true tlsn-tls-core.workspace = true tlsn-tls-client.workspace = true -notary-server = { tag = "v0.1.0-alpha.2", git = "https://github.com/tlsnotary/notary-server" } +notary-server = { path = "../../notary-server" } mpz-core.workspace = true futures.workspace = true diff --git a/tlsn/examples/README.md b/tlsn/examples/README.md index d6ecc53015..689cf8cf36 100644 --- a/tlsn/examples/README.md +++ b/tlsn/examples/README.md @@ -7,12 +7,11 @@ This folder contains examples showing how to use the TLSNotary protocol. ### Starting a notary server -Before running the examples please make sure that the Notary server is already running. The server can be started with: +Before running the examples please make sure that the Notary server is already running. The server can be started with the following command at the root level of this repository: ```shell -git clone https://github.com/tlsnotary/notary-server cd notary-server cargo run --release ``` -By default the server will be listening on 127.0.0.1:7047 \ No newline at end of file +By default the server will be listening on 127.0.0.1:7047 diff --git a/tlsn/examples/discord/README.md b/tlsn/examples/discord/README.md index 73765b76d8..78a08b1f5a 100644 --- a/tlsn/examples/discord/README.md +++ b/tlsn/examples/discord/README.md @@ -12,11 +12,11 @@ This involves 3 steps: In this tlsn/examples folder, create a `.env` file. Then in that `.env` file, set the values of the following constants by following the format shown in this [example env file](./.env.example). -| Name | Example | Location | -| --------------- | ------------------------------------------------------- |---------------------------------------------------------------------------------- | -| USER_AGENT | `Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0` | Look for `User-Agent` in a request headers | -| AUTHORIZATION | `MTE1NDe1Otg4N6NxNjczOTM2OA.GYbUBf.aDtcMUKDOmg6C2kxxFtlFSN1pgdMMBtpHgBBEs` | Look for `Authorization` in a request headers | -| CHANNEL_ID | `1154750485639745567` | URL | +| Name | Example | Location | +| ------------- | -------------------------------------------------------------------------------- | --------------------------------------------- | +| USER_AGENT | `Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0` | Look for `User-Agent` in a request headers | +| AUTHORIZATION | `MTE1NDe1Otg4N6NxNjczOTM2OA.GYbUBf.aDtcMUKDOmg6C2kxxFtlFSN1pgdMMBtpHgBBEs` | Look for `Authorization` in a request headers | +| CHANNEL_ID | `1154750485639745567` | URL | You can obtain these parameters by opening [Discord](https://discord.com/channels/@me) in your browser and accessing the message history you want to notarize. Please note that notarizing only works for short transcripts at the moment, so choose a contact with a short history. @@ -27,18 +27,15 @@ You can find the `CHANNEL_ID` directly in the url: `https://discord.com/channels/@me/{CHANNEL_ID)` ## Start the notary server - -Make sure you checkout a recent release and it matches the version of `tlsn`! - -``` -git clone https://github.com/tlsnotary/notary-server +At the root level of this repository, run +```sh cd notary-server cargo run --release ``` The notary server will now be running in the background waiting for connections. -For more information on how to configure the notary server, please refer to [this](https://github.com/tlsnotary/notary-server#running-the-server). +For more information on how to configure the `Notary` server, please refer to [this](../../../notary-server/README.md#running-the-server). ## Notarize diff --git a/tlsn/examples/discord/discord_dm.rs b/tlsn/examples/discord/discord_dm.rs index 61542f0452..fbba2f33fd 100644 --- a/tlsn/examples/discord/discord_dm.rs +++ b/tlsn/examples/discord/discord_dm.rs @@ -1,6 +1,6 @@ // This example shows how to notarize Discord DMs. // -// The example uses the notary server implemented in https://github.com/tlsnotary/notary-server +// The example uses the notary server implemented in ../../../notary-server use eyre::Result; use futures::AsyncWriteExt; @@ -26,7 +26,7 @@ use tlsn_prover::{Prover, ProverConfig}; // Setting of the application server const SERVER_DOMAIN: &str = "discord.com"; -// Setting of the notary server — make sure these are the same with those in the notary-server repository used (https://github.com/tlsnotary/notary-server) +// Setting of the notary server — make sure these are the same with those in the notary-server const NOTARY_DOMAIN: &str = "127.0.0.1"; const NOTARY_PORT: u16 = 7047; const NOTARY_CA_CERT_PATH: &str = "../rootCA.crt"; @@ -282,15 +282,19 @@ async fn setup_notary_connection() -> (tokio_rustls::client::TlsStream p256::PublicKey { - // from https://github.com/tlsnotary/notary-server/tree/main/src/fixture/notary/notary.key + // from ../../../notary-server/fixture/notary/notary.key // converted with `openssl ec -in notary.key -pubout -outform PEM` let pem = "-----BEGIN PUBLIC KEY----- diff --git a/tlsn/examples/quick_start.md b/tlsn/examples/quick_start.md index 053f1b84f0..1b1a2c4a9e 100644 --- a/tlsn/examples/quick_start.md +++ b/tlsn/examples/quick_start.md @@ -15,23 +15,26 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` # Guide +Clone this repository first + +```shell +git clone https://github.com/tlsnotary/tlsn +``` ### Start a Notary server: ```shell -git clone https://github.com/tlsnotary/notary-server -cd notary-server +cd tlsn/notary-server cargo run --release ``` The `Notary` server will now be running in the background waiting for connections from a `Prover`. You can switch to another console to run the `Prover`. -For more information on how to configure the `Notary` server, please refer to [this](https://github.com/tlsnotary/notary-server#running-the-server). +For more information on how to configure the `Notary` server, please refer to [this](../../notary-server/README.md#running-the-server). ### Run a simple Prover: ```shell -git clone https://github.com/tlsnotary/tlsn cd tlsn/tlsn/examples cargo run --release --example simple_prover ``` diff --git a/tlsn/examples/simple_notary.rs b/tlsn/examples/simple_notary.rs index 6955a8082a..caa67f6fba 100644 --- a/tlsn/examples/simple_notary.rs +++ b/tlsn/examples/simple_notary.rs @@ -1,5 +1,5 @@ /// This is a simple implementation of the notary server with minimal functionalities (without TLS, does not support WebSocket and configuration etc.) -/// For a more functional notary server implementation, please use https://github.com/tlsnotary/notary-server +/// For a more functional notary server implementation, please use the notary server in `../../notary-server` use std::env; use tokio::net::TcpListener; diff --git a/tlsn/examples/simple_prover.rs b/tlsn/examples/simple_prover.rs index 2d093a591c..57661e12b3 100644 --- a/tlsn/examples/simple_prover.rs +++ b/tlsn/examples/simple_prover.rs @@ -248,15 +248,19 @@ async fn connect_to_notary() -> (TlsStream, String) { // Request the notary to prepare for notarization via HTTP, where the underlying TCP connection // will be extracted later let request = Request::builder() - .uri(format!("https://{NOTARY_HOST}:{NOTARY_PORT}/notarize")) + // Need to specify the session_id so that notary server knows the right configuration to use + // as the configuration is set in the previous HTTP call + .uri(format!( + "https://{}:{}/notarize?sessionId={}", + NOTARY_HOST, + NOTARY_PORT, + configuration_response.session_id.clone() + )) .method("GET") .header("Host", NOTARY_HOST) .header("Connection", "Upgrade") // Need to specify this upgrade header for server to extract tcp connection later .header("Upgrade", "TCP") - // Need to specify the session_id so that notary server knows the right configuration to use - // as the configuration is set in the previous HTTP call - .header("X-Session-Id", configuration_response.session_id.clone()) .body(Body::empty()) .unwrap(); diff --git a/tlsn/examples/simple_verifier.rs b/tlsn/examples/simple_verifier.rs index 5a44b13e8f..ca237b667c 100644 --- a/tlsn/examples/simple_verifier.rs +++ b/tlsn/examples/simple_verifier.rs @@ -67,7 +67,7 @@ fn main() { /// Returns a Notary pubkey trusted by this Verifier fn notary_pubkey() -> p256::PublicKey { - // from https://github.com/tlsnotary/notary-server/tree/main/src/fixture/notary/notary.key + // from ../../notary-server/fixture/notary/notary.key // converted with `openssl ec -in notary.key -pubout -outform PEM` let pem = "-----BEGIN PUBLIC KEY----- diff --git a/tlsn/examples/twitter_dm.md b/tlsn/examples/twitter_dm.md index caf7d4a532..bc9b111f05 100644 --- a/tlsn/examples/twitter_dm.md +++ b/tlsn/examples/twitter_dm.md @@ -12,13 +12,13 @@ This involves 3 steps: In this tlsn/examples folder, create a `.env` file. Then in that `.env` file, set the values of the following constants by following the format shown in this [example env file](./.env.example). -| Name | Example | Location in Request Headers Section (within Network Tab of Developer Tools) | -| --------------- | ------------------------------------------------------- |---------------------------------------------------------------------------------- | -| CONVERSATION_ID | `20124652-973145016511139841` | Look for `Referer`, then extract the `ID` in `https://twitter.com/messages/` | -| CLIENT_UUID | `e6f00000-cccc-dddd-bbbb-eeeeeefaaa27` | Look for `X-Client-Uuid`, then copy the entire value | -| AUTH_TOKEN | `670ccccccbe2bbbbbbbc1025aaaaaafa55555551` | Look for `Cookie`, then extract the `token` in `;auth_token=;` | -| ACCESS_TOKEN | `AAAAAAAAAAAAAAAAAAAAANRILgAA...4puTs%3D1Zv7...WjCpTnA` | Look for `Authorization`, then extract the `token` in `Bearer ` | -| CSRF_TOKEN | `77d8ef46bd57f722ea7e9f...f4235a713040bfcaac1cd6909` | Look for `X-Csrf-Token`, then copy the entire value | +| Name | Example | Location in Request Headers Section (within Network Tab of Developer Tools) | +| --------------- | ------------------------------------------------------- | -------------------------------------------------------------------------------- | +| CONVERSATION_ID | `20124652-973145016511139841` | Look for `Referer`, then extract the `ID` in `https://twitter.com/messages/` | +| CLIENT_UUID | `e6f00000-cccc-dddd-bbbb-eeeeeefaaa27` | Look for `X-Client-Uuid`, then copy the entire value | +| AUTH_TOKEN | `670ccccccbe2bbbbbbbc1025aaaaaafa55555551` | Look for `Cookie`, then extract the `token` in `;auth_token=;` | +| ACCESS_TOKEN | `AAAAAAAAAAAAAAAAAAAAANRILgAA...4puTs%3D1Zv7...WjCpTnA` | Look for `Authorization`, then extract the `token` in `Bearer ` | +| CSRF_TOKEN | `77d8ef46bd57f722ea7e9f...f4235a713040bfcaac1cd6909` | Look for `X-Csrf-Token`, then copy the entire value | You can obtain these parameters by opening [Twitter](https://twitter.com/messages/) in your browser and accessing the message history you want to notarize. Please note that notarizing only works for short transcripts at the moment, so choose a contact with a short history. @@ -27,16 +27,15 @@ Next, open the **Developer Tools**, go to the **Network** tab, and refresh the p ![Screenshot](twitter_dm_browser.png) ## Start the notary server - -``` -git clone https://github.com/tlsnotary/notary-server +At the root level of this repository, run +```sh cd notary-server cargo run --release ``` The notary server will now be running in the background waiting for connections. -For more information on how to configure the notary server, please refer to [this](https://github.com/tlsnotary/notary-server#running-the-server). +For more information on how to configure the notary server, please refer to [this](../../notary-server/README.md#running-the-server). ## Notarize diff --git a/tlsn/examples/twitter_dm.rs b/tlsn/examples/twitter_dm.rs index b961436ccc..904709de7b 100644 --- a/tlsn/examples/twitter_dm.rs +++ b/tlsn/examples/twitter_dm.rs @@ -1,4 +1,4 @@ -/// This prover implementation talks to the notary server implemented in https://github.com/tlsnotary/notary-server, instead of the simple_notary.rs in this example directory +/// This prover implementation talks to the notary server implemented in ../../notary-server, instead of the simple_notary.rs in this example directory use eyre::Result; use futures::AsyncWriteExt; use hyper::{body::to_bytes, client::conn::Parts, Body, Request, StatusCode}; @@ -24,7 +24,7 @@ const SERVER_DOMAIN: &str = "twitter.com"; const ROUTE: &str = "i/api/1.1/dm/conversation"; const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; -// Setting of the notary server — make sure these are the same with those in the notary-server repository used (https://github.com/tlsnotary/notary-server) +// Setting of the notary server — make sure these are the same with those in the notary-server const NOTARY_DOMAIN: &str = "127.0.0.1"; const NOTARY_PORT: u16 = 7047; const NOTARY_CA_CERT_PATH: &str = "./rootCA.crt"; @@ -146,15 +146,19 @@ async fn main() { // Send notarization request via HTTP, where the underlying TCP connection will be extracted later let request = Request::builder() - .uri(format!("https://{NOTARY_DOMAIN}:{NOTARY_PORT}/notarize")) + // Need to specify the session_id so that notary server knows the right configuration to use + // as the configuration is set in the previous HTTP call + .uri(format!( + "https://{}:{}/notarize?sessionId={}", + NOTARY_DOMAIN, + NOTARY_PORT, + notarization_response.session_id.clone() + )) .method("GET") .header("Host", NOTARY_DOMAIN) .header("Connection", "Upgrade") // Need to specify this upgrade header for server to extract tcp connection later .header("Upgrade", "TCP") - // Need to specify the session_id so that notary server knows the right configuration to use - // as the configuration is set in the previous HTTP call - .header("X-Session-Id", notarization_response.session_id.clone()) .body(Body::empty()) .unwrap();