Migrate notary-server repo (#358)

* Migrate notary-server repo in, merge in github actions, update dockerfile and readme. * Correct formatting. * Move clippy back to original order. * Fix clippy warning. * The notary server was moved to this repo #358 * Modify comments. * Add session id query param PR changes. * Fix cd issue and client bug. --------- Co-authored-by: Hendrik Eeckhaut <hendrik@eeckhaut.org>
tlsnotary · Oct 13, 2023 · 4408dfa · 4408dfa
1 parent b603460
commit 4408dfa
Show file tree

Hide file tree

Showing 44 changed files with 2,729 additions and 60 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -0,0 +1,52 @@
+name: cd
+
+on:
+  push:
+    tags:
+      - "[v]?[0-9]+.[0-9]+.[0-9]+*"
+
+env:
+  CONTAINER_REGISTRY: ghcr.io
+
+jobs:
+  build_and_publish_notary_server_image:
+    name: Build and publish notary server's image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Wait for test workflow to succeed
+        uses: lewagon/wait-on-check-action@v1.3.1
+        with:
+          ref: ${{ github.ref }}
+          # Have to be specify '(notary-server)', as we are using matrix for build_and_test job in ci.yml, else it will fail, more details [here](https://github.com/lewagon/wait-on-check-action#check-name)
+          check-name: 'Build and test (notary-server)'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          # How frequent (in seconds) this job will call GitHub API to check the status of the job specified at 'check-name'
+          wait-interval: 60
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.CONTAINER_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker image of notary server
+        id: meta-notary-server
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.CONTAINER_REGISTRY }}/${{ github.repository }}/notary-server
+
+      - name: Build and push Docker image of notary server
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta-notary-server.outputs.tags }}
+          labels: ${{ steps.meta-notary-server.outputs.labels }}
+          file: ./notary-server/notary-server.Dockerfile
diff --git a/.github/workflows/rust.yml → .github/workflows/ci.yml b/.github/workflows/rust.yml → .github/workflows/ci.yml
@@ -1,17 +1,22 @@
-name: Rust
+name: ci
 
 on:
   push:
-    branches: [dev]
+    branches:
+      - dev
+    tags:
+      - "[v]?[0-9]+.[0-9]+.[0-9]+*"
   pull_request:
-    branches: [dev]
+    branches:
+      - dev
 
 env:
   CARGO_TERM_COLOR: always
   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
 
 jobs:
   build_and_test:
+    name: Build and test
     if: ( ! github.event.pull_request.draft )
     runs-on: ubuntu-latest
     strategy:
@@ -28,6 +33,7 @@ jobs:
           - components/prf
           - components/tls
           - tlsn
+          - notary-server
         include:
           - package: components/integration-tests
             release: true
@@ -37,9 +43,10 @@ jobs:
       run:
         working-directory: ${{ matrix.package }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
-      - name: Nightly with rustfmt
+      - name: Install nightly rust toolchain with rustfmt
         uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: nightly
@@ -48,13 +55,17 @@ jobs:
       - name: "Check formatting"
         run: cargo +nightly fmt --check --all
 
-      - name: Stable
+      - name: Install stable rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: stable
           components: clippy
 
-      - uses: Swatinem/rust-cache@v2.5.0
+      - name: "Clippy"
+        run: cargo clippy --all-features -- -D warnings
+
+      - name: Use caching
+        uses: Swatinem/rust-cache@v2.5.0
         with:
           workspaces: ${{ matrix.package }} -> target
 
@@ -79,6 +90,3 @@ jobs:
 
       - name: "Check that benches compile"
         run: cargo bench --no-run
-
-      - name: "Clippy"
-        run: cargo clippy --all-features -- -D warnings
diff --git a/.gitignore b/.gitignore
@@ -27,3 +27,6 @@ Cargo.lock
 
 # env var
 *.env
+
+# logs
+*.log
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@
 
 [mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg
 [apache-badge]: https://img.shields.io/github/license/saltstack/salt
-[actions-badge]: https://github.com/tlsnotary/tlsn/actions/workflows/rust.yml/badge.svg
+[actions-badge]: https://github.com/tlsnotary/tlsn/actions/workflows/ci.yml/badge.svg
 [actions-url]: https://github.com/tlsnotary/tlsn/actions?query=workflow%3Arust+branch%3Adev
 
 [Website](https://tlsnotary.org) |

diff --git a/notary-server/Cargo.toml b/notary-server/Cargo.toml
@@ -0,0 +1,45 @@
+[package]
+name = "notary-server"
+version = "0.1.0-alpha.2"
+edition = "2021"
+
+[dependencies]
+async-trait = "0.1.67"
+async-tungstenite = { version = "0.22.2", features = ["tokio-native-tls"] }
+axum = { version = "0.6.18", features = ["ws"] }
+axum-core = "0.3.4"
+axum-macros = "0.3.8"
+base64 = "0.21.0"
+eyre = "0.6.8"
+futures = "0.3"
+futures-util = "0.3.28"
+http = "0.2.9"
+hyper = { version = "0.14", features = ["client", "http1", "server", "tcp"] }
+opentelemetry = { version = "0.19" }
+p256 = "0.13"
+rustls = { version = "0.21" }
+rustls-pemfile = { version = "1.0.2" }
+serde = { version = "1.0.147", features = ["derive"] }
+serde_json = "1.0"
+serde_yaml = "0.9.21"
+sha1 = "0.10"
+structopt = "0.3.26"
+thiserror = "1"
+tlsn-notary = { path = "../tlsn/tlsn-notary" }
+tlsn-tls-core = { path = "../components/tls/tls-core" }
+tokio = { version = "1", features = ["full"] }
+tokio-rustls = { version = "0.24.1" }
+tokio-util = { version = "0.7", features = ["compat"] }
+tower = { version = "0.4.12", features = ["make"] }
+tracing = "0.1"
+tracing-opentelemetry = "0.19"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+uuid = { version = "1.4.1", features = ["v4", "fast-rng"] }
+ws_stream_tungstenite = { version = "0.10.0", features = ["tokio_io"] }
+
+[dev-dependencies]
+# specify vendored feature to use statically linked copy of OpenSSL
+hyper-tls = { version = "0.5.0", features = ["vendored"] }
+tls-server-fixture = { path = "../components/tls/tls-server-fixture" }
+tlsn-prover = { path = "../tlsn/tlsn-prover" }
+tokio-native-tls = { version = "0.3.1", features = ["vendored"] }
diff --git a/notary-server/README.md b/notary-server/README.md
@@ -0,0 +1,106 @@
+# notary-server
+
+An implementation of the notary server in Rust.
+
+## ⚠️ Notice
+
+This crate is currently under active development and should not be used in production. Expect bugs and regular major breaking changes.
+
+---
+## Running the server
+### Using Cargo
+1. Configure the server setting in this config [file](./config/config.yaml) — refer [here](./src/config.rs) for more information on the definition of the setting parameters.
+2. Start the server by running the following in a terminal at the root of this crate.
+```bash
+cargo run --release
+```
+3. To use a config file from a different location, run the following command to override the default config file location.
+```bash
+cargo run --release -- --config-file <path-of-new-config-file>
+```
+
+### Using Docker
+There are two ways to obtain the notary server's Docker image:
+- [GitHub](#obtaining-the-image-via-github)
+- [Building from source](#building-from-source)
+
+#### GitHub
+1. Obtain the latest image with:
+```bash
+docker pull ghcr.io/tlsnotary/tlsn/notary-server:latest
+```
+2. Run the docker container with:
+```bash
+docker run --init -p 127.0.0.1:7047:7047 ghcr.io/tlsnotary/tlsn/notary-server:latest
+```
+3. If you want to change the default configuration, create a `config` folder locally, that contains a `config.yaml`, whose content follows the format of the default config file [here](./config/config.yaml).
+4. Instead of step 2, run the docker container with the following (remember to change the port mapping if you have changed that in the config):
+```bash
+docker run --init -p 127.0.0.1:7047:7047 -v <your config folder path>:/root/.notary-server/config ghcr.io/tlsnotary/tlsn/notary-server:latest
+```
+
+#### Building from source
+1. Configure the server setting in this config [file](./config/config.yaml).
+2. Build the docker image by running the following in a terminal at the root of this *repository*.
+```bash
+docker build . -t notary-server:local -f notary-server/notary-server.Dockerfile
+```
+3. Run the docker container and specify the port specified in the config file, e.g. for the default port 7047
+```bash
+docker run --init -p 127.0.0.1:7047:7047 notary-server:local
+```
+
+### Using different key/cert for TLS or/and notarization with Docker
+1. Instead of changing the key/cert file path(s) in the config file, create a folder containing your key/cert by following the folder structure [here](./fixture/).
+2. When launching the docker container, mount your folder onto the docker container at the relevant path prefixed by `/root/.notary-server`.
+- Example 1: Using different key/cert for both TLS and notarization:
+```bash
+docker run --init -p 127.0.0.1:7047:7047 -v <your folder path>:/root/.notary-server/fixture notary-server:local
+```
+- Example 2: Using different key for notarization (your folder should only contain `notary.key`):
+```bash
+docker run --init -p 127.0.0.1:7047:7047 -v <your folder path>:/root/.notary-server/fixture/notary notary-server:local
+```
+---
+## API
+All APIs are TLS-protected, hence please use `https://` or `wss://`.
+### HTTP APIs
+Defined in the [OpenAPI specification](./openapi.yaml).
+
+### WebSocket APIs
+#### /notarize
+##### Description
+To perform notarization using the session id (unique id returned upon calling the `/session` endpoint successfully) submitted as a custom header.
+
+##### Query Parameter
+`sessionId`
+
+##### Query Parameter Type
+String
+
+---
+## Architecture
+### Objective
+The main objective of a notary server is to perform notarization together with a prover. In this case, the prover can either be
+1. TCP client — which has access and control over the transport layer, i.e. TCP
+2. WebSocket client — which has no access over TCP and instead uses WebSocket for notarization
+
+### Design Choices
+#### Web Framework
+Axum is chosen as the framework to serve HTTP and WebSocket requests from the prover clients due to its rich and well supported features, e.g. native integration with Tokio/Hyper/Tower, customizable middleware, ability to support lower level integration of TLS ([example](https://github.com/tokio-rs/axum/blob/main/examples/low-level-rustls/src/main.rs)). To simplify the notary server setup, a single Axum router is used to support both HTTP and WebSocket connections, i.e. all requests can be made to the same port of the notary server.
+
+#### WebSocket
+Axum's internal implementation of WebSocket uses [tokio_tungstenite](https://docs.rs/tokio-tungstenite/latest/tokio_tungstenite/), which provides a WebSocket struct that doesn't implement [AsyncRead](https://docs.rs/futures/latest/futures/io/trait.AsyncRead.html) and [AsyncWrite](https://docs.rs/futures/latest/futures/io/trait.AsyncWrite.html). Both these traits are required by TLSN core libraries for prover and notary. To overcome this, a [slight modification](./src/service/axum_websocket.rs) of Axum's implementation of WebSocket is used, where [async_tungstenite](https://docs.rs/async-tungstenite/latest/async_tungstenite/) is used instead so that [ws_stream_tungstenite](https://docs.rs/ws_stream_tungstenite/latest/ws_stream_tungstenite/index.html) can be used to wrap on top of the WebSocket struct to get AsyncRead and AsyncWrite implemented.
+
+#### Notarization Configuration
+To perform notarization, some parameters need to be configured by the prover and notary server (more details in the [OpenAPI specification](./openapi.yaml)), i.e.
+- maximum transcript size
+- unique session id
+
+To streamline this process, a single HTTP endpoint (`/session`) is used by both TCP and WebSocket clients.
+
+#### Notarization
+After calling the configuration endpoint above, prover can proceed to start notarization. For TCP client, that means calling the `/notarize` endpoint using HTTP (`https`), while WebSocket client should call the same endpoint but using WebSocket (`wss`). Example implementations of these clients can be found in the [integration test](./tests/integration_test.rs).
+
+#### Signatures
+Currently, both the private key (and cert) used to establish TLS connection with prover, and the private key used by notary server to sign the notarized transcript, are hardcoded PEM keys stored in this repository. Though the paths of these keys can be changed in the config to use different keys instead.
diff --git a/notary-server/config/config.yaml b/notary-server/config/config.yaml
@@ -0,0 +1,17 @@
+server:
+  name: "notary-server"
+  host: "0.0.0.0"
+  port: 7047
+
+notarization:
+  max-transcript-size: 16384
+
+tls-signature:
+  private-key-pem-path: "./fixture/tls/notary.key"
+  certificate-pem-path: "./fixture/tls/notary.crt"
+
+notary-signature:
+  private-key-pem-path: "./fixture/notary/notary.key"
+
+tracing:
+  default-level: DEBUG
diff --git a/notary-server/fixture/notary/notary.key b/notary-server/fixture/notary/notary.key
@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgEvBc/VMWn3E4PGfe
+ETc/ekdTRmRwNN9J6eKDPxJ98ZmhRANCAAQG/foUjhkWzMlrQNAUnfBYJe9UsWtx
+HMwbmRpN4cahLMO7pwWrHe4RZikUajoLQQ5SB/6YSBuS0utehy/nIfMq
+-----END PRIVATE KEY-----
diff --git a/notary-server/fixture/tls/notary.crt b/notary-server/fixture/tls/notary.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDTzCCAjegAwIBAgIJALo+PtyTmxELMA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNV
+BAoMCXRsc25vdGFyeTESMBAGA1UEAwwJdGxzbm90YXJ5MB4XDTIzMDYyNjE2MTI1
+N1oXDTI0MDYyNTE2MTI1N1owNDEYMBYGA1UECgwPdGxzbm90YXJ5c2VydmVyMRgw
+FgYDVQQDDA90bHNub3RhcnlzZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCqo+rOvL/l3ehVLrOBpzQrjWClV03rl+xiDIElEcVSz017gvoHX0ti
++etBHX+plJOhVRQrO+a3QeYv7NqDnQKIMozsStClkK4MagU1114JO+z4eArfQFDv
+Czq2VYwDYBmLj4Lz0y54oQLyy/O8ON/ganYaW/3quGhufo+d774m8qCjSvhdTBnL
+h1GxiZKfM8PFaRmBCMGa4mViTlpmnZq7eDzLumlh8WeOTFIWmbjNL4DkMJKu/gHK
+5uOCtIUkFPezIN88Pq6wC88jRihXM7hrJUofPZZKzkDpwydGxol9fS0kiMANG6L8
+CUIeQhMDElCV/XiAXHi4MtH93XWjTR3VAgMBAAGjcDBuMEIGA1UdIwQ7MDmhLKQq
+MCgxEjAQBgNVBAoMCXRsc25vdGFyeTESMBAGA1UEAwwJdGxzbm90YXJ5ggkAwxok
+9FN4wLMwCQYDVR0TBAIwADAdBgNVHREEFjAUghJ0bHNub3RhcnlzZXJ2ZXIuaW8w
+DQYJKoZIhvcNAQELBQADggEBAByvWsHE5qZYAJT1io1mwVQdXkDnlVjT/GAdu/Mx
+EoUPJ9Pt/1XiS1dWXJMIZFbfOiZJBnX+sKxPpy/flaI4kbnXJY8nB5gFPkLWI7ok
+V+r2iqEapsX3zrLx7x3AAM2kJbTieMLaGWe9g40wkGzmnpFJf5W8SgI2JEc4KlDo
+joQJtsJa85PeOGtMsKLXnqUofDHbvDR0ab9obkh4Ngw+D1CGVXEGduCx1+SwB1jO
+eDysCo+8ikyrrlzyDR1OyFJW28WVzLRJH0Z2bwldekM1RvCXqBYeLtAgNtS3Xb1w
+RVP9VAx7KlmNF6kG52R2dQ1Z7J7i8JIZEkBcKjITEmpKrfE=
+-----END CERTIFICATE-----
diff --git a/notary-server/fixture/tls/notary.csr b/notary-server/fixture/tls/notary.csr
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICeTCCAWECAQAwNDEYMBYGA1UECgwPdGxzbm90YXJ5c2VydmVyMRgwFgYDVQQD
+DA90bHNub3RhcnlzZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQCqo+rOvL/l3ehVLrOBpzQrjWClV03rl+xiDIElEcVSz017gvoHX0ti+etBHX+p
+lJOhVRQrO+a3QeYv7NqDnQKIMozsStClkK4MagU1114JO+z4eArfQFDvCzq2VYwD
+YBmLj4Lz0y54oQLyy/O8ON/ganYaW/3quGhufo+d774m8qCjSvhdTBnLh1GxiZKf
+M8PFaRmBCMGa4mViTlpmnZq7eDzLumlh8WeOTFIWmbjNL4DkMJKu/gHK5uOCtIUk
+FPezIN88Pq6wC88jRihXM7hrJUofPZZKzkDpwydGxol9fS0kiMANG6L8CUIeQhMD
+ElCV/XiAXHi4MtH93XWjTR3VAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAqkWE
+FyI9r3cY3tXt6j0/xGYZCX3X1AGje7vcEUeYzlED32putmH96Fkia+X2CMpEwcn7
+jaojJWvtAKGAk46p/cRpbPEOhLLebXn4znaeBVF5ph283WmeExRlhQml0e7kwTs9
+MwSniEFKBtvq4cSqO7BM1+NXDpjauVpaACl2+E9KTE8LcGG0BvH2eJOM/yW6wZmG
+ykgyMeSg5UV/i5STWlryeaGBLCCmXx4jVfkBgaXw2Zq4ve1F/qU/eQFNUPk/iRSh
+aQEQIfEC0hwqEe2Nc7X6PoVd7Py/x7Bke1JP9mRI7EPoN/IT0XHanJ08tusDCcYG
+omGrHBGk9mELh39TXQ==
+-----END CERTIFICATE REQUEST-----
diff --git a/notary-server/fixture/tls/notary.ext b/notary-server/fixture/tls/notary.ext
@@ -0,0 +1,5 @@
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = tlsnotaryserver.io
diff --git a/notary-server/fixture/tls/notary.key b/notary-server/fixture/tls/notary.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCqo+rOvL/l3ehV
+LrOBpzQrjWClV03rl+xiDIElEcVSz017gvoHX0ti+etBHX+plJOhVRQrO+a3QeYv
+7NqDnQKIMozsStClkK4MagU1114JO+z4eArfQFDvCzq2VYwDYBmLj4Lz0y54oQLy
+y/O8ON/ganYaW/3quGhufo+d774m8qCjSvhdTBnLh1GxiZKfM8PFaRmBCMGa4mVi
+TlpmnZq7eDzLumlh8WeOTFIWmbjNL4DkMJKu/gHK5uOCtIUkFPezIN88Pq6wC88j
+RihXM7hrJUofPZZKzkDpwydGxol9fS0kiMANG6L8CUIeQhMDElCV/XiAXHi4MtH9
+3XWjTR3VAgMBAAECggEBAIYDgk+nMVbIdsUfjl8PAAwMVpDEBjA2+rDufSat1Dj7
+EjEkZlUP5FbxTG+xSSfXxjH4bYSe4M2f9bZB4ENpNinc+YxCHadJ/0dEpJ7qa7H4
+3F0veepnyqhSO2Qjv3iPKsDOjtwLSP34BibFQsDaMgk/001UXhDPj0ToJMa3GLHg
+pw1G2ri4WO4NxQA354y61jBNy0D4mjHHlcnofi4iLOFr2Kf538f0RgUyw8EkZ2sE
+QyqL5HpHE93qIuLzl3/NxjQNHfO99dNNl6oWzPmXGi0nPGCMith3+8dMH7QiR/sS
+r2bjdusIccV3tlZqCJUdWDC/RVgVQKDV3pWBx+i/1gECgYEA19UrQAwgxaimcs7E
+NXISBzm2XgOOg9e5/W5EJvObu9zflqB3CBvrdZhhl1ZR+hTAbP5rHZQHWkMMAFbD
+dT+VYIqTWUCIDkFpcB7vNa41A5eIbSdz1W+V4ZdAOUuwFcaG2xQA/F37S5DvH12V
+JZ4ktJQklYUKmlUXSDTDgRUiApUCgYEAymWqTJjlSi1ADa3bAENaSKGaTazoeWBF
+OesnwTLYCT+Aap+3aMjnG5+gxlSbdfJ1odrahXA3VSZwUL0IvCg8HcJsvwc0Bw3/
+LUpwCk2yLlq+OsOtpQSgsOVOzKzXJTEnjHBZxyInJsuTb+Kf5qn7/zvGQSNvbePT
+h+YMAwGpHkECgYEAmg3DkzGU6sCYHfZLwkIrcBDXhH9RZ/XBAY2FA7B6Bjt/NApR
+K+6RwBwF/HlWhgPt3V4zoqcYIGse09caKEQ8IO6Igfo3osU5txe9cjloCapNbGvu
+l/fPqXfGFZ9ajhBoDVNX6MpEJgnLRD4NyQ358RKUkkyl5sa5mYZfzXECF4kCgYA+
+PcmDSLmqeAPssPxaNlw7XccQAA511Q804oYVOceKAIdDQt6qUK4RpqNQmpA8U1Wt
+cpok0v+RJgMAMUHQaychl7rNfC+Zw8onaW7PHFmhO7Koa6ioyKWKANqcwsJe46Df
+5WUWggA8Q/qRO8Ykrz2Zng431efciWVxs2MaQZZ6gQKBgA+QdsMadsrWoqh3tCZA
+uruQ7hXCfALJfgexWFAtLIwlHujXI81+YVICCutCe6riktPZ/zTz/nbEAyt7kIiz
+6BF7UYGT29qu1rC0MLzjfwK9ExuMSkvy9ZXGM0bCEgANIkZ0A/zTtTeRaFFGbx6l
+F1Y+ihMVuZ4rOGQbVUfQxz1F
+-----END PRIVATE KEY-----