diff --git a/.env.sample b/.env.sample index 43760dff..0cb4018c 100644 --- a/.env.sample +++ b/.env.sample @@ -1,19 +1,20 @@ +# General Configuration +DB_POOL_SIZE=5 +STORE_PAGINATION_LIMIT=100 +STORE_MAX_RETRIES=3 +STORE_INITIAL_BACKOFF_MS=100 +STREAM_THROTTLE_HISTORICAL=100 +STREAM_THROTTLE_LIVE=100 + # Authentication & Security KEYPAIR=generated-p2p-secret JWT_AUTH_SECRET=generated-secret -# AWS S3 Configuration -AWS_ACCESS_KEY_ID=test -AWS_SECRET_ACCESS_KEY=test -AWS_ENDPOINT_URL=http://localhost:4566 -AWS_REGION=us-east-1 -AWS_S3_ENABLED=false -AWS_S3_BUCKET_NAME=fuel-streams-local -STORAGE_MAX_RETRIES=5 +# Database Configuration +DATABASE_URL=postgresql://root@localhost:26257/defaultdb?sslmode=disable # NATS Configuration NATS_URL=nats://localhost:4222 -NATS_PUBLISHER_URL=nats://localhost:4333 NATS_SYSTEM_USER=sys NATS_SYSTEM_PASS=sys NATS_ADMIN_USER=admin diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7245f0fa..b81e482f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,17 +38,18 @@ jobs: refactor test scopes: | - benches repo deps release data-parser + message-broker fuel-streams core + domains executors macros - nats - storage + store + types consumer publisher webserver @@ -211,26 +212,20 @@ jobs: runs-on: ubuntu-latest env: NATS_URL: nats://127.0.0.1:4222 - NATS_PUBLISHER_URL: nats://127.0.0.1:4333 NATS_SYSTEM_USER: sys NATS_SYSTEM_PASSWORD: sys NATS_ADMIN_USER: admin NATS_ADMIN_PASS: admin NATS_PUBLIC_USER: default_user NATS_PUBLIC_PASS: "" - AWS_ACCESS_KEY_ID: test - AWS_SECRET_ACCESS_KEY: test - AWS_REGION: us-east-1 - AWS_ENDPOINT_URL: http://localhost:4566 - AWS_S3_BUCKET_NAME: fuel-streams-local strategy: fail-fast: false matrix: package: - - fuel-data-parser - fuel-streams - fuel-streams-core - fuel-streams-macros + - fuel-streams-store - sv-webserver - sv-publisher steps: diff --git a/.gitignore b/.gitignore index 8607f4ff..9971c637 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,6 @@ docs/ **/**/charts/**.tgz values-secrets.yaml values-publisher-env.yaml -localstack-data .vscode **/Cargo.lock +.sqlx diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3c46c7e0..7f6b217c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,6 @@ You can check the [./scripts/setup.sh](./scripts/setup.sh) file to see what is b Here's an overview of the project's directory structure: - `crates/`: Contains the main Rust crates for the project -- `benches/`: Benchmarking code and performance tests - `tests/`: Integration and end-to-end tests - `examples/`: Example code and usage demonstrations - `cluster/`: Kubernetes cluster configuration and deployment files @@ -91,9 +90,7 @@ This is a general rule used for commits. When you are creating a PR, ensure that - `core`: Changes that affect the core package. - `publisher`: Changes that affect the publisher package. - `fuel-streams`: Changes that affect the fuel-streams package. -- `benches`: Changes related to benchmarks. - `deps`: Changes related to dependencies. -- `data-parser`: Changes that affect the data-parser package. - `macros`: Changes that affect the macros package. ## 📜 Useful Commands @@ -111,7 +108,6 @@ To make your life easier, here are some commands to run common tasks in this pro | `make test-watch` | Run tests in watch mode | | `make clean` | Clean the build artifacts | | `make dev-watch` | Run the project in development mode with auto-reload | -| `make bench` | Run benchmarks for the project | | `make audit` | Run security audit on dependencies | | `make audit-fix` | Fix security vulnerabilities in dependencies | | `make version` | Show current version | diff --git a/Cargo.lock b/Cargo.lock index c3dc6999..70a82515 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "bytes", "futures-core", "futures-sink", @@ -56,7 +56,7 @@ dependencies = [ "actix-utils", "ahash 0.8.11", "base64 0.22.1", - "bitflags 2.6.0", + "bitflags 2.7.0", "brotli 6.0.0", "bytes", "bytestring", @@ -90,7 +90,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -207,7 +207,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -349,12 +349,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "0.6.18" @@ -461,7 +455,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "synstructure", ] @@ -473,7 +467,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -545,7 +539,7 @@ dependencies = [ "proc-macro2", "quote", "strum 0.26.3", - "syn 2.0.91", + "syn 2.0.96", "thiserror 1.0.69", ] @@ -658,18 +652,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -709,6 +703,15 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -754,7 +757,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -763,389 +766,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "aws-config" -version = "1.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json 0.60.7", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "hex", - "http 0.2.12", - "ring 0.17.8", - "time", - "tokio", - "tracing", - "url", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "zeroize", -] - -[[package]] -name = "aws-runtime" -version = "1.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-s3" -version = "1.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3ba2c5c0f2618937ce3d4a5ad574b86775576fa24006bcb3128c6e2cbf3c34e" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json 0.61.1", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "fastrand", - "hex", - "hmac 0.12.1", - "http 0.2.12", - "http-body 0.4.6", - "lru", - "once_cell", - "percent-encoding", - "regex-lite", - "sha2 0.10.8", - "tracing", - "url", -] - -[[package]] -name = "aws-sdk-sso" -version = "1.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json 0.61.1", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-ssooidc" -version = "1.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json 0.61.1", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "1.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json 0.61.1", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "1.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" -dependencies = [ - "aws-credential-types", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "crypto-bigint 0.5.5", - "form_urlencoded", - "hex", - "hmac 0.12.1", - "http 0.2.12", - "http 1.2.0", - "once_cell", - "p256 0.11.1", - "percent-encoding", - "ring 0.17.8", - "sha2 0.10.8", - "subtle", - "time", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-async" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa8ff1492fd9fb99ae28e8467af0dbbb7c31512b16fabf1a0f10d7bb6ef78bb" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.60.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc32c", - "crc32fast", - "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", - "pin-project-lite", - "sha1", - "sha2 0.10.8", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.60.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.60.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-json" -version = "0.61.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-query" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-runtime" -version = "1.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "fastrand", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "http-body 1.0.1", - "httparse", - "hyper 0.14.32", - "hyper-rustls 0.24.2", - "once_cell", - "pin-project-lite", - "pin-utils", - "rustls 0.21.12", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "1.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "bytes", - "http 0.2.12", - "http 1.2.0", - "pin-project-lite", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-types" -version = "1.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" -dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.2.0", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "itoa", - "num-integer", - "pin-project-lite", - "pin-utils", - "ryu", - "serde", - "time", - "tokio", - "tokio-util", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.60.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "rustc_version", - "tracing", -] - [[package]] name = "axum" version = "0.5.17" @@ -1307,12 +927,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" -[[package]] -name = "base16ct" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" - [[package]] name = "base16ct" version = "0.2.0" @@ -1337,16 +951,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] - [[package]] name = "base64ct" version = "1.6.0" @@ -1386,7 +990,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -1413,9 +1017,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" dependencies = [ "serde", ] @@ -1461,9 +1065,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2506947f73ad44e344215ccd6403ac2ae18cd8e046e581a441bf8d199f257f03" +checksum = "9fb65153674e51d3a42c8f27b05b9508cea85edfaade8aa46bc8fc18cecdfef3" dependencies = [ "borsh-derive", "cfg_aliases", @@ -1471,15 +1075,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2593a3b8b938bd68373196c9832f516be11fa487ef4ae745eb282e6a56a7244" +checksum = "a396e17ad94059c650db3d253bb6e25927f1eb462eede7e7a153bb6e75dce0a7" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -1573,16 +1177,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "bytestring" version = "1.4.0" @@ -1645,17 +1239,11 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" -version = "1.2.5" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" dependencies = [ "jobserver", "libc", @@ -1722,33 +1310,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "cipher" version = "0.4.4" @@ -1790,19 +1351,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" dependencies = [ "clap_builder", - "clap_derive 4.5.18", + "clap_derive 4.5.24", ] [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ "anstream", "anstyle", @@ -1825,14 +1386,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2110,7 +1671,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "462e1f6a8e005acc8835d32d60cbd7973ed65ea2a8d8473830e675f050956427" dependencies = [ "prost 0.13.4", - "tendermint-proto 0.40.0", + "tendermint-proto 0.40.1", "tonic 0.12.3", ] @@ -2122,15 +1683,15 @@ checksum = "210fbe6f98594963b46cc980f126a9ede5db9a3848ca65b71303bebdb01afcd9" dependencies = [ "bip32", "cosmos-sdk-proto", - "ecdsa 0.16.9", + "ecdsa", "eyre", "k256", "rand_core", "serde", "serde_json", - "signature 2.2.0", + "signature", "subtle-encoding", - "tendermint 0.40.0", + "tendermint 0.40.1", "thiserror 1.0.69", ] @@ -2284,14 +1845,20 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" dependencies = [ - "rustc_version", + "crc-catalog", ] +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -2301,44 +1868,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap 4.5.23", - "criterion-plot", - "futures", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - [[package]] name = "critical-section" version = "1.2.0" @@ -2364,6 +1893,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -2376,18 +1914,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array", - "rand_core", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.5.5" @@ -2454,7 +1980,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2555,7 +2081,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2577,7 +2103,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2606,19 +2132,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "data-parser" -version = "0.0.16" -dependencies = [ - "criterion", - "fuel-core-types 0.40.2", - "fuel-data-parser", - "rand", - "serde", - "strum 0.26.3", - "tokio", -] - [[package]] name = "debugid" version = "0.8.0" @@ -2628,16 +2141,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid", - "zeroize", -] - [[package]] name = "der" version = "0.7.9" @@ -2694,7 +2197,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2715,7 +2218,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "unicode-xid", ] @@ -2801,7 +2304,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2825,6 +2328,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" +[[package]] +name = "downcast-rs" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf" + [[package]] name = "dtoa" version = "1.0.9" @@ -2843,30 +2352,18 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" -[[package]] -name = "ecdsa" -version = "0.14.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" -dependencies = [ - "der 0.6.1", - "elliptic-curve 0.12.3", - "rfc6979 0.3.1", - "signature 1.6.4", -] - [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.7.9", + "der", "digest 0.10.7", - "elliptic-curve 0.13.8", - "rfc6979 0.4.0", - "signature 2.2.0", - "spki 0.7.3", + "elliptic-curve", + "rfc6979", + "signature", + "spki", ] [[package]] @@ -2875,8 +2372,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", + "pkcs8", + "signature", ] [[package]] @@ -2903,7 +2400,7 @@ dependencies = [ "rand_core", "serde", "sha2 0.10.8", - "signature 2.2.0", + "signature", "subtle", "zeroize", ] @@ -2913,6 +2410,9 @@ name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +dependencies = [ + "serde", +] [[package]] name = "elasticsearch" @@ -2925,7 +2425,7 @@ dependencies = [ "dyn-clone", "lazy_static", "percent-encoding", - "reqwest 0.12.9", + "reqwest 0.12.12", "rustc_version", "serde", "serde_json", @@ -2935,41 +2435,21 @@ dependencies = [ "void", ] -[[package]] -name = "elliptic-curve" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" -dependencies = [ - "base16ct 0.1.1", - "crypto-bigint 0.4.9", - "der 0.6.1", - "digest 0.10.7", - "ff 0.12.1", - "generic-array", - "group 0.12.1", - "pkcs8 0.9.0", - "rand_core", - "sec1 0.3.0", - "subtle", - "zeroize", -] - [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", + "base16ct", + "crypto-bigint", "digest 0.10.7", - "ff 0.13.0", + "ff", "generic-array", - "group 0.13.0", - "pkcs8 0.10.2", + "group", + "pkcs8", "rand_core", - "sec1 0.7.3", + "sec1", "subtle", "zeroize", ] @@ -3028,7 +2508,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -3048,7 +2528,7 @@ checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -3063,8 +2543,19 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ - "libc", - "windows-sys 0.59.0", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", ] [[package]] @@ -3150,7 +2641,7 @@ dependencies = [ "regex", "serde", "serde_json", - "syn 2.0.91", + "syn 2.0.96", "toml", "walkdir", ] @@ -3168,7 +2659,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -3182,7 +2673,7 @@ dependencies = [ "cargo_metadata", "chrono", "const-hex", - "elliptic-curve 0.13.8", + "elliptic-curve", "ethabi", "generic-array", "k256", @@ -3194,7 +2685,7 @@ dependencies = [ "serde", "serde_json", "strum 0.26.3", - "syn 2.0.91", + "syn 2.0.96", "tempfile", "thiserror 1.0.69", "tiny-keccak", @@ -3247,9 +2738,9 @@ checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c" [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", @@ -3288,16 +2779,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "ff" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" -dependencies = [ - "rand_core", - "subtle", -] - [[package]] name = "ff" version = "0.13.0" @@ -3373,6 +2854,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3421,7 +2913,7 @@ version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "122c27ab46707017063bf1c6e0b4f3de881e22e81b4059750a0dc95033d9cc26" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "fuel-types 0.56.0", "serde", "strum 0.24.1", @@ -3433,7 +2925,7 @@ version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f325971bf9047ec70004f80a989e03456316bc19cbef3ff3a39a38b192ab56e" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "fuel-types 0.58.2", "serde", "strum 0.24.1", @@ -3461,7 +2953,7 @@ dependencies = [ "async-graphql-value", "async-trait", "axum 0.5.17", - "clap 4.5.23", + "clap 4.5.26", "derive_more 0.99.18", "enum-iterator", "fuel-core-chain-config", @@ -3516,7 +3008,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75dd0a5d0512e90c466fad5bb6b84a971258d6bbf1e49ce6c95fd2c6f9c12d33" dependencies = [ "anyhow", - "clap 4.5.23", + "clap 4.5.26", "const_format", "dirs", "fuel-core", @@ -3648,7 +3140,7 @@ dependencies = [ "futures", "num_enum", "parking_lot", - "reqwest 0.12.9", + "reqwest 0.12.12", "serde", "strum 0.25.0", "strum_macros 0.25.3", @@ -3819,7 +3311,7 @@ dependencies = [ "futures", "postcard", "prost 0.12.6", - "reqwest 0.12.9", + "reqwest 0.12.12", "serde", "serde_json", "tendermint-rpc", @@ -3964,11 +3456,11 @@ version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33548590131674e8f272a3e056be4dbaa1de7cb364eab2b17987cd5c0dc31cb0" dependencies = [ - "ecdsa 0.16.9", + "ecdsa", "ed25519-dalek", "fuel-types 0.56.0", "k256", - "p256 0.13.2", + "p256", "serde", "sha2 0.10.8", "zeroize", @@ -3982,12 +3474,12 @@ checksum = "65e318850ca64890ff123a99b6b866954ef49da94ab9bc6827cf6ee045568585" dependencies = [ "coins-bip32", "coins-bip39", - "ecdsa 0.16.9", + "ecdsa", "ed25519-dalek", "fuel-types 0.58.2", "k256", "lazy_static", - "p256 0.13.2", + "p256", "rand", "secp256k1", "serde", @@ -4002,7 +3494,6 @@ dependencies = [ "async-compression", "async-trait", "bincode", - "displaydoc", "lazy_static", "paste", "postcard", @@ -4010,7 +3501,7 @@ dependencies = [ "serde_json", "strum 0.26.3", "strum_macros 0.26.4", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", ] @@ -4022,7 +3513,7 @@ checksum = "3f49fdbfc1615d88d2849650afc2b0ac2fecd69661ebadd31a073d8416747764" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "synstructure", ] @@ -4034,7 +3525,7 @@ checksum = "ab0bc46a3552964bae5169e79b383761a54bd115ea66951a1a7a229edcefa55a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "synstructure", ] @@ -4078,6 +3569,23 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "fuel-message-broker" +version = "0.0.16" +dependencies = [ + "async-nats", + "async-trait", + "bytes", + "dotenvy", + "futures", + "rand", + "serde", + "serde_json", + "thiserror 2.0.11", + "tokio", + "tracing", +] + [[package]] name = "fuel-sequencer-proto" version = "0.1.0" @@ -4106,14 +3614,14 @@ checksum = "2d0c46b5d76b3e11197bd31e036cd8b1cb46c4d822cacc48836638080c6d2b76" name = "fuel-streams" version = "0.0.16" dependencies = [ - "displaydoc", "fuel-streams-core", + "fuel-streams-store", "futures", - "reqwest 0.12.9", + "reqwest 0.12.12", "serde", "serde_json", "sv-webserver", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-tungstenite 0.26.1", "url", @@ -4125,30 +3633,46 @@ version = "0.0.16" dependencies = [ "anyhow", "async-nats", + "async-stream", "async-trait", - "displaydoc", "dotenvy", "fuel-core", "fuel-core-bin", - "fuel-core-client", "fuel-core-importer", "fuel-core-services", "fuel-core-storage", "fuel-core-types 0.40.2", - "fuel-data-parser", + "fuel-message-broker", + "fuel-streams-domains", "fuel-streams-macros", - "fuel-streams-nats", - "fuel-streams-storage", + "fuel-streams-store", + "fuel-streams-types", "futures", - "hex", "pretty_assertions", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", ] +[[package]] +name = "fuel-streams-domains" +version = "0.0.16" +dependencies = [ + "async-trait", + "fuel-core", + "fuel-core-types 0.40.2", + "fuel-streams-macros", + "fuel-streams-store", + "fuel-streams-types", + "serde", + "serde_json", + "sqlx", + "test-case", + "thiserror 2.0.11", +] + [[package]] name = "fuel-streams-examples" version = "0.0.16" @@ -4164,18 +3688,17 @@ name = "fuel-streams-executors" version = "0.0.16" dependencies = [ "anyhow", - "displaydoc", "dotenvy", "fuel-core", - "fuel-data-parser", "fuel-streams-core", + "fuel-streams-store", "futures", "num_cpus", "rayon", "serde", "serde_json", "sha2 0.10.8", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -4184,40 +3707,57 @@ dependencies = [ name = "fuel-streams-macros" version = "0.0.16" dependencies = [ + "downcast-rs", + "serde", + "serde_json", "subject-derive", + "test-case", + "thiserror 2.0.11", ] [[package]] -name = "fuel-streams-nats" +name = "fuel-streams-store" version = "0.0.16" dependencies = [ - "async-nats", - "displaydoc", + "async-stream", + "async-trait", "dotenvy", - "pretty_assertions", - "rand", - "serde_json", - "thiserror 2.0.9", + "fuel-data-parser", + "fuel-streams-macros", + "futures", + "serde", + "sqlx", + "test-case", + "thiserror 2.0.11", "tokio", - "tracing", ] [[package]] -name = "fuel-streams-storage" +name = "fuel-streams-test" version = "0.0.16" dependencies = [ - "async-trait", - "aws-config", - "aws-sdk-s3", - "displaydoc", - "dotenvy", - "pretty_assertions", + "anyhow", + "fuel-message-broker", + "fuel-streams-core", + "fuel-streams-domains", + "fuel-streams-macros", + "fuel-streams-store", + "futures", "rand", "serde_json", - "thiserror 2.0.9", "tokio", - "tracing", - "tracing-test", +] + +[[package]] +name = "fuel-streams-types" +version = "0.0.16" +dependencies = [ + "fuel-core", + "fuel-core-client", + "fuel-core-importer", + "fuel-core-types 0.40.2", + "hex", + "serde", ] [[package]] @@ -4226,7 +3766,7 @@ version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13aae44611588d199dd119e4a0ebd8eb7ae4cde6bf8b4d12715610b1f5e5b731" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "derivative", "derive_more 0.99.18", "fuel-asm 0.56.0", @@ -4248,7 +3788,7 @@ version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6723bb8710ba2b70516ac94d34459593225870c937670fb3afaf82e0354667ac" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "derivative", "derive_more 0.99.18", "fuel-asm 0.58.2", @@ -4296,7 +3836,7 @@ checksum = "64fc4695efac9207276f6229f2dd9811848b328a13604a698f7bce1d452bd986" dependencies = [ "async-trait", "backtrace", - "bitflags 2.6.0", + "bitflags 2.7.0", "derivative", "derive_more 0.99.18", "ethnum", @@ -4328,7 +3868,7 @@ dependencies = [ "anyhow", "async-trait", "backtrace", - "bitflags 2.6.0", + "bitflags 2.7.0", "derivative", "derive_more 0.99.18", "ethnum", @@ -4370,7 +3910,7 @@ dependencies = [ "dotenvy", "elasticsearch", "fuel-data-parser", - "fuel-streams-nats", + "fuel-message-broker", "futures", "futures-util", "jsonwebtoken 9.3.0", @@ -4383,7 +3923,7 @@ dependencies = [ "serde_json", "serde_prometheus", "sysinfo", - "thiserror 2.0.9", + "thiserror 2.0.11", "time", "tokio", "tokio-util", @@ -4453,6 +3993,17 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -4461,9 +4012,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ "futures-core", "pin-project-lite", @@ -4477,7 +4028,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -4487,7 +4038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pki-types", ] @@ -4604,9 +4155,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "gloo-timers" @@ -4630,24 +4181,13 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "group" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" -dependencies = [ - "ff 0.12.1", - "rand_core", - "subtle", -] - [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff 0.13.0", + "ff", "rand_core", "subtle", ] @@ -4690,16 +4230,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" -dependencies = [ - "cfg-if", - "crunchy", -] - [[package]] name = "hash32" version = "0.2.1" @@ -4767,6 +4297,15 @@ dependencies = [ "fxhash", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "heapless" version = "0.7.17" @@ -4924,6 +4463,15 @@ dependencies = [ "hmac 0.8.1", ] +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "hostname" version = "0.3.1" @@ -5069,9 +4617,7 @@ dependencies = [ "futures-util", "http 0.2.12", "hyper 0.14.32", - "log", "rustls 0.21.12", - "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] @@ -5086,7 +4632,7 @@ dependencies = [ "http 1.2.0", "hyper 1.5.2", "hyper-util", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pki-types", "tokio", "tokio-rustls 0.26.1", @@ -5292,7 +4838,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -5401,9 +4947,9 @@ dependencies = [ [[package]] name = "impl-more" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" +checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" [[package]] name = "impl-rlp" @@ -5432,7 +4978,7 @@ dependencies = [ "autocfg", "impl-tools-lib", "proc-macro-error2", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -5444,7 +4990,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -5455,7 +5001,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -5541,17 +5087,6 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" -[[package]] -name = "is-terminal" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" -dependencies = [ - "hermit-abi 0.4.0", - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -5602,9 +5137,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -5652,11 +5187,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ "cfg-if", - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", + "ecdsa", + "elliptic-curve", "once_cell", "sha2 0.10.8", - "signature 2.2.0", + "signature", ] [[package]] @@ -5679,6 +5214,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.8", +] [[package]] name = "lazycell" @@ -6042,7 +5580,7 @@ dependencies = [ "quinn", "rand", "ring 0.17.8", - "rustls 0.23.20", + "rustls 0.23.21", "socket2", "thiserror 1.0.69", "tokio", @@ -6102,7 +5640,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -6134,7 +5672,7 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.17.8", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-webpki 0.101.7", "thiserror 1.0.69", "x509-parser", @@ -6199,7 +5737,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "libc", ] @@ -6266,11 +5804,22 @@ dependencies = [ "libsecp256k1-core", ] +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-sys" -version = "1.1.20" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" dependencies = [ "cc", "pkg-config", @@ -6285,9 +5834,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -6759,6 +6308,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -6774,6 +6340,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.2" @@ -6792,6 +6369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -6822,7 +6400,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -6867,12 +6445,6 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "oorandom" -version = "11.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" - [[package]] name = "opaque-debug" version = "0.3.1" @@ -6910,7 +6482,7 @@ version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "cfg-if", "foreign-types", "libc", @@ -6927,7 +6499,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -6964,37 +6536,20 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -[[package]] -name = "outref" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" - [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "p256" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" -dependencies = [ - "ecdsa 0.14.8", - "elliptic-curve 0.12.3", - "sha2 0.10.8", -] - [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", + "ecdsa", + "elliptic-curve", "primeorder", "sha2 0.10.8", ] @@ -7144,7 +6699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.9", + "thiserror 2.0.11", "ucd-trie", ] @@ -7170,29 +6725,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -7201,13 +6756,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] -name = "pkcs8" -version = "0.9.0" +name = "pkcs1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.6.1", - "spki 0.6.0", + "der", + "pkcs8", + "spki", ] [[package]] @@ -7216,8 +6772,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.9", - "spki 0.7.3", + "der", + "spki", ] [[package]] @@ -7226,34 +6782,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "polling" version = "3.7.4" @@ -7388,12 +6916,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -7402,7 +6930,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve 0.13.8", + "elliptic-curve", ] [[package]] @@ -7471,14 +6999,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -7489,7 +7017,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "hex", "lazy_static", "procfs-core", @@ -7502,7 +7030,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "hex", ] @@ -7543,7 +7071,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -7552,7 +7080,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "lazy_static", "num-traits", "rand", @@ -7615,7 +7143,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -7628,7 +7156,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -7761,9 +7289,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.0", - "rustls 0.23.20", + "rustls 0.23.21", "socket2", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -7779,10 +7307,10 @@ dependencies = [ "rand", "ring 0.17.8", "rustc-hash 2.1.0", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pki-types", "slab", - "thiserror 2.0.9", + "thiserror 2.0.11", "tinyvec", "tracing", "web-time", @@ -7804,9 +7332,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -7894,7 +7422,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", ] [[package]] @@ -8026,9 +7554,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "async-compression", "base64 0.22.1", @@ -8055,7 +7583,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -8067,6 +7595,7 @@ dependencies = [ "tokio-native-tls", "tokio-rustls 0.26.1", "tokio-util", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -8086,17 +7615,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "rfc6979" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" -dependencies = [ - "crypto-bigint 0.4.9", - "hmac 0.12.1", - "zeroize", -] - [[package]] name = "rfc6979" version = "0.4.0" @@ -8222,6 +7740,26 @@ dependencies = [ "archery", ] +[[package]] +name = "rsa" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c75d7c5c6b673e58bf54d8544a9f432e3a925b0e80f7cd3602ab5c50c55519" +dependencies = [ + "const-oid", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rtnetlink" version = "0.13.1" @@ -8300,11 +7838,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "errno", "libc", "linux-raw-sys", @@ -8325,9 +7863,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "once_cell", "ring 0.17.8", @@ -8371,7 +7909,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.1.0", + "security-framework 3.2.0", ] [[package]] @@ -8424,9 +7962,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rw-stream-sink" @@ -8475,7 +8013,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -8531,23 +8069,9 @@ dependencies = [ [[package]] name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - -[[package]] -name = "sec1" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" -dependencies = [ - "base16ct 0.1.1", - "der 0.6.1", - "generic-array", - "pkcs8 0.9.0", - "subtle", - "zeroize", -] +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "sec1" @@ -8555,10 +8079,10 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct 0.2.0", - "der 0.7.9", + "base16ct", + "der", "generic-array", - "pkcs8 0.10.2", + "pkcs8", "subtle", "zeroize", ] @@ -8597,7 +8121,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -8606,11 +8130,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -8619,9 +8143,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -8650,9 +8174,9 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -8668,20 +8192,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -8709,16 +8233,16 @@ dependencies = [ [[package]] name = "serde_prometheus" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5a87b2190047e1d38f07e44814418be21033900dce2891cb737256af6550125" +checksum = "1ed13e91088fcf1395b4c92597cea91b7e173a0fabdf52cca559ccd11f6ea14e" dependencies = [ "heapless 0.8.0", "nom", "rpds", "serde", "serde_plain", - "thiserror 1.0.69", + "thiserror 2.0.11", ] [[package]] @@ -8729,7 +8253,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -8780,7 +8304,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -8858,22 +8382,12 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8 0.10.2", + "pkcs8", "rand_core", - "signature 2.2.0", + "signature", "zeroize", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest 0.10.7", - "rand_core", -] - [[package]] name = "signature" version = "2.2.0" @@ -8892,13 +8406,13 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "simple_asn1" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 1.0.69", + "thiserror 2.0.11", "time", ] @@ -8983,16 +8497,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "spki" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] - [[package]] name = "spki" version = "0.7.3" @@ -9000,7 +8504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.9", + "der", ] [[package]] @@ -9009,6 +8513,192 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" +[[package]] +name = "sqlx" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4410e73b3c0d8442c5f99b425d7a435b5ee0ae4167b3196771dd3f7a01be745f" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a007b6936676aa9ab40207cde35daab0a04b823be8ae004368c0793b96a61e0" +dependencies = [ + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.2", + "hashlink", + "indexmap 2.7.0", + "log", + "memchr", + "native-tls", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2 0.10.8", + "smallvec", + "thiserror 2.0.11", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3112e2ad78643fef903618d78cf0aec1cb3134b019730edb039b69eaf531f310" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.96", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e9f90acc5ab146a99bf5061a7eb4976b573f560bc898ef3bf8435448dd5e7ad" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.8", + "sqlx-core", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.96", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.7.0", + "byteorder", + "bytes", + "crc", + "digest 0.10.7", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac 0.12.1", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "sha1", + "sha2 0.10.8", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.11", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.7.0", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac 0.12.1", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2 0.10.8", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.11", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f85ca71d3a5b24e64e1d08dd8fe36c6c95c339a896cc33068148906784620540" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "tracing", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -9028,14 +8718,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7beae5182595e9a8b683fa98c4317f956c9a2dec3b9716990d20023cc60c766" [[package]] -name = "streams-tests" -version = "0.0.16" +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "fuel-core", - "fuel-streams", - "fuel-streams-core", - "pretty_assertions", - "tokio", + "unicode-bidi", + "unicode-normalization", + "unicode-properties", ] [[package]] @@ -9100,7 +8790,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -9113,7 +8803,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -9122,7 +8812,7 @@ version = "0.0.16" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -9151,23 +8841,23 @@ name = "sv-consumer" version = "0.0.16" dependencies = [ "anyhow", - "async-nats", "async-trait", - "clap 4.5.23", - "displaydoc", + "bincode", + "clap 4.5.26", "dotenvy", "fuel-core", + "fuel-message-broker", "fuel-streams-core", "fuel-streams-executors", + "fuel-streams-store", "fuel-web-utils", "futures", "hex", "num_cpus", "openssl", "prometheus", - "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-util", "tracing", @@ -9179,22 +8869,22 @@ name = "sv-publisher" version = "0.0.16" dependencies = [ "anyhow", - "async-nats", "async-trait", - "clap 4.5.23", - "displaydoc", + "clap 4.5.26", "fuel-core", "fuel-core-bin", "fuel-core-types 0.40.2", + "fuel-message-broker", "fuel-streams-core", "fuel-streams-executors", + "fuel-streams-store", "fuel-web-utils", "futures", "openssl", "prometheus", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-util", "tracing", @@ -9207,24 +8897,24 @@ dependencies = [ "actix-web", "actix-ws", "anyhow", - "async-nats", "async-trait", - "clap 4.5.23", + "bytes", + "clap 4.5.26", "displaydoc", "dotenvy", "fuel-data-parser", + "fuel-message-broker", "fuel-streams-core", - "fuel-streams-nats", - "fuel-streams-storage", + "fuel-streams-domains", + "fuel-streams-store", "fuel-web-utils", "futures", "num_cpus", "openssl", - "parking_lot", "prometheus", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "time", "tokio", "tracing", @@ -9235,9 +8925,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "12.12.4" +version = "12.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd33e73f154e36ec223c18013f7064a2c120f1162fc086ac9933542def186b00" +checksum = "bf08b42a6f9469bd8584daee39a1352c8133ccabc5151ccccb15896ef047d99a" dependencies = [ "debugid", "memmap2", @@ -9247,9 +8937,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.12.4" +version = "12.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e51191290147f071777e37fe111800bb82a9059f9c95b19d2dd41bfeddf477" +checksum = "32f73b5a5bd4da72720c45756a2d11edf110116b87f998bda59b97be8c2c7cf1" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -9269,9 +8959,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.91" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -9301,7 +8991,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -9336,7 +9026,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "core-foundation 0.9.4", "system-configuration-sys 0.6.0", ] @@ -9384,12 +9074,13 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -9416,7 +9107,7 @@ dependencies = [ "serde_json", "serde_repr", "sha2 0.10.8", - "signature 2.2.0", + "signature", "subtle", "subtle-encoding", "tendermint-proto 0.36.0", @@ -9426,9 +9117,9 @@ dependencies = [ [[package]] name = "tendermint" -version = "0.40.0" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d513ce7f9e41c67ab2dd3d554ef65f36fbcc61745af1e1f93eafdeefa1ce37" +checksum = "d9703e34d940c2a293804752555107f8dbe2b84ec4c6dd5203831235868105d2" dependencies = [ "bytes", "digest 0.10.7", @@ -9446,10 +9137,10 @@ dependencies = [ "serde_json", "serde_repr", "sha2 0.10.8", - "signature 2.2.0", + "signature", "subtle", "subtle-encoding", - "tendermint-proto 0.40.0", + "tendermint-proto 0.40.1", "time", "zeroize", ] @@ -9486,9 +9177,9 @@ dependencies = [ [[package]] name = "tendermint-proto" -version = "0.40.0" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c81ba1b023ec00763c3bc4f4376c67c0047f185cccf95c416c7a2f16272c4cbb" +checksum = "9ae9e1705aa0fa5ecb2c6aa7fb78c2313c4a31158ea5f02048bf318f849352eb" dependencies = [ "bytes", "flex-error", @@ -9547,6 +9238,39 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", + "test-case-core", +] + [[package]] name = "textwrap" version = "0.16.1" @@ -9564,11 +9288,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.9", + "thiserror-impl 2.0.11", ] [[package]] @@ -9579,18 +9303,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] name = "thiserror-impl" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -9675,16 +9399,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.8.1" @@ -9702,9 +9416,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -9730,13 +9444,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -9775,7 +9489,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.20", + "rustls 0.23.21", "tokio", ] @@ -9973,6 +9687,7 @@ dependencies = [ "futures-util", "pin-project-lite", "sync_wrapper 1.0.2", + "tokio", "tower-layer", "tower-service", ] @@ -10002,7 +9717,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "bytes", "futures-core", "futures-util", @@ -10061,7 +9776,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -10129,27 +9844,6 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "tracing-test" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "557b891436fe0d5e0e363427fc7f217abf9ccd510d5136549847bdcbcd011d68" -dependencies = [ - "tracing-core", - "tracing-subscriber", - "tracing-test-macro", -] - -[[package]] -name = "tracing-test-macro" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" -dependencies = [ - "quote", - "syn 2.0.91", -] - [[package]] name = "triomphe" version = "0.1.14" @@ -10207,7 +9901,7 @@ dependencies = [ "log", "rand", "sha1", - "thiserror 2.0.9", + "thiserror 2.0.11", "utf-8", ] @@ -10262,6 +9956,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -10361,9 +10061,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "b913a3b5fe84142e269d63cc62b64319ccaf89b748fc31fe025177f767a756c4" dependencies = [ "getrandom", "serde", @@ -10396,7 +10096,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -10423,12 +10123,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -[[package]] -name = "vsimd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" - [[package]] name = "walkdir" version = "2.5.0" @@ -10454,36 +10148,43 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -10494,9 +10195,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10504,22 +10205,25 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-encoder" @@ -10537,7 +10241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d28bc49ba1e5c5b61ffa7a2eace10820443c4b7d1c0b144109261d14570fdf8" dependencies = [ "ahash 0.8.11", - "bitflags 2.6.0", + "bitflags 2.7.0", "hashbrown 0.14.5", "indexmap 2.7.0", "semver", @@ -10562,7 +10266,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe501caefeb9f7b15360bdd7e47ad96e20223846f1c7db485ae5820ba5acc3d2" dependencies = [ "anyhow", - "bitflags 2.6.0", + "bitflags 2.7.0", "bumpalo", "cc", "cfg-if", @@ -10635,7 +10339,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -10734,7 +10438,7 @@ checksum = "a2bde986038b819bc43a21fef0610aeb47aabfe3ea09ca3533a7b81023b84ec6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -10751,9 +10455,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -10784,6 +10488,16 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +dependencies = [ + "redox_syscall", + "wasite", +] + [[package]] name = "widestring" version = "1.1.0" @@ -11039,9 +10753,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" dependencies = [ "memchr", ] @@ -11145,15 +10859,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" - -[[package]] -name = "xmlparser" -version = "0.13.6" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" [[package]] name = "xmltree" @@ -11239,7 +10947,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "synstructure", ] @@ -11261,7 +10969,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -11281,7 +10989,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "synstructure", ] @@ -11302,7 +11010,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -11324,7 +11032,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 43383a9a..b9dae3bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,7 @@ [workspace] default-members = ["crates/fuel-streams"] resolver = "2" -members = [ - "benches/*", - "crates/*", - "crates/fuel-streams-macros/subject-derive", - "examples", - "tests", -] +members = ["crates/*", "crates/fuel-streams-macros/subject-derive", "examples", "tests"] [workspace.package] authors = ["Fuel Labs "] @@ -26,6 +20,8 @@ actix-web = "4.9" anyhow = "1.0" async-nats = "0.38" async-trait = "0.1" +async-stream = "0.3" +bytes = "1.9" chrono = { version = "0.4", features = ["serde"] } clap = { version = "4.5", features = ["derive", "env"] } dotenvy = "0.15" @@ -58,22 +54,35 @@ sha2 = "0.10" strum = "0.26" strum_macros = "0.26" tokio = { version = "1.41", features = ["full"] } +tokio-stream = "0.1" tracing = "0.1" tracing-subscriber = "0.3" tracing-actix-web = "0.7" thiserror = "2.0" +regex = "1.10.3" +moka = { version = "0.12.9", features = ["sync"] } +bincode = "1.3" +test-case = "3.3" +sqlx = { version = "0.8.3", default-features = false, features = [ + "runtime-tokio", + "postgres", + "tls-native-tls", + "macros", +] } -fuel-streams = { version = "0.0.16", path = "crates/fuel-streams" } fuel-data-parser = { version = "0.0.16", path = "crates/fuel-data-parser" } -fuel-web-utils = { version = "0.0.16", path = "crates/fuel-web-utils" } +fuel-message-broker = { version = "0.0.16", path = "crates/fuel-message-broker" } +fuel-streams = { version = "0.0.16", path = "crates/fuel-streams" } fuel-streams-core = { version = "0.0.16", path = "crates/fuel-streams-core" } -fuel-streams-macros = { version = "0.0.16", path = "crates/fuel-streams-macros" } -fuel-streams-nats = { version = "0.0.16", path = "crates/fuel-streams-nats" } -fuel-streams-storage = { version = "0.0.16", path = "crates/fuel-streams-storage" } +fuel-streams-domains = { version = "0.0.16", path = "crates/fuel-streams-domains" } fuel-streams-executors = { version = "0.0.16", path = "crates/fuel-streams-executors" } +fuel-streams-macros = { version = "0.0.16", path = "crates/fuel-streams-macros" } +fuel-streams-store = { version = "0.0.16", path = "crates/fuel-streams-store" } +fuel-streams-types = { version = "0.0.16", path = "crates/fuel-streams-types" } +fuel-web-utils = { version = "0.0.16", path = "crates/fuel-web-utils" } subject-derive = { version = "0.0.16", path = "crates/fuel-streams-macros/subject-derive" } -sv-publisher = { version = "0.0.16", path = "crates/sv-publisher" } sv-consumer = { version = "0.0.16", path = "crates/sv-consumer" } +sv-publisher = { version = "0.0.16", path = "crates/sv-publisher" } sv-webserver = { version = "0.0.16", path = "crates/sv-webserver" } # Workspace projects diff --git a/Makefile b/Makefile index 59a5d530..1af35470 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ RUST_VERSION := 1.81.0 .PHONY: install validate-env check-commands check-network check-versions \ check-dev-env setup create-env version bump-version release dev-watch \ - clean clean-build cleanup-artifacts test-watch test bench helm-test \ + clean clean-build cleanup-artifacts test-watch test helm-test \ fmt fmt-cargo fmt-rust fmt-prettier fmt-markdown lint lint-cargo \ lint-rust lint-clippy lint-prettier lint-markdown lint-machete \ audit audit-fix-test audit-fix load-test run-publisher run-consumer \ @@ -138,9 +138,6 @@ test: cargo test --profile $(PROFILE) --doc -p $(PACKAGE) --all-features; \ fi -bench: - cargo bench -p data-parser - helm-test: helm unittest -f "tests/**/*.yaml" -f "tests/*.yaml" cluster/charts/fuel-streams @@ -213,42 +210,32 @@ run-publisher: PORT="4000" run-publisher: TELEMETRY_PORT="8080" run-publisher: NATS_URL="localhost:4222" run-publisher: EXTRA_ARGS="" +run-publisher: FROM_HEIGHT="0" run-publisher: check-network @./scripts/run_publisher.sh run-publisher-mainnet-dev: - $(MAKE) run-publisher NETWORK=mainnet MODE=dev + $(MAKE) run-publisher NETWORK=mainnet MODE=dev FROM_HEIGHT=0 run-publisher-mainnet-profiling: - $(MAKE) run-publisher NETWORK=mainnet MODE=profiling + $(MAKE) run-publisher NETWORK=mainnet MODE=profiling FROM_HEIGHT=0 run-publisher-testnet-dev: - $(MAKE) run-publisher NETWORK=testnet MODE=dev + $(MAKE) run-publisher NETWORK=testnet MODE=dev FROM_HEIGHT=0 run-publisher-testnet-profiling: - $(MAKE) run-publisher NETWORK=testnet MODE=profiling - -run-consumer: NATS_CORE_URL="localhost:4222" -run-consumer: NATS_PUBLISHER_URL="localhost:4223" -run-consumer: PORT="9003" -run-consumer: - cargo run --package sv-consumer --profile dev -- \ - --nats-core-url $(NATS_CORE_URL) \ - --port $(PORT) \ - --nats-publisher-url $(NATS_PUBLISHER_URL) + $(MAKE) run-publisher NETWORK=testnet MODE=profiling FROM_HEIGHT=0 # ------------------------------------------------------------ # Consumer Run Commands # ------------------------------------------------------------ run-consumer: NATS_URL="localhost:4222" -run-consumer: NATS_PUBLISHER_URL="localhost:4333" run-consumer: PORT="9003" run-consumer: cargo run --package sv-consumer --profile dev -- \ --nats-url $(NATS_URL) \ - --port $(PORT) \ - --nats-publisher-url $(NATS_PUBLISHER_URL) + --port $(PORT) # ------------------------------------------------------------ # Streamer Run Commands @@ -279,7 +266,7 @@ run-webserver-testnet-profiling: # ------------------------------------------------------------ # Define service profiles -DOCKER_SERVICES := nats localstack docker +DOCKER_SERVICES := nats docker cockroach run-docker-compose: PROFILE="all" run-docker-compose: @@ -308,6 +295,12 @@ $(foreach service,$(DOCKER_SERVICES),$(eval $(call make-docker-commands,$(servic reset-nats: clean-nats start-nats +setup-db: + @echo "Setting up database..." + @cd crates/fuel-streams-store && cargo sqlx migrate run + +reset-db: clean-docker start-docker setup-db + # ------------------------------------------------------------ # Local cluster (Minikube) # ------------------------------------------------------------ diff --git a/README.md b/README.md index dcb22bda..fc339ab5 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ With Fuel Data Systems, developers can build sophisticated applications that lev // Subscribe to blocks with last delivery policy let mut stream = connection - .subscribe::(subject, DeliverPolicy::Last) + .subscribe::(subject, DeliverPolicy::New) .await?; while let Some(block) = stream.next().await { diff --git a/Tiltfile b/Tiltfile index 8b9d1653..ebdeae46 100755 --- a/Tiltfile +++ b/Tiltfile @@ -91,41 +91,29 @@ RESOURCES = { 'ports': ['8080:8080'], 'labels': 'publisher', 'config_mode': ['minimal', 'full'], - 'deps': ['fuel-streams-nats-core', 'fuel-streams-nats-publisher'] + 'deps': ['fuel-streams-nats', ] }, 'consumer': { 'name': 'fuel-streams-sv-consumer', 'ports': ['8081:8080'], 'labels': 'consumer', 'config_mode': ['minimal', 'full'], - 'deps': ['fuel-streams-nats-core', 'fuel-streams-nats-publisher', 'fuel-streams-sv-publisher'] + 'deps': ['fuel-streams-nats', 'fuel-streams-sv-publisher'] }, 'sv-webserver': { 'name': 'fuel-streams-sv-webserver', 'ports': ['9003:9003'], 'labels': 'ws', 'config_mode': ['minimal', 'full'], - 'deps': ['fuel-streams-nats-core', 'fuel-streams-nats-publisher'] + 'deps': ['fuel-streams-nats'] }, - 'consumer': { - 'name': 'fuel-streams-sv-consumer', - 'ports': ['8082:8082'], - 'labels': 'consumer', - 'config_mode': ['minimal', 'full'] - }, - 'nats-core': { - 'name': 'fuel-streams-nats-core', + 'nats': { + 'name': 'fuel-streams-nats', 'ports': ['4222:4222', '6222:6222', '7422:7422'], 'labels': 'nats', 'config_mode': ['minimal', 'full'] }, - 'nats-publisher': { - 'name': 'fuel-streams-nats-publisher', - 'ports': ['4333:4222', '6222:6222', '7433:7422'], - 'labels': 'nats', - 'config_mode': ['minimal', 'full'], - 'deps': ['fuel-streams-nats-core'] - }, + } k8s_yaml(helm( diff --git a/benches/data-parser/Cargo.toml b/benches/data-parser/Cargo.toml deleted file mode 100644 index a56bf2d9..00000000 --- a/benches/data-parser/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[package] -name = "data-parser" -authors = { workspace = true } -keywords = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -repository = { workspace = true } -version = { workspace = true } -rust-version = { workspace = true } -publish = false - -[[bench]] -name = "serialize" -harness = false # do not use the default harness test -path = "benches/serialize.rs" - -[[bench]] -name = "deserialize" -harness = false # do not use the default harness test -path = "benches/deserialize.rs" - -[[bench]] -name = "serialize_compress" -harness = false # do not use the default harness test -path = "benches/serialize_compress.rs" - -[[bench]] -name = "deserialize_decompress" -harness = false # do not use the default harness test -path = "benches/deserialize_decompress.rs" - -[dependencies] -fuel-core-types = { workspace = true } -fuel-data-parser = { workspace = true, features = ["test-helpers", "bench-helpers"] } -rand = { workspace = true } -serde = { workspace = true } -strum = { workspace = true } -tokio = { workspace = true } - -[dev-dependencies] -criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } diff --git a/benches/data-parser/README.md b/benches/data-parser/README.md deleted file mode 100644 index c6607b41..00000000 --- a/benches/data-parser/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Running - -You can run all benchmarks with cargo: - -```sh -cargo bench -``` diff --git a/benches/data-parser/benches/deserialize.rs b/benches/data-parser/benches/deserialize.rs deleted file mode 100644 index 7df9efdd..00000000 --- a/benches/data-parser/benches/deserialize.rs +++ /dev/null @@ -1,74 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use data_parser::generate_test_block; -use fuel_core_types::{blockchain::block::Block, fuel_tx::Transaction}; -use fuel_data_parser::{ - DataParser, - SerializationType, - DEFAULT_COMPRESSION_STRATEGY, -}; -use strum::IntoEnumIterator; - -fn bench_deserialize(c: &mut Criterion) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - // Benchmarks for different serialization methods - let parametric_matrix = SerializationType::iter() - .map(|serialization_type| { - (serialization_type, DEFAULT_COMPRESSION_STRATEGY.clone()) - }) - .collect::>(); - - // Pre-serialize data for each serialization type - let serialized_data: Vec<_> = parametric_matrix - .iter() - .map(|(serialization_type, compression_strategy)| { - let test_block = generate_test_block(); - let data_parser = DataParser::default() - .with_compression_strategy(compression_strategy) - .with_serialization_type(serialization_type.clone()); - - // Perform serialization asynchronously and collect the results - let serialized = runtime.block_on(async { - data_parser - .serialize(&test_block) - .await - .expect("serialization failed") - }); - - (serialization_type.clone(), compression_strategy, serialized) - }) - .collect(); - - let mut group = c.benchmark_group("deserialize"); - - // benchmark each combination - for (serialization_type, compression_strategy, serialized) in - serialized_data - { - let bench_name = format!("[{}]", serialization_type); - group.bench_function(&bench_name, |b| { - let data_parser = DataParser::default() - .with_compression_strategy(compression_strategy) - .with_serialization_type(serialization_type.clone()); - - b.iter(|| { - // Perform deserialization - let result = runtime.block_on(async { - data_parser - .deserialize::>(&serialized) - .expect("deserialization failed") - }); - // Use black_box to make sure 'result' is considered used by the compiler - black_box(result); - }); - }); - } - - group.finish(); -} - -criterion_group!(benches, bench_deserialize); -criterion_main!(benches); diff --git a/benches/data-parser/benches/deserialize_decompress.rs b/benches/data-parser/benches/deserialize_decompress.rs deleted file mode 100644 index 1abc1978..00000000 --- a/benches/data-parser/benches/deserialize_decompress.rs +++ /dev/null @@ -1,72 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use data_parser::{generate_test_block, TestBlock}; -use fuel_data_parser::{ - DataParser, - SerializationType, - ALL_COMPRESSION_STRATEGIES, -}; -use strum::IntoEnumIterator; - -fn bench_decompress_deserialize(c: &mut Criterion) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - // Pre-serialize data for each combination type - let mut parametric_matrix = vec![]; - for serialization_type in SerializationType::iter() { - for compression_strategy in ALL_COMPRESSION_STRATEGIES.iter() { - let data_parser = DataParser::default() - .with_serialization_type(serialization_type.clone()) - .with_compression_strategy(compression_strategy); - - let serialized_and_compressed = runtime.block_on(async { - data_parser - .encode(&generate_test_block()) - .await - .expect("serialization failed") - }); - - parametric_matrix.push(( - serialization_type.clone(), - compression_strategy, - serialized_and_compressed, - )); - } - } - - let mut group = c.benchmark_group("decompress_deserialize"); - - // benchmark each combination - for (serialization_type, compression_strategy, serialized_and_compressed) in - parametric_matrix.iter() - { - let bench_name = format!( - "[{:?}][{:?}]", - serialization_type, - compression_strategy.name(), - ); - - group.bench_function(&bench_name, |b| { - let data_parser = DataParser::default() - .with_compression_strategy(compression_strategy) - .with_serialization_type(serialization_type.clone()); - - b.to_async(&runtime).iter(|| async { - let deserialized_and_decompressed = data_parser - .decode::(serialized_and_compressed) - .await - .expect("decompresison and deserialization"); - - // Use black_box to make sure 'result' is considered used by the compiler - black_box(deserialized_and_decompressed); - }); - }); - } - - group.finish(); -} - -criterion_group!(benches, bench_decompress_deserialize); -criterion_main!(benches); diff --git a/benches/data-parser/benches/serialize.rs b/benches/data-parser/benches/serialize.rs deleted file mode 100644 index 82778b16..00000000 --- a/benches/data-parser/benches/serialize.rs +++ /dev/null @@ -1,50 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use data_parser::generate_test_block; -use fuel_data_parser::{ - DataParser, - SerializationType, - DEFAULT_COMPRESSION_STRATEGY, -}; -use strum::IntoEnumIterator; - -fn bench_serialize(c: &mut Criterion) { - let mut group = c.benchmark_group("serialize"); - - let test_block = generate_test_block(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - // Benchmarks for different serialization methods - let parametric_matrix = SerializationType::iter() - .map(|serialization_type| { - (serialization_type, DEFAULT_COMPRESSION_STRATEGY.clone()) - }) - .collect::>(); - - for (serialization_type, compression_strategy) in parametric_matrix { - let bench_name = format!("[{}]", serialization_type); - - group.bench_function(bench_name, |b| { - let data_parser = DataParser::default() - .with_compression_strategy(&compression_strategy) - .with_serialization_type(serialization_type.clone()); - - b.to_async(&runtime).iter(|| async { - let result = data_parser - .serialize(&test_block) - .await - .expect("serialization"); - // Use black_box to make sure 'result' is considered used by the compiler - black_box(result.len()); // record size of the data - }); - }); - } - - group.finish(); -} - -criterion_group!(benches, bench_serialize); -criterion_main!(benches); diff --git a/benches/data-parser/benches/serialize_compress.rs b/benches/data-parser/benches/serialize_compress.rs deleted file mode 100644 index 9194bdce..00000000 --- a/benches/data-parser/benches/serialize_compress.rs +++ /dev/null @@ -1,50 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use data_parser::generate_test_block; -use fuel_data_parser::{ - DataParser, - SerializationType, - ALL_COMPRESSION_STRATEGIES, -}; -use strum::IntoEnumIterator; - -fn bench_serialize_compress(c: &mut Criterion) { - let mut group = c.benchmark_group("serialize_compress"); - - let test_block = generate_test_block(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - // build test matrix - for serialization_type in SerializationType::iter() { - for compression_strategy in ALL_COMPRESSION_STRATEGIES.iter() { - let bench_name = format!( - "[{:?}][{:?}]", - serialization_type.to_string(), - compression_strategy.name(), - ); - - group.bench_function(&bench_name, |b| { - let data_parser = DataParser::default() - .with_serialization_type(serialization_type.clone()) - .with_compression_strategy(compression_strategy); - - b.to_async(&runtime).iter(|| async { - let result = data_parser - .encode(&test_block) - .await - .expect("serialization and compression error"); - // Use black_box to make sure 'result' is considered used by the compiler - black_box(result.len()); // record size of the data - }); - }); - } - } - - group.finish(); -} - -criterion_group!(benches, bench_serialize_compress); -criterion_main!(benches); diff --git a/benches/data-parser/src/lib.rs b/benches/data-parser/src/lib.rs deleted file mode 100644 index 281b5c86..00000000 --- a/benches/data-parser/src/lib.rs +++ /dev/null @@ -1,56 +0,0 @@ -use fuel_core_types::{ - blockchain::{ - block::{Block, BlockV1}, - header::{ApplicationHeader, ConsensusHeader}, - primitives::DaBlockHeight, - }, - fuel_tx::{Bytes32, Transaction}, - fuel_types::BlockHeight, - tai64::Tai64, -}; -use fuel_data_parser::{DataEncoder, DataParserError}; -use rand::Rng; - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct TestBlock(Block); - -impl DataEncoder for TestBlock { - type Err = DataParserError; -} - -pub fn generate_test_block() -> TestBlock { - let mut rng = rand::thread_rng(); - let block_height: u32 = rng.gen_range(1..100); - let block_txs: u32 = rng.gen_range(1..100); - let previous_root: [u8; 32] = rng.gen(); - let tx_root: [u8; 32] = rng.gen(); - let mut block: Block = Block::V1(BlockV1::default()); - let txs = (0..block_txs) - .map(|_| Transaction::default_test_tx()) - .collect::>(); - *block.transactions_mut() = txs; - block - .header_mut() - .set_application_header(ApplicationHeader::default()); - block - .header_mut() - .set_block_height(BlockHeight::new(block_height)); - block - .header_mut() - .set_consensus_header(ConsensusHeader::default()); - block - .header_mut() - .set_da_height(DaBlockHeight::from(block_height as u64)); - block - .header_mut() - .set_previous_root(Bytes32::new(previous_root)); - block.header_mut().set_time(Tai64::now()); - block - .header_mut() - .set_transaction_root(Bytes32::new(tx_root)); - TestBlock(block) -} - -pub fn generate_test_tx() -> Transaction { - Transaction::default_test_tx() -} diff --git a/cluster/charts/fuel-streams/Chart.lock b/cluster/charts/fuel-streams/Chart.lock index 3b1e4e2c..2ea12bcd 100644 --- a/cluster/charts/fuel-streams/Chart.lock +++ b/cluster/charts/fuel-streams/Chart.lock @@ -2,11 +2,8 @@ dependencies: - name: nats repository: https://nats-io.github.io/k8s/helm/charts/ version: 1.2.8 -- name: nats - repository: https://nats-io.github.io/k8s/helm/charts/ - version: 1.2.8 -- name: nats - repository: https://nats-io.github.io/k8s/helm/charts/ - version: 1.2.8 -digest: sha256:a5f3dd64e1a20f7c9d58894359f6f909f33d14772355ee70033fd411219bcc7e -generated: "2024-12-18T16:59:13.903435-03:00" +- name: cockroachdb + repository: https://charts.cockroachdb.com/ + version: 15.0.3 +digest: sha256:31915b4f840d27a1b3c42639e007e420b5fd4300b2038990746c9643e784a2c7 +generated: "2025-01-10T23:33:35.594199-03:00" diff --git a/cluster/charts/fuel-streams/Chart.yaml b/cluster/charts/fuel-streams/Chart.yaml index a6537880..e15d3c35 100755 --- a/cluster/charts/fuel-streams/Chart.yaml +++ b/cluster/charts/fuel-streams/Chart.yaml @@ -2,15 +2,10 @@ apiVersion: v2 appVersion: "1.0" description: A Helm chart for Kubernetes name: fuel-streams -version: 0.8.8 +version: 0.9.0 dependencies: - name: nats version: 1.2.8 repository: https://nats-io.github.io/k8s/helm/charts/ - alias: nats-core - condition: nats-core.enabled - - name: nats - version: 1.2.8 - repository: https://nats-io.github.io/k8s/helm/charts/ - alias: nats-publisher - condition: nats-publisher.enabled + alias: nats + condition: nats.enabled diff --git a/cluster/charts/fuel-streams/templates/consumer/deployment.yaml b/cluster/charts/fuel-streams/templates/consumer/deployment.yaml index 39dcfecd..b9c9599e 100644 --- a/cluster/charts/fuel-streams/templates/consumer/deployment.yaml +++ b/cluster/charts/fuel-streams/templates/consumer/deployment.yaml @@ -43,8 +43,6 @@ spec: args: - "--nats-url" - "$(NATS_URL)" - - "--nats-publisher-url" - - "$(NATS_PUBLISHER_URL)" {{- with $consumer.image.args }} {{- toYaml . | nindent 10 }} {{- end }} diff --git a/cluster/charts/fuel-streams/values-local.yaml b/cluster/charts/fuel-streams/values-local.yaml index 46254d83..e7f6acb6 100644 --- a/cluster/charts/fuel-streams/values-local.yaml +++ b/cluster/charts/fuel-streams/values-local.yaml @@ -5,22 +5,11 @@ config: commonConfigMap: enabled: true data: - AWS_S3_BUCKET_NAME: "fuel-streams-staging" - AWS_ENDPOINT_URL: "https://localhost:9000" - AWS_REGION: "us-east-1" - AWS_S3_ENABLED: "false" USE_METRICS: "false" - NATS_URL: "fuel-streams-nats-core:4222" - NATS_PUBLISHER_URL: "fuel-streams-nats-publisher:4222" - NATS_SYSTEM_USER: "sys" - NATS_SYSTEM_PASS: "sys" - NATS_ADMIN_USER: "admin" - NATS_ADMIN_PASS: "admin" - NATS_PUBLIC_USER: "default_user" - NATS_PUBLIC_PASS: "" # Reduce storage requirements for local development publisher: + enabled: false image: repository: sv-publisher pullPolicy: IfNotPresent @@ -41,7 +30,7 @@ publisher: memory: 512Mi consumer: - enabled: true + enabled: false image: repository: sv-consumer pullPolicy: IfNotPresent @@ -58,7 +47,7 @@ consumer: memory: 512Mi webserver: - enabled: true + enabled: false image: repository: sv-webserver pullPolicy: IfNotPresent @@ -72,8 +61,8 @@ webserver: enabled: false # NATS Core configuration for local development -nats-core: - enabled: true +nats: + enabled: false container: env: GOMEMLIMIT: 1GiB @@ -103,33 +92,3 @@ nats-core: jetstream: max_file_store: << 10GiB >> max_memory_store: << 1GiB >> - -# NATS Publisher configuration for local development -nats-publisher: - enabled: true - container: - env: - GOMEMLIMIT: 1GiB - merge: - envFrom: - - configMapRef: - name: fuel-streams-config - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - cpu: 500m - memory: 1Gi - - config: - jetstream: - fileStore: - pvc: - size: 10Gi - storageClassName: "standard" - - merge: - jetstream: - max_file_store: << 10GiB >> - max_memory_store: << 1GiB >> diff --git a/cluster/charts/fuel-streams/values.yaml b/cluster/charts/fuel-streams/values.yaml index 5a6d0abc..8829a9f3 100755 --- a/cluster/charts/fuel-streams/values.yaml +++ b/cluster/charts/fuel-streams/values.yaml @@ -11,9 +11,9 @@ serviceAccount: create: true automount: true -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # Global configurations -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ labels: {} annotations: {} @@ -69,26 +69,23 @@ startupProbe: failureThreshold: 6 successThreshold: 1 -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # Global ConfigMap -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ commonConfigMap: enabled: true data: - AWS_S3_BUCKET_NAME: "fuel-streams-staging" - AWS_ENDPOINT_URL: "https://s3.us-east-1.amazonaws.com" - AWS_REGION: "us-east-1" - AWS_S3_ENABLED: "true" USE_METRICS: "false" - NATS_URL: "fuel-streams-nats-core:4222" - NATS_PUBLISHER_URL: "fuel-streams-nats-publisher:4222" + NATS_URL: "fuel-streams-nats:4222" NATS_SYSTEM_USER: "sys" NATS_SYSTEM_PASS: "sys" NATS_ADMIN_USER: "admin" NATS_ADMIN_PASS: "admin" NATS_PUBLIC_USER: "default_user" NATS_PUBLIC_PASS: "" + # For local purposes only, for production use fuel-streams-keys secret + DATABASE_URL: "postgresql://root:root@fuel-streams-cockroachdb:26257/fuel_streams?sslmode=disable" # This is a secret that is used for local development # It is not used in production @@ -96,16 +93,16 @@ localSecrets: enabled: false data: {} -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # Monitoring -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ monitoring: enabled: false -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # Publisher configuration -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ publisher: enabled: true @@ -169,9 +166,9 @@ publisher: podValue: 4 periodSeconds: 15 -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # Consumer configuration -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ consumer: enabled: true @@ -221,9 +218,9 @@ consumer: podValue: 4 periodSeconds: 15 -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # Consumer configuration -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ webserver: enabled: true @@ -294,11 +291,11 @@ webserver: podValue: 4 periodSeconds: 15 -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # NATS Core configuration -# ------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ -nats-core: +nats: enabled: true natsBox: @@ -335,10 +332,6 @@ nats-core: enabled: true cluster: enabled: true - websocket: - enabled: true - leafnodes: - enabled: true monitor: enabled: false @@ -350,10 +343,6 @@ nats-core: routeURLs: useFQDN: true - websocket: - enabled: true - port: 8443 - jetstream: enabled: true fileStore: @@ -363,10 +352,6 @@ nats-core: size: 2000Gi storageClassName: "gp3-generic" - leafnodes: - enabled: true - port: 7422 - monitor: enabled: false port: 8222 @@ -374,8 +359,7 @@ nats-core: merge: max_payload: << 32MiB >> jetstream: - domain: CORE - sync_interval: << 30s >> + sync_interval: always max_outstanding_catchup: << 512MiB >> max_file_store: << 2000GiB >> max_memory_store: << 7GiB >> @@ -386,81 +370,3 @@ nats-core: merge: $tplYaml: | {{- include "nats-accounts" . | nindent 8 }} - -# ------------------------------------------------------------------------------------------------- -# NATS Publisher configuration -# ------------------------------------------------------------------------------------------------- - -nats-publisher: - enabled: true - - natsBox: - enabled: false - - promExporter: - enabled: false - - statefulSet: - merge: - spec: - replicas: 5 - - container: - image: - repository: nats - tag: 2.10.24-alpine - env: - GOMEMLIMIT: 7GiB - merge: - resources: - requests: - cpu: 2 - memory: 8Gi - - service: - enabled: true - ports: - nats: - enabled: true - leafnodes: - enabled: true - monitor: - enabled: false - - config: - jetstream: - enabled: true - fileStore: - dir: /data - pvc: - enabled: true - size: 100Gi - storageClassName: "gp3-generic" - - leafnodes: - enabled: true - port: 7422 - merge: - remotes: - - urls: ["nats-leaf://admin:admin@fuel-streams-nats-core:7422"] - account: ADMIN - - monitor: - enabled: false - port: 8222 - - merge: - max_payload: << 32MiB >> - jetstream: - domain: PUBLISHER - sync_interval: << 30s >> - max_outstanding_catchup: << 512MiB >> - max_file_store: << 100GiB >> - max_memory_store: << 7GiB >> - system_account: SYS - $include: auth.conf - - configMap: - merge: - $tplYaml: | - {{- include "nats-accounts" . | nindent 8 }} diff --git a/cluster/docker/docker-compose.yml b/cluster/docker/docker-compose.yml index eff95417..36076d68 100644 --- a/cluster/docker/docker-compose.yml +++ b/cluster/docker/docker-compose.yml @@ -1,10 +1,10 @@ services: - nats-core: + nats: profiles: - all - nats image: nats:latest - container_name: nats-core + container_name: nats restart: always ports: - 4222:4222 @@ -12,55 +12,29 @@ services: - ./nats-config/core.conf:/etc/nats/nats.conf - ./nats-config/accounts.conf:/etc/nats/accounts.conf command: - - --name=fuel-streams-nats-core + - --name=fuel-streams-nats - --js - --config=/etc/nats/nats.conf - -D env_file: - ./../../.env - nats-publisher: + cockroach: profiles: - all - - nats - image: nats:latest - container_name: nats-publisher - restart: always + - cockroach + image: cockroachdb/cockroach:v23.2.2 ports: - - 4333:4222 + - "26257:26257" + - "8080:8080" + command: start-single-node --insecure volumes: - - ./nats-config/publisher.conf:/etc/nats/nats.conf - - ./nats-config/accounts.conf:/etc/nats/accounts.conf - command: - - --name=fuel-streams-nats-publisher - - --js - - --config=/etc/nats/nats.conf - - -D - env_file: - - ./../../.env - depends_on: - - nats-core + - cockroach_data:/cockroach/cockroach-data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health?ready=1"] + interval: 10s + timeout: 5s + retries: 5 - localstack: - profiles: - - all - - localstack - image: localstack/localstack:latest - container_name: localstack - restart: always - ports: - - "4566:4566" # LocalStack main gateway port - - "4572:4572" # S3 service port (optional) - environment: - - SERVICES=s3 # Enable just S3 service - - DEBUG=1 - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-test} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-test} - - DEFAULT_REGION=${AWS_REGION:-us-east-1} - - DEFAULT_BUCKETS=${AWS_S3_BUCKET_NAME:-fuel-streams-local} - volumes: - - ./localstack-data:/var/lib/localstack - - /var/run/docker.sock:/var/run/docker.sock - - ./init-localstack.sh:/etc/localstack/init/ready.d/init-localstack.sh - env_file: - - ./../../.env +volumes: + cockroach_data: diff --git a/cluster/docker/fuel-core.Dockerfile b/cluster/docker/fuel-core.Dockerfile index 0ec3f52b..d6c8af53 100644 --- a/cluster/docker/fuel-core.Dockerfile +++ b/cluster/docker/fuel-core.Dockerfile @@ -12,23 +12,21 @@ WORKDIR /build/ COPY --from=xx / / # hadolint ignore=DL3008 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - lld \ - clang \ - libclang-dev \ - && xx-apt-get update \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + lld \ + clang \ + libclang-dev \ + && xx-apt-get update \ && xx-apt-get install -y libc6-dev g++ binutils \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* - FROM chef AS planner ENV CARGO_NET_GIT_FETCH_WITH_CLI=true COPY . . RUN cargo chef prepare --recipe-path recipe.json - FROM chef AS builder ARG PACKAGE_NAME ARG DEBUG_SYMBOLS=false diff --git a/cluster/docker/init-localstack.sh b/cluster/docker/init-localstack.sh deleted file mode 100755 index d88f344c..00000000 --- a/cluster/docker/init-localstack.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -echo "Creating S3 bucket in LocalStack..." - -BUCKET_NAME=${AWS_S3_BUCKET_NAME:-fuel-streams-local} -awslocal s3 mb "s3://${BUCKET_NAME}" -echo "Bucket created: ${BUCKET_NAME}" diff --git a/cluster/docker/nats-config/client.conf b/cluster/docker/nats-config/client.conf deleted file mode 100644 index 08d9e2b3..00000000 --- a/cluster/docker/nats-config/client.conf +++ /dev/null @@ -1,18 +0,0 @@ -port: 4222 -server_name: client-server - -jetstream { - store_dir: "./data/store_client" - domain: CLIENT -} - -leafnodes { - remotes: [ - { - urls: ["nats://admin:admin@nats-core:7422"] - account: "ADMIN" - } - ] -} - -include ./accounts.conf diff --git a/cluster/docker/nats-config/core.conf b/cluster/docker/nats-config/core.conf index 2e8f11f4..e00e4f8b 100644 --- a/cluster/docker/nats-config/core.conf +++ b/cluster/docker/nats-config/core.conf @@ -3,11 +3,6 @@ server_name: core-server jetstream { store_dir: "./data/core" - domain: CORE -} - -leafnodes { - port: 7422 } include ./accounts.conf diff --git a/cluster/docker/nats-config/publisher.conf b/cluster/docker/nats-config/publisher.conf deleted file mode 100644 index 52de2113..00000000 --- a/cluster/docker/nats-config/publisher.conf +++ /dev/null @@ -1,18 +0,0 @@ -port: 4222 -server_name: leaf-server - -jetstream { - store_dir: "./data/store_leaf" - domain: LEAF -} - -leafnodes { - remotes: [ - { - urls: ["nats://admin:admin@nats-core:7422"] - account: "ADMIN" - } - ] -} - -include ./accounts.conf diff --git a/crates/fuel-data-parser/Cargo.toml b/crates/fuel-data-parser/Cargo.toml index 6140c616..564a7e51 100644 --- a/crates/fuel-data-parser/Cargo.toml +++ b/crates/fuel-data-parser/Cargo.toml @@ -14,7 +14,6 @@ rust-version = { workspace = true } async-compression = { version = "0.4", features = ["tokio"], optional = true } async-trait = { workspace = true } bincode = { version = "1.3", optional = true } -displaydoc = { workspace = true } lazy_static = "1.5" paste = "1.0" postcard = { version = "1.0", features = ["alloc"], optional = true } diff --git a/crates/fuel-data-parser/README.md b/crates/fuel-data-parser/README.md index 0eaebbc8..e189940e 100644 --- a/crates/fuel-data-parser/README.md +++ b/crates/fuel-data-parser/README.md @@ -74,7 +74,7 @@ async fn example_usage() -> Result<(), Box> { To run the benchmarks and measure performance of different serialization and compression strategies: ```sh -cargo bench -p data-parser -p nats-publisher -p bench-consumers +cargo bench -p data-parser ``` > [!INFO] diff --git a/crates/fuel-data-parser/src/error.rs b/crates/fuel-data-parser/src/error.rs index e6564a7f..e7455200 100644 --- a/crates/fuel-data-parser/src/error.rs +++ b/crates/fuel-data-parser/src/error.rs @@ -1,63 +1,60 @@ #![allow(dead_code)] -use displaydoc::Display as DisplayDoc; -use thiserror::Error; - /// Compression error types -#[derive(Debug, DisplayDoc, Error)] +#[derive(Debug, thiserror::Error)] pub enum CompressionError { #[cfg(feature = "zlib")] - /// Failed to compress or decompress data using zlib: {0} + #[error("Failed to compress or decompress data using zlib: {0}")] Zlib(std::io::Error), #[cfg(feature = "gzip")] - /// Failed to compress or decompress data using gzip: {0} + #[error("Failed to compress or decompress data using gzip: {0}")] Gzip(std::io::Error), #[cfg(feature = "brotli")] - /// Failed to compress or decompress data using brotli: {0} + #[error("Failed to compress or decompress data using brotli: {0}")] Brotli(std::io::Error), #[cfg(feature = "bzip2")] - /// Failed to compress or decompress data using bzip2: {0} + #[error("Failed to compress or decompress data using bzip2: {0}")] Bz(std::io::Error), #[cfg(feature = "lzma")] - /// Failed to compress or decompress data using lzma: {0} + #[error("Failed to compress or decompress data using lzma: {0}")] Lzma(std::io::Error), #[cfg(feature = "deflate")] - /// Failed to compress or decompress data using deflate: {0} + #[error("Failed to compress or decompress data using deflate: {0}")] Deflate(std::io::Error), #[cfg(feature = "zstd")] - /// Failed to compress or decompress data using zstd: {0} + #[error("Failed to compress or decompress data using zstd: {0}")] Zstd(std::io::Error), } /// Serialization/Deserialization error types. -#[derive(Debug, DisplayDoc, Error)] +#[derive(Debug, thiserror::Error)] pub enum SerdeError { #[cfg(feature = "bincode")] - /// Failed to serialize or deserialize data using bincode: {0} + #[error(transparent)] Bincode(#[from] bincode::ErrorKind), #[cfg(feature = "postcard")] - /// Failed to serialize or deserialize data using postcard: {0} + #[error(transparent)] Postcard(#[from] postcard::Error), #[cfg(feature = "json")] - /// Failed to serialize or deserialize data using JSON: {0} + #[error(transparent)] Json(#[from] serde_json::Error), } /// Data parser error types. -#[derive(Debug, DisplayDoc, Error)] +#[derive(Debug, thiserror::Error)] pub enum DataParserError { - /// An error occurred during data compression or decompression: {0} + #[error(transparent)] Compression(#[from] CompressionError), - /// An error occurred during data serialization or deserialization: {0} + #[error(transparent)] Serde(#[from] SerdeError), - /// An error occurred during data encoding: {0} + #[error("An error occurred during data encoding: {0}")] Encode(#[source] SerdeError), - /// An error occurred during data decoding: {0} + #[error("An error occurred during data decoding: {0}")] Decode(#[source] SerdeError), #[cfg(feature = "json")] - /// An error occurred during data encoding to JSON: {0} + #[error("An error occurred during data encoding to JSON: {0}")] EncodeJson(#[source] SerdeError), #[cfg(feature = "json")] - /// An error occurred during data decoding from JSON: {0} + #[error("An error occurred during data decoding from JSON: {0}")] DecodeJson(#[source] SerdeError), } diff --git a/crates/fuel-data-parser/src/lib.rs b/crates/fuel-data-parser/src/lib.rs index 5de469c5..4f7667f3 100644 --- a/crates/fuel-data-parser/src/lib.rs +++ b/crates/fuel-data-parser/src/lib.rs @@ -53,13 +53,6 @@ pub trait DataEncoder: Self::data_parser().encode_json(self).map_err(Into::into) } - #[cfg(feature = "json")] - fn encode_json_value(&self) -> Result { - Self::data_parser() - .encode_json_value(self) - .map_err(Into::into) - } - async fn decode(encoded: &[u8]) -> Result { Self::data_parser() .decode(encoded) @@ -73,12 +66,8 @@ pub trait DataEncoder: } #[cfg(feature = "json")] - fn decode_json_value( - encoded: &serde_json::Value, - ) -> Result { - Self::data_parser() - .decode_json_value(encoded) - .map_err(Into::into) + fn to_json_value(&self) -> Result { + Self::data_parser().to_json_value(self).map_err(Into::into) } } @@ -141,7 +130,7 @@ impl Default for DataParser { /// ``` fn default() -> Self { Self { - compression_strategy: Some(DEFAULT_COMPRESSION_STRATEGY.clone()), + compression_strategy: None, serialization_type: SerializationType::Json, } } @@ -255,7 +244,7 @@ impl DataParser { } #[cfg(feature = "json")] - pub fn encode_json_value( + pub fn to_json_value( &self, data: &T, ) -> Result { @@ -362,14 +351,6 @@ impl DataParser { self.deserialize_json(data) } - #[cfg(feature = "json")] - pub fn decode_json_value( - &self, - data: &serde_json::Value, - ) -> Result { - self.deserialize_json_value(data) - } - /// Deserializes the provided data according to the selected `SerializationType`. /// /// # Arguments @@ -404,15 +385,6 @@ impl DataParser { serde_json::from_slice(raw_data) .map_err(|e| DataParserError::DecodeJson(SerdeError::Json(e))) } - - #[cfg(feature = "json")] - fn deserialize_json_value( - &self, - raw_data: &serde_json::Value, - ) -> Result { - serde_json::from_value(raw_data.clone()) - .map_err(|e| DataParserError::DecodeJson(SerdeError::Json(e))) - } } #[cfg(test)] diff --git a/crates/fuel-streams-nats/Cargo.toml b/crates/fuel-message-broker/Cargo.toml similarity index 72% rename from crates/fuel-streams-nats/Cargo.toml rename to crates/fuel-message-broker/Cargo.toml index 60254899..eab72ea5 100644 --- a/crates/fuel-streams-nats/Cargo.toml +++ b/crates/fuel-message-broker/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "fuel-streams-nats" -description = "Strategies and adapters for storing fuel streams in NATS" +name = "fuel-message-broker" +description = "A message broker for the Fuel Streams" authors = { workspace = true } keywords = { workspace = true } edition = { workspace = true } @@ -12,18 +12,20 @@ rust-version = { workspace = true } [dependencies] async-nats = { workspace = true } -displaydoc = { workspace = true } +async-trait = { workspace = true } +bytes = { workspace = true } dotenvy = { workspace = true } +futures = { workspace = true } rand = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } [dev-dependencies] -pretty_assertions = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] } [features] default = [] test-helpers = [] -bench-helpers = [] diff --git a/crates/fuel-message-broker/src/lib.rs b/crates/fuel-message-broker/src/lib.rs new file mode 100644 index 00000000..41909706 --- /dev/null +++ b/crates/fuel-message-broker/src/lib.rs @@ -0,0 +1,8 @@ +mod msg_broker; +mod nats; +pub mod nats_metrics; +mod nats_opts; + +pub use msg_broker::*; +pub use nats::*; +pub use nats_opts::*; diff --git a/crates/fuel-message-broker/src/msg_broker.rs b/crates/fuel-message-broker/src/msg_broker.rs new file mode 100644 index 00000000..2ac0dc55 --- /dev/null +++ b/crates/fuel-message-broker/src/msg_broker.rs @@ -0,0 +1,150 @@ +use std::{fmt, sync::Arc}; + +use async_trait::async_trait; +use futures::Stream; + +/// Represents a namespace for message broker subjects/topics +#[derive(Debug, Clone, Default)] +pub enum Namespace { + Custom(String), + #[default] + None, +} + +impl fmt::Display for Namespace { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Namespace::Custom(s) => write!(f, "{s}"), + Namespace::None => write!(f, "none"), + } + } +} + +impl Namespace { + pub fn subject_name(&self, val: &str) -> String { + match self { + Namespace::Custom(s) => format!("{s}.{val}"), + Namespace::None => val.to_string(), + } + } + + pub fn queue_name(&self, val: &str) -> String { + match self { + Namespace::Custom(s) => format!("{s}_{val}"), + Namespace::None => val.to_string(), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum MessageBrokerError { + #[error("Failed to connect to broker: {0}")] + Connection(String), + #[error("Failed to setup broker infrastructure: {0}")] + Setup(String), + #[error("Failed to publish message: {0}")] + Publishing(String), + #[error("Failed to receive message: {0}")] + Receiving(String), + #[error("Failed to acknowledge message: {0}")] + Acknowledgment(String), + #[error("Failed to subscribe: {0}")] + Subscription(String), + #[error("Failed to flush: {0}")] + Flush(String), + #[error(transparent)] + Serde(#[from] serde_json::Error), + #[error(transparent)] + Other(#[from] Box), +} + +#[async_trait] +pub trait Message: std::fmt::Debug + Send + Sync { + fn payload(&self) -> Vec; + async fn ack(&self) -> Result<(), MessageBrokerError>; +} + +pub type MessageBlockStream = Box< + dyn Stream, MessageBrokerError>> + + Send + + Unpin, +>; + +pub type MessageStream = + Box, MessageBrokerError>> + Send + Unpin>; + +#[async_trait] +pub trait MessageBroker: std::fmt::Debug + Send + Sync + 'static { + /// Get the current namespace + fn namespace(&self) -> &Namespace; + + /// Setup required infrastructure (queues, exchanges, etc) + async fn setup(&self) -> Result<(), MessageBrokerError>; + + /// Check if the broker is connected + fn is_connected(&self) -> bool; + + /// Publish a block to the work queue for processing + /// Used by publisher to send blocks to consumers + async fn publish_block( + &self, + id: String, + payload: Vec, + ) -> Result<(), MessageBrokerError>; + + /// Receive a stream of blocks from the work queue + /// Used by consumer to process blocks + async fn receive_blocks_stream( + &self, + batch_size: usize, + ) -> Result; + + /// Publish an event to a topic for subscribers + /// Used by Stream implementation for pub/sub + async fn publish_event( + &self, + topic: &str, + payload: bytes::Bytes, + ) -> Result<(), MessageBrokerError>; + + /// Subscribe to events on a topic + /// Used by Stream implementation for pub/sub + async fn subscribe_to_events( + &self, + topic: &str, + ) -> Result; + + /// Flush all in-flight messages + async fn flush(&self) -> Result<(), MessageBrokerError>; + + /// Check if the broker is healthy + async fn is_healthy(&self) -> bool; + + /// Get health info + async fn get_health_info( + &self, + uptime_secs: u64, + ) -> Result; +} + +#[derive(Debug, Clone, Default)] +pub enum MessageBrokerClient { + #[default] + Nats, +} + +impl MessageBrokerClient { + pub async fn start( + &self, + url: &str, + ) -> Result, MessageBrokerError> { + match self { + MessageBrokerClient::Nats => { + let opts = crate::NatsOpts::new(url.to_string()); + let broker = crate::NatsMessageBroker::new(&opts).await?; + broker.setup().await?; + Ok(broker.arc()) + } + } + } +} diff --git a/crates/fuel-message-broker/src/nats.rs b/crates/fuel-message-broker/src/nats.rs new file mode 100644 index 00000000..ff9850cc --- /dev/null +++ b/crates/fuel-message-broker/src/nats.rs @@ -0,0 +1,469 @@ +use std::{sync::Arc, time::Duration}; + +use async_nats::{ + jetstream::{ + consumer::{pull::Config as ConsumerConfig, AckPolicy, PullConsumer}, + context::Publish, + stream::{Config as StreamConfig, RetentionPolicy}, + Context, + }, + Client, +}; +use async_trait::async_trait; +use futures::StreamExt; +use tracing::info; + +use crate::{ + nats_metrics::{NatsHealthInfo, StreamInfo}, + Message, + MessageBlockStream, + MessageBroker, + MessageBrokerError, + MessageStream, + Namespace, + NatsOpts, +}; + +#[derive(Debug)] +pub struct NatsMessage(async_nats::jetstream::Message); + +#[derive(Debug, Clone)] +pub struct NatsMessageBroker { + pub client: Client, + pub jetstream: Context, + pub namespace: Namespace, + pub opts: NatsOpts, +} + +impl NatsMessageBroker { + const BLOCKS_STREAM: &'static str = "block_importer"; + const BLOCKS_SUBJECT: &'static str = "block_submitted"; + + pub async fn new(opts: &NatsOpts) -> Result { + let url = &opts.url(); + let client = opts.connect_opts().connect(url).await.map_err(|e| { + MessageBrokerError::Connection(format!( + "Failed to connect to NATS at {}: {}", + url, e + )) + })?; + info!("Connected to NATS server at {}", url); + let jetstream = async_nats::jetstream::new(client.clone()); + Ok(Self { + client, + jetstream, + namespace: opts.namespace.clone(), + opts: opts.clone(), + }) + } + + fn stream_name(&self) -> String { + self.namespace().queue_name(Self::BLOCKS_STREAM) + } + + fn consumer_name(&self) -> String { + format!("{}_consumer", self.stream_name()) + } + + fn blocks_subject(&self) -> String { + self.namespace().subject_name(Self::BLOCKS_SUBJECT) + } + + async fn get_blocks_stream( + &self, + ) -> Result { + let subject_name = format!("{}.>", self.blocks_subject()); + let stream_name = self.stream_name(); + let stream = self + .jetstream + .get_or_create_stream(StreamConfig { + name: stream_name, + subjects: vec![subject_name], + retention: RetentionPolicy::WorkQueue, + duplicate_window: Duration::from_secs(1), + allow_rollup: true, + ..Default::default() + }) + .await + .map_err(|e| MessageBrokerError::Setup(e.to_string()))?; + Ok(stream) + } + + async fn get_blocks_consumer( + &self, + ) -> Result { + let consumer_name = self.consumer_name(); + let stream = self.get_blocks_stream().await?; + stream + .get_or_create_consumer(&consumer_name, ConsumerConfig { + durable_name: Some(consumer_name.to_string()), + ack_policy: AckPolicy::Explicit, + ack_wait: Duration::from_secs(self.opts.ack_wait_secs), + ..Default::default() + }) + .await + .map_err(|e| MessageBrokerError::Setup(e.to_string())) + } + + pub async fn get_stream_info( + &self, + ) -> Result, MessageBrokerError> { + let mut streams = self.jetstream.streams(); + let mut infos = vec![]; + while let Some(stream) = streams.next().await { + let stream = + stream.map_err(|e| MessageBrokerError::Setup(e.to_string()))?; + infos.push(StreamInfo { + stream_name: stream.config.name, + state: stream.state.into(), + }); + } + Ok(infos) + } + + pub fn arc(&self) -> Arc { + Arc::new(self.clone()) + } +} + +#[async_trait] +impl Message for NatsMessage { + fn payload(&self) -> Vec { + self.0.payload.to_vec() + } + + async fn ack(&self) -> Result<(), MessageBrokerError> { + self.0 + .ack() + .await + .map_err(|e| MessageBrokerError::Acknowledgment(e.to_string())) + } +} + +#[async_trait] +impl MessageBroker for NatsMessageBroker { + fn namespace(&self) -> &Namespace { + &self.namespace + } + + fn is_connected(&self) -> bool { + let state = self.client.connection_state(); + state == async_nats::connection::State::Connected + } + + async fn setup(&self) -> Result<(), MessageBrokerError> { + let _ = self.get_blocks_stream().await?; + Ok(()) + } + + async fn publish_block( + &self, + id: String, + payload: Vec, + ) -> Result<(), MessageBrokerError> { + let subject = format!("{}.{}", self.blocks_subject(), id); + let payload_id = format!("{}.block_{}", self.namespace(), id); + let publish = Publish::build() + .message_id(payload_id) + .payload(payload.into()); + self.jetstream + .send_publish(subject, publish) + .await + .map_err(|e| MessageBrokerError::Publishing(e.to_string()))? + .await + .map_err(|e| MessageBrokerError::Publishing(e.to_string()))?; + + Ok(()) + } + + async fn receive_blocks_stream( + &self, + batch_size: usize, + ) -> Result { + let consumer = self.get_blocks_consumer().await?; + let stream = consumer + .fetch() + .max_messages(batch_size) + .messages() + .await + .map_err(|e| MessageBrokerError::Receiving(e.to_string()))? + .filter_map(|msg| async { + msg.ok() + .map(|m| Ok(Box::new(NatsMessage(m)) as Box)) + }) + .boxed(); + Ok(Box::new(stream)) + } + + async fn publish_event( + &self, + topic: &str, + payload: bytes::Bytes, + ) -> Result<(), MessageBrokerError> { + let subject = self.namespace().subject_name(topic); + self.client + .publish(subject, payload) + .await + .map_err(|e| MessageBrokerError::Publishing(e.to_string()))?; + Ok(()) + } + + async fn subscribe_to_events( + &self, + topic: &str, + ) -> Result { + let subject = self.namespace().subject_name(topic); + let stream = self + .client + .subscribe(subject) + .await + .map_err(|e| MessageBrokerError::Subscription(e.to_string()))? + .map(|msg| Ok(msg.payload.to_vec())); + Ok(Box::new(stream)) + } + + async fn flush(&self) -> Result<(), MessageBrokerError> { + self.client.flush().await.map_err(|e| { + MessageBrokerError::Flush(format!( + "Failed to flush NATS client: {}", + e + )) + })?; + Ok(()) + } + + async fn is_healthy(&self) -> bool { + self.is_connected() + } + + async fn get_health_info( + &self, + uptime_secs: u64, + ) -> Result { + let infos = self.get_stream_info().await?; + let health_info = NatsHealthInfo { + uptime_secs, + is_healthy: self.is_healthy().await, + streams_info: infos, + }; + Ok(serde_json::to_value(health_info)?) + } +} + +#[cfg(test)] +mod tests { + use rand::Rng; + + use super::*; + const NATS_URL: &str = "nats://localhost:4222"; + + async fn setup_broker() -> Result { + let opts = NatsOpts::new(NATS_URL.to_string()) + .with_rdn_namespace() + .with_ack_wait(1); + let broker = NatsMessageBroker::new(&opts).await?; + broker.setup().await?; + Ok(broker) + } + + #[tokio::test] + async fn test_broker_connection() -> Result<(), MessageBrokerError> { + let _broker = setup_broker().await?; + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn test_pub_sub() -> Result<(), MessageBrokerError> { + let broker = setup_broker().await?; + let broker_clone = broker.clone(); + + // Spawn a task to receive events + let receiver = tokio::spawn(async move { + let mut events = + broker_clone.subscribe_to_events("test.topic").await?; + + tokio::time::timeout(Duration::from_secs(5), events.next()) + .await + .map_err(|_| { + MessageBrokerError::Receiving( + "Timeout waiting for message".into(), + ) + })? + .ok_or_else(|| { + MessageBrokerError::Receiving("No message received".into()) + })? + }); + + // Add a small delay to ensure subscriber is ready + tokio::time::sleep(Duration::from_millis(100)).await; + + broker + .publish_event("test.topic", vec![4, 5, 6].into()) + .await?; + let result = receiver.await.expect("receiver task panicked")?; + assert_eq!(result, vec![4, 5, 6]); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn test_work_queue() -> Result<(), MessageBrokerError> { + let broker = setup_broker().await?; + let broker_clone = broker.clone(); + + // Spawn a task to receive events + let receiver = tokio::spawn(async move { + let mut messages = Vec::new(); + let mut stream = broker_clone.receive_blocks_stream(3).await?; + while let Some(msg) = stream.next().await { + let msg = msg?; + messages.push(msg); + if messages.len() >= 3 { + break; + } + } + Ok::>, MessageBrokerError>(messages) + }); + + // Publish multiple messages + broker.publish_block("1".to_string(), vec![1, 2, 3]).await?; + broker.publish_block("2".to_string(), vec![4, 5, 6]).await?; + broker.publish_block("3".to_string(), vec![7, 8, 9]).await?; + + // Wait for receiver and check results + let messages = receiver.await.expect("receiver task panicked")?; + assert_eq!(messages.len(), 3, "Expected to receive 3 messages"); + assert_eq!(messages[0].payload(), &[1, 2, 3]); + assert_eq!(messages[1].payload(), &[4, 5, 6]); + assert_eq!(messages[2].payload(), &[7, 8, 9]); + + // Acknowledge all messages + for msg in messages { + msg.ack().await?; + } + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn test_work_queue_batch_size_limiting( + ) -> Result<(), MessageBrokerError> { + let broker = setup_broker().await?; + + // Publish 3 messages + broker.publish_block("1".to_string(), vec![1]).await?; + broker.publish_block("2".to_string(), vec![2]).await?; + broker.publish_block("3".to_string(), vec![3]).await?; + + // Receive with batch size of 2 + let mut stream = broker.receive_blocks_stream(2).await?; + let mut received = Vec::new(); + while let Some(msg) = stream.next().await { + let msg = msg?; + received.push(msg.payload().to_vec()); + msg.ack().await?; + } + + assert_eq!( + received.len(), + 2, + "Should only receive batch_size messages" + ); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn test_work_queue_unacked_message_redelivery( + ) -> Result<(), MessageBrokerError> { + let broker = setup_broker().await?; + broker.publish_block("1".to_string(), vec![1]).await?; + + { + let mut stream = broker.receive_blocks_stream(1).await?; + let msg = stream.next().await.unwrap(); + assert!(msg.is_ok()); + let msg = msg.unwrap(); + assert_eq!(msg.payload(), &[1]); + } + + // Message should be redelivered after ack wait of 1 second + tokio::time::sleep(Duration::from_secs(2)).await; + + // Receive message again + let mut stream = broker.receive_blocks_stream(1).await?; + let msg = stream.next().await.unwrap(); + assert!(msg.is_ok()); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn test_work_queue_multiple_consumers( + ) -> Result<(), MessageBrokerError> { + let broker = setup_broker().await?; + let broker1 = broker.clone(); + let broker2 = broker.clone(); + let broker3 = broker.clone(); + + // Spawn three consumer tasks + let consumer1 = tokio::spawn(async move { + let mut stream = broker1.receive_blocks_stream(1).await?; + let msg = stream.next().await.ok_or_else(|| { + MessageBrokerError::Receiving("No message received".into()) + })??; + msg.ack().await?; + Ok::, MessageBrokerError>(msg.payload().to_vec()) + }); + + let consumer2 = tokio::spawn(async move { + let mut stream = broker2.receive_blocks_stream(1).await?; + let msg = stream.next().await.ok_or_else(|| { + MessageBrokerError::Receiving("No message received".into()) + })??; + msg.ack().await?; + Ok::, MessageBrokerError>(msg.payload().to_vec()) + }); + + let consumer3 = tokio::spawn(async move { + let mut stream = broker3.receive_blocks_stream(1).await?; + let msg = stream.next().await.ok_or_else(|| { + MessageBrokerError::Receiving("No message received".into()) + })??; + msg.ack().await?; + Ok::, MessageBrokerError>(msg.payload().to_vec()) + }); + + let heights = (0..3).map(|_| random_height() as u8).collect::>(); + + // Publish three messages + broker + .publish_block(heights[0].to_string(), vec![heights[0]]) + .await?; + broker + .publish_block(heights[1].to_string(), vec![heights[1]]) + .await?; + broker + .publish_block(heights[2].to_string(), vec![heights[2]]) + .await?; + + // Collect results from all consumers + let msg1 = consumer1.await.expect("consumer1 task panicked")?; + let msg2 = consumer2.await.expect("consumer2 task panicked")?; + let msg3 = consumer3.await.expect("consumer3 task panicked")?; + + // Verify that each consumer got a different message + let mut received = vec![msg1[0], msg2[0], msg3[0]]; + let mut heights = heights.clone(); + received.sort(); + heights.sort(); + assert_eq!( + received, heights, + "Consumers should receive all messages, regardless of order" + ); + + Ok(()) + } + + fn random_height() -> u32 { + rand::thread_rng().gen_range(1..1000) + } +} diff --git a/crates/fuel-message-broker/src/nats_metrics.rs b/crates/fuel-message-broker/src/nats_metrics.rs new file mode 100644 index 00000000..c917b015 --- /dev/null +++ b/crates/fuel-message-broker/src/nats_metrics.rs @@ -0,0 +1,51 @@ +use async_nats::jetstream::stream::State; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct StreamInfo { + pub state: StreamState, + pub stream_name: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +pub struct StreamState { + /// The number of messages contained in this stream + pub messages: u64, + /// The number of bytes of all messages contained in this stream + pub bytes: u64, + /// The lowest sequence number still present in this stream + #[serde(rename = "first_seq")] + pub first_sequence: u64, + /// The time associated with the oldest message still present in this stream + #[serde(rename = "first_ts")] + pub first_timestamp: i64, + /// The last sequence number assigned to a message in this stream + #[serde(rename = "last_seq")] + pub last_sequence: u64, + /// The time that the last message was received by this stream + #[serde(rename = "last_ts")] + pub last_timestamp: i64, + /// The number of consumers configured to consume this stream + pub consumer_count: usize, +} + +impl From for StreamState { + fn from(state: State) -> Self { + StreamState { + messages: state.messages, + bytes: state.bytes, + first_sequence: state.first_sequence, + first_timestamp: state.first_timestamp.unix_timestamp(), + last_sequence: state.last_sequence, + last_timestamp: state.last_timestamp.unix_timestamp(), + consumer_count: state.consumer_count, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NatsHealthInfo { + pub uptime_secs: u64, + pub is_healthy: bool, + pub streams_info: Vec, +} diff --git a/crates/fuel-message-broker/src/nats_opts.rs b/crates/fuel-message-broker/src/nats_opts.rs new file mode 100644 index 00000000..83725746 --- /dev/null +++ b/crates/fuel-message-broker/src/nats_opts.rs @@ -0,0 +1,88 @@ +use std::time::Duration; + +use async_nats::ConnectOptions; + +use crate::Namespace; + +#[derive(Debug, Clone)] +pub struct NatsOpts { + pub(crate) url: String, + pub(crate) namespace: Namespace, + pub(crate) timeout_secs: u64, + pub(crate) ack_wait_secs: u64, +} + +impl NatsOpts { + pub fn new(url: String) -> Self { + Self { + url, + timeout_secs: 5, + ack_wait_secs: 10, + namespace: Namespace::None, + } + } + + pub fn from_env() -> Self { + let url = dotenvy::var("NATS_URL").expect("NATS_URL must be set"); + Self::new(url) + } + + pub fn url(&self) -> String { + self.url.clone() + } + + pub fn with_url>(self, url: S) -> Self { + Self { + url: url.into(), + ..self + } + } + + pub fn with_ack_wait(self, secs: u64) -> Self { + Self { + ack_wait_secs: secs, + ..self + } + } + + #[cfg(any(test, feature = "test-helpers"))] + pub fn with_rdn_namespace(self) -> Self { + let namespace = format!(r"namespace-{}", Self::random_int()); + self.with_namespace(&namespace) + } + + #[cfg(any(test, feature = "test-helpers"))] + pub fn with_namespace(self, namespace: &str) -> Self { + use crate::Namespace; + let namespace = Namespace::Custom(namespace.to_string()); + Self { namespace, ..self } + } + + pub fn with_timeout(self, secs: u64) -> Self { + Self { + timeout_secs: secs, + ..self + } + } + + pub(super) fn connect_opts(&self) -> ConnectOptions { + let user = dotenvy::var("NATS_ADMIN_USER") + .expect("NATS_ADMIN_USER must be set"); + let pass = dotenvy::var("NATS_ADMIN_PASS") + .expect("NATS_ADMIN_PASS must be set"); + let opts = ConnectOptions::with_user_and_password(user, pass); + opts.connection_timeout(Duration::from_secs(self.timeout_secs)) + .max_reconnects(1) + .name(Self::conn_id()) + } + + // This will be useful for debugging and monitoring connections + fn conn_id() -> String { + format!(r"connection-{}", Self::random_int()) + } + + fn random_int() -> u32 { + use rand::Rng; + rand::thread_rng().gen_range(0..1000000) + } +} diff --git a/crates/fuel-streams-core/Cargo.toml b/crates/fuel-streams-core/Cargo.toml index 62289927..a54dd380 100644 --- a/crates/fuel-streams-core/Cargo.toml +++ b/crates/fuel-streams-core/Cargo.toml @@ -13,8 +13,8 @@ rust-version = { workspace = true } [dependencies] anyhow = { workspace = true } async-nats = { workspace = true } +async-stream = { workspace = true } async-trait = { workspace = true } -displaydoc = { workspace = true } dotenvy = { workspace = true } fuel-core = { workspace = true, default-features = false, features = [ "p2p", @@ -27,18 +27,16 @@ fuel-core-bin = { workspace = true, default-features = false, features = [ "relayer", "rocksdb", ] } -fuel-core-client = { workspace = true, default-features = false, features = ["std"] } fuel-core-importer = { workspace = true } fuel-core-services = { workspace = true, default-features = false, features = ["test-helpers"] } fuel-core-storage = { workspace = true } fuel-core-types = { workspace = true, default-features = false, features = ["std", "serde"] } -fuel-data-parser = { workspace = true } +fuel-message-broker = { workspace = true } +fuel-streams-domains = { workspace = true } fuel-streams-macros = { workspace = true } -fuel-streams-nats = { workspace = true, features = ["test-helpers"] } -fuel-streams-storage = { workspace = true, features = ["test-helpers"] } +fuel-streams-store = { workspace = true } +fuel-streams-types = { workspace = true } futures = { workspace = true } -hex = { workspace = true } -pretty_assertions = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } @@ -51,5 +49,4 @@ serde_json = { workspace = true } [features] default = [] -test-helpers = ["dep:pretty_assertions"] -bench-helpers = ["dep:pretty_assertions"] +test-helpers = [] diff --git a/crates/fuel-streams-core/README.md b/crates/fuel-streams-core/README.md index 5bdf9516..5742a2a8 100644 --- a/crates/fuel-streams-core/README.md +++ b/crates/fuel-streams-core/README.md @@ -54,28 +54,28 @@ fuel-streams-core = "*" Here's a simple example to get you started with Fuel Streams Core: ```rust,no_run -use std::sync::Arc; use fuel_streams_core::prelude::*; +use fuel_streams_store::db::*; +use fuel_message_broker::*; use futures::StreamExt; #[tokio::main] -async fn main() -> BoxedResult<()> { +async fn main() -> anyhow::Result<()> { // Connect to NATS server - let nats_opts = NatsClientOpts::admin_opts(); - let nats_client = NatsClient::connect(&nats_opts).await?; - let storage_opts = S3StorageOpts::new(StorageEnv::Local, StorageRole::Admin); - let storage = Arc::new(S3Storage::new(storage_opts).await?); + let db = Db::new(DbConnectionOpts::default()).await?; + let broker = MessageBrokerClient::Nats.start("nats://localhost:4222").await?; + broker.setup().await?; - // Create a stream for blocks - let stream = Stream::::new(&nats_client, &storage).await; + // Create or get existing stream for blocks + let stream = Stream::::get_or_init(&broker, &db.arc()).await; // Subscribe to the stream - let wildcard = BlocksSubject::wildcard(None, None); // blocks.*.* - let mut subscription = stream.subscribe(None).await?; + let subject = BlocksSubject::new(); // blocks.*.* + let mut subscription = stream.subscribe(subject, DeliverPolicy::New).await; // Process incoming blocks while let Some(block) = subscription.next().await { - dbg!(block); + dbg!(block?); } Ok(()) diff --git a/crates/fuel-streams-core/src/blocks/mod.rs b/crates/fuel-streams-core/src/blocks/mod.rs deleted file mode 100644 index 6d8b5eef..00000000 --- a/crates/fuel-streams-core/src/blocks/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -pub mod subjects; -pub mod types; - -pub use subjects::*; - -use super::types::*; -use crate::{DataEncoder, StreamError, Streamable}; - -impl DataEncoder for Block { - type Err = StreamError; -} -impl Streamable for Block { - const NAME: &'static str = "blocks"; - const WILDCARD_LIST: &'static [&'static str] = &[BlocksSubject::WILDCARD]; -} - -#[cfg(test)] -mod tests { - use serde_json::{self, json}; - - use super::*; - - #[tokio::test] - async fn test_block_encode() { - let block = MockBlock::build(42); - let encoded = block.encode().await.unwrap(); - let decoded = Block::decode(&encoded).await.unwrap(); - assert_eq!(decoded, block, "Decoded block should match original"); - } - - #[tokio::test] - async fn test_serialization() { - let header = BlockHeader { - application_hash: [0u8; 32].into(), - consensus_parameters_version: 1, - da_height: 1000, - event_inbox_root: [1u8; 32].into(), - id: Default::default(), - height: 42, - message_outbox_root: [3u8; 32].into(), - message_receipt_count: 10, - prev_root: [4u8; 32].into(), - state_transition_bytecode_version: 2, - time: FuelCoreTai64(1697398400).into(), - transactions_count: 5, - transactions_root: [5u8; 32].into(), - version: BlockHeaderVersion::V1, - }; - - let block = Block { - consensus: Consensus::default(), - header: header.clone(), - height: 42, - id: Default::default(), - transaction_ids: vec![], - version: BlockVersion::V1, - }; - - let serialized_block = - serde_json::to_value(&block).expect("Failed to serialize Block"); - - let expected_json = json!({ - "consensus": { - "chainConfigHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "coinsRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", - "contractsRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", - "messagesRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionsRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", - "type": "Genesis" - }, - "header": { - "applicationHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "consensusParametersVersion": 1, - "daHeight": 1000, - "eventInboxRoot": "0x0101010101010101010101010101010101010101010101010101010101010101", - "id": "0x0000000000000000000000000000000000000000000000000000000000000000", - "height": 42, - "messageOutboxRoot": "0x0303030303030303030303030303030303030303030303030303030303030303", - "messageReceiptCount": 10, - "prevRoot": "0x0404040404040404040404040404040404040404040404040404040404040404", - "stateTransitionBytecodeVersion": 2, - "time": "1697398400", - "transactionsCount": 5, - "transactionsRoot": "0x0505050505050505050505050505050505050505050505050505050505050505", - "version": "V1" - }, - "height": 42, - "id": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIds": [], - "version": "V1" - }); - - assert_eq!(serialized_block, expected_json); - } -} diff --git a/crates/fuel-streams-core/src/blocks/subjects.rs b/crates/fuel-streams-core/src/blocks/subjects.rs deleted file mode 100644 index c9038c09..00000000 --- a/crates/fuel-streams-core/src/blocks/subjects.rs +++ /dev/null @@ -1,63 +0,0 @@ -use crate::prelude::*; - -/// Represents a NATS subject for blocks in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of blocks -/// based on their producer and height. -/// -/// # Examples -/// -/// Creating a subject for a specific block: -/// -/// ``` -/// # use fuel_streams_core::blocks::BlocksSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::IntoSubject; -/// let subject = BlocksSubject { -/// producer: Some(Address::zeroed()), -/// height: Some(23.into()), -/// }; -/// assert_eq!(subject.parse(), "blocks.0x0000000000000000000000000000000000000000000000000000000000000000.23"); -/// ``` -/// -/// All blocks wildcard: -/// -/// ``` -/// # use fuel_streams_core::blocks::BlocksSubject; -/// assert_eq!(BlocksSubject::WILDCARD, "blocks.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method for flexible parameter-based filtering -/// -/// ``` -/// # use fuel_streams_core::blocks::BlocksSubject; -/// # use fuel_streams_core::prelude::*; -/// let wildcard = BlocksSubject::wildcard(None, Some(23.into())); -/// assert_eq!(wildcard, "blocks.*.23"); -/// ``` -/// -/// Using the builder pattern for flexible subject construction: -/// This approach allows for step-by-step creation of a `BlocksSubject`, -/// -/// ``` -/// # use fuel_streams_core::blocks::BlocksSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = BlocksSubject::new() -/// .with_producer(Some(Address::zeroed())) -/// .with_height(Some(23.into())); -/// assert_eq!(subject.parse(), "blocks.0x0000000000000000000000000000000000000000000000000000000000000000.23"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "blocks.>"] -#[subject_format = "blocks.{producer}.{height}"] -pub struct BlocksSubject { - pub producer: Option
, - pub height: Option, -} - -impl From<&Block> for BlocksSubject { - fn from(block: &Block) -> Self { - BlocksSubject::new().with_height(Some(block.height.into())) - } -} diff --git a/crates/fuel-streams-core/src/fuel_core_like.rs b/crates/fuel-streams-core/src/fuel_core_like.rs index 85d2458f..544f9b8b 100644 --- a/crates/fuel-streams-core/src/fuel_core_like.rs +++ b/crates/fuel-streams-core/src/fuel_core_like.rs @@ -12,10 +12,9 @@ use fuel_core_types::{ blockchain::consensus::{Consensus, Sealed}, fuel_types::BlockHeight, }; +use fuel_streams_types::fuel_core::*; use tokio::{sync::broadcast::Receiver, time::sleep}; -use crate::types::*; - /// Interface for `fuel-core` related logic. /// This was introduced to simplify mocking and testing the `sv-publisher` crate. #[async_trait::async_trait] @@ -36,7 +35,9 @@ pub trait FuelCoreLike: Sync + Send { fn onchain_database(&self) -> &Database { self.database().on_chain() } - fn offchain_database(&self) -> anyhow::Result> { + fn offchain_database( + &self, + ) -> anyhow::Result> { Ok(Arc::new(self.database().off_chain().latest_view()?)) } @@ -94,7 +95,7 @@ pub trait FuelCoreLike: Sync + Send { fn get_block_and_producer( &self, sealed_block: Sealed, - ) -> (FuelCoreBlock, Address) { + ) -> (FuelCoreBlock, fuel_streams_types::primitives::Address) { let block = sealed_block.entity.clone(); let block_producer = sealed_block .consensus @@ -108,7 +109,7 @@ pub trait FuelCoreLike: Sync + Send { fn get_block_and_producer( &self, sealed_block: Sealed, - ) -> (FuelCoreBlock, Address) { + ) -> (FuelCoreBlock, fuel_streams_types::Address) { let block = sealed_block.entity.clone(); let block_producer = sealed_block .consensus diff --git a/crates/fuel-streams-core/src/inputs/mod.rs b/crates/fuel-streams-core/src/inputs/mod.rs deleted file mode 100644 index 87e7e369..00000000 --- a/crates/fuel-streams-core/src/inputs/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -pub mod subjects; -pub mod types; - -pub use subjects::*; - -use super::types::*; -use crate::{DataEncoder, StreamError, Streamable}; - -impl DataEncoder for Input { - type Err = StreamError; -} -impl Streamable for Input { - const NAME: &'static str = "inputs"; - const WILDCARD_LIST: &'static [&'static str] = &[ - InputsCoinSubject::WILDCARD, - InputsContractSubject::WILDCARD, - InputsMessageSubject::WILDCARD, - InputsByIdSubject::WILDCARD, - ]; -} diff --git a/crates/fuel-streams-core/src/inputs/subjects.rs b/crates/fuel-streams-core/src/inputs/subjects.rs deleted file mode 100644 index 69e99780..00000000 --- a/crates/fuel-streams-core/src/inputs/subjects.rs +++ /dev/null @@ -1,263 +0,0 @@ -use crate::prelude::*; - -/// Represents a subject for querying inputs by their identifier in the Fuel ecosystem. -/// -/// This struct is used to create and parse subjects related to inputs identified by -/// various types of IDs, which can be used for subscribing to or publishing events -/// about specific inputs. -/// -/// # Examples -/// -/// Creating and parsing a subject: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = InputsByIdSubject { -/// tx_id: Some([1u8; 32].into()), -/// index: Some(0), -/// id_kind: Some(IdentifierKind::AssetID), -/// id_value: Some([3u8; 32].into()), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "by_id.inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.asset_id.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// All inputs by ID wildcard: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsByIdSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(InputsByIdSubject::WILDCARD, "by_id.inputs.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = InputsByIdSubject::wildcard(Some([1u8; 32].into()), Some(0), Some(IdentifierKind::AssetID), None); -/// assert_eq!(wildcard, "by_id.inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.asset_id.*"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = InputsByIdSubject::new() -/// .with_tx_id(Some([1u8; 32].into())) -/// .with_index(Some(0)) -/// .with_id_kind(Some(IdentifierKind::AssetID)) -/// .with_id_value(Some([3u8; 32].into())); -/// assert_eq!(subject.parse(), "by_id.inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.asset_id.0x0303030303030303030303030303030303030303030303030303030303030303"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "by_id.inputs.>"] -#[subject_format = "by_id.inputs.{tx_id}.{index}.{id_kind}.{id_value}"] -pub struct InputsByIdSubject { - pub tx_id: Option, - pub index: Option, - pub id_kind: Option, - pub id_value: Option, -} - -/// Represents a subject for input coins in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of input coins -/// based on their transaction ID, index, owner, and asset ID. -/// -/// # Examples -/// -/// Creating a subject for a specific input coin: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsCoinSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = InputsCoinSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// owner: Some(Address::from([2u8; 32])), -/// asset_id: Some(AssetId::from([3u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.coin.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// All input coins wildcard: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsCoinSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(InputsCoinSubject::WILDCARD, "inputs.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsCoinSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = InputsCoinSubject::wildcard(None, Some(0), None, Some(AssetId::from([3u8; 32]))); -/// assert_eq!(wildcard, "inputs.*.0.coin.*.0x0303030303030303030303030303030303030303030303030303030303030303"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsCoinSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = InputsCoinSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_owner(Some(Address::from([2u8; 32]))) -/// .with_asset_id(Some(AssetId::from([3u8; 32]))); -/// assert_eq!(subject.parse(), "inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.coin.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "inputs.>"] -#[subject_format = "inputs.{tx_id}.{index}.coin.{owner}.{asset_id}"] -pub struct InputsCoinSubject { - pub tx_id: Option, - pub index: Option, - pub owner: Option
, - pub asset_id: Option, -} - -/// Represents a subject for input contracts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of input contracts -/// based on their transaction ID, index, and contract ID. -/// -/// # Examples -/// -/// Creating a subject for a specific input contract: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsContractSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = InputsContractSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// contract_id: Some(ContractId::from([4u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.contract.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -/// -/// All input contracts wildcard: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsContractSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(InputsContractSubject::WILDCARD, "inputs.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsContractSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = InputsContractSubject::wildcard(Some(Bytes32::from([1u8; 32])), None, None); -/// assert_eq!(wildcard, "inputs.0x0101010101010101010101010101010101010101010101010101010101010101.*.contract.*"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsContractSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = InputsContractSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_contract_id(Some(ContractId::from([4u8; 32]))); -/// assert_eq!(subject.parse(), "inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.contract.0x0404040404040404040404040404040404040404040404040404040404040404"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "inputs.>"] -#[subject_format = "inputs.{tx_id}.{index}.contract.{contract_id}"] -pub struct InputsContractSubject { - pub tx_id: Option, - pub index: Option, - pub contract_id: Option, -} - -/// Represents a subject for input messages in the Fuel ecosystem. -/// -/// This struct is used to create and parse subjects related to input messages, -/// which can be used for subscribing to or publishing events about input messages. -/// -/// # Examples -/// -/// Creating and parsing a subject: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsMessageSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = InputsMessageSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// sender: Some(Address::from([2u8; 32])), -/// recipient: Some(Address::from([3u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.message.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// All input messages wildcard: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsMessageSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(InputsMessageSubject::WILDCARD, "inputs.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsMessageSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = InputsMessageSubject::wildcard(Some(Bytes32::from([1u8; 32])), None, None, None); -/// assert_eq!(wildcard, "inputs.0x0101010101010101010101010101010101010101010101010101010101010101.*.message.*.*"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::inputs::subjects::InputsMessageSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = InputsMessageSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_sender(Some(Address::from([2u8; 32]))) -/// .with_recipient(Some(Address::from([3u8; 32]))); -/// assert_eq!(subject.parse(), "inputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.message.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "inputs.>"] -#[subject_format = "inputs.{tx_id}.{index}.message.{sender}.{recipient}"] -pub struct InputsMessageSubject { - pub tx_id: Option, - pub index: Option, - pub sender: Option
, - pub recipient: Option
, -} diff --git a/crates/fuel-streams-core/src/lib.rs b/crates/fuel-streams-core/src/lib.rs index c680eacc..21eeafdd 100644 --- a/crates/fuel-streams-core/src/lib.rs +++ b/crates/fuel-streams-core/src/lib.rs @@ -1,47 +1,52 @@ #![doc = include_str!("../README.md")] -pub mod blocks; -pub mod inputs; -pub mod logs; -pub mod outputs; -pub mod receipts; -pub mod transactions; -pub mod utxos; - -pub mod nats { - pub use fuel_streams_nats::*; -} - -pub mod storage { - pub use fuel_streams_storage::*; -} - -pub(crate) mod data_parser { - pub use fuel_data_parser::*; -} - -pub mod stream; -pub mod subjects; - pub mod fuel_core_like; -mod fuel_core_types; -mod primitive_types; -pub mod types; +pub mod stream; -pub(crate) use data_parser::*; +pub use fuel_core_like::*; pub use stream::*; pub mod prelude { - #[allow(unused_imports)] pub use fuel_streams_macros::subject::*; - pub use crate::{ - data_parser::*, - fuel_core_like::*, - nats::*, - storage::*, - stream::*, - subjects::*, - types::*, + pub use crate::{fuel_core_like::*, stream::*, subjects::*, types::*}; +} + +pub mod types { + pub use fuel_streams_domains::{ + blocks::types::*, + inputs::types::*, + outputs::types::*, + receipts::types::*, + transactions::types::*, + utxos::types::*, }; + pub use fuel_streams_types::*; +} + +pub mod subjects { + pub use fuel_streams_domains::{ + blocks::subjects::*, + inputs::subjects::*, + outputs::subjects::*, + receipts::subjects::*, + transactions::subjects::*, + utxos::subjects::*, + }; + pub use fuel_streams_macros::subject::*; } + +macro_rules! export_module { + ($module:ident) => { + pub mod $module { + pub use fuel_streams_domains::$module::subjects::*; + } + }; +} + +export_module!(blocks); +export_module!(inputs); +export_module!(outputs); +export_module!(receipts); +export_module!(transactions); +export_module!(utxos); diff --git a/crates/fuel-streams-core/src/logs/mod.rs b/crates/fuel-streams-core/src/logs/mod.rs deleted file mode 100644 index 49459fcf..00000000 --- a/crates/fuel-streams-core/src/logs/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -pub mod subjects; -pub mod types; - -pub use subjects::*; - -use super::types::*; -use crate::{DataEncoder, StreamError, Streamable}; - -impl DataEncoder for Log { - type Err = StreamError; -} -impl Streamable for Log { - const NAME: &'static str = "logs"; - const WILDCARD_LIST: &'static [&'static str] = &[LogsSubject::WILDCARD]; -} diff --git a/crates/fuel-streams-core/src/logs/subjects.rs b/crates/fuel-streams-core/src/logs/subjects.rs deleted file mode 100644 index 9b0fa56e..00000000 --- a/crates/fuel-streams-core/src/logs/subjects.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::prelude::*; - -/// Represents a subject for logs related to transactions in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of logs -/// based on the block height, transaction ID, the index of the receipt within the transaction, -/// and the unique log ID. -/// -/// # Examples -/// -/// Creating a subject for a specific log: -/// -/// ``` -/// # use fuel_streams_core::logs::subjects::LogsSubject; -/// # use fuel_streams_core::types::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = LogsSubject { -/// block_height: Some(1000.into()), -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// receipt_index: Some(0), -/// log_id: Some(Bytes32::from([2u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "logs.1000.0x0101010101010101010101010101010101010101010101010101010101010101.0.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// Wildcard for querying all logs: -/// -/// ``` -/// # use fuel_streams_core::logs::subjects::LogsSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(LogsSubject::WILDCARD, "logs.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::logs::subjects::LogsSubject; -/// # use fuel_streams_core::types::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = LogsSubject::wildcard( -/// Some(1000.into()), -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "logs.1000.0x0101010101010101010101010101010101010101010101010101010101010101.*.*" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::logs::subjects::LogsSubject; -/// # use fuel_streams_core::types::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = LogsSubject::new() -/// .with_block_height(Some(2310.into())) -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_receipt_index(Some(0)) -/// .with_log_id(Some(Bytes32::from([2u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "logs.2310.0x0101010101010101010101010101010101010101010101010101010101010101.0.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "logs.>"] -#[subject_format = "logs.{block_height}.{tx_id}.{receipt_index}.{log_id}"] -pub struct LogsSubject { - pub block_height: Option, - pub tx_id: Option, - pub receipt_index: Option, - pub log_id: Option, -} diff --git a/crates/fuel-streams-core/src/logs/types.rs b/crates/fuel-streams-core/src/logs/types.rs deleted file mode 100644 index 927216d4..00000000 --- a/crates/fuel-streams-core/src/logs/types.rs +++ /dev/null @@ -1,56 +0,0 @@ -use crate::types::*; - -/// A convenient aggregate type to represent a Fuel logs to allow users -/// think about them agnostic of receipts. -#[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum Log { - WithoutData { - id: ContractId, - ra: FuelCoreWord, - rb: FuelCoreWord, - rc: FuelCoreWord, - rd: FuelCoreWord, - pc: FuelCoreWord, - is: FuelCoreWord, - }, - WithData { - id: ContractId, - ra: FuelCoreWord, - rb: FuelCoreWord, - ptr: FuelCoreWord, - len: FuelCoreWord, - digest: Bytes32, - pc: FuelCoreWord, - is: FuelCoreWord, - data: Option>, - }, -} - -impl From for Log { - fn from(value: Receipt) -> Self { - match value { - Receipt::Log(log) => Log::WithoutData { - id: log.id, - ra: log.ra, - rb: log.rb, - rc: log.rc, - rd: log.rd, - pc: log.pc, - is: log.is, - }, - Receipt::LogData(log) => Log::WithData { - id: log.id, - ra: log.ra, - rb: log.rb, - ptr: log.ptr, - len: log.len, - digest: log.digest, - pc: log.pc, - is: log.is, - data: log.data, - }, - _ => panic!("Invalid receipt type"), - } - } -} diff --git a/crates/fuel-streams-core/src/outputs/mod.rs b/crates/fuel-streams-core/src/outputs/mod.rs deleted file mode 100644 index 160f656a..00000000 --- a/crates/fuel-streams-core/src/outputs/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub mod subjects; -pub mod types; - -pub use subjects::*; - -use super::types::*; -use crate::{DataEncoder, StreamError, Streamable}; - -impl DataEncoder for Output { - type Err = StreamError; -} -impl Streamable for Output { - const NAME: &'static str = "outputs"; - const WILDCARD_LIST: &'static [&'static str] = &[ - OutputsByIdSubject::WILDCARD, - OutputsCoinSubject::WILDCARD, - OutputsContractSubject::WILDCARD, - OutputsChangeSubject::WILDCARD, - OutputsVariableSubject::WILDCARD, - OutputsContractCreatedSubject::WILDCARD, - ]; -} diff --git a/crates/fuel-streams-core/src/outputs/subjects.rs b/crates/fuel-streams-core/src/outputs/subjects.rs deleted file mode 100644 index 26cb6ec4..00000000 --- a/crates/fuel-streams-core/src/outputs/subjects.rs +++ /dev/null @@ -1,303 +0,0 @@ -use crate::prelude::*; - -/// Represents a subject for querying outputs by their identifier in the Fuel ecosystem. -/// -/// This struct is used to create and parse subjects related to outputs identified by -/// various types of IDs, which can be used for subscribing to or publishing events -/// about specific outputs. -/// -/// # Examples -/// -/// Creating and parsing a subject: -/// -/// ``` -/// # use fuel_streams_core::outputs::subjects::OutputsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = OutputsByIdSubject { -/// tx_id: Some([1u8; 32].into()), -/// index: Some(0), -/// id_kind: Some(IdentifierKind::AssetID), -/// id_value: Some([3u8; 32].into()), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "by_id.outputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.asset_id.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// All outputs by ID wildcard: -/// -/// ``` -/// # use fuel_streams_core::outputs::subjects::OutputsByIdSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(OutputsByIdSubject::WILDCARD, "by_id.outputs.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::outputs::subjects::OutputsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = OutputsByIdSubject::wildcard(Some([1u8; 32].into()), Some(0), Some(IdentifierKind::AssetID), None); -/// assert_eq!(wildcard, "by_id.outputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.asset_id.*"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::outputs::subjects::OutputsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = OutputsByIdSubject::new() -/// .with_tx_id(Some([1u8; 32].into())) -/// .with_index(Some(0)) -/// .with_id_kind(Some(IdentifierKind::AssetID)) -/// .with_id_value(Some([3u8; 32].into())); -/// assert_eq!(subject.parse(), "by_id.outputs.0x0101010101010101010101010101010101010101010101010101010101010101.0.asset_id.0x0303030303030303030303030303030303030303030303030303030303030303"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "by_id.outputs.>"] -#[subject_format = "by_id.outputs.{tx_id}.{index}.{id_kind}.{id_value}"] -pub struct OutputsByIdSubject { - pub tx_id: Option, - pub index: Option, - pub id_kind: Option, - pub id_value: Option, -} - -/// Represents the NATS subject for coin outputs. -/// -/// This subject format allows for querying coin outputs based on transaction ID, -/// index, recipient address (`to`), and asset ID. -/// -/// # Examples -/// -/// **Creating a subject for a specific coin output:** -/// -/// ``` -/// use fuel_streams_core::outputs::subjects::OutputsCoinSubject; -/// use fuel_streams_core::prelude::*; -/// use fuel_streams_macros::subject::SubjectBuildable; -/// -/// let subject = OutputsCoinSubject::new() -/// .with_tx_id(Some(Bytes32::zeroed())) -/// .with_index(Some(0)) -/// .with_to(Some(Address::zeroed())) -/// .with_asset_id(Some(AssetId::zeroed())); -/// assert_eq!( -/// subject.to_string(), -/// "outputs.coin.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000.0x0000000000000000000000000000000000000000000000000000000000000000" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "outputs.>"] -#[subject_format = "outputs.coin.{tx_id}.{index}.{to}.{asset_id}"] -pub struct OutputsCoinSubject { - pub tx_id: Option, - pub index: Option, - pub to: Option
, - pub asset_id: Option, -} - -/// Represents the NATS subject for contract outputs. -/// -/// This subject format allows for querying contract outputs based on -/// transaction ID, index, and contract ID. -/// -/// # Examples -/// -/// **Creating a subject for a specific contract output:** -/// -/// ``` -/// use fuel_streams_core::outputs::subjects::OutputsContractSubject; -/// use fuel_streams_core::prelude::*; -/// use fuel_streams_macros::subject::SubjectBuildable; -/// -/// let subject = OutputsContractSubject::new() -/// .with_tx_id(Some(Bytes32::zeroed())) -/// .with_index(Some(0)) -/// .with_contract_id(Some(ContractId::zeroed())); -/// assert_eq!( -/// subject.to_string(), -/// "outputs.contract.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "outputs.>"] -#[subject_format = "outputs.contract.{tx_id}.{index}.{contract_id}"] -pub struct OutputsContractSubject { - pub tx_id: Option, - pub index: Option, - pub contract_id: Option, -} - -/// Represents the NATS subject for change outputs. -/// -/// This subject format allows for querying change outputs based on transaction ID, -/// index, recipient address (`to`), and asset ID. -/// -/// # Examples -/// -/// **Creating a subject for a specific change output:** -/// -/// ``` -/// use fuel_streams_core::outputs::subjects::OutputsChangeSubject; -/// use fuel_streams_core::prelude::*; -/// use fuel_streams_macros::subject::SubjectBuildable; -/// -/// let subject = OutputsChangeSubject::new() -/// .with_tx_id(Some(Bytes32::zeroed())) -/// .with_index(Some(0)) -/// .with_to(Some(Address::zeroed())) -/// .with_asset_id(Some(AssetId::zeroed())); -/// assert_eq!( -/// subject.to_string(), -/// "outputs.change.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000.0x0000000000000000000000000000000000000000000000000000000000000000" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "outputs.>"] -#[subject_format = "outputs.change.{tx_id}.{index}.{to}.{asset_id}"] -pub struct OutputsChangeSubject { - pub tx_id: Option, - pub index: Option, - pub to: Option
, - pub asset_id: Option, -} - -/// Represents the NATS subject for variable outputs. -/// -/// This subject format allows for querying variable outputs based on transaction -/// ID, index, recipient address (`to`), and asset ID. -/// -/// # Examples -/// -/// **Creating a subject for a specific variable output:** -/// -/// ``` -/// use fuel_streams_core::outputs::subjects::OutputsVariableSubject; -/// use fuel_streams_core::prelude::*; -/// use fuel_streams_macros::subject::SubjectBuildable; -/// -/// let subject = OutputsVariableSubject::new() -/// .with_tx_id(Some(Bytes32::zeroed())) -/// .with_index(Some(0)) -/// .with_to(Some(Address::zeroed())) -/// .with_asset_id(Some(AssetId::from([1u8; 32]))); -/// assert_eq!( -/// subject.to_string(), -/// "outputs.variable.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000.0x0101010101010101010101010101010101010101010101010101010101010101" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "outputs.>"] -#[subject_format = "outputs.variable.{tx_id}.{index}.{to}.{asset_id}"] -pub struct OutputsVariableSubject { - pub tx_id: Option, - pub index: Option, - pub to: Option
, - pub asset_id: Option, -} - -/// Represents the NATS subject for contract created outputs. -/// -/// This subject format allows for querying contract creation outputs based on -/// transaction ID, index, and contract ID. -/// -/// # Examples -/// -/// **Creating a subject for a specific contract creation output:** -/// -/// ``` -/// use fuel_streams_core::outputs::subjects::OutputsContractCreatedSubject; -/// use fuel_streams_core::prelude::*; -/// use fuel_streams_macros::subject::SubjectBuildable; -/// -/// let subject = OutputsContractCreatedSubject::new() -/// .with_tx_id(Some(Bytes32::zeroed())) -/// .with_index(Some(0)) -/// .with_contract_id(Some(ContractId::zeroed())); -/// assert_eq!( -/// subject.to_string(), -/// "outputs.contract_created.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "outputs.>"] -#[subject_format = "outputs.contract_created.{tx_id}.{index}.{contract_id}"] -pub struct OutputsContractCreatedSubject { - pub tx_id: Option, - pub index: Option, - pub contract_id: Option, -} - -#[cfg(test)] -mod tests { - use fuel_core_types::fuel_types::{Address, Bytes32}; - use fuel_streams_macros::subject::SubjectBuildable; - - use super::*; - - #[test] - fn test_output_subject_wildcard() { - assert_eq!(OutputsByIdSubject::WILDCARD, "by_id.outputs.>"); - assert_eq!(OutputsCoinSubject::WILDCARD, "outputs.>"); - assert_eq!(OutputsContractSubject::WILDCARD, "outputs.>"); - assert_eq!(OutputsChangeSubject::WILDCARD, "outputs.>"); - assert_eq!(OutputsVariableSubject::WILDCARD, "outputs.>"); - assert_eq!(OutputsContractCreatedSubject::WILDCARD, "outputs.>"); - } - - #[test] - fn test_outputs_coin_subject_creation() { - let coin_subject = OutputsCoinSubject::new() - .with_tx_id(Some(Bytes32::zeroed().into())) - .with_index(Some(0)) - .with_to(Some(Address::zeroed().into())) - .with_asset_id(Some(AssetId::zeroed())); - assert_eq!( - coin_subject.to_string(), - "outputs.coin.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000.0x0000000000000000000000000000000000000000000000000000000000000000" - ); - } - - #[test] - fn test_outputs_contract_created_subject_creation() { - let contract_created_subject = OutputsContractCreatedSubject::new() - .with_tx_id(Some(Bytes32::zeroed().into())) - .with_index(Some(0)) - .with_contract_id(Some(ContractId::zeroed())); - assert_eq!( - contract_created_subject.to_string(), - "outputs.contract_created.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000" - ); - } - - #[test] - fn test_output_subject_coin() { - let output_subject = OutputsCoinSubject::new() - .with_tx_id(Some(Bytes32::zeroed().into())) - .with_index(Some(0)) - .with_to(Some(Address::zeroed().into())) - .with_asset_id(Some(AssetId::zeroed())); - assert_eq!( - output_subject.to_string(), - "outputs.coin.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000.0x0000000000000000000000000000000000000000000000000000000000000000" - ); - } - - #[test] - fn test_output_subject_variable() { - let output_subject = OutputsVariableSubject::new() - .with_tx_id(Some(Bytes32::zeroed().into())) - .with_index(Some(0)) - .with_to(Some(Address::zeroed().into())) - .with_asset_id(Some(AssetId::zeroed())); - assert_eq!( - output_subject.to_string(), - "outputs.variable.0x0000000000000000000000000000000000000000000000000000000000000000.0.0x0000000000000000000000000000000000000000000000000000000000000000.0x0000000000000000000000000000000000000000000000000000000000000000" - ); - } -} diff --git a/crates/fuel-streams-core/src/primitive_types.rs b/crates/fuel-streams-core/src/primitive_types.rs deleted file mode 100644 index c03696e6..00000000 --- a/crates/fuel-streams-core/src/primitive_types.rs +++ /dev/null @@ -1,417 +0,0 @@ -use fuel_core_types::{ - fuel_asm::RawInstruction, - fuel_tx::PanicReason, - fuel_types, -}; -pub use serde::{Deserialize, Serialize}; - -use crate::fuel_core_types::*; - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Default, -)] -pub struct LongBytes(pub Vec); - -impl LongBytes { - pub fn zeroed() -> Self { - Self(vec![0; 32]) - } -} -impl AsRef<[u8]> for LongBytes { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} -impl AsMut<[u8]> for LongBytes { - fn as_mut(&mut self) -> &mut [u8] { - &mut self.0 - } -} -impl From> for LongBytes { - fn from(value: Vec) -> Self { - Self(value) - } -} -impl std::fmt::Display for LongBytes { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", hex::encode(&self.0)) - } -} -impl From<&[u8]> for LongBytes { - fn from(value: &[u8]) -> Self { - Self(value.to_vec()) - } -} - -macro_rules! common_wrapper_type { - ($wrapper_type:ident, $inner_type:ty) => { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct $wrapper_type(pub $inner_type); - - // Custom serialization - impl Serialize for $wrapper_type { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - if serializer.is_human_readable() { - serializer.serialize_str(&format!("0x{}", self.0)) - } else { - self.0.serialize(serializer) - } - } - } - - // Custom deserialization using FromStr - impl<'de> Deserialize<'de> for $wrapper_type { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - if deserializer.is_human_readable() { - let s = String::deserialize(deserializer)?; - s.parse().map_err(serde::de::Error::custom) - } else { - Ok($wrapper_type(<$inner_type>::deserialize(deserializer)?)) - } - } - } - - impl From<$inner_type> for $wrapper_type { - fn from(value: $inner_type) -> Self { - $wrapper_type(value) - } - } - - impl From<$wrapper_type> for $inner_type { - fn from(value: $wrapper_type) -> Self { - value.0 - } - } - - impl From<&$inner_type> for $wrapper_type { - fn from(value: &$inner_type) -> Self { - $wrapper_type(value.clone()) - } - } - - impl std::fmt::Display for $wrapper_type { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "0x{}", self.0) - } - } - - impl From<&str> for $wrapper_type { - fn from(s: &str) -> Self { - s.parse().unwrap_or_else(|e| { - panic!( - "Failed to parse {}: {}", - stringify!($wrapper_type), - e - ) - }) - } - } - - impl $wrapper_type { - pub fn zeroed() -> Self { - $wrapper_type(<$inner_type>::zeroed()) - } - - pub fn new(inner: $inner_type) -> Self { - $wrapper_type(inner) - } - } - - impl AsRef<$inner_type> for $wrapper_type { - fn as_ref(&self) -> &$inner_type { - &self.0 - } - } - - impl $wrapper_type { - pub fn into_inner(self) -> $inner_type { - self.0 - } - } - - impl Default for $wrapper_type { - fn default() -> Self { - $wrapper_type(<$inner_type>::zeroed()) - } - } - }; -} - -macro_rules! generate_byte_type_wrapper { - // Pattern with byte_size specified - ($wrapper_type:ident, $inner_type:ty, $byte_size:expr) => { - common_wrapper_type!($wrapper_type, $inner_type); - - impl From<[u8; $byte_size]> for $wrapper_type { - fn from(value: [u8; $byte_size]) -> Self { - $wrapper_type(<$inner_type>::from(value)) - } - } - - impl std::str::FromStr for $wrapper_type { - type Err = String; - - fn from_str(s: &str) -> Result { - let s = s.strip_prefix("0x").unwrap_or(s); - if s.len() != std::mem::size_of::<$inner_type>() * 2 { - return Err(format!( - "Invalid length for {}, expected {} characters", - stringify!($wrapper_type), - std::mem::size_of::<$inner_type>() * 2 - )); - } - let bytes = hex::decode(s).map_err(|e| { - format!("Failed to decode hex string: {}", e) - })?; - let array: [u8; $byte_size] = bytes - .try_into() - .map_err(|_| "Invalid byte length".to_string())?; - Ok($wrapper_type(<$inner_type>::from(array))) - } - } - }; - - ($wrapper_type:ident, $inner_type:ty) => { - common_wrapper_type!($wrapper_type, $inner_type); - - impl From> for $wrapper_type { - fn from(value: Vec) -> Self { - $wrapper_type(<$inner_type>::from(value)) - } - } - impl std::str::FromStr for $wrapper_type { - type Err = String; - fn from_str(s: &str) -> Result { - let s = s.strip_prefix("0x").unwrap_or(s); - let bytes = hex::decode(s).map_err(|e| { - format!("Failed to decode hex string: {}", e) - })?; - Ok($wrapper_type(bytes.into())) - } - } - }; -} - -generate_byte_type_wrapper!(Address, fuel_types::Address, 32); -generate_byte_type_wrapper!(Bytes32, fuel_types::Bytes32, 32); -generate_byte_type_wrapper!(ContractId, fuel_types::ContractId, 32); -generate_byte_type_wrapper!(AssetId, fuel_types::AssetId, 32); -generate_byte_type_wrapper!(BlobId, fuel_types::BlobId, 32); -generate_byte_type_wrapper!(Nonce, fuel_types::Nonce, 32); -generate_byte_type_wrapper!(Salt, fuel_types::Salt, 32); -generate_byte_type_wrapper!(MessageId, fuel_types::MessageId, 32); -generate_byte_type_wrapper!(BlockId, fuel_types::Bytes32, 32); -generate_byte_type_wrapper!(Signature, fuel_types::Bytes64, 64); -generate_byte_type_wrapper!(TxId, fuel_types::TxId, 32); -generate_byte_type_wrapper!(HexData, LongBytes); - -/// Implements bidirectional conversions between `Bytes32` and a given type. -/// -/// This macro generates implementations of the `From` trait to convert: -/// - From `Bytes32` to the target type -/// - From a reference to `Bytes32` to the target type -/// - From the target type to `Bytes32` -/// - From a reference of the target type to `Bytes32` -/// -/// The target type must be a 32-byte type that can be converted to/from `[u8; 32]`. -/// -/// # Example -/// ```ignore -/// impl_bytes32_conversions!(ContractId); -/// ``` -macro_rules! impl_bytes32_conversions { - ($type:ty) => { - impl From for $type { - fn from(value: Bytes32) -> Self { - let bytes: [u8; 32] = value.0.into(); - <$type>::from(bytes) - } - } - impl From<&Bytes32> for $type { - fn from(value: &Bytes32) -> Self { - value.clone().into() - } - } - impl From<$type> for Bytes32 { - fn from(value: $type) -> Self { - let bytes: [u8; 32] = value.0.into(); - Bytes32::from(bytes) - } - } - impl From<&$type> for Bytes32 { - fn from(value: &$type) -> Self { - value.clone().into() - } - } - }; -} - -impl_bytes32_conversions!(MessageId); -impl_bytes32_conversions!(ContractId); -impl_bytes32_conversions!(AssetId); -impl_bytes32_conversions!(Address); -impl_bytes32_conversions!(BlobId); -impl_bytes32_conversions!(Nonce); -impl_bytes32_conversions!(Salt); -impl_bytes32_conversions!(BlockId); -impl_bytes32_conversions!(TxId); - -impl From for BlockId { - fn from(value: FuelCoreBlockId) -> Self { - Self(FuelCoreBytes32::from(value)) - } -} - -#[derive( - Debug, - Default, - Copy, - Clone, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - Deserialize, - Serialize, -)] -pub struct TxPointer { - block_height: FuelCoreBlockHeight, - tx_index: u16, -} - -impl From for TxPointer { - fn from(value: FuelCoreTxPointer) -> Self { - Self { - block_height: value.block_height(), - tx_index: value.tx_index(), - } - } -} - -#[derive( - Debug, Default, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, -)] -pub struct UtxoId { - pub tx_id: Bytes32, - pub output_index: u16, -} -impl From<&UtxoId> for HexData { - fn from(value: &UtxoId) -> Self { - value.to_owned().into() - } -} -impl From for UtxoId { - fn from(value: FuelCoreUtxoId) -> Self { - Self::from(&value) - } -} -impl From<&FuelCoreUtxoId> for UtxoId { - fn from(value: &FuelCoreUtxoId) -> Self { - Self { - tx_id: value.tx_id().into(), - output_index: value.output_index(), - } - } -} -impl From for HexData { - fn from(value: UtxoId) -> Self { - let mut bytes = Vec::with_capacity(34); - bytes.extend_from_slice(value.tx_id.0.as_ref()); - bytes.extend_from_slice(&value.output_index.to_be_bytes()); - HexData(bytes.into()) - } -} - -#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] -pub struct PanicInstruction { - pub reason: PanicReason, - pub instruction: RawInstruction, -} -impl From for PanicInstruction { - fn from(value: FuelCorePanicInstruction) -> Self { - Self { - reason: value.reason().to_owned(), - instruction: value.instruction().to_owned(), - } - } -} - -#[derive( - Debug, - Copy, - Clone, - PartialEq, - Eq, - Hash, - Default, - serde::Serialize, - serde::Deserialize, -)] -#[repr(u64)] -pub enum ScriptExecutionResult { - Success, - Revert, - Panic, - // Generic failure case since any u64 is valid here - GenericFailure(u64), - #[default] - Unknown, -} -impl From for ScriptExecutionResult { - fn from(value: FuelCoreScriptExecutionResult) -> Self { - match value { - FuelCoreScriptExecutionResult::Success => Self::Success, - FuelCoreScriptExecutionResult::Revert => Self::Revert, - FuelCoreScriptExecutionResult::Panic => Self::Panic, - FuelCoreScriptExecutionResult::GenericFailure(value) => { - Self::GenericFailure(value) - } - } - } -} - -/// Macro to implement conversion from a type to `Bytes32`. -/// -/// This macro creates an implementation of the `From` trait, allowing for conversion -/// from the specified type into a `Bytes32` type. It is useful when working with -/// byte-based types, such as `ContractId`, in the Fuel ecosystem. -/// -/// The generated implementation allows conversion by dereferencing the input value -/// and creating a `Bytes32` type from it, making the conversion simple and efficient. -/// -/// # Usage -/// -/// impl_from_bytes32!(FromType); -/// -/// -/// Where `FromType` is the type that you want to be able to convert into a `Bytes32`. -// -/// # Notes -/// -/// The macro assumes that the type being converted can be dereferenced into a byte-based type -/// compatible with the `Bytes32` structure. -macro_rules! impl_from_bytes32 { - ($from_type:ty) => { - impl From<$from_type> for Bytes32 { - fn from(value: $from_type) -> Self { - Bytes32(fuel_core_types::fuel_tx::Bytes32::from(*value)) - } - } - impl From<&$from_type> for Bytes32 { - fn from(value: &$from_type) -> Self { - (*value).into() - } - } - }; -} - -impl_from_bytes32!(fuel_types::ContractId); -impl_from_bytes32!(fuel_types::AssetId); -impl_from_bytes32!(fuel_types::Address); diff --git a/crates/fuel-streams-core/src/receipts/mod.rs b/crates/fuel-streams-core/src/receipts/mod.rs deleted file mode 100644 index 6779f6b1..00000000 --- a/crates/fuel-streams-core/src/receipts/mod.rs +++ /dev/null @@ -1,30 +0,0 @@ -pub mod subjects; -pub mod types; - -pub use subjects::*; - -use super::types::*; -use crate::{DataEncoder, StreamError, Streamable}; - -impl DataEncoder for Receipt { - type Err = StreamError; -} -impl Streamable for Receipt { - const NAME: &'static str = "receipts"; - const WILDCARD_LIST: &'static [&'static str] = &[ - ReceiptsCallSubject::WILDCARD, - ReceiptsByIdSubject::WILDCARD, - ReceiptsBurnSubject::WILDCARD, - ReceiptsLogSubject::WILDCARD, - ReceiptsMintSubject::WILDCARD, - ReceiptsPanicSubject::WILDCARD, - ReceiptsReturnSubject::WILDCARD, - ReceiptsRevertSubject::WILDCARD, - ReceiptsLogDataSubject::WILDCARD, - ReceiptsTransferSubject::WILDCARD, - ReceiptsMessageOutSubject::WILDCARD, - ReceiptsReturnDataSubject::WILDCARD, - ReceiptsTransferOutSubject::WILDCARD, - ReceiptsScriptResultSubject::WILDCARD, - ]; -} diff --git a/crates/fuel-streams-core/src/receipts/subjects.rs b/crates/fuel-streams-core/src/receipts/subjects.rs deleted file mode 100644 index 6b6bd3e5..00000000 --- a/crates/fuel-streams-core/src/receipts/subjects.rs +++ /dev/null @@ -1,1051 +0,0 @@ -use crate::prelude::*; - -/// Represents a subject for querying receipts by their identifier in the Fuel ecosystem. -/// -/// This struct is used to create and parse subjects related to receipts identified by -/// various types of IDs, which can be used for subscribing to or publishing events -/// about specific receipts. -/// -/// # Examples -/// -/// Creating and parsing a subject: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsByIdSubject { -/// tx_id: Some([1u8; 32].into()), -/// index: Some(0), -/// id_kind: Some(IdentifierKind::ContractID), -/// id_value: Some([2u8; 32].into()), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "by_id.receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.contract_id.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// All receipts by ID wildcard: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsByIdSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsByIdSubject::WILDCARD, "by_id.receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsByIdSubject::wildcard(Some([1u8; 32].into()), Some(0), Some(IdentifierKind::ContractID), None); -/// assert_eq!(wildcard, "by_id.receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.contract_id.*"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsByIdSubject::new() -/// .with_tx_id(Some([1u8; 32].into())) -/// .with_index(Some(0)) -/// .with_id_kind(Some(IdentifierKind::ContractID)) -/// .with_id_value(Some([2u8; 32].into())); -/// assert_eq!(subject.parse(), "by_id.receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.contract_id.0x0202020202020202020202020202020202020202020202020202020202020202"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "by_id.receipts.>"] -#[subject_format = "by_id.receipts.{tx_id}.{index}.{id_kind}.{id_value}"] -pub struct ReceiptsByIdSubject { - pub tx_id: Option, - pub index: Option, - pub id_kind: Option, - pub id_value: Option, -} - -/// Represents a subject for receipts related to contract calls in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of receipts based -/// on the transaction ID, index, the contract initiating the call (`from`), the receiving contract (`to`), -/// and the asset ID involved in the transaction. -/// -/// # Examples -/// -/// Creating a subject for a specific call receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsCallSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsCallSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// from: Some(ContractId::from([2u8; 32])), -/// to: Some(ContractId::from([3u8; 32])), -/// asset_id: Some(AssetId::from([4u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.call.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -/// -/// Wildcard for querying all call receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsCallSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsCallSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsCallSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsCallSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// Some(0), -/// Some(ContractId::from([2u8; 32])), -/// Some(ContractId::from([3u8; 32])), -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.call.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.*" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsCallSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsCallSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_from(Some(ContractId::from([2u8; 32]))) -/// .with_to(Some(ContractId::from([3u8; 32]))) -/// .with_asset_id(Some(AssetId::from([4u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.call.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.call.{from}.{to}.{asset_id}"] -pub struct ReceiptsCallSubject { - pub tx_id: Option, - pub index: Option, - pub from: Option, - pub to: Option, - pub asset_id: Option, -} - -/// Represents a subject for receipts related to contract returns in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of return receipts -/// based on the transaction ID, index, and the contract ID (`id`) associated with the return. -/// -/// # Examples -/// -/// Creating a subject for a specific return receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsReturnSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsReturnSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// id: Some(ContractId::from([2u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.return.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// Wildcard for querying all return receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsReturnSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsReturnSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsReturnSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsReturnSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.return.*" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsReturnSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsReturnSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_id(Some(ContractId::from([2u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.return.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.return.{id}"] -pub struct ReceiptsReturnSubject { - pub tx_id: Option, - pub index: Option, - pub id: Option, -} - -// -/// This subject format allows for efficient querying and filtering of return data receipts -/// based on the transaction ID, index, and the contract ID (`id`) associated with the return data. -/// -/// # Examples -/// -/// Creating a subject for a specific return data receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsReturnDataSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsReturnDataSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// id: Some(ContractId::from([2u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.return_data.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// Wildcard for querying all return data receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsReturnDataSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsReturnDataSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsReturnDataSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsReturnDataSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.return_data.*" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsReturnDataSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsReturnDataSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_id(Some(ContractId::from([2u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.return_data.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.return_data.{id}"] -pub struct ReceiptsReturnDataSubject { - pub tx_id: Option, - pub index: Option, - pub id: Option, -} - -/// Represents a subject for receipts related to contract panics in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of panic receipts -/// based on the transaction ID, index, and the contract ID (`id`) associated with the panic event. -/// -/// # Examples -/// -/// Creating a subject for a specific panic receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsPanicSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsPanicSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// id: Some(ContractId::from([2u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.panic.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// Wildcard for querying all panic receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsPanicSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsPanicSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsPanicSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsPanicSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.panic.*" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsPanicSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsPanicSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_id(Some(ContractId::from([2u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.panic.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.panic.{id}"] -pub struct ReceiptsPanicSubject { - pub tx_id: Option, - pub index: Option, - pub id: Option, -} - -/// Represents a subject for receipts related to contract reverts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of revert receipts -/// based on the transaction ID, index, and the contract ID (`id`) associated with the revert event. -/// -/// # Examples -/// -/// Creating a subject for a specific revert receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsRevertSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsRevertSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// id: Some(ContractId::from([2u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.revert.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// Wildcard for querying all revert receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsRevertSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsRevertSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsRevertSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsRevertSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.revert.*" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsRevertSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsRevertSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_id(Some(ContractId::from([2u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.revert.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.revert.{id}"] -pub struct ReceiptsRevertSubject { - pub tx_id: Option, - pub index: Option, - pub id: Option, -} - -/// Represents a subject for log receipts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of log receipts -/// based on the transaction ID, index, and the contract ID (`id`) associated with the log event. -/// -/// # Examples -/// -/// Creating a subject for a specific log receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsLogSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsLogSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// id: Some(ContractId::from([2u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.log.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// Wildcard for querying all log receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsLogSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsLogSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsLogSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsLogSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.log.*" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsLogSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsLogSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_id(Some(ContractId::from([2u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.log.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.log.{id}"] -pub struct ReceiptsLogSubject { - pub tx_id: Option, - pub index: Option, - pub id: Option, -} - -/// Represents a subject for log data receipts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of log data receipts -/// based on the transaction ID, index, and the contract ID (`id`) associated with the log data. -/// -/// # Examples -/// -/// Creating a subject for a specific log data receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsLogDataSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsLogDataSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// id: Some(ContractId::from([2u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.log_data.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// Wildcard for querying all log data receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsLogDataSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsLogDataSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsLogDataSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsLogDataSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.log_data.*" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsLogDataSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsLogDataSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_id(Some(ContractId::from([2u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.log_data.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.log_data.{id}"] -pub struct ReceiptsLogDataSubject { - pub tx_id: Option, - pub index: Option, - pub id: Option, -} - -/// Represents a subject for transfer receipts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of transfer receipts -/// based on the transaction ID, index, the contract ID of the sender (`from`), the contract ID of the receiver (`to`), -/// and the asset ID involved in the transfer. -/// -/// # Examples -/// -/// Creating a subject for a specific transfer receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsTransferSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsTransferSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// from: Some(ContractId::from([2u8; 32])), -/// to: Some(ContractId::from([3u8; 32])), -/// asset_id: Some(AssetId::from([4u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.transfer.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -/// -/// Wildcard for querying all transfer receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsTransferSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsTransferSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsTransferSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsTransferSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// Some(ContractId::from([2u8; 32])), -/// Some(ContractId::from([3u8; 32])), -/// Some(AssetId::from([4u8; 32])) -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.transfer.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsTransferSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsTransferSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_from(Some(ContractId::from([2u8; 32]))) -/// .with_to(Some(ContractId::from([3u8; 32]))) -/// .with_asset_id(Some(AssetId::from([4u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.transfer.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.transfer.{from}.{to}.{asset_id}"] -pub struct ReceiptsTransferSubject { - pub tx_id: Option, - pub index: Option, - pub from: Option, - pub to: Option, - pub asset_id: Option, -} - -/// Represents a subject for transfer-out receipts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of transfer-out receipts -/// based on the transaction ID, index, the contract ID of the sender (`from`), the address of the receiver (`to`), -/// and the asset ID involved in the transfer-out. -/// -/// # Examples -/// -/// Creating a subject for a specific transfer-out receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsTransferOutSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsTransferOutSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// from: Some(ContractId::from([2u8; 32])), -/// to: Some(Address::from([3u8; 32])), -/// asset_id: Some(AssetId::from([4u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.transfer_out.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -/// -/// Wildcard for querying all transfer-out receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsTransferOutSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsTransferOutSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsTransferOutSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsTransferOutSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// Some(ContractId::from([2u8; 32])), -/// Some(Address::from([3u8; 32])), -/// Some(AssetId::from([4u8; 32])) -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.transfer_out.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsTransferOutSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsTransferOutSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_from(Some(ContractId::from([2u8; 32]))) -/// .with_to(Some(Address::from([3u8; 32]))) -/// .with_asset_id(Some(AssetId::from([4u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.transfer_out.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303.0x0404040404040404040404040404040404040404040404040404040404040404" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.transfer_out.{from}.{to}.{asset_id}"] -pub struct ReceiptsTransferOutSubject { - pub tx_id: Option, - pub index: Option, - pub from: Option, - pub to: Option
, - pub asset_id: Option, -} - -/// Represents a subject for script result receipts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of script result receipts -/// based on the transaction ID (`tx_id`) and index (`index`) within the transaction. -/// -/// # Examples -/// -/// Creating a subject for a specific script result receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsScriptResultSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsScriptResultSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.script_result" -/// ); -/// ``` -/// -/// Wildcard for querying all script result receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsScriptResultSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsScriptResultSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsScriptResultSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsScriptResultSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.script_result" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsScriptResultSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsScriptResultSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.script_result" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.script_result"] -pub struct ReceiptsScriptResultSubject { - pub tx_id: Option, - pub index: Option, -} - -/// Represents a subject for message-out receipts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of message-out receipts -/// based on the transaction ID (`tx_id`), index (`index`), sender address (`sender`), and recipient address (`recipient`). -/// -/// # Examples -/// -/// Creating a subject for a specific message-out receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsMessageOutSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsMessageOutSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// sender: Some(Address::from([2u8; 32])), -/// recipient: Some(Address::from([3u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.message_out.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// Wildcard for querying all message-out receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsMessageOutSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsMessageOutSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsMessageOutSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsMessageOutSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// Some(Address::from([2u8; 32])), -/// Some(Address::from([3u8; 32])), -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.message_out.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsMessageOutSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsMessageOutSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_sender(Some(Address::from([2u8; 32]))) -/// .with_recipient(Some(Address::from([3u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.message_out.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.message_out.{sender}.{recipient}"] -pub struct ReceiptsMessageOutSubject { - pub tx_id: Option, - pub index: Option, - pub sender: Option
, - pub recipient: Option
, -} - -/// Represents a subject for mint receipts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of mint receipts -/// based on the transaction ID (`tx_id`), index (`index`), contract ID (`contract_id`), and sub ID (`sub_id`). -/// -/// # Examples -/// -/// Creating a subject for a specific mint receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsMintSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsMintSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// contract_id: Some(ContractId::from([2u8; 32])), -/// sub_id: Some(Bytes32::from([3u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.mint.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// Wildcard for querying all mint receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsMintSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsMintSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsMintSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsMintSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// Some(ContractId::from([2u8; 32])), -/// Some(Bytes32::from([3u8; 32])), -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.mint.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsMintSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsMintSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_contract_id(Some(ContractId::from([2u8; 32]))) -/// .with_sub_id(Some(Bytes32::from([3u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.mint.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.mint.{contract_id}.{sub_id}"] -pub struct ReceiptsMintSubject { - pub tx_id: Option, - pub index: Option, - pub contract_id: Option, - pub sub_id: Option, -} - -/// Represents a subject for burn receipts in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of burn receipts -/// based on the transaction ID (`tx_id`), index (`index`), contract ID (`contract_id`), and sub ID (`sub_id`). -/// -/// # Examples -/// -/// Creating a subject for a specific burn receipt: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsBurnSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsBurnSubject { -/// tx_id: Some(Bytes32::from([1u8; 32])), -/// index: Some(0), -/// contract_id: Some(ContractId::from([2u8; 32])), -/// sub_id: Some(Bytes32::from([3u8; 32])), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.burn.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// Wildcard for querying all burn receipts: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsBurnSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(ReceiptsBurnSubject::WILDCARD, "receipts.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsBurnSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = ReceiptsBurnSubject::wildcard( -/// Some(Bytes32::from([1u8; 32])), -/// None, -/// Some(ContractId::from([2u8; 32])), -/// Some(Bytes32::from([3u8; 32])), -/// ); -/// assert_eq!( -/// wildcard, -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.*.burn.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::receipts::subjects::ReceiptsBurnSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = ReceiptsBurnSubject::new() -/// .with_tx_id(Some(Bytes32::from([1u8; 32]))) -/// .with_index(Some(0)) -/// .with_contract_id(Some(ContractId::from([2u8; 32]))) -/// .with_sub_id(Some(Bytes32::from([3u8; 32]))); -/// assert_eq!( -/// subject.parse(), -/// "receipts.0x0101010101010101010101010101010101010101010101010101010101010101.0.burn.0x0202020202020202020202020202020202020202020202020202020202020202.0x0303030303030303030303030303030303030303030303030303030303030303" -/// ); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "receipts.>"] -#[subject_format = "receipts.{tx_id}.{index}.burn.{contract_id}.{sub_id}"] -pub struct ReceiptsBurnSubject { - pub tx_id: Option, - pub index: Option, - pub contract_id: Option, - pub sub_id: Option, -} diff --git a/crates/fuel-streams-core/src/stream/config.rs b/crates/fuel-streams-core/src/stream/config.rs new file mode 100644 index 00000000..e770e907 --- /dev/null +++ b/crates/fuel-streams-core/src/stream/config.rs @@ -0,0 +1,15 @@ +use std::sync::LazyLock; + +pub static STREAM_THROTTLE_HISTORICAL: LazyLock = LazyLock::new(|| { + dotenvy::var("STREAM_THROTTLE_HISTORICAL") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(150) +}); + +pub static STREAM_THROTTLE_LIVE: LazyLock = LazyLock::new(|| { + dotenvy::var("STREAM_THROTTLE_LIVE") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(0) +}); diff --git a/crates/fuel-streams-core/src/stream/deliver_policy.rs b/crates/fuel-streams-core/src/stream/deliver_policy.rs new file mode 100644 index 00000000..87645a91 --- /dev/null +++ b/crates/fuel-streams-core/src/stream/deliver_policy.rs @@ -0,0 +1,165 @@ +use serde::{self, Deserialize, Deserializer, Serialize}; + +#[derive(Debug, thiserror::Error)] +pub enum DeliverPolicyError { + #[error("Invalid delivery policy format. Expected 'new', 'from_block:', or 'from_block='")] + InvalidFormat, + #[error("Block height cannot be empty")] + EmptyBlockHeight, + #[error("Invalid block height '{0}': must be a positive number")] + InvalidBlockHeight(String), +} + +#[derive(Debug, Default, Serialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum DeliverPolicy { + #[default] + New, + FromBlock { + #[serde(rename = "blockHeight")] + block_height: u64, + }, +} + +impl std::fmt::Display for DeliverPolicy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DeliverPolicy::New => write!(f, "new"), + DeliverPolicy::FromBlock { block_height } => { + write!(f, "from_block:{}", block_height) + } + } + } +} + +impl std::str::FromStr for DeliverPolicy { + type Err = DeliverPolicyError; + fn from_str(value: &str) -> Result { + match value { + "new" => Ok(DeliverPolicy::New), + value + if value.starts_with("from_block:") + || value.starts_with("from_block=") => + { + let block_height = value + .strip_prefix("from_block:") + .or_else(|| value.strip_prefix("from_block=")) + .ok_or(DeliverPolicyError::InvalidFormat)? + .trim(); + + if block_height.is_empty() { + return Err(DeliverPolicyError::EmptyBlockHeight); + } + + let height = block_height.parse::().map_err(|_| { + DeliverPolicyError::InvalidBlockHeight( + block_height.to_string(), + ) + })?; + + Ok(DeliverPolicy::FromBlock { + block_height: height, + }) + } + _ => Err(DeliverPolicyError::InvalidFormat), + } + } +} + +// Add custom deserialization +impl<'de> Deserialize<'de> for DeliverPolicy { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum PolicyHelper { + String(String), + Object { + #[serde(rename = "fromBlock")] + from_block: BlockHeight, + }, + } + + #[derive(Deserialize)] + struct BlockHeight { + #[serde(rename = "blockHeight")] + block_height: u64, + } + + let helper = PolicyHelper::deserialize(deserializer)?; + match helper { + PolicyHelper::String(s) => { + s.parse().map_err(serde::de::Error::custom) + } + PolicyHelper::Object { from_block } => { + Ok(DeliverPolicy::FromBlock { + block_height: from_block.block_height, + }) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_string_deserialization() { + // Test "new" string format + let json = r#""new""#; + let policy: DeliverPolicy = serde_json::from_str(json).unwrap(); + assert_eq!(policy, DeliverPolicy::New); + + // Test "from_block:123" string format + let json = r#""from_block:123""#; + let policy: DeliverPolicy = serde_json::from_str(json).unwrap(); + assert_eq!(policy, DeliverPolicy::FromBlock { block_height: 123 }); + + // Test "from_block=123" string format + let json = r#""from_block=123""#; + let policy: DeliverPolicy = serde_json::from_str(json).unwrap(); + assert_eq!(policy, DeliverPolicy::FromBlock { block_height: 123 }); + } + + #[test] + fn test_object_deserialization() { + // Test object format + let json = r#"{"fromBlock": {"blockHeight": 123}}"#; + let policy: DeliverPolicy = serde_json::from_str(json).unwrap(); + assert_eq!(policy, DeliverPolicy::FromBlock { block_height: 123 }); + } + + #[test] + fn test_invalid_formats() { + // Test invalid string format + let json = r#""invalid_format""#; + let result: Result = serde_json::from_str(json); + assert!(result.is_err()); + + // Test invalid block height + let json = r#""from_block:invalid""#; + let result: Result = serde_json::from_str(json); + assert!(result.is_err()); + + // Test empty block height + let json = r#""from_block:""#; + let result: Result = serde_json::from_str(json); + assert!(result.is_err()); + } + + #[test] + fn test_serialization() { + // Test New variant serialization + let policy = DeliverPolicy::New; + let json = serde_json::to_string(&policy).unwrap(); + assert_eq!(json, r#""new""#); + + // Test FromBlock variant serialization + let policy = DeliverPolicy::FromBlock { block_height: 123 }; + let json = serde_json::to_string(&policy).unwrap(); + assert_eq!(json, r#"{"fromBlock":{"blockHeight":123}}"#); + } +} diff --git a/crates/fuel-streams-core/src/stream/error.rs b/crates/fuel-streams-core/src/stream/error.rs index 03625d9d..fc552a58 100644 --- a/crates/fuel-streams-core/src/stream/error.rs +++ b/crates/fuel-streams-core/src/stream/error.rs @@ -1,42 +1,19 @@ -use async_nats::{ - error, - jetstream::{ - consumer::StreamErrorKind, - context::{CreateKeyValueErrorKind, CreateStreamErrorKind}, - kv::{CreateError, CreateErrorKind, PutError, WatchErrorKind}, - stream::{ConsumerErrorKind, LastRawMessageErrorKind}, - }, -}; -use displaydoc::Display as DisplayDoc; -use fuel_data_parser::DataParserError; -use thiserror::Error; +use async_nats::SubscribeError; +use fuel_message_broker::MessageBrokerError; +use fuel_streams_store::{db::DbError, store::StoreError}; -#[derive(Error, DisplayDoc, Debug)] +use crate::DeliverPolicyError; + +#[derive(Debug, thiserror::Error)] pub enum StreamError { - /// Failed to publish to stream: {subject_name}, error: {source} - PublishFailed { - subject_name: String, - #[source] - source: error::Error, - }, - /// Failed to publish to storage: {0} - Storage(#[from] fuel_streams_storage::StorageError), - /// Failed to retrieve last published message from stream: {0} - GetLastPublishedFailed(#[from] error::Error), - /// Failed to create Key-Value Store: {0} - StoreCreation(#[from] error::Error), - /// Failed to publish item to Key-Value Store: {0} - StorePublish(#[from] PutError), - /// Failed to subscribe to subject in Key-Value Store: {0} - StoreSubscribe(#[from] error::Error), - /// Failed to publish item to stream: {0} - StreamPublish(#[from] CreateError), - /// Failed to create stream: {0} - StreamCreation(#[from] error::Error), - /// Failed to create consumer for stream: {0} - ConsumerCreate(#[from] error::Error), - /// Failed to consume messages from stream: {0} - ConsumerMessages(#[from] error::Error), - /// Failed to encode or decode data: {0} - Encoder(#[from] DataParserError), + #[error(transparent)] + Db(#[from] DbError), + #[error(transparent)] + Store(#[from] StoreError), + #[error(transparent)] + Subscribe(#[from] SubscribeError), + #[error(transparent)] + DeliverPolicy(#[from] DeliverPolicyError), + #[error(transparent)] + MessageBrokerClient(#[from] MessageBrokerError), } diff --git a/crates/fuel-streams-core/src/stream/fuel_streams.rs b/crates/fuel-streams-core/src/stream/fuel_streams.rs index 92ecb7f4..fbbc719c 100644 --- a/crates/fuel-streams-core/src/stream/fuel_streams.rs +++ b/crates/fuel-streams-core/src/stream/fuel_streams.rs @@ -1,187 +1,42 @@ use std::sync::Arc; -use async_nats::{ - jetstream::{context::CreateStreamErrorKind, stream::State as StreamState}, - RequestErrorKind, -}; -use futures::stream::BoxStream; +use fuel_message_broker::MessageBroker; +use fuel_streams_store::db::Db; -use crate::prelude::*; +use super::Stream; +use crate::types::*; #[derive(Clone, Debug)] pub struct FuelStreams { - pub transactions: Stream, pub blocks: Stream, + pub transactions: Stream, pub inputs: Stream, pub outputs: Stream, pub receipts: Stream, pub utxos: Stream, - pub logs: Stream, -} - -pub struct FuelStreamsUtils; -impl FuelStreamsUtils { - pub fn is_within_subject_names(subject_name: &str) -> bool { - let subject_names = Self::subjects_names(); - subject_names.contains(&subject_name) - } - - pub fn subjects_names() -> &'static [&'static str] { - &[ - Transaction::NAME, - Block::NAME, - Input::NAME, - Receipt::NAME, - Utxo::NAME, - Log::NAME, - ] - } - - pub fn wildcards() -> Vec<&'static str> { - let nested_wildcards = [ - Transaction::WILDCARD_LIST, - Block::WILDCARD_LIST, - Input::WILDCARD_LIST, - Receipt::WILDCARD_LIST, - Utxo::WILDCARD_LIST, - Log::WILDCARD_LIST, - ]; - nested_wildcards - .into_iter() - .flatten() - .copied() - .collect::>() - } + pub msg_broker: Arc, + pub db: Arc, } impl FuelStreams { - pub async fn new( - nats_client: &NatsClient, - storage: &Arc, - ) -> Self { + pub async fn new(broker: &Arc, db: &Arc) -> Self { Self { - transactions: Stream::::new(nats_client, storage) - .await, - blocks: Stream::::new(nats_client, storage).await, - inputs: Stream::::new(nats_client, storage).await, - outputs: Stream::::new(nats_client, storage).await, - receipts: Stream::::new(nats_client, storage).await, - utxos: Stream::::new(nats_client, storage).await, - logs: Stream::::new(nats_client, storage).await, - } - } - - pub async fn setup_all( - core_client: &NatsClient, - publisher_client: &NatsClient, - storage: &Arc, - ) -> (Self, Self) { - let core_stream = Self::new(core_client, storage).await; - let publisher_stream = Self::new(publisher_client, storage).await; - (core_stream, publisher_stream) - } - - pub async fn subscribe_raw( - &self, - sub_subject: &str, - subscription_config: Option, - ) -> Result, String, NatsMessage)>, StreamError> - { - match sub_subject { - Transaction::NAME => { - self.transactions.subscribe_raw(subscription_config).await - } - Block::NAME => self.blocks.subscribe_raw(subscription_config).await, - Input::NAME => self.inputs.subscribe_raw(subscription_config).await, - Output::NAME => { - self.outputs.subscribe_raw(subscription_config).await - } - Receipt::NAME => { - self.receipts.subscribe_raw(subscription_config).await - } - Utxo::NAME => self.utxos.subscribe_raw(subscription_config).await, - Log::NAME => self.logs.subscribe_raw(subscription_config).await, - _ => Err(StreamError::StreamCreation( - CreateStreamErrorKind::InvalidStreamName.into(), - )), + blocks: Stream::::get_or_init(broker, db).await, + transactions: Stream::::get_or_init(broker, db).await, + inputs: Stream::::get_or_init(broker, db).await, + outputs: Stream::::get_or_init(broker, db).await, + receipts: Stream::::get_or_init(broker, db).await, + utxos: Stream::::get_or_init(broker, db).await, + msg_broker: Arc::clone(broker), + db: Arc::clone(db), } } pub fn arc(self) -> Arc { Arc::new(self) } -} - -#[async_trait::async_trait] -pub trait FuelStreamsExt: Sync + Send { - fn blocks(&self) -> &Stream; - fn transactions(&self) -> &Stream; - fn inputs(&self) -> &Stream; - fn outputs(&self) -> &Stream; - fn receipts(&self) -> &Stream; - fn utxos(&self) -> &Stream; - fn logs(&self) -> &Stream; - - async fn get_last_published_block(&self) -> anyhow::Result>; - async fn get_consumers_and_state( - &self, - ) -> Result, StreamState)>, RequestErrorKind>; - - #[cfg(any(test, feature = "test-helpers"))] - async fn is_empty(&self) -> bool; -} - -#[async_trait::async_trait] -impl FuelStreamsExt for FuelStreams { - fn blocks(&self) -> &Stream { - &self.blocks - } - fn transactions(&self) -> &Stream { - &self.transactions - } - fn inputs(&self) -> &Stream { - &self.inputs - } - fn outputs(&self) -> &Stream { - &self.outputs - } - fn receipts(&self) -> &Stream { - &self.receipts - } - fn utxos(&self) -> &Stream { - &self.utxos - } - fn logs(&self) -> &Stream { - &self.logs - } - - async fn get_last_published_block(&self) -> anyhow::Result> { - self.blocks - .get_last_published(BlocksSubject::WILDCARD) - .await - .map_err(|e| e.into()) - } - - async fn get_consumers_and_state( - &self, - ) -> Result, StreamState)>, RequestErrorKind> { - Ok(vec![ - self.transactions.get_consumers_and_state().await?, - self.blocks.get_consumers_and_state().await?, - self.inputs.get_consumers_and_state().await?, - self.outputs.get_consumers_and_state().await?, - self.receipts.get_consumers_and_state().await?, - self.utxos.get_consumers_and_state().await?, - self.logs.get_consumers_and_state().await?, - ]) - } - #[cfg(any(test, feature = "test-helpers"))] - async fn is_empty(&self) -> bool { - self.blocks.is_empty(BlocksSubject::WILDCARD).await - && self - .transactions - .is_empty(TransactionsSubject::WILDCARD) - .await + pub fn broker(&self) -> Arc { + self.msg_broker.clone() } } diff --git a/crates/fuel-streams-core/src/stream/mod.rs b/crates/fuel-streams-core/src/stream/mod.rs index 028c58b6..c96c3578 100644 --- a/crates/fuel-streams-core/src/stream/mod.rs +++ b/crates/fuel-streams-core/src/stream/mod.rs @@ -1,7 +1,10 @@ +pub(super) mod config; +mod deliver_policy; mod error; mod fuel_streams; mod stream_impl; +pub use deliver_policy::*; pub use error::*; pub use fuel_streams::*; pub use stream_impl::*; diff --git a/crates/fuel-streams-core/src/stream/stream_impl.rs b/crates/fuel-streams-core/src/stream/stream_impl.rs index 636e6c61..49f7b17f 100644 --- a/crates/fuel-streams-core/src/stream/stream_impl.rs +++ b/crates/fuel-streams-core/src/stream/stream_impl.rs @@ -1,527 +1,112 @@ -use std::sync::{Arc, LazyLock}; +use std::{sync::Arc, time::Duration}; -use async_nats::{ - jetstream::{ - consumer::AckPolicy, - kv::{self, CreateErrorKind}, - stream::{self, LastRawMessageErrorKind, State}, - }, - RequestErrorKind, -}; -use async_trait::async_trait; +pub use async_nats::Subscriber as StreamLiveSubscriber; +use fuel_message_broker::MessageBroker; use fuel_streams_macros::subject::IntoSubject; -use futures::{stream::BoxStream, StreamExt, TryStreamExt}; -use tokio::sync::OnceCell; - -use crate::prelude::*; - -pub static MAX_ACK_PENDING: LazyLock = LazyLock::new(|| { - dotenvy::var("MAX_ACK_PENDING") - .ok() - .and_then(|val| val.parse().ok()) - .unwrap_or(5) -}); - -#[derive(Debug, Clone)] -pub struct PublishPacket { - pub subject: Arc, - pub payload: Arc, -} - -impl PublishPacket { - pub fn new(payload: T, subject: Arc) -> Self { - Self { - payload: Arc::new(payload), - subject, - } - } - - pub fn get_s3_path(&self) -> String { - let subject = self.subject.parse(); - format!("{}.json.zstd", subject.replace('.', "/")) - } -} -pub trait StreamEncoder: DataEncoder {} -impl> StreamEncoder for T {} +use fuel_streams_store::{ + db::{Db, DbItem}, + record::{Record, RecordPacket}, + store::Store, +}; +use futures::{ + stream::{BoxStream, Stream as FStream}, + StreamExt, +}; +use tokio::{sync::OnceCell, time::sleep}; -/// Trait for types that can be streamed. -/// -/// # Examples -/// -/// ```no_run -/// use async_trait::async_trait; -/// use fuel_streams_core::prelude::*; -/// use fuel_data_parser::*; -/// -/// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -/// struct MyStreamable { -/// data: String, -/// } -/// -/// impl DataEncoder for MyStreamable { -/// type Err = StreamError; -/// } -/// -/// #[async_trait] -/// impl Streamable for MyStreamable { -/// const NAME: &'static str = "my_streamable"; -/// const WILDCARD_LIST: &'static [&'static str] = &["*"]; -/// } -/// ``` -#[async_trait] -pub trait Streamable: StreamEncoder + std::marker::Sized { - const NAME: &'static str; - const WILDCARD_LIST: &'static [&'static str]; +use super::{config, StreamError}; +use crate::DeliverPolicy; - fn to_packet(&self, subject: Arc) -> PublishPacket { - PublishPacket::new(self.clone(), subject) - } -} +pub type BoxedStreamItem = Result, StreamError>; +pub type BoxedStream = Box + Send + Unpin>; -/// Houses nats-agnostic APIs for publishing and consuming a streamable type -/// -/// # Examples -/// -/// ```no_run -/// use std::sync::Arc; -/// use fuel_streams_core::prelude::*; -/// use fuel_streams_macros::subject::IntoSubject; -/// use fuel_data_parser::*; -/// use futures::StreamExt; -/// -/// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -/// struct MyStreamable { -/// data: String, -/// } -/// -/// impl DataEncoder for MyStreamable { -/// type Err = StreamError; -/// } -/// -/// #[async_trait::async_trait] -/// impl Streamable for MyStreamable { -/// const NAME: &'static str = "my_streamable"; -/// const WILDCARD_LIST: &'static [&'static str] = &["*"]; -/// } -/// -/// async fn example(nats_client: &NatsClient, storage: &Arc) { -/// let stream = Stream::::new(nats_client, storage).await; -/// -/// // Publish -/// let subject = BlocksSubject::new().with_height(Some(23.into())).arc(); -/// let packet = MyStreamable { data: "foo".into() }.to_packet(subject); -/// stream.publish(&packet).await.unwrap(); -/// -/// // Subscribe -/// let mut subscription = stream.subscribe(None).await.unwrap(); -/// while let Some(message) = subscription.next().await { -/// // Process message -/// } -/// } -/// ``` -/// -/// TODO: Split this into two traits StreamPublisher + StreamSubscriber -/// TODO: Rename as FuelStream? #[derive(Debug, Clone)] -pub struct Stream { - store: Arc, - storage: Arc, +pub struct Stream { + store: Arc>, + broker: Arc, _marker: std::marker::PhantomData, } -impl Stream { +impl Stream { #[allow(clippy::declare_interior_mutable_const)] const INSTANCE: OnceCell = OnceCell::const_new(); pub async fn get_or_init( - nats_client: &NatsClient, - storage: &Arc, + broker: &Arc, + db: &Arc, ) -> Self { let cell = Self::INSTANCE; - cell.get_or_init(|| async { - Self::new(nats_client, storage).await.to_owned() - }) - .await - .to_owned() - } - - pub async fn new( - nats_client: &NatsClient, - storage: &Arc, - ) -> Self { - let namespace = &nats_client.namespace; - let bucket_name = namespace.stream_name(S::NAME); - let config = kv::Config { - bucket: bucket_name.to_owned(), - storage: stream::StorageType::File, - history: 1, - ..Default::default() - }; - - let store = nats_client - .get_or_create_kv_store(config) + cell.get_or_init(|| async { Self::new(broker, db).await.to_owned() }) .await - .expect("Streams must be created"); + .to_owned() + } + pub async fn new(broker: &Arc, db: &Arc) -> Self { + let store = Arc::new(Store::new(db)); + let broker = Arc::clone(broker); Self { - store: Arc::new(store), - storage: Arc::clone(storage), + store, + broker, _marker: std::marker::PhantomData, } } - pub async fn publish( - &self, - packet: &PublishPacket, - ) -> Result { - let payload = &packet.payload; - let s3_path = packet.get_s3_path(); - let subject_name = &packet.subject.parse(); - let encoded = payload.encode().await?; - self.storage.store(&s3_path, encoded).await?; - self.publish_s3_path_to_nats(subject_name, &s3_path).await + #[cfg(any(test, feature = "test-helpers"))] + pub fn store(&self) -> &Store { + &self.store } - async fn publish_s3_path_to_nats( - &self, - subject_name: &str, - s3_path: &str, - ) -> Result { - tracing::debug!("S3 path published: {:?}", s3_path); - let data = s3_path.to_string().into_bytes(); - let data_size = data.len(); - let result = self.store.create(subject_name, data.into()).await; - - match result { - Ok(_) => Ok(data_size), - Err(e) if e.kind() == CreateErrorKind::AlreadyExists => { - Ok(data_size) - } - Err(e) => Err(StreamError::PublishFailed { - subject_name: subject_name.to_string(), - source: e, - }), - } + pub fn arc(&self) -> Arc { + Arc::new(self.to_owned()) } - pub async fn get_consumers_and_state( + pub async fn publish( &self, - ) -> Result<(String, Vec, State), RequestErrorKind> { - let mut consumers = vec![]; - while let Ok(Some(consumer)) = - self.store.stream.consumer_names().try_next().await - { - consumers.push(consumer); - } - - let state = self.store.stream.cached_info().state.clone(); - let stream_name = self.get_stream_name().to_string(); - Ok((stream_name, consumers, state)) - } - - pub fn get_stream_name(&self) -> &str { - self.store.stream_name.as_str() + packet: &Arc>, + ) -> Result { + let broker = self.broker.clone(); + let db_record = self.store.insert_record(packet).await?; + let encoded_value = db_record.encoded_value().to_vec(); + let subject = packet.subject_str(); + broker.publish_event(&subject, encoded_value.into()).await?; + Ok(db_record) } - pub async fn subscribe_raw( + pub async fn subscribe_dynamic( &self, - subscription_config: Option, - ) -> Result, String, NatsMessage)>, StreamError> - { - let config = self.get_consumer_config(subscription_config); - let config = self.prefix_filter_subjects(config); - let consumer = self.store.stream.create_consumer(config).await?; - let messages = consumer.messages().await?; - let storage = Arc::clone(&self.storage); - - Ok(messages - .then(move |message| { - let message = message.expect("Message must be valid"); - let nats_payload = message.payload.to_vec(); - let storage = storage.clone(); - async move { - let s3_path = String::from_utf8(nats_payload) - .expect("Must be S3 path"); - ( - storage - .retrieve(&s3_path) - .await - .expect("S3 object must exist"), - s3_path, - message, - ) + subject: Arc, + deliver_policy: DeliverPolicy, + ) -> BoxStream<'static, Result, StreamError>> { + let store = self.store.clone(); + let broker = self.broker.clone(); + let subject_clone = subject.clone(); + let stream = async_stream::try_stream! { + if let DeliverPolicy::FromBlock { block_height } = deliver_policy { + let height = Some(block_height); + let mut historical = store.stream_by_subject(subject_clone, height); + while let Some(result) = historical.next().await { + let item = result.map_err(StreamError::Store)?; + yield item.encoded_value().to_vec(); + let throttle_time = *config::STREAM_THROTTLE_HISTORICAL; + sleep(Duration::from_millis(throttle_time as u64)).await; } - }) - .boxed()) - } - - pub async fn subscribe( - &self, - subscription_config: Option, - ) -> Result, StreamError> { - let raw_stream = self.subscribe_raw(subscription_config).await?; - Ok(raw_stream - .then(|(s3_data, s3_path, message)| async move { - let item = S::decode(&s3_data).await.expect("Failed to decode"); - let _ = message.ack().await; - (item, s3_path) - }) - .boxed()) - } - - pub fn get_consumer_config( - &self, - subscription_config: Option, - ) -> PullConsumerConfig { - let filter_subjects = match subscription_config.clone() { - Some(subscription_config) => subscription_config.filter_subjects, - None => vec![S::WILDCARD_LIST[0].to_string()], - }; - let delivery_policy = match subscription_config.clone() { - Some(subscription_config) => subscription_config.deliver_policy, - None => NatsDeliverPolicy::New, + } + let mut live = broker.subscribe_to_events(&subject.parse()).await?; + while let Some(msg) = live.next().await { + yield msg?; + let throttle_time = *config::STREAM_THROTTLE_LIVE; + sleep(Duration::from_millis(throttle_time as u64)).await; + } }; - PullConsumerConfig { - filter_subjects, - deliver_policy: delivery_policy, - ack_policy: AckPolicy::Explicit, - max_ack_pending: *MAX_ACK_PENDING as i64, - ..Default::default() - } + Box::pin(stream) } - #[cfg(feature = "test-helpers")] - /// Fetch all old messages from this stream - pub async fn catchup( + pub async fn subscribe( &self, - number_of_messages: usize, - ) -> Result>, StreamError> { - let config = PullConsumerConfig { - filter_subjects: self.all_filter_subjects(), - deliver_policy: NatsDeliverPolicy::All, - ack_policy: AckPolicy::None, - ..Default::default() - }; - let config = self.prefix_filter_subjects(config); - let consumer = self.store.stream.create_consumer(config).await?; - - let stream = consumer - .messages() - .await? - .take(number_of_messages) - .then(|message| async { - if let Ok(message) = message { - Some( - self.get_payload_from_s3(message.payload.to_vec()) - .await - .unwrap(), - ) - } else { - None - } - }) - .boxed(); - - Ok(stream) - } - - // TODO: Make this interface more Stream-like and Nats agnostic - pub async fn create_consumer( - &self, - config: PullConsumerConfig, - ) -> Result, StreamError> { - let config = self.prefix_filter_subjects(config); - Ok(self.store.stream.create_consumer(config).await?) - } - - #[cfg(feature = "test-helpers")] - fn all_filter_subjects(&self) -> Vec { - S::WILDCARD_LIST.iter().map(|s| s.to_string()).collect() - } - - #[cfg(feature = "test-helpers")] - pub async fn is_empty(&self, wildcard: &str) -> bool - where - S: for<'de> serde::Deserialize<'de>, - { - self.get_last_published(wildcard) - .await - .is_ok_and(|result| result.is_none()) - } - - pub async fn get_last_published( - &self, - wildcard: &str, - ) -> Result, StreamError> { - let subject_name = &Self::prefix_filter_subject(wildcard); - let message = self - .store - .stream - .get_last_raw_message_by_subject(subject_name) - .await; - - match message { - Ok(message) => Ok(Some( - self.get_payload_from_s3(message.payload.to_vec()).await?, - )), - Err(error) => match &error.kind() { - LastRawMessageErrorKind::NoMessageFound => Ok(None), - _ => Err(error.into()), - }, - } - } - - async fn get_payload_from_s3( - &self, - nats_payload: Vec, - ) -> Result { - let s3_path = String::from_utf8(nats_payload).expect("Must be S3 path"); - let s3_object = self - .storage - .retrieve(&s3_path) - .await - .expect("S3 object must exist"); - Ok(S::decode(&s3_object).await.expect("Failed to decode")) - } - - #[cfg(any(test, feature = "test-helpers"))] - pub async fn assert_has_stream( - &self, - names: &std::collections::HashSet, - ) { - let mut stream = self.store.stream.clone(); - let info = stream.info().await.unwrap(); - let has_stream = names.iter().any(|n| n.eq(&info.config.name)); - assert!(has_stream) - } - - fn prefix_filter_subjects( - &self, - mut config: PullConsumerConfig, - ) -> PullConsumerConfig { - config.filter_subjects = config - .filter_subjects - .iter() - .map(Self::prefix_filter_subject) - .collect(); - config - } - - fn prefix_filter_subject(subject: impl Into) -> String { - // An hack to ensure we keep the KV namespace when reading - // from the KV store's stream - let subject = subject.into(); - format!("$KV.*.{subject}") - } - - #[cfg(any(test, feature = "test-helpers"))] - pub fn store(&self) -> &kv::Store { - &self.store - } - - pub fn arc(&self) -> Arc { - Arc::new(self.to_owned()) - } -} - -/// Configuration for subscribing to a consumer. -/// -/// # Examples -/// -/// ``` -/// use fuel_streams_core::stream::SubscriptionConfig; -/// use async_nats::jetstream::consumer::DeliverPolicy as NatsDeliverPolicy; -/// -/// let config = SubscriptionConfig { -/// filter_subjects: vec!["example.*".to_string()], -/// deliver_policy: NatsDeliverPolicy::All, -/// }; -/// ``` -#[derive(Debug, Clone, Default)] -pub struct SubscriptionConfig { - pub filter_subjects: Vec, - pub deliver_policy: NatsDeliverPolicy, -} - -#[cfg(any(test, feature = "test-helpers"))] -mod tests { - use serde::{Deserialize, Serialize}; - - use super::*; - - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] - struct TestStreamable { - data: String, - } - - impl DataEncoder for TestStreamable { - type Err = StreamError; - } - - #[async_trait] - impl Streamable for TestStreamable { - const NAME: &'static str = "test_streamable"; - const WILDCARD_LIST: &'static [&'static str] = &["*"]; - } - - #[tokio::test] - async fn test_stream_item_s3_encoding_flow() { - let (stream, _, test_data, subject) = setup_test().await; - let packet = test_data.to_packet(subject); - - // Publish (this will encode and store in S3) - stream.publish(&packet).await.unwrap(); - - // Get the S3 path that was used - let s3_path = packet.get_s3_path(); - - // Retrieve directly from S3 and verify encoding - let raw_s3_data = stream.storage.retrieve(&s3_path).await.unwrap(); - let decoded = TestStreamable::decode(&raw_s3_data).await.unwrap(); - assert_eq!(decoded, test_data, "Retrieved data should match original"); - } - - #[tokio::test] - async fn test_stream_item_json_encoding_flow() { - use fuel_data_parser::DataParser; - let (_, _, test_data, _) = setup_test().await; - let encoded = test_data.encode().await.unwrap(); - let decoded = TestStreamable::decode(&encoded).await.unwrap(); - assert_eq!(decoded, test_data, "Decoded data should match original"); - - let json = DataParser::default().encode_json(&test_data).unwrap(); - let json_str = String::from_utf8(json).unwrap(); - let expected_json = r#"{"data":"test content"}"#; - assert_eq!( - json_str, expected_json, - "JSON structure should exactly match expected format" - ); - } - - #[cfg(test)] - async fn setup_test() -> ( - Stream, - Arc, - TestStreamable, - Arc, - ) { - let storage = S3Storage::new_for_testing().await.unwrap(); - let nats_client_opts = - NatsClientOpts::admin_opts().with_rdn_namespace(); - let nats_client = NatsClient::connect(&nats_client_opts).await.unwrap(); - let stream = Stream::::new( - &nats_client, - &Arc::new(storage.clone()), - ) - .await; - let test_data = TestStreamable { - data: "test content".to_string(), - }; - let subject = Arc::new( - BlocksSubject::new() - .with_producer(Some(Address::zeroed())) - .with_height(Some(1.into())), - ); - (stream, Arc::new(storage), test_data, subject) + subject: S, + deliver_policy: DeliverPolicy, + ) -> BoxStream<'static, Result, StreamError>> { + let subject = Arc::new(subject); + self.subscribe_dynamic(subject, deliver_policy).await } } diff --git a/crates/fuel-streams-core/src/transactions/mod.rs b/crates/fuel-streams-core/src/transactions/mod.rs deleted file mode 100644 index 2094d796..00000000 --- a/crates/fuel-streams-core/src/transactions/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -pub mod subjects; -pub mod types; - -pub use subjects::*; - -use super::types::*; -use crate::{DataEncoder, StreamError, Streamable}; - -impl DataEncoder for Transaction { - type Err = StreamError; -} -impl Streamable for Transaction { - const NAME: &'static str = "transactions"; - const WILDCARD_LIST: &'static [&'static str] = &[ - TransactionsSubject::WILDCARD, - TransactionsByIdSubject::WILDCARD, - ]; -} diff --git a/crates/fuel-streams-core/src/transactions/subjects.rs b/crates/fuel-streams-core/src/transactions/subjects.rs deleted file mode 100644 index a258f0cf..00000000 --- a/crates/fuel-streams-core/src/transactions/subjects.rs +++ /dev/null @@ -1,162 +0,0 @@ -use crate::prelude::*; - -/// Represents a subject for querying transactions by their identifier in the Fuel ecosystem. -/// -/// This struct is used to create and parse subjects related to transactions identified by -/// various types of IDs, which can be used for subscribing to or publishing events -/// about specific transactions. -/// -/// # Examples -/// -/// Creating and parsing a subject: -/// -/// ``` -/// # use fuel_streams_core::transactions::subjects::TransactionsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = TransactionsByIdSubject { -/// tx_id: Some([1u8; 32].into()), -/// index: Some(0), -/// id_kind: Some(IdentifierKind::ContractID), -/// id_value: Some([2u8; 32].into()), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "by_id.transactions.0x0101010101010101010101010101010101010101010101010101010101010101.0.contract_id.0x0202020202020202020202020202020202020202020202020202020202020202" -/// ); -/// ``` -/// -/// All transactions by ID wildcard: -/// -/// ``` -/// # use fuel_streams_core::transactions::subjects::TransactionsByIdSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(TransactionsByIdSubject::WILDCARD, "by_id.transactions.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::transactions::subjects::TransactionsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = TransactionsByIdSubject::wildcard(Some([1u8; 32].into()), Some(0), Some(IdentifierKind::ContractID), None); -/// assert_eq!(wildcard, "by_id.transactions.0x0101010101010101010101010101010101010101010101010101010101010101.0.contract_id.*"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::transactions::subjects::TransactionsByIdSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = TransactionsByIdSubject::new() -/// .with_tx_id(Some([1u8; 32].into())) -/// .with_index(Some(0)) -/// .with_id_kind(Some(IdentifierKind::ContractID)) -/// .with_id_value(Some([2u8; 32].into())); -/// assert_eq!(subject.parse(), "by_id.transactions.0x0101010101010101010101010101010101010101010101010101010101010101.0.contract_id.0x0202020202020202020202020202020202020202020202020202020202020202"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "by_id.transactions.>"] -#[subject_format = "by_id.transactions.{tx_id}.{index}.{id_kind}.{id_value}"] -pub struct TransactionsByIdSubject { - pub tx_id: Option, - pub index: Option, - pub id_kind: Option, - pub id_value: Option, -} - -/// Represents a subject for publishing transactions that happen in the Fuel network. -/// -/// This subject format allows for efficient querying and filtering of transactions -/// based on their height, index, ID, status, and kind. -/// -/// # Examples -/// -/// Creating a subject for a specific transaction: -/// -/// ``` -/// # use fuel_streams_core::transactions::TransactionsSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::IntoSubject; -/// let subject = TransactionsSubject { -/// block_height: Some(23.into()), -/// index: Some(1), -/// tx_id: Some(Bytes32::zeroed()), -/// status: Some(TransactionStatus::Success), -/// kind: Some(TransactionKind::Script), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "transactions.23.1.0x0000000000000000000000000000000000000000000000000000000000000000.success.script" -/// ); -/// ``` -/// -/// All transactions wildcard: -/// -/// ``` -/// # use fuel_streams_core::transactions::TransactionsSubject; -/// assert_eq!(TransactionsSubject::WILDCARD, "transactions.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::transactions::TransactionsSubject; -/// # use fuel_streams_core::prelude::*; -/// let wildcard = TransactionsSubject::wildcard(None, None, Some(Bytes32::zeroed()), None, None); -/// assert_eq!(wildcard, "transactions.*.*.0x0000000000000000000000000000000000000000000000000000000000000000.*.*"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::transactions::TransactionsSubject; -/// # use fuel_streams_core::prelude::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = TransactionsSubject::new() -/// .with_block_height(Some(23.into())) -/// .with_index(Some(1)) -/// .with_tx_id(Some(Bytes32::zeroed())) -/// .with_status(Some(TransactionStatus::Success)) -/// .with_kind(Some(TransactionKind::Script)); -/// assert_eq!(subject.parse(), "transactions.23.1.0x0000000000000000000000000000000000000000000000000000000000000000.success.script"); -/// ``` -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "transactions.>"] -#[subject_format = "transactions.{block_height}.{index}.{tx_id}.{status}.{kind}"] -pub struct TransactionsSubject { - pub block_height: Option, - pub index: Option, - pub tx_id: Option, - pub status: Option, - pub kind: Option, -} - -impl From<&Transaction> for TransactionsSubject { - fn from(transaction: &Transaction) -> Self { - let subject = TransactionsSubject::new(); - subject - .with_tx_id(Some(transaction.id.clone())) - .with_kind(Some(transaction.kind.clone())) - } -} - -#[cfg(test)] -mod test { - use pretty_assertions::assert_eq; - - use super::*; - - #[test] - fn transactions_subjects_from_transaction() { - let mock_tx = MockTransaction::build(); - let subject = TransactionsSubject::from(&mock_tx); - assert!(subject.block_height.is_none()); - assert!(subject.index.is_none()); - assert!(subject.status.is_none()); - assert!(subject.kind.is_some()); - assert_eq!(subject.tx_id.unwrap(), mock_tx.to_owned().id); - } -} diff --git a/crates/fuel-streams-core/src/types.rs b/crates/fuel-streams-core/src/types.rs deleted file mode 100644 index 9d1c880f..00000000 --- a/crates/fuel-streams-core/src/types.rs +++ /dev/null @@ -1,18 +0,0 @@ -pub use crate::{ - blocks::types::*, - fuel_core_types::*, - inputs::types::*, - logs::types::*, - nats::types::*, - outputs::types::*, - primitive_types::*, - receipts::types::*, - transactions::types::*, - utxos::types::*, -}; - -// ------------------------------------------------------------------------ -// General -// ------------------------------------------------------------------------ -pub type BoxedError = Box; -pub type BoxedResult = Result; diff --git a/crates/fuel-streams-core/src/utxos/mod.rs b/crates/fuel-streams-core/src/utxos/mod.rs deleted file mode 100644 index f52e7414..00000000 --- a/crates/fuel-streams-core/src/utxos/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -pub mod subjects; -pub mod types; - -pub use subjects::*; - -use super::types::*; -use crate::{DataEncoder, StreamError, Streamable}; - -impl DataEncoder for Utxo { - type Err = StreamError; -} -impl Streamable for Utxo { - const NAME: &'static str = "utxos"; - const WILDCARD_LIST: &'static [&'static str] = &[UtxosSubject::WILDCARD]; -} diff --git a/crates/fuel-streams-core/src/utxos/subjects.rs b/crates/fuel-streams-core/src/utxos/subjects.rs deleted file mode 100644 index eec48b98..00000000 --- a/crates/fuel-streams-core/src/utxos/subjects.rs +++ /dev/null @@ -1,112 +0,0 @@ -use fuel_streams_macros::subject::{IntoSubject, Subject}; - -use crate::types::*; - -/// Represents a subject for utxos messages in the Fuel ecosystem. -/// -/// This struct is used to create and parse subjects related to utxos messages, -/// which can be used for subscribing to or publishing events about utxos messages. -/// -/// # Examples -/// -/// Creating and parsing a subject: -/// -/// ``` -/// # use fuel_streams_core::utxos::subjects::UtxosSubject; -/// # use fuel_streams_core::types::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = UtxosSubject { -/// utxo_id: Some(HexData::zeroed()), -/// utxo_type: Some(UtxoType::Message), -/// }; -/// assert_eq!( -/// subject.parse(), -/// "utxos.message.0x0000000000000000000000000000000000000000000000000000000000000000" -/// ); -/// ``` -/// -/// All utxos messages wildcard: -/// -/// ``` -/// # use fuel_streams_core::utxos::subjects::UtxosSubject; -/// # use fuel_streams_macros::subject::*; -/// assert_eq!(UtxosSubject::WILDCARD, "utxos.>"); -/// ``` -/// -/// Creating a subject query using the `wildcard` method: -/// -/// ``` -/// # use fuel_streams_core::utxos::subjects::UtxosSubject; -/// # use fuel_streams_core::types::*; -/// # use fuel_streams_macros::subject::*; -/// let wildcard = UtxosSubject::wildcard( -/// Some(HexData::zeroed()), -/// None, -/// ); -/// assert_eq!(wildcard, "utxos.*.0x0000000000000000000000000000000000000000000000000000000000000000"); -/// ``` -/// -/// Using the builder pattern: -/// -/// ``` -/// # use fuel_streams_core::utxos::subjects::UtxosSubject; -/// # use fuel_streams_core::types::*; -/// # use fuel_streams_macros::subject::*; -/// let subject = UtxosSubject::new() -/// .with_utxo_id(Some(HexData::zeroed())) -/// .with_utxo_type(Some(UtxoType::Message)); -/// assert_eq!(subject.parse(), "utxos.message.0x0000000000000000000000000000000000000000000000000000000000000000"); -/// ``` - -#[derive(Subject, Debug, Clone, Default)] -#[subject_wildcard = "utxos.>"] -#[subject_format = "utxos.{utxo_type}.{utxo_id}"] -pub struct UtxosSubject { - pub utxo_id: Option, - pub utxo_type: Option, -} - -#[cfg(test)] -mod tests { - use fuel_streams_macros::subject::SubjectBuildable; - - use super::*; - - #[test] - fn test_utxos_subject_wildcard() { - assert_eq!(UtxosSubject::WILDCARD, "utxos.>"); - } - - #[test] - fn test_utxos_message_subject_creation() { - let utxo_subject = UtxosSubject::new() - .with_utxo_id(Some(HexData::zeroed())) - .with_utxo_type(Some(UtxoType::Message)); - assert_eq!( - utxo_subject.to_string(), - "utxos.message.0x0000000000000000000000000000000000000000000000000000000000000000" - ); - } - - #[test] - fn test_utxos_coin_subject_creation() { - let utxo_subject = UtxosSubject::new() - .with_utxo_id(Some(HexData::zeroed())) - .with_utxo_type(Some(UtxoType::Coin)); - assert_eq!( - utxo_subject.to_string(), - "utxos.coin.0x0000000000000000000000000000000000000000000000000000000000000000" - ); - } - - #[test] - fn test_utxos_contract_subject_creation() { - let utxo_subject = UtxosSubject::new() - .with_utxo_id(Some(HexData::zeroed())) - .with_utxo_type(Some(UtxoType::Contract)); - assert_eq!( - utxo_subject.to_string(), - "utxos.contract.0x0000000000000000000000000000000000000000000000000000000000000000" - ); - } -} diff --git a/crates/fuel-streams-core/src/utxos/types.rs b/crates/fuel-streams-core/src/utxos/types.rs deleted file mode 100644 index 4c7dcac7..00000000 --- a/crates/fuel-streams-core/src/utxos/types.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::prelude::*; - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Utxo { - pub utxo_id: UtxoId, - pub sender: Option
, - pub recipient: Option
, - pub nonce: Option, - pub data: Option, - pub amount: Option, - pub tx_id: TxId, -} - -#[derive(Debug, Clone, Default)] -pub enum UtxoType { - Contract, - Coin, - #[default] - Message, -} - -impl std::fmt::Display for UtxoType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let value: &'static str = match self { - UtxoType::Contract => "contract", - UtxoType::Coin => "coin", - UtxoType::Message => "message", - }; - write!(f, "{value}") - } -} diff --git a/crates/fuel-streams-domains/Cargo.toml b/crates/fuel-streams-domains/Cargo.toml new file mode 100644 index 00000000..40af6dae --- /dev/null +++ b/crates/fuel-streams-domains/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "fuel-streams-domains" +description = "Domains definitions for fuel streams" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +async-trait = { workspace = true } +fuel-core = { workspace = true, default-features = false, features = [ + "p2p", + "relayer", + "rocksdb", + "test-helpers", +] } +fuel-core-types = { workspace = true, default-features = false, features = ["std", "serde"] } +fuel-streams-macros = { workspace = true } +fuel-streams-store = { workspace = true } +fuel-streams-types = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sqlx = { workspace = true, features = ["runtime-tokio", "postgres", "tls-native-tls", "macros"] } +thiserror = { workspace = true } + +[dev-dependencies] +test-case = { workspace = true } + +[features] +default = [] +test-helpers = [] diff --git a/crates/fuel-streams-domains/src/blocks/db_item.rs b/crates/fuel-streams-domains/src/blocks/db_item.rs new file mode 100644 index 00000000..267a2b33 --- /dev/null +++ b/crates/fuel-streams-domains/src/blocks/db_item.rs @@ -0,0 +1,70 @@ +use std::cmp::Ordering; + +use fuel_streams_store::{ + db::{DbError, DbItem}, + record::{DataEncoder, RecordEntity, RecordPacket, RecordPacketError}, +}; +use serde::{Deserialize, Serialize}; + +use super::Block; +use crate::Subjects; + +#[derive( + Debug, Clone, Serialize, Deserialize, PartialEq, Eq, sqlx::FromRow, +)] +pub struct BlockDbItem { + pub subject: String, + pub value: Vec, + pub block_height: i64, + pub producer_address: String, +} + +impl DataEncoder for BlockDbItem { + type Err = DbError; +} + +impl DbItem for BlockDbItem { + fn entity(&self) -> &RecordEntity { + &RecordEntity::Block + } + + fn encoded_value(&self) -> &[u8] { + &self.value + } + + fn subject_str(&self) -> String { + self.subject.clone() + } +} + +impl PartialOrd for BlockDbItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for BlockDbItem { + fn cmp(&self, other: &Self) -> Ordering { + self.block_height.cmp(&other.block_height) + } +} + +impl TryFrom<&RecordPacket> for BlockDbItem { + type Error = RecordPacketError; + fn try_from(packet: &RecordPacket) -> Result { + let record = packet.record.as_ref(); + let subject: Subjects = packet + .try_into() + .map_err(|_| RecordPacketError::SubjectMismatch)?; + + match subject { + Subjects::Block(_) => Ok(BlockDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode block"), + block_height: record.height.clone().into(), + producer_address: record.producer.to_string(), + }), + _ => Err(RecordPacketError::SubjectMismatch), + } + } +} diff --git a/crates/fuel-streams-domains/src/blocks/mod.rs b/crates/fuel-streams-domains/src/blocks/mod.rs new file mode 100644 index 00000000..5f5af414 --- /dev/null +++ b/crates/fuel-streams-domains/src/blocks/mod.rs @@ -0,0 +1,8 @@ +mod db_item; +mod record_impl; +pub mod subjects; +pub mod types; + +pub use db_item::*; +pub use subjects::*; +pub use types::*; diff --git a/crates/fuel-streams-domains/src/blocks/record_impl.rs b/crates/fuel-streams-domains/src/blocks/record_impl.rs new file mode 100644 index 00000000..9ad58c63 --- /dev/null +++ b/crates/fuel-streams-domains/src/blocks/record_impl.rs @@ -0,0 +1,43 @@ +use async_trait::async_trait; +use fuel_streams_store::{ + db::{Db, DbError, DbResult}, + record::{DataEncoder, Record, RecordEntity, RecordPacket}, +}; + +use super::{Block, BlockDbItem}; + +impl DataEncoder for Block { + type Err = DbError; +} + +#[async_trait] +impl Record for Block { + type DbItem = BlockDbItem; + + const ENTITY: RecordEntity = RecordEntity::Block; + const ORDER_PROPS: &'static [&'static str] = &["block_height"]; + + async fn insert( + &self, + db: &Db, + packet: &RecordPacket, + ) -> DbResult { + let db_item = BlockDbItem::try_from(packet)?; + let record = sqlx::query_as::<_, Self::DbItem>( + r#" + INSERT INTO blocks (subject, producer_address, block_height, value) + VALUES ($1, $2, $3, $4) + RETURNING subject, producer_address, block_height, value + "#, + ) + .bind(db_item.subject) + .bind(db_item.producer_address) + .bind(db_item.block_height) + .bind(db_item.value) + .fetch_one(&db.pool) + .await + .map_err(DbError::Insert)?; + + Ok(record) + } +} diff --git a/crates/fuel-streams-domains/src/blocks/subjects.rs b/crates/fuel-streams-domains/src/blocks/subjects.rs new file mode 100644 index 00000000..9de2dbf2 --- /dev/null +++ b/crates/fuel-streams-domains/src/blocks/subjects.rs @@ -0,0 +1,20 @@ +use fuel_streams_macros::subject::*; +use fuel_streams_types::*; +use serde::{Deserialize, Serialize}; + +use super::types::*; + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "blocks"] +#[subject_wildcard = "blocks.>"] +#[subject_format = "blocks.{producer}.{block_height}"] +pub struct BlocksSubject { + pub producer: Option
, + pub block_height: Option, +} + +impl From<&Block> for BlocksSubject { + fn from(block: &Block) -> Self { + BlocksSubject::new().with_block_height(Some(block.height.clone())) + } +} diff --git a/crates/fuel-streams-core/src/blocks/types.rs b/crates/fuel-streams-domains/src/blocks/types.rs similarity index 85% rename from crates/fuel-streams-core/src/blocks/types.rs rename to crates/fuel-streams-domains/src/blocks/types.rs index de0d1d3f..1e0fcb44 100644 --- a/crates/fuel-streams-core/src/blocks/types.rs +++ b/crates/fuel-streams-domains/src/blocks/types.rs @@ -1,4 +1,7 @@ -use crate::types::*; +use std::str::FromStr; + +use fuel_streams_types::{fuel_core::*, primitives::*}; +use serde::{Deserialize, Serialize}; // Block type #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -6,17 +9,19 @@ use crate::types::*; pub struct Block { pub consensus: Consensus, pub header: BlockHeader, - pub height: u32, + pub height: BlockHeight, pub id: BlockId, - pub transaction_ids: Vec, + pub transaction_ids: Vec, pub version: BlockVersion, + pub producer: Address, } impl Block { pub fn new( block: &fuel_core_types::blockchain::block::Block, consensus: Consensus, - transaction_ids: Vec, + transaction_ids: Vec, + producer: Address, ) -> Self { let header: BlockHeader = block.header().into(); let height = header.height; @@ -30,15 +35,16 @@ impl Block { Self { consensus, header: header.to_owned(), - height, + height: height.into(), id: header.id, transaction_ids, version, + producer, } } } -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct BlockHeight(String); impl From for BlockHeight { @@ -50,7 +56,19 @@ impl From for BlockHeight { impl From for BlockHeight { fn from(value: u32) -> Self { - BlockHeight::from(FuelCoreBlockHeight::from(value)) + BlockHeight(value.to_string()) + } +} + +impl From for u32 { + fn from(value: BlockHeight) -> Self { + value.0.parse::().unwrap() + } +} + +impl From for i64 { + fn from(value: BlockHeight) -> Self { + value.0.parse::().unwrap() } } @@ -60,6 +78,14 @@ impl std::fmt::Display for BlockHeight { } } +impl FromStr for BlockHeight { + type Err = String; + fn from_str(s: &str) -> Result { + let height = s.parse::().map_err(|_| "Invalid block height")?; + Ok(BlockHeight(height.to_string())) + } +} + // Consensus enum #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] @@ -206,6 +232,6 @@ impl MockBlock { .collect::>(); *block.transactions_mut() = txs; - Block::new(&block, Consensus::default(), Vec::new()) + Block::new(&block, Consensus::default(), Vec::new(), Address::default()) } } diff --git a/crates/fuel-streams-domains/src/inputs/db_item.rs b/crates/fuel-streams-domains/src/inputs/db_item.rs new file mode 100644 index 00000000..1e12f822 --- /dev/null +++ b/crates/fuel-streams-domains/src/inputs/db_item.rs @@ -0,0 +1,120 @@ +use std::cmp::Ordering; + +use fuel_streams_store::{ + db::{DbError, DbItem}, + record::{DataEncoder, RecordEntity, RecordPacket, RecordPacketError}, +}; +use serde::{Deserialize, Serialize}; + +use super::Input; +use crate::Subjects; + +#[derive( + Debug, Clone, Serialize, Deserialize, PartialEq, Eq, sqlx::FromRow, +)] +pub struct InputDbItem { + pub subject: String, + pub value: Vec, + pub block_height: i64, + pub tx_id: String, + pub tx_index: i64, + pub input_index: i64, + pub input_type: String, + pub owner_id: Option, // for coin inputs + pub asset_id: Option, // for coin inputs + pub contract_id: Option, // for contract inputs + pub sender: Option, // for message inputs + pub recipient: Option, // for message inputs +} + +impl DataEncoder for InputDbItem { + type Err = DbError; +} + +impl DbItem for InputDbItem { + fn entity(&self) -> &RecordEntity { + &RecordEntity::Input + } + + fn encoded_value(&self) -> &[u8] { + &self.value + } + + fn subject_str(&self) -> String { + self.subject.clone() + } +} + +impl PartialOrd for InputDbItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for InputDbItem { + fn cmp(&self, other: &Self) -> Ordering { + // Order by block height first + self.block_height + .cmp(&other.block_height) + // Then by transaction index within the block + .then(self.tx_index.cmp(&other.tx_index)) + // Finally by input index within the transaction + .then(self.input_index.cmp(&other.input_index)) + } +} + +impl TryFrom<&RecordPacket> for InputDbItem { + type Error = RecordPacketError; + fn try_from(packet: &RecordPacket) -> Result { + let record = packet.record.as_ref(); + let subject: Subjects = packet + .try_into() + .map_err(|_| RecordPacketError::SubjectMismatch)?; + + match subject { + Subjects::InputsCoin(subject) => Ok(InputDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode input"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + input_index: subject.input_index.unwrap() as i64, + input_type: "coin".to_string(), + owner_id: Some(subject.owner_id.unwrap().to_string()), + asset_id: Some(subject.asset_id.unwrap().to_string()), + contract_id: None, + sender: None, + recipient: None, + }), + Subjects::InputsContract(subject) => Ok(InputDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode input"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + input_index: subject.input_index.unwrap() as i64, + input_type: "contract".to_string(), + owner_id: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sender: None, + recipient: None, + }), + Subjects::InputsMessage(subject) => Ok(InputDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode input"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + input_index: subject.input_index.unwrap() as i64, + input_type: "message".to_string(), + owner_id: None, + asset_id: None, + contract_id: None, + sender: Some(subject.sender.unwrap().to_string()), + recipient: Some(subject.recipient.unwrap().to_string()), + }), + _ => Err(RecordPacketError::SubjectMismatch), + } + } +} diff --git a/crates/fuel-streams-domains/src/inputs/mod.rs b/crates/fuel-streams-domains/src/inputs/mod.rs new file mode 100644 index 00000000..5f5af414 --- /dev/null +++ b/crates/fuel-streams-domains/src/inputs/mod.rs @@ -0,0 +1,8 @@ +mod db_item; +mod record_impl; +pub mod subjects; +pub mod types; + +pub use db_item::*; +pub use subjects::*; +pub use types::*; diff --git a/crates/fuel-streams-domains/src/inputs/record_impl.rs b/crates/fuel-streams-domains/src/inputs/record_impl.rs new file mode 100644 index 00000000..6c7131c8 --- /dev/null +++ b/crates/fuel-streams-domains/src/inputs/record_impl.rs @@ -0,0 +1,58 @@ +use async_trait::async_trait; +use fuel_streams_store::{ + db::{Db, DbError, DbResult}, + record::{DataEncoder, Record, RecordEntity, RecordPacket}, +}; + +use super::{Input, InputDbItem}; + +impl DataEncoder for Input { + type Err = DbError; +} + +#[async_trait] +impl Record for Input { + type DbItem = InputDbItem; + + const ENTITY: RecordEntity = RecordEntity::Input; + const ORDER_PROPS: &'static [&'static str] = + &["block_height", "tx_index", "input_index"]; + + async fn insert( + &self, + db: &Db, + packet: &RecordPacket, + ) -> DbResult { + let db_item = InputDbItem::try_from(packet)?; + let record = sqlx::query_as::<_, Self::DbItem>( + r#" + INSERT INTO inputs ( + subject, value, block_height, tx_id, tx_index, + input_index, input_type, owner_id, asset_id, + contract_id, sender, recipient + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + RETURNING subject, value, block_height, tx_id, tx_index, + input_index, input_type, owner_id, asset_id, + contract_id, sender, recipient + "#, + ) + .bind(db_item.subject) + .bind(db_item.value) + .bind(db_item.block_height) + .bind(db_item.tx_id) + .bind(db_item.tx_index) + .bind(db_item.input_index) + .bind(db_item.input_type) + .bind(db_item.owner_id) + .bind(db_item.asset_id) + .bind(db_item.contract_id) + .bind(db_item.sender) + .bind(db_item.recipient) + .fetch_one(&db.pool) + .await + .map_err(DbError::Insert)?; + + Ok(record) + } +} diff --git a/crates/fuel-streams-domains/src/inputs/subjects.rs b/crates/fuel-streams-domains/src/inputs/subjects.rs new file mode 100644 index 00000000..64a69381 --- /dev/null +++ b/crates/fuel-streams-domains/src/inputs/subjects.rs @@ -0,0 +1,43 @@ +use fuel_streams_macros::subject::*; +use fuel_streams_types::*; +use serde::{Deserialize, Serialize}; + +use crate::blocks::types::*; + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "inputs_coin"] +#[subject_wildcard = "inputs.>"] +#[subject_format = "inputs.coin.{block_height}.{tx_id}.{tx_index}.{input_index}.{owner_id}.{asset_id}"] +pub struct InputsCoinSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub input_index: Option, + pub owner_id: Option
, + pub asset_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "inputs_contract"] +#[subject_wildcard = "inputs.>"] +#[subject_format = "inputs.contract.{block_height}.{tx_id}.{tx_index}.{input_index}.{contract_id}"] +pub struct InputsContractSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub input_index: Option, + pub contract_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "inputs_message"] +#[subject_wildcard = "inputs.>"] +#[subject_format = "inputs.message.{block_height}.{tx_id}.{tx_index}.{input_index}.{sender}.{recipient}"] +pub struct InputsMessageSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub input_index: Option, + pub sender: Option
, + pub recipient: Option
, +} diff --git a/crates/fuel-streams-core/src/inputs/types.rs b/crates/fuel-streams-domains/src/inputs/types.rs similarity index 97% rename from crates/fuel-streams-core/src/inputs/types.rs rename to crates/fuel-streams-domains/src/inputs/types.rs index 1781e773..e10a5385 100644 --- a/crates/fuel-streams-core/src/inputs/types.rs +++ b/crates/fuel-streams-domains/src/inputs/types.rs @@ -1,8 +1,6 @@ -use fuel_core_types::fuel_crypto; +use fuel_streams_types::{fuel_core::*, primitives::*}; +use serde::{Deserialize, Serialize}; -use crate::types::*; - -// Input enum #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] pub enum Input { @@ -180,7 +178,7 @@ pub struct InputMessage { impl InputMessage { pub fn compute_message_id(&self) -> MessageId { - let hasher = fuel_crypto::Hasher::default() + let hasher = fuel_core_types::fuel_crypto::Hasher::default() .chain(self.sender.as_ref()) .chain(self.recipient.as_ref()) .chain(self.nonce.as_ref()) diff --git a/crates/fuel-streams-domains/src/lib.rs b/crates/fuel-streams-domains/src/lib.rs new file mode 100644 index 00000000..1274bbed --- /dev/null +++ b/crates/fuel-streams-domains/src/lib.rs @@ -0,0 +1,9 @@ +pub mod blocks; +pub mod inputs; +pub mod outputs; +pub mod receipts; +pub mod transactions; +pub mod utxos; + +pub use subjects::*; +mod subjects; diff --git a/crates/fuel-streams-domains/src/outputs/db_item.rs b/crates/fuel-streams-domains/src/outputs/db_item.rs new file mode 100644 index 00000000..826f75d7 --- /dev/null +++ b/crates/fuel-streams-domains/src/outputs/db_item.rs @@ -0,0 +1,136 @@ +use std::cmp::Ordering; + +use fuel_streams_store::{ + db::{DbError, DbItem}, + record::{DataEncoder, RecordEntity, RecordPacket, RecordPacketError}, +}; +use serde::{Deserialize, Serialize}; + +use super::Output; +use crate::Subjects; + +#[derive( + Debug, Clone, Serialize, Deserialize, PartialEq, Eq, sqlx::FromRow, +)] +pub struct OutputDbItem { + pub subject: String, + pub value: Vec, + pub block_height: i64, + pub tx_id: String, + pub tx_index: i64, + pub output_index: i64, + pub output_type: String, + pub to_address: Option, // for coin, change, and variable outputs + pub asset_id: Option, // for coin, change, and variable outputs + pub contract_id: Option, /* for contract and contract_created outputs */ +} + +impl DataEncoder for OutputDbItem { + type Err = DbError; +} + +impl DbItem for OutputDbItem { + fn entity(&self) -> &RecordEntity { + &RecordEntity::Output + } + + fn encoded_value(&self) -> &[u8] { + &self.value + } + + fn subject_str(&self) -> String { + self.subject.clone() + } +} + +impl PartialOrd for OutputDbItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for OutputDbItem { + fn cmp(&self, other: &Self) -> Ordering { + // Order by block height first + self.block_height + .cmp(&other.block_height) + // Then by transaction index within the block + .then(self.tx_index.cmp(&other.tx_index)) + // Finally by output index within the transaction + .then(self.output_index.cmp(&other.output_index)) + } +} + +impl TryFrom<&RecordPacket> for OutputDbItem { + type Error = RecordPacketError; + fn try_from(packet: &RecordPacket) -> Result { + let record = packet.record.as_ref(); + let subject: Subjects = packet + .try_into() + .map_err(|_| RecordPacketError::SubjectMismatch)?; + + match subject { + Subjects::OutputsCoin(subject) => Ok(OutputDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode output"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + output_index: subject.output_index.unwrap() as i64, + output_type: "coin".to_string(), + to_address: Some(subject.to_address.unwrap().to_string()), + asset_id: Some(subject.asset_id.unwrap().to_string()), + contract_id: None, + }), + Subjects::OutputsContract(subject) => Ok(OutputDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode output"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + output_index: subject.output_index.unwrap() as i64, + output_type: "contract".to_string(), + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + }), + Subjects::OutputsChange(subject) => Ok(OutputDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode output"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + output_index: subject.output_index.unwrap() as i64, + output_type: "change".to_string(), + to_address: Some(subject.to_address.unwrap().to_string()), + asset_id: Some(subject.asset_id.unwrap().to_string()), + contract_id: None, + }), + Subjects::OutputsVariable(subject) => Ok(OutputDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode output"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + output_index: subject.output_index.unwrap() as i64, + output_type: "variable".to_string(), + to_address: Some(subject.to_address.unwrap().to_string()), + asset_id: Some(subject.asset_id.unwrap().to_string()), + contract_id: None, + }), + Subjects::OutputsContractCreated(subject) => Ok(OutputDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode output"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + output_index: subject.output_index.unwrap() as i64, + output_type: "contract_created".to_string(), + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + }), + _ => Err(RecordPacketError::SubjectMismatch), + } + } +} diff --git a/crates/fuel-streams-domains/src/outputs/mod.rs b/crates/fuel-streams-domains/src/outputs/mod.rs new file mode 100644 index 00000000..5f5af414 --- /dev/null +++ b/crates/fuel-streams-domains/src/outputs/mod.rs @@ -0,0 +1,8 @@ +mod db_item; +mod record_impl; +pub mod subjects; +pub mod types; + +pub use db_item::*; +pub use subjects::*; +pub use types::*; diff --git a/crates/fuel-streams-domains/src/outputs/record_impl.rs b/crates/fuel-streams-domains/src/outputs/record_impl.rs new file mode 100644 index 00000000..476a7022 --- /dev/null +++ b/crates/fuel-streams-domains/src/outputs/record_impl.rs @@ -0,0 +1,54 @@ +use async_trait::async_trait; +use fuel_streams_store::{ + db::{Db, DbError, DbResult}, + record::{DataEncoder, Record, RecordEntity, RecordPacket}, +}; + +use super::{Output, OutputDbItem}; + +impl DataEncoder for Output { + type Err = DbError; +} + +#[async_trait] +impl Record for Output { + type DbItem = OutputDbItem; + + const ENTITY: RecordEntity = RecordEntity::Output; + const ORDER_PROPS: &'static [&'static str] = + &["block_height", "tx_index", "output_index"]; + + async fn insert( + &self, + db: &Db, + packet: &RecordPacket, + ) -> DbResult { + let db_item = OutputDbItem::try_from(packet)?; + let record = sqlx::query_as::<_, Self::DbItem>( + r#" + INSERT INTO outputs ( + subject, value, block_height, tx_id, tx_index, + output_index, output_type, to_address, asset_id, contract_id + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + RETURNING subject, value, block_height, tx_id, tx_index, + output_index, output_type, to_address, asset_id, contract_id + "#, + ) + .bind(db_item.subject) + .bind(db_item.value) + .bind(db_item.block_height) + .bind(db_item.tx_id) + .bind(db_item.tx_index) + .bind(db_item.output_index) + .bind(db_item.output_type) + .bind(db_item.to_address) + .bind(db_item.asset_id) + .bind(db_item.contract_id) + .fetch_one(&db.pool) + .await + .map_err(DbError::Insert)?; + + Ok(record) + } +} diff --git a/crates/fuel-streams-domains/src/outputs/subjects.rs b/crates/fuel-streams-domains/src/outputs/subjects.rs new file mode 100644 index 00000000..f05247a8 --- /dev/null +++ b/crates/fuel-streams-domains/src/outputs/subjects.rs @@ -0,0 +1,68 @@ +use fuel_streams_macros::subject::*; +use fuel_streams_types::*; +use serde::{Deserialize, Serialize}; + +use crate::blocks::types::*; + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "outputs_coin"] +#[subject_wildcard = "outputs.>"] +#[subject_format = "outputs.coin.{block_height}.{tx_id}.{tx_index}.{output_index}.{to_address}.{asset_id}"] +pub struct OutputsCoinSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub output_index: Option, + pub to_address: Option
, + pub asset_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "outputs_contract"] +#[subject_wildcard = "outputs.>"] +#[subject_format = "outputs.contract.{block_height}.{tx_id}.{tx_index}.{output_index}.{contract_id}"] +pub struct OutputsContractSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub output_index: Option, + pub contract_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "outputs_change"] +#[subject_wildcard = "outputs.>"] +#[subject_format = "outputs.change.{block_height}.{tx_id}.{tx_index}.{output_index}.{to_address}.{asset_id}"] +pub struct OutputsChangeSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub output_index: Option, + pub to_address: Option
, + pub asset_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "outputs_variable"] +#[subject_wildcard = "outputs.>"] +#[subject_format = "outputs.variable.{block_height}.{tx_id}.{tx_index}.{output_index}.{to_address}.{asset_id}"] +pub struct OutputsVariableSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub output_index: Option, + pub to_address: Option
, + pub asset_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "outputs_contract_created"] +#[subject_wildcard = "outputs.>"] +#[subject_format = "outputs.contract_created.{block_height}.{tx_id}.{tx_index}.{output_index}.{contract_id}"] +pub struct OutputsContractCreatedSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub output_index: Option, + pub contract_id: Option, +} diff --git a/crates/fuel-streams-core/src/outputs/types.rs b/crates/fuel-streams-domains/src/outputs/types.rs similarity index 96% rename from crates/fuel-streams-core/src/outputs/types.rs rename to crates/fuel-streams-domains/src/outputs/types.rs index ddedd9a0..ffed95f7 100644 --- a/crates/fuel-streams-core/src/outputs/types.rs +++ b/crates/fuel-streams-domains/src/outputs/types.rs @@ -1,4 +1,5 @@ -use crate::types::*; +use fuel_streams_types::{fuel_core::*, primitives::*}; +use serde::{Deserialize, Serialize}; // Output enum #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/crates/fuel-streams-domains/src/receipts/db_item.rs b/crates/fuel-streams-domains/src/receipts/db_item.rs new file mode 100644 index 00000000..e2f00dc4 --- /dev/null +++ b/crates/fuel-streams-domains/src/receipts/db_item.rs @@ -0,0 +1,316 @@ +use std::cmp::Ordering; + +use fuel_streams_store::{ + db::{DbError, DbItem}, + record::{DataEncoder, RecordEntity, RecordPacket, RecordPacketError}, +}; +use serde::{Deserialize, Serialize}; + +use super::Receipt; +use crate::Subjects; + +#[derive( + Debug, Clone, Serialize, Deserialize, PartialEq, Eq, sqlx::FromRow, +)] +pub struct ReceiptDbItem { + pub subject: String, + pub value: Vec, + pub block_height: i64, + pub tx_id: String, + pub tx_index: i64, + pub receipt_index: i64, + pub receipt_type: String, + pub from_contract_id: Option, // for call/transfer/transfer_out + pub to_contract_id: Option, // for call/transfer + pub to_address: Option, // for transfer_out + pub asset_id: Option, // for call/transfer/transfer_out + pub contract_id: Option, /* for return/return_data/panic/revert/log/log_data/mint/burn */ + pub sub_id: Option, // for mint/burn + pub sender_address: Option, // for message_out + pub recipient_address: Option, // for message_out +} + +impl DataEncoder for ReceiptDbItem { + type Err = DbError; +} + +impl DbItem for ReceiptDbItem { + fn entity(&self) -> &RecordEntity { + &RecordEntity::Receipt + } + + fn encoded_value(&self) -> &[u8] { + &self.value + } + + fn subject_str(&self) -> String { + self.subject.clone() + } +} + +impl PartialOrd for ReceiptDbItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ReceiptDbItem { + fn cmp(&self, other: &Self) -> Ordering { + // Order by block height first + self.block_height + .cmp(&other.block_height) + // Then by transaction index within the block + .then(self.tx_index.cmp(&other.tx_index)) + // Finally by receipt index within the transaction + .then(self.receipt_index.cmp(&other.receipt_index)) + } +} + +impl TryFrom<&RecordPacket> for ReceiptDbItem { + type Error = RecordPacketError; + fn try_from(packet: &RecordPacket) -> Result { + let record = packet.record.as_ref(); + let subject: Subjects = packet + .try_into() + .map_err(|_| RecordPacketError::SubjectMismatch)?; + + match subject { + Subjects::ReceiptsCall(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "call".to_string(), + from_contract_id: Some( + subject.from_contract_id.unwrap().to_string(), + ), + to_contract_id: Some( + subject.to_contract_id.unwrap().to_string(), + ), + asset_id: Some(subject.asset_id.unwrap().to_string()), + to_address: None, + contract_id: None, + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsReturn(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "return".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsReturnData(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "return_data".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsPanic(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "panic".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsRevert(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "revert".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsLog(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "log".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsLogData(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "log_data".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsTransfer(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "transfer".to_string(), + from_contract_id: Some( + subject.from_contract_id.unwrap().to_string(), + ), + to_contract_id: Some( + subject.to_contract_id.unwrap().to_string(), + ), + asset_id: Some(subject.asset_id.unwrap().to_string()), + contract_id: None, + sub_id: None, + sender_address: None, + recipient_address: None, + to_address: None, + }), + Subjects::ReceiptsTransferOut(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "transfer_out".to_string(), + from_contract_id: Some( + subject.from_contract_id.unwrap().to_string(), + ), + to_contract_id: None, + to_address: Some(subject.to_address.unwrap().to_string()), + asset_id: Some(subject.asset_id.unwrap().to_string()), + contract_id: None, + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsScriptResult(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "script_result".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: None, + sub_id: None, + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsMessageOut(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "message_out".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: None, + sub_id: None, + sender_address: Some( + subject.sender_address.unwrap().to_string(), + ), + recipient_address: Some( + subject.recipient_address.unwrap().to_string(), + ), + }), + Subjects::ReceiptsMint(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "mint".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sub_id: Some(subject.sub_id.unwrap().to_string()), + sender_address: None, + recipient_address: None, + }), + Subjects::ReceiptsBurn(subject) => Ok(ReceiptDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode receipt"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + receipt_index: subject.receipt_index.unwrap() as i64, + receipt_type: "burn".to_string(), + from_contract_id: None, + to_contract_id: None, + to_address: None, + asset_id: None, + contract_id: Some(subject.contract_id.unwrap().to_string()), + sub_id: Some(subject.sub_id.unwrap().to_string()), + sender_address: None, + recipient_address: None, + }), + _ => Err(RecordPacketError::SubjectMismatch), + } + } +} diff --git a/crates/fuel-streams-domains/src/receipts/mod.rs b/crates/fuel-streams-domains/src/receipts/mod.rs new file mode 100644 index 00000000..5f5af414 --- /dev/null +++ b/crates/fuel-streams-domains/src/receipts/mod.rs @@ -0,0 +1,8 @@ +mod db_item; +mod record_impl; +pub mod subjects; +pub mod types; + +pub use db_item::*; +pub use subjects::*; +pub use types::*; diff --git a/crates/fuel-streams-domains/src/receipts/record_impl.rs b/crates/fuel-streams-domains/src/receipts/record_impl.rs new file mode 100644 index 00000000..fba60b03 --- /dev/null +++ b/crates/fuel-streams-domains/src/receipts/record_impl.rs @@ -0,0 +1,60 @@ +use async_trait::async_trait; +use fuel_streams_store::{ + db::{Db, DbError, DbResult}, + record::{DataEncoder, Record, RecordEntity, RecordPacket}, +}; + +use super::{Receipt, ReceiptDbItem}; + +impl DataEncoder for Receipt { + type Err = DbError; +} + +#[async_trait] +impl Record for Receipt { + type DbItem = ReceiptDbItem; + + const ENTITY: RecordEntity = RecordEntity::Receipt; + const ORDER_PROPS: &'static [&'static str] = + &["block_height", "tx_index", "receipt_index"]; + + async fn insert( + &self, + db: &Db, + packet: &RecordPacket, + ) -> DbResult { + let db_item = ReceiptDbItem::try_from(packet)?; + let record = sqlx::query_as::<_, Self::DbItem>( + r#" + INSERT INTO receipts ( + subject, value, block_height, tx_id, tx_index, receipt_index, + receipt_type, from_contract_id, to_contract_id, + asset_id, contract_id, sub_id, sender_address, recipient_address + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + RETURNING subject, value, block_height, tx_id, tx_index, receipt_index, + receipt_type, from_contract_id, to_contract_id, + asset_id, contract_id, sub_id, sender_address, recipient_address + "#, + ) + .bind(db_item.subject) + .bind(db_item.value) + .bind(db_item.block_height) + .bind(db_item.tx_id) + .bind(db_item.tx_index) + .bind(db_item.receipt_index) + .bind(db_item.receipt_type) + .bind(db_item.from_contract_id) + .bind(db_item.to_contract_id) + .bind(db_item.asset_id) + .bind(db_item.contract_id) + .bind(db_item.sub_id) + .bind(db_item.sender_address) + .bind(db_item.recipient_address) + .fetch_one(&db.pool) + .await + .map_err(DbError::Insert)?; + + Ok(record) + } +} diff --git a/crates/fuel-streams-domains/src/receipts/subjects.rs b/crates/fuel-streams-domains/src/receipts/subjects.rs new file mode 100644 index 00000000..f2638872 --- /dev/null +++ b/crates/fuel-streams-domains/src/receipts/subjects.rs @@ -0,0 +1,169 @@ +use fuel_streams_macros::subject::*; +use fuel_streams_types::*; +use serde::{Deserialize, Serialize}; + +use crate::blocks::types::*; + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_call"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.call.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{from_contract_id}.{to_contract_id}.{asset_id}"] +pub struct ReceiptsCallSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub from_contract_id: Option, + pub to_contract_id: Option, + pub asset_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_return"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.return.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{contract_id}"] +pub struct ReceiptsReturnSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub contract_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_return_data"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.return_data.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{contract_id}"] +pub struct ReceiptsReturnDataSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub contract_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_panic"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.panic.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{contract_id}"] +pub struct ReceiptsPanicSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub contract_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_revert"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.revert.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{contract_id}"] +pub struct ReceiptsRevertSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub contract_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_log"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.log.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{contract_id}"] +pub struct ReceiptsLogSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub contract_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_log_data"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.log_data.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{contract_id}"] +pub struct ReceiptsLogDataSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub contract_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_transfer"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.transfer.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{from_contract_id}.{to_contract_id}.{asset_id}"] +pub struct ReceiptsTransferSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub from_contract_id: Option, + pub to_contract_id: Option, + pub asset_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_transfer_out"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.transfer_out.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{from_contract_id}.{to_address}.{asset_id}"] +pub struct ReceiptsTransferOutSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub from_contract_id: Option, + pub to_address: Option
, + pub asset_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_script_result"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.script_result.{block_height}.{tx_id}.{tx_index}.{receipt_index}"] +pub struct ReceiptsScriptResultSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_message_out"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.message_out.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{sender_address}.{recipient_address}"] +pub struct ReceiptsMessageOutSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub sender_address: Option
, + pub recipient_address: Option
, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_mint"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.mint.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{contract_id}.{sub_id}"] +pub struct ReceiptsMintSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub contract_id: Option, + pub sub_id: Option, +} + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "receipts_burn"] +#[subject_wildcard = "receipts.>"] +#[subject_format = "receipts.burn.{block_height}.{tx_id}.{tx_index}.{receipt_index}.{contract_id}.{sub_id}"] +pub struct ReceiptsBurnSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub receipt_index: Option, + pub contract_id: Option, + pub sub_id: Option, +} diff --git a/crates/fuel-streams-core/src/receipts/types.rs b/crates/fuel-streams-domains/src/receipts/types.rs similarity index 86% rename from crates/fuel-streams-core/src/receipts/types.rs rename to crates/fuel-streams-domains/src/receipts/types.rs index 2f9f5f45..da4a9ff6 100644 --- a/crates/fuel-streams-core/src/receipts/types.rs +++ b/crates/fuel-streams-domains/src/receipts/types.rs @@ -1,7 +1,5 @@ -use fuel_core_types::fuel_asm::Word; -use serde::{self, Deserialize, Serialize}; - -use crate::types::*; +use fuel_streams_types::{fuel_core::*, primitives::*}; +use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] @@ -27,33 +25,33 @@ pub enum Receipt { pub struct CallReceipt { pub id: ContractId, pub to: ContractId, - pub amount: Word, + pub amount: FuelCoreWord, pub asset_id: AssetId, - pub gas: Word, - pub param1: Word, - pub param2: Word, - pub pc: Word, - pub is: Word, + pub gas: FuelCoreWord, + pub param1: FuelCoreWord, + pub param2: FuelCoreWord, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ReturnReceipt { pub id: ContractId, - pub val: Word, - pub pc: Word, - pub is: Word, + pub val: FuelCoreWord, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ReturnDataReceipt { pub id: ContractId, - pub ptr: Word, - pub len: Word, + pub ptr: FuelCoreWord, + pub len: FuelCoreWord, pub digest: Bytes32, - pub pc: Word, - pub is: Word, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, pub data: Option>, } @@ -62,8 +60,8 @@ pub struct ReturnDataReceipt { pub struct PanicReceipt { pub id: ContractId, pub reason: PanicInstruction, - pub pc: Word, - pub is: Word, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, pub contract_id: Option, } @@ -71,34 +69,34 @@ pub struct PanicReceipt { #[serde(rename_all = "camelCase")] pub struct RevertReceipt { pub id: ContractId, - pub ra: Word, - pub pc: Word, - pub is: Word, + pub ra: FuelCoreWord, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct LogReceipt { pub id: ContractId, - pub ra: Word, - pub rb: Word, - pub rc: Word, - pub rd: Word, - pub pc: Word, - pub is: Word, + pub ra: FuelCoreWord, + pub rb: FuelCoreWord, + pub rc: FuelCoreWord, + pub rd: FuelCoreWord, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct LogDataReceipt { pub id: ContractId, - pub ra: Word, - pub rb: Word, - pub ptr: Word, - pub len: Word, + pub ra: FuelCoreWord, + pub rb: FuelCoreWord, + pub ptr: FuelCoreWord, + pub len: FuelCoreWord, pub digest: Bytes32, - pub pc: Word, - pub is: Word, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, pub data: Option>, } @@ -107,10 +105,10 @@ pub struct LogDataReceipt { pub struct TransferReceipt { pub id: ContractId, pub to: ContractId, - pub amount: Word, + pub amount: FuelCoreWord, pub asset_id: AssetId, - pub pc: Word, - pub is: Word, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] @@ -118,17 +116,17 @@ pub struct TransferReceipt { pub struct TransferOutReceipt { pub id: ContractId, pub to: Address, - pub amount: Word, + pub amount: FuelCoreWord, pub asset_id: AssetId, - pub pc: Word, - pub is: Word, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ScriptResultReceipt { pub result: ScriptExecutionResult, - pub gas_used: Word, + pub gas_used: FuelCoreWord, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] @@ -136,9 +134,9 @@ pub struct ScriptResultReceipt { pub struct MessageOutReceipt { pub sender: Address, pub recipient: Address, - pub amount: Word, + pub amount: FuelCoreWord, pub nonce: Nonce, - pub len: Word, + pub len: FuelCoreWord, pub digest: Bytes32, pub data: Option>, } @@ -148,9 +146,9 @@ pub struct MessageOutReceipt { pub struct MintReceipt { pub sub_id: Bytes32, pub contract_id: ContractId, - pub val: Word, - pub pc: Word, - pub is: Word, + pub val: FuelCoreWord, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] @@ -158,9 +156,9 @@ pub struct MintReceipt { pub struct BurnReceipt { pub sub_id: Bytes32, pub contract_id: ContractId, - pub val: Word, - pub pc: Word, - pub is: Word, + pub val: FuelCoreWord, + pub pc: FuelCoreWord, + pub is: FuelCoreWord, } impl From for Receipt { diff --git a/crates/fuel-streams-domains/src/subjects.rs b/crates/fuel-streams-domains/src/subjects.rs new file mode 100644 index 00000000..791c706a --- /dev/null +++ b/crates/fuel-streams-domains/src/subjects.rs @@ -0,0 +1,368 @@ +use std::{str::FromStr, sync::Arc}; + +use fuel_streams_macros::subject::{FromJsonString, IntoSubject}; +use fuel_streams_store::record::{RecordEntity, RecordPacket}; +use thiserror::Error; + +use crate::{ + blocks::*, + inputs::*, + outputs::*, + receipts::*, + transactions::*, + utxos::*, +}; + +#[derive(Error, Debug, PartialEq, Eq)] +pub enum SubjectPayloadError { + #[error("Unknown subject: {0}")] + UnknownSubject(String), + #[error(transparent)] + ParseError(#[from] fuel_streams_macros::subject::SubjectError), +} + +#[derive(Debug, Clone)] +pub enum Subjects { + Block(BlocksSubject), + InputsCoin(InputsCoinSubject), + InputsContract(InputsContractSubject), + InputsMessage(InputsMessageSubject), + OutputsCoin(OutputsCoinSubject), + OutputsContract(OutputsContractSubject), + OutputsChange(OutputsChangeSubject), + OutputsVariable(OutputsVariableSubject), + OutputsContractCreated(OutputsContractCreatedSubject), + ReceiptsCall(ReceiptsCallSubject), + ReceiptsReturn(ReceiptsReturnSubject), + ReceiptsReturnData(ReceiptsReturnDataSubject), + ReceiptsPanic(ReceiptsPanicSubject), + ReceiptsRevert(ReceiptsRevertSubject), + ReceiptsLog(ReceiptsLogSubject), + ReceiptsLogData(ReceiptsLogDataSubject), + ReceiptsTransfer(ReceiptsTransferSubject), + ReceiptsTransferOut(ReceiptsTransferOutSubject), + ReceiptsScriptResult(ReceiptsScriptResultSubject), + ReceiptsMessageOut(ReceiptsMessageOutSubject), + ReceiptsMint(ReceiptsMintSubject), + ReceiptsBurn(ReceiptsBurnSubject), + Transactions(TransactionsSubject), + Utxos(UtxosSubject), +} + +impl From for Arc { + fn from(subject: Subjects) -> Self { + match subject { + Subjects::Block(s) => s.dyn_arc(), + Subjects::InputsCoin(s) => s.dyn_arc(), + Subjects::InputsContract(s) => s.dyn_arc(), + Subjects::InputsMessage(s) => s.dyn_arc(), + Subjects::OutputsCoin(s) => s.dyn_arc(), + Subjects::OutputsContract(s) => s.dyn_arc(), + Subjects::OutputsChange(s) => s.dyn_arc(), + Subjects::OutputsVariable(s) => s.dyn_arc(), + Subjects::OutputsContractCreated(s) => s.dyn_arc(), + Subjects::ReceiptsCall(s) => s.dyn_arc(), + Subjects::ReceiptsReturn(s) => s.dyn_arc(), + Subjects::ReceiptsReturnData(s) => s.dyn_arc(), + Subjects::ReceiptsPanic(s) => s.dyn_arc(), + Subjects::ReceiptsRevert(s) => s.dyn_arc(), + Subjects::ReceiptsLog(s) => s.dyn_arc(), + Subjects::ReceiptsLogData(s) => s.dyn_arc(), + Subjects::ReceiptsTransfer(s) => s.dyn_arc(), + Subjects::ReceiptsTransferOut(s) => s.dyn_arc(), + Subjects::ReceiptsScriptResult(s) => s.dyn_arc(), + Subjects::ReceiptsMessageOut(s) => s.dyn_arc(), + Subjects::ReceiptsMint(s) => s.dyn_arc(), + Subjects::ReceiptsBurn(s) => s.dyn_arc(), + Subjects::Transactions(s) => s.dyn_arc(), + Subjects::Utxos(s) => s.dyn_arc(), + } + } +} + +#[derive(Debug, Clone)] +pub struct SubjectPayload { + pub subject: String, + pub params: serde_json::Value, + record_entity: RecordEntity, +} +impl SubjectPayload { + pub fn new( + subject: String, + params: serde_json::Value, + ) -> Result { + let record_entity = Self::record_from_subject_str(&subject)?; + Ok(Self { + record_entity, + subject, + params, + }) + } + + pub fn into_subject(&self) -> Arc { + let subject: Subjects = self.clone().try_into().unwrap(); + subject.into() + } + + pub fn record_entity(&self) -> &RecordEntity { + &self.record_entity + } + + pub fn parsed_subject(&self) -> String { + let subject_item = self.into_subject(); + subject_item.parse() + } + + fn record_from_subject_str( + subject: &str, + ) -> Result { + let subject = subject.to_lowercase(); + let subject_entity = if subject.contains("_") { + subject.split("_").next().unwrap() + } else { + &subject + }; + RecordEntity::from_str(subject_entity) + .map_err(|_| SubjectPayloadError::UnknownSubject(subject)) + } +} + +impl TryFrom for Subjects { + type Error = SubjectPayloadError; + fn try_from(json: SubjectPayload) -> Result { + match json.subject.as_str() { + BlocksSubject::ID => Ok(Subjects::Block(BlocksSubject::from_json( + &json.params.to_string(), + )?)), + InputsCoinSubject::ID => Ok(Subjects::InputsCoin( + InputsCoinSubject::from_json(&json.params.to_string())?, + )), + InputsContractSubject::ID => Ok(Subjects::InputsContract( + InputsContractSubject::from_json(&json.params.to_string())?, + )), + InputsMessageSubject::ID => Ok(Subjects::InputsMessage( + InputsMessageSubject::from_json(&json.params.to_string())?, + )), + OutputsCoinSubject::ID => Ok(Subjects::OutputsCoin( + OutputsCoinSubject::from_json(&json.params.to_string())?, + )), + OutputsContractSubject::ID => Ok(Subjects::OutputsContract( + OutputsContractSubject::from_json(&json.params.to_string())?, + )), + OutputsChangeSubject::ID => Ok(Subjects::OutputsChange( + OutputsChangeSubject::from_json(&json.params.to_string())?, + )), + OutputsVariableSubject::ID => Ok(Subjects::OutputsVariable( + OutputsVariableSubject::from_json(&json.params.to_string())?, + )), + OutputsContractCreatedSubject::ID => { + Ok(Subjects::OutputsContractCreated( + OutputsContractCreatedSubject::from_json( + &json.params.to_string(), + )?, + )) + } + ReceiptsCallSubject::ID => Ok(Subjects::ReceiptsCall( + ReceiptsCallSubject::from_json(&json.params.to_string())?, + )), + ReceiptsReturnSubject::ID => Ok(Subjects::ReceiptsReturn( + ReceiptsReturnSubject::from_json(&json.params.to_string())?, + )), + ReceiptsReturnDataSubject::ID => Ok(Subjects::ReceiptsReturnData( + ReceiptsReturnDataSubject::from_json(&json.params.to_string())?, + )), + ReceiptsPanicSubject::ID => Ok(Subjects::ReceiptsPanic( + ReceiptsPanicSubject::from_json(&json.params.to_string())?, + )), + ReceiptsRevertSubject::ID => Ok(Subjects::ReceiptsRevert( + ReceiptsRevertSubject::from_json(&json.params.to_string())?, + )), + ReceiptsLogSubject::ID => Ok(Subjects::ReceiptsLog( + ReceiptsLogSubject::from_json(&json.params.to_string())?, + )), + ReceiptsLogDataSubject::ID => Ok(Subjects::ReceiptsLogData( + ReceiptsLogDataSubject::from_json(&json.params.to_string())?, + )), + ReceiptsTransferSubject::ID => Ok(Subjects::ReceiptsTransfer( + ReceiptsTransferSubject::from_json(&json.params.to_string())?, + )), + ReceiptsTransferOutSubject::ID => { + Ok(Subjects::ReceiptsTransferOut( + ReceiptsTransferOutSubject::from_json( + &json.params.to_string(), + )?, + )) + } + ReceiptsScriptResultSubject::ID => { + Ok(Subjects::ReceiptsScriptResult( + ReceiptsScriptResultSubject::from_json( + &json.params.to_string(), + )?, + )) + } + ReceiptsMessageOutSubject::ID => Ok(Subjects::ReceiptsMessageOut( + ReceiptsMessageOutSubject::from_json(&json.params.to_string())?, + )), + ReceiptsMintSubject::ID => Ok(Subjects::ReceiptsMint( + ReceiptsMintSubject::from_json(&json.params.to_string())?, + )), + ReceiptsBurnSubject::ID => Ok(Subjects::ReceiptsBurn( + ReceiptsBurnSubject::from_json(&json.params.to_string())?, + )), + TransactionsSubject::ID => Ok(Subjects::Transactions( + TransactionsSubject::from_json(&json.params.to_string())?, + )), + UtxosSubject::ID => Ok(Subjects::Utxos(UtxosSubject::from_json( + &json.params.to_string(), + )?)), + _ => Err(SubjectPayloadError::UnknownSubject(json.subject)), + } + } +} + +macro_rules! impl_try_from_record_packet { + ($type:ty, [$(($subject_type:ty, $variant:ident)),+ $(,)?]) => { + impl TryFrom<&RecordPacket<$type>> for Subjects { + type Error = SubjectPayloadError; + fn try_from(packet: &RecordPacket<$type>) -> Result { + $( + if let Ok(subject) = packet.subject_matches::<$subject_type>() { + return Ok(Subjects::$variant(subject)); + } + )+ + Err(SubjectPayloadError::UnknownSubject(packet.subject_str())) + } + } + }; +} + +// Usage for each type +impl_try_from_record_packet!(Block, [(BlocksSubject, Block)]); + +impl_try_from_record_packet!(Input, [ + (InputsCoinSubject, InputsCoin), + (InputsContractSubject, InputsContract), + (InputsMessageSubject, InputsMessage) +]); + +impl_try_from_record_packet!(Output, [ + (OutputsCoinSubject, OutputsCoin), + (OutputsContractSubject, OutputsContract), + (OutputsChangeSubject, OutputsChange), + (OutputsVariableSubject, OutputsVariable), + (OutputsContractCreatedSubject, OutputsContractCreated) +]); + +impl_try_from_record_packet!(Receipt, [ + (ReceiptsCallSubject, ReceiptsCall), + (ReceiptsReturnSubject, ReceiptsReturn), + (ReceiptsReturnDataSubject, ReceiptsReturnData), + (ReceiptsPanicSubject, ReceiptsPanic), + (ReceiptsRevertSubject, ReceiptsRevert), + (ReceiptsLogSubject, ReceiptsLog), + (ReceiptsLogDataSubject, ReceiptsLogData), + (ReceiptsTransferSubject, ReceiptsTransfer), + (ReceiptsTransferOutSubject, ReceiptsTransferOut), + (ReceiptsScriptResultSubject, ReceiptsScriptResult), + (ReceiptsMessageOutSubject, ReceiptsMessageOut), + (ReceiptsMintSubject, ReceiptsMint), + (ReceiptsBurnSubject, ReceiptsBurn) +]); + +impl_try_from_record_packet!(Transaction, [( + TransactionsSubject, + Transactions +)]); + +impl_try_from_record_packet!(Utxo, [(UtxosSubject, Utxos)]); + +#[cfg(test)] +mod tests { + use fuel_streams_store::record::RecordEntity; + use serde_json::json; + use test_case::test_case; + + use super::*; + + #[test] + fn test_subject_json_conversion() { + // Test block subject + let block_json = SubjectPayload::new( + BlocksSubject::ID.to_string(), + json!({ + "producer": "0x0101010101010101010101010101010101010101010101010101010101010101", + "height": 123 + }), + ).unwrap(); + let subject: Subjects = block_json.try_into().unwrap(); + assert!(matches!(subject, Subjects::Block(_))); + + // Test inputs_coin subject + let inputs_coin_json = SubjectPayload::new( + InputsCoinSubject::ID.to_string(), + json!({ + "block_height": 123, + "tx_id": "0x0202020202020202020202020202020202020202020202020202020202020202", + "tx_index": 0, + "input_index": 1, + "owner": "0x0303030303030303030303030303030303030303030303030303030303030303", + "asset_id": "0x0404040404040404040404040404040404040404040404040404040404040404" + }), + ).unwrap(); + let subject: Subjects = inputs_coin_json.try_into().unwrap(); + assert!(matches!(subject, Subjects::InputsCoin(_))); + + // Test with empty params + let empty_block_json = + SubjectPayload::new(BlocksSubject::ID.to_string(), json!({})) + .unwrap(); + let subject: Subjects = empty_block_json.try_into().unwrap(); + assert!(matches!(subject, Subjects::Block(_))); + + // Test invalid subject + let result = + SubjectPayload::new("invalid_subject".to_string(), json!({})); + assert!(matches!( + result, + Err(SubjectPayloadError::UnknownSubject(_)) + )); + } + + #[test_case("blocks" => Ok(RecordEntity::Block); "blocks subject")] + #[test_case("inputs_coin" => Ok(RecordEntity::Input); "inputs_coin subject")] + #[test_case("inputs_contract" => Ok(RecordEntity::Input); "inputs_contract subject")] + #[test_case("inputs_message" => Ok(RecordEntity::Input); "inputs_message subject")] + #[test_case("outputs_coin" => Ok(RecordEntity::Output); "outputs_coin subject")] + #[test_case("outputs_contract" => Ok(RecordEntity::Output); "outputs_contract subject")] + #[test_case("outputs_change" => Ok(RecordEntity::Output); "outputs_change subject")] + #[test_case("outputs_variable" => Ok(RecordEntity::Output); "outputs_variable subject")] + #[test_case("outputs_contract_created" => Ok(RecordEntity::Output); "outputs_contract_created subject")] + #[test_case("receipts_call" => Ok(RecordEntity::Receipt); "receipts_call subject")] + #[test_case("receipts_return" => Ok(RecordEntity::Receipt); "receipts_return subject")] + #[test_case("receipts_return_data" => Ok(RecordEntity::Receipt); "receipts_return_data subject")] + #[test_case("receipts_panic" => Ok(RecordEntity::Receipt); "receipts_panic subject")] + #[test_case("receipts_revert" => Ok(RecordEntity::Receipt); "receipts_revert subject")] + #[test_case("receipts_log" => Ok(RecordEntity::Receipt); "receipts_log subject")] + #[test_case("receipts_log_data" => Ok(RecordEntity::Receipt); "receipts_log_data subject")] + #[test_case("receipts_transfer" => Ok(RecordEntity::Receipt); "receipts_transfer subject")] + #[test_case("receipts_transfer_out" => Ok(RecordEntity::Receipt); "receipts_transfer_out subject")] + #[test_case("receipts_script_result" => Ok(RecordEntity::Receipt); "receipts_script_result subject")] + #[test_case("receipts_message_out" => Ok(RecordEntity::Receipt); "receipts_message_out subject")] + #[test_case("receipts_mint" => Ok(RecordEntity::Receipt); "receipts_mint subject")] + #[test_case("receipts_burn" => Ok(RecordEntity::Receipt); "receipts_burn subject")] + #[test_case("transactions" => Ok(RecordEntity::Transaction); "transactions subject")] + #[test_case("utxos" => Ok(RecordEntity::Utxo); "utxos subject")] + // Case variations + #[test_case("BLOCKS" => Ok(RecordEntity::Block); "uppercase subject")] + #[test_case("Inputs_Coin" => Ok(RecordEntity::Input); "mixed case subject")] + #[test_case("RECEIPTS_TRANSFER" => Ok(RecordEntity::Receipt); "uppercase with underscore")] + // Invalid cases + #[test_case("invalid" => Err(SubjectPayloadError::UnknownSubject("invalid".to_string())); "invalid subject")] + #[test_case("invalid_subject" => Err(SubjectPayloadError::UnknownSubject("invalid_subject".to_string())); "invalid subject with type")] + #[test_case("" => Err(SubjectPayloadError::UnknownSubject("".to_string())); "empty subject")] + fn test_record_entity_parsing( + input: &str, + ) -> Result { + SubjectPayload::record_from_subject_str(input) + } +} diff --git a/crates/fuel-streams-domains/src/transactions/db_item.rs b/crates/fuel-streams-domains/src/transactions/db_item.rs new file mode 100644 index 00000000..b53e4199 --- /dev/null +++ b/crates/fuel-streams-domains/src/transactions/db_item.rs @@ -0,0 +1,84 @@ +use std::cmp::Ordering; + +use fuel_streams_store::{ + db::{DbError, DbItem}, + record::{DataEncoder, RecordEntity, RecordPacket, RecordPacketError}, +}; +use serde::{Deserialize, Serialize}; + +use super::Transaction; +use crate::Subjects; + +#[derive( + Debug, Clone, Serialize, Deserialize, PartialEq, Eq, sqlx::FromRow, +)] +pub struct TransactionDbItem { + pub subject: String, + pub value: Vec, + pub block_height: i64, + pub tx_id: String, + pub tx_index: i64, + pub tx_status: String, + pub kind: String, +} + +impl DataEncoder for TransactionDbItem { + type Err = DbError; +} + +impl DbItem for TransactionDbItem { + fn entity(&self) -> &RecordEntity { + &RecordEntity::Transaction + } + + fn encoded_value(&self) -> &[u8] { + &self.value + } + + fn subject_str(&self) -> String { + self.subject.clone() + } +} + +impl PartialOrd for TransactionDbItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for TransactionDbItem { + fn cmp(&self, other: &Self) -> Ordering { + // Order by block height first + self.block_height + .cmp(&other.block_height) + // Then by transaction index within the block + .then(self.tx_index.cmp(&other.tx_index)) + } +} + +impl TryFrom<&RecordPacket> for TransactionDbItem { + type Error = RecordPacketError; + fn try_from( + packet: &RecordPacket, + ) -> Result { + let record = packet.record.as_ref(); + let subject: Subjects = packet + .try_into() + .map_err(|_| RecordPacketError::SubjectMismatch)?; + + match subject { + Subjects::Transactions(subject) => Ok(TransactionDbItem { + subject: packet.subject_str(), + value: record + .encode_json() + .expect("Failed to encode transaction"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + tx_status: subject.tx_status.unwrap().to_string(), + kind: subject.kind.unwrap().to_string(), + }), + _ => Err(RecordPacketError::SubjectMismatch), + } + } +} diff --git a/crates/fuel-streams-domains/src/transactions/mod.rs b/crates/fuel-streams-domains/src/transactions/mod.rs new file mode 100644 index 00000000..5f5af414 --- /dev/null +++ b/crates/fuel-streams-domains/src/transactions/mod.rs @@ -0,0 +1,8 @@ +mod db_item; +mod record_impl; +pub mod subjects; +pub mod types; + +pub use db_item::*; +pub use subjects::*; +pub use types::*; diff --git a/crates/fuel-streams-domains/src/transactions/record_impl.rs b/crates/fuel-streams-domains/src/transactions/record_impl.rs new file mode 100644 index 00000000..9ef97eaf --- /dev/null +++ b/crates/fuel-streams-domains/src/transactions/record_impl.rs @@ -0,0 +1,48 @@ +use async_trait::async_trait; +use fuel_streams_store::{ + db::{Db, DbError, DbResult}, + record::{DataEncoder, Record, RecordEntity, RecordPacket}, +}; + +use super::{Transaction, TransactionDbItem}; + +impl DataEncoder for Transaction { + type Err = DbError; +} + +#[async_trait] +impl Record for Transaction { + type DbItem = TransactionDbItem; + + const ENTITY: RecordEntity = RecordEntity::Transaction; + const ORDER_PROPS: &'static [&'static str] = &["block_height", "tx_index"]; + + async fn insert( + &self, + db: &Db, + packet: &RecordPacket, + ) -> DbResult { + let db_item = TransactionDbItem::try_from(packet)?; + let record = sqlx::query_as::<_, Self::DbItem>( + r#" + INSERT INTO transactions ( + subject, value, block_height, tx_id, tx_index, tx_status, kind + ) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING subject, value, block_height, tx_id, tx_index, tx_status, kind + "#, + ) + .bind(db_item.subject) + .bind(db_item.value) + .bind(db_item.block_height) + .bind(db_item.tx_id) + .bind(db_item.tx_index) + .bind(db_item.tx_status) + .bind(db_item.kind) + .fetch_one(&db.pool) + .await + .map_err(DbError::Insert)?; + + Ok(record) + } +} diff --git a/crates/fuel-streams-domains/src/transactions/subjects.rs b/crates/fuel-streams-domains/src/transactions/subjects.rs new file mode 100644 index 00000000..00833d55 --- /dev/null +++ b/crates/fuel-streams-domains/src/transactions/subjects.rs @@ -0,0 +1,26 @@ +use fuel_streams_macros::subject::*; +use fuel_streams_types::*; +use serde::{Deserialize, Serialize}; + +use crate::{blocks::types::*, transactions::types::*}; + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "transactions"] +#[subject_wildcard = "transactions.>"] +#[subject_format = "transactions.{block_height}.{tx_id}.{tx_index}.{tx_status}.{kind}"] +pub struct TransactionsSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub tx_status: Option, + pub kind: Option, +} + +impl From<&Transaction> for TransactionsSubject { + fn from(transaction: &Transaction) -> Self { + let subject = TransactionsSubject::new(); + subject + .with_tx_id(Some(transaction.id.clone())) + .with_kind(Some(transaction.kind.clone())) + } +} diff --git a/crates/fuel-streams-core/src/transactions/types.rs b/crates/fuel-streams-domains/src/transactions/types.rs similarity index 87% rename from crates/fuel-streams-core/src/transactions/types.rs rename to crates/fuel-streams-domains/src/transactions/types.rs index bb536110..25f58b91 100644 --- a/crates/fuel-streams-core/src/transactions/types.rs +++ b/crates/fuel-streams-domains/src/transactions/types.rs @@ -1,7 +1,8 @@ -pub use fuel_core_client::client::types::TransactionStatus as ClientTransactionStatus; use fuel_core_types::fuel_tx; +use fuel_streams_types::{fuel_core::*, primitives::*}; +use serde::{Deserialize, Serialize}; -use crate::types::*; +use crate::{inputs::types::*, outputs::types::*, receipts::types::*}; #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] pub struct StorageSlot { @@ -27,7 +28,7 @@ impl From<&FuelCoreStorageSlot> for StorageSlot { #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Transaction { - pub id: Bytes32, + pub id: TxId, #[serde(rename = "type")] pub kind: TransactionKind, pub bytecode_root: Option, @@ -371,7 +372,7 @@ impl Transaction { }; Transaction { - id: id.to_owned(), + id: id.to_owned().into(), kind: transaction.into(), bytecode_root, bytecode_witness_index, @@ -423,17 +424,22 @@ pub enum TransactionKind { Blob, } +impl TransactionKind { + fn as_str(&self) -> &'static str { + match self { + Self::Create => "create", + Self::Mint => "mint", + Self::Script => "script", + Self::Upgrade => "upgrade", + Self::Upload => "upload", + Self::Blob => "blob", + } + } +} + impl std::fmt::Display for TransactionKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let value: &'static str = match self { - TransactionKind::Create => "create", - TransactionKind::Mint => "mint", - TransactionKind::Script => "script", - TransactionKind::Upgrade => "upgrade", - TransactionKind::Upload => "upload", - TransactionKind::Blob => "blob", - }; - write!(f, "{value}") + write!(f, "{}", self.as_str()) } } @@ -450,14 +456,18 @@ impl From<&FuelCoreTransaction> for TransactionKind { } } -#[derive(Debug, Clone)] -#[cfg(any(test, feature = "test-helpers"))] -pub struct MockTransaction(pub Block); - -#[cfg(any(test, feature = "test-helpers"))] -impl MockTransaction { - pub fn build() -> Transaction { - Transaction::default() +impl std::str::FromStr for TransactionKind { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + s if s == Self::Create.as_str() => Ok(Self::Create), + s if s == Self::Mint.as_str() => Ok(Self::Mint), + s if s == Self::Script.as_str() => Ok(Self::Script), + s if s == Self::Upgrade.as_str() => Ok(Self::Upgrade), + s if s == Self::Upload.as_str() => Ok(Self::Upload), + s if s == Self::Blob.as_str() => Ok(Self::Blob), + _ => Err(format!("Invalid transaction kind: {s}")), + } } } @@ -474,14 +484,33 @@ pub enum TransactionStatus { impl std::fmt::Display for TransactionStatus { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let value: &'static str = match self { + write!(f, "{}", self.as_str()) + } +} + +impl TransactionStatus { + fn as_str(&self) -> &'static str { + match self { TransactionStatus::Failed => "failed", TransactionStatus::Submitted => "submitted", TransactionStatus::SqueezedOut => "squeezed_out", TransactionStatus::Success => "success", TransactionStatus::None => "none", - }; - write!(f, "{value}") + } + } +} + +impl std::str::FromStr for TransactionStatus { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + s if s == Self::Failed.as_str() => Ok(Self::Failed), + s if s == Self::Submitted.as_str() => Ok(Self::Submitted), + s if s == Self::SqueezedOut.as_str() => Ok(Self::SqueezedOut), + s if s == Self::Success.as_str() => Ok(Self::Success), + s if s == Self::None.as_str() => Ok(Self::None), + _ => Err(format!("Invalid transaction status: {s}")), + } } } @@ -504,27 +533,27 @@ impl From<&FuelCoreTransactionStatus> for TransactionStatus { } } -impl From<&ClientTransactionStatus> for TransactionStatus { - fn from(value: &ClientTransactionStatus) -> Self { +impl From<&FuelCoreClientTransactionStatus> for TransactionStatus { + fn from(value: &FuelCoreClientTransactionStatus) -> Self { match value { - ClientTransactionStatus::Failure { .. } => { + FuelCoreClientTransactionStatus::Failure { .. } => { TransactionStatus::Failed } - ClientTransactionStatus::Submitted { .. } => { + FuelCoreClientTransactionStatus::Submitted { .. } => { TransactionStatus::Submitted } - ClientTransactionStatus::SqueezedOut { .. } => { + FuelCoreClientTransactionStatus::SqueezedOut { .. } => { TransactionStatus::SqueezedOut } - ClientTransactionStatus::Success { .. } => { + FuelCoreClientTransactionStatus::Success { .. } => { TransactionStatus::Success } } } } -impl From for TransactionStatus { - fn from(value: ClientTransactionStatus) -> Self { +impl From for TransactionStatus { + fn from(value: FuelCoreClientTransactionStatus) -> Self { (&value).into() } } @@ -560,3 +589,14 @@ impl FuelCoreTransactionExt for FuelCoreTransaction { } } } + +#[derive(Debug, Clone)] +#[cfg(any(test, feature = "test-helpers"))] +pub struct MockTransaction(pub crate::blocks::Block); + +#[cfg(any(test, feature = "test-helpers"))] +impl MockTransaction { + pub fn build() -> Transaction { + Transaction::default() + } +} diff --git a/crates/fuel-streams-domains/src/utxos/db_item.rs b/crates/fuel-streams-domains/src/utxos/db_item.rs new file mode 100644 index 00000000..3f30aa4c --- /dev/null +++ b/crates/fuel-streams-domains/src/utxos/db_item.rs @@ -0,0 +1,84 @@ +use std::cmp::Ordering; + +use fuel_streams_store::{ + db::{DbError, DbItem}, + record::{DataEncoder, RecordEntity, RecordPacket, RecordPacketError}, +}; +use serde::{Deserialize, Serialize}; + +use super::Utxo; +use crate::Subjects; + +#[derive( + Debug, Clone, Serialize, Deserialize, PartialEq, Eq, sqlx::FromRow, +)] +pub struct UtxoDbItem { + pub subject: String, + pub value: Vec, + pub block_height: i64, + pub tx_id: String, + pub tx_index: i64, + pub input_index: i64, + pub utxo_type: String, + pub utxo_id: String, +} + +impl DataEncoder for UtxoDbItem { + type Err = DbError; +} + +impl DbItem for UtxoDbItem { + fn entity(&self) -> &RecordEntity { + &RecordEntity::Utxo + } + + fn encoded_value(&self) -> &[u8] { + &self.value + } + + fn subject_str(&self) -> String { + self.subject.clone() + } +} + +impl PartialOrd for UtxoDbItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for UtxoDbItem { + fn cmp(&self, other: &Self) -> Ordering { + // Order by block height first + self.block_height + .cmp(&other.block_height) + // Then by transaction index within the block + .then(self.tx_index.cmp(&other.tx_index)) + // Finally by input index + .then(self.input_index.cmp(&other.input_index)) + } +} + +impl TryFrom<&RecordPacket> for UtxoDbItem { + type Error = RecordPacketError; + fn try_from(packet: &RecordPacket) -> Result { + let record = packet.record.as_ref(); + let subject: Subjects = packet + .try_into() + .map_err(|_| RecordPacketError::SubjectMismatch)?; + + match subject { + Subjects::Utxos(subject) => Ok(UtxoDbItem { + subject: packet.subject_str(), + value: record.encode_json().expect("Failed to encode utxo"), + block_height: subject.block_height.unwrap().into(), + tx_id: subject.tx_id.unwrap().to_string(), + tx_index: subject.tx_index.unwrap() as i64, + input_index: subject.input_index.unwrap() as i64, + utxo_type: subject.utxo_type.unwrap().to_string(), + utxo_id: subject.utxo_id.unwrap().to_string(), + }), + _ => Err(RecordPacketError::SubjectMismatch), + } + } +} diff --git a/crates/fuel-streams-domains/src/utxos/mod.rs b/crates/fuel-streams-domains/src/utxos/mod.rs new file mode 100644 index 00000000..5f5af414 --- /dev/null +++ b/crates/fuel-streams-domains/src/utxos/mod.rs @@ -0,0 +1,8 @@ +mod db_item; +mod record_impl; +pub mod subjects; +pub mod types; + +pub use db_item::*; +pub use subjects::*; +pub use types::*; diff --git a/crates/fuel-streams-domains/src/utxos/record_impl.rs b/crates/fuel-streams-domains/src/utxos/record_impl.rs new file mode 100644 index 00000000..3a5872e6 --- /dev/null +++ b/crates/fuel-streams-domains/src/utxos/record_impl.rs @@ -0,0 +1,52 @@ +use async_trait::async_trait; +use fuel_streams_store::{ + db::{Db, DbError, DbResult}, + record::{DataEncoder, Record, RecordEntity, RecordPacket}, +}; + +use super::{Utxo, UtxoDbItem}; + +impl DataEncoder for Utxo { + type Err = DbError; +} + +#[async_trait] +impl Record for Utxo { + type DbItem = UtxoDbItem; + + const ENTITY: RecordEntity = RecordEntity::Utxo; + const ORDER_PROPS: &'static [&'static str] = + &["block_height", "tx_index", "input_index"]; + + async fn insert( + &self, + db: &Db, + packet: &RecordPacket, + ) -> DbResult { + let db_item = UtxoDbItem::try_from(packet)?; + let record = sqlx::query_as::<_, Self::DbItem>( + r#" + INSERT INTO utxos ( + subject, value, block_height, tx_id, tx_index, + input_index, utxo_type, utxo_id + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + RETURNING subject, value, block_height, tx_id, tx_index, + input_index, utxo_type, utxo_id + "#, + ) + .bind(db_item.subject) + .bind(db_item.value) + .bind(db_item.block_height) + .bind(db_item.tx_id) + .bind(db_item.tx_index) + .bind(db_item.input_index) + .bind(db_item.utxo_type) + .bind(db_item.utxo_id) + .fetch_one(&db.pool) + .await + .map_err(DbError::Insert)?; + + Ok(record) + } +} diff --git a/crates/fuel-streams-domains/src/utxos/subjects.rs b/crates/fuel-streams-domains/src/utxos/subjects.rs new file mode 100644 index 00000000..a67ae374 --- /dev/null +++ b/crates/fuel-streams-domains/src/utxos/subjects.rs @@ -0,0 +1,19 @@ +use fuel_streams_macros::subject::*; +use fuel_streams_types::*; +use serde::{Deserialize, Serialize}; + +use super::types::*; +use crate::blocks::types::*; + +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "utxos"] +#[subject_wildcard = "utxos.>"] +#[subject_format = "utxos.{block_height}.{tx_id}.{tx_index}.{input_index}.{utxo_type}.{utxo_id}"] +pub struct UtxosSubject { + pub block_height: Option, + pub tx_id: Option, + pub tx_index: Option, + pub input_index: Option, + pub utxo_type: Option, + pub utxo_id: Option, +} diff --git a/crates/fuel-streams-domains/src/utxos/types.rs b/crates/fuel-streams-domains/src/utxos/types.rs new file mode 100644 index 00000000..d9b82431 --- /dev/null +++ b/crates/fuel-streams-domains/src/utxos/types.rs @@ -0,0 +1,50 @@ +use fuel_streams_types::primitives::*; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Utxo { + pub utxo_id: UtxoId, + pub sender: Option
, + pub recipient: Option
, + pub nonce: Option, + pub data: Option, + pub amount: Option, + pub tx_id: TxId, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub enum UtxoType { + Contract, + Coin, + #[default] + Message, +} + +impl UtxoType { + fn as_str(&self) -> &'static str { + match self { + UtxoType::Contract => "contract", + UtxoType::Coin => "coin", + UtxoType::Message => "message", + } + } +} + +impl std::fmt::Display for UtxoType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +impl std::str::FromStr for UtxoType { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + s if s == Self::Contract.as_str() => Ok(Self::Contract), + s if s == Self::Coin.as_str() => Ok(Self::Coin), + s if s == Self::Message.as_str() => Ok(Self::Message), + _ => Err(format!("Invalid UTXO type: {s}")), + } + } +} diff --git a/crates/fuel-streams-executors/Cargo.toml b/crates/fuel-streams-executors/Cargo.toml index 98c0f71b..b8354d83 100644 --- a/crates/fuel-streams-executors/Cargo.toml +++ b/crates/fuel-streams-executors/Cargo.toml @@ -13,11 +13,10 @@ publish = false [dependencies] anyhow = { workspace = true } -displaydoc = { workspace = true } dotenvy = { workspace = true } fuel-core = { workspace = true } -fuel-data-parser = { workspace = true, features = ["test-helpers"] } -fuel-streams-core = { workspace = true, features = ["test-helpers"] } +fuel-streams-core = { workspace = true } +fuel-streams-store = { workspace = true } futures = { workspace = true } num_cpus = { workspace = true } rayon = { workspace = true } diff --git a/crates/fuel-streams-executors/src/blocks.rs b/crates/fuel-streams-executors/src/blocks.rs index 5a145e2d..48a14a22 100644 --- a/crates/fuel-streams-executors/src/blocks.rs +++ b/crates/fuel-streams-executors/src/blocks.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use fuel_streams_core::prelude::*; +use fuel_streams_core::{subjects::*, types::*, FuelStreams}; use futures::stream::FuturesUnordered; use tokio::task::JoinHandle; @@ -12,29 +12,25 @@ impl Executor { let block = self.block(); let block_height = (*metadata.block_height).clone(); let block_producer = (*metadata.block_producer).clone(); - let packet = PublishPacket::::new( - block.to_owned(), - BlocksSubject { - height: Some(block_height), - producer: Some(block_producer), - } - .arc(), - ); - self.publish(&packet) + let subject = BlocksSubject { + block_height: Some(block_height), + producer: Some(block_producer), + } + .arc(); + self.publish(&block.to_packet(subject)) } pub fn process_all( payload: Arc, - fuel_streams: &Arc, + fuel_streams: &Arc, semaphore: &Arc, ) -> FuturesUnordered>> { - let block_stream = fuel_streams.blocks().arc(); - let tx_stream = fuel_streams.transactions().arc(); - let input_stream = fuel_streams.inputs().arc(); - let output_stream = fuel_streams.outputs().arc(); - let receipt_stream = fuel_streams.receipts().arc(); - let log_stream = fuel_streams.logs().arc(); - let utxo_stream = fuel_streams.utxos().arc(); + let block_stream = fuel_streams.blocks.arc(); + let tx_stream = fuel_streams.transactions.arc(); + let input_stream = fuel_streams.inputs.arc(); + let output_stream = fuel_streams.outputs.arc(); + let receipt_stream = fuel_streams.receipts.arc(); + let utxo_stream = fuel_streams.utxos.arc(); let block_executor = Executor::new(&payload, &block_stream, semaphore); let tx_executor = Executor::new(&payload, &tx_stream, semaphore); @@ -43,24 +39,23 @@ impl Executor { Executor::new(&payload, &output_stream, semaphore); let receipt_executor = Executor::new(&payload, &receipt_stream, semaphore); - let log_executor = Executor::new(&payload, &log_stream, semaphore); let utxo_executor = Executor::new(&payload, &utxo_stream, semaphore); let transactions = payload.transactions.to_owned(); - let tx_tasks = - transactions - .iter() - .enumerate() - .flat_map(|tx_item @ (_, tx)| { - vec![ - tx_executor.process(tx_item), - input_executor.process(tx), - output_executor.process(tx), - receipt_executor.process(tx), - log_executor.process(tx), - utxo_executor.process(tx), - ] - }); + let tx_tasks = transactions.iter().enumerate().flat_map(|tx_item| { + let tx_task = tx_executor.process(tx_item); + let input_tasks = input_executor.process(tx_item); + let output_tasks = output_executor.process(tx_item); + let receipt_tasks = receipt_executor.process(tx_item); + let utxo_tasks = utxo_executor.process(tx_item); + vec![ + tx_task, + input_tasks, + output_tasks, + receipt_tasks, + utxo_tasks, + ] + }); let block_task = block_executor.process(); std::iter::once(block_task) diff --git a/crates/fuel-streams-executors/src/inputs.rs b/crates/fuel-streams-executors/src/inputs.rs index 33394171..e59696c9 100644 --- a/crates/fuel-streams-executors/src/inputs.rs +++ b/crates/fuel-streams-executors/src/inputs.rs @@ -1,6 +1,4 @@ -use std::sync::Arc; - -use fuel_streams_core::prelude::*; +use fuel_streams_core::{subjects::*, types::*}; use rayon::prelude::*; use tokio::task::JoinHandle; @@ -9,30 +7,23 @@ use crate::*; impl Executor { pub fn process( &self, - tx: &Transaction, + (tx_index, tx): (usize, &Transaction), ) -> Vec>> { + let block_height = self.block_height(); let tx_id = tx.id.clone(); let packets = tx .inputs .par_iter() .enumerate() - .flat_map(move |(index, input)| { - let main_subject = main_subject(input, tx_id.clone(), index); - let identifier_subjects = - identifiers(input, &tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: InputsByIdSubject| subject.arc()) - .collect::>(); - - let mut packets = vec![input.to_packet(main_subject)]; - packets.extend( - identifier_subjects - .into_iter() - .map(|subject| input.to_packet(subject)), + .flat_map(move |(input_index, input)| { + let main_subject = main_subject( + block_height.clone(), + tx_index as u32, + input_index as u32, + tx_id.clone(), + input, ); - - packets + vec![input.to_packet(main_subject)] }) .collect::>(); @@ -41,27 +32,35 @@ impl Executor { } fn main_subject( + block_height: BlockHeight, + tx_index: u32, + input_index: u32, + tx_id: TxId, input: &Input, - tx_id: Bytes32, - index: usize, ) -> Arc { match input { Input::Contract(contract) => InputsContractSubject { + block_height: Some(block_height), tx_id: Some(tx_id), - index: Some(index), + tx_index: Some(tx_index), + input_index: Some(input_index), contract_id: Some(contract.contract_id.to_owned().into()), } .arc(), Input::Coin(coin) => InputsCoinSubject { + block_height: Some(block_height), tx_id: Some(tx_id), - index: Some(index), - owner: Some(coin.owner.to_owned()), + tx_index: Some(tx_index), + input_index: Some(input_index), + owner_id: Some(coin.owner.to_owned()), asset_id: Some(coin.asset_id.to_owned()), } .arc(), Input::Message(message) => InputsMessageSubject { + block_height: Some(block_height), tx_id: Some(tx_id), - index: Some(index), + tx_index: Some(tx_index), + input_index: Some(input_index), sender: Some(message.sender.to_owned()), recipient: Some(message.recipient.to_owned()), } @@ -69,61 +68,61 @@ fn main_subject( } } -pub fn identifiers( - input: &Input, - tx_id: &Bytes32, - index: u8, -) -> Vec { - let mut identifiers = match input { - Input::Coin(coin) => { - vec![ - Identifier::Address( - tx_id.to_owned(), - index, - coin.owner.to_owned().into(), - ), - Identifier::AssetID( - tx_id.to_owned(), - index, - coin.asset_id.to_owned().into(), - ), - ] - } - Input::Message(message) => { - vec![ - Identifier::Address( - tx_id.to_owned(), - index, - message.sender.to_owned().into(), - ), - Identifier::Address( - tx_id.to_owned(), - index, - message.recipient.to_owned().into(), - ), - ] - } - Input::Contract(contract) => { - vec![Identifier::ContractID( - tx_id.to_owned(), - index, - contract.contract_id.to_owned(), - )] - } - }; +// pub fn identifiers( +// input: &Input, +// tx_id: &Bytes32, +// index: u8, +// ) -> Vec { +// let mut identifiers = match input { +// Input::Coin(coin) => { +// vec![ +// Identifier::Address( +// tx_id.to_owned(), +// index, +// coin.owner.to_owned().into(), +// ), +// Identifier::AssetID( +// tx_id.to_owned(), +// index, +// coin.asset_id.to_owned().into(), +// ), +// ] +// } +// Input::Message(message) => { +// vec![ +// Identifier::Address( +// tx_id.to_owned(), +// index, +// message.sender.to_owned().into(), +// ), +// Identifier::Address( +// tx_id.to_owned(), +// index, +// message.recipient.to_owned().into(), +// ), +// ] +// } +// Input::Contract(contract) => { +// vec![Identifier::ContractID( +// tx_id.to_owned(), +// index, +// contract.contract_id.to_owned(), +// )] +// } +// }; - match input { - Input::Coin(InputCoin { predicate, .. }) - | Input::Message(InputMessage { predicate, .. }) => { - let predicate_tag = super::sha256(&predicate.0 .0); - identifiers.push(Identifier::PredicateID( - tx_id.to_owned(), - index, - predicate_tag, - )); - } - _ => {} - }; +// match input { +// Input::Coin(InputCoin { predicate, .. }) +// | Input::Message(InputMessage { predicate, .. }) => { +// let predicate_tag = super::sha256(&predicate.0 .0); +// identifiers.push(Identifier::PredicateID( +// tx_id.to_owned(), +// index, +// predicate_tag, +// )); +// } +// _ => {} +// }; - identifiers -} +// identifiers +// } diff --git a/crates/fuel-streams-executors/src/lib.rs b/crates/fuel-streams-executors/src/lib.rs index d5fa6a36..0dbf89cb 100644 --- a/crates/fuel-streams-executors/src/lib.rs +++ b/crates/fuel-streams-executors/src/lib.rs @@ -1,6 +1,5 @@ pub mod blocks; pub mod inputs; -pub mod logs; pub mod outputs; pub mod receipts; pub mod transactions; @@ -11,9 +10,13 @@ use std::{ sync::{Arc, LazyLock}, }; -use displaydoc::Display as DisplayDoc; -use fuel_data_parser::DataParserError; -use fuel_streams_core::prelude::*; +use fuel_streams_core::{fuel_core_like::FuelCoreLike, types::*, Stream}; +use fuel_streams_store::record::{ + DataEncoder, + EncoderError, + Record, + RecordPacket, +}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tokio::task::JoinHandle; @@ -38,22 +41,22 @@ pub fn sha256(bytes: &[u8]) -> Bytes32 { bytes.into() } -#[derive(Debug, thiserror::Error, DisplayDoc)] +#[derive(Debug, thiserror::Error)] pub enum ExecutorError { - /// Failed to publish: {0} + #[error("Failed to publish: {0}")] PublishFailed(String), - /// Failed to acquire semaphore: {0} + #[error(transparent)] SemaphoreError(#[from] tokio::sync::AcquireError), - /// Failed to serialize block payload: {0} + #[error(transparent)] Serialization(#[from] serde_json::Error), - /// Failed to fetch transaction status: {0} + #[error("Failed to fetch transaction status: {0}")] TransactionStatus(String), - /// Failed to access offchain database: {0} + #[error(transparent)] OffchainDatabase(#[from] anyhow::Error), - /// Failed to join tasks: {0} + #[error(transparent)] JoinError(#[from] tokio::task::JoinError), - /// Failed to encode or decode data: {0} - Encoder(#[from] DataParserError), + #[error(transparent)] + Encoder(#[from] EncoderError), } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -111,7 +114,8 @@ impl BlockPayload { let txs_ids = txs.iter().map(|i| i.id.clone()).collect(); let block_height = block.header().height(); let consensus = fuel_core.get_consensus(block_height)?; - let block = Block::new(&block, consensus.into(), txs_ids); + let producer = consensus.block_producer(&block.id())?.into(); + let block = Block::new(&block, consensus.into(), txs_ids, producer); Ok(Self { block, transactions: txs, @@ -119,7 +123,7 @@ impl BlockPayload { }) } - pub fn tx_ids(&self) -> Vec { + pub fn tx_ids(&self) -> Vec { self.transactions .iter() .map(|tx| tx.id.clone()) @@ -141,8 +145,8 @@ impl BlockPayload { &self.metadata } - pub fn block_height(&self) -> u32 { - self.block.height + pub fn block_height(&self) -> BlockHeight { + self.block.height.clone() } pub fn arc(&self) -> Arc { @@ -177,17 +181,17 @@ impl BlockPayload { } } -pub struct Executor { - pub stream: Arc>, +pub struct Executor { + pub stream: Arc>, payload: Arc, semaphore: Arc, - __marker: PhantomData, + __marker: PhantomData, } -impl Executor { +impl Executor { pub fn new( payload: &Arc, - stream: &Arc>, + stream: &Arc>, semaphore: &Arc, ) -> Self { Self { @@ -200,14 +204,13 @@ impl Executor { fn publish( &self, - packet: &PublishPacket, + packet: &RecordPacket, ) -> JoinHandle> { - let wildcard = packet.subject.parse(); let stream = Arc::clone(&self.stream); let permit = Arc::clone(&self.semaphore); - // TODO: add telemetry back again - let packet = packet.clone(); + let wildcard = packet.subject_str(); + let packet = Arc::new(packet.clone()); tokio::spawn({ async move { let _permit = permit.acquire().await?; @@ -230,14 +233,16 @@ impl Executor { pub fn payload(&self) -> Arc { Arc::clone(&self.payload) } + pub fn metadata(&self) -> &Metadata { &self.payload.metadata } + pub fn block(&self) -> &Block { &self.payload.block } + pub fn block_height(&self) -> BlockHeight { - let height = self.block().height; - BlockHeight::from(height) + self.block().height.clone() } } diff --git a/crates/fuel-streams-executors/src/logs.rs b/crates/fuel-streams-executors/src/logs.rs deleted file mode 100644 index f4ac0308..00000000 --- a/crates/fuel-streams-executors/src/logs.rs +++ /dev/null @@ -1,38 +0,0 @@ -use fuel_streams_core::prelude::*; -use rayon::prelude::*; -use tokio::task::JoinHandle; - -use crate::*; - -impl Executor { - pub fn process( - &self, - tx: &Transaction, - ) -> Vec>> { - let block_height = self.block_height(); - let tx_id = tx.id.clone(); - let receipts = tx.receipts.clone(); - let packets = receipts - .par_iter() - .enumerate() - .filter_map(|(index, receipt)| match receipt { - Receipt::Log(LogReceipt { id, .. }) - | Receipt::LogData(LogDataReceipt { id, .. }) => { - Some(PublishPacket::new( - receipt.to_owned().into(), - LogsSubject { - block_height: Some(block_height.clone()), - tx_id: Some(tx_id.to_owned()), - receipt_index: Some(index), - log_id: Some(id.into()), - } - .arc(), - )) - } - _ => None, - }) - .collect::>(); - - packets.iter().map(|packet| self.publish(packet)).collect() - } -} diff --git a/crates/fuel-streams-executors/src/outputs.rs b/crates/fuel-streams-executors/src/outputs.rs index 48e9baa8..fddd7164 100644 --- a/crates/fuel-streams-executors/src/outputs.rs +++ b/crates/fuel-streams-executors/src/outputs.rs @@ -1,6 +1,4 @@ -use std::sync::Arc; - -use fuel_streams_core::prelude::*; +use fuel_streams_core::{subjects::*, types::*}; use rayon::prelude::*; use tokio::task::JoinHandle; @@ -9,48 +7,46 @@ use crate::*; impl Executor { pub fn process( &self, - tx: &Transaction, + (tx_index, tx): (usize, &Transaction), ) -> Vec>> { + let block_height = self.block_height(); let tx_id = tx.id.clone(); - let packets: Vec> = tx + let packets = tx .outputs .par_iter() .enumerate() - .flat_map(|(index, output)| { - let main_subject = main_subject(output, tx, &tx_id, index); - let identifier_subjects = - identifiers(output, tx, &tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: OutputsByIdSubject| subject.arc()) - .collect::>(); - - let mut packets = vec![output.to_packet(main_subject)]; - packets.extend( - identifier_subjects - .into_iter() - .map(|subject| output.to_packet(subject)), + .flat_map(|(output_index, output)| { + let main_subject = main_subject( + block_height.clone(), + tx_index as u32, + output_index as u32, + tx_id.to_owned(), + tx, + output, ); - - packets + vec![output.to_packet(main_subject)] }) - .collect(); + .collect::>(); packets.iter().map(|packet| self.publish(packet)).collect() } } fn main_subject( - output: &Output, + block_height: BlockHeight, + tx_index: u32, + output_index: u32, + tx_id: TxId, transaction: &Transaction, - tx_id: &Bytes32, - index: usize, + output: &Output, ) -> Arc { match output { Output::Coin(OutputCoin { to, asset_id, .. }) => OutputsCoinSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), - to: Some(to.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + output_index: Some(output_index), + to_address: Some(to.to_owned()), asset_id: Some(asset_id.to_owned()), } .arc(), @@ -69,26 +65,32 @@ fn main_subject( }; OutputsContractSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + output_index: Some(output_index), contract_id: Some(contract_id), } .arc() } Output::Change(OutputChange { to, asset_id, .. }) => { OutputsChangeSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), - to: Some(to.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + output_index: Some(output_index), + to_address: Some(to.to_owned()), asset_id: Some(asset_id.to_owned()), } .arc() } Output::Variable(OutputVariable { to, asset_id, .. }) => { OutputsVariableSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), - to: Some(to.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + output_index: Some(output_index), + to_address: Some(to.to_owned()), asset_id: Some(asset_id.to_owned()), } .arc() @@ -96,49 +98,51 @@ fn main_subject( Output::ContractCreated(OutputContractCreated { contract_id, .. }) => OutputsContractCreatedSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + output_index: Some(output_index), contract_id: Some(contract_id.to_owned()), } .arc(), } } -pub fn identifiers( - output: &Output, - tx: &Transaction, - tx_id: &Bytes32, - index: u8, -) -> Vec { - match output { - Output::Change(OutputChange { to, asset_id, .. }) - | Output::Variable(OutputVariable { to, asset_id, .. }) - | Output::Coin(OutputCoin { to, asset_id, .. }) => { - vec![ - Identifier::Address(tx_id.to_owned(), index, to.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - Output::Contract(contract) => find_output_contract_id(tx, contract) - .map(|contract_id| { - vec![Identifier::ContractID( - tx_id.to_owned(), - index, - contract_id.into(), - )] - }) - .unwrap_or_default(), - Output::ContractCreated(OutputContractCreated { - contract_id, .. - }) => { - vec![Identifier::ContractID( - tx_id.to_owned(), - index, - contract_id.into(), - )] - } - } -} +// pub fn identifiers( +// output: &Output, +// tx: &Transaction, +// tx_id: &Bytes32, +// index: u8, +// ) -> Vec { +// match output { +// Output::Change(OutputChange { to, asset_id, .. }) +// | Output::Variable(OutputVariable { to, asset_id, .. }) +// | Output::Coin(OutputCoin { to, asset_id, .. }) => { +// vec![ +// Identifier::Address(tx_id.to_owned(), index, to.into()), +// Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), +// ] +// } +// Output::Contract(contract) => find_output_contract_id(tx, contract) +// .map(|contract_id| { +// vec![Identifier::ContractID( +// tx_id.to_owned(), +// index, +// contract_id.into(), +// )] +// }) +// .unwrap_or_default(), +// Output::ContractCreated(OutputContractCreated { +// contract_id, .. +// }) => { +// vec![Identifier::ContractID( +// tx_id.to_owned(), +// index, +// contract_id.into(), +// )] +// } +// } +// } pub fn find_output_contract_id( tx: &Transaction, diff --git a/crates/fuel-streams-executors/src/receipts.rs b/crates/fuel-streams-executors/src/receipts.rs index c99f03fc..528e6c79 100644 --- a/crates/fuel-streams-executors/src/receipts.rs +++ b/crates/fuel-streams-executors/src/receipts.rs @@ -1,6 +1,4 @@ -use std::sync::Arc; - -use fuel_streams_core::prelude::*; +use fuel_streams_core::{subjects::*, types::*}; use rayon::prelude::*; use tokio::task::JoinHandle; @@ -9,42 +7,37 @@ use crate::*; impl Executor { pub fn process( &self, - tx: &Transaction, + (tx_index, tx): (usize, &Transaction), ) -> Vec>> { + let block_height = self.block_height(); let tx_id = tx.id.clone(); let receipts = tx.receipts.clone(); - let packets: Vec> = receipts + let packets = receipts .par_iter() .enumerate() - .flat_map(|(index, receipt)| { - let main_subject = main_subject(receipt, &tx_id, index); - let identifier_subjects = - identifiers(receipt, &tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: ReceiptsByIdSubject| subject.arc()) - .collect::>(); - - let receipt: Receipt = receipt.to_owned(); - let mut packets = vec![receipt.to_packet(main_subject)]; - packets.extend( - identifier_subjects - .into_iter() - .map(|subject| receipt.to_packet(subject)), + .flat_map(|(receipt_index, receipt)| { + let main_subject = main_subject( + block_height.clone(), + tx_index as u32, + receipt_index as u32, + tx_id.clone(), + receipt, ); - - packets + let receipt: Receipt = receipt.to_owned(); + vec![receipt.to_packet(main_subject)] }) - .collect(); + .collect::>(); packets.iter().map(|packet| self.publish(packet)).collect() } } fn main_subject( + block_height: BlockHeight, + tx_index: u32, + receipt_index: u32, + tx_id: TxId, receipt: &Receipt, - tx_id: &Bytes32, - index: usize, ) -> Arc { match receipt { Receipt::Call(CallReceipt { @@ -53,49 +46,63 @@ fn main_subject( asset_id, .. }) => ReceiptsCallSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - from: Some(from.to_owned()), - to: Some(to.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + from_contract_id: Some(from.to_owned()), + to_contract_id: Some(to.to_owned()), asset_id: Some(asset_id.to_owned()), } .arc(), Receipt::Return(ReturnReceipt { id, .. }) => ReceiptsReturnSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + contract_id: Some(id.to_owned()), } .arc(), Receipt::ReturnData(ReturnDataReceipt { id, .. }) => { ReceiptsReturnDataSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + contract_id: Some(id.to_owned()), } .arc() } Receipt::Panic(PanicReceipt { id, .. }) => ReceiptsPanicSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + contract_id: Some(id.to_owned()), } .arc(), Receipt::Revert(RevertReceipt { id, .. }) => ReceiptsRevertSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + contract_id: Some(id.to_owned()), } .arc(), Receipt::Log(LogReceipt { id, .. }) => ReceiptsLogSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + contract_id: Some(id.to_owned()), } .arc(), Receipt::LogData(LogDataReceipt { id, .. }) => ReceiptsLogDataSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + contract_id: Some(id.to_owned()), } .arc(), Receipt::Transfer(TransferReceipt { @@ -104,10 +111,12 @@ fn main_subject( asset_id, .. }) => ReceiptsTransferSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - from: Some(from.to_owned()), - to: Some(to.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + from_contract_id: Some(from.to_owned()), + to_contract_id: Some(to.to_owned()), asset_id: Some(asset_id.to_owned()), } .arc(), @@ -118,28 +127,34 @@ fn main_subject( asset_id, .. }) => ReceiptsTransferOutSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - from: Some(from.to_owned()), - to: Some(to.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + from_contract_id: Some(from.to_owned()), + to_address: Some(to.to_owned()), asset_id: Some(asset_id.to_owned()), } .arc(), Receipt::ScriptResult(ScriptResultReceipt { .. }) => { ReceiptsScriptResultSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), } .arc() } Receipt::MessageOut(MessageOutReceipt { sender, recipient, .. }) => ReceiptsMessageOutSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - sender: Some(sender.to_owned()), - recipient: Some(recipient.to_owned()), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), + sender_address: Some(sender.to_owned()), + recipient_address: Some(recipient.to_owned()), } .arc(), Receipt::Mint(MintReceipt { @@ -147,8 +162,10 @@ fn main_subject( sub_id, .. }) => ReceiptsMintSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), contract_id: Some(contract_id.to_owned()), sub_id: Some((*sub_id).to_owned()), } @@ -158,8 +175,10 @@ fn main_subject( sub_id, .. }) => ReceiptsBurnSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + receipt_index: Some(receipt_index), contract_id: Some(contract_id.to_owned()), sub_id: Some((*sub_id).to_owned()), } @@ -167,72 +186,72 @@ fn main_subject( } } -pub fn identifiers( - receipt: &Receipt, - tx_id: &Bytes32, - index: u8, -) -> Vec { - match receipt { - Receipt::Call(CallReceipt { - id: from, - to, - asset_id, - .. - }) => { - vec![ - Identifier::ContractID(tx_id.to_owned(), index, from.into()), - Identifier::ContractID(tx_id.to_owned(), index, to.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - Receipt::Return(ReturnReceipt { id, .. }) - | Receipt::ReturnData(ReturnDataReceipt { id, .. }) - | Receipt::Panic(PanicReceipt { id, .. }) - | Receipt::Revert(RevertReceipt { id, .. }) - | Receipt::Log(LogReceipt { id, .. }) - | Receipt::LogData(LogDataReceipt { id, .. }) => { - vec![Identifier::ContractID(tx_id.to_owned(), index, id.into())] - } - Receipt::Transfer(TransferReceipt { - id: from, - to, - asset_id, - .. - }) => { - vec![ - Identifier::ContractID(tx_id.to_owned(), index, from.into()), - Identifier::ContractID(tx_id.to_owned(), index, to.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - Receipt::TransferOut(TransferOutReceipt { - id: from, - to, - asset_id, - .. - }) => { - vec![ - Identifier::ContractID(tx_id.to_owned(), index, from.into()), - Identifier::ContractID(tx_id.to_owned(), index, to.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - Receipt::MessageOut(MessageOutReceipt { - sender, recipient, .. - }) => { - vec![ - Identifier::Address(tx_id.to_owned(), index, sender.into()), - Identifier::Address(tx_id.to_owned(), index, recipient.into()), - ] - } - Receipt::Mint(MintReceipt { contract_id, .. }) - | Receipt::Burn(BurnReceipt { contract_id, .. }) => { - vec![Identifier::ContractID( - tx_id.to_owned(), - index, - contract_id.into(), - )] - } - _ => Vec::new(), - } -} +// pub fn identifiers( +// receipt: &Receipt, +// tx_id: &Bytes32, +// index: u8, +// ) -> Vec { +// match receipt { +// Receipt::Call(CallReceipt { +// id: from, +// to, +// asset_id, +// .. +// }) => { +// vec![ +// Identifier::ContractID(tx_id.to_owned(), index, from.into()), +// Identifier::ContractID(tx_id.to_owned(), index, to.into()), +// Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), +// ] +// } +// Receipt::Return(ReturnReceipt { id, .. }) +// | Receipt::ReturnData(ReturnDataReceipt { id, .. }) +// | Receipt::Panic(PanicReceipt { id, .. }) +// | Receipt::Revert(RevertReceipt { id, .. }) +// | Receipt::Log(LogReceipt { id, .. }) +// | Receipt::LogData(LogDataReceipt { id, .. }) => { +// vec![Identifier::ContractID(tx_id.to_owned(), index, id.into())] +// } +// Receipt::Transfer(TransferReceipt { +// id: from, +// to, +// asset_id, +// .. +// }) => { +// vec![ +// Identifier::ContractID(tx_id.to_owned(), index, from.into()), +// Identifier::ContractID(tx_id.to_owned(), index, to.into()), +// Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), +// ] +// } +// Receipt::TransferOut(TransferOutReceipt { +// id: from, +// to, +// asset_id, +// .. +// }) => { +// vec![ +// Identifier::ContractID(tx_id.to_owned(), index, from.into()), +// Identifier::ContractID(tx_id.to_owned(), index, to.into()), +// Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), +// ] +// } +// Receipt::MessageOut(MessageOutReceipt { +// sender, recipient, .. +// }) => { +// vec![ +// Identifier::Address(tx_id.to_owned(), index, sender.into()), +// Identifier::Address(tx_id.to_owned(), index, recipient.into()), +// ] +// } +// Receipt::Mint(MintReceipt { contract_id, .. }) +// | Receipt::Burn(BurnReceipt { contract_id, .. }) => { +// vec![Identifier::ContractID( +// tx_id.to_owned(), +// index, +// contract_id.into(), +// )] +// } +// _ => Vec::new(), +// } +// } diff --git a/crates/fuel-streams-executors/src/transactions.rs b/crates/fuel-streams-executors/src/transactions.rs index fc1f2763..17c7c8a4 100644 --- a/crates/fuel-streams-executors/src/transactions.rs +++ b/crates/fuel-streams-executors/src/transactions.rs @@ -1,5 +1,4 @@ -use fuel_streams_core::prelude::*; -use rayon::prelude::*; +use fuel_streams_core::{subjects::*, types::*}; use tokio::task::JoinHandle; use crate::*; @@ -7,76 +6,43 @@ use crate::*; impl Executor { pub fn process( &self, - tx_item: (usize, &Transaction), + (tx_index, tx): (usize, &Transaction), ) -> Vec>> { let block_height = self.block_height(); - packets_from_tx(tx_item, &block_height) - .iter() - .map(|packet| self.publish(packet)) - .collect() + let packet = packet_from_tx((tx_index, tx), &block_height); + vec![self.publish(&packet)] } } -fn packets_from_tx( - (index, tx): (usize, &Transaction), +fn packet_from_tx( + (tx_index, tx): (usize, &Transaction), block_height: &BlockHeight, -) -> Vec> { - let estimated_capacity = - 1 + tx.inputs.len() + tx.outputs.len() + tx.receipts.len(); +) -> RecordPacket { let tx_id = tx.id.clone(); let tx_status = tx.status.clone(); - let receipts = tx.receipts.clone(); - - // Main subject - let mut packets = Vec::with_capacity(estimated_capacity); - packets.push( - tx.to_packet( - TransactionsSubject { - block_height: Some(block_height.to_owned()), - index: Some(index), - tx_id: Some(tx_id.to_owned()), - status: Some(tx_status), - kind: Some(tx.kind.to_owned()), - } - .arc(), - ), - ); - - let index_u8 = index as u8; - let mut additional_packets: Vec> = - rayon::iter::once(&tx.kind) - .flat_map(|kind| identifiers(tx, kind, &tx_id, index_u8)) - .chain( - tx.inputs.par_iter().flat_map(|input| { - inputs::identifiers(input, &tx_id, index_u8) - }), - ) - .chain(tx.outputs.par_iter().flat_map(|output| { - outputs::identifiers(output, tx, &tx_id, index_u8) - })) - .chain(receipts.par_iter().flat_map(|receipt| { - receipts::identifiers(receipt, &tx_id, index_u8) - })) - .map(|identifier| TransactionsByIdSubject::from(identifier).arc()) - .map(|subject| tx.to_packet(subject)) - .collect(); - - packets.append(&mut additional_packets); - packets -} - -fn identifiers( - tx: &Transaction, - kind: &TransactionKind, - tx_id: &Bytes32, - index: u8, -) -> Vec { - match kind { - TransactionKind::Script => { - let script_data = &tx.script_data.to_owned().unwrap_or_default().0; - let script_tag = sha256(&script_data.0); - vec![Identifier::ScriptID(tx_id.to_owned(), index, script_tag)] - } - _ => Vec::new(), + let main_subject = TransactionsSubject { + block_height: Some(block_height.to_owned()), + tx_index: Some(tx_index as u32), + tx_id: Some(tx_id.to_owned()), + tx_status: Some(tx_status), + kind: Some(tx.kind.to_owned()), } + .arc(); + tx.to_packet(main_subject) } + +// fn identifiers( +// tx: &transaction, +// kind: &transactionkind, +// tx_id: &bytes32, +// index: u8, +// ) -> vec { +// match kind { +// transactionkind::script => { +// let script_data = &tx.script_data.to_owned().unwrap_or_default().0; +// let script_tag = sha256(&script_data.0); +// vec![identifier::scriptid(tx_id.to_owned(), index, script_tag)] +// } +// _ => vec::new(), +// } +// } diff --git a/crates/fuel-streams-executors/src/utxos.rs b/crates/fuel-streams-executors/src/utxos.rs index cdd01afe..7eabe1a6 100644 --- a/crates/fuel-streams-executors/src/utxos.rs +++ b/crates/fuel-streams-executors/src/utxos.rs @@ -1,4 +1,4 @@ -use fuel_streams_core::prelude::*; +use fuel_streams_core::{subjects::*, types::*}; use rayon::prelude::*; use tokio::task::JoinHandle; @@ -7,13 +7,24 @@ use crate::*; impl Executor { pub fn process( &self, - tx: &Transaction, + (tx_index, tx): (usize, &Transaction), ) -> Vec>> { + let block_height = self.block_height(); let tx_id = tx.id.clone(); let packets = tx .inputs .par_iter() - .filter_map(|input| utxo_packet(input, &tx_id)) + .enumerate() + .map(|(input_index, input)| { + let (utxo, subject) = main_subject( + block_height.clone(), + tx_index as u32, + input_index as u32, + tx_id.clone(), + input, + ); + utxo.to_packet(subject) + }) .collect::>(); packets @@ -23,20 +34,30 @@ impl Executor { } } -fn utxo_packet(input: &Input, tx_id: &Bytes32) -> Option> { +fn main_subject( + block_height: BlockHeight, + tx_index: u32, + input_index: u32, + tx_id: TxId, + input: &Input, +) -> (Utxo, Arc) { match input { Input::Contract(InputContract { utxo_id, .. }) => { let utxo = Utxo { utxo_id: utxo_id.to_owned(), - tx_id: tx_id.to_owned().into(), + tx_id: tx_id.to_owned(), ..Default::default() }; let subject = UtxosSubject { + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + input_index: Some(input_index), utxo_type: Some(UtxoType::Contract), utxo_id: Some(utxo_id.into()), } .arc(); - Some(utxo.to_packet(subject)) + (utxo, subject) } Input::Coin(InputCoin { utxo_id, amount, .. @@ -44,15 +65,19 @@ fn utxo_packet(input: &Input, tx_id: &Bytes32) -> Option> { let utxo = Utxo { utxo_id: utxo_id.to_owned(), amount: Some(*amount), - tx_id: tx_id.to_owned().into(), + tx_id: tx_id.to_owned(), ..Default::default() }; let subject = UtxosSubject { + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + input_index: Some(input_index), utxo_type: Some(UtxoType::Coin), utxo_id: Some(utxo_id.into()), } .arc(); - Some(utxo.to_packet(subject)) + (utxo, subject) } Input::Message( input @ InputMessage { @@ -66,7 +91,7 @@ fn utxo_packet(input: &Input, tx_id: &Bytes32) -> Option> { ) => { let utxo_id = input.computed_utxo_id(); let utxo = Utxo { - tx_id: tx_id.to_owned().into(), + tx_id: tx_id.to_owned(), utxo_id: utxo_id.to_owned(), sender: Some(sender.to_owned()), recipient: Some(recipient.to_owned()), @@ -75,11 +100,15 @@ fn utxo_packet(input: &Input, tx_id: &Bytes32) -> Option> { data: Some(data.to_owned()), }; let subject = UtxosSubject { + block_height: Some(block_height), + tx_id: Some(tx_id), + tx_index: Some(tx_index), + input_index: Some(input_index), utxo_type: Some(UtxoType::Message), - utxo_id: None, + utxo_id: Some(utxo_id.into()), } .arc(); - Some(utxo.to_packet(subject)) + (utxo, subject) } } } diff --git a/crates/fuel-streams-macros/Cargo.toml b/crates/fuel-streams-macros/Cargo.toml index 508a9d8a..3af67074 100644 --- a/crates/fuel-streams-macros/Cargo.toml +++ b/crates/fuel-streams-macros/Cargo.toml @@ -11,4 +11,14 @@ version = { workspace = true } rust-version = { workspace = true } [dependencies] +downcast-rs = "2.0.1" +serde = { workspace = true } +serde_json = { workspace = true } subject-derive = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +test-case = { workspace = true } + +[package.metadata.cargo-machete] +ignored = ["serde", "serde_json"] diff --git a/crates/fuel-streams-macros/README.md b/crates/fuel-streams-macros/README.md index bb9e5908..a24ae28a 100644 --- a/crates/fuel-streams-macros/README.md +++ b/crates/fuel-streams-macros/README.md @@ -47,9 +47,10 @@ The `Subject` derive macro allows you to easily implement the `Subject` trait fo Example: ```rust -use fuel_streams_macros::subject::{Subject, IntoSubject, SubjectBuildable}; +use fuel_streams_macros::subject::*; -#[derive(Subject, Debug, Clone, Default)] +#[derive(Subject, Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +#[subject_id = "test"] #[subject_wildcard = "test.>"] #[subject_format = "test.{field1}.{field2}"] struct TestSubject { diff --git a/crates/fuel-streams-macros/src/lib.rs b/crates/fuel-streams-macros/src/lib.rs index 74d1e248..32325c5c 100644 --- a/crates/fuel-streams-macros/src/lib.rs +++ b/crates/fuel-streams-macros/src/lib.rs @@ -1,16 +1,43 @@ #![doc = include_str!("../README.md")] pub mod subject { + pub use std::fmt::Debug; + + use downcast_rs::{impl_downcast, Downcast}; + pub use serde_json; pub use subject_derive::*; - /// This trait is used internally by the `Subject` derive macro to convert a struct into a - /// standard NATS subject. - pub trait IntoSubject: std::fmt::Debug + Send + Sync { + #[derive(thiserror::Error, Debug, PartialEq, Eq)] + pub enum SubjectError { + #[error("Invalid JSON conversion: {0}")] + InvalidJsonConversion(String), + #[error("Expected JSON object")] + ExpectedJsonObject, + } + + pub trait IntoSubject: Debug + Downcast + Send + Sync + 'static { + fn id(&self) -> &'static str; fn parse(&self) -> String; fn wildcard(&self) -> &'static str; + fn to_sql_where(&self) -> String; + } + impl_downcast!(IntoSubject); + + pub trait FromJsonString: + serde::Serialize + + serde::de::DeserializeOwned + + Clone + + Sized + + Debug + + Send + + Sync + + 'static + { + fn from_json(json: &str) -> Result; + fn to_json(&self) -> String; } - pub trait SubjectBuildable: std::fmt::Debug { + pub trait SubjectBuildable: Debug { fn new() -> Self; } diff --git a/crates/fuel-streams-macros/subject-derive/src/into_subject.rs b/crates/fuel-streams-macros/subject-derive/src/into_subject.rs index d1902737..631566ec 100644 --- a/crates/fuel-streams-macros/subject-derive/src/into_subject.rs +++ b/crates/fuel-streams-macros/subject-derive/src/into_subject.rs @@ -12,6 +12,9 @@ pub fn parse_fn(input: &DeriveInput, field_names: &[&Ident]) -> TokenStream { quote! { fn parse(&self) -> String { + if [#(&self.#field_names.is_none()),*].iter().all(|&x| *x) { + return Self::WILDCARD.to_string(); + } #(#parse_fields)* format!(#format_str) } @@ -21,7 +24,83 @@ pub fn parse_fn(input: &DeriveInput, field_names: &[&Ident]) -> TokenStream { pub fn wildcard_fn() -> TokenStream { quote! { fn wildcard(&self) -> &'static str { - Self::WILDCARD + Self::WILDCARD + } + } +} + +pub fn to_sql_where_fn(field_names: &[&Ident]) -> TokenStream { + let field_props = field_names.iter().map(|name| { + quote! { + match &self.#name { + Some(val) => Some(format!("{} = '{}'", stringify!(#name), val)), + None => None, + } + } + }); + + quote! { + fn to_sql_where(&self) -> String { + let pattern = self.parse(); + if pattern.ends_with(".>") { + return "TRUE".to_string(); + } + + let conditions = vec![#(#field_props),*].into_iter().filter_map(|x| x).collect::>(); + conditions.join(" AND ") + } + } +} + +pub fn from_json_fn(field_names: &[&Ident]) -> TokenStream { + let parse_fields = field_names.iter().map(|name| { + let name_str = name.to_string(); + quote! { + let #name = if let Some(value) = obj.get(#name_str) { + if value.is_null() { + None + } else { + let str_val = value.to_string().trim_matches('"').to_string(); + Some(str_val) + } + } else { + None + }; + } + }); + + quote! { + fn from_json(json: &str) -> Result { + let parsed: fuel_streams_macros::subject::serde_json::Value = + fuel_streams_macros::subject::serde_json::from_str(json) + .map_err(|e| SubjectError::InvalidJsonConversion(e.to_string()))?; + + let obj = match parsed.as_object() { + Some(obj) => obj, + None => return Err(SubjectError::ExpectedJsonObject), + }; + + #(#parse_fields)* + + Ok(Self::build( + #(#field_names.and_then(|v| v.parse().ok()),)* + )) + } + } +} + +pub fn id_fn() -> TokenStream { + quote! { + fn id(&self) -> &'static str { + Self::ID + } + } +} + +pub fn to_json_fn() -> TokenStream { + quote! { + fn to_json(&self) -> String { + fuel_streams_macros::subject::serde_json::to_string(self).unwrap() } } } diff --git a/crates/fuel-streams-macros/subject-derive/src/lib.rs b/crates/fuel-streams-macros/subject-derive/src/lib.rs index 3961a391..02007ed9 100644 --- a/crates/fuel-streams-macros/subject-derive/src/lib.rs +++ b/crates/fuel-streams-macros/subject-derive/src/lib.rs @@ -7,7 +7,10 @@ use proc_macro::TokenStream; use quote::quote; use syn::{parse_macro_input, DeriveInput}; -#[proc_macro_derive(Subject, attributes(subject_wildcard, subject_format))] +#[proc_macro_derive( + Subject, + attributes(subject_id, subject_wildcard, subject_format) +)] pub fn subject_derive(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); let name = &input.ident; @@ -16,9 +19,12 @@ pub fn subject_derive(input: TokenStream) -> TokenStream { let field_names = fields::names_from_fields(fields); let field_types = fields::types_from_fields(fields); - let parse_fn = into_subject::parse_fn(&input, &field_names); + let id_fn = into_subject::id_fn(); let wildcard_fn = into_subject::wildcard_fn(); - + let parse_fn = into_subject::parse_fn(&input, &field_names); + let to_sql_where_fn = into_subject::to_sql_where_fn(&field_names); + let from_json_fn = into_subject::from_json_fn(&field_names); + let to_json_fn = into_subject::to_json_fn(); let subject_expanded = subject::expanded(name, &field_names, &field_types, &input.attrs); @@ -34,11 +40,16 @@ pub fn subject_derive(input: TokenStream) -> TokenStream { } impl IntoSubject for #name { + #id_fn #parse_fn - #wildcard_fn + #to_sql_where_fn } + impl FromJsonString for #name { + #from_json_fn + #to_json_fn + } } .into() } diff --git a/crates/fuel-streams-macros/subject-derive/src/subject.rs b/crates/fuel-streams-macros/subject-derive/src/subject.rs index ee068544..910f25f3 100644 --- a/crates/fuel-streams-macros/subject-derive/src/subject.rs +++ b/crates/fuel-streams-macros/subject-derive/src/subject.rs @@ -20,6 +20,23 @@ fn create_with_methods<'a>( }) } +fn create_get_methods<'a>( + field_names: &'a [&'a Ident], + field_types: &'a [&'a Type], +) -> impl Iterator + 'a { + field_names + .iter() + .zip(field_types.iter()) + .map(|(name, ty)| { + let method_name = format_ident!("get_{}", name); + quote! { + pub fn #method_name(&self) -> &#ty { + &self.#name + } + } + }) +} + pub fn expanded<'a>( name: &'a Ident, field_names: &'a [&'a Ident], @@ -27,10 +44,13 @@ pub fn expanded<'a>( attrs: &'a [syn::Attribute], ) -> TokenStream { let with_methods = create_with_methods(field_names, field_types); + let get_methods = create_get_methods(field_names, field_types); let wildcard = crate::attrs::subject_attr("wildcard", attrs); + let id = crate::attrs::subject_attr("id", attrs); quote! { impl #name { + pub const ID: &'static str = #id; pub const WILDCARD: &'static str = #wildcard; pub fn build( @@ -55,7 +75,12 @@ pub fn expanded<'a>( std::sync::Arc::new(self) } + pub fn dyn_arc(self) -> std::sync::Arc { + self.arc() as std::sync::Arc + } + #(#with_methods)* + #(#get_methods)* } impl std::fmt::Display for #name { diff --git a/crates/fuel-streams-macros/tests/subject-derive.rs b/crates/fuel-streams-macros/tests/subject-derive.rs index b58d3423..d1714530 100644 --- a/crates/fuel-streams-macros/tests/subject-derive.rs +++ b/crates/fuel-streams-macros/tests/subject-derive.rs @@ -1,11 +1,14 @@ use fuel_streams_macros::subject::*; +use serde::{Deserialize, Serialize}; -#[derive(Subject, Debug, Clone, Default)] +#[derive(Subject, Debug, Clone, Default, Serialize, Deserialize)] +#[subject_id = "test"] #[subject_wildcard = "test.>"] -#[subject_format = "test.{field1}.{field2}"] +#[subject_format = "test.{field1}.{field2}.{field3}"] struct TestSubject { - field1: Option, - field2: Option, + pub field1: Option, + pub field2: Option, + pub field3: Option, } #[test] @@ -13,36 +16,193 @@ fn subject_derive_parse() { let subject = TestSubject { field1: Some("foo".to_string()), field2: Some(55), + field3: Some("bar".to_string()), }; assert_eq!(TestSubject::WILDCARD, "test.>"); - assert_eq!(subject.parse(), "test.foo.55"); + assert_eq!(subject.parse(), "test.foo.55.bar"); } #[test] fn subject_derive_wildcard() { - let wildcard = TestSubject::wildcard(None, Some(10)); - assert_eq!(wildcard, "test.*.10"); + let wildcard = TestSubject::wildcard(None, Some(10), None); + assert_eq!(wildcard, "test.*.10.*"); } #[test] fn subject_derive_build() { - let subject = TestSubject::build(Some("foo".into()), Some(55)); - assert_eq!(subject.parse(), "test.foo.55"); + let subject = + TestSubject::build(Some("foo".into()), Some(55), Some("bar".into())); + assert_eq!(subject.parse(), "test.foo.55.bar"); } #[test] fn subject_derive_builder() { let subject = TestSubject::new() .with_field1(Some("foo".into())) - .with_field2(Some(55)); - assert_eq!(subject.parse(), "test.foo.55"); + .with_field2(Some(55)) + .with_field3(Some("bar".into())); + assert_eq!(subject.parse(), "test.foo.55.bar"); } #[test] fn subject_derive_to_string() { - let subject = TestSubject::new() - .with_field1(Some("foo".into())) - .with_field2(Some(55)); - assert_eq!(&subject.to_string(), "test.foo.55") + let subject = TestSubject::new().with_field1(Some("foo".into())); + assert_eq!(&subject.to_string(), "test.foo.*.*") +} + +#[test] +fn subject_derive_sql_where_exact_match() { + let subject = TestSubject { + field1: Some("foo".to_string()), + field2: Some(55), + field3: Some("bar".to_string()), + }; + + assert_eq!(subject.parse(), "test.foo.55.bar"); + assert_eq!( + subject.to_sql_where(), + "field1 = 'foo' AND field2 = '55' AND field3 = 'bar'" + ); +} + +#[test] +fn subject_derive_sql_where_wildcards() { + let subject = TestSubject { + field1: None, + field2: Some(55), + field3: Some("bar".to_string()), + }; + + assert_eq!(subject.parse(), "test.*.55.bar"); + assert_eq!(subject.to_sql_where(), "field2 = '55' AND field3 = 'bar'"); +} + +#[test] +fn subject_derive_sql_where_greater_than() { + let subject = TestSubject { + field1: Some("foo".to_string()), + field2: None, + field3: Some("bar".to_string()), + }; + + assert_eq!(subject.to_sql_where(), "field1 = 'foo' AND field3 = 'bar'"); +} + +#[test] +fn subject_derive_sql_where_table_only() { + let subject = TestSubject { + field1: None, + field2: None, + field3: None, + }; + + assert_eq!(subject.parse(), "test.>"); + assert_eq!(subject.to_sql_where(), "TRUE"); + + let subject2 = TestSubject::default(); + assert_eq!(subject2.parse(), "test.>"); + assert_eq!(subject2.to_sql_where(), "TRUE"); + + let subject3 = TestSubject::new(); + assert_eq!(subject3.parse(), "test.>"); + assert_eq!(subject3.to_sql_where(), "TRUE"); +} + +#[test] +fn subject_derive_from_json() { + // Test with all fields + let subject = TestSubject::from_json( + r#"{"field1": "foo", "field2": 55, "field3": "bar"}"#, + ) + .unwrap(); + assert_eq!(subject.parse(), "test.foo.55.bar"); + + // Test with partial fields + let subject = TestSubject::from_json(r#"{"field1": "foo"}"#).unwrap(); + assert_eq!(subject.parse(), "test.foo.*.*"); + + // Test with empty object + let subject = TestSubject::from_json("{}").unwrap(); + assert_eq!(subject.parse(), "test.>"); +} + +#[test] +fn subject_derive_from_json_error() { + // Test error cases + let invalid_json = TestSubject::from_json("{invalid}"); + assert!(matches!( + invalid_json, + Err(SubjectError::InvalidJsonConversion(_)) + )); + + let invalid_type = TestSubject::from_json("[1, 2, 3]"); + assert!(matches!( + invalid_type, + Err(SubjectError::ExpectedJsonObject) + )); +} + +#[test] +fn subject_derive_id() { + let subject = TestSubject::new(); + assert_eq!(TestSubject::ID, "test"); + assert_eq!(subject.id(), "test"); +} + +#[test] +fn subject_derive_to_json() { + // Test with all fields + let subject = TestSubject { + field1: Some("foo".to_string()), + field2: Some(55), + field3: Some("bar".to_string()), + }; + assert_eq!( + subject.to_json(), + r#"{"field1":"foo","field2":55,"field3":"bar"}"# + ); + + // Test with partial fields + let subject = TestSubject { + field1: Some("foo".to_string()), + field2: None, + field3: None, + }; + assert_eq!( + subject.to_json(), + r#"{"field1":"foo","field2":null,"field3":null}"# + ); + + // Test with no fields + let subject = TestSubject::new(); + assert_eq!( + subject.to_json(), + r#"{"field1":null,"field2":null,"field3":null}"# + ); +} + +#[test] +fn subject_derive_json_roundtrip() { + // Create a subject with mixed values and nulls + let original = TestSubject { + field1: Some("test".to_string()), + field2: None, + field3: Some("value".to_string()), + }; + + // Convert to JSON string + let json_str = original.to_json(); + + // Convert back from JSON string + let roundtrip = TestSubject::from_json(&json_str).unwrap(); + + // Verify the fields match + assert_eq!(roundtrip.field1, original.field1); + assert_eq!(roundtrip.field2, original.field2); + assert_eq!(roundtrip.field3, original.field3); + + // Verify the parsed subject string is the same + assert_eq!(roundtrip.parse(), "test.test.*.value"); + assert_eq!(original.parse(), "test.test.*.value"); } diff --git a/crates/fuel-streams-nats/src/error.rs b/crates/fuel-streams-nats/src/error.rs deleted file mode 100644 index c7159900..00000000 --- a/crates/fuel-streams-nats/src/error.rs +++ /dev/null @@ -1,56 +0,0 @@ -use async_nats::{ - error, - jetstream::{ - consumer::StreamErrorKind, - context::{ - CreateKeyValueErrorKind, - CreateStreamErrorKind, - PublishError, - }, - kv::{PutError, WatchErrorKind}, - stream::ConsumerErrorKind, - }, - ConnectErrorKind, -}; -use displaydoc::Display as DisplayDoc; -use thiserror::Error; - -use super::types::PayloadSize; - -#[derive(Error, DisplayDoc, Debug)] -pub enum NatsError { - /// Payload size exceeds maximum allowed: subject '{subject_name}' has size {payload_size} which is larger than the maximum of {max_payload_size} - PayloadTooLarge { - subject_name: String, - payload_size: PayloadSize, - max_payload_size: PayloadSize, - }, - - /// Failed to connect to NATS server at {url} - ConnectionError { - url: String, - #[source] - source: error::Error, - }, - - /// Failed to create Key-Value Store in NATS - StoreCreation(#[from] error::Error), - - /// Failed to publish item to Key-Value Store - StorePublish(#[from] PutError), - - /// Failed to subscribe to subject in Key-Value Store - StoreSubscribe(#[from] error::Error), - - /// Failed to publish item to NATS stream - StreamPublish(#[from] PublishError), - - /// Failed to create NATS stream - StreamCreation(#[from] error::Error), - - /// Failed to create consumer for NATS stream - ConsumerCreate(#[from] error::Error), - - /// Failed to consume messages from NATS stream - ConsumerMessages(#[from] error::Error), -} diff --git a/crates/fuel-streams-nats/src/lib.rs b/crates/fuel-streams-nats/src/lib.rs deleted file mode 100644 index 762ae550..00000000 --- a/crates/fuel-streams-nats/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -/// Houses shared APIs for interacting with NATS for sv-publisher and fuel-streams crates -/// As much as possible, the public interface/APIS should be agnostic of NATS. These can then be extended -/// in the sv-publisher and fuel-streams crates to provide a more opinionated API towards -/// their specific use-cases. -pub mod error; -pub mod nats_client; -pub mod nats_client_opts; -pub mod nats_namespace; -pub mod types; - -pub use error::*; -pub use nats_client::*; -pub use nats_client_opts::*; -pub use nats_namespace::*; -pub use types::*; diff --git a/crates/fuel-streams-nats/src/nats_client.rs b/crates/fuel-streams-nats/src/nats_client.rs deleted file mode 100644 index fd3474fa..00000000 --- a/crates/fuel-streams-nats/src/nats_client.rs +++ /dev/null @@ -1,105 +0,0 @@ -use async_nats::{ - error, - jetstream::{context::CreateKeyValueErrorKind, kv}, -}; -use tracing::info; - -use super::{types::*, NatsClientOpts, NatsError, NatsNamespace}; - -/// NatsClient is a wrapper around the NATS client that provides additional functionality -/// geared towards fuel-streaming use-cases -/// -/// # Examples -/// -/// Creating a new `NatsClient`: -/// -/// ```no_run -/// use fuel_streams_nats::*; -/// -/// async fn example() -> Result<(), Box> { -/// let opts = NatsClientOpts::public_opts(); -/// let client = NatsClient::connect(&opts).await?; -/// Ok(()) -/// } -/// ``` -/// -/// Creating a key-value store: -/// -/// ```no_run -/// use fuel_streams_nats::*; -/// use async_nats::jetstream::kv; -/// -/// async fn example() -> Result<(), Box> { -/// let opts = NatsClientOpts::public_opts(); -/// let client = NatsClient::connect(&opts).await?; -/// let kv_config = kv::Config { -/// bucket: "my-bucket".into(), -/// ..Default::default() -/// }; -/// -/// let store = client.get_or_create_kv_store(kv_config).await?; -/// Ok(()) -/// } -/// ``` -#[derive(Debug, Clone)] -pub struct NatsClient { - /// The underlying NATS client - pub nats_client: async_nats::Client, - /// The JetStream context for this client - pub jetstream: JetStreamContext, - /// The namespace used for this client - pub namespace: NatsNamespace, - /// The options used to create this client - pub opts: NatsClientOpts, -} - -impl NatsClient { - pub async fn connect(opts: &NatsClientOpts) -> Result { - let url = &opts.get_url(); - let namespace = opts.namespace.clone(); - let nats_client = - opts.connect_opts().connect(url).await.map_err(|e| { - NatsError::ConnectionError { - url: url.to_string(), - source: e, - } - })?; - - let jetstream = match opts.domain.clone() { - None => async_nats::jetstream::new(nats_client.clone()), - Some(domain) => { - async_nats::jetstream::with_domain(nats_client.clone(), domain) - } - }; - info!("Connected to NATS server at {}", url); - - Ok(Self { - nats_client, - jetstream, - opts: opts.to_owned(), - namespace, - }) - } - - pub async fn get_or_create_kv_store( - &self, - options: kv::Config, - ) -> Result> { - let bucket = options.bucket.clone(); - let store = self.jetstream.get_key_value(&bucket).await; - let store = match store { - Ok(store) => store, - Err(_) => self.jetstream.create_key_value(options).await?, - }; - - Ok(store) - } - - pub fn is_connected(&self) -> bool { - self.state() == ConnectionState::Connected - } - - fn state(&self) -> ConnectionState { - self.nats_client.connection_state() - } -} diff --git a/crates/fuel-streams-nats/src/nats_client_opts.rs b/crates/fuel-streams-nats/src/nats_client_opts.rs deleted file mode 100644 index f6aaa0a3..00000000 --- a/crates/fuel-streams-nats/src/nats_client_opts.rs +++ /dev/null @@ -1,227 +0,0 @@ -use std::time::Duration; - -use async_nats::ConnectOptions; - -use super::NatsNamespace; - -#[derive(Debug, Clone, Eq, PartialEq, Default)] -pub enum NatsAuth { - Admin, - System, - #[default] - Public, - Custom(String, String), -} - -impl NatsAuth { - fn credentials_from_env(&self) -> (String, String) { - match self { - NatsAuth::Admin => ( - dotenvy::var("NATS_ADMIN_USER") - .expect("NATS_ADMIN_USER must be set"), - dotenvy::var("NATS_ADMIN_PASS") - .expect("NATS_ADMIN_PASS must be set"), - ), - NatsAuth::System => ( - dotenvy::var("NATS_SYSTEM_USER") - .expect("NATS_SYSTEM_USER must be set"), - dotenvy::var("NATS_SYSTEM_PASS") - .expect("NATS_SYSTEM_PASS must be set"), - ), - NatsAuth::Public => ("default_user".to_string(), "".to_string()), - NatsAuth::Custom(user, pass) => { - (user.to_string(), pass.to_string()) - } - } - } -} - -/// Configuration options for connecting to NATS -/// -/// # Examples -/// -/// ```no_run -/// use fuel_streams_nats::*; -/// -/// // Create with URL -/// let opts = NatsClientOpts::new("nats://localhost:4222".to_string(), Some(NatsAuth::Admin)); -/// -/// // Create with admin credentials from environment -/// let opts = NatsClientOpts::admin_opts(); -/// -/// // Create with system credentials from environment -/// let opts = NatsClientOpts::system_opts(); -/// -/// // Create with public credentials -/// let opts = NatsClientOpts::public_opts(); -/// ``` -/// -/// Customize options: -/// -/// ```no_run -/// use fuel_streams_nats::*; -/// -/// let opts = NatsClientOpts::new("nats://localhost:4222".to_string(), Some(NatsAuth::Admin)) -/// .with_domain("mydomain") -/// .with_user("myuser") -/// .with_password("mypass") -/// .with_timeout(10); -/// ``` -#[derive(Debug, Clone)] -pub struct NatsClientOpts { - /// The URL of the NATS server. - pub(crate) url: String, - /// The namespace used as a prefix for NATS streams, consumers, and subject names. - pub(crate) namespace: NatsNamespace, - /// The timeout in seconds for NATS operations. - pub(crate) timeout_secs: u64, - /// The domain to use for the NATS client. - pub(crate) domain: Option, - /// The user to use for the NATS client. - pub(crate) user: Option, - /// The password to use for the NATS client. - pub(crate) password: Option, -} - -impl NatsClientOpts { - pub fn new(url: String, auth: Option) -> Self { - let (user, pass) = auth.unwrap_or_default().credentials_from_env(); - Self { - url, - namespace: NatsNamespace::default(), - timeout_secs: 5, - domain: None, - user: Some(user), - password: Some(pass), - } - } - - pub fn from_env(auth: Option) -> Self { - let url = dotenvy::var("NATS_URL").expect("NATS_URL must be set"); - Self::new(url, auth) - } - pub fn admin_opts() -> Self { - Self::from_env(Some(NatsAuth::Admin)) - } - pub fn system_opts() -> Self { - Self::from_env(Some(NatsAuth::System)) - } - pub fn public_opts() -> Self { - Self::from_env(Some(NatsAuth::Public)) - } - - pub fn get_url(&self) -> String { - self.url.clone() - } - - pub fn with_url>(self, url: S) -> Self { - Self { - url: url.into(), - ..self - } - } - - pub fn with_domain>(self, domain: S) -> Self { - Self { - domain: Some(domain.into()), - ..self - } - } - - pub fn with_user>(self, user: S) -> Self { - Self { - user: Some(user.into()), - ..self - } - } - - pub fn with_password>(self, password: S) -> Self { - Self { - password: Some(password.into()), - ..self - } - } - - #[cfg(any(test, feature = "test-helpers"))] - pub fn with_rdn_namespace(self) -> Self { - let namespace = format!(r"namespace-{}", Self::random_int()); - self.with_namespace(&namespace) - } - - #[cfg(any(test, feature = "test-helpers"))] - pub fn with_namespace(self, namespace: &str) -> Self { - let namespace = NatsNamespace::Custom(namespace.to_string()); - Self { namespace, ..self } - } - - pub fn with_timeout(self, secs: u64) -> Self { - Self { - timeout_secs: secs, - ..self - } - } - - pub(super) fn connect_opts(&self) -> ConnectOptions { - let opts = match (self.user.clone(), self.password.clone()) { - (Some(user), Some(pass)) => { - ConnectOptions::with_user_and_password(user, pass) - } - _ => ConnectOptions::new(), - }; - - opts.connection_timeout(Duration::from_secs(self.timeout_secs)) - .max_reconnects(1) - .name(Self::conn_id()) - } - - // This will be useful for debugging and monitoring connections - fn conn_id() -> String { - format!(r"connection-{}", Self::random_int()) - } - - fn random_int() -> u32 { - use rand::Rng; - rand::thread_rng().gen() - } -} - -#[cfg(test)] -mod tests { - use std::env; - - use super::*; - - #[test] - fn test_role_credentials() { - // Setup - env::set_var("NATS_ADMIN_USER", "admin"); - env::set_var("NATS_ADMIN_PASS", "admin_pass"); - - // Test Admin role credentials - let (user, pass) = NatsAuth::Admin.credentials_from_env(); - assert_eq!(user, "admin"); - assert_eq!(pass, "admin_pass"); - - // Cleanup - env::remove_var("NATS_ADMIN_USER"); - env::remove_var("NATS_ADMIN_PASS"); - } - - #[test] - fn test_from_env_with_role() { - // Setup - env::set_var("NATS_URL", "nats://localhost:4222"); - env::set_var("NATS_ADMIN_USER", "admin"); - env::set_var("NATS_ADMIN_PASS", "admin_pass"); - - // Test Admin role - let opts = NatsClientOpts::from_env(Some(NatsAuth::Admin)); - assert_eq!(opts.user, Some("admin".to_string())); - assert_eq!(opts.password, Some("admin_pass".to_string())); - - // Cleanup - env::remove_var("NATS_URL"); - env::remove_var("NATS_ADMIN_USER"); - env::remove_var("NATS_ADMIN_PASS"); - } -} diff --git a/crates/fuel-streams-nats/src/nats_namespace.rs b/crates/fuel-streams-nats/src/nats_namespace.rs deleted file mode 100644 index 947e8760..00000000 --- a/crates/fuel-streams-nats/src/nats_namespace.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::fmt; - -static DEFAULT_NAMESPACE: &str = "fuel"; - -/// Represents a namespace for NATS subjects and streams. -/// -/// # Examples -/// -/// ``` -/// use fuel_streams_nats::NatsNamespace; -/// -/// let default_namespace = NatsNamespace::default(); -/// assert_eq!(default_namespace.to_string(), "fuel"); -/// -/// let custom_namespace = NatsNamespace::Custom("my_custom_namespace".to_string()); -/// assert_eq!(custom_namespace.to_string(), "my_custom_namespace"); -/// ``` -#[derive(Debug, Clone, Default)] -pub enum NatsNamespace { - #[default] - Fuel, - Custom(String), -} - -impl fmt::Display for NatsNamespace { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let value = match self { - NatsNamespace::Fuel => DEFAULT_NAMESPACE, - NatsNamespace::Custom(s) => s, - }; - write!(f, "{value}") - } -} - -impl From for String { - fn from(val: NatsNamespace) -> Self { - val.to_string() - } -} - -impl NatsNamespace { - /// Creates a subject name by combining the namespace with the given value. - /// - /// # Examples - /// - /// ``` - /// use fuel_streams_nats::NatsNamespace; - /// - /// let namespace = NatsNamespace::default(); - /// assert_eq!(namespace.subject_name("test"), "fuel.test"); - /// - /// let custom_namespace = NatsNamespace::Custom("custom".to_string()); - /// assert_eq!(custom_namespace.subject_name("test"), "custom.test"); - /// ``` - pub fn subject_name(&self, val: &str) -> String { - format!("{self}.{}", val) - } - - /// Creates a stream name by combining the namespace with the given value. - /// - /// # Examples - /// - /// ``` - /// use fuel_streams_nats::NatsNamespace; - /// - /// let namespace = NatsNamespace::default(); - /// assert_eq!(namespace.stream_name("test"), "fuel_test"); - /// - /// let custom_namespace = NatsNamespace::Custom("custom".to_string()); - /// assert_eq!(custom_namespace.stream_name("test"), "custom_test"); - /// ``` - pub fn stream_name(&self, val: &str) -> String { - format!("{self}_{val}") - } -} diff --git a/crates/fuel-streams-nats/src/types.rs b/crates/fuel-streams-nats/src/types.rs deleted file mode 100644 index 5240a200..00000000 --- a/crates/fuel-streams-nats/src/types.rs +++ /dev/null @@ -1,24 +0,0 @@ -pub use async_nats::{ - connection::State as ConnectionState, - jetstream::{ - consumer::{ - pull::{ - Config as PullConsumerConfig, - MessagesError, - Stream as PullConsumerStream, - }, - AckPolicy, - Config as ConsumerConfig, - Consumer as NatsConsumer, - DeliverPolicy as NatsDeliverPolicy, - }, - kv::Config as KvStoreConfig, - stream::Config as NatsStreamConfig, - Context as JetStreamContext, - Message as NatsMessage, - }, - Client as AsyncNatsClient, - ConnectOptions as NatsConnectOpts, -}; - -pub type PayloadSize = usize; diff --git a/crates/fuel-streams-storage/Cargo.toml b/crates/fuel-streams-storage/Cargo.toml deleted file mode 100644 index 7eb8af8d..00000000 --- a/crates/fuel-streams-storage/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -name = "fuel-streams-storage" -description = "Srategies and adapters for storing fuel streams in transient and file storage systems (i.e. NATS and S3)" -authors = { workspace = true } -keywords = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -repository = { workspace = true } -version = { workspace = true } -rust-version = { workspace = true } - -[dependencies] -async-trait = "0.1.83" -aws-config = { version = "1.5.10", features = ["behavior-version-latest"] } -aws-sdk-s3 = "1.65.0" -displaydoc = { workspace = true } -dotenvy = { workspace = true } -rand = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } - -[dev-dependencies] -pretty_assertions = { workspace = true } -serde_json = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] } -tracing-test = "0.2.0" - -[features] -default = [] -test-helpers = [] -bench-helpers = [] diff --git a/crates/fuel-streams-storage/src/lib.rs b/crates/fuel-streams-storage/src/lib.rs deleted file mode 100644 index 130a9ba0..00000000 --- a/crates/fuel-streams-storage/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -// TODO: Introduce Adapters for Transient and FileStorage (NATS and S3 clients would implement those) -pub mod retry; -pub mod s3; -pub mod storage; -pub mod storage_config; - -pub use s3::*; -pub use storage::*; -pub use storage_config::*; diff --git a/crates/fuel-streams-storage/src/retry.rs b/crates/fuel-streams-storage/src/retry.rs deleted file mode 100644 index f50f25fb..00000000 --- a/crates/fuel-streams-storage/src/retry.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::{future::Future, sync::LazyLock, time::Duration}; - -use tracing; - -pub static STORAGE_MAX_RETRIES: LazyLock = LazyLock::new(|| { - dotenvy::var("STORAGE_MAX_RETRIES") - .ok() - .and_then(|val| val.parse().ok()) - .unwrap_or(5) -}); - -#[derive(Debug, Clone)] -pub struct RetryConfig { - pub max_retries: u32, - pub initial_backoff: Duration, -} - -impl Default for RetryConfig { - fn default() -> Self { - Self { - max_retries: *STORAGE_MAX_RETRIES as u32, - initial_backoff: Duration::from_millis(100), - } - } -} - -pub async fn with_retry( - config: &RetryConfig, - operation_name: &str, - f: F, -) -> Result -where - F: Fn() -> Fut, - Fut: Future>, - E: std::fmt::Display, -{ - let mut attempt = 0; - let mut last_error = None; - while attempt < config.max_retries { - match f().await { - Ok(result) => return Ok(result), - Err(e) => { - last_error = Some(e); - attempt += 1; - - if attempt < config.max_retries { - let backoff = - config.initial_backoff * 2u32.pow(attempt - 1); - tracing::warn!( - "{} failed, attempt {}/{}: {}. Retrying in {:?}", - operation_name, - attempt, - config.max_retries, - last_error.as_ref().unwrap(), - backoff - ); - tokio::time::sleep(backoff).await; - } - } - } - } - - Err(last_error.unwrap()) -} - -#[cfg(test)] -mod tests { - use std::sync::{ - atomic::{AtomicU32, Ordering}, - Arc, - }; - - use super::*; - - #[tokio::test] - async fn test_retry_mechanism() { - let config = RetryConfig { - max_retries: 3, - initial_backoff: Duration::from_millis(10), /* Shorter duration for tests */ - }; - - let attempt_counter = Arc::new(AtomicU32::new(0)); - let counter_clone = attempt_counter.clone(); - - let result: Result<(), String> = with_retry(&config, "test", || { - let value = counter_clone.clone(); - async move { - let attempt = value.fetch_add(1, Ordering::SeqCst); - if attempt < 2 { - // Fail first two attempts - Err("Simulated failure".to_string()) - } else { - // Succeed on third attempt - Ok(()) - } - } - }) - .await; - - assert!(result.is_ok()); - assert_eq!(attempt_counter.load(Ordering::SeqCst), 3); - } - - #[tokio::test] - async fn test_retry_exhaustion() { - let config = RetryConfig { - max_retries: 3, - initial_backoff: Duration::from_millis(10), - }; - - let result: Result<(), String> = - with_retry(&config, "test", || async { - Err("Always fails".to_string()) - }) - .await; - - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), "Always fails"); - } -} diff --git a/crates/fuel-streams-storage/src/s3/mod.rs b/crates/fuel-streams-storage/src/s3/mod.rs deleted file mode 100644 index ff459c80..00000000 --- a/crates/fuel-streams-storage/src/s3/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod s3_client; -mod s3_client_opts; - -pub use s3_client::*; -pub use s3_client_opts::*; diff --git a/crates/fuel-streams-storage/src/s3/s3_client.rs b/crates/fuel-streams-storage/src/s3/s3_client.rs deleted file mode 100644 index 0483ee5d..00000000 --- a/crates/fuel-streams-storage/src/s3/s3_client.rs +++ /dev/null @@ -1,409 +0,0 @@ -use async_trait::async_trait; -use aws_config::BehaviorVersion; -use aws_sdk_s3::Client; - -use super::s3_client_opts::S3StorageOpts; -use crate::{ - retry::{with_retry, RetryConfig}, - storage::{Storage, StorageError}, - StorageConfig, -}; - -#[derive(Debug, Clone)] -pub struct S3Storage { - client: Client, - config: S3StorageOpts, - retry_config: RetryConfig, -} - -#[async_trait] -impl Storage for S3Storage { - type Config = S3StorageOpts; - - async fn new(config: Self::Config) -> Result { - let aws_config = aws_config::defaults(BehaviorVersion::latest()) - .endpoint_url(config.endpoint_url()) - .region(config.region()) - .no_credentials() - .load() - .await; - - let s3_config = aws_sdk_s3::config::Builder::from(&aws_config) - .force_path_style(true) - .disable_s3_express_session_auth(true) - .build(); - - let client = aws_sdk_s3::Client::from_conf(s3_config); - Ok(Self { - client, - config, - retry_config: RetryConfig::default(), - }) - } - - async fn store( - &self, - key: &str, - data: Vec, - ) -> Result<(), StorageError> { - with_retry(&self.retry_config, "store", || { - let data = data.clone(); - async move { - #[allow(clippy::identity_op)] - const LARGE_FILE_THRESHOLD: usize = 1 * 1024 * 1024; // 1MB - if data.len() >= LARGE_FILE_THRESHOLD { - tracing::debug!( - "Uploading file to S3 using multipart_upload" - ); - self.upload_multipart(key, data).await - } else { - tracing::debug!("Uploading file to S3 using put_object"); - self.put_object(key, data).await - } - } - }) - .await - } - - async fn retrieve(&self, key: &str) -> Result, StorageError> { - with_retry(&self.retry_config, "retrieve", || async { - let result = self - .client - .get_object() - .bucket(self.config.bucket()) - .key(key) - .send() - .await - .map_err(|e| StorageError::RetrieveError(e.to_string()))?; - - Ok(result - .body - .collect() - .await - .map_err(|e| StorageError::RetrieveError(e.to_string()))? - .into_bytes() - .to_vec()) - }) - .await - } - - async fn delete(&self, key: &str) -> Result<(), StorageError> { - with_retry(&self.retry_config, "delete", || async { - self.client - .delete_object() - .bucket(self.config.bucket()) - .key(key) - .send() - .await - .map_err(|e| StorageError::DeleteError(e.to_string()))?; - Ok(()) - }) - .await - } -} - -impl S3Storage { - pub async fn create_bucket(&self) -> Result<(), StorageError> { - self.client - .create_bucket() - .bucket(self.config.bucket()) - .send() - .await - .map_err(|e| StorageError::StoreError(e.to_string()))?; - Ok(()) - } - - async fn put_object( - &self, - key: &str, - object: Vec, - ) -> Result<(), StorageError> { - self.client - .put_object() - .bucket(self.config.bucket()) - .key(key) - .body(object.into()) - .send() - .await - .map_err(|e| StorageError::StoreError(e.to_string()))?; - - Ok(()) - } - - async fn upload_multipart( - &self, - key: &str, - data: Vec, - ) -> Result<(), StorageError> { - const CHUNK_SIZE: usize = 5 * 1024 * 1024; // 5MB chunks - - // Create multipart upload - let create_multipart = self - .client - .create_multipart_upload() - .bucket(self.config.bucket()) - .key(key) - .send() - .await - .map_err(|e| { - StorageError::StoreError(format!( - "Failed to create multipart upload: {}", - e - )) - })?; - - let upload_id = create_multipart.upload_id().ok_or_else(|| { - StorageError::StoreError("Failed to get upload ID".to_string()) - })?; - - let mut completed_parts = Vec::new(); - let chunks = data.chunks(CHUNK_SIZE); - let total_chunks = chunks.len(); - - // Upload parts - for (i, chunk) in chunks.enumerate() { - let part_number = (i + 1) as i32; - - match self - .client - .upload_part() - .bucket(self.config.bucket()) - .key(key) - .upload_id(upload_id) - .body(chunk.to_vec().into()) - .part_number(part_number) - .send() - .await - { - Ok(response) => { - if let Some(e_tag) = response.e_tag() { - completed_parts.push( - aws_sdk_s3::types::CompletedPart::builder() - .e_tag(e_tag) - .part_number(part_number) - .build(), - ); - } - } - Err(err) => { - // Abort the multipart upload if a part fails - self.client - .abort_multipart_upload() - .bucket(self.config.bucket()) - .key(key) - .upload_id(upload_id) - .send() - .await - .map_err(|e| { - StorageError::StoreError(format!( - "Failed to abort multipart upload: {}", - e - )) - })?; - - return Err(StorageError::StoreError(format!( - "Failed to upload part: {}", - err - ))); - } - } - - tracing::debug!( - "Uploaded part {}/{} for key={}", - part_number, - total_chunks, - key - ); - } - - // Complete multipart upload - self.client - .complete_multipart_upload() - .bucket(self.config.bucket()) - .key(key) - .upload_id(upload_id) - .multipart_upload( - aws_sdk_s3::types::CompletedMultipartUpload::builder() - .set_parts(Some(completed_parts)) - .build(), - ) - .send() - .await - .map_err(|e| { - StorageError::StoreError(format!( - "Failed to complete multipart upload: {}", - e - )) - })?; - - Ok(()) - } - - #[cfg(any(test, feature = "test-helpers"))] - pub async fn new_for_testing() -> Result { - dotenvy::dotenv().ok(); - - use crate::{StorageEnv, StorageRole}; - let config = S3StorageOpts::new(StorageEnv::Local, StorageRole::Admin) - .with_random_namespace(); - - let aws_config = aws_config::defaults(BehaviorVersion::latest()) - .endpoint_url(config.endpoint_url()) - .region(config.region()) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - "test", "test", None, None, "static", - )) - .load() - .await; - - let s3_config = aws_sdk_s3::config::Builder::from(&aws_config) - .force_path_style(true) - .disable_s3_express_session_auth(true) - .build(); - - let client = aws_sdk_s3::Client::from_conf(s3_config); - - // Ensure bucket exists before running tests - let storage = Self { - client, - config, - retry_config: RetryConfig::default(), - }; - storage.ensure_bucket().await?; - Ok(storage) - } - - #[cfg(any(test, feature = "test-helpers"))] - pub async fn ensure_bucket(&self) -> Result<(), StorageError> { - // Check if bucket exists - let exists = self - .client - .head_bucket() - .bucket(self.config.bucket()) - .send() - .await - .is_ok(); - - // Create bucket if it doesn't exist - if !exists { - self.create_bucket().await?; - } - Ok(()) - } - - pub fn with_retry_config(mut self, config: RetryConfig) -> Self { - self.retry_config = config; - self - } -} - -#[cfg(test)] -mod tests { - use tracing_test::traced_test; - - use super::*; - use crate::storage::Storage; - - #[tokio::test] - async fn test_basic_operations() { - let storage = S3Storage::new_for_testing().await.unwrap(); - - // Test store and retrieve - let key = "test-key"; - let content = b"Hello, Storage!".to_vec(); - - storage.store(key, content.clone()).await.unwrap(); - let retrieved = storage.retrieve(key).await.unwrap(); - assert_eq!(retrieved, content); - - // Test delete - storage.delete(key).await.unwrap(); - let result = storage.retrieve(key).await; - assert!(result.is_err()); - } - - #[tokio::test] - #[traced_test] - async fn test_file_size_threshold() { - let storage = S3Storage::new_for_testing().await.unwrap(); - - // Test small file (under 1MB) - let small_content = vec![0u8; 500 * 1024]; - storage - .store("small-file", small_content.clone()) - .await - .unwrap(); - assert!(logs_contain("put_object")); - - // Verify small file was stored correctly - let retrieved_small = storage.retrieve("small-file").await.unwrap(); - assert_eq!(retrieved_small, small_content); - - // Test large file (over 1MB) - let large_content = vec![0u8; 2 * 1024 * 1024]; - storage - .store("large-file", large_content.clone()) - .await - .unwrap(); - assert!(logs_contain("multipart_upload")); - - // Verify large file was stored correctly - let retrieved_large = storage.retrieve("large-file").await.unwrap(); - assert_eq!(retrieved_large, large_content); - } - - #[tokio::test] - async fn test_multipart_upload_with_multiple_chunks() { - let storage = S3Storage::new_for_testing().await.unwrap(); - - // Create a file that will require exactly 3 chunks (15MB + 1 byte) - // Since chunk size is 5MB, this will create 3 chunks: - // Chunk 1: 5MB - // Chunk 2: 5MB - // Chunk 3: 5MB + 1 byte - let content_size = (5 * 1024 * 1024 * 3) + 1; - let content: Vec = (0..content_size) - .map(|i| (i % 255) as u8) // Create pattern to verify data integrity - .collect(); - - let key = "multiple-chunks"; - - // Store the file - storage.store(key, content.clone()).await.unwrap(); - - // Retrieve and verify the file immediately after upload - let retrieved_after_upload = storage.retrieve(key).await.unwrap(); - assert_eq!( - retrieved_after_upload.len(), - content.len(), - "Retrieved file size should match original" - ); - assert_eq!( - retrieved_after_upload, content, - "Retrieved file content should match original" - ); - - // Wait a moment and retrieve again to verify persistence - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - let retrieved_after_wait = storage.retrieve(key).await.unwrap(); - assert_eq!( - retrieved_after_wait.len(), - content.len(), - "Retrieved file size should still match after waiting" - ); - assert_eq!( - retrieved_after_wait, content, - "Retrieved file content should still match after waiting" - ); - - // Clean up - storage.delete(key).await.unwrap(); - - // Verify deletion - let result = storage.retrieve(key).await; - assert!( - result.is_err(), - "File should no longer exist after deletion" - ); - } -} diff --git a/crates/fuel-streams-storage/src/s3/s3_client_opts.rs b/crates/fuel-streams-storage/src/s3/s3_client_opts.rs deleted file mode 100644 index 782c4f00..00000000 --- a/crates/fuel-streams-storage/src/s3/s3_client_opts.rs +++ /dev/null @@ -1,157 +0,0 @@ -use std::str::FromStr; - -use aws_config::Region; - -use crate::{StorageConfig, StorageEnv, StorageRole}; - -#[derive(Debug, Clone, Default)] -pub struct S3StorageOpts { - pub env: StorageEnv, - pub role: StorageRole, - pub namespace: Option, -} - -impl StorageConfig for S3StorageOpts { - fn new(env: StorageEnv, role: StorageRole) -> Self { - Self { - env, - role, - namespace: None, - } - } - - fn from_env(role: Option) -> Self { - let env = std::env::var("NETWORK") - .map(|s| StorageEnv::from_str(&s).unwrap_or_default()) - .unwrap_or_default(); - - Self { - env, - role: role.unwrap_or_default(), - namespace: None, - } - } - - fn endpoint_url(&self) -> String { - match self.role { - StorageRole::Admin => dotenvy::var("AWS_ENDPOINT_URL") - .expect("AWS_ENDPOINT_URL must be set for admin role"), - StorageRole::Public => { - match self.env { - StorageEnv::Local => "http://localhost:4566".to_string(), - StorageEnv::Testnet | StorageEnv::Mainnet => { - let bucket = self.bucket(); - let region = self.region(); - format!("https://{bucket}.s3-website-{region}.amazonaws.com") - } - } - } - } - } - - fn environment(&self) -> &StorageEnv { - &self.env - } - - fn role(&self) -> &StorageRole { - &self.role - } -} - -impl S3StorageOpts { - pub fn with_namespace(mut self, namespace: impl Into) -> Self { - self.namespace = Some(namespace.into()); - self - } - - pub fn region(&self) -> Region { - let region = match &self.role { - StorageRole::Admin => dotenvy::var("AWS_REGION") - .expect("AWS_REGION must be set for admin role"), - StorageRole::Public => "us-east-1".to_string(), - }; - Region::new(region) - } - - pub fn bucket(&self) -> String { - if matches!(self.role, StorageRole::Admin) { - return dotenvy::var("AWS_S3_BUCKET_NAME") - .expect("AWS_S3_BUCKET_NAME must be set for admin role"); - } - - let base_bucket = match self.env { - StorageEnv::Local => "fuel-streams-local", - StorageEnv::Testnet => "fuel-streams-testnet", - StorageEnv::Mainnet => "fuel-streams", - }; - - match &self.namespace { - Some(ns) => format!("{base_bucket}-{ns}"), - None => base_bucket.to_string(), - } - } - - pub fn credentials(&self) -> Option { - match self.role { - StorageRole::Admin => Some(aws_sdk_s3::config::Credentials::new( - dotenvy::var("AWS_ACCESS_KEY_ID") - .expect("AWS_ACCESS_KEY_ID must be set for admin role"), - dotenvy::var("AWS_SECRET_ACCESS_KEY") - .expect("AWS_SECRET_ACCESS_KEY must be set for admin role"), - None, - None, - "static", - )), - StorageRole::Public => None, - } - } - - #[cfg(any(test, feature = "test-helpers"))] - pub fn with_random_namespace(mut self) -> Self { - let random_namespace = { - use rand::Rng; - let random_int: u32 = rand::thread_rng().gen(); - format!("namespace-{}", random_int) - }; - self.namespace = Some(random_namespace); - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_bucket_names() { - let opts = S3StorageOpts::new(StorageEnv::Local, StorageRole::Public); - assert_eq!(opts.bucket(), "fuel-streams-local"); - - let opts = opts.with_namespace("test"); - assert_eq!(opts.bucket(), "fuel-streams-local-test"); - - let opts = S3StorageOpts::new(StorageEnv::Testnet, StorageRole::Public); - assert_eq!(opts.bucket(), "fuel-streams-testnet"); - - let opts = S3StorageOpts::new(StorageEnv::Mainnet, StorageRole::Public); - assert_eq!(opts.bucket(), "fuel-streams"); - } - - #[test] - fn test_public_endpoint_urls() { - let opts = S3StorageOpts::new(StorageEnv::Local, StorageRole::Public); - assert_eq!(opts.endpoint_url(), "http://localhost:4566"); - - let opts = S3StorageOpts::new(StorageEnv::Testnet, StorageRole::Public); - assert_eq!( - opts.endpoint_url(), - "https://fuel-streams-testnet.s3-website-us-east-1.amazonaws.com" - ); - - let opts = S3StorageOpts::new(StorageEnv::Mainnet, StorageRole::Public); - assert_eq!( - opts.endpoint_url(), - "https://fuel-streams.s3-website-us-east-1.amazonaws.com" - ); - } -} diff --git a/crates/fuel-streams-storage/src/storage.rs b/crates/fuel-streams-storage/src/storage.rs deleted file mode 100644 index b1e1afcd..00000000 --- a/crates/fuel-streams-storage/src/storage.rs +++ /dev/null @@ -1,47 +0,0 @@ -use async_trait::async_trait; -use displaydoc::Display as DisplayDoc; -use thiserror::Error; - -use crate::StorageConfig; - -#[derive(Error, Debug, DisplayDoc)] -pub enum StorageError { - /// Failed to store object: {0} - StoreError(String), - /// Failed to retrieve object: {0} - RetrieveError(String), - /// Failed to delete object: {0} - DeleteError(String), - /// Failed to initialize storage: {0} - InitError(String), -} - -#[async_trait] -pub trait Storage: std::fmt::Debug + Send + Sync { - type Config: StorageConfig; - - async fn new(config: Self::Config) -> Result - where - Self: Sized; - - async fn new_admin() -> Result - where - Self: Sized, - { - Self::new(Self::Config::admin_opts()).await - } - - async fn new_public() -> Result - where - Self: Sized, - { - Self::new(Self::Config::public_opts()).await - } - - async fn store(&self, key: &str, data: Vec) - -> Result<(), StorageError>; - - async fn retrieve(&self, key: &str) -> Result, StorageError>; - - async fn delete(&self, key: &str) -> Result<(), StorageError>; -} diff --git a/crates/fuel-streams-storage/src/storage_config.rs b/crates/fuel-streams-storage/src/storage_config.rs deleted file mode 100644 index 3ed09426..00000000 --- a/crates/fuel-streams-storage/src/storage_config.rs +++ /dev/null @@ -1,46 +0,0 @@ -use std::str::FromStr; - -#[derive(Debug, Clone, Default)] -pub enum StorageRole { - Admin, - #[default] - Public, -} - -#[derive(Debug, Clone, Default)] -pub enum StorageEnv { - #[default] - Local, - Testnet, - Mainnet, -} - -impl FromStr for StorageEnv { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "local" => Ok(StorageEnv::Local), - "testnet" => Ok(StorageEnv::Testnet), - "mainnet" => Ok(StorageEnv::Mainnet), - _ => Err(format!("unknown environment type: {}", s)), - } - } -} - -pub trait StorageConfig: Send + Sync + std::fmt::Debug + Sized { - fn new(env: StorageEnv, role: StorageRole) -> Self; - fn from_env(role: Option) -> Self; - - fn admin_opts() -> Self { - Self::from_env(Some(StorageRole::Admin)) - } - - fn public_opts() -> Self { - Self::from_env(Some(StorageRole::Public)) - } - - fn endpoint_url(&self) -> String; - fn environment(&self) -> &StorageEnv; - fn role(&self) -> &StorageRole; -} diff --git a/crates/fuel-streams-store/Cargo.toml b/crates/fuel-streams-store/Cargo.toml new file mode 100644 index 00000000..c93a3141 --- /dev/null +++ b/crates/fuel-streams-store/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "fuel-streams-store" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +async-stream = { workspace = true } +async-trait = { workspace = true } +dotenvy = { workspace = true } +fuel-data-parser = { workspace = true } +fuel-streams-macros = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +sqlx = { workspace = true, default-features = false, features = [ + "runtime-tokio", + "postgres", + "sqlite", + "any", + "tls-native-tls", + "macros", +] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] } + +[dev-dependencies] +test-case = { workspace = true } + +[features] +default = [] +test-helpers = [] # Feature for test utilities diff --git a/crates/fuel-streams-store/migrations/20250105033516_create_blocks_table.sql b/crates/fuel-streams-store/migrations/20250105033516_create_blocks_table.sql new file mode 100644 index 00000000..6143c998 --- /dev/null +++ b/crates/fuel-streams-store/migrations/20250105033516_create_blocks_table.sql @@ -0,0 +1,12 @@ +-- Create records table +CREATE TABLE IF NOT EXISTS blocks ( + _id SERIAL PRIMARY KEY, + subject TEXT NOT NULL, + block_height BIGINT NOT NULL, + producer_address TEXT NOT NULL, + value BYTEA NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_blocks_subject ON blocks (subject); +CREATE INDEX IF NOT EXISTS idx_blocks_producer_address ON blocks (producer_address); +CREATE INDEX IF NOT EXISTS idx_blocks_block_height ON blocks (block_height); diff --git a/crates/fuel-streams-store/migrations/20250108203444_create_transactions_table.sql b/crates/fuel-streams-store/migrations/20250108203444_create_transactions_table.sql new file mode 100644 index 00000000..7d577fb2 --- /dev/null +++ b/crates/fuel-streams-store/migrations/20250108203444_create_transactions_table.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS transactions ( + _id SERIAL PRIMARY KEY, + subject TEXT NOT NULL, + block_height BIGINT NOT NULL, + tx_id TEXT NOT NULL, + tx_index INTEGER NOT NULL, + tx_status TEXT NOT NULL, + kind TEXT NOT NULL, + value BYTEA NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_transactions_subject ON transactions (subject); +CREATE INDEX IF NOT EXISTS idx_transactions_block_height ON transactions (block_height); +CREATE INDEX IF NOT EXISTS idx_transactions_tx_id ON transactions (tx_id); +CREATE INDEX IF NOT EXISTS idx_transactions_tx_index ON transactions (tx_index); +CREATE INDEX IF NOT EXISTS idx_transactions_tx_status ON transactions (tx_status); +CREATE INDEX IF NOT EXISTS idx_transactions_kind ON transactions (kind); diff --git a/crates/fuel-streams-store/migrations/20250108203637_create_inputs_table.sql b/crates/fuel-streams-store/migrations/20250108203637_create_inputs_table.sql new file mode 100644 index 00000000..6ee0f21e --- /dev/null +++ b/crates/fuel-streams-store/migrations/20250108203637_create_inputs_table.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS inputs ( + _id SERIAL PRIMARY KEY, + subject TEXT NOT NULL, + block_height BIGINT NOT NULL, + tx_id TEXT NOT NULL, + tx_index INTEGER NOT NULL, + input_index INTEGER NOT NULL, + input_type TEXT NOT NULL, -- 'coin', 'contract', or 'message' + owner_id TEXT, -- for coin + asset_id TEXT, -- for coin + contract_id TEXT, -- for contract + sender TEXT, -- for message + recipient TEXT, -- for message + value BYTEA NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_inputs_subject ON inputs (subject); +CREATE INDEX IF NOT EXISTS idx_inputs_block_height ON inputs (block_height); +CREATE INDEX IF NOT EXISTS idx_inputs_tx_id ON inputs (tx_id); +CREATE INDEX IF NOT EXISTS idx_inputs_tx_index ON inputs (tx_index); +CREATE INDEX IF NOT EXISTS idx_inputs_input_index ON inputs (input_index); +CREATE INDEX IF NOT EXISTS idx_inputs_input_type ON inputs (input_type); +CREATE INDEX IF NOT EXISTS idx_inputs_owner_id ON inputs (owner_id); +CREATE INDEX IF NOT EXISTS idx_inputs_asset_id ON inputs (asset_id); +CREATE INDEX IF NOT EXISTS idx_inputs_contract_id ON inputs (contract_id); +CREATE INDEX IF NOT EXISTS idx_inputs_sender ON inputs (sender); +CREATE INDEX IF NOT EXISTS idx_inputs_recipient ON inputs (recipient); diff --git a/crates/fuel-streams-store/migrations/20250108204045_create_outputs_table.sql b/crates/fuel-streams-store/migrations/20250108204045_create_outputs_table.sql new file mode 100644 index 00000000..976de112 --- /dev/null +++ b/crates/fuel-streams-store/migrations/20250108204045_create_outputs_table.sql @@ -0,0 +1,23 @@ +CREATE TABLE IF NOT EXISTS outputs ( + _id SERIAL PRIMARY KEY, + subject TEXT NOT NULL, + block_height BIGINT NOT NULL, + tx_id TEXT NOT NULL, + tx_index INTEGER NOT NULL, + output_index INTEGER NOT NULL, + output_type TEXT NOT NULL, -- 'coin', 'contract', 'change', 'variable', or 'contract_created' + to_address TEXT, -- for coin, change, and variable + asset_id TEXT, -- for coin, change, and variable + contract_id TEXT, -- for contract and contract_created + value BYTEA NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_outputs_subject ON outputs (subject); +CREATE INDEX IF NOT EXISTS idx_outputs_block_height ON outputs (block_height); +CREATE INDEX IF NOT EXISTS idx_outputs_tx_id ON outputs (tx_id); +CREATE INDEX IF NOT EXISTS idx_outputs_tx_index ON outputs (tx_index); +CREATE INDEX IF NOT EXISTS idx_outputs_output_index ON outputs (output_index); +CREATE INDEX IF NOT EXISTS idx_outputs_output_type ON outputs (output_type); +CREATE INDEX IF NOT EXISTS idx_outputs_to_address ON outputs (to_address); +CREATE INDEX IF NOT EXISTS idx_outputs_asset_id ON outputs (asset_id); +CREATE INDEX IF NOT EXISTS idx_outputs_contract_id ON outputs (contract_id); diff --git a/crates/fuel-streams-store/migrations/20250108204242_create_utxos_table.sql b/crates/fuel-streams-store/migrations/20250108204242_create_utxos_table.sql new file mode 100644 index 00000000..bf49e4b6 --- /dev/null +++ b/crates/fuel-streams-store/migrations/20250108204242_create_utxos_table.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS utxos ( + _id SERIAL PRIMARY KEY, + subject TEXT NOT NULL, + block_height BIGINT NOT NULL, + tx_id TEXT NOT NULL, + tx_index INTEGER NOT NULL, + input_index INTEGER NOT NULL, + utxo_type TEXT NOT NULL, -- 'message', 'coin', or 'contract' + utxo_id TEXT NOT NULL, -- hex string of the UTXO identifier + value BYTEA NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_utxos_subject ON utxos (subject); +CREATE INDEX IF NOT EXISTS idx_utxos_block_height ON utxos (block_height); +CREATE INDEX IF NOT EXISTS idx_utxos_tx_id ON utxos (tx_id); +CREATE INDEX IF NOT EXISTS idx_utxos_tx_index ON utxos (tx_index); +CREATE INDEX IF NOT EXISTS idx_utxos_input_index ON utxos (input_index); +CREATE INDEX IF NOT EXISTS idx_utxos_utxo_type ON utxos (utxo_type); +CREATE INDEX IF NOT EXISTS idx_utxos_utxo_id ON utxos (utxo_id); diff --git a/crates/fuel-streams-store/migrations/20250108204327_create_receipts_table.sql b/crates/fuel-streams-store/migrations/20250108204327_create_receipts_table.sql new file mode 100644 index 00000000..adf5f6e4 --- /dev/null +++ b/crates/fuel-streams-store/migrations/20250108204327_create_receipts_table.sql @@ -0,0 +1,34 @@ +CREATE TABLE IF NOT EXISTS receipts ( + _id SERIAL PRIMARY KEY, + subject TEXT NOT NULL, + block_height BIGINT NOT NULL, + tx_id TEXT NOT NULL, + tx_index INTEGER NOT NULL, + receipt_index INTEGER NOT NULL, + receipt_type TEXT NOT NULL, -- 'call', 'return', 'return_data', 'panic', 'revert', 'log', 'log_data', + -- 'transfer', 'transfer_out', 'script_result', 'message_out', 'mint', 'burn' + from_contract_id TEXT, -- ContractId for call/transfer/transfer_out + to_contract_id TEXT, -- ContractId for call/transfer + to_address TEXT, -- Address for transfer_out + asset_id TEXT, -- for call/transfer/transfer_out + contract_id TEXT, -- ContractId for return/return_data/panic/revert/log/log_data/mint/burn + sub_id TEXT, -- for mint/burn + sender_address TEXT, -- Address for message_out + recipient_address TEXT, -- Address for message_out + value BYTEA NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_receipts_subject ON receipts (subject); +CREATE INDEX IF NOT EXISTS idx_receipts_block_height ON receipts (block_height); +CREATE INDEX IF NOT EXISTS idx_receipts_tx_id ON receipts (tx_id); +CREATE INDEX IF NOT EXISTS idx_receipts_tx_index ON receipts (tx_index); +CREATE INDEX IF NOT EXISTS idx_receipts_receipt_index ON receipts (receipt_index); +CREATE INDEX IF NOT EXISTS idx_receipts_receipt_type ON receipts (receipt_type); +CREATE INDEX IF NOT EXISTS idx_receipts_from_contract_id ON receipts (from_contract_id); +CREATE INDEX IF NOT EXISTS idx_receipts_to_contract_id ON receipts (to_contract_id); +CREATE INDEX IF NOT EXISTS idx_receipts_to_address ON receipts (to_address); +CREATE INDEX IF NOT EXISTS idx_receipts_asset_id ON receipts (asset_id); +CREATE INDEX IF NOT EXISTS idx_receipts_contract_id ON receipts (contract_id); +CREATE INDEX IF NOT EXISTS idx_receipts_sub_id ON receipts (sub_id); +CREATE INDEX IF NOT EXISTS idx_receipts_sender_address ON receipts (sender_address); +CREATE INDEX IF NOT EXISTS idx_receipts_recipient_address ON receipts (recipient_address); diff --git a/crates/fuel-streams-store/src/db/db_impl.rs b/crates/fuel-streams-store/src/db/db_impl.rs new file mode 100644 index 00000000..a94f3569 --- /dev/null +++ b/crates/fuel-streams-store/src/db/db_impl.rs @@ -0,0 +1,86 @@ +use std::sync::{Arc, LazyLock}; + +use sqlx::{Pool, Postgres}; + +use crate::record::{EncoderError, RecordPacketError}; + +#[derive(thiserror::Error, Debug)] +pub enum DbError { + #[error("Failed to open database")] + Open(#[source] sqlx::Error), + #[error("Failed to insert data")] + Insert(#[source] sqlx::Error), + #[error("Record not found: {0}")] + NotFound(String), + #[error("Failed to find many records by pattern")] + FindManyByPattern(#[source] sqlx::Error), + #[error("Failed to encode/decode data")] + EncodeDecode(#[from] EncoderError), + #[error("Other error: {0}")] + Other(String), + #[error(transparent)] + DbItemFromPacket(#[from] RecordPacketError), + #[error("Failed to truncate table")] + TruncateTable(#[source] sqlx::Error), + #[error("Failed to execute query")] + Query(#[source] sqlx::Error), +} + +pub type DbResult = Result; + +pub static DB_POOL_SIZE: LazyLock = LazyLock::new(|| { + dotenvy::var("DB_POOL_SIZE") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(5) +}); + +#[derive(Debug, Clone)] +pub struct DbConnectionOpts { + pub connection_str: String, + pub pool_size: Option, +} + +impl Default for DbConnectionOpts { + fn default() -> Self { + Self { + pool_size: Some(*DB_POOL_SIZE as u32), + connection_str: dotenvy::var("DATABASE_URL") + .expect("DATABASE_URL not set"), + } + } +} + +#[derive(Debug, Clone)] +pub struct Db { + pub pool: Pool, +} + +impl Db { + pub async fn new(opts: DbConnectionOpts) -> DbResult { + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(opts.pool_size.unwrap_or_default()) + .connect(&opts.connection_str) + .await + .map_err(DbError::Open)?; + + Ok(Self { pool }) + } + + pub fn arc(self) -> Arc { + Arc::new(self) + } + + pub async fn truncate_table(&self, table_name: &str) -> DbResult<()> { + let query = format!("TRUNCATE TABLE {}", table_name); + sqlx::query(&query) + .execute(&self.pool) + .await + .map_err(DbError::TruncateTable)?; + Ok(()) + } + + pub fn pool_ref(&self) -> &Pool { + &self.pool + } +} diff --git a/crates/fuel-streams-store/src/db/db_item.rs b/crates/fuel-streams-store/src/db/db_item.rs new file mode 100644 index 00000000..ac580006 --- /dev/null +++ b/crates/fuel-streams-store/src/db/db_item.rs @@ -0,0 +1,28 @@ +use async_trait::async_trait; +use fuel_data_parser::DataEncoder; +use sqlx::postgres::PgRow; + +use super::DbError; +use crate::record::RecordEntity; + +#[async_trait] +pub trait DbItem: + DataEncoder + + Unpin + + std::fmt::Debug + + PartialEq + + Eq + + Ord + + PartialOrd + + Send + + Sync + + Sized + + serde::Serialize + + serde::de::DeserializeOwned + + for<'r> sqlx::FromRow<'r, PgRow> + + 'static +{ + fn entity(&self) -> &RecordEntity; + fn encoded_value(&self) -> &[u8]; + fn subject_str(&self) -> String; +} diff --git a/crates/fuel-streams-store/src/db/mod.rs b/crates/fuel-streams-store/src/db/mod.rs new file mode 100644 index 00000000..a409509e --- /dev/null +++ b/crates/fuel-streams-store/src/db/mod.rs @@ -0,0 +1,5 @@ +mod db_impl; +mod db_item; + +pub use db_impl::*; +pub use db_item::*; diff --git a/crates/fuel-streams-store/src/lib.rs b/crates/fuel-streams-store/src/lib.rs new file mode 100644 index 00000000..0c76c683 --- /dev/null +++ b/crates/fuel-streams-store/src/lib.rs @@ -0,0 +1,3 @@ +pub mod db; +pub mod record; +pub mod store; diff --git a/crates/fuel-streams-store/src/record/mod.rs b/crates/fuel-streams-store/src/record/mod.rs new file mode 100644 index 00000000..336bad76 --- /dev/null +++ b/crates/fuel-streams-store/src/record/mod.rs @@ -0,0 +1,9 @@ +mod query_options; +mod record_entity; +mod record_impl; +mod record_packet; + +pub use query_options::*; +pub use record_entity::*; +pub use record_impl::*; +pub use record_packet::*; diff --git a/crates/fuel-streams-store/src/record/query_options.rs b/crates/fuel-streams-store/src/record/query_options.rs new file mode 100644 index 00000000..89182727 --- /dev/null +++ b/crates/fuel-streams-store/src/record/query_options.rs @@ -0,0 +1,39 @@ +#[derive(Debug, Clone)] +pub struct QueryOptions { + pub offset: i64, + pub limit: i64, + pub from_block: Option, + pub namespace: Option, +} +impl Default for QueryOptions { + fn default() -> Self { + Self { + offset: 0, + limit: 100, + from_block: None, + namespace: None, + } + } +} + +impl QueryOptions { + pub fn with_offset(mut self, offset: i64) -> Self { + self.offset = offset.max(0); + self + } + pub fn with_limit(mut self, limit: i64) -> Self { + self.limit = limit.max(1); + self + } + pub fn with_from_block(mut self, from_block: Option) -> Self { + self.from_block = from_block; + self + } + pub fn with_namespace(mut self, namespace: Option) -> Self { + self.namespace = namespace; + self + } + pub fn increment_offset(&mut self) { + self.offset += self.limit; + } +} diff --git a/crates/fuel-streams-store/src/record/record_entity.rs b/crates/fuel-streams-store/src/record/record_entity.rs new file mode 100644 index 00000000..03c0a063 --- /dev/null +++ b/crates/fuel-streams-store/src/record/record_entity.rs @@ -0,0 +1,52 @@ +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, sqlx::Type)] +#[sqlx(type_name = "record_entity", rename_all = "lowercase")] +pub enum RecordEntity { + Block, + Transaction, + Input, + Output, + Receipt, + Utxo, +} + +impl std::fmt::Display for RecordEntity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl RecordEntity { + pub fn as_str(&self) -> &'static str { + match self { + Self::Block => "block", + Self::Transaction => "transaction", + Self::Input => "input", + Self::Output => "output", + Self::Receipt => "receipt", + Self::Utxo => "utxo", + } + } + + pub fn table_name(&self) -> String { + format!("{}{}", self.as_str(), "s") + } +} + +impl FromStr for RecordEntity { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + "block" | "blocks" => Ok(Self::Block), + "transaction" | "transactions" => Ok(Self::Transaction), + "input" | "inputs" => Ok(Self::Input), + "output" | "outputs" => Ok(Self::Output), + "receipt" | "receipts" => Ok(Self::Receipt), + "utxo" | "utxos" => Ok(Self::Utxo), + _ => Err(format!("Invalid record entity: {}", s)), + } + } +} diff --git a/crates/fuel-streams-store/src/record/record_impl.rs b/crates/fuel-streams-store/src/record/record_impl.rs new file mode 100644 index 00000000..15989865 --- /dev/null +++ b/crates/fuel-streams-store/src/record/record_impl.rs @@ -0,0 +1,94 @@ +use std::sync::Arc; + +use async_trait::async_trait; +pub use fuel_data_parser::{DataEncoder, DataParserError as EncoderError}; +use fuel_streams_macros::subject::IntoSubject; +use sqlx::{Postgres, QueryBuilder}; + +use super::{QueryOptions, RecordEntity, RecordPacket}; +use crate::db::{Db, DbError, DbItem, DbResult}; + +pub trait RecordEncoder: DataEncoder {} +impl> RecordEncoder for T {} + +#[async_trait] +pub trait Record: RecordEncoder + 'static { + type DbItem: DbItem; + + const ENTITY: RecordEntity; + const ORDER_PROPS: &'static [&'static str]; + + fn to_packet(&self, subject: Arc) -> RecordPacket { + RecordPacket::new(subject, self) + } + + async fn from_db_item(record: &Self::DbItem) -> DbResult { + Self::decode(record.encoded_value()).await + } + + async fn insert( + &self, + db: &Db, + packet: &RecordPacket, + ) -> DbResult; + + fn build_find_many_query( + subject: Arc, + options: QueryOptions, + ) -> QueryBuilder<'static, Postgres> { + let mut query_builder: QueryBuilder = QueryBuilder::default(); + let select = format!("SELECT * FROM {}", Self::ENTITY.table_name()); + query_builder.push(select); + query_builder.push(" WHERE "); + query_builder.push(subject.to_sql_where()); + + if let Some(block) = options.from_block { + query_builder.push(" AND block_height >= "); + query_builder.push_bind(block as i64); + } + + if cfg!(any(test, feature = "test-helpers")) { + if let Some(ns) = options.namespace { + query_builder.push(" AND subject LIKE "); + query_builder.push_bind(format!("{}%", ns)); + } + } + + query_builder.push(" ORDER BY "); + query_builder.push(Self::ORDER_PROPS.join(", ")); + query_builder.push(" ASC LIMIT "); + query_builder.push_bind(options.limit); + query_builder.push(" OFFSET "); + query_builder.push_bind(options.offset); + query_builder + } + + async fn find_last_record( + db: &Db, + options: QueryOptions, + ) -> DbResult> { + let mut query_builder = sqlx::QueryBuilder::new(format!( + "SELECT * FROM {}", + Self::ENTITY.table_name() + )); + + if let Some(ns) = options.namespace { + query_builder + .push(" WHERE subject LIKE ") + .push_bind(format!("{}%", ns)); + } + + query_builder + .push(" ORDER BY ") + .push(Self::ORDER_PROPS.join(", ")) + .push(" DESC LIMIT 1"); + + let query = query_builder.build_query_as::(); + let record = query + .fetch_optional(&db.pool) + .await + .map_err(DbError::FindManyByPattern)?; + + Ok(record) + } +} diff --git a/crates/fuel-streams-store/src/record/record_packet.rs b/crates/fuel-streams-store/src/record/record_packet.rs new file mode 100644 index 00000000..f434b31d --- /dev/null +++ b/crates/fuel-streams-store/src/record/record_packet.rs @@ -0,0 +1,65 @@ +use std::{fmt::Debug, sync::Arc}; + +use fuel_streams_macros::subject::IntoSubject; + +use crate::record::Record; + +#[derive(Debug, thiserror::Error)] +pub enum RecordPacketError { + #[error("Failed to downcast subject")] + DowncastError, + #[error("Subject mismatch")] + SubjectMismatch, +} + +#[derive(Debug, Clone)] +pub struct RecordPacket { + pub record: Arc, + pub subject: Arc, + namespace: Option, +} + +impl RecordPacket { + pub fn new(subject: Arc, record: &R) -> Self { + Self { + subject: Arc::clone(&subject), + record: Arc::new(record.clone()), + namespace: None, + } + } + + pub fn with_namespace(mut self, namespace: &str) -> Self { + self.namespace = Some(namespace.to_string()); + self + } + + pub fn subject_matches( + &self, + ) -> Result { + if let Some(subject) = self.subject.downcast_ref::() { + Ok(subject.clone()) + } else { + Err(RecordPacketError::DowncastError) + } + } + + pub fn subject_str(&self) -> String { + if cfg!(any(test, feature = "test-helpers")) { + let mut subject = self.subject.parse(); + if let Some(namespace) = &self.namespace { + subject = format!("{}.{}", namespace, subject); + } + subject + } else { + self.subject.parse() + } + } + + pub fn namespace(&self) -> Option<&str> { + self.namespace.as_deref() + } + + pub fn arc(&self) -> Arc { + Arc::new(self.clone()) + } +} diff --git a/crates/fuel-streams-store/src/store/config.rs b/crates/fuel-streams-store/src/store/config.rs new file mode 100644 index 00000000..b8b7e8fe --- /dev/null +++ b/crates/fuel-streams-store/src/store/config.rs @@ -0,0 +1,22 @@ +use std::sync::LazyLock; + +pub static STORE_PAGINATION_LIMIT: LazyLock = LazyLock::new(|| { + dotenvy::var("STORE_PAGINATION_LIMIT") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(100) +}); + +pub static STORE_MAX_RETRIES: LazyLock = LazyLock::new(|| { + dotenvy::var("STORE_MAX_RETRIES") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(3) +}); + +pub static STORE_INITIAL_BACKOFF_MS: LazyLock = LazyLock::new(|| { + dotenvy::var("STORE_INITIAL_BACKOFF_MS") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(100) +}); diff --git a/crates/fuel-streams-store/src/store/errors.rs b/crates/fuel-streams-store/src/store/errors.rs new file mode 100644 index 00000000..9d2e5b93 --- /dev/null +++ b/crates/fuel-streams-store/src/store/errors.rs @@ -0,0 +1,9 @@ +use crate::db::DbError; + +#[derive(thiserror::Error, Debug)] +pub enum StoreError { + #[error(transparent)] + Db(#[from] DbError), + #[error(transparent)] + Stream(#[from] sqlx::Error), +} diff --git a/crates/fuel-streams-store/src/store/mod.rs b/crates/fuel-streams-store/src/store/mod.rs new file mode 100644 index 00000000..581ab5af --- /dev/null +++ b/crates/fuel-streams-store/src/store/mod.rs @@ -0,0 +1,6 @@ +pub(super) mod config; +mod errors; +mod store_impl; + +pub use errors::*; +pub use store_impl::*; diff --git a/crates/fuel-streams-store/src/store/store_impl.rs b/crates/fuel-streams-store/src/store/store_impl.rs new file mode 100644 index 00000000..d4fe3d19 --- /dev/null +++ b/crates/fuel-streams-store/src/store/store_impl.rs @@ -0,0 +1,113 @@ +use std::sync::Arc; + +use fuel_streams_macros::subject::IntoSubject; +use futures::{stream::BoxStream, StreamExt, TryStreamExt}; + +use super::{config, StoreError}; +use crate::{ + db::Db, + record::{QueryOptions, Record, RecordPacket}, +}; + +pub type StoreResult = Result; + +#[derive(Debug, Clone)] +pub struct Store { + pub db: Arc, + pub namespace: Option, + _marker: std::marker::PhantomData, +} + +impl Store { + pub fn new(db: &Arc) -> Self { + Self { + db: Arc::clone(db), + namespace: None, + _marker: std::marker::PhantomData, + } + } + + #[cfg(any(test, feature = "test-helpers"))] + pub fn with_namespace(&mut self, namespace: &str) -> &mut Self { + self.namespace = Some(namespace.to_string()); + self + } + + pub async fn insert_record( + &self, + packet: &RecordPacket, + ) -> StoreResult { + with_retry(|| packet.record.insert(&self.db, packet)).await + } + + #[cfg(any(test, feature = "test-helpers"))] + pub async fn find_many_by_subject( + &self, + subject: &Arc, + mut options: QueryOptions, + ) -> StoreResult> { + options = options.with_namespace(self.namespace.clone()); + R::build_find_many_query(subject.clone(), options.clone()) + .build_query_as::() + .fetch_all(&self.db.pool) + .await + .map_err(StoreError::from) + } + + pub fn stream_by_subject( + &self, + subject: Arc, + from_block: Option, + ) -> BoxStream<'static, Result> { + let db = Arc::clone(&self.db); + let namespace = self.namespace.clone(); + async_stream::stream! { + let options = QueryOptions::default() + .with_namespace(namespace) + .with_from_block(from_block) + .with_limit(*config::STORE_PAGINATION_LIMIT); + let mut query = R::build_find_many_query(subject, options.clone()); + let mut stream = query + .build_query_as::() + .fetch(&db.pool); + while let Some(result) = stream.try_next().await? { + yield Ok(result); + } + } + .boxed() + } + + pub async fn find_last_record(&self) -> StoreResult> { + let options = + QueryOptions::default().with_namespace(self.namespace.clone()); + R::find_last_record(&self.db, options) + .await + .map_err(StoreError::from) + } +} + +async fn with_retry(f: F) -> StoreResult +where + F: Fn() -> Fut, + Fut: std::future::Future>, + StoreError: From, +{ + let mut attempt = 0; + loop { + match f().await { + Ok(result) => return Ok(result), + Err(err) => { + attempt += 1; + if attempt >= *config::STORE_MAX_RETRIES { + return Err(StoreError::from(err)); + } + + // Exponential backoff: 100ms, 200ms, 400ms + let initial_backoff_ms = *config::STORE_INITIAL_BACKOFF_MS; + let delay = initial_backoff_ms * (1 << (attempt - 1)); + tokio::time::sleep(std::time::Duration::from_millis(delay)) + .await; + } + } + } +} diff --git a/crates/fuel-streams-types/Cargo.toml b/crates/fuel-streams-types/Cargo.toml new file mode 100644 index 00000000..b3c3fc82 --- /dev/null +++ b/crates/fuel-streams-types/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "fuel-streams-types" +description = "Types for fuel streams" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +fuel-core = { workspace = true, default-features = false, features = [ + "p2p", + "relayer", + "rocksdb", + "test-helpers", +] } +fuel-core-client = { workspace = true, default-features = false, features = ["std"] } +fuel-core-importer = { workspace = true } +fuel-core-types = { workspace = true, default-features = false, features = ["std", "serde"] } +hex = { workspace = true } +serde = { workspace = true } + +[features] +default = [] +test-helpers = [] diff --git a/crates/fuel-streams-core/src/fuel_core_types.rs b/crates/fuel-streams-types/src/fuel_core.rs similarity index 89% rename from crates/fuel-streams-core/src/fuel_core_types.rs rename to crates/fuel-streams-types/src/fuel_core.rs index b5461427..51c780e5 100644 --- a/crates/fuel-streams-core/src/fuel_core_types.rs +++ b/crates/fuel-streams-types/src/fuel_core.rs @@ -2,7 +2,10 @@ use fuel_core::state::{ generic_database::GenericDatabase, iterable_key_value_view::IterableKeyValueViewWrapper, }; -pub use fuel_core_client::client::schema::Tai64Timestamp as FuelCoreTai64Timestamp; +pub use fuel_core_client::client::{ + schema::Tai64Timestamp as FuelCoreTai64Timestamp, + types::TransactionStatus as FuelCoreClientTransactionStatus, +}; pub use fuel_core_importer::ImporterResult as FuelCoreImporterResult; pub use fuel_core_types::{ blockchain::{ @@ -16,6 +19,7 @@ pub use fuel_core_types::{ primitives::BlockId as FuelCoreBlockId, SealedBlock as FuelCoreSealedBlock, }, + fuel_asm::Word as FuelCoreWord, fuel_crypto::Signature as FuelCoreSignature, fuel_tx::{ field::{Inputs as FuelCoreInputs, Outputs as FuelCoreOutputs}, @@ -41,7 +45,6 @@ pub use fuel_core_types::{ UniqueIdentifier as FuelCoreUniqueIdentifier, UpgradePurpose as FuelCoreUpgradePurpose, UtxoId as FuelCoreUtxoId, - Word as FuelCoreWord, }, fuel_types::{ BlockHeight as FuelCoreBlockHeight, @@ -57,7 +60,7 @@ pub use fuel_core_types::{ tai64::Tai64 as FuelCoreTai64, }; -pub type OffchainDatabase = GenericDatabase< +pub type FuelCoreOffchainDatabase = GenericDatabase< IterableKeyValueViewWrapper< fuel_core::fuel_core_graphql_api::storage::Column, >, diff --git a/crates/fuel-streams-types/src/lib.rs b/crates/fuel-streams-types/src/lib.rs new file mode 100644 index 00000000..a9d956eb --- /dev/null +++ b/crates/fuel-streams-types/src/lib.rs @@ -0,0 +1,5 @@ +pub mod fuel_core; +pub mod primitives; + +pub use fuel_core::*; +pub use primitives::*; diff --git a/crates/fuel-streams-types/src/primitives/bytes.rs b/crates/fuel-streams-types/src/primitives/bytes.rs new file mode 100644 index 00000000..ed95b0ca --- /dev/null +++ b/crates/fuel-streams-types/src/primitives/bytes.rs @@ -0,0 +1,57 @@ +use fuel_core_types::fuel_types; + +use super::{LongBytes, UtxoId}; +use crate::{ + fuel_core::*, + generate_byte_type_wrapper, + impl_bytes32_to_type, + impl_from_type_to_bytes32, +}; + +generate_byte_type_wrapper!(Address, fuel_types::Address, 32); +generate_byte_type_wrapper!(Bytes32, fuel_types::Bytes32, 32); +generate_byte_type_wrapper!(ContractId, fuel_types::ContractId, 32); +generate_byte_type_wrapper!(AssetId, fuel_types::AssetId, 32); +generate_byte_type_wrapper!(BlobId, fuel_types::BlobId, 32); +generate_byte_type_wrapper!(Nonce, fuel_types::Nonce, 32); +generate_byte_type_wrapper!(Salt, fuel_types::Salt, 32); +generate_byte_type_wrapper!(MessageId, fuel_types::MessageId, 32); +generate_byte_type_wrapper!(BlockId, fuel_types::Bytes32, 32); +generate_byte_type_wrapper!(Signature, fuel_types::Bytes64, 64); +generate_byte_type_wrapper!(TxId, fuel_types::TxId, 32); +generate_byte_type_wrapper!(HexData, LongBytes); + +impl From<&UtxoId> for HexData { + fn from(value: &UtxoId) -> Self { + value.to_owned().into() + } +} + +impl From for HexData { + fn from(value: UtxoId) -> Self { + let mut bytes = Vec::with_capacity(34); + bytes.extend_from_slice(value.tx_id.0.as_ref()); + bytes.extend_from_slice(&value.output_index.to_be_bytes()); + HexData(bytes.into()) + } +} + +impl_bytes32_to_type!(MessageId); +impl_bytes32_to_type!(ContractId); +impl_bytes32_to_type!(AssetId); +impl_bytes32_to_type!(Address); +impl_bytes32_to_type!(BlobId); +impl_bytes32_to_type!(Nonce); +impl_bytes32_to_type!(Salt); +impl_bytes32_to_type!(BlockId); +impl_bytes32_to_type!(TxId); + +impl_from_type_to_bytes32!(fuel_types::ContractId); +impl_from_type_to_bytes32!(fuel_types::AssetId); +impl_from_type_to_bytes32!(fuel_types::Address); + +impl From for BlockId { + fn from(value: FuelCoreBlockId) -> Self { + Self(FuelCoreBytes32::from(value)) + } +} diff --git a/crates/fuel-streams-types/src/primitives/bytes_long.rs b/crates/fuel-streams-types/src/primitives/bytes_long.rs new file mode 100644 index 00000000..958497ed --- /dev/null +++ b/crates/fuel-streams-types/src/primitives/bytes_long.rs @@ -0,0 +1,42 @@ +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, + Default, +)] +pub struct LongBytes(pub Vec); + +impl LongBytes { + pub fn zeroed() -> Self { + Self(vec![0; 32]) + } +} +impl AsRef<[u8]> for LongBytes { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} +impl AsMut<[u8]> for LongBytes { + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0 + } +} +impl From> for LongBytes { + fn from(value: Vec) -> Self { + Self(value) + } +} +impl std::fmt::Display for LongBytes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", hex::encode(&self.0)) + } +} +impl From<&[u8]> for LongBytes { + fn from(value: &[u8]) -> Self { + Self(value.to_vec()) + } +} diff --git a/crates/fuel-streams-types/src/primitives/common.rs b/crates/fuel-streams-types/src/primitives/common.rs new file mode 100644 index 00000000..94ecf8be --- /dev/null +++ b/crates/fuel-streams-types/src/primitives/common.rs @@ -0,0 +1,206 @@ +/// Common wrapper type macro that implements basic traits and conversions +#[macro_export] +macro_rules! common_wrapper_type { + ($wrapper_type:ident, $inner_type:ty) => { + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct $wrapper_type(pub $inner_type); + + // Custom serialization + impl serde::Serialize for $wrapper_type { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.serialize_str(&format!("0x{}", self.0)) + } else { + self.0.serialize(serializer) + } + } + } + + // Custom deserialization using FromStr + impl<'de> serde::Deserialize<'de> for $wrapper_type { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + if deserializer.is_human_readable() { + let s = String::deserialize(deserializer)?; + s.parse().map_err(serde::de::Error::custom) + } else { + Ok($wrapper_type(<$inner_type>::deserialize(deserializer)?)) + } + } + } + + impl From<$inner_type> for $wrapper_type { + fn from(value: $inner_type) -> Self { + $wrapper_type(value) + } + } + + impl From<$wrapper_type> for $inner_type { + fn from(value: $wrapper_type) -> Self { + value.0 + } + } + + impl From<&$inner_type> for $wrapper_type { + fn from(value: &$inner_type) -> Self { + $wrapper_type(value.clone()) + } + } + + impl std::fmt::Display for $wrapper_type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "0x{}", self.0) + } + } + + impl From<&str> for $wrapper_type { + fn from(s: &str) -> Self { + s.parse().unwrap_or_else(|e| { + panic!( + "Failed to parse {}: {}", + stringify!($wrapper_type), + e + ) + }) + } + } + + impl $wrapper_type { + pub fn zeroed() -> Self { + $wrapper_type(<$inner_type>::zeroed()) + } + + pub fn new(inner: $inner_type) -> Self { + $wrapper_type(inner) + } + } + + impl AsRef<$inner_type> for $wrapper_type { + fn as_ref(&self) -> &$inner_type { + &self.0 + } + } + + impl $wrapper_type { + pub fn into_inner(self) -> $inner_type { + self.0 + } + } + + impl Default for $wrapper_type { + fn default() -> Self { + $wrapper_type(<$inner_type>::zeroed()) + } + } + }; +} + +/// Macro for generating byte type wrappers with optional byte size specification +#[macro_export] +macro_rules! generate_byte_type_wrapper { + // Pattern with byte_size specified + ($wrapper_type:ident, $inner_type:ty, $byte_size:expr) => { + $crate::common_wrapper_type!($wrapper_type, $inner_type); + + impl From<[u8; $byte_size]> for $wrapper_type { + fn from(value: [u8; $byte_size]) -> Self { + $wrapper_type(<$inner_type>::from(value)) + } + } + + impl std::str::FromStr for $wrapper_type { + type Err = String; + + fn from_str(s: &str) -> Result { + let s = s.strip_prefix("0x").unwrap_or(s); + if s.len() != std::mem::size_of::<$inner_type>() * 2 { + return Err(format!( + "Invalid length for {}, expected {} characters", + stringify!($wrapper_type), + std::mem::size_of::<$inner_type>() * 2 + )); + } + let bytes = hex::decode(s).map_err(|e| { + format!("Failed to decode hex string: {}", e) + })?; + let array: [u8; $byte_size] = bytes + .try_into() + .map_err(|_| "Invalid byte length".to_string())?; + Ok($wrapper_type(<$inner_type>::from(array))) + } + } + }; + + // Pattern without byte_size + ($wrapper_type:ident, $inner_type:ty) => { + $crate::common_wrapper_type!($wrapper_type, $inner_type); + + impl From> for $wrapper_type { + fn from(value: Vec) -> Self { + $wrapper_type(<$inner_type>::from(value)) + } + } + + impl std::str::FromStr for $wrapper_type { + type Err = String; + fn from_str(s: &str) -> Result { + let s = s.strip_prefix("0x").unwrap_or(s); + let bytes = hex::decode(s).map_err(|e| { + format!("Failed to decode hex string: {}", e) + })?; + Ok($wrapper_type(bytes.into())) + } + } + }; +} + +/// Macro for implementing Bytes32 conversions +#[macro_export] +macro_rules! impl_bytes32_to_type { + ($type:ty) => { + impl From for $type { + fn from(value: Bytes32) -> Self { + let bytes: [u8; 32] = value.0.into(); + <$type>::from(bytes) + } + } + impl From<&Bytes32> for $type { + fn from(value: &Bytes32) -> Self { + value.clone().into() + } + } + impl From<$type> for Bytes32 { + fn from(value: $type) -> Self { + let bytes: [u8; 32] = value.0.into(); + Bytes32::from(bytes) + } + } + impl From<&$type> for Bytes32 { + fn from(value: &$type) -> Self { + value.clone().into() + } + } + }; +} + +/// Macro for implementing From for Bytes32 +#[macro_export] +macro_rules! impl_from_type_to_bytes32 { + ($from_type:ty) => { + impl From<$from_type> for Bytes32 { + fn from(value: $from_type) -> Self { + Bytes32(fuel_core_types::fuel_tx::Bytes32::from(*value)) + } + } + impl From<&$from_type> for Bytes32 { + fn from(value: &$from_type) -> Self { + (*value).into() + } + } + }; +} diff --git a/crates/fuel-streams-core/src/subjects.rs b/crates/fuel-streams-types/src/primitives/identifier.rs similarity index 53% rename from crates/fuel-streams-core/src/subjects.rs rename to crates/fuel-streams-types/src/primitives/identifier.rs index 6eb58229..8d573a51 100644 --- a/crates/fuel-streams-core/src/subjects.rs +++ b/crates/fuel-streams-types/src/primitives/identifier.rs @@ -1,15 +1,4 @@ -pub use fuel_streams_macros::subject::*; - -use crate::primitive_types::*; -pub use crate::{ - blocks::subjects::*, - inputs::subjects::*, - logs::subjects::*, - outputs::subjects::*, - receipts::subjects::*, - transactions::subjects::*, - utxos::subjects::*, -}; +use crate::Bytes32; // ------------------------------------------------------------------------ // Identifier @@ -52,50 +41,51 @@ pub enum Identifier { #[macro_export] macro_rules! impl_from_identifier_for { ($subject:ident) => { - impl From for $subject { - fn from(identifier: Identifier) -> Self { + impl From<$crate::Identifier> for $subject { + fn from(identifier: $crate::Identifier) -> Self { match identifier { - Identifier::Address(tx_id, index, id) => $subject::build( - Some(tx_id), - Some(index), - Some(IdentifierKind::Address), - Some(id), - ), - Identifier::ContractID(tx_id, index, id) => { + $crate::Identifier::Address(tx_id, index, id) => { + $subject::build( + Some(tx_id), + Some(index), + Some($crate::IdentifierKind::Address), + Some(id), + ) + } + $crate::Identifier::ContractID(tx_id, index, id) => { + $subject::build( + Some(tx_id), + Some(index), + Some($crate::IdentifierKind::ContractID), + Some(id), + ) + } + $crate::Identifier::AssetID(tx_id, index, id) => { + $subject::build( + Some(tx_id), + Some(index), + Some($crate::IdentifierKind::AssetID), + Some(id), + ) + } + $crate::Identifier::PredicateID(tx_id, index, id) => { $subject::build( Some(tx_id), Some(index), - Some(IdentifierKind::ContractID), + Some($crate::IdentifierKind::PredicateID), Some(id), ) } - Identifier::AssetID(tx_id, index, id) => $subject::build( - Some(tx_id), - Some(index), - Some(IdentifierKind::AssetID), - Some(id), - ), - Identifier::PredicateID(tx_id, index, id) => { + $crate::Identifier::ScriptID(tx_id, index, id) => { $subject::build( Some(tx_id), Some(index), - Some(IdentifierKind::PredicateID), + Some($crate::IdentifierKind::ScriptID), Some(id), ) } - Identifier::ScriptID(tx_id, index, id) => $subject::build( - Some(tx_id), - Some(index), - Some(IdentifierKind::ScriptID), - Some(id), - ), } } } }; } - -impl_from_identifier_for!(TransactionsByIdSubject); -impl_from_identifier_for!(InputsByIdSubject); -impl_from_identifier_for!(OutputsByIdSubject); -impl_from_identifier_for!(ReceiptsByIdSubject); diff --git a/crates/fuel-streams-types/src/primitives/mod.rs b/crates/fuel-streams-types/src/primitives/mod.rs new file mode 100644 index 00000000..442e857c --- /dev/null +++ b/crates/fuel-streams-types/src/primitives/mod.rs @@ -0,0 +1,17 @@ +pub mod bytes; +pub mod bytes_long; +pub mod common; +pub mod identifier; +pub mod script_execution; +pub mod tx_pointer; +pub mod utxo_id; + +pub use bytes::*; +pub use bytes_long::*; +pub use identifier::*; +pub use script_execution::*; +pub use tx_pointer::*; +pub use utxo_id::*; + +pub type BoxedError = Box; +pub type BoxedResult = Result; diff --git a/crates/fuel-streams-types/src/primitives/script_execution.rs b/crates/fuel-streams-types/src/primitives/script_execution.rs new file mode 100644 index 00000000..9d1f0a80 --- /dev/null +++ b/crates/fuel-streams-types/src/primitives/script_execution.rs @@ -0,0 +1,53 @@ +use fuel_core_types::{fuel_asm::RawInstruction, fuel_tx::PanicReason}; + +use crate::fuel_core::*; + +#[derive( + Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize, +)] +pub struct PanicInstruction { + pub reason: PanicReason, + pub instruction: RawInstruction, +} +impl From for PanicInstruction { + fn from(value: FuelCorePanicInstruction) -> Self { + Self { + reason: value.reason().to_owned(), + instruction: value.instruction().to_owned(), + } + } +} + +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + Default, + serde::Serialize, + serde::Deserialize, +)] +#[repr(u64)] +pub enum ScriptExecutionResult { + Success, + Revert, + Panic, + // Generic failure case since any u64 is valid here + GenericFailure(u64), + #[default] + Unknown, +} +impl From for ScriptExecutionResult { + fn from(value: FuelCoreScriptExecutionResult) -> Self { + match value { + FuelCoreScriptExecutionResult::Success => Self::Success, + FuelCoreScriptExecutionResult::Revert => Self::Revert, + FuelCoreScriptExecutionResult::Panic => Self::Panic, + FuelCoreScriptExecutionResult::GenericFailure(value) => { + Self::GenericFailure(value) + } + } + } +} diff --git a/crates/fuel-streams-types/src/primitives/tx_pointer.rs b/crates/fuel-streams-types/src/primitives/tx_pointer.rs new file mode 100644 index 00000000..165048b6 --- /dev/null +++ b/crates/fuel-streams-types/src/primitives/tx_pointer.rs @@ -0,0 +1,28 @@ +use crate::fuel_core::*; + +#[derive( + Debug, + Default, + Copy, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Deserialize, + serde::Serialize, +)] +pub struct TxPointer { + block_height: FuelCoreBlockHeight, + tx_index: u16, +} + +impl From for TxPointer { + fn from(value: FuelCoreTxPointer) -> Self { + Self { + block_height: value.block_height(), + tx_index: value.tx_index(), + } + } +} diff --git a/crates/fuel-streams-types/src/primitives/utxo_id.rs b/crates/fuel-streams-types/src/primitives/utxo_id.rs new file mode 100644 index 00000000..128df4c9 --- /dev/null +++ b/crates/fuel-streams-types/src/primitives/utxo_id.rs @@ -0,0 +1,32 @@ +use super::Bytes32; +use crate::fuel_core::*; + +#[derive( + Debug, + Default, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, +)] +pub struct UtxoId { + pub tx_id: Bytes32, + pub output_index: u16, +} + +impl From for UtxoId { + fn from(value: FuelCoreUtxoId) -> Self { + Self::from(&value) + } +} + +impl From<&FuelCoreUtxoId> for UtxoId { + fn from(value: &FuelCoreUtxoId) -> Self { + Self { + tx_id: value.tx_id().into(), + output_index: value.output_index(), + } + } +} diff --git a/crates/fuel-streams/Cargo.toml b/crates/fuel-streams/Cargo.toml index 85a5138d..72ad50da 100644 --- a/crates/fuel-streams/Cargo.toml +++ b/crates/fuel-streams/Cargo.toml @@ -11,8 +11,8 @@ rust-version = { workspace = true } version = "0.0.16" [dependencies] -displaydoc = { workspace = true } fuel-streams-core = { workspace = true } +fuel-streams-store = { workspace = true } futures = { workspace = true } reqwest = "0.12.9" serde = { workspace = true } diff --git a/crates/fuel-streams/README.md b/crates/fuel-streams/README.md index 9e6dbaa4..010760f1 100644 --- a/crates/fuel-streams/README.md +++ b/crates/fuel-streams/README.md @@ -79,7 +79,7 @@ async fn main() -> Result<(), Box> { // Subscribe to blocks with last delivery policy let mut stream = connection - .subscribe::(subject, DeliverPolicy::Last) + .subscribe::(subject, DeliverPolicy::New) .await?; while let Some(block) = stream.next().await { @@ -129,7 +129,7 @@ async fn main() -> Result<(), Box> { // Subscribe to the filtered transaction stream let mut stream = connection - .subscribe::(subject, DeliverPolicy::Last) + .subscribe::(subject, DeliverPolicy::New) .await?; while let Some(transaction) = stream.next().await { diff --git a/crates/fuel-streams/src/client/connection.rs b/crates/fuel-streams/src/client/connection.rs index dd07b221..184b7f2b 100644 --- a/crates/fuel-streams/src/client/connection.rs +++ b/crates/fuel-streams/src/client/connection.rs @@ -1,4 +1,5 @@ -use fuel_streams_core::{subjects::IntoSubject, Streamable}; +use fuel_streams_core::subjects::*; +use fuel_streams_store::record::Record; use futures::{ stream::{SplitSink, SplitStream}, SinkExt, @@ -82,15 +83,16 @@ impl Connection { Ok(()) } - pub async fn subscribe( + pub async fn subscribe( &mut self, - subject: impl IntoSubject, + subject: impl IntoSubject + FromJsonString, deliver_policy: DeliverPolicy, ) -> Result> + '_ + Send + Unpin, ClientError> { let message = ClientMessage::Subscribe(SubscriptionPayload { - wildcard: subject.parse(), deliver_policy, + subject: subject.id().to_string(), + params: subject.to_json().into(), }); self.send_client_message(message).await?; @@ -136,13 +138,14 @@ impl Connection { Ok(Box::pin(stream)) } - pub async fn unsubscribe( + pub async fn unsubscribe( &self, - subject: S, + subject: impl IntoSubject + FromJsonString, deliver_policy: DeliverPolicy, ) -> Result<(), ClientError> { let message = ClientMessage::Unsubscribe(SubscriptionPayload { - wildcard: subject.parse(), + subject: subject.id().to_string(), + params: subject.to_json().into(), deliver_policy, }); self.send_client_message(message).await?; diff --git a/crates/fuel-streams/src/client/error.rs b/crates/fuel-streams/src/client/error.rs index a4f78938..1d184102 100644 --- a/crates/fuel-streams/src/client/error.rs +++ b/crates/fuel-streams/src/client/error.rs @@ -1,35 +1,23 @@ -use displaydoc::Display as DisplayDoc; -use thiserror::Error; - -#[derive(Debug, Error, DisplayDoc)] +#[derive(Debug, thiserror::Error)] pub enum ClientError { - /// Failed to convert JSON to string: {0} + #[error(transparent)] JsonToString(#[from] serde_json::Error), - - /// Failed to parse URL: {0} + #[error(transparent)] UrlParse(#[from] url::ParseError), - - /// Failed to API response: {0} + #[error(transparent)] ApiResponse(#[from] reqwest::Error), - - /// Failed to connect to WebSocket: {0} + #[error(transparent)] WebSocketConnect(#[from] tokio_tungstenite::tungstenite::Error), - - /// Invalid header value: {0} + #[error(transparent)] InvalidHeaderValue(#[from] reqwest::header::InvalidHeaderValue), - - /// Failed to parse host from URL + #[error("Failed to parse host from URL")] HostParseFailed, - - /// Missing JWT token + #[error("Missing JWT token")] MissingJwtToken, - - /// Missing write sink + #[error("Missing write sink")] MissingWriteSink, - - /// Missing read stream + #[error("Missing read stream")] MissingReadStream, - - /// Missing WebSocket connection + #[error("Missing WebSocket connection")] MissingWebSocketConnection, } diff --git a/crates/fuel-streams/src/client/types.rs b/crates/fuel-streams/src/client/types.rs index 938aa620..130bf762 100644 --- a/crates/fuel-streams/src/client/types.rs +++ b/crates/fuel-streams/src/client/types.rs @@ -1,9 +1,8 @@ -pub use sv_webserver::server::{ - http::models::{LoginRequest, LoginResponse}, - ws::models::{ - ClientMessage, - DeliverPolicy, - ServerMessage, - SubscriptionPayload, - }, +pub use fuel_streams_core::DeliverPolicy; +pub use sv_webserver::server::types::{ + ClientMessage, + LoginRequest, + LoginResponse, + ServerMessage, + SubscriptionPayload, }; diff --git a/crates/fuel-streams/src/error.rs b/crates/fuel-streams/src/error.rs index 46b5cf8f..c93bdff7 100644 --- a/crates/fuel-streams/src/error.rs +++ b/crates/fuel-streams/src/error.rs @@ -1,8 +1,5 @@ -use displaydoc::Display as DisplayDoc; -use thiserror::Error as ThisError; - -#[derive(Debug, ThisError, DisplayDoc)] +#[derive(Debug, thiserror::Error)] pub enum Error { - /// WebSocket client error: {0} + #[error(transparent)] Client(#[from] crate::client::error::ClientError), } diff --git a/crates/fuel-streams/src/lib.rs b/crates/fuel-streams/src/lib.rs index f4e965f8..9cc63a4b 100644 --- a/crates/fuel-streams/src/lib.rs +++ b/crates/fuel-streams/src/lib.rs @@ -18,24 +18,6 @@ pub mod types { pub use crate::client::types::*; } -macro_rules! export_module { - ($module:ident, $($submodule:ident),+) => { - pub mod $module { - $( - pub use fuel_streams_core::$module::$submodule::*; - )+ - } - }; -} - -export_module!(blocks, subjects, types); -export_module!(inputs, subjects, types); -export_module!(logs, subjects, types); -export_module!(outputs, subjects, types); -export_module!(receipts, subjects); -export_module!(transactions, subjects, types); -export_module!(utxos, subjects, types); - #[cfg(any(test, feature = "test-helpers"))] pub mod prelude { pub use crate::{client::*, error::*, networks::*, subjects::*, types::*}; diff --git a/crates/fuel-web-utils/Cargo.toml b/crates/fuel-web-utils/Cargo.toml index d6176b14..32a4d316 100644 --- a/crates/fuel-web-utils/Cargo.toml +++ b/crates/fuel-web-utils/Cargo.toml @@ -23,7 +23,7 @@ displaydoc = { workspace = true } dotenvy = { workspace = true } elasticsearch = "8.15.0-alpha.1" fuel-data-parser = { workspace = true } -fuel-streams-nats = { workspace = true, features = ["test-helpers"] } +fuel-message-broker = { workspace = true } futures = { workspace = true } futures-util = { workspace = true } jsonwebtoken = "9.3.0" diff --git a/crates/fuel-web-utils/src/server/middlewares/auth/jwt.rs b/crates/fuel-web-utils/src/server/middlewares/auth/jwt.rs index b1110f64..26186ba5 100755 --- a/crates/fuel-web-utils/src/server/middlewares/auth/jwt.rs +++ b/crates/fuel-web-utils/src/server/middlewares/auth/jwt.rs @@ -6,7 +6,6 @@ use actix_web::{ ResponseError, }; use chrono::Utc; -use displaydoc::Display as DisplayDoc; use jsonwebtoken::{ decode, encode, @@ -17,7 +16,6 @@ use jsonwebtoken::{ Validation, }; use serde::{Deserialize, Serialize}; -use thiserror::Error; use uuid::Uuid; const BEARER: &str = "Bearer"; @@ -31,23 +29,23 @@ pub enum UserType { } /// User-related errors -#[derive(Clone, Debug, DisplayDoc, Error, PartialEq)] +#[derive(Clone, Debug, thiserror::Error, PartialEq)] pub enum UserError { - /// User not found + #[error("User not found")] UserNotFound, - /// Unknown User Role: `{0}` + #[error("Unknown User Role: {0}")] UnknownUserRole(String), - /// Unknown User Status: `{0}` + #[error("Unknown User Status: {0}")] UnknownUserStatus(String), - /// Unallowed User Role: `{0}` + #[error("Unallowed User Role: {0}")] UnallowedUserRole(String), - /// Missing password + #[error("Missing password")] MissingPassword, - /// Missing username + #[error("Missing username")] MissingUsername, - /// Wrong password + #[error("Wrong password")] WrongPassword, - /// User is not verified + #[error("User is not verified")] UnverifiedUser, } @@ -83,25 +81,25 @@ impl ResponseError for UserError { } /// Auth errors -#[derive(Clone, Debug, DisplayDoc, Error, PartialEq)] +#[derive(Clone, Debug, thiserror::Error, PartialEq)] pub enum AuthError { - /// Wrong Credentials + #[error("Wrong Credentials")] WrongCredentialsError, - /// JWT Token not valid + #[error("JWT Token not valid")] JWTTokenError, - /// JWT Token Creation Error + #[error("JWT Token Creation Error")] JWTTokenCreationError, - /// No Auth Header + #[error("No Auth Header")] NoAuthHeaderError, - /// Invalid Auth Header + #[error("Invalid Auth Header")] InvalidAuthHeaderError, - /// No Permission + #[error("No Permission")] NoPermissionError, - /// Expired Token + #[error("Expired Token")] ExpiredToken, - /// Bad Encoded User Role: `{0}` + #[error("Bad Encoded User Role: {0}")] BadEncodedUserRole(String), - /// Unparsable UUID error: `{0}` + #[error("Unparsable UUID error: {0}")] UnparsableUuid(String), } diff --git a/crates/fuel-web-utils/src/shutdown.rs b/crates/fuel-web-utils/src/shutdown.rs index 19ce2fb1..a2ab539f 100644 --- a/crates/fuel-web-utils/src/shutdown.rs +++ b/crates/fuel-web-utils/src/shutdown.rs @@ -1,14 +1,14 @@ use std::{sync::Arc, time::Duration}; -use fuel_streams_nats::NatsClient; +use fuel_message_broker::MessageBroker; use tokio_util::sync::CancellationToken; pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(90); -pub async fn shutdown_nats_with_timeout(nats_client: &NatsClient) { +pub async fn shutdown_broker_with_timeout(broker: &Arc) { let _ = tokio::time::timeout(GRACEFUL_SHUTDOWN_TIMEOUT, async { - tracing::info!("Flushing in-flight messages to nats ..."); - match nats_client.nats_client.flush().await { + tracing::info!("Flushing in-flight messages to broker ..."); + match broker.flush().await { Ok(_) => { tracing::info!("Flushed all streams successfully!"); } diff --git a/crates/fuel-web-utils/src/telemetry/system.rs b/crates/fuel-web-utils/src/telemetry/system.rs index f795e20e..c0942d68 100644 --- a/crates/fuel-web-utils/src/telemetry/system.rs +++ b/crates/fuel-web-utils/src/telemetry/system.rs @@ -7,7 +7,6 @@ use std::{ }; use derive_more::Deref; -use displaydoc::Display as DisplayDoc; use rust_decimal::{ prelude::{FromPrimitive, ToPrimitive}, Decimal, @@ -23,7 +22,6 @@ use sysinfo::{ RefreshKind, SystemExt, }; -use thiserror::Error; use tokio::time; // TODO: move this to web interface as `SystemsMetricsResponse` ? @@ -38,9 +36,9 @@ impl From for SystemMetricsWrapper { } } -#[derive(Debug, Error, DisplayDoc)] +#[derive(Debug, thiserror::Error)] pub enum Error { - /// The process {0} could not be found + #[error("The process {0} could not be found")] ProcessNotFound(Pid), } diff --git a/crates/sv-consumer/Cargo.toml b/crates/sv-consumer/Cargo.toml index 21f20c74..133eaf1c 100644 --- a/crates/sv-consumer/Cargo.toml +++ b/crates/sv-consumer/Cargo.toml @@ -17,20 +17,20 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } -async-nats = { workspace = true } async-trait = { workspace = true } +bincode = { workspace = true } clap = { workspace = true } -displaydoc = { workspace = true } dotenvy = { workspace = true } fuel-core = { workspace = true, default-features = false, features = ["p2p", "relayer", "rocksdb"] } +fuel-message-broker = { workspace = true } fuel-streams-core = { workspace = true, features = ["test-helpers"] } fuel-streams-executors = { workspace = true, features = ["test-helpers"] } +fuel-streams-store = { workspace = true } fuel-web-utils = { workspace = true } futures = { workspace = true } hex = { workspace = true } num_cpus = { workspace = true } prometheus = { version = "0.13", features = ["process"] } -serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/crates/sv-consumer/src/cli.rs b/crates/sv-consumer/src/cli.rs index b7e2b709..8f04a0fd 100644 --- a/crates/sv-consumer/src/cli.rs +++ b/crates/sv-consumer/src/cli.rs @@ -20,15 +20,15 @@ pub struct Cli { help = "NATS URL to connect to." )] pub nats_url: String, - /// Nats publisher URL + /// Database URL to connect to. #[arg( long, - value_name = "NATS_PUBLISHER_URL", - env = "NATS_PUBLISHER_URL", - default_value = "localhost:4333", - help = "NATS Publisher URL to connect to." + value_name = "DATABASE_URL", + env = "DATABASE_URL", + default_value = "postgresql://root@localhost:26257/defaultdb?sslmode=disable", + help = "Database URL to connect to." )] - pub nats_publisher_url: String, + pub db_url: String, /// Use metrics #[arg( long, diff --git a/crates/sv-consumer/src/lib.rs b/crates/sv-consumer/src/lib.rs index 3fa43b53..8575ea14 100644 --- a/crates/sv-consumer/src/lib.rs +++ b/crates/sv-consumer/src/lib.rs @@ -1,35 +1,3 @@ -use std::sync::Arc; - -use fuel_streams_core::prelude::*; - pub mod cli; pub mod metrics; pub mod state; - -#[derive(Debug, Clone, Default)] -pub enum Client { - #[default] - Core, - Publisher, -} - -impl Client { - pub fn url(&self, cli: &cli::Cli) -> String { - match self { - Client::Core => cli.nats_url.clone(), - Client::Publisher => cli.nats_publisher_url.clone(), - } - } - pub async fn create( - &self, - cli: &cli::Cli, - ) -> Result, NatsError> { - let url = self.url(cli); - let opts = NatsClientOpts::admin_opts() - .with_url(url) - .with_domain("CORE".to_string()) - .with_user("admin".to_string()) - .with_password("admin".to_string()); - Ok(Arc::new(NatsClient::connect(&opts).await?)) - } -} diff --git a/crates/sv-consumer/src/main.rs b/crates/sv-consumer/src/main.rs index 0084784e..e440305c 100644 --- a/crates/sv-consumer/src/main.rs +++ b/crates/sv-consumer/src/main.rs @@ -1,58 +1,43 @@ -use std::{ - sync::{Arc, LazyLock}, - time::Duration, -}; +use std::sync::{Arc, LazyLock}; -use async_nats::jetstream::{ - consumer::{ - pull::{BatchErrorKind, Config as ConsumerConfig}, - Consumer, - }, - context::CreateStreamErrorKind, - stream::{ConsumerErrorKind, RetentionPolicy}, -}; use clap::Parser; -use displaydoc::Display as DisplayDoc; -use fuel_streams_core::prelude::*; +use fuel_message_broker::{MessageBroker, MessageBrokerClient}; +use fuel_streams_core::{types::*, FuelStreams}; use fuel_streams_executors::*; +use fuel_streams_store::{ + db::{Db, DbConnectionOpts}, + record::DataEncoder, +}; use fuel_web_utils::{ server::api::build_and_spawn_web_server, - shutdown::{shutdown_nats_with_timeout, ShutdownController}, + shutdown::{shutdown_broker_with_timeout, ShutdownController}, telemetry::Telemetry, }; use futures::{future::try_join_all, stream::FuturesUnordered, StreamExt}; -use sv_consumer::{cli::Cli, state::ServerState, Client}; +use sv_consumer::{cli::Cli, state::ServerState}; use tokio_util::sync::CancellationToken; use tracing::level_filters::LevelFilter; use tracing_subscriber::fmt::time; -#[derive(thiserror::Error, Debug, DisplayDoc)] +#[derive(thiserror::Error, Debug)] pub enum ConsumerError { - /// Failed to receive batch of messages from NATS: {0} - BatchStream(#[from] async_nats::error::Error), - /// Failed to create stream: {0} - CreateStream(#[from] async_nats::error::Error), - /// Failed to create consumer: {0} - CreateConsumer(#[from] async_nats::error::Error), - /// Failed to connect to NATS client: {0} - NatsClient(#[from] NatsError), - /// Failed to communicate with NATS server: {0} - Nats(#[from] async_nats::Error), - /// Failed to deserialize block payload from message: {0} - Deserialization(#[from] serde_json::Error), - /// Failed to decode UTF-8: {0} + #[error(transparent)] + Deserialization(#[from] bincode::Error), + #[error(transparent)] Utf8(#[from] std::str::Utf8Error), - /// Failed to execute executor tasks: {0} + #[error(transparent)] Executor(#[from] ExecutorError), - /// Failed to join tasks: {0} + #[error(transparent)] JoinTasks(#[from] tokio::task::JoinError), - /// Failed to acquire semaphore: {0} + #[error(transparent)] Semaphore(#[from] tokio::sync::AcquireError), - /// Failed to setup storage: {0} - Storage(#[from] fuel_streams_core::storage::StorageError), - /// Failed to start telemetry + #[error(transparent)] + Db(#[from] fuel_streams_store::db::DbError), + #[error(transparent)] + MessageBrokerClient(#[from] fuel_message_broker::MessageBrokerError), + #[error("Failed to start telemetry")] TelemetryStart, - /// Failed to start web server + #[error("Failed to start web server")] WebServerStart, } @@ -99,42 +84,22 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -async fn setup_storage() -> Result, ConsumerError> { - let storage_opts = S3StorageOpts::admin_opts(); - let storage = S3Storage::new(storage_opts).await?; - Ok(Arc::new(storage)) +async fn setup_db(db_url: &str) -> Result, ConsumerError> { + let db = Db::new(DbConnectionOpts { + connection_str: db_url.to_string(), + pool_size: Some(5), + }) + .await?; + Ok(db.arc()) } -async fn setup_nats( - cli: &Cli, -) -> Result< - (Arc, Arc, Consumer), - ConsumerError, -> { - let core_client = Client::Core.create(cli).await?; - let publisher_client = Client::Publisher.create(cli).await?; - let stream_name = publisher_client.namespace.stream_name("block_importer"); - let stream = publisher_client - .jetstream - .get_or_create_stream(async_nats::jetstream::stream::Config { - name: stream_name, - subjects: vec!["block_submitted.>".to_string()], - retention: RetentionPolicy::WorkQueue, - duplicate_window: Duration::from_secs(1), - allow_rollup: true, - ..Default::default() - }) - .await?; - - let consumer = stream - .get_or_create_consumer("block_importer", ConsumerConfig { - durable_name: Some("block_importer".to_string()), - ack_policy: AckPolicy::Explicit, - ..Default::default() - }) - .await?; - - Ok((core_client, publisher_client, consumer)) +async fn setup_message_broker( + nats_url: &str, +) -> Result, ConsumerError> { + let broker = MessageBrokerClient::Nats; + let broker = broker.start(nats_url).await?; + broker.setup().await?; + Ok(broker) } pub static CONSUMER_MAX_THREADS: LazyLock = LazyLock::new(|| { @@ -149,12 +114,9 @@ async fn process_messages( cli: &Cli, token: &CancellationToken, ) -> Result<(), ConsumerError> { - let (core_client, publisher_client, consumer) = setup_nats(cli).await?; - let storage = setup_storage().await?; - let (_, publisher_stream) = - FuelStreams::setup_all(&core_client, &publisher_client, &storage).await; - let fuel_streams: Arc = publisher_stream.arc(); - + let db = setup_db(&cli.db_url).await?; + let message_broker = setup_message_broker(&cli.nats_url).await?; + let fuel_streams = FuelStreams::new(&message_broker, &db).await.arc(); let telemetry = Telemetry::new(None) .await .map_err(|_| ConsumerError::TelemetryStart)?; @@ -164,7 +126,7 @@ async fn process_messages( .map_err(|_| ConsumerError::TelemetryStart)?; let server_state = ServerState::new( - publisher_client.clone(), + message_broker.clone(), Arc::clone(&telemetry), Arc::clone(&fuel_streams), ); @@ -175,21 +137,21 @@ async fn process_messages( let semaphore = Arc::new(tokio::sync::Semaphore::new(64)); while !token.is_cancelled() { let mut messages = - consumer.fetch().max_messages(100).messages().await?.fuse(); + message_broker.receive_blocks_stream(100).await?.fuse(); let mut futs = FuturesUnordered::new(); while let Some(msg) = messages.next().await { let msg = msg?; + let payload = msg.payload(); let fuel_streams = fuel_streams.clone(); let semaphore = semaphore.clone(); - tracing::debug!( "Received message payload length: {}", - msg.payload.len() + payload.len() ); let future = async move { - match BlockPayload::decode(&msg.payload).await { + match BlockPayload::decode(&payload).await { Ok(payload) => { let payload = Arc::new(payload); let start_time = std::time::Instant::now(); @@ -207,7 +169,7 @@ async fn process_messages( tracing::error!("Failed to decode payload: {:?}", e); tracing::debug!( "Raw payload (hex): {:?}", - hex::encode(&msg.payload) + hex::encode(payload) ); Err(e) } @@ -223,7 +185,7 @@ async fn process_messages( tracing::info!("Stopping actix server ..."); server_handle.stop(true).await; tracing::info!("Actix server stopped. Goodbye!"); - shutdown_nats_with_timeout(&publisher_client).await; + shutdown_broker_with_timeout(&message_broker).await; Ok(()) } diff --git a/crates/sv-consumer/src/state.rs b/crates/sv-consumer/src/state.rs index ef373d86..9455f603 100644 --- a/crates/sv-consumer/src/state.rs +++ b/crates/sv-consumer/src/state.rs @@ -3,80 +3,29 @@ use std::{ time::{Duration, Instant}, }; -use async_nats::jetstream::stream::State; use async_trait::async_trait; -use fuel_streams_core::{nats::NatsClient, FuelStreamsExt}; +use fuel_message_broker::MessageBroker; +use fuel_streams_core::FuelStreams; use fuel_web_utils::{server::state::StateProvider, telemetry::Telemetry}; -use serde::{Deserialize, Serialize}; use crate::metrics::Metrics; -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct StreamInfo { - consumers: Vec, - state: StreamState, - stream_name: String, -} - -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] -pub struct StreamState { - /// The number of messages contained in this stream - pub messages: u64, - /// The number of bytes of all messages contained in this stream - pub bytes: u64, - /// The lowest sequence number still present in this stream - #[serde(rename = "first_seq")] - pub first_sequence: u64, - /// The time associated with the oldest message still present in this stream - #[serde(rename = "first_ts")] - pub first_timestamp: i64, - /// The last sequence number assigned to a message in this stream - #[serde(rename = "last_seq")] - pub last_sequence: u64, - /// The time that the last message was received by this stream - #[serde(rename = "last_ts")] - pub last_timestamp: i64, - /// The number of consumers configured to consume this stream - pub consumer_count: usize, -} - -impl From for StreamState { - fn from(state: State) -> Self { - StreamState { - messages: state.messages, - bytes: state.bytes, - first_sequence: state.first_sequence, - first_timestamp: state.first_timestamp.unix_timestamp(), - last_sequence: state.last_sequence, - last_timestamp: state.last_timestamp.unix_timestamp(), - consumer_count: state.consumer_count, - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct HealthResponse { - pub uptime_secs: u64, - pub is_healthy: bool, - pub streams_info: Vec, -} - pub struct ServerState { pub start_time: Instant, - pub nats_client: Arc, + pub msg_broker: Arc, pub telemetry: Arc>, - pub fuel_streams: Arc, + pub fuel_streams: Arc, } impl ServerState { pub fn new( - nats_client: Arc, + msg_broker: Arc, telemetry: Arc>, - fuel_streams: Arc, + fuel_streams: Arc, ) -> Self { Self { start_time: Instant::now(), - nats_client, + msg_broker, telemetry, fuel_streams, } @@ -90,29 +39,14 @@ impl ServerState { #[async_trait] impl StateProvider for ServerState { async fn is_healthy(&self) -> bool { - self.nats_client.is_connected() + self.msg_broker.is_healthy().await } async fn get_health(&self) -> serde_json::Value { - let streams_info = self - .fuel_streams - .get_consumers_and_state() + self.msg_broker + .get_health_info(self.uptime().as_secs()) .await - .unwrap_or_default() - .into_iter() - .map(|res| StreamInfo { - consumers: res.1, - state: res.2.into(), - stream_name: res.0, - }) - .collect::>(); - - let resp = HealthResponse { - uptime_secs: self.uptime().as_secs(), - is_healthy: self.is_healthy().await, - streams_info, - }; - serde_json::to_value(resp).unwrap_or(serde_json::json!({})) + .unwrap_or(serde_json::json!({})) } async fn get_metrics(&self) -> String { diff --git a/crates/sv-publisher/Cargo.toml b/crates/sv-publisher/Cargo.toml index a5098ecb..fe70352f 100644 --- a/crates/sv-publisher/Cargo.toml +++ b/crates/sv-publisher/Cargo.toml @@ -17,10 +17,8 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } -async-nats = { workspace = true } async-trait = { workspace = true } clap = { workspace = true } -displaydoc = { workspace = true } fuel-core = { workspace = true, default-features = false, features = ["p2p", "relayer", "rocksdb"] } fuel-core-bin = { workspace = true, default-features = false, features = [ "p2p", @@ -28,8 +26,10 @@ fuel-core-bin = { workspace = true, default-features = false, features = [ "rocksdb", ] } fuel-core-types = { workspace = true, default-features = false, features = ["std", "serde"] } +fuel-message-broker = { workspace = true } fuel-streams-core = { workspace = true, features = ["test-helpers"] } fuel-streams-executors = { workspace = true, features = ["test-helpers"] } +fuel-streams-store = { workspace = true } fuel-web-utils = { workspace = true } futures = { workspace = true } prometheus = { version = "0.13", features = ["process"] } diff --git a/crates/sv-publisher/src/cli.rs b/crates/sv-publisher/src/cli.rs index 35ab6a36..f60c9e5a 100644 --- a/crates/sv-publisher/src/cli.rs +++ b/crates/sv-publisher/src/cli.rs @@ -30,6 +30,23 @@ pub struct Cli { help = "NATS URL to connect to." )] pub nats_url: String, + /// Database URL to connect to. + #[arg( + long, + value_name = "DATABASE_URL", + env = "DATABASE_URL", + default_value = "postgresql://root@localhost:26257/defaultdb?sslmode=disable", + help = "Database URL to connect to." + )] + pub db_url: String, + /// Start from block height + #[arg( + long, + value_name = "FROM_HEIGHT", + default_value = "0", + help = "Start from block height" + )] + pub from_height: u32, /// Use metrics #[arg( long, diff --git a/crates/sv-publisher/src/main.rs b/crates/sv-publisher/src/main.rs index d9dc54aa..fcef3347 100644 --- a/crates/sv-publisher/src/main.rs +++ b/crates/sv-publisher/src/main.rs @@ -1,31 +1,43 @@ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; -use async_nats::jetstream::{ - context::{Publish, PublishErrorKind}, - stream::RetentionPolicy, - Context, -}; use clap::Parser; -use displaydoc::Display as DisplayDoc; use fuel_core_types::blockchain::SealedBlock; -use fuel_streams_core::prelude::*; +use fuel_message_broker::{ + MessageBroker, + MessageBrokerClient, + MessageBrokerError, +}; +use fuel_streams_core::{types::*, FuelCore, FuelCoreLike}; use fuel_streams_executors::*; +use fuel_streams_store::{ + db::{Db, DbConnectionOpts}, + record::{DataEncoder, EncoderError, QueryOptions, Record}, +}; use fuel_web_utils::{ server::api::build_and_spawn_web_server, - shutdown::{shutdown_nats_with_timeout, ShutdownController}, + shutdown::{shutdown_broker_with_timeout, ShutdownController}, telemetry::Telemetry, }; use futures::StreamExt; use sv_publisher::{cli::Cli, metrics::Metrics, state::ServerState}; -use thiserror::Error; use tokio_util::sync::CancellationToken; -#[derive(Error, Debug, DisplayDoc)] -pub enum LiveBlockProcessingError { - /// Failed to publish block: {0} - PublishError(#[from] PublishError), - /// Processing was cancelled +#[derive(thiserror::Error, Debug)] +pub enum PublishError { + #[error(transparent)] + BlockPayload(#[from] ExecutorError), + #[error("Failed to access offchain database: {0}")] + OffchainDatabase(String), + #[error(transparent)] + Db(#[from] fuel_streams_store::db::DbError), + #[error("Failed to initialize Fuel Core: {0}")] + FuelCore(String), + #[error(transparent)] + Encoder(#[from] EncoderError), + #[error("Processing was cancelled")] Cancelled, + #[error(transparent)] + MessageBrokerClient(#[from] MessageBrokerError), } #[tokio::main] @@ -35,21 +47,18 @@ async fn main() -> anyhow::Result<()> { let fuel_core: Arc = FuelCore::new(config).await?; fuel_core.start().await?; + let db = setup_db(&cli.db_url).await?; + let message_broker = setup_message_broker(&cli.nats_url).await?; let telemetry = Telemetry::new(None).await?; telemetry.start().await?; - let storage = setup_storage().await?; - let nats_client = setup_nats(&cli.nats_url).await?; - let server_state = - ServerState::new(nats_client.clone(), Arc::clone(&telemetry)); + ServerState::new(message_broker.clone(), Arc::clone(&telemetry)); let server_handle = build_and_spawn_web_server(cli.port, server_state).await?; let last_block_height = Arc::new(fuel_core.get_latest_block_height()?); - let last_published = - Arc::new(find_last_published_height(&nats_client, &storage).await?); - + let last_published = Arc::new(find_last_published_height(&db).await?); let shutdown = Arc::new(ShutdownController::new()); shutdown.clone().spawn_signal_handler(); @@ -58,7 +67,8 @@ async fn main() -> anyhow::Result<()> { tokio::select! { result = async { let historical = process_historical_blocks( - &nats_client, + cli.from_height, + &message_broker, fuel_core.clone(), last_block_height, last_published, @@ -67,7 +77,7 @@ async fn main() -> anyhow::Result<()> { ); let live = process_live_blocks( - &nats_client.jetstream, + &message_broker, fuel_core.clone(), shutdown.token().clone(), Arc::clone(&telemetry) @@ -84,7 +94,7 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Stopping actix server ..."); server_handle.stop(true).await; tracing::info!("Actix server stopped. Goodbye!"); - shutdown_nats_with_timeout(&nats_client).await; + shutdown_broker_with_timeout(&message_broker).await; } } @@ -92,52 +102,50 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -async fn setup_storage() -> anyhow::Result> { - let storage_opts = S3StorageOpts::admin_opts(); - let storage = S3Storage::new(storage_opts).await?; - Ok(Arc::new(storage)) +async fn setup_db(db_url: &str) -> Result { + let db = Db::new(DbConnectionOpts { + connection_str: db_url.to_string(), + ..Default::default() + }) + .await?; + Ok(db) } -async fn setup_nats(nats_url: &str) -> anyhow::Result { - let opts = NatsClientOpts::admin_opts() - .with_url(nats_url.to_string()) - .with_domain("CORE".to_string()); - let nats_client = NatsClient::connect(&opts).await?; - let stream_name = nats_client.namespace.stream_name("block_importer"); - nats_client - .jetstream - .get_or_create_stream(async_nats::jetstream::stream::Config { - name: stream_name, - subjects: vec!["block_submitted.>".to_string()], - retention: RetentionPolicy::WorkQueue, - duplicate_window: Duration::from_secs(1), - ..Default::default() - }) - .await?; - - Ok(nats_client) +async fn setup_message_broker( + nats_url: &str, +) -> Result, PublishError> { + let broker = MessageBrokerClient::Nats; + let broker = broker.start(nats_url).await?; + broker.setup().await?; + Ok(broker) } -async fn find_last_published_height( - nats_client: &NatsClient, - storage: &Arc, -) -> anyhow::Result { - let block_stream = Stream::::get_or_init(nats_client, storage).await; - let last_publish_height = block_stream - .get_last_published(BlocksSubject::WILDCARD) - .await?; - match last_publish_height { - Some(block) => Ok(block.height), +async fn find_last_published_height(db: &Db) -> Result { + let record = Block::find_last_record(db, QueryOptions::default()).await?; + match record { + Some(record) => Ok(record.block_height as u32), None => Ok(0), } } fn get_historical_block_range( + from_height: u32, last_published_height: Arc, last_block_height: Arc, ) -> Option> { - let last_published_height = *last_published_height; - let last_block_height = *last_block_height; + let last_published_height = if *last_published_height > from_height { + *last_published_height + } else { + from_height + }; + + let last_block_height = if *last_block_height > from_height { + *last_block_height + } else { + tracing::error!("Last block height is less than from height"); + *last_block_height + }; + let start_height = last_published_height + 1; let end_height = last_block_height; if start_height > end_height { @@ -153,16 +161,19 @@ fn get_historical_block_range( } fn process_historical_blocks( - nats_client: &NatsClient, + from_height: u32, + message_broker: &Arc, fuel_core: Arc, last_block_height: Arc, last_published_height: Arc, token: CancellationToken, telemetry: Arc>, ) -> tokio::task::JoinHandle<()> { - let jetstream = nats_client.jetstream.clone(); + let message_broker = message_broker.clone(); + let fuel_core = fuel_core.clone(); tokio::spawn(async move { let Some(heights) = get_historical_block_range( + from_height, last_published_height, last_block_height, ) else { @@ -170,14 +181,14 @@ fn process_historical_blocks( }; futures::stream::iter(heights) .map(|height| { - let jetstream = jetstream.clone(); + let message_broker = message_broker.clone(); let fuel_core = fuel_core.clone(); let sealed_block = fuel_core.get_sealed_block_by_height(height); let sealed_block = Arc::new(sealed_block); let telemetry = telemetry.clone(); async move { publish_block( - &jetstream, + &message_broker, &fuel_core, &sealed_block, telemetry, @@ -193,35 +204,30 @@ fn process_historical_blocks( } async fn process_live_blocks( - jetstream: &Context, + message_broker: &Arc, fuel_core: Arc, token: CancellationToken, telemetry: Arc>, -) -> Result<(), LiveBlockProcessingError> { +) -> Result<(), PublishError> { let mut subscription = fuel_core.blocks_subscription(); while let Ok(data) = subscription.recv().await { if token.is_cancelled() { break; } let sealed_block = Arc::new(data.sealed_block.clone()); - publish_block(jetstream, &fuel_core, &sealed_block, telemetry.clone()) - .await?; + publish_block( + message_broker, + &fuel_core, + &sealed_block, + telemetry.clone(), + ) + .await?; } Ok(()) } -#[derive(Error, Debug, DisplayDoc)] -pub enum PublishError { - /// Failed to publish block to NATS server: {0} - NatsPublish(#[from] async_nats::error::Error), - /// Failed to create block payload due to: {0} - BlockPayload(#[from] ExecutorError), - /// Failed to access offchain database: {0} - OffchainDatabase(String), -} - async fn publish_block( - jetstream: &Context, + message_broker: &Arc, fuel_core: &Arc, sealed_block: &Arc, telemetry: Arc>, @@ -229,22 +235,17 @@ async fn publish_block( let metadata = Metadata::new(fuel_core, sealed_block); let fuel_core = Arc::clone(fuel_core); let payload = BlockPayload::new(fuel_core, sealed_block, &metadata)?; - let encoded_payload = payload.encode().await?; - let payload_size = encoded_payload.len(); - let publish = Publish::build() - .message_id(payload.message_id()) - .payload(encoded_payload.into()); - - jetstream - .send_publish(payload.subject(), publish) - .await - .map_err(PublishError::NatsPublish)? - .await - .map_err(PublishError::NatsPublish)?; + let encoded = payload.encode().await?; + + message_broker + .publish_block(payload.message_id(), encoded.clone()) + .await?; if let Some(metrics) = telemetry.base_metrics() { - metrics - .update_publisher_success_metrics(&payload.subject(), payload_size); + metrics.update_publisher_success_metrics( + &payload.subject(), + encoded.len(), + ); } tracing::info!("New block submitted: {}", payload.block_height()); diff --git a/crates/sv-publisher/src/state.rs b/crates/sv-publisher/src/state.rs index 6d6e86ed..bbcaa9b6 100644 --- a/crates/sv-publisher/src/state.rs +++ b/crates/sv-publisher/src/state.rs @@ -4,7 +4,7 @@ use std::{ }; use async_trait::async_trait; -use fuel_streams_core::nats::NatsClient; +use fuel_message_broker::MessageBroker; use fuel_web_utils::{server::state::StateProvider, telemetry::Telemetry}; use serde::{Deserialize, Serialize}; @@ -18,18 +18,18 @@ pub struct HealthResponse { pub struct ServerState { pub start_time: Instant, - pub nats_client: NatsClient, + pub msg_broker: Arc, pub telemetry: Arc>, } impl ServerState { pub fn new( - nats_client: NatsClient, + msg_broker: Arc, telemetry: Arc>, ) -> Self { Self { start_time: Instant::now(), - nats_client, + msg_broker, telemetry, } } @@ -42,7 +42,7 @@ impl ServerState { #[async_trait] impl StateProvider for ServerState { async fn is_healthy(&self) -> bool { - self.nats_client.is_connected() + self.msg_broker.is_healthy().await } async fn get_health(&self) -> serde_json::Value { diff --git a/crates/sv-webserver/Cargo.toml b/crates/sv-webserver/Cargo.toml index 26c00403..7af00901 100644 --- a/crates/sv-webserver/Cargo.toml +++ b/crates/sv-webserver/Cargo.toml @@ -18,19 +18,19 @@ path = "src/main.rs" actix-web = { workspace = true } actix-ws = "0.3.0" anyhow = { workspace = true } -async-nats = { workspace = true } async-trait = { workspace = true } +bytes = { workspace = true } clap = { workspace = true } displaydoc = { workspace = true } dotenvy = { workspace = true } fuel-data-parser = { workspace = true } +fuel-message-broker = { workspace = true } fuel-streams-core = { workspace = true, features = ["test-helpers"] } -fuel-streams-nats = { workspace = true, features = ["test-helpers"] } -fuel-streams-storage = { workspace = true, features = ["test-helpers"] } +fuel-streams-domains = { workspace = true } +fuel-streams-store = { workspace = true } fuel-web-utils = { workspace = true } futures = { workspace = true } num_cpus = { workspace = true } -parking_lot = { version = "0.12", features = ["serde"] } prometheus = { version = "0.13", features = ["process"] } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/sv-webserver/src/cli.rs b/crates/sv-webserver/src/cli.rs index 37cad8bd..f722c919 100644 --- a/crates/sv-webserver/src/cli.rs +++ b/crates/sv-webserver/src/cli.rs @@ -23,15 +23,15 @@ pub struct Cli { )] pub nats_url: String, - /// Enable S3 + /// Database URL to connect to. #[arg( long, - value_name = "AWS_S3_ENABLED", - env = "AWS_S3_ENABLED", - default_value = "true", - help = "Enable S3 integration" + value_name = "DATABASE_URL", + env = "DATABASE_URL", + default_value = "postgresql://root@localhost:26257/defaultdb?sslmode=disable", + help = "Database URL to connect to." )] - pub s3_enabled: bool, + pub db_url: String, /// JWT secret #[arg( diff --git a/crates/sv-webserver/src/config.rs b/crates/sv-webserver/src/config.rs index eeaada1b..4d0420aa 100644 --- a/crates/sv-webserver/src/config.rs +++ b/crates/sv-webserver/src/config.rs @@ -33,7 +33,12 @@ pub struct AuthConfig { } #[derive(Clone, Debug)] -pub struct NatsConfig { +pub struct BrokerConfig { + pub url: String, +} + +#[derive(Clone, Debug)] +pub struct DbConfig { pub url: String, } @@ -41,8 +46,8 @@ pub struct NatsConfig { pub struct Config { pub api: ApiConfig, pub auth: AuthConfig, - pub s3: S3Config, - pub nats: NatsConfig, + pub broker: BrokerConfig, + pub db: DbConfig, } impl Config { @@ -60,11 +65,11 @@ impl Config { auth: AuthConfig { jwt_secret: cli.jwt_secret.clone(), }, - nats: NatsConfig { + broker: BrokerConfig { url: cli.nats_url.clone(), }, - s3: S3Config { - enabled: cli.s3_enabled, + db: DbConfig { + url: cli.db_url.clone(), }, }) } diff --git a/crates/sv-webserver/src/main.rs b/crates/sv-webserver/src/main.rs index 9b908dac..a9997231 100644 --- a/crates/sv-webserver/src/main.rs +++ b/crates/sv-webserver/src/main.rs @@ -1,7 +1,7 @@ use fuel_web_utils::server::api::{spawn_web_server, ApiServerBuilder}; use sv_webserver::{ config::Config, - server::{state::ServerState, svc::svc_handlers}, + server::{handlers, state::ServerState}, }; use tracing::level_filters::LevelFilter; use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; @@ -25,7 +25,7 @@ async fn main() -> anyhow::Result<()> { let config = Config::load()?; let server_state = ServerState::new(&config).await?; let server = ApiServerBuilder::new(config.api.port, server_state.clone()) - .with_dynamic_routes(svc_handlers(server_state)) + .with_dynamic_routes(handlers::create_services(server_state)) .build()?; let server_handle = server.handle(); let server_task = spawn_web_server(server).await; diff --git a/crates/sv-webserver/src/metrics.rs b/crates/sv-webserver/src/metrics.rs index 0b23c56b..d5c09403 100644 --- a/crates/sv-webserver/src/metrics.rs +++ b/crates/sv-webserver/src/metrics.rs @@ -1,13 +1,23 @@ +use std::time::Duration; + use async_trait::async_trait; use fuel_web_utils::telemetry::metrics::TelemetryMetrics; use prometheus::{ + register_histogram_vec, register_int_counter_vec, register_int_gauge_vec, + HistogramVec, IntCounterVec, IntGaugeVec, Registry, }; +#[derive(Debug)] +pub enum SubscriptionChange { + Added, + Removed, +} + #[derive(Clone, Debug)] pub struct Metrics { pub registry: Registry, @@ -15,6 +25,10 @@ pub struct Metrics { pub user_subscribed_messages: IntGaugeVec, pub subs_messages_throughput: IntCounterVec, pub subs_messages_error_rates: IntCounterVec, + pub connection_duration: HistogramVec, + pub duplicate_subscription_attempts: IntCounterVec, + pub user_active_subscriptions: IntGaugeVec, + pub subscription_lifetime: HistogramVec, } impl Default for Metrics { @@ -77,12 +91,46 @@ impl Metrics { ) .expect("metric must be created"); + let connection_duration = register_histogram_vec!( + format!("{}ws_connection_duration_seconds", metric_prefix), + "Duration of WebSocket connections in seconds", + &["user_id"], + vec![0.1, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0] + ) + .expect("metric must be created"); + + let duplicate_subscription_attempts = register_int_counter_vec!( + format!("{}ws_duplicate_subscription_attempts", metric_prefix), + "Number of attempts to create duplicate subscriptions", + &["user_id", "subscription_id"] + ) + .expect("metric must be created"); + + let user_active_subscriptions = register_int_gauge_vec!( + format!("{}ws_user_active_subscriptions", metric_prefix), + "Number of active subscriptions per user", + &["user_id"] + ) + .expect("metric must be created"); + + let subscription_lifetime = register_histogram_vec!( + format!("{}ws_subscription_lifetime_seconds", metric_prefix), + "Duration of individual subscriptions in seconds", + &["user_id", "subscription_id"], + vec![0.1, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0] + ) + .expect("metric must be created"); + let registry = Registry::new_custom(prefix, None).expect("registry to be created"); registry.register(Box::new(total_ws_subs.clone()))?; registry.register(Box::new(user_subscribed_messages.clone()))?; registry.register(Box::new(subs_messages_throughput.clone()))?; registry.register(Box::new(subs_messages_error_rates.clone()))?; + registry.register(Box::new(connection_duration.clone()))?; + registry.register(Box::new(duplicate_subscription_attempts.clone()))?; + registry.register(Box::new(user_active_subscriptions.clone()))?; + registry.register(Box::new(subscription_lifetime.clone()))?; Ok(Self { registry, @@ -90,6 +138,10 @@ impl Metrics { user_subscribed_messages, subs_messages_throughput, subs_messages_error_rates, + connection_duration, + duplicate_subscription_attempts, + user_active_subscriptions, + subscription_lifetime, }) } @@ -127,7 +179,7 @@ impl Metrics { } pub fn decrement_subscriptions_count(&self) { - self.total_ws_subs.with_label_values(&[]).inc(); + self.total_ws_subs.with_label_values(&[]).dec(); } pub fn update_unsubscribed( @@ -149,6 +201,55 @@ impl Metrics { .with_label_values(&[&user_id.to_string(), subject_wildcard]) .inc(); } + + pub fn track_connection_duration(&self, user_id: &str, duration: Duration) { + self.connection_duration + .with_label_values(&[user_id]) + .observe(duration.as_secs_f64()); + } + + pub fn track_duplicate_subscription( + &self, + user_id: uuid::Uuid, + subscription_id: &str, + ) { + self.duplicate_subscription_attempts + .with_label_values(&[&user_id.to_string(), subscription_id]) + .inc(); + } + + pub fn update_user_subscription_count( + &self, + user_id: uuid::Uuid, + subject: &str, + change: &SubscriptionChange, + ) { + let delta = match change { + SubscriptionChange::Added => 1, + SubscriptionChange::Removed => -1, + }; + + // Update per-user subscription count + self.user_active_subscriptions + .with_label_values(&[&user_id.to_string()]) + .add(delta); + + // Update subject-specific count + self.user_subscribed_messages + .with_label_values(&[&user_id.to_string(), subject]) + .add(delta); + } + + pub fn track_subscription_lifetime( + &self, + user_id: uuid::Uuid, + subscription_id: &str, + duration: Duration, + ) { + self.subscription_lifetime + .with_label_values(&[&user_id.to_string(), subscription_id]) + .observe(duration.as_secs_f64()); + } } #[cfg(test)] diff --git a/crates/sv-webserver/src/server/errors.rs b/crates/sv-webserver/src/server/errors.rs new file mode 100644 index 00000000..21f064d3 --- /dev/null +++ b/crates/sv-webserver/src/server/errors.rs @@ -0,0 +1,108 @@ +use actix_ws::{CloseCode, CloseReason, Closed, ProtocolError}; +use fuel_streams_core::stream::StreamError; +use fuel_streams_domains::SubjectPayloadError; +use fuel_streams_store::{ + db::DbError, + record::EncoderError, + store::StoreError, +}; + +/// Ws Subscription-related errors +#[derive(Debug, thiserror::Error)] +pub enum WebsocketError { + #[error("Stream error: {0}")] + StreamError(#[from] StreamError), + #[error("Unserializable payload: {0}")] + UnserializablePayload(#[from] serde_json::Error), + #[error("Connection closed with reason: {code} - {description}")] + ClosedWithReason { code: u16, description: String }, + #[error(transparent)] + Encoder(#[from] EncoderError), + #[error(transparent)] + Database(#[from] DbError), + #[error(transparent)] + Store(#[from] StoreError), + #[error(transparent)] + SubjectPayload(#[from] SubjectPayloadError), + #[error("Connection closed")] + Closed(#[from] Closed), + #[error("Unsupported message type")] + UnsupportedMessageType, + #[error(transparent)] + ProtocolError(#[from] ProtocolError), + #[error("Failed to send message")] + SendError, + #[error("Client timeout")] + Timeout, +} + +impl From for CloseReason { + fn from(error: WebsocketError) -> Self { + CloseReason { + code: match &error { + // Stream and data handling errors + WebsocketError::StreamError(_) => CloseCode::Error, + WebsocketError::UnserializablePayload(_) => { + CloseCode::Unsupported + } + WebsocketError::Encoder(_) => CloseCode::Invalid, + WebsocketError::SubjectPayload(_) => CloseCode::Invalid, + + // Connection state errors + WebsocketError::ClosedWithReason { code, .. } => { + CloseCode::Other(code.to_owned()) + } + WebsocketError::Closed(_) => CloseCode::Away, + + // Infrastructure errors + WebsocketError::Database(_) => CloseCode::Error, + WebsocketError::Store(_) => CloseCode::Error, + WebsocketError::UnsupportedMessageType => { + CloseCode::Unsupported + } + WebsocketError::ProtocolError(_) => CloseCode::Protocol, + WebsocketError::SendError => CloseCode::Error, + WebsocketError::Timeout => CloseCode::Away, + }, + description: Some(match &error { + WebsocketError::StreamError(e) => { + format!("Stream error: {}", e) + } + WebsocketError::UnserializablePayload(e) => { + format!("Failed to serialize payload: {}", e) + } + WebsocketError::ClosedWithReason { description, .. } => { + description.clone() + } + WebsocketError::Encoder(e) => format!("Encoding error: {}", e), + WebsocketError::Database(e) => format!("Database error: {}", e), + WebsocketError::Store(e) => format!("Store error: {}", e), + WebsocketError::SubjectPayload(e) => { + format!("Subject payload error: {}", e) + } + WebsocketError::Closed(_) => { + "Connection closed by peer".to_string() + } + WebsocketError::UnsupportedMessageType => { + "Unsupported message type".to_string() + } + WebsocketError::ProtocolError(_) => { + "Protocol error".to_string() + } + WebsocketError::SendError => { + "Failed to send message".to_string() + } + WebsocketError::Timeout => "Client timeout".to_string(), + }), + } + } +} + +impl From for WebsocketError { + fn from(reason: CloseReason) -> Self { + WebsocketError::ClosedWithReason { + code: reason.code.into(), + description: reason.description.unwrap_or_default(), + } + } +} diff --git a/crates/sv-webserver/src/server/http/handlers.rs b/crates/sv-webserver/src/server/handlers/http.rs similarity index 93% rename from crates/sv-webserver/src/server/http/handlers.rs rename to crates/sv-webserver/src/server/handlers/http.rs index 74b7084d..68b7a968 100644 --- a/crates/sv-webserver/src/server/http/handlers.rs +++ b/crates/sv-webserver/src/server/handlers/http.rs @@ -9,10 +9,9 @@ use fuel_web_utils::server::middlewares::auth::jwt::{ }; use uuid::Uuid; -use super::models::{LoginRequest, LoginResponse}; use crate::server::{ - // auth::{create_jwt, AuthError, UserError, UserRole}, state::ServerState, + types::{LoginRequest, LoginResponse}, }; pub static AUTH_DATA: LazyLock> = diff --git a/crates/sv-webserver/src/server/svc.rs b/crates/sv-webserver/src/server/handlers/mod.rs similarity index 67% rename from crates/sv-webserver/src/server/svc.rs rename to crates/sv-webserver/src/server/handlers/mod.rs index 0a139385..06e37d47 100644 --- a/crates/sv-webserver/src/server/svc.rs +++ b/crates/sv-webserver/src/server/handlers/mod.rs @@ -1,26 +1,30 @@ +pub mod http; +pub mod websocket; + use actix_web::web; use fuel_web_utils::server::{ api::with_prefixed_route, middlewares::auth::transform::JwtAuth, }; -use super::http::handlers::request_jwt; -use crate::server::{state::ServerState, ws::handlers::get_ws}; +use super::handlers; +use crate::server::state::ServerState; -pub fn svc_handlers( +pub fn create_services( state: ServerState, ) -> impl Fn(&mut web::ServiceConfig) + Send + Sync + 'static { move |cfg: &mut web::ServiceConfig| { + cfg.app_data(web::Data::new(state.clone())); cfg.service( web::resource(with_prefixed_route("jwt")) - .route(web::post().to(request_jwt)), + .route(web::post().to(handlers::http::request_jwt)), ); cfg.service( web::resource(with_prefixed_route("ws")) .wrap(JwtAuth::new(state.jwt_secret.clone())) .route(web::get().to({ move |req, body, state: web::Data| { - get_ws(req, body, state) + handlers::websocket::get_websocket(req, body, state) } })), ); diff --git a/crates/sv-webserver/src/server/handlers/websocket.rs b/crates/sv-webserver/src/server/handlers/websocket.rs new file mode 100644 index 00000000..2d9c55a2 --- /dev/null +++ b/crates/sv-webserver/src/server/handlers/websocket.rs @@ -0,0 +1,181 @@ +use std::{pin::pin, sync::Arc, time::Instant}; + +use actix_web::{ + web::{self, Bytes}, + HttpRequest, + Responder, +}; +use actix_ws::{CloseCode, CloseReason, Message, MessageStream, Session}; +use fuel_streams_core::FuelStreams; +use fuel_web_utils::telemetry::Telemetry; +use futures::{ + future::{self, Either}, + StreamExt as _, +}; +use uuid::Uuid; + +use crate::{ + metrics::Metrics, + server::{ + errors::WebsocketError, + state::ServerState, + types::ClientMessage, + websocket::{subscribe, unsubscribe, WsController}, + }, +}; + +#[derive(Debug)] +enum CloseAction { + Error(WebsocketError), + Closed(Option), + Unsubscribe, + Timeout, +} +impl From for CloseReason { + fn from(action: CloseAction) -> Self { + match action { + CloseAction::Closed(reason) => { + reason.unwrap_or(CloseCode::Normal.into()) + } + CloseAction::Error(_) => CloseCode::Away.into(), + CloseAction::Unsubscribe => CloseCode::Normal.into(), + CloseAction::Timeout => CloseCode::Away.into(), + } + } +} + +pub async fn get_websocket( + req: HttpRequest, + body: web::Payload, + state: web::Data, +) -> actix_web::Result { + let user_id = WsController::user_id_from_req(&req)?; + let (response, session, msg_stream) = actix_ws::handle(&req, body)?; + let fuel_streams = state.fuel_streams.clone(); + let telemetry = state.telemetry.clone(); + actix_web::rt::spawn(handler( + session, + msg_stream, + telemetry, + fuel_streams, + user_id, + )); + Ok(response) +} + +async fn handler( + mut session: actix_ws::Session, + msg_stream: actix_ws::MessageStream, + telemetry: Arc>, + fuel_streams: Arc, + user_id: Uuid, +) -> Result<(), WebsocketError> { + let mut ctx = WsController::new(user_id, telemetry, fuel_streams); + tracing::info!( + %user_id, + event = "websocket_connection_opened", + "WebSocket connection opened" + ); + + let action = handle_messages(&mut ctx, &mut session, msg_stream).await; + if let Some(action) = action { + if let CloseAction::Error(err) = &action { + ctx.send_error_msg(&mut session, err).await?; + } + ctx.close_session(session, action.into()).await; + } + Ok(()) +} + +async fn handle_messages( + ctx: &mut WsController, + session: &mut Session, + msg_stream: MessageStream, +) -> Option { + let mut last_heartbeat = Instant::now(); + let mut interval = tokio::time::interval(ctx.heartbeat_interval()); + let mut msg_stream = msg_stream.max_frame_size(ctx.max_frame_size()); + let mut msg_stream = pin!(msg_stream); + + loop { + let tick = pin!(interval.tick()); + match future::select(msg_stream.next(), tick).await { + Either::Left((Some(Ok(msg)), _)) => match msg { + Message::Text(msg) => { + let msg = Bytes::from(msg.as_bytes().to_vec()); + match handle_client_msg(session, ctx, msg).await { + Err(err) => break Some(CloseAction::Error(err)), + Ok(Some(close_action)) => break Some(close_action), + Ok(None) => {} + } + } + Message::Binary(msg) => { + match handle_client_msg(session, ctx, msg).await { + Err(err) => break Some(CloseAction::Error(err)), + Ok(Some(close_action)) => break Some(close_action), + Ok(None) => {} + } + } + Message::Ping(bytes) => { + last_heartbeat = Instant::now(); + if let Err(err) = session.pong(&bytes).await { + let err = WebsocketError::from(err); + break Some(CloseAction::Error(err)); + } + } + Message::Pong(_) => { + last_heartbeat = Instant::now(); + } + Message::Close(reason) => { + break Some(CloseAction::Closed(reason)); + } + Message::Continuation(_) => { + let user_id = ctx.user_id(); + tracing::warn!(%user_id, "Continuation frames not supported"); + let err = WebsocketError::UnsupportedMessageType; + break Some(CloseAction::Error(err)); + } + Message::Nop => {} + }, + Either::Left((Some(Err(err)), _)) => { + let user_id = ctx.user_id(); + tracing::error!(%user_id, error = %err, "WebSocket protocol error"); + break Some(CloseAction::Error(WebsocketError::from(err))); + } + Either::Left((None, _)) => { + let user_id = ctx.user_id(); + tracing::info!(%user_id, "Client disconnected"); + break None; + } + Either::Right((_inst, _)) => { + if let Err(err) = ctx.heartbeat(session, last_heartbeat).await { + match err { + WebsocketError::Timeout => { + break Some(CloseAction::Timeout) + } + _ => break Some(CloseAction::Error(err)), + } + } + } + } + } +} + +async fn handle_client_msg( + session: &mut Session, + ctx: &mut WsController, + msg: Bytes, +) -> Result, WebsocketError> { + tracing::info!("Received binary {:?}", msg); + let msg = serde_json::from_slice(&msg)?; + match msg { + ClientMessage::Subscribe(payload) => { + subscribe(session, ctx, payload).await?; + Ok(None) + } + ClientMessage::Unsubscribe(payload) => { + unsubscribe(session, ctx, payload).await?; + Ok(Some(CloseAction::Unsubscribe)) + } + } +} diff --git a/crates/sv-webserver/src/server/http/mod.rs b/crates/sv-webserver/src/server/http/mod.rs deleted file mode 100644 index 759a498a..00000000 --- a/crates/sv-webserver/src/server/http/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod handlers; -pub mod models; diff --git a/crates/sv-webserver/src/server/http/models.rs b/crates/sv-webserver/src/server/http/models.rs deleted file mode 100644 index 11d8ebb8..00000000 --- a/crates/sv-webserver/src/server/http/models.rs +++ /dev/null @@ -1,17 +0,0 @@ -use serde::{Deserialize, Serialize}; -use validator::Validate; - -#[derive(Debug, Serialize, Deserialize, Validate)] -#[serde(rename_all = "camelCase")] -pub struct LoginRequest { - pub username: String, - pub password: String, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct LoginResponse { - pub id: uuid::Uuid, - pub username: String, - pub jwt_token: String, -} diff --git a/crates/sv-webserver/src/server/mod.rs b/crates/sv-webserver/src/server/mod.rs index 7869e3b1..da4efcd7 100644 --- a/crates/sv-webserver/src/server/mod.rs +++ b/crates/sv-webserver/src/server/mod.rs @@ -1,4 +1,5 @@ -pub mod http; +pub mod errors; +pub mod handlers; pub mod state; -pub mod svc; -pub mod ws; +pub mod types; +pub mod websocket; diff --git a/crates/sv-webserver/src/server/state.rs b/crates/sv-webserver/src/server/state.rs index 911cc9fa..714e67a3 100644 --- a/crates/sv-webserver/src/server/state.rs +++ b/crates/sv-webserver/src/server/state.rs @@ -3,104 +3,47 @@ use std::{ time::{Duration, Instant}, }; -use async_nats::jetstream::stream::State; use async_trait::async_trait; -use fuel_streams_core::{nats::NatsClient, FuelStreams, FuelStreamsExt}; -use fuel_streams_nats::NatsClientOpts; -use fuel_streams_storage::{S3Storage, S3StorageOpts, Storage, StorageConfig}; +use fuel_message_broker::{MessageBroker, MessageBrokerClient}; +use fuel_streams_core::FuelStreams; +use fuel_streams_store::db::{Db, DbConnectionOpts}; use fuel_web_utils::{server::state::StateProvider, telemetry::Telemetry}; -use parking_lot::RwLock; -use serde::{Deserialize, Serialize}; use crate::{config::Config, metrics::Metrics}; -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct StreamInfo { - consumers: Vec, - state: StreamState, - stream_name: String, -} - -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] -pub struct StreamState { - /// The number of messages contained in this stream - pub messages: u64, - /// The number of bytes of all messages contained in this stream - pub bytes: u64, - /// The lowest sequence number still present in this stream - #[serde(rename = "first_seq")] - pub first_sequence: u64, - /// The time associated with the oldest message still present in this stream - #[serde(rename = "first_ts")] - pub first_timestamp: i64, - /// The last sequence number assigned to a message in this stream - #[serde(rename = "last_seq")] - pub last_sequence: u64, - /// The time that the last message was received by this stream - #[serde(rename = "last_ts")] - pub last_timestamp: i64, - /// The number of consumers configured to consume this stream - pub consumer_count: usize, -} - -impl From for StreamState { - fn from(state: State) -> Self { - StreamState { - messages: state.messages, - bytes: state.bytes, - first_sequence: state.first_sequence, - first_timestamp: state.first_timestamp.unix_timestamp(), - last_sequence: state.last_sequence, - last_timestamp: state.last_timestamp.unix_timestamp(), - consumer_count: state.consumer_count, - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct HealthResponse { - pub uptime_secs: u64, - pub is_healthy: bool, - pub streams_info: Vec, -} - #[derive(Clone)] pub struct ServerState { pub start_time: Instant, - pub nats_client: NatsClient, - pub telemetry: Arc>, + pub msg_broker: Arc, pub fuel_streams: Arc, - pub connection_count: Arc>, - pub storage: Option>, + pub telemetry: Arc>, + pub db: Arc, pub jwt_secret: String, } impl ServerState { pub async fn new(config: &Config) -> anyhow::Result { - let nats_client_opts = NatsClientOpts::admin_opts() - .with_url(config.nats.url.clone()) - .with_domain("CORE"); - let nats_client = NatsClient::connect(&nats_client_opts).await?; - let storage_opts = S3StorageOpts::admin_opts(); - let storage = Arc::new(S3Storage::new(storage_opts).await?); - let fuel_streams = - Arc::new(FuelStreams::new(&nats_client, &storage).await); + let msg_broker = + MessageBrokerClient::Nats.start(&config.broker.url).await?; + let db = Db::new(DbConnectionOpts { + connection_str: config.db.url.clone(), + ..Default::default() + }) + .await? + .arc(); + + let fuel_streams = FuelStreams::new(&msg_broker, &db).await.arc(); let metrics = Metrics::new_with_random_prefix()?; let telemetry = Telemetry::new(Some(metrics)).await?; telemetry.start().await?; Ok(Self { start_time: Instant::now(), + msg_broker, fuel_streams, - nats_client, telemetry, - storage: if config.s3.enabled { - Some(storage) - } else { - None - }, + db, jwt_secret: config.auth.jwt_secret.clone(), - connection_count: Arc::new(RwLock::new(0)), }) } @@ -112,29 +55,14 @@ impl ServerState { #[async_trait] impl StateProvider for ServerState { async fn is_healthy(&self) -> bool { - self.nats_client.is_connected() + self.msg_broker.is_healthy().await } async fn get_health(&self) -> serde_json::Value { - let streams_info = self - .fuel_streams - .get_consumers_and_state() + self.msg_broker + .get_health_info(self.uptime().as_secs()) .await - .unwrap_or_default() - .into_iter() - .map(|res| StreamInfo { - consumers: res.1, - state: res.2.into(), - stream_name: res.0, - }) - .collect::>(); - - let resp = HealthResponse { - uptime_secs: self.uptime().as_secs(), - is_healthy: self.is_healthy().await, - streams_info, - }; - serde_json::to_value(resp).unwrap_or(serde_json::json!({})) + .unwrap_or(serde_json::json!({})) } async fn get_metrics(&self) -> String { diff --git a/crates/sv-webserver/src/server/types.rs b/crates/sv-webserver/src/server/types.rs new file mode 100644 index 00000000..c715f7e4 --- /dev/null +++ b/crates/sv-webserver/src/server/types.rs @@ -0,0 +1,98 @@ +use fuel_streams_core::DeliverPolicy; +use fuel_streams_domains::{SubjectPayload, SubjectPayloadError}; +use serde::{Deserialize, Serialize}; +use validator::Validate; + +#[derive(Debug, Serialize, Deserialize, Validate)] +#[serde(rename_all = "camelCase")] +pub struct LoginRequest { + pub username: String, + pub password: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LoginResponse { + pub id: uuid::Uuid, + pub username: String, + pub jwt_token: String, +} + +#[derive(Eq, PartialEq, Debug, Deserialize, Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionPayload { + pub deliver_policy: DeliverPolicy, + pub subject: String, + pub params: serde_json::Value, +} +impl TryFrom for SubjectPayload { + type Error = SubjectPayloadError; + fn try_from(payload: SubscriptionPayload) -> Result { + SubjectPayload::new(payload.subject, payload.params) + } +} +impl std::fmt::Display for SubscriptionPayload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = serde_json::to_string(self).map_err(|_| std::fmt::Error)?; + write!(f, "{s}") + } +} +impl TryFrom for SubscriptionPayload { + type Error = serde_json::Error; + fn try_from(subscription_id: String) -> Result { + serde_json::from_str(&subscription_id) + } +} + +#[derive(Eq, PartialEq, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum ClientMessage { + Subscribe(SubscriptionPayload), + Unsubscribe(SubscriptionPayload), +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum ServerMessage { + Subscribed(SubscriptionPayload), + Unsubscribed(SubscriptionPayload), + Error(String), + Response(ResponseMessage), +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ResponseMessage { + pub subject: String, + pub payload: serde_json::Value, +} + +#[cfg(test)] +mod tests { + use super::{ClientMessage, DeliverPolicy, SubscriptionPayload}; + + #[test] + fn test_sub_ser() { + let stream_topic_wildcard = "blocks.*.*".to_owned(); + let msg = ClientMessage::Subscribe(SubscriptionPayload { + subject: stream_topic_wildcard.clone(), + params: serde_json::Value::Null, + deliver_policy: DeliverPolicy::New, + }); + let ser_str_value = serde_json::to_string(&msg).unwrap(); + println!("Ser value {:?}", ser_str_value); + let expected_value = serde_json::json!({ + "subscribe": { + "subject": stream_topic_wildcard, + "params": serde_json::Value::Null, + "deliverPolicy": "new" + } + }); + let deser_msg_val = + serde_json::from_value::(expected_value).unwrap(); + assert!(msg.eq(&deser_msg_val)); + + let deser_msg_str = + serde_json::from_str::(&ser_str_value).unwrap(); + assert!(msg.eq(&deser_msg_str)); + } +} diff --git a/crates/sv-webserver/src/server/websocket/controller.rs b/crates/sv-webserver/src/server/websocket/controller.rs new file mode 100644 index 00000000..3fe883c4 --- /dev/null +++ b/crates/sv-webserver/src/server/websocket/controller.rs @@ -0,0 +1,431 @@ +use std::{ + collections::HashSet, + sync::Arc, + time::{Duration, Instant}, +}; + +use actix_web::{HttpMessage, HttpRequest}; +use actix_ws::{CloseCode, CloseReason, Session}; +use fuel_streams_core::FuelStreams; +use fuel_web_utils::telemetry::Telemetry; +use tokio::sync::{broadcast, Mutex}; +use uuid::Uuid; + +use crate::{ + metrics::{Metrics, SubscriptionChange}, + server::{ + errors::WebsocketError, + types::{ServerMessage, SubscriptionPayload}, + }, +}; + +#[derive(Clone)] +struct AuthManager { + user_id: Uuid, +} + +impl AuthManager { + fn new(user_id: Uuid) -> Self { + Self { user_id } + } + + pub fn user_id(&self) -> &Uuid { + &self.user_id + } + + pub fn user_id_from_req( + req: &HttpRequest, + ) -> Result { + match req.extensions().get::() { + Some(user_id) => { + tracing::info!( + user_id = %user_id, + "Authenticated WebSocket connection" + ); + Ok(*user_id) + } + None => { + tracing::warn!("Unauthenticated WebSocket connection attempt"); + Err(actix_web::error::ErrorUnauthorized( + "Missing or invalid JWT", + )) + } + } + } +} + +#[derive(Clone)] +struct MessageHandler { + user_id: Uuid, +} + +impl MessageHandler { + fn new(user_id: Uuid) -> Self { + Self { user_id } + } + + async fn send_message( + &self, + session: &mut Session, + message: ServerMessage, + ) -> Result<(), WebsocketError> { + let msg_encoded = serde_json::to_vec(&message) + .map_err(WebsocketError::UnserializablePayload)?; + session.binary(msg_encoded).await?; + Ok(()) + } + + async fn send_error( + &self, + session: &mut Session, + error: &WebsocketError, + ) -> Result<(), WebsocketError> { + let error_msg = ServerMessage::Error(error.to_string()); + if let Err(send_err) = self.send_message(session, error_msg).await { + tracing::error!( + %self.user_id, + error = %send_err, + "Failed to send error message" + ); + return Err(WebsocketError::SendError); + } + Ok(()) + } +} + +#[derive(Clone)] +struct MetricsHandler { + telemetry: Arc>, + user_id: Uuid, +} + +impl MetricsHandler { + fn new(telemetry: Arc>, user_id: Uuid) -> Self { + Self { telemetry, user_id } + } + + fn track_subscription( + &self, + payload: &SubscriptionPayload, + change: SubscriptionChange, + ) { + if let Some(metrics) = self.telemetry.base_metrics() { + let subject = payload.subject.clone(); + metrics.update_user_subscription_count( + self.user_id, + &subject, + &change, + ); + match change { + SubscriptionChange::Added => { + metrics.increment_subscriptions_count() + } + SubscriptionChange::Removed => { + metrics.decrement_subscriptions_count() + } + } + } + } + + fn track_connection_duration(&self, duration: Duration) { + if let Some(metrics) = self.telemetry.base_metrics() { + metrics + .track_connection_duration(&self.user_id.to_string(), duration); + } + } + + fn track_duplicate_subscription(&self, payload: &SubscriptionPayload) { + if let Some(metrics) = self.telemetry.base_metrics() { + metrics.track_duplicate_subscription( + self.user_id, + &payload.to_string(), + ); + } + } +} + +// Connection management +#[derive(Clone)] +struct ConnectionManager { + user_id: Uuid, + start_time: Instant, + tx: broadcast::Sender, + active_subscriptions: Arc>>, + metrics_handler: MetricsHandler, +} + +impl ConnectionManager { + pub const CLIENT_TIMEOUT: Duration = Duration::from_secs(10); + pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); + pub const MAX_FRAME_SIZE: usize = 8 * 1024 * 1024; // 8MB + pub const CHANNEL_CAPACITY: usize = 100; + + fn new(user_id: Uuid, metrics_handler: MetricsHandler) -> Self { + let (tx, _) = broadcast::channel(Self::CHANNEL_CAPACITY); + Self { + user_id, + start_time: Instant::now(), + tx, + active_subscriptions: Arc::new(Mutex::new(HashSet::new())), + metrics_handler, + } + } + + fn subscribe(&self) -> broadcast::Receiver { + self.tx.subscribe() + } + + async fn is_subscribed(&self, subscription_id: &str) -> bool { + self.active_subscriptions + .lock() + .await + .contains(subscription_id) + } + + async fn add_subscription( + &self, + payload: &SubscriptionPayload, + ) -> Result<(), WebsocketError> { + let subscription_id = payload.to_string(); + self.active_subscriptions + .lock() + .await + .insert(subscription_id); + self.metrics_handler + .track_subscription(payload, SubscriptionChange::Added); + Ok(()) + } + + async fn remove_subscription(&self, payload: &SubscriptionPayload) { + self.shutdown(self.user_id).await; + let subscription_id = payload.to_string(); + if self + .active_subscriptions + .lock() + .await + .remove(&subscription_id) + { + self.metrics_handler + .track_subscription(payload, SubscriptionChange::Removed); + } + } + + async fn clear_subscriptions(&self) { + let subscriptions = self.active_subscriptions.lock().await; + for subscription_id in subscriptions.iter() { + let payload = + SubscriptionPayload::try_from(subscription_id.clone()); + if let Ok(payload) = payload { + self.remove_subscription(&payload).await; + self.metrics_handler + .track_subscription(&payload, SubscriptionChange::Removed); + } + } + } + + async fn shutdown(&self, user_id: Uuid) { + let _ = self.tx.send(user_id); + } + + fn connection_duration(&self) -> Duration { + self.start_time.elapsed() + } + + async fn check_duplicate_subscription( + &self, + session: &mut Session, + payload: &SubscriptionPayload, + message_handler: &MessageHandler, + ) -> Result { + let subscription_id = payload.to_string(); + if self.is_subscribed(&subscription_id).await { + self.metrics_handler.track_duplicate_subscription(payload); + let warning_msg = ServerMessage::Error(format!( + "Already subscribed to {}", + subscription_id + )); + message_handler.send_message(session, warning_msg).await?; + return Ok(true); + } + Ok(false) + } + + async fn heartbeat( + &self, + user_id: &Uuid, + session: &mut Session, + last_heartbeat: Instant, + ) -> Result<(), WebsocketError> { + let duration = Instant::now().duration_since(last_heartbeat); + if duration > Self::CLIENT_TIMEOUT { + tracing::warn!( + %user_id, + timeout = ?Self::CLIENT_TIMEOUT, + "Client timeout; disconnecting" + ); + return Err(WebsocketError::Timeout); + } + session.ping(b"").await.map_err(WebsocketError::from) + } +} + +#[derive(Clone)] +pub struct WsController { + auth: AuthManager, + messaging: MessageHandler, + connection: ConnectionManager, + pub streams: Arc, +} + +impl WsController { + pub fn new( + user_id: Uuid, + telemetry: Arc>, + streams: Arc, + ) -> Self { + let metrics = MetricsHandler::new(telemetry, user_id); + let connection = ConnectionManager::new(user_id, metrics); + Self { + auth: AuthManager::new(user_id), + messaging: MessageHandler::new(user_id), + connection, + streams, + } + } + + pub fn receiver(&self) -> broadcast::Receiver { + self.connection.subscribe() + } + + pub fn user_id(&self) -> &Uuid { + &self.auth.user_id() + } + + pub async fn send_message( + &self, + session: &mut Session, + message: ServerMessage, + ) -> Result<(), WebsocketError> { + self.messaging.send_message(session, message).await + } + + pub async fn send_error_msg( + &self, + session: &mut Session, + error: &WebsocketError, + ) -> Result<(), WebsocketError> { + self.messaging.send_error(session, error).await + } + + pub async fn is_subscribed(&self, subscription_id: &str) -> bool { + self.connection.is_subscribed(subscription_id).await + } + + pub async fn add_subscription( + &self, + payload: &SubscriptionPayload, + ) -> Result<(), WebsocketError> { + self.connection.add_subscription(payload).await + } + + pub async fn remove_subscription(&self, payload: &SubscriptionPayload) { + self.connection.remove_subscription(payload).await + } + + pub async fn check_duplicate_subscription( + &self, + session: &mut Session, + payload: &SubscriptionPayload, + ) -> Result { + self.connection + .check_duplicate_subscription(session, payload, &self.messaging) + .await + } + + pub async fn shutdown_subscription( + &self, + session: &mut Session, + payload: &SubscriptionPayload, + ) -> Result<(), WebsocketError> { + let user_id = self.auth.user_id(); + if !self.is_subscribed(&payload.to_string()).await { + let warning_msg = ServerMessage::Error(format!( + "No active subscription found for {}", + payload.to_string() + )); + self.send_message(session, warning_msg).await?; + return Ok(()); + } + self.connection.shutdown(user_id.to_owned()).await; + Ok(()) + } + + pub async fn close_session( + self, + session: Session, + close_reason: CloseReason, + ) { + let _ = session.close(Some(close_reason.clone())).await; + self.connection.clear_subscriptions().await; + + let duration = self.connection.connection_duration(); + self.connection + .metrics_handler + .track_connection_duration(duration); + + self.log_connection_close(duration, &close_reason); + } + + fn log_connection_close( + &self, + duration: Duration, + close_reason: &CloseReason, + ) { + let user_id = self.auth.user_id().to_string(); + let description = close_reason.description.as_deref(); + + if close_reason.code == CloseCode::Normal { + tracing::info!( + target: "websocket", + %user_id, + event = "websocket_connection_closed", + duration_secs = duration.as_secs_f64(), + close_reason = description, + "WebSocket connection closed" + ); + } else { + tracing::error!( + target: "websocket", + %user_id, + event = "websocket_connection_closed", + duration_secs = duration.as_secs_f64(), + close_reason = description, + "WebSocket connection closed" + ); + } + } + + pub fn user_id_from_req( + req: &HttpRequest, + ) -> Result { + AuthManager::user_id_from_req(req) + } + + pub async fn heartbeat( + &self, + session: &mut Session, + last_heartbeat: Instant, + ) -> Result<(), WebsocketError> { + self.connection + .heartbeat(self.user_id(), session, last_heartbeat) + .await + } + + pub fn heartbeat_interval(&self) -> Duration { + ConnectionManager::HEARTBEAT_INTERVAL + } + + pub fn max_frame_size(&self) -> usize { + ConnectionManager::MAX_FRAME_SIZE + } +} diff --git a/crates/sv-webserver/src/server/websocket/decoder.rs b/crates/sv-webserver/src/server/websocket/decoder.rs new file mode 100644 index 00000000..dda672a0 --- /dev/null +++ b/crates/sv-webserver/src/server/websocket/decoder.rs @@ -0,0 +1,51 @@ +use fuel_streams_core::types::*; +use fuel_streams_domains::SubjectPayload; +use fuel_streams_store::record::{DataEncoder, RecordEntity}; + +use crate::server::{ + errors::WebsocketError, + types::{ResponseMessage, ServerMessage, SubscriptionPayload}, +}; + +pub async fn decode_and_responde( + payload: SubscriptionPayload, + data: Vec, +) -> Result { + let subject = payload.subject.clone(); + let payload = decode_to_json_value(&payload.try_into()?, data).await?; + let response_message = ResponseMessage { subject, payload }; + Ok(ServerMessage::Response(response_message)) +} + +async fn decode_to_json_value( + payload: &SubjectPayload, + data: Vec, +) -> Result { + let value = match payload.record_entity() { + RecordEntity::Block => { + let payload: Block = Block::decode(&data).await?; + payload.to_json_value()? + } + RecordEntity::Transaction => { + let payload: Transaction = Transaction::decode(&data).await?; + payload.to_json_value()? + } + RecordEntity::Input => { + let payload: Input = Input::decode(&data).await?; + payload.to_json_value()? + } + RecordEntity::Output => { + let payload: Output = Output::decode(&data).await?; + payload.to_json_value()? + } + RecordEntity::Receipt => { + let payload: Receipt = Receipt::decode(&data).await?; + payload.to_json_value()? + } + RecordEntity::Utxo => { + let payload: Utxo = Utxo::decode(&data).await?; + payload.to_json_value()? + } + }; + Ok(value) +} diff --git a/crates/sv-webserver/src/server/websocket/mod.rs b/crates/sv-webserver/src/server/websocket/mod.rs new file mode 100644 index 00000000..b2962d10 --- /dev/null +++ b/crates/sv-webserver/src/server/websocket/mod.rs @@ -0,0 +1,6 @@ +mod controller; +pub(super) mod decoder; +mod subscribe; + +pub use controller::*; +pub use subscribe::*; diff --git a/crates/sv-webserver/src/server/websocket/subscribe.rs b/crates/sv-webserver/src/server/websocket/subscribe.rs new file mode 100644 index 00000000..052cd4a6 --- /dev/null +++ b/crates/sv-webserver/src/server/websocket/subscribe.rs @@ -0,0 +1,153 @@ +use std::sync::Arc; + +use actix_ws::{CloseReason, Session}; +use fuel_streams_core::{BoxedStream, DeliverPolicy, FuelStreams}; +use fuel_streams_domains::SubjectPayload; +use fuel_streams_store::record::RecordEntity; +use futures::StreamExt; + +use super::decoder::decode_and_responde; +use crate::server::{ + errors::WebsocketError, + types::{ServerMessage, SubscriptionPayload}, + websocket::WsController, +}; + +pub async fn unsubscribe( + session: &mut Session, + ctx: &mut WsController, + payload: SubscriptionPayload, +) -> Result<(), WebsocketError> { + ctx.remove_subscription(&payload).await; + let msg = ServerMessage::Unsubscribed(payload.clone()); + ctx.send_message(session, msg).await?; + Ok(()) +} + +pub async fn subscribe( + session: &mut Session, + ctx: &mut WsController, + payload: SubscriptionPayload, +) -> Result<(), WebsocketError> { + tracing::info!("Received subscribe message: {:?}", payload); + if ctx.check_duplicate_subscription(session, &payload).await? { + return Ok(()); + } + + // Subscribe to the subject + let subject_payload: SubjectPayload = payload.clone().try_into()?; + let sub = create_subscriber( + &ctx.streams, + &subject_payload, + payload.deliver_policy, + ) + .await?; + + // Send the subscription message to the client + let subscribed_msg = ServerMessage::Subscribed(payload.clone()); + ctx.send_message(session, subscribed_msg).await?; + ctx.add_subscription(&payload).await?; + + // Spawn a task to process messages + spawn_subscription_process(session, ctx, payload, sub); + Ok(()) +} + +fn spawn_subscription_process( + session: &mut Session, + ctx: &mut WsController, + payload: SubscriptionPayload, + mut sub: BoxedStream, +) -> tokio::task::JoinHandle<()> { + let user_id = *ctx.user_id(); + let mut shutdown_rx = ctx.receiver(); + actix_web::rt::spawn({ + let mut session = session.to_owned(); + let mut ctx = ctx.to_owned(); + async move { + let result: Result<(), WebsocketError> = tokio::select! { + shutdown = shutdown_rx.recv() => match shutdown { + Ok(shutdown_user_id) if shutdown_user_id == user_id => { + tracing::info!(%user_id, "Subscription gracefully shutdown"); + Ok(()) + } + _ => process_msgs(&mut session, &mut ctx, &mut sub, &payload).await + }, + process_result = process_msgs(&mut session, &mut ctx, &mut sub, &payload) => { + tracing::debug!(%user_id, "Process messages completed"); + process_result + } + }; + + if let Err(err) = result { + tracing::error!(%user_id, error = %err, "Subscription processing error"); + let _ = session.close(Some(CloseReason::from(err))).await; + } + } + }) +} + +async fn process_msgs( + session: &mut Session, + ctx: &mut WsController, + sub: &mut BoxedStream, + payload: &SubscriptionPayload, +) -> Result<(), WebsocketError> { + while let Some(result) = sub.next().await { + let result = result?; + let payload = decode_and_responde(payload.to_owned(), result).await?; + ctx.send_message(session, payload).await?; + } + + ctx.remove_subscription(&payload).await; + let msg = ServerMessage::Unsubscribed(payload.clone()); + ctx.send_message(session, msg).await?; + Ok(()) +} + +async fn create_subscriber( + streams: &Arc, + subject_payload: &SubjectPayload, + deliver_policy: DeliverPolicy, +) -> Result { + let subject = subject_payload.into_subject(); + let stream = match subject_payload.record_entity() { + RecordEntity::Block => { + streams + .blocks + .subscribe_dynamic(subject, deliver_policy) + .await + } + RecordEntity::Transaction => { + streams + .transactions + .subscribe_dynamic(subject, deliver_policy) + .await + } + RecordEntity::Input => { + streams + .inputs + .subscribe_dynamic(subject, deliver_policy) + .await + } + RecordEntity::Output => { + streams + .outputs + .subscribe_dynamic(subject, deliver_policy) + .await + } + RecordEntity::Receipt => { + streams + .receipts + .subscribe_dynamic(subject, deliver_policy) + .await + } + RecordEntity::Utxo => { + streams + .utxos + .subscribe_dynamic(subject, deliver_policy) + .await + } + }; + Ok(Box::new(stream)) +} diff --git a/crates/sv-webserver/src/server/ws/errors.rs b/crates/sv-webserver/src/server/ws/errors.rs deleted file mode 100644 index 05f00c4c..00000000 --- a/crates/sv-webserver/src/server/ws/errors.rs +++ /dev/null @@ -1,18 +0,0 @@ -use displaydoc::Display as DisplayDoc; -use fuel_streams_core::StreamError; -use thiserror::Error; - -/// Ws Subscription-related errors -#[derive(Debug, DisplayDoc, Error)] -pub enum WsSubscriptionError { - /// Unknown subject name: `{0}` - UnknownSubjectName(String), - /// Unsupported wildcard pattern: `{0}` - UnsupportedWildcardPattern(String), - /// Unserializable payload: `{0}` - UnserializablePayload(#[from] serde_json::Error), - /// Stream Error: `{0}` - Stream(#[from] StreamError), - /// Closed by client with reason: `{0}` - ClosedWithReason(String), -} diff --git a/crates/sv-webserver/src/server/ws/handlers.rs b/crates/sv-webserver/src/server/ws/handlers.rs deleted file mode 100644 index 85a06b74..00000000 --- a/crates/sv-webserver/src/server/ws/handlers.rs +++ /dev/null @@ -1,394 +0,0 @@ -use std::sync::{atomic::AtomicUsize, Arc}; - -use actix_web::{ - web::{self, Bytes}, - HttpMessage, - HttpRequest, - Responder, -}; -use actix_ws::{Message, Session}; -use fuel_streams_core::prelude::*; -use fuel_web_utils::telemetry::Telemetry; -use futures::StreamExt; -use uuid::Uuid; - -use super::{ - errors::WsSubscriptionError, - models::{ClientMessage, ResponseMessage}, -}; -use crate::{ - metrics::Metrics, - server::{ - state::ServerState, - ws::models::{ServerMessage, SubscriptionPayload}, - }, -}; - -static _NEXT_USER_ID: AtomicUsize = AtomicUsize::new(1); - -pub async fn get_ws( - req: HttpRequest, - body: web::Payload, - state: web::Data, -) -> actix_web::Result { - // extract user id - let user_id = match req.extensions().get::() { - Some(user_id) => { - tracing::info!( - "Authenticated WebSocket connection for user: {:?}", - user_id.to_string() - ); - user_id.to_owned() - } - None => { - tracing::info!("Unauthenticated WebSocket connection"); - return Err(actix_web::error::ErrorUnauthorized( - "Missing or invalid JWT", - )); - } - }; - - // split the request into response, session, and message stream - let (response, session, mut msg_stream) = actix_ws::handle(&req, body)?; - - // spawm an actor handling the ws connection - let streams = state.fuel_streams.clone(); - let telemetry = state.telemetry.clone(); - actix_web::rt::spawn(async move { - tracing::info!("Ws opened for user id {:?}", user_id.to_string()); - while let Some(Ok(msg)) = msg_stream.recv().await { - let mut session = session.clone(); - match msg { - Message::Ping(bytes) => { - tracing::info!("Received ping, {:?}", bytes); - if session.pong(&bytes).await.is_err() { - tracing::error!("Error sending pong, {:?}", bytes); - } - } - Message::Pong(bytes) => { - tracing::info!("Received pong, {:?}", bytes); - } - Message::Text(string) => { - tracing::info!("Received text, {string}"); - let bytes = Bytes::from(string.as_bytes().to_vec()); - let _ = handle_binary_message( - bytes, - user_id, - session, - Arc::clone(&telemetry), - Arc::clone(&streams), - ) - .await; - } - Message::Binary(bytes) => { - let _ = handle_binary_message( - bytes, - user_id, - session, - Arc::clone(&telemetry), - Arc::clone(&streams), - ) - .await; - } - Message::Close(reason) => { - tracing::info!( - "Got close event, terminating session with reason {:?}", - reason - ); - let reason_str = - reason.and_then(|r| r.description).unwrap_or_default(); - close_socket_with_error( - WsSubscriptionError::ClosedWithReason( - reason_str.to_string(), - ), - user_id, - session, - None, - telemetry, - ) - .await; - return; - } - _ => { - tracing::error!("Received unknown message type"); - close_socket_with_error( - WsSubscriptionError::ClosedWithReason( - "Unknown message type".to_string(), - ), - user_id, - session, - None, - telemetry, - ) - .await; - return; - } - }; - } - }); - - Ok(response) -} - -async fn handle_binary_message( - msg: Bytes, - user_id: uuid::Uuid, - mut session: Session, - telemetry: Arc>, - streams: Arc, -) -> Result<(), WsSubscriptionError> { - tracing::info!("Received binary {:?}", msg); - let client_message = match parse_client_message(msg) { - Ok(msg) => msg, - Err(e) => { - close_socket_with_error(e, user_id, session, None, telemetry).await; - return Ok(()); - } - }; - - tracing::info!("Message parsed: {:?}", client_message); - // handle the client message - match client_message { - ClientMessage::Subscribe(payload) => { - tracing::info!("Received subscribe message: {:?}", payload); - let subject_wildcard = payload.wildcard; - let deliver_policy = payload.deliver_policy; - - // verify the subject name - let sub_subject = - match verify_and_extract_subject_name(&subject_wildcard) { - Ok(res) => res, - Err(e) => { - close_socket_with_error( - e, - user_id, - session, - Some(subject_wildcard.clone()), - telemetry, - ) - .await; - return Ok(()); - } - }; - - // start the streamer async - let mut stream_session = session.clone(); - - // reply to socket with subscription - send_message_to_socket( - &mut session, - ServerMessage::Subscribed(SubscriptionPayload { - wildcard: subject_wildcard.clone(), - deliver_policy, - }), - ) - .await; - - // receive streaming in a background thread - let streams = streams.clone(); - let telemetry = telemetry.clone(); - actix_web::rt::spawn(async move { - // update metrics - if let Some(metrics) = telemetry.base_metrics() { - metrics.update_user_subscription_metrics( - user_id, - &subject_wildcard, - ); - } - - // subscribe to the stream - let config = SubscriptionConfig { - deliver_policy: deliver_policy.into(), - filter_subjects: vec![subject_wildcard.clone()], - }; - - let mut sub = match streams - .subscribe_raw(&sub_subject, Some(config)) - .await - { - Ok(sub) => sub, - Err(e) => { - close_socket_with_error( - WsSubscriptionError::Stream(e), - user_id, - session, - Some(subject_wildcard.clone()), - telemetry, - ) - .await; - return; - } - }; - - // consume and forward to the ws - while let Some((s3_serialized_payload, s3_path, message)) = - sub.next().await - { - // decode and serialize back to ws payload - let serialized_ws_payload = match decode( - &subject_wildcard, - s3_serialized_payload, - s3_path, - ) - .await - { - Ok(res) => res, - Err(e) => { - if let Some(metrics) = telemetry.base_metrics() { - metrics.update_error_metrics( - &subject_wildcard, - &e.to_string(), - ); - } - tracing::error!("Error serializing received stream message: {:?}", e); - continue; - } - }; - - // send the payload over the stream - let _ = stream_session.binary(serialized_ws_payload).await; - let _ = message.ack().await; - } - }); - Ok(()) - } - ClientMessage::Unsubscribe(payload) => { - tracing::info!("Received unsubscribe message: {:?}", payload); - let subject_wildcard = payload.wildcard; - let deliver_policy = payload.deliver_policy; - if let Err(e) = verify_and_extract_subject_name(&subject_wildcard) { - close_socket_with_error( - e, - user_id, - session, - Some(subject_wildcard.clone()), - telemetry, - ) - .await; - return Ok(()); - } - - // TODO: implement session management for the same user_id - // send a message to the client to confirm unsubscribing - send_message_to_socket( - &mut session, - ServerMessage::Unsubscribed(SubscriptionPayload { - wildcard: subject_wildcard, - deliver_policy, - }), - ) - .await; - Ok(()) - } - } -} - -fn parse_client_message( - msg: Bytes, -) -> Result { - let msg = serde_json::from_slice::(&msg) - .map_err(WsSubscriptionError::UnserializablePayload)?; - Ok(msg) -} - -pub fn verify_and_extract_subject_name( - subject_wildcard: &str, -) -> Result { - let mut subject_parts = subject_wildcard.split('.'); - // TODO: more advanced checks here with Regex - if subject_parts.clone().count() == 1 { - return Err(WsSubscriptionError::UnsupportedWildcardPattern( - subject_wildcard.to_string(), - )); - } - let subject_name = subject_parts.next().unwrap_or_default(); - if !FuelStreamsUtils::is_within_subject_names(subject_name) { - return Err(WsSubscriptionError::UnknownSubjectName( - subject_wildcard.to_string(), - )); - } - Ok(subject_name.to_string()) -} - -async fn close_socket_with_error( - e: WsSubscriptionError, - user_id: uuid::Uuid, - mut session: Session, - subject_wildcard: Option, - telemetry: Arc>, -) { - tracing::error!("ws subscription error: {:?}", e.to_string()); - if let Some(subject_wildcard) = subject_wildcard { - if let Some(metrics) = telemetry.base_metrics() { - metrics.update_error_metrics(&subject_wildcard, &e.to_string()); - metrics.update_unsubscribed(user_id, &subject_wildcard); - } - } - - if let Some(metrics) = telemetry.base_metrics() { - metrics.decrement_subscriptions_count(); - } - send_message_to_socket(&mut session, ServerMessage::Error(e.to_string())) - .await; - let _ = session.close(None).await; -} - -async fn send_message_to_socket(session: &mut Session, message: ServerMessage) { - let data = serde_json::to_vec(&message).ok().unwrap_or_default(); - let _ = session.binary(data).await; -} - -async fn decode( - subject_wildcard: &str, - s3_payload: Vec, - s3_path: String, -) -> Result, WsSubscriptionError> { - let subject = verify_and_extract_subject_name(subject_wildcard)?; - let payload = match subject.as_str() { - Block::NAME => { - let entity = Block::decode(&s3_payload).await?; - Ok(entity.encode_json_value()?) - } - Transaction::NAME => { - let entity = Transaction::decode(&s3_payload).await?; - Ok(entity.encode_json_value()?) - } - Input::NAME => { - let entity = Input::decode(&s3_payload).await?; - Ok(entity.encode_json_value()?) - } - Output::NAME => { - let entity = Output::decode(&s3_payload).await?; - Ok(entity.encode_json_value()?) - } - Receipt::NAME => { - let entity = Receipt::decode(&s3_payload).await?; - Ok(entity.encode_json_value()?) - } - Utxo::NAME => { - let entity = Utxo::decode(&s3_payload).await?; - Ok(entity.encode_json_value()?) - } - Log::NAME => { - let entity = Log::decode(&s3_payload).await?; - Ok(entity.encode_json_value()?) - } - _ => Err(WsSubscriptionError::UnknownSubjectName( - subject_wildcard.to_string(), - )), - }; - - let subject = s3_path_to_subject(s3_path); - serde_json::to_vec(&ServerMessage::Response(ResponseMessage { - subject, - payload: payload?, - })) - .map_err(WsSubscriptionError::UnserializablePayload) -} - -fn s3_path_to_subject(s3_path: String) -> String { - s3_path - .replace("/", ".") - .replace(".json.zstd", "") - .replace(".json", "") -} diff --git a/crates/sv-webserver/src/server/ws/mod.rs b/crates/sv-webserver/src/server/ws/mod.rs deleted file mode 100644 index 4fef2941..00000000 --- a/crates/sv-webserver/src/server/ws/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod errors; -pub mod handlers; -pub mod models; diff --git a/crates/sv-webserver/src/server/ws/models.rs b/crates/sv-webserver/src/server/ws/models.rs deleted file mode 100644 index 4cb57a7d..00000000 --- a/crates/sv-webserver/src/server/ws/models.rs +++ /dev/null @@ -1,112 +0,0 @@ -use fuel_streams_nats::NatsDeliverPolicy; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] -#[repr(u8)] -#[serde(rename_all = "camelCase")] -pub enum DeliverPolicy { - All, - Last, - New, - ByStartSequence { - #[serde(rename = "optStartSeq")] - start_sequence: u64, - }, - ByStartTime { - #[serde(rename = "optStartTime")] - start_time: time::OffsetDateTime, - }, - LastPerSubject, -} - -impl From for NatsDeliverPolicy { - fn from(policy: DeliverPolicy) -> Self { - match policy { - DeliverPolicy::All => NatsDeliverPolicy::All, - DeliverPolicy::Last => NatsDeliverPolicy::Last, - DeliverPolicy::New => NatsDeliverPolicy::New, - DeliverPolicy::ByStartSequence { start_sequence } => { - NatsDeliverPolicy::ByStartSequence { start_sequence } - } - DeliverPolicy::ByStartTime { start_time } => { - NatsDeliverPolicy::ByStartTime { start_time } - } - DeliverPolicy::LastPerSubject => NatsDeliverPolicy::LastPerSubject, - } - } -} - -impl From for DeliverPolicy { - fn from(policy: NatsDeliverPolicy) -> Self { - match policy { - NatsDeliverPolicy::All => DeliverPolicy::All, - NatsDeliverPolicy::Last => DeliverPolicy::Last, - NatsDeliverPolicy::New => DeliverPolicy::New, - NatsDeliverPolicy::ByStartSequence { start_sequence } => { - DeliverPolicy::ByStartSequence { start_sequence } - } - NatsDeliverPolicy::ByStartTime { start_time } => { - DeliverPolicy::ByStartTime { start_time } - } - NatsDeliverPolicy::LastPerSubject => DeliverPolicy::LastPerSubject, - } - } -} - -#[derive(Eq, PartialEq, Debug, Deserialize, Serialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct SubscriptionPayload { - pub wildcard: String, - pub deliver_policy: DeliverPolicy, -} - -#[derive(Eq, PartialEq, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum ClientMessage { - Subscribe(SubscriptionPayload), - Unsubscribe(SubscriptionPayload), -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum ServerMessage { - Subscribed(SubscriptionPayload), - Unsubscribed(SubscriptionPayload), - Response(ResponseMessage), - Error(String), -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct ResponseMessage { - pub subject: String, - pub payload: serde_json::Value, -} - -#[cfg(test)] -mod tests { - use super::{ClientMessage, DeliverPolicy, SubscriptionPayload}; - - #[test] - fn test_sub_ser() { - let stream_topic_wildcard = "blocks.*.*".to_owned(); - let msg = ClientMessage::Subscribe(SubscriptionPayload { - wildcard: stream_topic_wildcard.clone(), - deliver_policy: DeliverPolicy::All, - }); - let ser_str_value = serde_json::to_string(&msg).unwrap(); - println!("Ser value {:?}", ser_str_value); - let expected_value = serde_json::json!({ - "subscribe": { - "wildcard": stream_topic_wildcard, - "deliverPolicy": "all" - } - }); - let deser_msg_val = - serde_json::from_value::(expected_value).unwrap(); - assert!(msg.eq(&deser_msg_val)); - - let deser_msg_str = - serde_json::from_str::(&ser_str_value).unwrap(); - assert!(msg.eq(&deser_msg_str)); - } -} diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 0d57c4ea..c5658549 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -39,7 +39,3 @@ path = "outputs.rs" [[example]] name = "utxos" path = "utxos.rs" - -[[example]] -name = "logs" -path = "logs.rs" diff --git a/examples/inputs.rs b/examples/inputs.rs index 05841026..6c8860f8 100644 --- a/examples/inputs.rs +++ b/examples/inputs.rs @@ -15,7 +15,7 @@ async fn main() -> anyhow::Result<()> { let subject = InputsCoinSubject::new(); // Subscribe to the input stream with the specified configuration let mut stream = connection - .subscribe::(subject, DeliverPolicy::Last) + .subscribe::(subject, DeliverPolicy::New) .await?; // Process incoming inputs diff --git a/examples/logs.rs b/examples/logs.rs deleted file mode 100644 index 116545a9..00000000 --- a/examples/logs.rs +++ /dev/null @@ -1,27 +0,0 @@ -use fuel_streams::prelude::*; -use futures::StreamExt; - -// This example demonstrates how to use the fuel-streams library to stream -// logs from a Fuel network. It connects to a streaming service, -// subscribes to a log stream, and prints incoming logs. -#[tokio::main] -async fn main() -> anyhow::Result<()> { - // Initialize a client connection to the Fuel streaming service - let mut client = Client::new(FuelNetwork::Staging).await?; - let mut connection = client.connect().await?; - - println!("Listening for logs..."); - - let subject = LogsSubject::new(); - // Subscribe to the log stream with the specified configuration - let mut stream = connection - .subscribe::(subject, DeliverPolicy::Last) - .await?; - - // Process incoming logs - while let Some(msg) = stream.next().await { - println!("Received log: {:?}", msg.payload); - } - - Ok(()) -} diff --git a/examples/outputs.rs b/examples/outputs.rs index a7b81c7d..7bd56cca 100644 --- a/examples/outputs.rs +++ b/examples/outputs.rs @@ -15,7 +15,7 @@ async fn main() -> anyhow::Result<()> { let subject = OutputsCoinSubject::new(); // Subscribe to the output stream with the specified configuration let mut stream = connection - .subscribe::(subject, DeliverPolicy::Last) + .subscribe::(subject, DeliverPolicy::New) .await?; // Process incoming outputs diff --git a/examples/receipts.rs b/examples/receipts.rs index 6d81528a..cf210b87 100644 --- a/examples/receipts.rs +++ b/examples/receipts.rs @@ -1,8 +1,7 @@ use fuel_streams::prelude::*; use futures::StreamExt; -/// The contract ID to stream the receipts for. For this example, we're using the contract ID of the https://thundernft.market/ -const CONTRACT_ID: &str = +const TX_ID: &str = "0x243ef4c2301f44eecbeaf1c39fee9379664b59a2e5b75317e8c7e7f26a25ed4d"; #[tokio::main] @@ -14,13 +13,11 @@ async fn main() -> anyhow::Result<()> { println!("Listening for receipts..."); // Create a subject for all receipt types related to the contract - let subject = ReceiptsByIdSubject::new() - .with_id_kind(Some(IdentifierKind::ContractID)) - .with_id_value(Some(CONTRACT_ID.into())); + let subject = ReceiptsReturnSubject::new().with_tx_id(Some(TX_ID.into())); // Subscribe to the receipt stream with the specified configuration let mut stream = connection - .subscribe::(subject, DeliverPolicy::All) + .subscribe::(subject, DeliverPolicy::New) .await?; // Process incoming receipts diff --git a/examples/transactions.rs b/examples/transactions.rs index fb234fe3..3ee4470d 100644 --- a/examples/transactions.rs +++ b/examples/transactions.rs @@ -18,7 +18,7 @@ async fn main() -> anyhow::Result<()> { // Subscribe to the transaction stream with the specified configuration let mut stream = connection - .subscribe::(subject, DeliverPolicy::Last) + .subscribe::(subject, DeliverPolicy::New) .await?; // Process incoming transactions diff --git a/examples/utxos.rs b/examples/utxos.rs index 86cce293..77cebd6f 100644 --- a/examples/utxos.rs +++ b/examples/utxos.rs @@ -17,7 +17,7 @@ async fn main() -> anyhow::Result<()> { // Subscribe to the UTXO stream with the specified configuration let mut stream = connection - .subscribe::(subject, DeliverPolicy::Last) + .subscribe::(subject, DeliverPolicy::New) .await?; // Process incoming UTXOs diff --git a/package.json b/package.json index a31d2e32..6cbfb26e 100644 --- a/package.json +++ b/package.json @@ -6,13 +6,13 @@ "scripts": { "prettier:fix": "prettier --write \"**/*.@(json|md|sh|toml)\"", "prettier:validate": "prettier --check \"**/*.@(json|md|sh|toml)\"", - "md:lint": "markdownlint -c .markdownlint.json \"**/*.md\" \".github/**/*.md\" -i \"target\" -i \"node_modules\" -i \"CHANGELOG.md\"", + "md:lint": "markdownlint -c .markdownlint.json **/*.md -d -i target -i node_modules -i CHANGELOG.md", "md:fix": "pnpm md:lint --fix" }, "devDependencies": { "@commitlint/config-conventional": "^19.6.0", "commitlint": "^19.6.0", - "markdownlint": "^0.36.1", + "markdownlint": "0.37.3", "markdownlint-cli": "^0.43.0", "prettier": "^3.4.1", "prettier-plugin-sh": "^0.14.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 42d3417f..96706ff4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,8 +15,8 @@ importers: specifier: ^19.6.0 version: 19.6.0(@types/node@22.5.0)(typescript@5.5.4) markdownlint: - specifier: ^0.36.1 - version: 0.36.1 + specifier: 0.37.3 + version: 0.37.3 markdownlint-cli: specifier: ^0.43.0 version: 0.43.0 @@ -126,9 +126,21 @@ packages: '@types/conventional-commits-parser@5.0.0': resolution: {integrity: sha512-loB369iXNmAZglwWATL+WRe+CRMmmBPtpolYzIebFaX4YA3x+BEfLqhUAV9WanycKI3TG1IMr5bMJDajDKLlUQ==} + '@types/debug@4.1.12': + resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} + + '@types/katex@0.16.7': + resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} + + '@types/ms@0.7.34': + resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} + '@types/node@22.5.0': resolution: {integrity: sha512-DkFrJOe+rfdHTqqMg0bSNlGlQ85hSoh2TPzZyhHsXnMtligRWpxUySiyw8FY14ITt24HVCiQPWxS3KO/QlGmWg==} + '@types/unist@2.0.11': + resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + JSONStream@1.3.5: resolution: {integrity: sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==} hasBin: true @@ -180,6 +192,15 @@ packages: resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + character-entities-legacy@3.0.0: + resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} + + character-entities@2.0.2: + resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} + + character-reference-invalid@2.0.1: + resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==} + cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -201,6 +222,10 @@ packages: resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} engines: {node: '>=18'} + commander@8.3.0: + resolution: {integrity: sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==} + engines: {node: '>= 12'} + commitlint@19.6.0: resolution: {integrity: sha512-0gOMRBSpnCw3Su0rfVeDqCe4ck/fkhGGC9UxVDeSyyCemFXs4U3BDuwMWvYcw4qsEAkPuDjQNoU8KWyPtHBq/w==} engines: {node: '>=v18'} @@ -247,10 +272,29 @@ packages: resolution: {integrity: sha512-wAV9QHOsNbwnWdNW2FYvE1P56wtgSbM+3SZcdGiWQILwVjACCXDCI3Ai8QlCjMDB8YK5zySiXZYBiwGmNY3lnw==} engines: {node: '>=12'} + debug@4.4.0: + resolution: {integrity: sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + decode-named-character-reference@1.0.2: + resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==} + deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + devlop@1.1.0: + resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + dot-prop@5.3.0: resolution: {integrity: sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==} engines: {node: '>=8'} @@ -334,13 +378,25 @@ packages: resolution: {integrity: sha512-QQnnxNyfvmHFIsj7gkPcYymR8Jdw/o7mp5ZFihxn6h8Ci6fh3Dx4E1gPjpQEpIuPo9XVNY/ZUwh4BPMjGyL01g==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + is-alphabetical@2.0.1: + resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} + + is-alphanumerical@2.0.1: + resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + is-decimal@2.0.1: + resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} + is-fullwidth-code-point@3.0.0: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} + is-hexadecimal@2.0.1: + resolution: {integrity: sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==} + is-obj@2.0.0: resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==} engines: {node: '>=8'} @@ -384,6 +440,10 @@ packages: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} engines: {node: '>=0.10.0'} + katex@0.16.19: + resolution: {integrity: sha512-3IA6DYVhxhBabjSLTNO9S4+OliA3Qvb8pBQXMfC4WxXJgLwZgnfDl0BmB4z6nBMdznBsZ+CGM8DrGZ5hcguDZg==} + hasBin: true + lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} @@ -442,6 +502,10 @@ packages: resolution: {integrity: sha512-s73fU2CQN7WCgjhaQUQ8wYESQNzGRNOKDd+3xgVqu8kuTEhmwepd/mxOv1LR2oV046ONrTLBFsM7IoKWNvmy5g==} engines: {node: '>=18'} + markdownlint@0.37.3: + resolution: {integrity: sha512-eoQqH0291YCCjd+Pe1PUQ9AmWthlVmS0XWgcionkZ8q34ceZyRI+pYvsWksXJJL8OBkWCPwp1h/pnXxrPFC4oA==} + engines: {node: '>=18'} + mdurl@2.0.0: resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} @@ -449,6 +513,81 @@ packages: resolution: {integrity: sha512-BhXM0Au22RwUneMPwSCnyhTOizdWoIEPU9sp0Aqa1PnDMR5Wv2FGXYDjuzJEIX+Eo2Rb8xuYe5jrnm5QowQFkw==} engines: {node: '>=16.10'} + micromark-core-commonmark@2.0.2: + resolution: {integrity: sha512-FKjQKbxd1cibWMM1P9N+H8TwlgGgSkWZMmfuVucLCHaYqeSvJ0hFeHsIa65pA2nYbes0f8LDHPMrd9X7Ujxg9w==} + + micromark-extension-directive@3.0.2: + resolution: {integrity: sha512-wjcXHgk+PPdmvR58Le9d7zQYWy+vKEU9Se44p2CrCDPiLr2FMyiT4Fyb5UFKFC66wGB3kPlgD7q3TnoqPS7SZA==} + + micromark-extension-gfm-autolink-literal@2.1.0: + resolution: {integrity: sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==} + + micromark-extension-gfm-footnote@2.1.0: + resolution: {integrity: sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==} + + micromark-extension-gfm-table@2.1.0: + resolution: {integrity: sha512-Ub2ncQv+fwD70/l4ou27b4YzfNaCJOvyX4HxXU15m7mpYY+rjuWzsLIPZHJL253Z643RpbcP1oeIJlQ/SKW67g==} + + micromark-extension-math@3.1.0: + resolution: {integrity: sha512-lvEqd+fHjATVs+2v/8kg9i5Q0AP2k85H0WUOwpIVvUML8BapsMvh1XAogmQjOCsLpoKRCVQqEkQBB3NhVBcsOg==} + + micromark-factory-destination@2.0.1: + resolution: {integrity: sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==} + + micromark-factory-label@2.0.1: + resolution: {integrity: sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==} + + micromark-factory-space@2.0.1: + resolution: {integrity: sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==} + + micromark-factory-title@2.0.1: + resolution: {integrity: sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==} + + micromark-factory-whitespace@2.0.1: + resolution: {integrity: sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==} + + micromark-util-character@2.1.1: + resolution: {integrity: sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==} + + micromark-util-chunked@2.0.1: + resolution: {integrity: sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==} + + micromark-util-classify-character@2.0.1: + resolution: {integrity: sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==} + + micromark-util-combine-extensions@2.0.1: + resolution: {integrity: sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==} + + micromark-util-decode-numeric-character-reference@2.0.2: + resolution: {integrity: sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==} + + micromark-util-encode@2.0.1: + resolution: {integrity: sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==} + + micromark-util-html-tag-name@2.0.1: + resolution: {integrity: sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==} + + micromark-util-normalize-identifier@2.0.1: + resolution: {integrity: sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==} + + micromark-util-resolve-all@2.0.1: + resolution: {integrity: sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==} + + micromark-util-sanitize-uri@2.0.1: + resolution: {integrity: sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==} + + micromark-util-subtokenize@2.0.3: + resolution: {integrity: sha512-VXJJuNxYWSoYL6AJ6OQECCFGhIU2GGHMw8tahogePBrjkG8aCCas3ibkp7RnVOSTClg2is05/R7maAhF1XyQMg==} + + micromark-util-symbol@2.0.1: + resolution: {integrity: sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==} + + micromark-util-types@2.0.1: + resolution: {integrity: sha512-534m2WhVTddrcKVepwmVEVnUAmtrx9bfIjNoQHRqfnvdaHQiFytEhJoTgpWJvDEXCO5gLTQh3wYC1PgOJA4NSQ==} + + micromark@4.0.1: + resolution: {integrity: sha512-eBPdkcoCNvYcxQOAKAlceo5SNdzZWfF+FcSupREAzdAh9rRmE239CEQAiTwIgblwnoM8zzj35sZ5ZwvSEOF6Kw==} + minimatch@10.0.1: resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==} engines: {node: 20 || >=22} @@ -460,6 +599,9 @@ packages: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + mvdan-sh@0.10.1: resolution: {integrity: sha512-kMbrH0EObaKmK3nVRKUIIya1dpASHIEusM13S4V1ViHFuxuNxCo+arxoa6j/dbV22YBGjl7UKJm9QQKJ2Crzhg==} @@ -478,6 +620,9 @@ packages: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} + parse-entities@4.0.2: + resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + parse-json@5.2.0: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} @@ -793,10 +938,20 @@ snapshots: dependencies: '@types/node': 22.5.0 + '@types/debug@4.1.12': + dependencies: + '@types/ms': 0.7.34 + + '@types/katex@0.16.7': {} + + '@types/ms@0.7.34': {} + '@types/node@22.5.0': dependencies: undici-types: 6.19.8 + '@types/unist@2.0.11': {} + JSONStream@1.3.5: dependencies: jsonparse: 1.3.1 @@ -843,6 +998,12 @@ snapshots: chalk@5.3.0: {} + character-entities-legacy@3.0.0: {} + + character-entities@2.0.2: {} + + character-reference-invalid@2.0.1: {} + cliui@8.0.1: dependencies: string-width: 4.2.3 @@ -863,6 +1024,8 @@ snapshots: commander@12.1.0: {} + commander@8.3.0: {} + commitlint@19.6.0(@types/node@22.5.0)(typescript@5.5.4): dependencies: '@commitlint/cli': 19.6.0(@types/node@22.5.0)(typescript@5.5.4) @@ -915,8 +1078,22 @@ snapshots: dargs@8.1.0: {} + debug@4.4.0: + dependencies: + ms: 2.1.3 + + decode-named-character-reference@1.0.2: + dependencies: + character-entities: 2.0.2 + deep-extend@0.6.0: {} + dequal@2.0.3: {} + + devlop@1.1.0: + dependencies: + dequal: 2.0.3 + dot-prop@5.3.0: dependencies: is-obj: 2.0.0 @@ -988,10 +1165,21 @@ snapshots: ini@4.1.1: {} + is-alphabetical@2.0.1: {} + + is-alphanumerical@2.0.1: + dependencies: + is-alphabetical: 2.0.1 + is-decimal: 2.0.1 + is-arrayish@0.2.1: {} + is-decimal@2.0.1: {} + is-fullwidth-code-point@3.0.0: {} + is-hexadecimal@2.0.1: {} + is-obj@2.0.0: {} is-text-path@2.0.0: @@ -1022,6 +1210,10 @@ snapshots: jsonpointer@5.0.1: {} + katex@0.16.19: + dependencies: + commander: 8.3.0 + lines-and-columns@1.2.4: {} linkify-it@5.0.0: @@ -1081,10 +1273,196 @@ snapshots: markdown-it: 14.1.0 markdownlint-micromark: 0.1.12 + markdownlint@0.37.3: + dependencies: + markdown-it: 14.1.0 + micromark: 4.0.1 + micromark-core-commonmark: 2.0.2 + micromark-extension-directive: 3.0.2 + micromark-extension-gfm-autolink-literal: 2.1.0 + micromark-extension-gfm-footnote: 2.1.0 + micromark-extension-gfm-table: 2.1.0 + micromark-extension-math: 3.1.0 + micromark-util-types: 2.0.1 + transitivePeerDependencies: + - supports-color + mdurl@2.0.0: {} meow@12.1.1: {} + micromark-core-commonmark@2.0.2: + dependencies: + decode-named-character-reference: 1.0.2 + devlop: 1.1.0 + micromark-factory-destination: 2.0.1 + micromark-factory-label: 2.0.1 + micromark-factory-space: 2.0.1 + micromark-factory-title: 2.0.1 + micromark-factory-whitespace: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-chunked: 2.0.1 + micromark-util-classify-character: 2.0.1 + micromark-util-html-tag-name: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-subtokenize: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-extension-directive@3.0.2: + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-factory-whitespace: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + parse-entities: 4.0.2 + + micromark-extension-gfm-autolink-literal@2.1.0: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-extension-gfm-footnote@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-core-commonmark: 2.0.2 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-extension-gfm-table@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-extension-math@3.1.0: + dependencies: + '@types/katex': 0.16.7 + devlop: 1.1.0 + katex: 0.16.19 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-factory-destination@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-factory-label@2.0.1: + dependencies: + devlop: 1.1.0 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-factory-space@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-types: 2.0.1 + + micromark-factory-title@2.0.1: + dependencies: + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-factory-whitespace@2.0.1: + dependencies: + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-util-character@2.1.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-util-chunked@2.0.1: + dependencies: + micromark-util-symbol: 2.0.1 + + micromark-util-classify-character@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-util-combine-extensions@2.0.1: + dependencies: + micromark-util-chunked: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-util-decode-numeric-character-reference@2.0.2: + dependencies: + micromark-util-symbol: 2.0.1 + + micromark-util-encode@2.0.1: {} + + micromark-util-html-tag-name@2.0.1: {} + + micromark-util-normalize-identifier@2.0.1: + dependencies: + micromark-util-symbol: 2.0.1 + + micromark-util-resolve-all@2.0.1: + dependencies: + micromark-util-types: 2.0.1 + + micromark-util-sanitize-uri@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-encode: 2.0.1 + micromark-util-symbol: 2.0.1 + + micromark-util-subtokenize@2.0.3: + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + + micromark-util-symbol@2.0.1: {} + + micromark-util-types@2.0.1: {} + + micromark@4.0.1: + dependencies: + '@types/debug': 4.1.12 + debug: 4.4.0 + decode-named-character-reference: 1.0.2 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.2 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-chunked: 2.0.1 + micromark-util-combine-extensions: 2.0.1 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-encode: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-subtokenize: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.1 + transitivePeerDependencies: + - supports-color + minimatch@10.0.1: dependencies: brace-expansion: 2.0.1 @@ -1093,6 +1471,8 @@ snapshots: minipass@7.1.2: {} + ms@2.1.3: {} + mvdan-sh@0.10.1: {} p-limit@4.0.0: @@ -1109,6 +1489,16 @@ snapshots: dependencies: callsites: 3.1.0 + parse-entities@4.0.2: + dependencies: + '@types/unist': 2.0.11 + character-entities-legacy: 3.0.0 + character-reference-invalid: 2.0.1 + decode-named-character-reference: 1.0.2 + is-alphanumerical: 2.0.1 + is-decimal: 2.0.1 + is-hexadecimal: 2.0.1 + parse-json@5.2.0: dependencies: '@babel/code-frame': 7.24.7 diff --git a/scripts/run_publisher.sh b/scripts/run_publisher.sh index bc2cd3b6..7b6608c7 100755 --- a/scripts/run_publisher.sh +++ b/scripts/run_publisher.sh @@ -18,6 +18,8 @@ usage() { echo " --telemetry-port : Specify the telemetry port number" echo " Default: 8080" echo " --extra-args : Optional additional arguments to append (in quotes)" + echo " --from-height : Specify the starting block height" + echo " Default: 0" echo "" echo "Examples:" echo " $0 # Runs with all defaults" @@ -25,6 +27,7 @@ usage() { echo " $0 --port 4001 # Runs on port 4001" echo " $0 --telemetry-port 8081 # uses telemetry port 8081" echo " $0 --network mainnet --port 4001 --telemetry-port 8081 --mode dev # Custom network, port, telemetry-port and mode" + echo " $0 --from-height 1000 # Start from block height 1000" exit 1 } @@ -33,6 +36,7 @@ NETWORK=${NETWORK:-"testnet"} MODE=${MODE:-"profiling"} PORT=${PORT:-"4004"} TELEMETRY_PORT=${TELEMETRY_PORT:-"8080"} +FROM_HEIGHT=${FROM_HEIGHT:-"0"} while [[ "$#" -gt 0 ]]; do case $1 in @@ -56,6 +60,10 @@ while [[ "$#" -gt 0 ]]; do EXTRA_ARGS="$2" shift 2 ;; + --from-height) + FROM_HEIGHT="$2" + shift 2 + ;; --help) usage ;; @@ -82,6 +90,7 @@ echo " → Network: $NETWORK" echo " → Mode: $MODE" echo " → Port: $PORT" echo " → Telemetry Port: $TELEMETRY_PORT" +echo " → From Height: $FROM_HEIGHT" if [ -n "$EXTRA_ARGS" ]; then echo "→ Extra Arguments: $EXTRA_ARGS" fi @@ -123,6 +132,7 @@ COMMON_ARGS=( # Application specific "--nats-url" "nats://localhost:4222" # "--telemetry-port" "${TELEMETRY_PORT}" + "--from-height" "${FROM_HEIGHT}" ) # Execute based on mode diff --git a/tarpaulin.toml b/tarpaulin.toml index ce0c18cd..1bedfb9d 100644 --- a/tarpaulin.toml +++ b/tarpaulin.toml @@ -1,19 +1,3 @@ -[cov_fuel_data_parser] -name = "Fuel Data Parser Coverage Analysis" -packages = ["fuel-data-parser"] -all-features = true -run-types = ["Lib", "Tests"] -timeout = "120s" -color = "Always" -locked = true -count = true -no-dead-code = true -fail-immediately = true -skip-clean = true -engine = "Llvm" -#fail-under = 80 - -# ========================================== [cov_fuel_streams] name = "Fuel Streams Coverage Analysis" packages = ["fuel-streams"] diff --git a/tests/Cargo.toml b/tests/Cargo.toml index bb1077fe..3b001818 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -1,27 +1,27 @@ [package] -name = "streams-tests" -authors = { workspace = true } -keywords = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -repository = { workspace = true } +name = "fuel-streams-test" version = { workspace = true } -rust-version = { workspace = true } -publish = false -autobenches = false -autotests = false +edition = { workspace = true } [[test]] harness = true name = "integration_tests" path = "tests/lib.rs" +[features] +test-helpers = [] + [dependencies] -fuel-core = { workspace = true, features = ["test-helpers"] } -fuel-streams = { workspace = true, features = ["test-helpers"] } +anyhow = { workspace = true } +fuel-message-broker = { workspace = true } fuel-streams-core = { workspace = true, features = ["test-helpers"] } +fuel-streams-domains = { workspace = true, features = ["test-helpers"] } +fuel-streams-macros = { workspace = true } +fuel-streams-store = { workspace = true, features = ["test-helpers"] } +futures = { workspace = true } +rand = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] } -[dev-dependencies] -pretty_assertions = { workspace = true } +[package.metadata.cargo-machete] +ignored = ["fuel-streams-macros"] diff --git a/tests/src/lib.rs b/tests/src/lib.rs index 23376c6b..1bd20031 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -1,112 +1,79 @@ -use std::{sync::Arc, time::Duration}; - -use fuel_streams::{client::Client, Connection, FuelNetwork}; -use fuel_streams_core::{ - nats::NatsClient, - prelude::*, - types::Transaction, - Stream, +use std::sync::Arc; + +use fuel_message_broker::{MessageBroker, MessageBrokerClient}; +use fuel_streams_core::{stream::*, subjects::*, types::Block}; +use fuel_streams_domains::blocks::{subjects::BlocksSubject, types::MockBlock}; +use fuel_streams_store::{ + db::{Db, DbConnectionOpts, DbResult}, + record::Record, + store::Store, }; -use tokio::task::JoinHandle; - -type PublishedBlocksResult = - BoxedResult<(Vec<(BlocksSubject, Block)>, JoinHandle<()>)>; -type PublishedTxsResult = - BoxedResult<(Vec<(TransactionsSubject, Transaction)>, JoinHandle<()>)>; - -#[derive(Debug, Clone)] -pub struct Streams { - pub blocks: Stream, - pub transactions: Stream, +use rand::Rng; + +// ----------------------------------------------------------------------------- +// Setup +// ----------------------------------------------------------------------------- + +pub async fn setup_db() -> DbResult { + let opts = DbConnectionOpts { + pool_size: Some(10), + ..Default::default() + }; + Db::new(opts).await } -impl Streams { - pub async fn new( - nats_client: &NatsClient, - storage: &Arc, - ) -> Self { - let blocks = Stream::::get_or_init(nats_client, storage).await; - let transactions = - Stream::::get_or_init(nats_client, storage).await; - Self { - transactions, - blocks, - } - } +pub async fn setup_store() -> DbResult> { + let db = setup_db().await?; + let store = Store::new(&db.arc()); + Ok(store) } -pub async fn server_setup() -> BoxedResult<(NatsClient, Streams, Connection)> { - let nats_client_opts = NatsClientOpts::admin_opts().with_rdn_namespace(); - let nats_client = NatsClient::connect(&nats_client_opts).await?; - - let storage_opts = S3StorageOpts::admin_opts().with_random_namespace(); - let storage = Arc::new(S3Storage::new(storage_opts).await?); - storage.create_bucket().await?; - - let streams = Streams::new(&nats_client, &storage).await; - - let mut client = Client::new(FuelNetwork::Local).await?; - let connection = client.connect().await?; - - Ok((nats_client, streams, connection)) +pub async fn setup_message_broker( + nats_url: &str, +) -> anyhow::Result> { + let broker = MessageBrokerClient::Nats.start(nats_url).await?; + broker.setup().await?; + Ok(broker) } -pub fn publish_items( - stream: &Stream, - items: Vec<(impl IntoSubject + Clone + 'static, T)>, -) -> JoinHandle<()> { - tokio::task::spawn({ - let stream = stream.clone(); - let items = items.clone(); - async move { - for item in items { - tokio::time::sleep(Duration::from_millis(50)).await; - let payload = item.1.clone(); - let subject = Arc::new(item.0); - let packet = payload.to_packet(subject); - - stream.publish(&packet).await.unwrap(); - } - } - }) +pub async fn setup_stream(nats_url: &str) -> anyhow::Result> { + let db = setup_db().await?; + let nats_client = setup_message_broker(nats_url).await?; + let stream = Stream::::get_or_init(&nats_client, &db.arc()).await; + Ok(stream) } -pub fn publish_blocks( - stream: &Stream, - producer: Option
, - use_height: Option, -) -> PublishedBlocksResult { - let mut items = Vec::new(); - for i in 0..10 { - let block_item = MockBlock::build(use_height.unwrap_or(i)); - let subject = BlocksSubject::build( - producer.clone(), - Some((use_height.unwrap_or(i)).into()), - ); - items.push((subject, block_item)); - } +// ----------------------------------------------------------------------------- +// Test data +// ----------------------------------------------------------------------------- - let join_handle = publish_items::(stream, items.clone()); +pub fn create_test_data(height: u32) -> (BlocksSubject, Block) { + let block = MockBlock::build(height); + let subject = BlocksSubject::from(&block); + (subject, block) +} - Ok((items, join_handle)) +pub fn create_multiple_test_data( + count: usize, + start_height: u32, +) -> Vec<(BlocksSubject, Block)> { + (0..count) + .map(|idx| create_test_data(start_height + idx as u32)) + .collect() } -pub fn publish_transactions( - stream: &Stream, - mock_block: &Block, - use_index: Option, -) -> PublishedTxsResult { - let mut items = Vec::new(); - for i in 0..10 { - let tx = MockTransaction::build(); - let subject = TransactionsSubject::from(&tx) - .with_block_height(Some(mock_block.height.into())) - .with_index(Some(use_index.unwrap_or(i) as usize)) - .with_status(Some(TransactionStatus::Success)); - items.push((subject, tx)); +pub async fn add_test_records( + store: &Store, + prefix: &str, + records: &[(Arc, Block)], +) -> anyhow::Result<()> { + for (subject, block) in records { + let packet = block.to_packet(subject.clone()).with_namespace(prefix); + store.insert_record(&packet).await?; } + Ok(()) +} - let join_handle = publish_items::(stream, items.clone()); - - Ok((items, join_handle)) +pub fn create_random_db_name() -> String { + format!("test_{}", rand::thread_rng().gen_range(0..1000000)) } diff --git a/tests/tests/client.rs b/tests/tests/client.rs deleted file mode 100644 index 9bcfd6c3..00000000 --- a/tests/tests/client.rs +++ /dev/null @@ -1,342 +0,0 @@ -// use std::{collections::HashSet, sync::Arc, time::Duration}; - -// use fuel_streams::prelude::*; -// use fuel_streams_core::prelude::{types, *}; -// use futures::{ -// future::{try_join_all, BoxFuture}, -// FutureExt, -// StreamExt, -// TryStreamExt, -// }; -// use rand::{distributions::Alphanumeric, Rng}; -// use streams_tests::{publish_blocks, server_setup}; -// use tokio::time::timeout; - -// fn gen_random_string(size: usize) -> String { -// rand::thread_rng() -// .sample_iter(&Alphanumeric) -// .take(size) -// .map(char::from) -// .collect() -// } - -// #[tokio::test] -// async fn conn_streams_has_required_streams() -> BoxedResult<()> { -// let (client, streams, _) = server_setup().await.unwrap(); -// let mut context_streams = client.jetstream.stream_names(); - -// let mut names = HashSet::new(); -// while let Some(name) = context_streams.try_next().await? { -// names.insert(name); -// } -// streams.blocks.assert_has_stream(&names).await; -// streams.transactions.assert_has_stream(&names).await; - -// for name in names.iter() { -// let empty = streams.blocks.is_empty(name).await; -// assert!(empty, "stream must be empty after creation"); -// } -// Ok(()) -// } - -// #[tokio::test] -// async fn fuel_streams_client_connection() -> BoxedResult<()> { -// let nats_opts = NatsClientOpts::admin_opts(); -// let client = NatsClient::connect(&nats_opts).await?; -// assert!(client.is_connected()); -// let s3_opts = Arc::new(S3StorageOpts::admin_opts()); -// let client = Client::with_opts(&nats_opts, &s3_opts).await?; -// assert!(client.nats_conn.is_connected()); -// Ok(()) -// } - -// #[tokio::test] -// async fn multiple_client_connections() -> BoxedResult<()> { -// let nats_opts = NatsClientOpts::admin_opts(); -// let s3_opts = Arc::new(S3StorageOpts::admin_opts()); -// let tasks: Vec<_> = (0..100) -// .map(|_| { -// let nats_opts = nats_opts.clone(); -// let s3_opts = s3_opts.clone(); -// async move { -// let client = -// Client::with_opts(&nats_opts, &s3_opts).await.unwrap(); -// assert!(client.nats_conn.is_connected()); -// Ok::<(), NatsError>(()) -// } -// }) -// .collect(); - -// assert!(try_join_all(tasks).await.is_ok()); -// Ok(()) -// } - -// #[tokio::test] -// async fn public_user_cannot_create_streams() -> BoxedResult<()> { -// let network = FuelNetwork::Local; -// let opts = NatsClientOpts::public_opts() -// .with_url(network.to_nats_url()) -// .with_rdn_namespace() -// .with_timeout(1); -// let client = NatsClient::connect(&opts).await?; -// let (random_stream_title, random_subject) = -// (gen_random_string(6), gen_random_string(6)); - -// assert!(client -// .jetstream -// .create_stream(types::NatsStreamConfig { -// name: random_stream_title, -// subjects: vec![random_subject], -// ..Default::default() -// }) -// .await -// .is_err()); - -// Ok(()) -// } - -// #[tokio::test] -// async fn public_user_cannot_create_stores() -> BoxedResult<()> { -// let network = FuelNetwork::Local; -// let opts = NatsClientOpts::public_opts() -// .with_url(network.to_nats_url()) -// .with_rdn_namespace() -// .with_timeout(1); - -// let random_bucket_title = gen_random_string(6); - -// let client = NatsClient::connect(&opts).await?; -// assert!(client -// .jetstream -// .create_key_value(types::KvStoreConfig { -// bucket: random_bucket_title, -// ..Default::default() -// }) -// .await -// .is_err()); - -// Ok(()) -// } - -// #[tokio::test] -// async fn public_user_cannot_delete_stores() -> BoxedResult<()> { -// let network = FuelNetwork::Local; -// let opts = NatsClientOpts::admin_opts() -// .with_url(network.to_nats_url()) -// .with_rdn_namespace() -// .with_timeout(1); - -// let random_bucket_title = gen_random_string(6); - -// let client = NatsClient::connect(&opts).await?; -// client -// .jetstream -// .create_key_value(types::KvStoreConfig { -// bucket: random_bucket_title.clone(), -// ..Default::default() -// }) -// .await?; - -// let opts = NatsClientOpts::public_opts() -// .with_url(network.to_nats_url()) -// .with_rdn_namespace() -// .with_timeout(1); -// let client = NatsClient::connect(&opts).await?; - -// assert!(client -// .jetstream -// .delete_key_value(&random_bucket_title) -// .await -// .is_err()); - -// Ok(()) -// } - -// #[tokio::test] -// async fn public_user_cannot_delete_stream() -> BoxedResult<()> { -// let opts = NatsClientOpts::admin_opts() -// .with_rdn_namespace() -// .with_timeout(1); -// let client = NatsClient::connect(&opts).await?; - -// let (random_stream_title, random_subject) = -// (gen_random_string(6), gen_random_string(6)); - -// client -// .jetstream -// .create_stream(types::NatsStreamConfig { -// name: random_stream_title.clone(), -// subjects: vec![random_subject], -// ..Default::default() -// }) -// .await?; - -// let network = FuelNetwork::Local; -// let public_opts = -// NatsClientOpts::public_opts().with_url(network.to_nats_url()); -// let public_client = NatsClient::connect(&public_opts).await?; - -// assert!( -// public_client -// .jetstream -// .delete_stream(&random_stream_title) -// .await -// .is_err(), -// "Stream must be deleted at this point" -// ); - -// Ok(()) -// } - -// #[tokio::test] -// async fn public_user_can_access_streams_after_created() { -// let network = FuelNetwork::Local; -// let admin_opts = NatsClientOpts::admin_opts() -// .with_url(network.to_nats_url()) -// .with_rdn_namespace() -// .with_timeout(1); - -// let public_opts = NatsClientOpts::public_opts() -// .with_url(network.to_nats_url()) -// .with_rdn_namespace() -// .with_timeout(1); - -// assert!(NatsClient::connect(&admin_opts).await.is_ok()); -// assert!(NatsClient::connect(&public_opts).await.is_ok()); -// } - -// #[tokio::test] -// async fn public_and_admin_user_can_access_streams_after_created( -// ) -> BoxedResult<()> { -// let network = FuelNetwork::Local; -// let admin_opts = NatsClientOpts::admin_opts() -// .with_url(network.to_nats_url()) -// .with_rdn_namespace() -// .with_timeout(1); -// let s3_opts = Arc::new(S3StorageOpts::admin_opts()); -// let admin_tasks: Vec>> = (0..100) -// .map(|_| { -// let opts: NatsClientOpts = admin_opts.clone(); -// let s3_opts = s3_opts.clone(); -// async move { -// let client = Client::with_opts(&opts, &s3_opts).await.unwrap(); -// assert!(client.nats_conn.is_connected()); -// Ok::<(), NatsError>(()) -// } -// .boxed() -// }) -// .collect(); - -// let public_opts = NatsClientOpts::public_opts() -// .with_url(network.to_nats_url()) -// .with_rdn_namespace() -// .with_timeout(1); -// let s3_public_opts = -// Arc::new(S3StorageOpts::new(S3Env::Local, S3Role::Public)); -// let public_tasks: Vec>> = (0..100) -// .map(|_| { -// let opts: NatsClientOpts = public_opts.clone(); -// let s3_opts = s3_public_opts.clone(); -// async move { -// let client = Client::with_opts(&opts, &s3_opts).await.unwrap(); -// assert!(client.nats_conn.is_connected()); -// Ok::<(), NatsError>(()) -// } -// .boxed() -// }) -// .collect(); - -// // Combine both vectors into one -// let mut all_tasks = -// Vec::with_capacity(admin_tasks.len() + public_tasks.len()); -// all_tasks.extend(admin_tasks); -// all_tasks.extend(public_tasks); - -// assert!(try_join_all(all_tasks).await.is_ok()); -// Ok(()) -// } - -// #[tokio::test] -// async fn admin_user_can_delete_stream() -> BoxedResult<()> { -// let opts = NatsClientOpts::admin_opts() -// .with_rdn_namespace() -// .with_timeout(1); -// let client = NatsClient::connect(&opts).await?; - -// let (random_stream_title, random_subject) = -// (gen_random_string(6), gen_random_string(6)); - -// client -// .jetstream -// .create_stream(types::NatsStreamConfig { -// name: random_stream_title.clone(), -// subjects: vec![random_subject], -// ..Default::default() -// }) -// .await?; - -// let status = client.jetstream.delete_stream(&random_stream_title).await?; -// assert!(status.success, "Stream must be deleted at this point"); - -// Ok(()) -// } - -// #[tokio::test] -// async fn admin_user_can_delete_stores() -> BoxedResult<()> { -// let opts = NatsClientOpts::admin_opts() -// .with_rdn_namespace() -// .with_timeout(1); - -// let random_bucket_title = gen_random_string(6); - -// let client = NatsClient::connect(&opts).await?; -// client -// .jetstream -// .create_key_value(types::KvStoreConfig { -// bucket: random_bucket_title.clone(), -// ..Default::default() -// }) -// .await?; - -// assert!(client -// .jetstream -// .delete_key_value(&random_bucket_title) -// .await -// .is_ok()); - -// Ok(()) -// } - -// #[tokio::test] -// async fn ensure_deduplication_when_publishing() -> BoxedResult<()> { -// let (_, _, client) = server_setup().await.unwrap(); -// let stream = fuel_streams::Stream::::new(&client).await; -// let producer = Some(Address::zeroed()); -// let const_block_height = 1001; -// let items = -// publish_blocks(stream.stream(), producer, Some(const_block_height)) -// .unwrap() -// .0; - -// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); -// let timeout_duration = Duration::from_secs(1); - -// // ensure just one message was published -// 'l: loop { -// match timeout(timeout_duration, sub.next()).await { -// Ok(Some((idx, entry))) => { -// let decoded_msg = Block::decode_raw(entry).unwrap(); -// let (subject, _block) = items[idx].to_owned(); -// let height = decoded_msg.payload.height; -// assert_eq!(decoded_msg.subject, subject.parse()); -// assert_eq!(height, const_block_height); -// assert!(idx < 1); -// } -// _ => { -// break 'l; -// } -// } -// } - -// Ok(()) -// } diff --git a/tests/tests/lib.rs b/tests/tests/lib.rs index 6a618b55..eff55a0c 100644 --- a/tests/tests/lib.rs +++ b/tests/tests/lib.rs @@ -1,3 +1,3 @@ -mod client; -mod publisher; +#![cfg(test)] +mod store; mod stream; diff --git a/tests/tests/publisher.rs b/tests/tests/publisher.rs deleted file mode 100644 index 61611acd..00000000 --- a/tests/tests/publisher.rs +++ /dev/null @@ -1,416 +0,0 @@ -// use std::{collections::HashMap, sync::Arc}; - -// use fuel_core::{ -// combined_database::CombinedDatabase, -// service::{Config, FuelService}, -// ShutdownListener, -// }; -// use fuel_core_importer::ImporterResult; -// use fuel_core_types::blockchain::SealedBlock; -// use fuel_streams_core::prelude::*; -// use tokio::sync::broadcast::{self, Receiver, Sender}; - -// // TODO - Re-implement with `mockall` and `mock` macros -// struct TestFuelCore { -// fuel_service: FuelService, -// chain_id: FuelCoreChainId, -// base_asset_id: FuelCoreAssetId, -// database: CombinedDatabase, -// blocks_broadcaster: Sender, -// receipts: Option>, -// } - -// impl TestFuelCore { -// fn default( -// blocks_broadcaster: Sender, -// ) -> Self { -// let mut shutdown = ShutdownListener::spawn(); -// let service = FuelService::new( -// Default::default(), -// Config::local_node(), -// &mut shutdown, -// ) -// .unwrap(); -// Self { -// fuel_service: service, -// chain_id: FuelCoreChainId::default(), -// base_asset_id: FuelCoreAssetId::zeroed(), -// database: CombinedDatabase::default(), -// blocks_broadcaster, -// receipts: None, -// } -// } -// fn with_receipts(mut self, receipts: Vec) -> Self { -// self.receipts = Some(receipts); -// self -// } -// fn arc(self) -> Arc { -// Arc::new(self) -// } -// } - -// #[async_trait::async_trait] -// impl FuelCoreLike for TestFuelCore { -// async fn start(&self) -> anyhow::Result<()> { -// Ok(()) -// } -// fn is_started(&self) -> bool { -// true -// } -// fn fuel_service(&self) -> &FuelService { -// &self.fuel_service -// } -// async fn await_synced_at_least_once( -// &self, -// _historical: bool, -// ) -> anyhow::Result<()> { -// Ok(()) -// } -// async fn stop(&self) {} - -// async fn await_offchain_db_sync( -// &self, -// _block_id: &FuelCoreBlockId, -// ) -> anyhow::Result<()> { -// Ok(()) -// } - -// fn base_asset_id(&self) -> &FuelCoreAssetId { -// &self.base_asset_id -// } -// fn chain_id(&self) -> &FuelCoreChainId { -// &self.chain_id -// } - -// fn database(&self) -> &CombinedDatabase { -// &self.database -// } - -// fn blocks_subscription( -// &self, -// ) -> Receiver { -// self.blocks_broadcaster.subscribe() -// } - -// fn get_receipts( -// &self, -// _tx_id: &FuelCoreBytes32, -// ) -> anyhow::Result>> { -// Ok(self.receipts.clone()) -// } - -// fn get_tx_status( -// &self, -// _tx_id: &FuelCoreBytes32, -// ) -> anyhow::Result> { -// Ok(Some(FuelCoreTransactionStatus::Success { -// receipts: self.receipts.clone().unwrap_or_default(), -// block_height: 0.into(), -// result: None, -// time: FuelCoreTai64::now(), -// total_gas: 0, -// total_fee: 0, -// })) -// } -// } - -// #[tokio::test(flavor = "multi_thread")] -// async fn doesnt_publish_any_message_when_no_block_has_been_mined() { -// let (blocks_broadcaster, _) = broadcast::channel::(1); -// let storage = Arc::new(S3Storage::new_for_testing().await); -// let publisher = new_publisher(blocks_broadcaster.clone(), &storage).await; - -// let shutdown_controller = start_publisher(&publisher).await; -// stop_publisher(shutdown_controller).await; - -// assert!(publisher.get_fuel_streams().is_empty().await); -// } - -// #[tokio::test(flavor = "multi_thread")] -// async fn publishes_a_block_message_when_a_single_block_has_been_mined() { -// let (blocks_broadcaster, _) = broadcast::channel::(1); -// let storage = Arc::new(S3Storage::new_for_testing().await); -// let publisher = new_publisher(blocks_broadcaster.clone(), &storage).await; - -// publish_block(&publisher, &blocks_broadcaster).await; - -// assert!(publisher -// .get_fuel_streams() -// .blocks() -// .get_last_published(BlocksSubject::WILDCARD) -// .await -// .is_ok_and(|result| result.is_some())); -// storage.cleanup_after_testing().await; -// } - -// #[tokio::test(flavor = "multi_thread")] -// async fn publishes_transaction_for_each_published_block() { -// let (blocks_broadcaster, _) = broadcast::channel::(1); -// let storage = Arc::new(S3Storage::new_for_testing().await); -// let publisher = new_publisher(blocks_broadcaster.clone(), &storage).await; - -// publish_block(&publisher, &blocks_broadcaster).await; - -// assert!(publisher -// .get_fuel_streams() -// .transactions() -// .get_last_published(TransactionsSubject::WILDCARD) -// .await -// .is_ok_and(|result| result.is_some())); -// storage.cleanup_after_testing().await; -// } - -// #[tokio::test(flavor = "multi_thread")] -// async fn publishes_receipts() { -// let (blocks_broadcaster, _) = broadcast::channel::(1); - -// let receipts = [ -// FuelCoreReceipt::Call { -// id: FuelCoreContractId::default(), -// to: Default::default(), -// amount: 0, -// asset_id: Default::default(), -// gas: 0, -// param1: 0, -// param2: 0, -// pc: 0, -// is: 0, -// }, -// FuelCoreReceipt::Return { -// id: FuelCoreContractId::default(), -// val: 0, -// pc: 0, -// is: 0, -// }, -// FuelCoreReceipt::ReturnData { -// id: FuelCoreContractId::default(), -// ptr: 0, -// len: 0, -// digest: FuelCoreBytes32::default(), -// pc: 0, -// is: 0, -// data: None, -// }, -// FuelCoreReceipt::Revert { -// id: FuelCoreContractId::default(), -// ra: 0, -// pc: 0, -// is: 0, -// }, -// FuelCoreReceipt::Log { -// id: FuelCoreContractId::default(), -// ra: 0, -// rb: 0, -// rc: 0, -// rd: 0, -// pc: 0, -// is: 0, -// }, -// FuelCoreReceipt::LogData { -// id: FuelCoreContractId::default(), -// ra: 0, -// rb: 0, -// ptr: 0, -// len: 0, -// digest: FuelCoreBytes32::default(), -// pc: 0, -// is: 0, -// data: None, -// }, -// FuelCoreReceipt::Transfer { -// id: FuelCoreContractId::default(), -// to: FuelCoreContractId::default(), -// amount: 0, -// asset_id: FuelCoreAssetId::default(), -// pc: 0, -// is: 0, -// }, -// FuelCoreReceipt::TransferOut { -// id: FuelCoreContractId::default(), -// to: FuelCoreAddress::default(), -// amount: 0, -// asset_id: FuelCoreAssetId::default(), -// pc: 0, -// is: 0, -// }, -// FuelCoreReceipt::Mint { -// sub_id: FuelCoreBytes32::default(), -// contract_id: FuelCoreContractId::default(), -// val: 0, -// pc: 0, -// is: 0, -// }, -// FuelCoreReceipt::Burn { -// sub_id: FuelCoreBytes32::default(), -// contract_id: FuelCoreContractId::default(), -// val: 0, -// pc: 0, -// is: 0, -// }, -// ]; - -// let fuel_core = TestFuelCore::default(blocks_broadcaster.clone()) -// .with_receipts(receipts.to_vec()) -// .arc(); - -// let storage = Arc::new(S3Storage::new_for_testing().await); -// let publisher = -// Publisher::new_for_testing(&nats_client().await, &storage, fuel_core) -// .await -// .unwrap(); - -// publish_block(&publisher, &blocks_broadcaster).await; - -// let mut receipts_stream = publisher -// .get_fuel_streams() -// .receipts() -// .catchup(10) -// .await -// .unwrap(); - -// let expected_receipts: Vec = -// receipts.iter().map(Into::into).collect(); -// let mut found_receipts = Vec::new(); - -// while let Some(Some(receipt)) = receipts_stream.next().await { -// found_receipts.push(receipt); -// } -// assert_eq!( -// found_receipts.len(), -// expected_receipts.len(), -// "Number of receipts doesn't match" -// ); - -// // Create sets of receipt identifiers -// let found_ids: std::collections::HashSet<_> = found_receipts -// .into_iter() -// .map(|r| match r { -// Receipt::Call(r) => r.id, -// Receipt::Return(r) => r.id, -// Receipt::ReturnData(r) => r.id, -// Receipt::Revert(r) => r.id, -// Receipt::Log(r) => r.id, -// Receipt::LogData(r) => r.id, -// Receipt::Transfer(r) => r.id, -// Receipt::TransferOut(r) => r.id, -// Receipt::Mint(r) => r.contract_id, -// Receipt::Burn(r) => r.contract_id, -// Receipt::Panic(r) => r.id, -// _ => unreachable!(), -// }) -// .collect(); - -// let expected_ids: std::collections::HashSet<_> = expected_receipts -// .into_iter() -// .map(|r| match r { -// Receipt::Call(r) => r.id, -// Receipt::Return(r) => r.id, -// Receipt::ReturnData(r) => r.id, -// Receipt::Revert(r) => r.id, -// Receipt::Log(r) => r.id, -// Receipt::LogData(r) => r.id, -// Receipt::Transfer(r) => r.id, -// Receipt::TransferOut(r) => r.id, -// Receipt::Mint(r) => r.contract_id, -// Receipt::Burn(r) => r.contract_id, -// Receipt::Panic(r) => r.id, -// _ => unreachable!(), -// }) -// .collect(); - -// assert_eq!( -// found_ids, expected_ids, -// "Published receipt IDs don't match expected IDs" -// ); - -// storage.cleanup_after_testing().await; -// } - -// #[tokio::test(flavor = "multi_thread")] -// async fn publishes_inputs() { -// let (blocks_broadcaster, _) = broadcast::channel::(1); -// let storage = Arc::new(S3Storage::new_for_testing().await); -// let publisher = new_publisher(blocks_broadcaster.clone(), &storage).await; - -// publish_block(&publisher, &blocks_broadcaster).await; - -// assert!(publisher -// .get_fuel_streams() -// .inputs() -// .get_last_published(InputsByIdSubject::WILDCARD) -// .await -// .is_ok_and(|result| result.is_some())); -// storage.cleanup_after_testing().await; -// } - -// async fn new_publisher( -// broadcaster: Sender, -// storage: &Arc, -// ) -> Publisher { -// let fuel_core = TestFuelCore::default(broadcaster).arc(); -// Publisher::new_for_testing(&nats_client().await, storage, fuel_core) -// .await -// .unwrap() -// } - -// async fn publish_block( -// publisher: &Publisher, -// blocks_broadcaster: &Sender, -// ) { -// let shutdown_controller = start_publisher(publisher).await; -// send_block(blocks_broadcaster); -// stop_publisher(shutdown_controller).await; -// } - -// async fn start_publisher(publisher: &Publisher) -> ShutdownController { -// let (shutdown_controller, shutdown_token) = get_controller_and_token(); -// tokio::spawn({ -// let publisher = publisher.clone(); -// async move { -// publisher.run(shutdown_token, true).await.unwrap(); -// } -// }); -// wait_for_publisher_to_start().await; -// shutdown_controller -// } -// async fn stop_publisher(shutdown_controller: ShutdownController) { -// wait_for_publisher_to_process_block().await; - -// assert!(shutdown_controller.initiate_shutdown().is_ok()); -// } - -// async fn wait_for_publisher_to_start() { -// tokio::time::sleep(std::time::Duration::from_secs(1)).await; -// } -// async fn wait_for_publisher_to_process_block() { -// tokio::time::sleep(std::time::Duration::from_secs(1)).await; -// } - -// fn send_block(broadcaster: &Sender) { -// let block = create_test_block(); -// assert!(broadcaster.send(block).is_ok()); -// } -// fn create_test_block() -> ImporterResult { -// let mut block_entity = FuelCoreBlock::default(); -// let tx = FuelCoreTransaction::default_test_tx(); - -// *block_entity.transactions_mut() = vec![tx]; - -// ImporterResult { -// shared_result: Arc::new(FuelCoreImportResult { -// sealed_block: SealedBlock { -// entity: block_entity, -// ..Default::default() -// }, -// ..Default::default() -// }), -// changes: Arc::new(HashMap::new()), -// } -// } - -// async fn nats_client() -> NatsClient { -// let opts = NatsClientOpts::admin_opts().with_rdn_namespace(); -// NatsClient::connect(&opts) -// .await -// .expect("NATS connection failed") -// } diff --git a/tests/tests/store/mod.rs b/tests/tests/store/mod.rs new file mode 100644 index 00000000..c66d9a1f --- /dev/null +++ b/tests/tests/store/mod.rs @@ -0,0 +1,2 @@ +mod pattern_matching; +mod record; diff --git a/tests/tests/store/pattern_matching.rs b/tests/tests/store/pattern_matching.rs new file mode 100644 index 00000000..8dec3228 --- /dev/null +++ b/tests/tests/store/pattern_matching.rs @@ -0,0 +1,61 @@ +use std::sync::Arc; + +use fuel_streams_core::{subjects::*, types::Block}; +use fuel_streams_domains::blocks::{subjects::BlocksSubject, types::MockBlock}; +use fuel_streams_store::record::{QueryOptions, Record}; +use fuel_streams_test::{create_random_db_name, setup_store}; + +#[tokio::test] +async fn test_asterisk_wildcards() -> anyhow::Result<()> { + let prefix = create_random_db_name(); + let mut store = setup_store::().await?; + store.with_namespace(&prefix); + + // Create and insert test blocks with different subjects + let blocks = vec![ + ( + MockBlock::build(1), + BlocksSubject::new().with_block_height(Some(1.into())), + ), + ( + MockBlock::build(2), + BlocksSubject::new().with_block_height(Some(2.into())), + ), + ( + MockBlock::build(3), + BlocksSubject::new().with_block_height(Some(3.into())), + ), + ]; + + for (block, subject) in blocks { + let packet = block.to_packet(Arc::new(subject)).with_namespace(&prefix); + store.insert_record(&packet).await?; + } + + // Test wildcard matching + let subject = BlocksSubject::new().with_block_height(None).dyn_arc(); + let records = store + .find_many_by_subject(&subject, QueryOptions::default()) + .await?; + assert_eq!(records.len(), 3); + + Ok(()) +} + +#[tokio::test] +async fn test_nonexistent_subjects() -> anyhow::Result<()> { + let prefix = create_random_db_name(); + let mut store = setup_store::().await?; + store.with_namespace(&prefix); + + // Test finding with a subject that doesn't exist + let nonexistent_subject = BlocksSubject::new() + .with_block_height(Some(999.into())) + .dyn_arc(); + let records = store + .find_many_by_subject(&nonexistent_subject, QueryOptions::default()) + .await?; + assert!(records.is_empty()); + + Ok(()) +} diff --git a/tests/tests/store/record.rs b/tests/tests/store/record.rs new file mode 100644 index 00000000..1c0bd4c1 --- /dev/null +++ b/tests/tests/store/record.rs @@ -0,0 +1,171 @@ +use std::sync::Arc; + +use fuel_streams_core::{subjects::*, types::Block}; +use fuel_streams_domains::blocks::{ + subjects::BlocksSubject, + types::MockBlock, + BlockDbItem, +}; +use fuel_streams_store::record::{QueryOptions, Record, RecordPacket}; +use fuel_streams_test::{create_random_db_name, setup_store}; + +#[tokio::test] +async fn test_block_db_item_conversion() -> anyhow::Result<()> { + let block = MockBlock::build(1); + let subject = BlocksSubject::from(&block); + + // Create Arc explicitly and use RecordPacket::new + let subject_arc: Arc = Arc::new(subject.clone()); + let packet = RecordPacket::new(subject_arc, &block); + + // Test direct conversion + let db_item = BlockDbItem::try_from(&packet) + .expect("Failed to convert packet to BlockDbItem"); + + let height: i64 = block.height.clone().into(); + assert_eq!(db_item.subject, subject.parse()); + assert_eq!(db_item.block_height, height); + assert_eq!(db_item.producer_address, block.producer.to_string()); + + // Verify we can decode the value back to a block + let decoded_block: Block = serde_json::from_slice(&db_item.value) + .expect("Failed to decode block from value"); + assert_eq!(decoded_block, block); + Ok(()) +} + +#[tokio::test] +async fn test_basic_insert() -> anyhow::Result<()> { + let store = setup_store().await?; + let block = MockBlock::build(1); + let subject = BlocksSubject::from(&block); + let prefix = create_random_db_name(); + let packet = block + .to_packet(Arc::new(subject.clone())) + .with_namespace(&prefix); + + let db_record = store.insert_record(&packet).await?; + assert_eq!(db_record.subject, packet.subject_str()); + assert_eq!(Block::from_db_item(&db_record).await?, block); + + Ok(()) +} + +#[tokio::test] +async fn test_multiple_inserts() -> anyhow::Result<()> { + let prefix = create_random_db_name(); + let mut store = setup_store().await?; + store.with_namespace(&prefix); + let subject = BlocksSubject::from(&MockBlock::build(1)); + + // Insert first block + let block1 = MockBlock::build(1); + let packet = block1 + .to_packet(Arc::new(subject.clone())) + .with_namespace(&prefix); + let db_record1 = store.insert_record(&packet).await?; + + // Insert second block + let block2 = MockBlock::build(2); + let packet = block2 + .to_packet(Arc::new(subject.clone())) + .with_namespace(&prefix); + let db_record2 = store.insert_record(&packet).await?; + + // Verify both records exist and are correct + assert_eq!(Block::from_db_item(&db_record1).await?, block1); + assert_eq!(Block::from_db_item(&db_record2).await?, block2); + + // Verify both records are found + let subject = BlocksSubject::new().with_block_height(None).dyn_arc(); + let records = store + .find_many_by_subject(&subject, QueryOptions::default()) + .await?; + assert_eq!(records.len(), 2); + + Ok(()) +} + +#[tokio::test] +async fn test_find_many_by_subject() -> anyhow::Result<()> { + let prefix = create_random_db_name(); + let mut store = setup_store().await?; + store.with_namespace(&prefix); + let subject1 = BlocksSubject::from(&MockBlock::build(1)); + let subject2 = BlocksSubject::from(&MockBlock::build(2)); + + // Insert blocks with different subjects + let block1 = MockBlock::build(1); + let block2 = MockBlock::build(2); + let packet1 = block1 + .to_packet(Arc::new(subject1.clone())) + .with_namespace(&prefix); + let packet2 = block2 + .to_packet(Arc::new(subject2.clone())) + .with_namespace(&prefix); + + store.insert_record(&packet1).await?; + store.insert_record(&packet2).await?; + + // Test finding by subject1 + let records = store + .find_many_by_subject(&subject1.dyn_arc(), QueryOptions::default()) + .await?; + assert_eq!(records.len(), 1); + assert_eq!(Block::from_db_item(&records[0]).await?, block1); + + // Test finding by subject2 + let records = store + .find_many_by_subject(&subject2.dyn_arc(), QueryOptions::default()) + .await?; + assert_eq!(records.len(), 1); + assert_eq!(Block::from_db_item(&records[0]).await?, block2); + + Ok(()) +} + +#[tokio::test] +async fn test_find_last_record() -> anyhow::Result<()> { + let prefix = create_random_db_name(); + let mut store = setup_store().await?; + store.with_namespace(&prefix); + let subject = BlocksSubject::from(&MockBlock::build(1)); + + // Insert multiple blocks + let blocks = vec![ + MockBlock::build(1), + MockBlock::build(2), + MockBlock::build(3), + ]; + + for block in &blocks { + let packet = block + .to_packet(Arc::new(subject.clone())) + .with_namespace(&prefix); + store.insert_record(&packet).await?; + } + + // Test finding last record + let last_record = store.find_last_record().await?; + assert!(last_record.is_some()); + let last_block = Block::from_db_item(&last_record.unwrap()).await?; + assert_eq!(last_block, blocks.last().unwrap().clone()); + + Ok(()) +} + +#[tokio::test] +async fn test_subject_matching() -> anyhow::Result<()> { + let block = MockBlock::build(1); + let subject = BlocksSubject::from(&block); + let packet = block.to_packet(Arc::new(subject.clone())); + + // Test subject matching + let matched_subject: BlocksSubject = packet + .subject_matches() + .expect("Failed to match BlocksSubject"); + + assert_eq!(matched_subject.parse(), subject.parse()); + + Ok(()) +} diff --git a/tests/tests/stream.rs b/tests/tests/stream.rs deleted file mode 100644 index 2df55a23..00000000 --- a/tests/tests/stream.rs +++ /dev/null @@ -1,242 +0,0 @@ -// use fuel_streams::prelude::*; -// use fuel_streams_core::prelude::*; -// use futures::{future::try_join_all, StreamExt}; -// use pretty_assertions::assert_eq; -// use streams_tests::{publish_blocks, publish_transactions, server_setup}; - -// #[tokio::test] -// async fn blocks_streams_subscribe() { -// let (_, _, client) = server_setup().await.unwrap(); -// let stream = fuel_streams::Stream::::new(&client).await; -// let producer = Some(Address::zeroed()); -// let items = publish_blocks(stream.stream(), producer, None).unwrap().0; - -// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); - -// while let Some((i, bytes)) = sub.next().await { -// let decoded_msg = Block::decode_raw(bytes).unwrap(); -// let (subject, block) = items[i].to_owned(); -// let height = decoded_msg.payload.height; - -// assert_eq!(decoded_msg.subject, subject.parse()); -// assert_eq!(decoded_msg.payload, block); -// assert_eq!(height, i as u32); -// if i == 9 { -// break; -// } -// } -// } - -// #[tokio::test] -// async fn blocks_streams_subscribe_with_filter() { -// let (_, _, client) = server_setup().await.unwrap(); -// let mut stream = fuel_streams::Stream::::new(&client).await; -// let producer = Some(Address::zeroed()); - -// // publishing 10 blocks -// publish_blocks(stream.stream(), producer, None).unwrap(); - -// // filtering by producer 0x000 and height 5 -// let filter = Filter::::build() -// .with_producer(Some(Address::zeroed())) -// .with_height(Some(5.into())); - -// // creating subscription -// let mut sub = stream -// .with_filter(filter) -// .subscribe_raw_with_config(StreamConfig::default()) -// .await -// .unwrap() -// .take(10); - -// // result should be just 1 single message with height 5 -// while let Some(bytes) = sub.next().await { -// let decoded_msg = Block::decode_raw(bytes).unwrap(); -// let height = decoded_msg.payload.height; -// assert_eq!(height, 5); -// if height == 5 { -// break; -// } -// } -// } - -// #[tokio::test] -// async fn transactions_streams_subscribe() { -// let (_, _, client) = server_setup().await.unwrap(); -// let stream = fuel_streams::Stream::::new(&client).await; - -// let mock_block = MockBlock::build(1); -// let items = publish_transactions(stream.stream(), &mock_block, None) -// .unwrap() -// .0; - -// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); -// while let Some((i, bytes)) = sub.next().await { -// let decoded_msg = Transaction::decode_raw(bytes).unwrap(); - -// let (_, transaction) = items[i].to_owned(); -// assert_eq!(decoded_msg.payload, transaction); -// if i == 9 { -// break; -// } -// } -// } - -// #[tokio::test] -// async fn transactions_streams_subscribe_with_filter() { -// let (_, _, client) = server_setup().await.unwrap(); -// let mut stream = fuel_streams::Stream::::new(&client).await; - -// // publishing 10 transactions -// let mock_block = MockBlock::build(5); -// let items = publish_transactions(stream.stream(), &mock_block, None) -// .unwrap() -// .0; - -// // filtering by transaction on block with height 5 -// let filter = Filter::::build() -// .with_block_height(Some(5.into())); - -// // creating subscription -// let mut sub = stream -// .with_filter(filter) -// .subscribe_raw_with_config(StreamConfig::default()) -// .await -// .unwrap() -// .take(10) -// .enumerate(); - -// // result should be 10 transactions messages -// while let Some((i, bytes)) = sub.next().await { -// let decoded_msg = Transaction::decode(bytes).unwrap(); - -// let (_, transaction) = items[i].to_owned(); -// assert_eq!(decoded_msg, transaction); -// if i == 9 { -// break; -// } -// } -// } - -// #[tokio::test] -// async fn multiple_subscribers_same_subject() { -// let (_, _, client) = server_setup().await.unwrap(); -// let stream = fuel_streams::Stream::::new(&client).await; -// let producer = Some(Address::zeroed()); -// let items = publish_blocks(stream.stream(), producer.clone(), None) -// .unwrap() -// .0; - -// let clients_count = 100; -// let done_signal = 99; -// let mut handles = Vec::new(); -// for _ in 0..clients_count { -// let stream = stream.clone(); -// let items = items.clone(); -// handles.push(tokio::spawn(async move { -// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); -// while let Some((i, bytes)) = sub.next().await { -// let decoded_msg = Block::decode_raw(bytes).unwrap(); -// let (subject, block) = items[i].to_owned(); -// let height = decoded_msg.payload.height; - -// assert_eq!(decoded_msg.subject, subject.parse()); -// assert_eq!(decoded_msg.payload, block); -// assert_eq!(height, i as u32); -// if i == 9 { -// return done_signal; -// } -// } -// done_signal + 1 -// })); -// } - -// let mut client_results = try_join_all(handles).await.unwrap(); -// assert!( -// client_results.len() == clients_count, -// "must have all clients subscribed to one subject" -// ); -// client_results.dedup(); -// assert!( -// client_results.len() == 1, -// "all clients must have the same result" -// ); -// assert!( -// client_results.first().cloned().unwrap() == done_signal, -// "all clients must have the same received the complete signal" -// ); -// } - -// #[tokio::test] -// async fn multiple_subscribers_different_subjects() { -// let (_, _, client) = server_setup().await.unwrap(); -// let producer = Some(Address::zeroed()); -// let block_stream = fuel_streams::Stream::::new(&client).await; -// let block_items = -// publish_blocks(block_stream.stream(), producer.clone(), None) -// .unwrap() -// .0; - -// let txs_stream = fuel_streams::Stream::::new(&client).await; -// let mock_block = MockBlock::build(1); -// let txs_items = -// publish_transactions(txs_stream.stream(), &mock_block, None) -// .unwrap() -// .0; - -// let clients_count = 100; -// let done_signal = 99; -// let mut handles = Vec::new(); -// for _ in 0..clients_count { -// // blocks stream -// let stream = block_stream.clone(); -// let items = block_items.clone(); -// handles.push(tokio::spawn(async move { -// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); -// while let Some((i, bytes)) = sub.next().await { -// let decoded_msg = Block::decode_raw(bytes).unwrap(); -// let (subject, block) = items[i].to_owned(); -// let height = decoded_msg.payload.height; - -// assert_eq!(decoded_msg.subject, subject.parse()); -// assert_eq!(decoded_msg.payload, block); -// assert_eq!(height, i as u32); -// if i == 9 { -// return done_signal; -// } -// } -// done_signal + 1 -// })); - -// // txs stream -// let stream = txs_stream.clone(); -// let items = txs_items.clone(); -// handles.push(tokio::spawn(async move { -// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); -// while let Some((i, bytes)) = sub.next().await { -// let decoded_msg = Transaction::decode_raw(bytes).unwrap(); -// let (_, transaction) = items[i].to_owned(); -// assert_eq!(decoded_msg.payload, transaction); -// if i == 9 { -// return done_signal; -// } -// } -// done_signal + 1 -// })); -// } - -// let mut client_results = try_join_all(handles).await.unwrap(); -// assert!( -// client_results.len() == 2 * clients_count, -// "must have all clients subscribed to two subjects" -// ); -// client_results.dedup(); -// assert!( -// client_results.len() == 1, -// "all clients must have the same result" -// ); -// assert!( -// client_results.first().cloned().unwrap() == done_signal, -// "all clients must have the same received the complete signal" -// ); -// } diff --git a/tests/tests/stream/live_data.rs b/tests/tests/stream/live_data.rs new file mode 100644 index 00000000..e5f041be --- /dev/null +++ b/tests/tests/stream/live_data.rs @@ -0,0 +1,41 @@ +use fuel_streams_core::{subjects::*, types::Block, DeliverPolicy}; +use fuel_streams_store::record::{DataEncoder, Record}; +use fuel_streams_test::{create_multiple_test_data, setup_stream}; +use futures::StreamExt; + +const NATS_URL: &str = "nats://localhost:4222"; + +#[tokio::test] +async fn test_streaming_live_data() -> anyhow::Result<()> { + let stream = setup_stream(NATS_URL).await?; + let data = create_multiple_test_data(10, 0); + + tokio::spawn({ + let data = data.clone(); + let stream = stream.clone(); + async move { + let subject = BlocksSubject::new().with_block_height(None); + let mut subscriber = stream + .subscribe(subject, DeliverPolicy::New) + .await + .enumerate(); + + while let Some((index, record)) = subscriber.next().await { + let record = record.unwrap(); + let expected_block = &data[index].1; + let decoded_block = Block::decode(&record).await.unwrap(); + assert_eq!(decoded_block, *expected_block); + if index == data.len() - 1 { + break; + } + } + } + }); + + for (subject, block) in data { + let packet = block.to_packet(subject.arc()).arc(); + stream.publish(&packet).await?; + } + + Ok(()) +} diff --git a/tests/tests/stream/mod.rs b/tests/tests/stream/mod.rs new file mode 100644 index 00000000..b025511b --- /dev/null +++ b/tests/tests/stream/mod.rs @@ -0,0 +1,2 @@ +mod live_data; +mod store_stream; diff --git a/tests/tests/stream/store_stream.rs b/tests/tests/stream/store_stream.rs new file mode 100644 index 00000000..d12de9a0 --- /dev/null +++ b/tests/tests/stream/store_stream.rs @@ -0,0 +1,41 @@ +use fuel_streams_core::types::Block; +use fuel_streams_store::record::Record; +use fuel_streams_test::{ + create_multiple_test_data, + create_random_db_name, + setup_store, +}; +use futures::StreamExt; + +#[tokio::test] +async fn test_stream_by_subject() -> anyhow::Result<()> { + // Setup store and test data + let prefix = create_random_db_name(); + let mut store = setup_store::().await?; + store.with_namespace(&prefix); + let data = create_multiple_test_data(10, 0); + + // Insert test records + for (subject, block) in &data { + let packet = block + .to_packet(subject.clone().dyn_arc()) + .with_namespace(&prefix); + store.insert_record(&packet).await?; + } + + // Test streaming with the first subject + let subject = data[0].0.clone(); + let mut stream = store.stream_by_subject(subject.arc(), Some(0)); + let mut count = 0; + while let Some(result) = stream.next().await { + let record = result?; + let height: u32 = data[count].1.height.clone().into(); + assert_eq!(record.block_height as u32, height); + count += 1; + } + + // Verify we got all records for this subject + assert_eq!(count, 1); // We should only get one record since we're querying by specific subject + + Ok(()) +}