From c28bd818d42393306675083b17f82187a24dc59b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 19 May 2023 15:14:52 +1000 Subject: [PATCH] Logging via the HTTP API (#4074) Squashed commit of the following: commit 16771dc8858ca82b0787672ad34ae0e366b8e51c Author: Age Manning Date: Wed May 17 15:27:47 2023 +1000 fmt commit 1deeafe2d515d64a1360e7f20aa34d5109311d42 Merge: 9f5ca3de7 b29bb2e03 Author: Age Manning Date: Wed May 17 15:26:13 2023 +1000 Merge latest unstable commit 9f5ca3de7ed7b3cea237e335f30711120b6526bd Author: Age Manning Date: Wed May 17 15:24:06 2023 +1000 Enable SSE logs only on GUI flag and misc reviewers comments commit 5eeca709cdc85382fb69ec063929bdf76cf4621a Merge: 3efdbe2b2 d64be0dd7 Author: Age Manning Date: Tue May 9 10:08:05 2023 +1000 Merge latest unstable commit 3efdbe2b22e3c5b363dd03da5c693ce7d61eb6c4 Author: Age Manning Date: Mon May 8 17:13:12 2023 +1000 fmt commit 202aad89a0ee4e4cfe27538fc8fcf598a24dbdd1 Author: Age Manning Date: Wed May 3 16:30:19 2023 +1000 Remove api token requirement for sse logs commit 5411d5a9a1d7aa3628160b0366189bb803fb2ce4 Author: Age Manning Date: Wed May 3 15:57:59 2023 +1000 Remove double json encoding commit 29fb985d91d7837f2f8a48fd57bbf20b15648c5d Author: Age Manning Date: Wed May 3 14:32:31 2023 +1000 fmt commit 3e10b2625b7a9f150256a07864e06c29175e9ba8 Merge: 37ec9b779 826e74862 Author: Age Manning Date: Wed May 3 14:30:41 2023 +1000 Merge latest unstable commit 37ec9b779936ee00b97c2e291805528f079ed2ba Author: Age Manning Date: Tue Mar 14 16:25:45 2023 +1100 Revert old clippy change commit b3d0167344cffd9f943b571576da6360e8e77261 Author: Age Manning Date: Tue Mar 14 14:45:31 2023 +1100 Appease clippy commit 1df3191a4397ebaf9838dcd2e7ac900eaee30f88 Author: Age Manning Date: Tue Mar 14 14:08:28 2023 +1100 Fix tests commit fc7645e7bf44cb2fcf3d8c7b791eba343b6b087b Author: Age Manning Date: Tue Mar 14 13:43:52 2023 +1100 Fix logging tests via tokio upgrade commit 4988a157e7fc44c5906b75b49b8db42110d00023 Merge: 6cf93387c 90cef1db8 Author: Age Manning Date: Mon Mar 13 15:47:26 2023 +1100 Merge latest unstable commit 6cf93387c3d3ac9dd2e56f7c032c39b43f050a2a Merge: 5deabe0b7 3ad77fb52 Author: Age Manning Date: Mon Mar 13 15:39:51 2023 +1100 Merge latest unstable commit 5deabe0b72573ac1307a00c73d59404505787b9b Author: Age Manning Date: Mon Mar 13 15:39:45 2023 +1100 fmt commit 5f24f65f2e157c821c4cc407977806163856ae3c Author: Age Manning Date: Mon Mar 13 15:38:36 2023 +1100 A working version commit 4da3351ec2ae2c8d7a0d267eb456418062f53950 Author: Age Manning Date: Thu Mar 9 13:44:22 2023 +1100 Tokio broadcast version commit e34aab2034b04659dc8977893ed55cd5df92b96f Merge: cd6edbc5f 9d821ce15 Author: Age Manning Date: Thu Mar 9 13:17:38 2023 +1100 Merge pull request #3 from michaelsproul/sse-logs Workaround format_args! with closure commit 9d821ce15b237f585314fbdef1409b99e3995755 Author: Michael Sproul Date: Thu Mar 9 11:57:51 2023 +1100 Workaround format_args! with closure commit cd6edbc5fa8f06a89380ccbb7e2c643a443a1a84 Author: Age Manning Date: Thu Mar 9 11:18:02 2023 +1100 Getting closer commit e9eb455d75d991ac613b66045fbfd91366a16d7c Author: Age Manning Date: Wed Mar 8 16:38:58 2023 +1100 Crossbeam version commit b688cfba5b3265f96cbb959781d99e24e3441abb Author: Age Manning Date: Tue Mar 7 14:56:17 2023 +1100 More temp commits commit 228b9301e70f26384360f39d9e1aa8c4316ba4e9 Merge: 035ca3764 17d9a620c Author: Age Manning Date: Wed Mar 1 13:05:45 2023 +1100 Merge latest unstable commit 035ca376404c95339ce608acfd0fcb36e8d4f7a2 Author: Age Manning Date: Wed Feb 8 20:24:39 2023 +1100 Temp progress --- Cargo.lock | 428 +++++++++++-------- beacon_node/client/Cargo.toml | 2 +- beacon_node/client/src/builder.rs | 2 + beacon_node/http_api/Cargo.toml | 4 +- beacon_node/http_api/src/lib.rs | 44 ++ beacon_node/http_api/src/test_utils.rs | 1 + beacon_node/src/cli.rs | 2 +- book/src/api-lighthouse.md | 28 ++ book/src/api-vc-endpoints.md | 30 ++ common/logging/Cargo.toml | 7 + common/logging/src/async_record.rs | 309 +++++++++++++ common/logging/src/lib.rs | 4 + common/logging/src/sse_logging_components.rs | 46 ++ lcli/src/main.rs | 1 + lighthouse/environment/src/lib.rs | 96 +++-- lighthouse/src/main.rs | 11 + testing/simulator/src/eth1_sim.rs | 1 + testing/simulator/src/no_eth1_sim.rs | 1 + testing/simulator/src/sync_sim.rs | 1 + validator_client/Cargo.toml | 2 + validator_client/src/http_api/mod.rs | 55 ++- validator_client/src/http_api/tests.rs | 3 +- validator_client/src/lib.rs | 1 + 23 files changed, 863 insertions(+), 216 deletions(-) create mode 100644 common/logging/src/async_record.rs create mode 100644 common/logging/src/sse_logging_components.rs diff --git a/Cargo.lock b/Cargo.lock index d6e21d4d221..3487ac150e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,16 +197,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.8", + "getrandom 0.2.9", "once_cell", "version_check", ] [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -236,9 +236,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "arbitrary" @@ -335,9 +335,9 @@ dependencies = [ [[package]] name = "asn1_der" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22d1f4b888c298a027c99dc9048015fac177587de20fc30232a057dfbe24a21" +checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" [[package]] name = "async-io" @@ -370,9 +370,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", @@ -381,13 +381,13 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -398,7 +398,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -427,9 +427,9 @@ dependencies = [ [[package]] name = "atomic-waker" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599" +checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" [[package]] name = "attohttpc" @@ -456,9 +456,9 @@ dependencies = [ [[package]] name = "auto_impl" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8c1df849285fbacd587de7818cc7d13be6cd2cbcd47a04fb1801b0e2706e33" +checksum = "fee3da8ef1276b0bee5dd1c7258010d8fffd31801447323115a25560e1327b89" dependencies = [ "proc-macro-error", "proc-macro2", @@ -538,7 +538,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.6.2", "object", "rustc-demangle", ] @@ -883,9 +883,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" [[package]] name = "byte-slice-cast" @@ -1202,9 +1202,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" dependencies = [ "crossbeam-utils", ] @@ -1248,9 +1248,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -1317,9 +1317,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1507,7 +1507,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -1524,7 +1524,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -1674,7 +1674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4355c25cbf99edcb6b4a0e906f6bdc6956eda149e84455bea49696429b2f8e8" dependencies = [ "futures", - "tokio-util 0.7.7", + "tokio-util 0.7.8", ] [[package]] @@ -1797,9 +1797,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4391a22b19c916e50bec4d6140f29bdda3e3bb187223fe6e3ea0b6e4d1021c04" +checksum = "72eb77396836a4505da85bae0712fa324b74acfe1876d7c2f7e694ef3d0ee373" dependencies = [ "bitflags", "byteorder", @@ -2077,7 +2077,7 @@ dependencies = [ "rand 0.8.5", "rlp", "serde", - "sha3 0.10.6", + "sha3 0.10.7", "zeroize", ] @@ -2097,7 +2097,7 @@ dependencies = [ "rand 0.8.5", "rlp", "serde", - "sha3 0.10.6", + "sha3 0.10.7", "zeroize", ] @@ -2160,13 +2160,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2385,7 +2385,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sha3 0.10.6", + "sha3 0.10.7", "thiserror", "uint", ] @@ -2549,7 +2549,7 @@ dependencies = [ "dunce", "ethers-core", "eyre", - "getrandom 0.2.8", + "getrandom 0.2.9", "hex", "proc-macro2", "quote", @@ -2621,7 +2621,7 @@ dependencies = [ "futures-core", "futures-timer", "futures-util", - "getrandom 0.2.8", + "getrandom 0.2.9", "hashers", "hex", "http", @@ -2837,13 +2837,13 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.7.1", ] [[package]] @@ -2964,9 +2964,9 @@ checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-lite" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ "fastrand", "futures-core", @@ -2985,7 +2985,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -3090,9 +3090,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "js-sys", @@ -3181,7 +3181,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.7.7", + "tokio-util 0.7.8", "tracing", ] @@ -3493,9 +3493,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ "bytes", "futures-channel", @@ -3543,16 +3543,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.54" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows 0.46.0", + "windows 0.48.0", ] [[package]] @@ -3787,13 +3787,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" dependencies = [ "hermit-abi 0.3.1", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -3893,7 +3893,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "sha2 0.10.6", - "sha3 0.10.6", + "sha3 0.10.7", ] [[package]] @@ -3993,9 +3993,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.140" +version = "0.2.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" [[package]] name = "libflate" @@ -4063,7 +4063,7 @@ dependencies = [ "bytes", "futures", "futures-timer", - "getrandom 0.2.8", + "getrandom 0.2.9", "instant", "libp2p-core 0.38.0", "libp2p-dns", @@ -4159,9 +4159,9 @@ dependencies = [ [[package]] name = "libp2p-core" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7f8b7d65c070a5a1b5f8f0510648189da08f787b8963f8e21219e0710733af" +checksum = "3c1df63c0b582aa434fb09b2d86897fa2b419ffeccf934b36f87fcedc8e835c2" dependencies = [ "either", "fnv", @@ -4252,18 +4252,18 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8ea433ae0cea7e3315354305237b9897afe45278b2118a7a57ca744e70fd27" +checksum = "9e2d584751cecb2aabaa56106be6be91338a60a0f4e420cf2af639204f596fc1" dependencies = [ "bs58", "ed25519-dalek", "log", "multiaddr 0.17.1", "multihash 0.17.0", - "prost", "quick-protobuf", "rand 0.8.5", + "sha2 0.10.6", "thiserror", "zeroize", ] @@ -4437,7 +4437,7 @@ checksum = "ff08d13d0dc66e5e9ba6279c1de417b84fa0d0adc3b03e5732928c180ec02781" dependencies = [ "futures", "futures-rustls", - "libp2p-core 0.39.1", + "libp2p-core 0.39.2", "libp2p-identity", "rcgen 0.10.0", "ring", @@ -4475,7 +4475,7 @@ dependencies = [ "thiserror", "tinytemplate", "tokio", - "tokio-util 0.7.7", + "tokio-util 0.7.8", "webrtc", ] @@ -4573,9 +4573,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" dependencies = [ "cc", "pkg-config", @@ -4708,9 +4708,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" [[package]] name = "lmdb-rkv" @@ -4764,11 +4764,18 @@ dependencies = [ name = "logging" version = "0.2.0" dependencies = [ + "chrono", "lazy_static", "lighthouse_metrics", + "parking_lot 0.12.1", + "serde", + "serde_json", "slog", + "slog-async", "slog-term", "sloggers", + "take_mut", + "tokio", ] [[package]] @@ -5019,6 +5026,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.6" @@ -5143,9 +5159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" dependencies = [ "core2", - "digest 0.10.6", "multihash-derive", - "sha2 0.10.6", "unsigned-varint 0.7.1", ] @@ -5421,9 +5435,9 @@ dependencies = [ [[package]] name = "ntapi" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" dependencies = [ "winapi", ] @@ -5596,9 +5610,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.49" +version = "0.10.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2f106ab837a24e03672c59b1239669a0596406ff657c3c0835b6b7f0f35a33" +checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" dependencies = [ "bitflags", "cfg-if", @@ -5617,7 +5631,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -5628,18 +5642,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.25.2+1.1.1t" +version = "111.25.3+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320708a054ad9b3bf314688b5db87cf4d6683d64cfc835e2337924ae62bf4431" +checksum = "924757a6a226bf60da5f7dd0311a34d2b52283dd82ddeb103208ddc66362f80c" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.84" +version = "0.9.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a20eace9dc2d82904039cb76dcf50fb1a0bba071cfd1629720b5d6f1ddba0fa" +checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" dependencies = [ "cc", "libc", @@ -5764,9 +5778,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" [[package]] name = "parking_lot" @@ -6004,9 +6018,9 @@ dependencies = [ [[package]] name = "polling" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e1f879b2998099c2d69ab9605d145d5b661195627eccc680002c4918a7fb6fa" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg 1.1.0", "bitflags", @@ -6015,7 +6029,7 @@ dependencies = [ "libc", "log", "pin-project-lite 0.2.9", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -6090,9 +6104,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pq-sys" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b845d6d8ec554f972a2c5298aad68953fd64e7441e846075450b44656a016d1" +checksum = "31c0052426df997c0cbd30789eb44ca097e3541717a7b8fa36b1c464ee7edebd" dependencies = [ "vcpkg", ] @@ -6176,9 +6190,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.55" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d0dd4be24fcdcfeaa12a432d588dc59bbad6cad3510c67e74a2b6b2fc950564" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" dependencies = [ "unicode-ident", ] @@ -6235,9 +6249,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", "prost-derive", @@ -6245,9 +6259,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", "heck", @@ -6280,9 +6294,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools", @@ -6293,9 +6307,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ "prost", ] @@ -6507,7 +6521,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.8", + "getrandom 0.2.9", ] [[package]] @@ -6599,20 +6613,20 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.8", + "getrandom 0.2.9", "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.1", ] [[package]] @@ -6621,7 +6635,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -6630,11 +6644,17 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" dependencies = [ "base64 0.21.0", "bytes", @@ -6663,7 +6683,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls 0.23.4", - "tokio-util 0.7.7", + "tokio-util 0.7.8", "tower-service", "url", "wasm-bindgen", @@ -6803,9 +6823,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a36c42d1873f9a77c53bde094f9664d9891bc604a45b4798fd2c389ed12e5b" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustc-hash" @@ -6848,16 +6868,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.6" +version = "0.37.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" +checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -6947,9 +6967,9 @@ dependencies = [ [[package]] name = "scale-info" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cfdffd972d76b22f3d7f81c8be34b2296afd3a25e0a547bd9abe340a4dbbe97" +checksum = "dfdef77228a4c05dc94211441595746732131ad7f6530c6c18f045da7b7ab937" dependencies = [ "cfg-if", "derive_more", @@ -6959,9 +6979,9 @@ dependencies = [ [[package]] name = "scale-info-derive" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fa974aea2d63dd18a4ec3a49d59af9f34178c73a4f56d2f18205628d00681e" +checksum = "53012eae69e5aa5c14671942a5dd47de59d4cdcff8532a6dd0e081faf1119482" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -7126,9 +7146,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" dependencies = [ "serde_derive", ] @@ -7155,20 +7175,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] name = "serde_json" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -7183,7 +7203,7 @@ checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -7305,9 +7325,9 @@ dependencies = [ [[package]] name = "sha3" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf0c33fae925bdc080598b84bc15c55e7b9a4a43b3c704da051f977469691c9" +checksum = "54c2bb1a323307527314a36bfb73f24febb08ce2b8a554bf4ffd6f51ad15198c" dependencies = [ "digest 0.10.6", "keccak", @@ -7611,12 +7631,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc8d618c6641ae355025c449427f9e96b98abf99a772be3cef6708d15c77147a" +checksum = "6d283f86695ae989d1e18440a943880967156325ba025f05049946bff47bcc2b" dependencies = [ "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -7891,9 +7911,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.13" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" dependencies = [ "proc-macro2", "quote", @@ -8097,7 +8117,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -8225,9 +8245,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" dependencies = [ "autocfg 1.1.0", "bytes", @@ -8239,7 +8259,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.4.9", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -8254,13 +8274,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -8292,9 +8312,9 @@ dependencies = [ "pin-project-lite 0.2.9", "postgres-protocol", "postgres-types", - "socket2 0.5.1", + "socket2 0.5.2", "tokio", - "tokio-util 0.7.7", + "tokio-util 0.7.8", ] [[package]] @@ -8321,14 +8341,14 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite 0.2.9", "tokio", - "tokio-util 0.7.7", + "tokio-util 0.7.8", ] [[package]] @@ -8378,9 +8398,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -8463,13 +8483,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -8505,9 +8525,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", @@ -8888,17 +8908,17 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.8", + "getrandom 0.2.9", "serde", ] [[package]] name = "uuid" -version = "1.3.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" dependencies = [ - "getrandom 0.2.8", + "getrandom 0.2.9", ] [[package]] @@ -8948,6 +8968,7 @@ dependencies = [ "task_executor", "tempfile", "tokio", + "tokio-stream", "tree_hash", "types", "url", @@ -9412,7 +9433,7 @@ dependencies = [ "tokio", "turn", "url", - "uuid 1.3.0", + "uuid 1.3.2", "waitgroup", "webrtc-mdns", "webrtc-util", @@ -9433,18 +9454,15 @@ dependencies = [ [[package]] name = "webrtc-media" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a3c157a040324e5049bcbd644ffc9079e6738fa2cfab2bcff64e5cc4c00d7" +checksum = "f72e1650a8ae006017d1a5280efb49e2610c19ccc3c0905b03b648aee9554991" dependencies = [ "byteorder", "bytes", - "derive_builder", - "displaydoc", "rand 0.8.5", "rtp", "thiserror", - "webrtc-util", ] [[package]] @@ -9584,11 +9602,11 @@ dependencies = [ [[package]] name = "windows" -version = "0.46.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", ] [[package]] @@ -9609,12 +9627,12 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", + "windows_aarch64_gnullvm 0.42.2", "windows_aarch64_msvc 0.42.2", "windows_i686_gnu 0.42.2", "windows_i686_msvc 0.42.2", "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm", + "windows_x86_64_gnullvm 0.42.2", "windows_x86_64_msvc 0.42.2", ] @@ -9624,7 +9642,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", ] [[package]] @@ -9633,21 +9660,42 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm", + "windows_aarch64_gnullvm 0.42.2", "windows_aarch64_msvc 0.42.2", "windows_i686_gnu 0.42.2", "windows_i686_msvc 0.42.2", "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm", + "windows_x86_64_gnullvm 0.42.2", "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.34.0" @@ -9660,6 +9708,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.34.0" @@ -9672,6 +9726,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.34.0" @@ -9684,6 +9744,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.34.0" @@ -9696,12 +9762,24 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.34.0" @@ -9714,6 +9792,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "winreg" version = "0.10.1" @@ -9856,9 +9940,9 @@ dependencies = [ [[package]] name = "yasna" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ "time 0.3.20", ] @@ -9880,7 +9964,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 876458eea52..d39bb2e3e28 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -6,7 +6,6 @@ edition = "2021" [dev-dependencies] serde_yaml = "0.8.13" -logging = { path = "../../common/logging" } state_processing = { path = "../../consensus/state_processing" } operation_pool = { path = "../operation_pool" } tokio = "1.14.0" @@ -17,6 +16,7 @@ store = { path = "../store" } network = { path = "../network" } timer = { path = "../timer" } lighthouse_network = { path = "../lighthouse_network" } +logging = { path = "../../common/logging" } parking_lot = "0.12.0" types = { path = "../../consensus/types" } eth2_config = { path = "../../common/eth2_config" } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 5ef1f28fb4b..215244b9bad 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -478,6 +478,7 @@ where network_globals: None, eth1_service: Some(genesis_service.eth1_service.clone()), log: context.log().clone(), + sse_logging_components: runtime_context.sse_logging_components.clone(), }); // Discard the error from the oneshot. @@ -698,6 +699,7 @@ where network_senders: self.network_senders.clone(), network_globals: self.network_globals.clone(), eth1_service: self.eth1_service.clone(), + sse_logging_components: runtime_context.sse_logging_components.clone(), log: log.clone(), }); diff --git a/beacon_node/http_api/Cargo.toml b/beacon_node/http_api/Cargo.toml index 8f253e2f245..2b117b26cef 100644 --- a/beacon_node/http_api/Cargo.toml +++ b/beacon_node/http_api/Cargo.toml @@ -36,11 +36,11 @@ tree_hash = "0.5.0" sysinfo = "0.26.5" system_health = { path = "../../common/system_health" } directory = { path = "../../common/directory" } +logging = { path = "../../common/logging" } ethereum_serde_utils = "0.5.0" operation_pool = { path = "../operation_pool" } sensitive_url = { path = "../../common/sensitive_url" } unused_port = {path = "../../common/unused_port"} -logging = { path = "../../common/logging" } store = { path = "../store" } [dev-dependencies] @@ -51,4 +51,4 @@ genesis = { path = "../genesis" } [[test]] name = "bn_http_api_tests" -path = "tests/main.rs" +path = "tests/main.rs" \ No newline at end of file diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index be1463f0c31..55e00bab340 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -36,6 +36,7 @@ use eth2::types::{ }; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; +use logging::SSELoggingComponents; use network::{NetworkMessage, NetworkSenders, ValidatorSubscriptionMessage}; use operation_pool::ReceivedPreCapella; use parking_lot::RwLock; @@ -108,6 +109,7 @@ pub struct Context { pub network_senders: Option>, pub network_globals: Option>>, pub eth1_service: Option, + pub sse_logging_components: Option, pub log: Logger, } @@ -448,6 +450,9 @@ pub fn serve( let inner_ctx = ctx.clone(); let log_filter = warp::any().map(move || inner_ctx.log.clone()); + let inner_components = ctx.sse_logging_components.clone(); + let sse_component_filter = warp::any().map(move || inner_components.clone()); + // Create a `warp` filter that provides access to local system information. let system_info = Arc::new(RwLock::new(sysinfo::System::new())); { @@ -3729,6 +3734,44 @@ pub fn serve( }, ); + // Subscribe to logs via Server Side Events + // /lighthouse/logs + let lighthouse_log_events = warp::path("lighthouse") + .and(warp::path("logs")) + .and(warp::path::end()) + .and(sse_component_filter) + .and_then(|sse_component: Option| { + blocking_response_task(move || { + if let Some(logging_components) = sse_component { + // Build a JSON stream + let s = + BroadcastStream::new(logging_components.sender.subscribe()).map(|msg| { + match msg { + Ok(data) => { + // Serialize to json + match data.to_json_string() { + // Send the json as a Server Side Event + Ok(json) => Ok(Event::default().data(json)), + Err(e) => Err(warp_utils::reject::server_sent_event_error( + format!("Unable to serialize to JSON {}", e), + )), + } + } + Err(e) => Err(warp_utils::reject::server_sent_event_error( + format!("Unable to receive event {}", e), + )), + } + }); + + Ok::<_, warp::Rejection>(warp::sse::reply(warp::sse::keep_alive().stream(s))) + } else { + Err(warp_utils::reject::custom_server_error( + "SSE Logging is not enabled".to_string(), + )) + } + }) + }); + // Define the ultimate set of routes that will be provided to the server. // Use `uor` rather than `or` in order to simplify types (see `UnifyingOrFilter`). let routes = warp::get() @@ -3796,6 +3839,7 @@ pub fn serve( .uor(get_lighthouse_block_packing_efficiency) .uor(get_lighthouse_merge_readiness) .uor(get_events) + .uor(lighthouse_log_events.boxed()) .recover(warp_utils::reject::handle_rejection), ) .boxed() diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index 8dc9be7dd43..9880a8ca617 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -198,6 +198,7 @@ pub async fn create_api_server_on_port( network_senders: Some(network_senders), network_globals: Some(network_globals), eth1_service: Some(eth1_service), + sse_logging_components: None, log, }); diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index b20b5c0a95e..59a5f4b2e0f 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1081,7 +1081,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("gui") .hidden(true) .help("Enable the graphical user interface and all its requirements. \ - This is equivalent to --http and --validator-monitor-auto.") + This enables --http and --validator-monitor-auto and enables SSE logging.") .takes_value(false) ) .arg( diff --git a/book/src/api-lighthouse.md b/book/src/api-lighthouse.md index e67a79c8f06..47fe62f50b0 100644 --- a/book/src/api-lighthouse.md +++ b/book/src/api-lighthouse.md @@ -679,3 +679,31 @@ Caveats: This is because the state _prior_ to the `start_epoch` needs to be loaded from the database, and loading a state on a boundary is most efficient. + +### `/lighthouse/logs` + +This is a Server Side Event subscription endpoint. This allows a user to read +the Lighthouse logs directly from the HTTP API endpoint. This currently +exposes INFO and higher level logs. It is only enabled when the `--gui` flag is set in the CLI. + +Example: + +```bash +curl -N "http://localhost:5052/lighthouse/logs" +``` + +Should provide an output that emits log events as they occur: +```json +{ +"data": { + "time": "Mar 13 15:28:41", + "level": "INFO", + "msg": "Syncing", + "service": "slot_notifier", + "est_time": "1 hr 27 mins", + "speed": "5.33 slots/sec", + "distance": "28141 slots (3 days 21 hrs)", + "peers": "8" + } +} +``` diff --git a/book/src/api-vc-endpoints.md b/book/src/api-vc-endpoints.md index 80a14ae7710..d5d76e4ef43 100644 --- a/book/src/api-vc-endpoints.md +++ b/book/src/api-vc-endpoints.md @@ -578,3 +578,33 @@ The following fields may be omitted or nullified to obtain default values: ### Example Response Body *No data is included in the response body.* + +## `GET /lighthouse/logs` + +Provides a subscription to receive logs as Server Side Events. Currently the +logs emitted are INFO level or higher. + +### HTTP Specification + +| Property | Specification | +|-------------------|--------------------------------------------| +| Path | `/lighthouse/logs` | +| Method | GET | +| Required Headers | None | +| Typical Responses | 200 | + +### Example Response Body + +```json +{ + "data": { + "time": "Mar 13 15:26:53", + "level": "INFO", + "msg": "Connected to beacon node(s)", + "service": "notifier", + "synced": 1, + "available": 1, + "total": 1 + } +} +``` \ No newline at end of file diff --git a/common/logging/Cargo.toml b/common/logging/Cargo.toml index e56a1a23583..b6179d9e782 100644 --- a/common/logging/Cargo.toml +++ b/common/logging/Cargo.toml @@ -10,6 +10,13 @@ test_logger = [] # Print log output to stderr when running tests instead of drop [dependencies] slog = "2.5.2" slog-term = "2.6.0" +tokio = { version = "1.26.0", features = ["sync"] } lighthouse_metrics = { path = "../lighthouse_metrics" } lazy_static = "1.4.0" sloggers = { version = "2.1.1", features = ["json"] } +slog-async = "2.7.0" +take_mut = "0.2.2" +parking_lot = "0.12.1" +serde = "1.0.153" +serde_json = "1.0.94" +chrono = "0.4.23" diff --git a/common/logging/src/async_record.rs b/common/logging/src/async_record.rs new file mode 100644 index 00000000000..6f998c61915 --- /dev/null +++ b/common/logging/src/async_record.rs @@ -0,0 +1,309 @@ +//! An object that can be used to pass through a channel and be cloned. It can therefore be used +//! via the broadcast channel. + +use parking_lot::Mutex; +use serde::ser::SerializeMap; +use serde::serde_if_integer128; +use serde::Serialize; +use slog::{BorrowedKV, Key, Level, OwnedKVList, Record, RecordStatic, Serializer, SingleKV, KV}; +use std::cell::RefCell; +use std::fmt; +use std::fmt::Write; +use std::sync::Arc; +use take_mut::take; + +thread_local! { + static TL_BUF: RefCell = RefCell::new(String::with_capacity(128)) +} + +/// Serialized record. +#[derive(Clone)] +pub struct AsyncRecord { + msg: String, + level: Level, + location: Box, + tag: String, + logger_values: OwnedKVList, + kv: Arc>, +} + +impl AsyncRecord { + /// Serializes a `Record` and an `OwnedKVList`. + pub fn from(record: &Record, logger_values: &OwnedKVList) -> Self { + let mut ser = ToSendSerializer::new(); + record + .kv() + .serialize(record, &mut ser) + .expect("`ToSendSerializer` can't fail"); + + AsyncRecord { + msg: fmt::format(*record.msg()), + level: record.level(), + location: Box::new(*record.location()), + tag: String::from(record.tag()), + logger_values: logger_values.clone(), + kv: Arc::new(Mutex::new(ser.finish())), + } + } + + pub fn to_json_string(&self) -> Result { + serde_json::to_string(&self).map_err(|e| format!("{:?}", e)) + } +} + +pub struct ToSendSerializer { + kv: Box, +} + +impl ToSendSerializer { + fn new() -> Self { + ToSendSerializer { kv: Box::new(()) } + } + + fn finish(self) -> Box { + self.kv + } +} + +impl Serializer for ToSendSerializer { + fn emit_bool(&mut self, key: Key, val: bool) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_unit(&mut self, key: Key) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, ())))); + Ok(()) + } + fn emit_none(&mut self, key: Key) -> slog::Result { + let val: Option<()> = None; + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_char(&mut self, key: Key, val: char) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_u8(&mut self, key: Key, val: u8) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_i8(&mut self, key: Key, val: i8) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_u16(&mut self, key: Key, val: u16) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_i16(&mut self, key: Key, val: i16) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_u32(&mut self, key: Key, val: u32) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_i32(&mut self, key: Key, val: i32) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_f32(&mut self, key: Key, val: f32) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_u64(&mut self, key: Key, val: u64) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_i64(&mut self, key: Key, val: i64) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_f64(&mut self, key: Key, val: f64) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + #[cfg(integer128)] + fn emit_u128(&mut self, key: Key, val: u128) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + #[cfg(integer128)] + fn emit_i128(&mut self, key: Key, val: i128) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_usize(&mut self, key: Key, val: usize) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_isize(&mut self, key: Key, val: isize) -> slog::Result { + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_str(&mut self, key: Key, val: &str) -> slog::Result { + let val = val.to_owned(); + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } + fn emit_arguments(&mut self, key: Key, val: &fmt::Arguments) -> slog::Result { + let val = fmt::format(*val); + take(&mut self.kv, |kv| Box::new((kv, SingleKV(key, val)))); + Ok(()) + } +} + +impl Serialize for AsyncRecord { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + // Get the current time + let dt = chrono::Local::now().format("%b %e %T").to_string(); + + let rs = RecordStatic { + location: &self.location, + level: self.level, + tag: &self.tag, + }; + let mut map_serializer = SerdeSerializer::new(serializer)?; + + // Serialize the time and log level first + map_serializer.serialize_entry("time", &dt)?; + map_serializer.serialize_entry("level", self.level.as_short_str())?; + + let kv = self.kv.lock(); + + // Convoluted pattern to avoid binding `format_args!` to a temporary. + // See: https://stackoverflow.com/questions/56304313/cannot-use-format-args-due-to-temporary-value-is-freed-at-the-end-of-this-state + let mut f = |msg: std::fmt::Arguments| { + map_serializer.serialize_entry("msg", &msg.to_string())?; + + let record = Record::new(&rs, &msg, BorrowedKV(&(*kv))); + self.logger_values + .serialize(&record, &mut map_serializer) + .map_err(serde::ser::Error::custom)?; + record + .kv() + .serialize(&record, &mut map_serializer) + .map_err(serde::ser::Error::custom) + }; + f(format_args!("{}", self.msg))?; + map_serializer.end() + } +} + +struct SerdeSerializer { + /// Current state of map serializing: `serde::Serializer::MapState` + ser_map: S::SerializeMap, +} + +impl SerdeSerializer { + fn new(ser: S) -> Result { + let ser_map = ser.serialize_map(None)?; + Ok(SerdeSerializer { ser_map }) + } + + fn serialize_entry(&mut self, key: K, value: V) -> Result<(), S::Error> + where + K: serde::Serialize, + V: serde::Serialize, + { + self.ser_map.serialize_entry(&key, &value) + } + + /// Finish serialization, and return the serializer + fn end(self) -> Result { + self.ser_map.end() + } +} + +// NOTE: This is borrowed from slog_json +macro_rules! impl_m( + ($s:expr, $key:expr, $val:expr) => ({ + let k_s: &str = $key.as_ref(); + $s.ser_map.serialize_entry(k_s, $val) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("serde serialization error: {}", e)))?; + Ok(()) + }); +); + +impl slog::Serializer for SerdeSerializer +where + S: serde::Serializer, +{ + fn emit_bool(&mut self, key: Key, val: bool) -> slog::Result { + impl_m!(self, key, &val) + } + + fn emit_unit(&mut self, key: Key) -> slog::Result { + impl_m!(self, key, &()) + } + + fn emit_char(&mut self, key: Key, val: char) -> slog::Result { + impl_m!(self, key, &val) + } + + fn emit_none(&mut self, key: Key) -> slog::Result { + let val: Option<()> = None; + impl_m!(self, key, &val) + } + fn emit_u8(&mut self, key: Key, val: u8) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_i8(&mut self, key: Key, val: i8) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_u16(&mut self, key: Key, val: u16) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_i16(&mut self, key: Key, val: i16) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_usize(&mut self, key: Key, val: usize) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_isize(&mut self, key: Key, val: isize) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_u32(&mut self, key: Key, val: u32) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_i32(&mut self, key: Key, val: i32) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_f32(&mut self, key: Key, val: f32) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_u64(&mut self, key: Key, val: u64) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_i64(&mut self, key: Key, val: i64) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_f64(&mut self, key: Key, val: f64) -> slog::Result { + impl_m!(self, key, &val) + } + serde_if_integer128! { + fn emit_u128(&mut self, key: Key, val: u128) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_i128(&mut self, key: Key, val: i128) -> slog::Result { + impl_m!(self, key, &val) + } + } + fn emit_str(&mut self, key: Key, val: &str) -> slog::Result { + impl_m!(self, key, &val) + } + fn emit_arguments(&mut self, key: Key, val: &fmt::Arguments) -> slog::Result { + TL_BUF.with(|buf| { + let mut buf = buf.borrow_mut(); + + buf.write_fmt(*val).unwrap(); + + let res = { || impl_m!(self, key, &*buf) }(); + buf.clear(); + res + }) + } +} diff --git a/common/logging/src/lib.rs b/common/logging/src/lib.rs index 85c42557445..a9ad25f3f3e 100644 --- a/common/logging/src/lib.rs +++ b/common/logging/src/lib.rs @@ -11,6 +11,10 @@ use std::time::{Duration, Instant}; pub const MAX_MESSAGE_WIDTH: usize = 40; +pub mod async_record; +mod sse_logging_components; +pub use sse_logging_components::SSELoggingComponents; + /// The minimum interval between log messages indicating that a queue is full. const LOG_DEBOUNCE_INTERVAL: Duration = Duration::from_secs(30); diff --git a/common/logging/src/sse_logging_components.rs b/common/logging/src/sse_logging_components.rs new file mode 100644 index 00000000000..244d09fbd12 --- /dev/null +++ b/common/logging/src/sse_logging_components.rs @@ -0,0 +1,46 @@ +//! This module provides an implementation of `slog::Drain` that optionally writes to a channel if +//! there are subscribers to a HTTP SSE stream. + +use crate::async_record::AsyncRecord; +use slog::{Drain, OwnedKVList, Record}; +use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use tokio::sync::broadcast::Sender; + +/// Default log level for SSE Events. +// NOTE: Made this a constant. Debug level seems to be pretty intense. Can make this +// configurable later if needed. +const LOG_LEVEL: slog::Level = slog::Level::Info; + +/// The components required in the HTTP API task to receive logged events. +#[derive(Clone)] +pub struct SSELoggingComponents { + /// The channel to receive events from. + pub sender: Arc>>, +} + +impl SSELoggingComponents { + /// Create a new SSE drain. + pub fn new(channel_size: usize) -> Self { + let (sender, _receiver) = tokio::sync::broadcast::channel(channel_size); + + let sender = Arc::new(AssertUnwindSafe(sender)); + SSELoggingComponents { sender } + } +} + +impl Drain for SSELoggingComponents { + type Ok = (); + type Err = &'static str; + + fn log(&self, record: &Record, logger_values: &OwnedKVList) -> Result { + if record.level().is_at_least(LOG_LEVEL) { + // Attempt to send the logs + match self.sender.send(AsyncRecord::from(record, logger_values)) { + Ok(_num_sent) => {} // Everything got sent + Err(_err) => {} // There are no subscribers, do nothing + } + } + Ok(()) + } +} diff --git a/lcli/src/main.rs b/lcli/src/main.rs index bc39c34e262..d072beaa4e1 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -881,6 +881,7 @@ fn run( max_log_number: 0, compression: false, is_restricted: true, + sse_logging: false, // No SSE Logging in LCLI }) .map_err(|e| format!("should start logger: {:?}", e))? .build() diff --git a/lighthouse/environment/src/lib.rs b/lighthouse/environment/src/lib.rs index 8ef67e82ddb..53915b52d96 100644 --- a/lighthouse/environment/src/lib.rs +++ b/lighthouse/environment/src/lib.rs @@ -12,6 +12,7 @@ use eth2_network_config::Eth2NetworkConfig; use futures::channel::mpsc::{channel, Receiver, Sender}; use futures::{future, StreamExt}; +use logging::SSELoggingComponents; use serde_derive::{Deserialize, Serialize}; use slog::{error, info, o, warn, Drain, Duplicate, Level, Logger}; use sloggers::{file::FileLoggerBuilder, types::Format, types::Severity, Build}; @@ -36,6 +37,7 @@ use {futures::channel::oneshot, std::cell::RefCell}; pub use task_executor::test_utils::null_logger; const LOG_CHANNEL_SIZE: usize = 2048; +const SSE_LOG_CHANNEL_SIZE: usize = 2048; /// The maximum time in seconds the client will wait for all internal tasks to shutdown. const MAXIMUM_SHUTDOWN_TIME: u64 = 15; @@ -57,6 +59,7 @@ pub struct LoggerConfig { pub max_log_number: usize, pub compression: bool, pub is_restricted: bool, + pub sse_logging: bool, } impl Default for LoggerConfig { fn default() -> Self { @@ -72,14 +75,54 @@ impl Default for LoggerConfig { max_log_number: 5, compression: false, is_restricted: true, + sse_logging: false, } } } +/// An execution context that can be used by a service. +/// +/// Distinct from an `Environment` because a `Context` is not able to give a mutable reference to a +/// `Runtime`, instead it only has access to a `Runtime`. +#[derive(Clone)] +pub struct RuntimeContext { + pub executor: TaskExecutor, + pub eth_spec_instance: E, + pub eth2_config: Eth2Config, + pub eth2_network_config: Option>, + pub sse_logging_components: Option, +} + +impl RuntimeContext { + /// Returns a sub-context of this context. + /// + /// The generated service will have the `service_name` in all it's logs. + pub fn service_context(&self, service_name: String) -> Self { + Self { + executor: self.executor.clone_with_name(service_name), + eth_spec_instance: self.eth_spec_instance.clone(), + eth2_config: self.eth2_config.clone(), + eth2_network_config: self.eth2_network_config.clone(), + sse_logging_components: self.sse_logging_components.clone(), + } + } + + /// Returns the `eth2_config` for this service. + pub fn eth2_config(&self) -> &Eth2Config { + &self.eth2_config + } + + /// Returns a reference to the logger for this service. + pub fn log(&self) -> &slog::Logger { + self.executor.log() + } +} + /// Builds an `Environment`. pub struct EnvironmentBuilder { runtime: Option>, log: Option, + sse_logging_components: Option, eth_spec_instance: E, eth2_config: Eth2Config, eth2_network_config: Option, @@ -91,6 +134,7 @@ impl EnvironmentBuilder { Self { runtime: None, log: None, + sse_logging_components: None, eth_spec_instance: MinimalEthSpec, eth2_config: Eth2Config::minimal(), eth2_network_config: None, @@ -104,6 +148,7 @@ impl EnvironmentBuilder { Self { runtime: None, log: None, + sse_logging_components: None, eth_spec_instance: MainnetEthSpec, eth2_config: Eth2Config::mainnet(), eth2_network_config: None, @@ -117,6 +162,7 @@ impl EnvironmentBuilder { Self { runtime: None, log: None, + sse_logging_components: None, eth_spec_instance: GnosisEthSpec, eth2_config: Eth2Config::gnosis(), eth2_network_config: None, @@ -265,7 +311,7 @@ impl EnvironmentBuilder { .build() .map_err(|e| format!("Unable to build file logger: {}", e))?; - let log = Logger::root(Duplicate::new(stdout_logger, file_logger).fuse(), o!()); + let mut log = Logger::root(Duplicate::new(stdout_logger, file_logger).fuse(), o!()); info!( log, @@ -273,6 +319,14 @@ impl EnvironmentBuilder { "path" => format!("{:?}", path) ); + // If the http API is enabled, we may need to send logs to be consumed by subscribers. + if config.sse_logging { + let sse_logger = SSELoggingComponents::new(SSE_LOG_CHANNEL_SIZE); + self.sse_logging_components = Some(sse_logger.clone()); + + log = Logger::root(Duplicate::new(log, sse_logger).fuse(), o!()); + } + self.log = Some(log); Ok(self) @@ -315,6 +369,7 @@ impl EnvironmentBuilder { signal: Some(signal), exit, log: self.log.ok_or("Cannot build environment without log")?, + sse_logging_components: self.sse_logging_components, eth_spec_instance: self.eth_spec_instance, eth2_config: self.eth2_config, eth2_network_config: self.eth2_network_config.map(Arc::new), @@ -322,42 +377,6 @@ impl EnvironmentBuilder { } } -/// An execution context that can be used by a service. -/// -/// Distinct from an `Environment` because a `Context` is not able to give a mutable reference to a -/// `Runtime`, instead it only has access to a `Runtime`. -#[derive(Clone)] -pub struct RuntimeContext { - pub executor: TaskExecutor, - pub eth_spec_instance: E, - pub eth2_config: Eth2Config, - pub eth2_network_config: Option>, -} - -impl RuntimeContext { - /// Returns a sub-context of this context. - /// - /// The generated service will have the `service_name` in all it's logs. - pub fn service_context(&self, service_name: String) -> Self { - Self { - executor: self.executor.clone_with_name(service_name), - eth_spec_instance: self.eth_spec_instance.clone(), - eth2_config: self.eth2_config.clone(), - eth2_network_config: self.eth2_network_config.clone(), - } - } - - /// Returns the `eth2_config` for this service. - pub fn eth2_config(&self) -> &Eth2Config { - &self.eth2_config - } - - /// Returns a reference to the logger for this service. - pub fn log(&self) -> &slog::Logger { - self.executor.log() - } -} - /// An environment where Lighthouse services can run. Used to start a production beacon node or /// validator client, or to run tests that involve logging and async task execution. pub struct Environment { @@ -369,6 +388,7 @@ pub struct Environment { signal: Option, exit: exit_future::Exit, log: Logger, + sse_logging_components: Option, eth_spec_instance: E, pub eth2_config: Eth2Config, pub eth2_network_config: Option>, @@ -395,6 +415,7 @@ impl Environment { eth_spec_instance: self.eth_spec_instance.clone(), eth2_config: self.eth2_config.clone(), eth2_network_config: self.eth2_network_config.clone(), + sse_logging_components: self.sse_logging_components.clone(), } } @@ -410,6 +431,7 @@ impl Environment { eth_spec_instance: self.eth_spec_instance.clone(), eth2_config: self.eth2_config.clone(), eth2_network_config: self.eth2_network_config.clone(), + sse_logging_components: self.sse_logging_components.clone(), } } diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index f55e39bfb7a..b814639ceb0 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -483,6 +483,16 @@ fn run( }; } + let sse_logging = { + if let Some(bn_matches) = matches.subcommand_matches("beacon_node") { + bn_matches.is_present("gui") + } else if let Some(vc_matches) = matches.subcommand_matches("validator_client") { + vc_matches.is_present("http") + } else { + false + } + }; + let logger_config = LoggerConfig { path: log_path, debug_level: String::from(debug_level), @@ -495,6 +505,7 @@ fn run( max_log_number: logfile_max_number, compression: logfile_compress, is_restricted: logfile_restricted, + sse_logging, }; let builder = environment_builder.initialize_logger(logger_config.clone())?; diff --git a/testing/simulator/src/eth1_sim.rs b/testing/simulator/src/eth1_sim.rs index a5462da396f..3e764d27d02 100644 --- a/testing/simulator/src/eth1_sim.rs +++ b/testing/simulator/src/eth1_sim.rs @@ -72,6 +72,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { max_log_number: 0, compression: false, is_restricted: true, + sse_logging: false, })? .multi_threaded_tokio_runtime()? .build()?; diff --git a/testing/simulator/src/no_eth1_sim.rs b/testing/simulator/src/no_eth1_sim.rs index b7598f9fa74..fc18b1cd489 100644 --- a/testing/simulator/src/no_eth1_sim.rs +++ b/testing/simulator/src/no_eth1_sim.rs @@ -54,6 +54,7 @@ pub fn run_no_eth1_sim(matches: &ArgMatches) -> Result<(), String> { max_log_number: 0, compression: false, is_restricted: true, + sse_logging: false, })? .multi_threaded_tokio_runtime()? .build()?; diff --git a/testing/simulator/src/sync_sim.rs b/testing/simulator/src/sync_sim.rs index 5eaed809df5..78f7e1ee9fb 100644 --- a/testing/simulator/src/sync_sim.rs +++ b/testing/simulator/src/sync_sim.rs @@ -58,6 +58,7 @@ fn syncing_sim( max_log_number: 0, compression: false, is_restricted: true, + sse_logging: false, })? .multi_threaded_tokio_runtime()? .build()?; diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index e0172afd2a1..494ebcb3dfc 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -25,6 +25,7 @@ bincode = "1.3.1" serde_json = "1.0.58" slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] } tokio = { version = "1.14.0", features = ["time"] } +tokio-stream = { version = "0.1.3", features = ["sync"] } futures = "0.3.7" dirs = "3.0.1" directory = { path = "../common/directory" } @@ -61,4 +62,5 @@ url = "2.2.2" malloc_utils = { path = "../common/malloc_utils" } sysinfo = "0.26.5" system_health = { path = "../common/system_health" } +logging = { path = "../common/logging" } diff --git a/validator_client/src/http_api/mod.rs b/validator_client/src/http_api/mod.rs index 15b3f9fe09e..fa6cde3ed54 100644 --- a/validator_client/src/http_api/mod.rs +++ b/validator_client/src/http_api/mod.rs @@ -18,6 +18,7 @@ use eth2::lighthouse_vc::{ types::{self as api_types, GenericResponse, Graffiti, PublicKey, PublicKeyBytes}, }; use lighthouse_version::version_with_platform; +use logging::SSELoggingComponents; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use slog::{crit, info, warn, Logger}; @@ -31,6 +32,7 @@ use std::sync::Arc; use sysinfo::{System, SystemExt}; use system_health::observe_system_health_vc; use task_executor::TaskExecutor; +use tokio_stream::{wrappers::BroadcastStream, StreamExt}; use types::{ChainSpec, ConfigAndPreset, EthSpec}; use validator_dir::Builder as ValidatorDirBuilder; use warp::{ @@ -39,6 +41,7 @@ use warp::{ response::Response, StatusCode, }, + sse::Event, Filter, }; @@ -73,6 +76,7 @@ pub struct Context { pub spec: ChainSpec, pub config: Config, pub log: Logger, + pub sse_logging_components: Option, pub slot_clock: T, pub _phantom: PhantomData, } @@ -201,6 +205,10 @@ pub fn serve( let api_token_path_inner = api_token_path.clone(); let api_token_path_filter = warp::any().map(move || api_token_path_inner.clone()); + // Filter for SEE Logging events + let inner_components = ctx.sse_logging_components.clone(); + let sse_component_filter = warp::any().map(move || inner_components.clone()); + // Create a `warp` filter that provides access to local system information. let system_info = Arc::new(RwLock::new(sysinfo::System::new())); { @@ -1021,6 +1029,49 @@ pub fn serve( }) }); + // Subscribe to get VC logs via Server side events + // /lighthouse/logs + let get_log_events = warp::path("lighthouse") + .and(warp::path("logs")) + .and(warp::path::end()) + .and(sse_component_filter) + .and_then(|sse_component: Option| { + warp_utils::task::blocking_task(move || { + if let Some(logging_components) = sse_component { + // Build a JSON stream + let s = + BroadcastStream::new(logging_components.sender.subscribe()).map(|msg| { + match msg { + Ok(data) => { + // Serialize to json + match data.to_json_string() { + // Send the json as a Server Sent Event + Ok(json) => Event::default().json_data(json).map_err(|e| { + warp_utils::reject::server_sent_event_error(format!( + "{:?}", + e + )) + }), + Err(e) => Err(warp_utils::reject::server_sent_event_error( + format!("Unable to serialize to JSON {}", e), + )), + } + } + Err(e) => Err(warp_utils::reject::server_sent_event_error( + format!("Unable to receive event {}", e), + )), + } + }); + + Ok::<_, warp::Rejection>(warp::sse::reply(warp::sse::keep_alive().stream(s))) + } else { + Err(warp_utils::reject::custom_server_error( + "SSE Logging is not enabled".to_string(), + )) + } + }) + }); + let routes = warp::any() .and(authorization_header_filter) // Note: it is critical that the `authorization_header_filter` is applied to all routes. @@ -1061,8 +1112,8 @@ pub fn serve( .or(delete_std_remotekeys), )), ) - // The auth route is the only route that is allowed to be accessed without the API token. - .or(warp::get().and(get_auth)) + // The auth route and logs are the only routes that are allowed to be accessed without the API token. + .or(warp::get().and(get_auth.or(get_log_events.boxed()))) // Maps errors into HTTP responses. .recover(warp_utils::reject::handle_rejection) // Add a `Server` header. diff --git a/validator_client/src/http_api/tests.rs b/validator_client/src/http_api/tests.rs index 1c593b1a4ec..84d2fe437d5 100644 --- a/validator_client/src/http_api/tests.rs +++ b/validator_client/src/http_api/tests.rs @@ -134,7 +134,8 @@ impl ApiTester { listen_port: 0, allow_origin: None, }, - log: log.clone(), + sse_logging_components: None, + log, slot_clock: slot_clock.clone(), _phantom: PhantomData, }); diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 6563d2fea50..3dde49f2270 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -576,6 +576,7 @@ impl ProductionValidatorClient { graffiti_flag: self.config.graffiti, spec: self.context.eth2_config.spec.clone(), config: self.config.http_api.clone(), + sse_logging_components: self.context.sse_logging_components.clone(), slot_clock: self.slot_clock.clone(), log: log.clone(), _phantom: PhantomData,