diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5662a130b09..4ffac753ead 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,8 @@ jobs: steps: - uses: actions/checkout@v4 name: Checkout code + - name: Install protoc + uses: arduino/setup-protoc@v3 - name: Run linter run: | make lint-rust format-rust-ci diff --git a/Dockerfile b/Dockerfile index f73bccf8496..77dea2c07b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN set -ex; \ make \ g++ \ gnupg \ + protobuf-compiler \ '; \ runtimeDeps=' \ curl \ diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 986b739c9ac..4f37ae477e7 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -13,7 +13,7 @@ steps: "-f", "./Dockerfile", ] - timeout: 1200s + timeout: 2400s # Pull docker image again, so we can get build info. # https://github.com/GoogleCloudPlatform/cloud-builders-community/issues/212#issuecomment-1478828752 @@ -27,4 +27,4 @@ images: [ timeout: 2640s options: # We need more memory for Webpack builds & e2e self-hosted tests - machineType: "E2_HIGHCPU_8" + machineType: "E2_HIGHCPU_32" diff --git a/requirements.txt b/requirements.txt index 94f53a82fbe..3f01e857941 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ python-dateutil==2.8.2 python-rapidjson==1.8 redis==4.5.4 sentry-arroyo==2.19.12 -sentry-kafka-schemas==0.1.129 +sentry-kafka-schemas==1.0.3 sentry-protos==0.1.58 sentry-redis-tools==0.3.0 sentry-relay==0.9.5 diff --git a/rust_snuba/Cargo.lock b/rust_snuba/Cargo.lock index dd4c63b6f86..490ba195cd7 100644 --- a/rust_snuba/Cargo.lock +++ b/rust_snuba/Cargo.lock @@ -337,6 +337,28 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] + [[package]] name = "async-task" version = "4.7.0" @@ -366,6 +388,53 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.2", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -1292,6 +1361,12 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "gloo-timers" version = "0.2.6" @@ -1323,6 +1398,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.4.1" @@ -1427,11 +1521,24 @@ dependencies = [ "http 1.1.0", ] +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "pin-project-lite", +] + [[package]] name = "httparse" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "httpdate" @@ -1483,7 +1590,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -1499,14 +1606,36 @@ dependencies = [ [[package]] name = "hyper" -version = "1.2.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", + "futures-channel", + "futures-util", + "h2 0.4.7", "http 1.1.0", "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -1522,6 +1651,26 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyper-util" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.6.0", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2047,6 +2196,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md5" version = "0.7.0" @@ -2131,6 +2286,12 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + [[package]] name = "native-tls" version = "0.2.11" @@ -2516,6 +2677,26 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" +[[package]] +name = "pin-project" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2667,6 +2848,58 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "prost" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" +dependencies = [ + "heck 0.5.0", + "itertools 0.11.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.58", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "prost-types" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" +dependencies = [ + "prost", +] + [[package]] name = "pyo3" version = "0.18.3" @@ -2963,7 +3196,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", @@ -2980,7 +3213,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", @@ -3007,7 +3240,7 @@ dependencies = [ "data-encoding", "futures", "httpmock", - "hyper 1.2.0", + "hyper 1.6.0", "insta", "json-schema-diff", "md5", @@ -3033,6 +3266,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-stream", + "tonic", "tracing", "tracing-subscriber", "uuid", @@ -3269,13 +3503,15 @@ dependencies = [ [[package]] name = "sentry-kafka-schemas" -version = "0.1.129" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790627715d4ea0e58e252dcb657a44146fde401b5520bbbc3b6500764ef71c86" +checksum = "de02462bd68f5ca1dc44be8f7a3114e80071de42b8e6091064878ebe7b8ae892" dependencies = [ "jsonschema", "prettyplease", + "prost", "schemars", + "sentry_protos", "serde", "serde_json", "serde_yaml", @@ -3345,6 +3581,21 @@ dependencies = [ "uuid", ] +[[package]] +name = "sentry_protos" +version = "0.1.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a7e373d284d5cddfeaa22c983f496ae5569d9902c2f8620492887313960de3" +dependencies = [ + "glob", + "prost", + "prost-types", + "regex", + "tokio", + "tonic", + "tonic-build", +] + [[package]] name = "sentry_usage_accountant" version = "0.1.0" @@ -3653,6 +3904,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.1" @@ -3875,9 +4132,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -3915,6 +4172,76 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.7", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.6.0", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.6", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.2" diff --git a/rust_snuba/Cargo.toml b/rust_snuba/Cargo.toml index 53bb0960c11..fd7ddd371dd 100644 --- a/rust_snuba/Cargo.toml +++ b/rust_snuba/Cargo.toml @@ -32,7 +32,7 @@ procspawn = { version = "1.0.0", features = ["json"] } pyo3 = { version = "0.18.1", features = ["chrono"] } reqwest = { version = "0.11.11", features = ["stream"] } sentry = { version = "0.32.0", features = ["anyhow", "tracing"] } -sentry-kafka-schemas = "0.1.129" +sentry-kafka-schemas = "1.0.3" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } thiserror = "1.0" @@ -55,6 +55,7 @@ serde_with = "3.8.1" seq-macro = "0.3" sentry_arroyo = "2.19.11" regex = "1.11.1" +tonic = "0.12.3" [dev-dependencies] diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__events-.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__events-.snap index 6a13ba9296e..ac21162b1bf 100644 --- a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__events-.snap +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__events-.snap @@ -3,13 +3,6 @@ source: src/processors/mod.rs expression: diff --- [ - Change { - path: "..2.data.contexts.", - change: PropertyAdd { - lhs_additional_properties: true, - added: "flags", - }, - }, Change { path: "..2.data.contexts..trace.", change: PropertyAdd { diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-uptime-results-UptimeMonitorChecksProcessor-snuba-uptime-results__1__failure.json.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-uptime-results-UptimeMonitorChecksProcessor-snuba-uptime-results__1__failure.json.snap index f78549abc60..1bc09c69ead 100644 --- a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-uptime-results-UptimeMonitorChecksProcessor-snuba-uptime-results__1__failure.json.snap +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-uptime-results-UptimeMonitorChecksProcessor-snuba-uptime-results__1__failure.json.snap @@ -1,6 +1,6 @@ --- source: src/processors/mod.rs -description: "{\n \"guid\": \"54afc7ed9c53491481919c931f75bae1\",\n \"subscription_id\": \"5421b5df80744113a6b57014f01a3a42\",\n \"status\": \"failure\",\n \"status_reason\": {\n \"type\": \"dns_error\",\n \"description\": \"Unable to resolve hostname example.xyz\"\n },\n \"trace_id\": \"947efba02dac463b9c1d886a44bafc94\",\n \"span_id\": \"58e84098e63f42e1\",\n \"scheduled_check_time_ms\": 1717614062978,\n \"actual_check_time_ms\": 1717614068008,\n \"duration_ms\": 100,\n \"request_info\": {\n \"request_type\": \"HEAD\",\n \"http_status_code\": 500\n },\n \"organization_id\": 123,\n \"project_id\": 456,\n \"retention_days\": 90,\n \"region_slug\": \"us-east-1\"\n}\n" +description: "{\n \"guid\": \"54afc7ed9c53491481919c931f75bae1\",\n \"subscription_id\": \"5421b5df80744113a6b57014f01a3a42\",\n \"status\": \"failure\",\n \"status_reason\": {\n \"type\": \"dns_error\",\n \"description\": \"Unable to resolve hostname example.xyz\"\n },\n \"trace_id\": \"947efba02dac463b9c1d886a44bafc94\",\n \"span_id\": \"58e84098e63f42e1\",\n \"scheduled_check_time_ms\": 1717614062978,\n \"actual_check_time_ms\": 1717614068008,\n \"duration_ms\": 100,\n \"request_info\": {\n \"request_type\": \"HEAD\",\n \"http_status_code\": 500\n },\n \"organization_id\": 123,\n \"project_id\": 456,\n \"retention_days\": 90,\n \"region\": \"us-east-1\",\n \"environment\": \"production\"\n}\n" expression: snapshot_payload --- [ @@ -8,11 +8,11 @@ expression: snapshot_payload "check_status": "failure", "check_status_reason": "dns_error", "duration_ms": 100, - "environment": null, + "environment": "production", "http_status_code": 500, "organization_id": 123, "project_id": 456, - "region": "", + "region": "us-east-1", "retention_days": 90, "scheduled_check_time": 1717614062, "timestamp": 1717614068008, diff --git a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-uptime-results-UptimeMonitorChecksProcessor-snuba-uptime-results__1__success.json.snap b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-uptime-results-UptimeMonitorChecksProcessor-snuba-uptime-results__1__success.json.snap index 78634574691..6a774b3ce87 100644 --- a/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-uptime-results-UptimeMonitorChecksProcessor-snuba-uptime-results__1__success.json.snap +++ b/rust_snuba/src/processors/snapshots/rust_snuba__processors__tests__schemas@snuba-uptime-results-UptimeMonitorChecksProcessor-snuba-uptime-results__1__success.json.snap @@ -1,6 +1,6 @@ --- source: src/processors/mod.rs -description: "{\n \"guid\": \"54afc7ed9c53491481919c931f75bae1\",\n \"subscription_id\": \"5421b5df80744113a6b57014f01a3a42\",\n \"status\": \"success\",\n \"status_reason\": null,\n \"trace_id\": \"947efba02dac463b9c1d886a44bafc94\",\n \"span_id\": \"58e84098e63f42e1\",\n \"scheduled_check_time_ms\": 1717614062978,\n \"actual_check_time_ms\": 1717614068008,\n \"duration_ms\": 50,\n \"request_info\": {\n \"request_type\": \"HEAD\",\n \"http_status_code\": 200\n },\n \"organization_id\": 123,\n \"project_id\": 456,\n \"retention_days\": 90,\n \"region_slug\": \"us-east-1\"\n}\n" +description: "{\n \"guid\": \"54afc7ed9c53491481919c931f75bae1\",\n \"subscription_id\": \"5421b5df80744113a6b57014f01a3a42\",\n \"status\": \"success\",\n \"status_reason\": null,\n \"trace_id\": \"947efba02dac463b9c1d886a44bafc94\",\n \"span_id\": \"58e84098e63f42e1\",\n \"scheduled_check_time_ms\": 1717614062978,\n \"actual_check_time_ms\": 1717614068008,\n \"duration_ms\": 50,\n \"request_info\": {\n \"request_type\": \"HEAD\",\n \"http_status_code\": 200\n },\n \"organization_id\": 123,\n \"project_id\": 456,\n \"retention_days\": 90,\n \"region\": \"us-east-1\",\n \"environment\": \"production\"\n}\n" expression: snapshot_payload --- [ @@ -8,11 +8,11 @@ expression: snapshot_payload "check_status": "success", "check_status_reason": "", "duration_ms": 50, - "environment": null, + "environment": "production", "http_status_code": 200, "organization_id": 123, "project_id": 456, - "region": "", + "region": "us-east-1", "retention_days": 90, "scheduled_check_time": 1717614062, "timestamp": 1717614068008,