From 85b2f7fed1a11f7266d6fbdb002b1b7fa324af51 Mon Sep 17 00:00:00 2001 From: pvannierop Date: Fri, 4 Oct 2024 15:15:45 +0200 Subject: [PATCH] Add support for Sentry monitoring Includes addition to documentation. --- README.md | 51 ++++++++++++++++++--- radar-gateway/build.gradle.kts | 3 ++ radar-gateway/src/main/resources/log4j2.xml | 39 ++++++++++++++++ 3 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 radar-gateway/src/main/resources/log4j2.xml diff --git a/README.md b/README.md index a0f3bb6..a1e1eb9 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,20 @@ # RADAR-Gateway -REST Gateway to the Apache Kafka, similar to the REST Proxy provided by Confluent. In addition, it does authentication and authorization, content validation and decompression if needed. It is available as a [docker image](https://hub.docker.com/r/radarbase/radar-gateway). +REST Gateway to the Apache Kafka, similar to the REST Proxy provided by Confluent. In addition, it does authentication +and authorization, content validation and decompression if needed. It is available as +a [docker image](https://hub.docker.com/r/radarbase/radar-gateway). + + +* [RADAR-Gateway](#radar-gateway) + * [Configuration](#configuration) + * [Usage](#usage) + * [Sentry monitoring](#sentry-monitoring) + ## Configuration -The [RADAR-Auth] library is used for authentication and authorization of users. Refer to the documentation there for a full description of the configuration options. +The [RADAR-Auth] library is used for authentication and authorization of users. Refer to the documentation there for a +full description of the configuration options. ## Usage @@ -21,22 +31,51 @@ TOPIC=test docker-compose exec kafka-1 kafka-topics --create --topic $TOPIC --bootstrap-server kafka-1:9092 ``` -Now the gateway is accessible through and the [ManagementPortal] is available through +Now the gateway is accessible through and the [ManagementPortal] is available +through -The access token should be generated by the aforementioned Management portal. The access token is a JWT (JSON Web Token) that should contain the `MEASUREMENT.CREATE` scope for resource `res_gateway`, and list all applicable sources to submit data for. The gateway does content validation for posted data. It requires to use the Avro format with JSON serialization, using the `application/vnd.kafka.avro.v1+json` or `application/vnd.kafka.avro.v2+json` media types, as described in the [REST Proxy documentation]. It also requires messages to have both a key and a value with schemas. The key should have a `userId` and `sourceId` field. The `userId` should match the `sub` field in the OAuth2 JWT access token. That JWT should also contain a `sources` array claim which should contain the given `sourceId`. Sources can be added in the ManagementPortal or be generated by the app dynamically and then registered with the ManagementPortal. +The access token should be generated by the aforementioned Management portal. The access token is a JWT (JSON Web Token) +that should contain the `MEASUREMENT.CREATE` scope for resource `res_gateway`, and list all applicable sources to submit +data for. The gateway does content validation for posted data. It requires to use the Avro format with JSON +serialization, using the `application/vnd.kafka.avro.v1+json` or `application/vnd.kafka.avro.v2+json` media types, as +described in the [REST Proxy documentation]. It also requires messages to have both a key and a value with schemas. The +key should have a `userId` and `sourceId` field. The `userId` should match the `sub` field in the OAuth2 JWT access +token. That JWT should also contain a `sources` array claim which should contain the given `sourceId`. Sources can be +added in the ManagementPortal or be generated by the app dynamically and then registered with the ManagementPortal. Now you can access the gateway: + ```shell TOKEN= curl -H "Authorization: Bearer $TOKEN" http://localhost:8090/radar-gateway/topics ``` -Data compressed with GZIP is decompressed if the `Content-Encoding: gzip` header is present. With `curl`, use the `-H "Content-Encoding: gzip" --data-binary @data.json.gz` flags. It can be activated in `radar-commons` Java `RestClient` by setting `RestClient.Builder.gzipCompression(true)`. Likewise it accepts Apple LZFSE encoded data by adding the header `Content-Encoding: lzfse`. +Data compressed with GZIP is decompressed if the `Content-Encoding: gzip` header is present. With `curl`, use the +`-H "Content-Encoding: gzip" --data-binary @data.json.gz` flags. It can be activated in `radar-commons` Java +`RestClient` by setting `RestClient.Builder.gzipCompression(true)`. Likewise it accepts Apple LZFSE encoded data by +adding the header `Content-Encoding: lzfse`. Otherwise, it accepts all the same Avro messages and headers as specified in the Kafka [REST Proxy documentation]. -Finally, the gateway accepts a custom binary format for data ingestion. The data must follow the binary Avro serialization of the [RecordSet schema](https://github.com/RADAR-base/RADAR-Schemas/blob/master/commons/kafka/record_set.avsc). Data in this format can be posted by using the content type `application/vnd.radarbase.avro.v1+binary`. It will construct an `ObservationKey` based on the user data in the `RecordSet`, and read the binary data values using the schema version provided in the `RecordSet`. This data sending mode can be activated in Java by using radar-commons `RestSender.Builder.useBinaryContent(true)`. Using binary mode has the added benefit of having a much more efficient GZIP encoding for many datasets. +Finally, the gateway accepts a custom binary format for data ingestion. The data must follow the binary Avro +serialization of +the [RecordSet schema](https://github.com/RADAR-base/RADAR-Schemas/blob/master/commons/kafka/record_set.avsc). Data in +this format can be posted by using the content type `application/vnd.radarbase.avro.v1+binary`. It will construct an +`ObservationKey` based on the user data in the `RecordSet`, and read the binary data values using the schema version +provided in the `RecordSet`. This data sending mode can be activated in Java by using radar-commons +`RestSender.Builder.useBinaryContent(true)`. Using binary mode has the added benefit of having a much more efficient +GZIP encoding for many datasets. [REST Proxy documentation]: https://docs.confluent.io/current/kafka-rest/api.html + [RADAR-Auth]: https://github.com/RADAR-base/ManagementPortal/tree/master/radar-auth + [ManagementPortal]: https://github.com/RADAR-base/ManagementPortal + +## Sentry monitoring + +To enable Sentry monitoring: + +1. Set a `SENTRY_DSN` environment variable that points to the desired Sentry DSN. +2. (Optional) Set the `SENTRY_LOG_LEVEL` environment variable to control the minimum log level of events sent to Sentry. + The default log level for Sentry is `WARN`. Possible values are `TRACE`, `DEBUG`, `INFO`, `WARN`, and `ERROR`. diff --git a/radar-gateway/build.gradle.kts b/radar-gateway/build.gradle.kts index 35c9a3a..2dfc405 100644 --- a/radar-gateway/build.gradle.kts +++ b/radar-gateway/build.gradle.kts @@ -4,6 +4,9 @@ plugins { application kotlin("plugin.serialization") version Versions.kotlin id("com.avast.gradle.docker-compose") version Versions.dockerCompose + // TODO Remove this when new release of radar-commons is available and used in this project. + // This version has Sentry support built in for radar-kotlin plugin. + id("io.sentry.jvm.gradle") version "4.11.0" } description = "RADAR Gateway to handle secured data flow to backend." diff --git a/radar-gateway/src/main/resources/log4j2.xml b/radar-gateway/src/main/resources/log4j2.xml new file mode 100644 index 0000000..66bc4fc --- /dev/null +++ b/radar-gateway/src/main/resources/log4j2.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + +