Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(postgres sink): Add postgres sink #21248

Open
wants to merge 26 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
f47611a
feat(sinks): Initial postgres log sink implementation
jorgehermo9 Sep 8, 2024
0a0576b
feat(sinks): Initial postgres log sink implementation
jorgehermo9 Sep 8, 2024
bf2b7af
test(sinks): Add integration test for postgres sink
jorgehermo9 Sep 9, 2024
2ecbd56
feat(sinks): postgres sink config unit tests
jorgehermo9 Sep 9, 2024
a3be3ea
Merge branch 'master' into feature/postgres-sink
jorgehermo9 Sep 9, 2024
cee4b39
docs(website): update components cue
jorgehermo9 Sep 9, 2024
548e996
chore: update spelling checklist
jorgehermo9 Sep 9, 2024
ea82ad4
style: cargo fmt
jorgehermo9 Sep 9, 2024
ae68fc4
fix: clippy lint
jorgehermo9 Sep 9, 2024
89e63e3
fix: clippy lint
jorgehermo9 Sep 9, 2024
f68f04b
docs: add changelog
jorgehermo9 Sep 9, 2024
f4cd4c7
docs: add changelog
jorgehermo9 Sep 9, 2024
9d78d67
Merge with master
jorgehermo9 Nov 22, 2024
984750d
chore: rename postgres integration tests feature flag
jorgehermo9 Nov 22, 2024
5b1ca45
chore: rename postgres integration tests feature flag
jorgehermo9 Nov 23, 2024
21f3ad3
docs: include connection pool reference
jorgehermo9 Nov 24, 2024
8fc8a10
Merge with master
jorgehermo9 Dec 6, 2024
0bea2f7
fix: fix compilation issue in benches
jorgehermo9 Dec 6, 2024
e45055d
chore: rename feature flag
jorgehermo9 Dec 6, 2024
6805961
feat: update
jorgehermo9 Dec 6, 2024
c88bccc
test: refactored test
jorgehermo9 Dec 6, 2024
f58a51b
test: add new test & store timestamp instead of text
jorgehermo9 Dec 15, 2024
95768bb
test: add new test & store timestamp instead of text
jorgehermo9 Dec 15, 2024
270d32c
test: add new test & store timestamp instead of text
jorgehermo9 Dec 15, 2024
0f07f64
test: add reference link
jorgehermo9 Dec 15, 2024
603fc5c
test: add more tests
jorgehermo9 Dec 22, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/actions/spelling/expect.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1019,6 +1019,7 @@ splitn
SPOF
spog
springframework
sqlx
srcport
SREs
sret
Expand Down
206 changes: 202 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,7 @@ smallvec = { version = "1", default-features = false, features = ["union", "serd
snafu = { version = "0.7.5", default-features = false, features = ["futures", "std"] }
snap = { version = "1.1.1", default-features = false }
socket2 = { version = "0.5.7", default-features = false }
sqlx = { version = "0.8.2", default-features= false, features = ["derive", "postgres", "runtime-tokio"], optional=true }
stream-cancel = { version = "0.8.2", default-features = false }
strip-ansi-escapes = { version = "0.2.0", default-features = false }
syslog = { version = "6.1.1", default-features = false, optional = true }
Expand Down Expand Up @@ -699,6 +700,7 @@ sinks-logs = [
"sinks-new_relic_logs",
"sinks-new_relic",
"sinks-papertrail",
"sinks-postgres",
"sinks-pulsar",
"sinks-redis",
"sinks-sematext",
Expand Down Expand Up @@ -765,6 +767,7 @@ sinks-new_relic_logs = ["sinks-http"]
sinks-new_relic = []
sinks-papertrail = ["dep:syslog"]
sinks-prometheus = ["dep:base64", "vector-lib/prometheus"]
sinks-postgres = ["dep:sqlx"]
sinks-pulsar = ["dep:apache-avro", "dep:pulsar", "dep:lru"]
sinks-redis = ["dep:redis"]
sinks-sematext = ["sinks-elasticsearch", "sinks-influxdb"]
Expand Down Expand Up @@ -812,6 +815,7 @@ all-integration-tests = [
"nginx-integration-tests",
"opentelemetry-integration-tests",
"postgresql_metrics-integration-tests",
"postgres-integration-tests",
"prometheus-integration-tests",
"pulsar-integration-tests",
"redis-integration-tests",
Expand Down Expand Up @@ -877,6 +881,7 @@ nats-integration-tests = ["sinks-nats", "sources-nats"]
nginx-integration-tests = ["sources-nginx_metrics"]
opentelemetry-integration-tests = ["sources-opentelemetry"]
postgresql_metrics-integration-tests = ["sources-postgresql_metrics"]
postgres-integration-tests = ["sinks-postgres"]
prometheus-integration-tests = ["sinks-prometheus", "sources-prometheus", "sinks-influxdb"]
pulsar-integration-tests = ["sinks-pulsar", "sources-pulsar"]
redis-integration-tests = ["sinks-redis", "sources-redis"]
Expand Down
3 changes: 3 additions & 0 deletions changelog.d/15765_postgres_sink.feature.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Add a new postgres sink which allows to send log events to a postgres database.

authors: jorgehermo9
2 changes: 2 additions & 0 deletions scripts/integration/postgres/test.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
features:
- postgresql_metrics-integration-tests
- postgres-integration-tests
jorgehermo9 marked this conversation as resolved.
Show resolved Hide resolved

test_filter: ::postgres

Expand All @@ -18,6 +19,7 @@ matrix:
# expressions are evaluated using https://github.com/micromatch/picomatch
paths:
- "src/internal_events/postgresql_metrics.rs"
- "src/sinks/postgres/**"
- "src/sources/postgresql_metrics.rs"
- "src/sources/util/**"
- "scripts/integration/postgres/**"
2 changes: 2 additions & 0 deletions src/sinks/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ pub mod new_relic;
pub mod opendal_common;
#[cfg(feature = "sinks-papertrail")]
pub mod papertrail;
#[cfg(feature = "sinks-postgres")]
pub mod postgres;
#[cfg(feature = "sinks-prometheus")]
pub mod prometheus;
#[cfg(feature = "sinks-pulsar")]
Expand Down
Loading
Loading