Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Replace lib/pq with pgx in a PostgreSQL scaler #4705

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
- **Pulsar Scaler**: Improve error messages for unsuccessful connections ([#4563](https://github.com/kedacore/keda/issues/4563))
- **Security**: Enable secret scanning in GitHub repo
- **RabbitMQ Scaler**: Add support for `unsafeSsl` in trigger metadata ([#4448](https://github.com/kedacore/keda/issues/4448))
- **PostgreSQL Scaler**: Replace `lib/pq` with `pgx` ([#4704](https://github.com/kedacore/keda/issues/4704))
- **Prometheus Metrics**: Add new metric with KEDA build info ([#4647](https://github.com/kedacore/keda/issues/4647))
- **Prometheus Scaler**: Add support for Google Managed Prometheus ([#4675](https://github.com/kedacore/keda/pull/4675))

Expand Down
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ require (
github.com/hashicorp/vault/api v1.9.2
github.com/imdario/mergo v0.3.16
github.com/influxdata/influxdb-client-go/v2 v2.12.3
github.com/jackc/pgx/v5 v5.4.0
github.com/joho/godotenv v1.5.1
github.com/lib/pq v1.10.9
github.com/microsoft/ApplicationInsights-Go v0.4.4
github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5
github.com/mitchellh/hashstructure v1.1.0
Expand Down Expand Up @@ -212,6 +212,8 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.7.6 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,12 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0
github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0=
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU=
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.4.0 h1:BSr+GCm4N6QcgIwv0DyTFHK9ugfEFF9DzSbbzxOiXU0=
github.com/jackc/pgx/v5 v5.4.0/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
Expand Down Expand Up @@ -525,8 +531,6 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
Expand Down
4 changes: 2 additions & 2 deletions pkg/scalers/postgresql_scaler.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

"github.com/go-logr/logr"
// PostreSQL drive required for this scaler
_ "github.com/lib/pq"
_ "github.com/jackc/pgx/v5/stdlib"
v2 "k8s.io/api/autoscaling/v2"
"k8s.io/metrics/pkg/apis/external_metrics"

Expand Down Expand Up @@ -146,7 +146,7 @@ func parsePostgreSQLMetadata(config *ScalerConfig) (*postgreSQLMetadata, error)
}

func getConnection(meta *postgreSQLMetadata, logger logr.Logger) (*sql.DB, error) {
db, err := sql.Open("postgres", meta.connection)
db, err := sql.Open("pgx", meta.connection)
if err != nil {
logger.Error(err, fmt.Sprintf("Found error opening postgreSQL: %s", err))
return nil, err
Expand Down
2 changes: 2 additions & 0 deletions pkg/scalers/postgresql_scaler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ var testPostgreSQLConnectionstring = []postgreSQLConnectionStringTestData{
{metadata: map[string]string{"query": "test_query", "targetQueryValue": "5"}, authParam: map[string]string{"connection": "test_connection_from_auth"}, connectionString: "test_connection_from_auth"},
// from meta
{metadata: map[string]string{"query": "test_query", "targetQueryValue": "5", "host": "localhost", "port": "1234", "dbName": "testDb", "userName": "user", "sslmode": "required"}, connectionString: "host=localhost port=1234 user=user dbname=testDb sslmode=required password="},
// from meta, multiple hosts
{metadata: map[string]string{"query": "test_query", "targetQueryValue": "5", "host": "host1,host2", "port": "1234", "dbName": "testDb", "userName": "user", "sslmode": "required"}, connectionString: "host=host1,host2 port=1234 user=user dbname=testDb sslmode=required password="},
}

func TestPosgresSQLConnectionStringGeneration(t *testing.T) {
Expand Down
219 changes: 219 additions & 0 deletions tests/scalers/postgresql/helper/postgresql_helper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
//go:build e2e
// +build e2e

package helper

import (
"fmt"
"strings"
)

const (
PostgresqlImage = "postgres:10.5"
)

const (
DeploymentTemplate = `apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: postgresql-update-worker
name: {{.DeploymentName}}
namespace: {{.TestNamespace}}
spec:
replicas: 0
selector:
matchLabels:
app: postgresql-update-worker
template:
metadata:
labels:
app: postgresql-update-worker
spec:
containers:
- image: ghcr.io/kedacore/tests-postgresql
imagePullPolicy: Always
name: postgresql-processor-test
command:
- /app
- update
env:
- name: TASK_INSTANCES_COUNT
value: "6000"
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: {{.SecretName}}
key: postgresql_conn_str
`
ScaledObjectTemplate = `apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{.ScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
scaleTargetRef:
name: {{.DeploymentName}}
pollingInterval: 5
cooldownPeriod: 10
minReplicaCount: {{.MinReplicaCount}}
maxReplicaCount: {{.MaxReplicaCount}}
triggers:
- type: postgresql
metadata:
targetQueryValue: "4"
activationTargetQueryValue: "5"
query: "SELECT CEIL(COUNT(*) / 5) FROM task_instance WHERE state='running' OR state='queued'"
authenticationRef:
name: {{.TriggerAuthenticationName}}
`
LowLevelRecordsJobTemplate = `apiVersion: batch/v1
kind: Job
metadata:
labels:
app: postgresql-insert-low-level-job
name: postgresql-insert-low-level-job
namespace: {{.TestNamespace}}
spec:
template:
metadata:
labels:
app: postgresql-insert-low-level-job
spec:
containers:
- image: ghcr.io/kedacore/tests-postgresql
imagePullPolicy: Always
name: postgresql-processor-test
command:
- /app
- insert
env:
- name: TASK_INSTANCES_COUNT
value: "20"
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: {{.SecretName}}
key: postgresql_conn_str
restartPolicy: Never
backoffLimit: 4
`
InsertRecordsJobTemplate = `apiVersion: batch/v1
kind: Job
metadata:
labels:
app: postgresql-insert-job
name: postgresql-insert-job
namespace: {{.TestNamespace}}
spec:
template:
metadata:
labels:
app: postgresql-insert-job
spec:
containers:
- image: ghcr.io/kedacore/tests-postgresql
imagePullPolicy: Always
name: postgresql-processor-test
command:
- /app
- insert
env:
- name: TASK_INSTANCES_COUNT
value: "10000"
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: {{.SecretName}}
key: postgresql_conn_str
restartPolicy: Never
backoffLimit: 4
`
TriggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: {{.TriggerAuthenticationName}}
namespace: {{.TestNamespace}}
spec:
secretTargetRef:
- parameter: connection
name: {{.SecretName}}
key: {{.SecretKey}}
`
PostgreSQLStatefulSetTemplate = `apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: {{.PostgreSQLStatefulSetName}}
name: {{.PostgreSQLStatefulSetName}}
namespace: {{.TestNamespace}}
spec:
replicas: 1
serviceName: {{.PostgreSQLStatefulSetName}}
selector:
matchLabels:
app: {{.PostgreSQLStatefulSetName}}
template:
metadata:
labels:
app: {{.PostgreSQLStatefulSetName}}
spec:
containers:
- name: postgresql
image: {{.PostgreSQLImage}}
env:
- name: POSTGRES_USER
value: {{.PostgreSQLUsername}}
- name: POSTGRES_PASSWORD
value: {{.PostgreSQLPassword}}
- name: POSTGRES_DB
value: {{.PostgreSQLDatabase}}
ports:
- name: postgresql
protocol: TCP
containerPort: 5432
readinessProbe:
tcpSocket:
port: 5432
initialDelaySeconds: 5
timeoutSeconds: 5
livenessProbe:
tcpSocket:
port: 5432
initialDelaySeconds: 5
timeoutSeconds: 5
volumeMounts:
- name: init-scripts
mountPath: /docker-entrypoint-initdb.d
volumes:
- name: init-scripts
configMap:
name: master-slave-config
optional: true
`
PostgreSQLServiceTemplate = `apiVersion: v1
kind: Service
metadata:
labels:
app: {{.PostgreSQLStatefulSetName}}
name: {{.PostgreSQLStatefulSetName}}
namespace: {{.TestNamespace}}
spec:
ports:
- port: 5432
protocol: TCP
targetPort: 5432
selector:
app: {{.PostgreSQLStatefulSetName}}
type: ClusterIP
`
)

func GetConnectionString(username string, password string, svcNames []string, namespace string, database string) string {
svcFQDNs := make([]string, 0, len(svcNames))
for _, svcName := range svcNames {
svcFQDNs = append(svcFQDNs, fmt.Sprintf("%s.%s.svc.cluster.local", svcName, namespace))
}
fqdns := strings.Join(svcFQDNs, ",")
return fmt.Sprintf("postgresql://%s:%s@%s:5432/%s?sslmode=disable", username, password, fqdns, database)
}
Loading