From 5e7c8a59a6d5aac128dba45be1888aacf1ca17fb Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 14 Oct 2024 10:25:39 -0700 Subject: [PATCH 1/3] support valkey:// scheme --- Makefile | 5 +++-- exporter/redis.go | 7 +++++++ exporter/redis_test.go | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a3968ad7..c1a58b05 100644 --- a/Makefile +++ b/Makefile @@ -32,12 +32,13 @@ docker-test: test: contrib/tls/gen-test-certs.sh + TEST_VALKEY7_URI="valkey://localhost:16384" \ + TEST_VALKEY8_URI="valkey://localhost:16382" \ + TEST_REDIS_URI="redis://localhost:16385" \ TEST_REDIS7_URI="redis://localhost:16385" \ TEST_REDIS5_URI="redis://localhost:16383" \ TEST_REDIS6_URI="redis://localhost:16379" \ - TEST_VALKEY7_URI="redis://localhost:16384" \ - TEST_VALKEY8_URI="redis://localhost:16382" \ TEST_REDIS_2_8_URI="redis://localhost:16381" \ TEST_KEYDB01_URI="redis://localhost:16401" \ TEST_KEYDB02_URI="redis://localhost:16402" \ diff --git a/exporter/redis.go b/exporter/redis.go index 3d9be768..05bc6b98 100644 --- a/exporter/redis.go +++ b/exporter/redis.go @@ -45,6 +45,13 @@ func (e *Exporter) connectToRedis() (redis.Conn, error) { uri = "redis://" + uri } + switch { + case strings.HasPrefix(uri, "valkey://"): + uri = strings.Replace(uri, "valkey://", "redis://", 1) + case strings.HasPrefix(uri, "valkeys://"): + uri = strings.Replace(uri, "valkeys://", "rediss://", 1) + } + options, err := e.configureOptions(uri) if err != nil { return nil, err diff --git a/exporter/redis_test.go b/exporter/redis_test.go index 5c32d6e4..b265a910 100644 --- a/exporter/redis_test.go +++ b/exporter/redis_test.go @@ -31,6 +31,23 @@ func TestHostVariations(t *testing.T) { } } +// todo: also test valkeys://... +func TestValkeyScheme(t *testing.T) { + host := os.Getenv("TEST_VALKEY8_URI") + + e, _ := NewRedisExporter(host, Options{SkipTLSVerification: true}) + c, err := e.connectToRedis() + if err != nil { + t.Fatalf("connectToRedis() err: %s", err) + } + + if _, err := c.Do("PING", ""); err != nil { + t.Errorf("PING err: %s", err) + } + + c.Close() +} + func TestPasswordProtectedInstance(t *testing.T) { userAddr := os.Getenv("TEST_USER_PWD_REDIS_URI") if userAddr == "" { From 4d260d8b069d1739c41eab1c914a0973bc2819d1 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 14 Oct 2024 15:50:32 -0700 Subject: [PATCH 2/3] fix Dockerfile location --- .github/workflows/tests.yml | 9 +++--- Makefile | 3 +- docker-compose.yml | 26 ++++++++++++++++++ exporter/exporter.go | 13 +++++++-- exporter/exporter_test.go | 4 +-- exporter/http_test.go | 5 ++++ exporter/redis.go | 7 ----- exporter/redis_test.go | 1 - exporter/tls_test.go | 55 +++++++++++++++++++++++++++++++++++++ 9 files changed, 106 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7082e00c..6649308a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,10 +42,11 @@ jobs: make test - - name: Run tests - valkey 7 + - name: Run tests - valkey 8 env: LOG_LEVEL: "info" - TEST_REDIS_URI: "redis://localhost:16384" + TEST_REDIS_URI: "redis://localhost:16382" + TEST_VALKEY8_TLS_URI: "valkeys://localhost:16386" TEST_PWD_REDIS_URI: "redis://:redis-password@localhost:16380" run: | go test -v -race -p 1 ./... @@ -120,7 +121,7 @@ jobs: push: false target: alpine tags: user/app:tst - file: docker/Dockerfile + file: Dockerfile build-args: "GOARCH=amd64" - name: Test Docker Image Build - Scratch @@ -129,5 +130,5 @@ jobs: push: false target: scratch-release tags: user/app:tst - file: docker/Dockerfile + file: Dockerfile build-args: "GOARCH=amd64" diff --git a/Makefile b/Makefile index c1a58b05..58490303 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,8 @@ test: TEST_VALKEY7_URI="valkey://localhost:16384" \ TEST_VALKEY8_URI="valkey://localhost:16382" \ - + TEST_VALKEY8_TLS_URI="valkeys://localhost:16386" \ + TEST_REDIS7_TLS_URI="rediss://localhost:16386" \ TEST_REDIS_URI="redis://localhost:16385" \ TEST_REDIS7_URI="redis://localhost:16385" \ TEST_REDIS5_URI="redis://localhost:16383" \ diff --git a/docker-compose.yml b/docker-compose.yml index 493c9b66..fe625d0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,12 +7,38 @@ services: - "16385:6379" - "6379:6379" + redis74-tls: + image: redis:7.4 + volumes: + - ./contrib/tls:/tls + command: | + valkey-server --enable-debug-command yes --protected-mode no + --tls-port 6379 --port 0 + --tls-cert-file /tls/redis.crt + --tls-key-file /tls/redis.key + --tls-ca-cert-file /tls/ca.crt + ports: + - "16387:6379" + valkey8: image: valkey/valkey:8 command: "valkey-server --enable-debug-command yes --protected-mode no" ports: - "16382:6379" + valkey8-tls: + image: valkey/valkey:8 + volumes: + - ./contrib/tls:/tls + command: | + valkey-server --enable-debug-command yes --protected-mode no + --tls-port 6379 --port 0 + --tls-cert-file /tls/redis.crt + --tls-key-file /tls/redis.key + --tls-ca-cert-file /tls/ca.crt + ports: + - "16386:6379" + valkey7: image: valkey/valkey:7.2 command: "valkey-server --enable-debug-command yes --protected-mode no" diff --git a/exporter/exporter.go b/exporter/exporter.go index 1796c80e..f0252d4c 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -86,11 +86,20 @@ type Options struct { } // NewRedisExporter returns a new exporter of Redis metrics. -func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) { +func NewRedisExporter(uri string, opts Options) (*Exporter, error) { log.Debugf("NewRedisExporter options: %#v", opts) + switch { + case strings.HasPrefix(uri, "valkey://"): + uri = strings.Replace(uri, "valkey://", "redis://", 1) + case strings.HasPrefix(uri, "valkeys://"): + uri = strings.Replace(uri, "valkeys://", "rediss://", 1) + } + + log.Debugf("NewRedisExporter = using redis uri: %s", uri) + e := &Exporter{ - redisAddr: redisURI, + redisAddr: uri, options: opts, namespace: opts.Namespace, diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index 6cdec3d0..2a8ef161 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -160,7 +160,7 @@ func setupDBKeys(t *testing.T, uri string) error { } func setupDBKeysCluster(t *testing.T, uri string) error { - e := Exporter{redisAddr: uri} + e, _ := NewRedisExporter(uri, Options{}) c, err := e.connectToRedisCluster() if err != nil { return err @@ -193,7 +193,7 @@ func deleteKeysFromDB(t *testing.T, addr string) error { } func deleteKeysFromDBCluster(addr string) error { - e := Exporter{redisAddr: addr} + e, _ := NewRedisExporter(addr, Options{}) c, err := e.connectToRedisCluster() if err != nil { return err diff --git a/exporter/http_test.go b/exporter/http_test.go index e5924a4c..89e0d20f 100644 --- a/exporter/http_test.go +++ b/exporter/http_test.go @@ -229,6 +229,11 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) { os.Getenv("TEST_REDIS_URI"), os.Getenv("TEST_REDIS_2_8_URI"), + os.Getenv("TEST_REDIS7_URI"), + os.Getenv("TEST_REDIS7_TLS_URI"), + + os.Getenv("TEST_VALKEY8_URI"), + os.Getenv("TEST_KEYDB01_URI"), os.Getenv("TEST_KEYDB02_URI"), diff --git a/exporter/redis.go b/exporter/redis.go index 05bc6b98..3d9be768 100644 --- a/exporter/redis.go +++ b/exporter/redis.go @@ -45,13 +45,6 @@ func (e *Exporter) connectToRedis() (redis.Conn, error) { uri = "redis://" + uri } - switch { - case strings.HasPrefix(uri, "valkey://"): - uri = strings.Replace(uri, "valkey://", "redis://", 1) - case strings.HasPrefix(uri, "valkeys://"): - uri = strings.Replace(uri, "valkeys://", "rediss://", 1) - } - options, err := e.configureOptions(uri) if err != nil { return nil, err diff --git a/exporter/redis_test.go b/exporter/redis_test.go index b265a910..76515ff3 100644 --- a/exporter/redis_test.go +++ b/exporter/redis_test.go @@ -31,7 +31,6 @@ func TestHostVariations(t *testing.T) { } } -// todo: also test valkeys://... func TestValkeyScheme(t *testing.T) { host := os.Getenv("TEST_VALKEY8_URI") diff --git a/exporter/tls_test.go b/exporter/tls_test.go index 0789a217..5133e3c2 100644 --- a/exporter/tls_test.go +++ b/exporter/tls_test.go @@ -1,7 +1,11 @@ package exporter import ( + "os" + "strings" "testing" + + "github.com/prometheus/client_golang/prometheus" ) func TestCreateClientTLSConfig(t *testing.T) { @@ -40,6 +44,57 @@ func TestCreateClientTLSConfig(t *testing.T) { } } +func TestValkeyTLSScheme(t *testing.T) { + + for _, host := range []string{ + os.Getenv("TEST_REDIS7_TLS_URI"), + os.Getenv("TEST_VALKEY8_TLS_URI"), + } { + + e, _ := NewRedisExporter(host, + Options{ + SkipTLSVerification: true, + ClientCertFile: "../contrib/tls/redis.crt", + ClientKeyFile: "../contrib/tls/redis.key", + }, + ) + c, err := e.connectToRedis() + if err != nil { + t.Fatalf("connectToRedis() err: %s", err) + } + + if _, err := c.Do("PING", ""); err != nil { + t.Errorf("PING err: %s", err) + } + + c.Close() + + chM := make(chan prometheus.Metric) + go func() { + e.Collect(chM) + close(chM) + }() + + tsts := []struct { + in string + found bool + }{ + {in: "db_keys"}, + {in: "commands_total"}, + {in: "total_connections_received"}, + {in: "used_memory"}, + } + for m := range chM { + desc := m.Desc().String() + for i := range tsts { + if strings.Contains(desc, tsts[i].in) { + tsts[i].found = true + } + } + } + } +} + func TestCreateServerTLSConfig(t *testing.T) { e := getTestExporter() From f6ce5ff39a7aafe52de4705739d002b0feb9865a Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 14 Oct 2024 16:28:43 -0700 Subject: [PATCH 3/3] test valkey 8 --- .github/workflows/tests.yml | 13 +++++- Makefile | 8 ++-- docker-compose.yml | 6 +-- exporter/http_test.go | 3 +- exporter/streams_test.go | 4 +- exporter/tls_test.go | 82 +++++++++++++++++++------------------ 6 files changed, 65 insertions(+), 51 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6649308a..50ecb447 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,6 @@ on: jobs: test-stuff: runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 @@ -21,6 +20,12 @@ jobs: - name: Set up Docker Compose run: sudo apt-get install docker-compose + # need to do this before we start the services as they need the TLS creds + - name: Create test certs for TLS + run: | + make test-certs + chmod 777 ./contrib/tls/* + - name: Start services run: docker-compose up -d working-directory: ./ @@ -33,6 +38,12 @@ jobs: - name: Install Dependencies run: go mod tidy + - name: Docker logs + run: | + echo "${{ toJson(job) }}" + docker logs "redis_exporter_redis7-tls_1" + docker logs "redis_exporter_valkey8-tls_1" + - name: Run tests env: LOG_LEVEL: "info" diff --git a/Makefile b/Makefile index 58490303..369a080c 100644 --- a/Makefile +++ b/Makefile @@ -27,15 +27,17 @@ docker-test: $(DOCKER_COMPOSE) -f docker-compose.yml run --rm tests bash -c 'make test' +.PHONY: test-certs +test-certs: + contrib/tls/gen-test-certs.sh + .PHONY: test test: - contrib/tls/gen-test-certs.sh - TEST_VALKEY7_URI="valkey://localhost:16384" \ TEST_VALKEY8_URI="valkey://localhost:16382" \ TEST_VALKEY8_TLS_URI="valkeys://localhost:16386" \ - TEST_REDIS7_TLS_URI="rediss://localhost:16386" \ + TEST_REDIS7_TLS_URI="rediss://localhost:16387" \ TEST_REDIS_URI="redis://localhost:16385" \ TEST_REDIS7_URI="redis://localhost:16385" \ TEST_REDIS5_URI="redis://localhost:16383" \ diff --git a/docker-compose.yml b/docker-compose.yml index fe625d0d..d06bfde0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,18 @@ services: - redis74: + redis7: image: redis:7.4 command: "redis-server --enable-debug-command yes --protected-mode no" ports: - "16385:6379" - "6379:6379" - redis74-tls: + redis7-tls: image: redis:7.4 volumes: - ./contrib/tls:/tls command: | - valkey-server --enable-debug-command yes --protected-mode no + redis-server --enable-debug-command yes --protected-mode no --tls-port 6379 --port 0 --tls-cert-file /tls/redis.crt --tls-key-file /tls/redis.key diff --git a/exporter/http_test.go b/exporter/http_test.go index 89e0d20f..236b3f95 100644 --- a/exporter/http_test.go +++ b/exporter/http_test.go @@ -168,7 +168,6 @@ func TestHTTPScrapeMetricsEndpoints(t *testing.T) { // labels and label values `redis_mode`, - `standalone`, `cmd="config`, "maxmemory_policy", @@ -230,8 +229,8 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) { os.Getenv("TEST_REDIS_2_8_URI"), os.Getenv("TEST_REDIS7_URI"), - os.Getenv("TEST_REDIS7_TLS_URI"), + os.Getenv("TEST_VALKEY7_URI"), os.Getenv("TEST_VALKEY8_URI"), os.Getenv("TEST_KEYDB01_URI"), diff --git a/exporter/streams_test.go b/exporter/streams_test.go index 0454b6d6..78dc230b 100644 --- a/exporter/streams_test.go +++ b/exporter/streams_test.go @@ -106,7 +106,7 @@ func TestStreamsGetStreamInfoUsingValKey7(t *testing.T) { t.Skipf("TEST_VALKEY7_URI not set - skipping") } - addr := os.Getenv("TEST_VALKEY7_URI") + addr := strings.Replace(os.Getenv("TEST_VALKEY7_URI"), "valkey://", "redis://", 1) c, err := redis.DialURL(addr) if err != nil { t.Fatalf("Couldn't connect to %#v: %#v", addr, err) @@ -285,7 +285,7 @@ func TestStreamsScanStreamGroupsUsingValKey7(t *testing.T) { if os.Getenv("TEST_VALKEY7_URI") == "" { t.Skipf("TEST_VALKEY7_URI not set - skipping") } - addr := os.Getenv("TEST_VALKEY7_URI") + addr := strings.Replace(os.Getenv("TEST_VALKEY7_URI"), "valkey://", "redis://", 1) db := dbNumStr c, err := redis.DialURL(addr) diff --git a/exporter/tls_test.go b/exporter/tls_test.go index 5133e3c2..a490f9fc 100644 --- a/exporter/tls_test.go +++ b/exporter/tls_test.go @@ -45,53 +45,55 @@ func TestCreateClientTLSConfig(t *testing.T) { } func TestValkeyTLSScheme(t *testing.T) { - for _, host := range []string{ os.Getenv("TEST_REDIS7_TLS_URI"), os.Getenv("TEST_VALKEY8_TLS_URI"), } { + t.Run(host, func(t *testing.T) { + + e, _ := NewRedisExporter(host, + Options{ + SkipTLSVerification: true, + ClientCertFile: "../contrib/tls/redis.crt", + ClientKeyFile: "../contrib/tls/redis.key", + }, + ) + c, err := e.connectToRedis() + if err != nil { + t.Fatalf("connectToRedis() err: %s", err) + } + + if _, err := c.Do("PING", ""); err != nil { + t.Errorf("PING err: %s", err) + } - e, _ := NewRedisExporter(host, - Options{ - SkipTLSVerification: true, - ClientCertFile: "../contrib/tls/redis.crt", - ClientKeyFile: "../contrib/tls/redis.key", - }, - ) - c, err := e.connectToRedis() - if err != nil { - t.Fatalf("connectToRedis() err: %s", err) - } - - if _, err := c.Do("PING", ""); err != nil { - t.Errorf("PING err: %s", err) - } - - c.Close() - - chM := make(chan prometheus.Metric) - go func() { - e.Collect(chM) - close(chM) - }() - - tsts := []struct { - in string - found bool - }{ - {in: "db_keys"}, - {in: "commands_total"}, - {in: "total_connections_received"}, - {in: "used_memory"}, - } - for m := range chM { - desc := m.Desc().String() - for i := range tsts { - if strings.Contains(desc, tsts[i].in) { - tsts[i].found = true + c.Close() + + chM := make(chan prometheus.Metric) + go func() { + e.Collect(chM) + close(chM) + }() + + tsts := []struct { + in string + found bool + }{ + {in: "db_keys"}, + {in: "commands_total"}, + {in: "total_connections_received"}, + {in: "used_memory"}, + } + for m := range chM { + desc := m.Desc().String() + for i := range tsts { + if strings.Contains(desc, tsts[i].in) { + tsts[i].found = true + } } } - } + + }) } }