Use e2edb.NewMinio to disable SSE-S3 in e2e tests (thanos-io#6055)

* Use e2edb.NewMinio to disable SSE Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com> * Use temp fork for TLS Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com> * Fix broken rules api fanout test Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com> * Fix broken query compatibility test Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com> * Remove fork Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com> Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>
ngraham20 · Apr 17, 2023 · 18c6cce · 18c6cce
1 parent a7e9ea8
commit 18c6cce
Show file tree

Hide file tree

Showing 9 changed files with 49 additions and 172 deletions.
diff --git a/go.mod b/go.mod
@@ -18,7 +18,7 @@ require (
 	github.com/chromedp/chromedp v0.8.2
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.0
-	github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a
+	github.com/efficientgo/e2e v0.14.1-0.20230119090947-fa7ceb0197c5
 	github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/fatih/structtag v1.2.0

diff --git a/go.sum b/go.sum
@@ -247,8 +247,8 @@ github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 h1:rydBwnBoywKQMjWF0z8SriYtQ+uUcaFsxuijMjJr5PI=
 github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4/go.mod h1:kQa0V74HNYMfuJH6jiPiwNdpWXl4xd/K4tzlrcvYDQI=
-github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a h1:cnJajqeh/HjvJLhI3wPvWG9OQ4gU79+4pELRD5Pkih8=
-github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a/go.mod h1:Hi+sz0REtlhVZ8zcdeTC3j6LUEEpJpPtNjOaOKuNcgI=
+github.com/efficientgo/e2e v0.14.1-0.20230119090947-fa7ceb0197c5 h1:N1fHVcNEPMJNB93sxT6icl5yvoFxa2sp3/1NnVlrAFc=
+github.com/efficientgo/e2e v0.14.1-0.20230119090947-fa7ceb0197c5/go.mod h1:plsKU0YHE9uX+7utvr7SiDtVBSHJyEfHRO4UnUgDmts=
 github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd h1:VaYzzXeUbC5fVheskcKVNOyJMEYD+HgrJNzIAg/mRIM=
 github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd/go.mod h1:ZV0utlglOczUWv3ih2AbqPSoLoFzdplUYxwV62eZi6Q=
 github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=

diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go
@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/efficientgo/e2e"
+	e2edb "github.com/efficientgo/e2e/db"
 	e2emon "github.com/efficientgo/e2e/monitoring"
 	"github.com/efficientgo/e2e/monitoring/matchers"
 	"github.com/go-kit/log"
@@ -347,11 +348,11 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) {
 	testutil.Ok(t, os.MkdirAll(dir, os.ModePerm))
 
 	const bucket = "compact-test"
-	m := e2ethanos.NewMinio(e, "minio", bucket)
+	m := e2edb.NewMinio(e, "minio", bucket, e2edb.WithMinioTLS())
 	testutil.Ok(t, e2e.StartAndWaitReady(m))
 
 	bkt, err := s3.NewBucketWithConfig(logger,
-		e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test-feed")
+		e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed")
 	testutil.Ok(t, err)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
@@ -454,7 +455,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) {
 
 	bktConfig := client.BucketConfig{
 		Type:   client.S3,
-		Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), m.InternalDir()),
+		Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
 	}
 
 	// Crank down the deletion mark delay since deduplication can miss blocks in the presence of replica labels it doesn't know about.

diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go
@@ -4,14 +4,8 @@
 package e2ethanos
 
 import (
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
 	"encoding/json"
-	"encoding/pem"
 	"fmt"
-	"math/big"
-	"net"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -31,9 +25,9 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/thanos-io/objstore/client"
-	"github.com/thanos-io/objstore/exthttp"
 	"github.com/thanos-io/objstore/providers/s3"
 
+	"github.com/thanos-io/objstore/exthttp"
 	"github.com/thanos-io/thanos/pkg/alert"
 	"github.com/thanos-io/thanos/pkg/httpconfig"
 
@@ -945,57 +939,6 @@ http {
 	}), "http")
 }
 
-// NewMinio returns minio server, used as a local replacement for S3.
-// TODO(@matej-g): This is a temporary workaround for https://github.com/efficientgo/e2e/issues/11;
-// after this is addresses fixed all calls should be replaced with e2edb.NewMinio.
-func NewMinio(e e2e.Environment, name, bktName string) *e2emon.InstrumentedRunnable {
-	image := "minio/minio:RELEASE.2022-07-30T05-21-40Z"
-	minioKESGithubContent := "https://raw.githubusercontent.com/minio/kes/master"
-
-	httpsPort := 8090
-	consolePort := 8080
-	f := e.Runnable(fmt.Sprintf("minio-%s", name)).
-		WithPorts(map[string]int{"https": httpsPort, "console": consolePort}).
-		Future()
-
-	if err := os.MkdirAll(filepath.Join(f.Dir(), "certs", "CAs"), 0750); err != nil {
-		return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "create certs dir"))}
-	}
-
-	if err := genCerts(
-		filepath.Join(f.Dir(), "certs", "public.crt"),
-		filepath.Join(f.Dir(), "certs", "private.key"),
-		filepath.Join(f.Dir(), "certs", "CAs", "ca.crt"),
-		fmt.Sprintf("%s-minio-%s", e.Name(), name),
-	); err != nil {
-		return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "fail to generate certs"))}
-	}
-
-	commands := []string{
-		fmt.Sprintf("curl -sSL --tlsv1.2 -O '%s/root.key' -O '%s/root.cert'", minioKESGithubContent, minioKESGithubContent),
-		fmt.Sprintf("mkdir -p /data/%s && minio server --certs-dir %s/certs --address :%v --console-address :%v /data", bktName, f.InternalDir(), httpsPort, consolePort),
-	}
-
-	minio := e2emon.AsInstrumented(f.Init(e2e.StartOptions{
-		Image: image,
-		// Create the required bucket before starting minio.
-		Command:   e2e.NewCommandWithoutEntrypoint("sh", "-c", strings.Join(commands, " && ")),
-		Readiness: e2e.NewHTTPSReadinessProbe("console", "/", 200, 200),
-		EnvVars: map[string]string{
-			"MINIO_ROOT_USER":     e2edb.MinioAccessKey,
-			"MINIO_ROOT_PASSWORD": e2edb.MinioSecretKey,
-			"MINIO_BROWSER":       "on",
-			"ENABLE_HTTPS":        "1",
-			// https://docs.min.io/docs/minio-kms-quickstart-guide.html
-			"MINIO_KMS_KES_ENDPOINT":  "https://play.min.io:7373",
-			"MINIO_KMS_KES_KEY_FILE":  "root.key",
-			"MINIO_KMS_KES_CERT_FILE": "root.cert",
-			"MINIO_KMS_KES_KEY_NAME":  "my-minio-key",
-		},
-	}), "https")
-	return minio
-}
-
 func NewMemcached(e e2e.Environment, name string) *e2emon.InstrumentedRunnable {
 	return e2emon.AsInstrumented(e.Runnable(fmt.Sprintf("memcached-%s", name)).
 		WithPorts(map[string]int{"memcached": 11211}).
@@ -1061,76 +1004,6 @@ func NewToolsBucketWeb(
 	})), "http")
 }
 
-// genCerts generates certificates and writes those to the provided paths.
-func genCerts(certPath, privkeyPath, caPath, serverName string) error {
-	var caRoot = &x509.Certificate{
-		SerialNumber:          big.NewInt(2019),
-		NotAfter:              time.Now().AddDate(10, 0, 0),
-		IsCA:                  true,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
-		BasicConstraintsValid: true,
-	}
-
-	var cert = &x509.Certificate{
-		SerialNumber: big.NewInt(1658),
-		DNSNames:     []string{serverName},
-		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
-		NotAfter:     time.Now().AddDate(10, 0, 0),
-		SubjectKeyId: []byte{1, 2, 3},
-		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
-		KeyUsage:     x509.KeyUsageDigitalSignature,
-	}
-
-	caPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		return err
-	}
-
-	certPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		return err
-	}
-	// Generate CA cert.
-	caBytes, err := x509.CreateCertificate(rand.Reader, caRoot, caRoot, &caPrivKey.PublicKey, caPrivKey)
-	if err != nil {
-		return err
-	}
-	caPEM := pem.EncodeToMemory(&pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: caBytes,
-	})
-	err = os.WriteFile(caPath, caPEM, 0644)
-	if err != nil {
-		return err
-	}
-
-	// Sign the cert with the CA private key.
-	certBytes, err := x509.CreateCertificate(rand.Reader, cert, caRoot, &certPrivKey.PublicKey, caPrivKey)
-	if err != nil {
-		return err
-	}
-	certPEM := pem.EncodeToMemory(&pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: certBytes,
-	})
-	err = os.WriteFile(certPath, certPEM, 0644)
-	if err != nil {
-		return err
-	}
-
-	certPrivKeyPEM := pem.EncodeToMemory(&pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
-	})
-	err = os.WriteFile(privkeyPath, certPrivKeyPEM, 0644)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func NewS3Config(bucket, endpoint, basePath string) s3.Config {
 	httpDefaultConf := s3.DefaultConfig.HTTPConfig
 	httpDefaultConf.TLSConfig = exthttp.TLSConfig{

diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go
@@ -19,6 +19,7 @@ import (
 	"github.com/thanos-io/objstore/client"
 
 	"github.com/efficientgo/core/testutil"
+	e2edb "github.com/efficientgo/e2e/db"
 	"github.com/thanos-io/thanos/pkg/query"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	"github.com/thanos-io/thanos/test/e2e/e2ethanos"
@@ -37,14 +38,14 @@ func TestInfo(t *testing.T) {
 	testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3))
 
 	const bucket = "info-api-test"
-	m := e2ethanos.NewMinio(e, "thanos-minio", bucket)
+	m := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
 	testutil.Ok(t, e2e.StartAndWaitReady(m))
 	store := e2ethanos.NewStoreGW(
 		e,
 		"1",
 		client.BucketConfig{
 			Type:   client.S3,
-			Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), m.InternalDir()),
+			Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
 		},
 		"",
 		nil,

diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go
@@ -542,7 +542,7 @@ config:
 				ruleAndAssert(t, ctx, q.Endpoint("http"), "", []*rulespb.RuleGroup{
 					{
 						Name: "example_abort",
-						File: "/shared/data/querier-1/rules/rules.yaml",
+						File: q.Dir() + "/rules/rules.yaml",
 						Rules: []*rulespb.Rule{
 							rulespb.NewAlertingRule(&rulespb.Alert{
 								Name:  "TestAlert_AbortOnPartialResponse",
@@ -591,11 +591,11 @@ func TestQueryStoreMetrics(t *testing.T) {
 	t.Cleanup(cancel)
 
 	bucket := "store-gw-test"
-	minio := e2ethanos.NewMinio(e, "thanos-minio", bucket)
+	minio := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
 	testutil.Ok(t, e2e.StartAndWaitReady(minio))
 
 	l := log.NewLogfmtLogger(os.Stdout)
-	bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("https"), minio.Dir()), "test")
+	bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test")
 	testutil.Ok(t, err)
 
 	// Preparing 2 different blocks for the tests.
@@ -639,7 +639,7 @@ func TestQueryStoreMetrics(t *testing.T) {
 		"s1",
 		client.BucketConfig{
 			Type:   client.S3,
-			Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("https"), minio.InternalDir()),
+			Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("http"), minio.InternalDir()),
 		},
 		"",
 		nil,
@@ -719,7 +719,7 @@ func TestSidecarStorePushdown(t *testing.T) {
 	testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1))
 
 	const bucket = "store-gateway-test"
-	m := e2ethanos.NewMinio(e, "thanos-minio", bucket)
+	m := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
 	testutil.Ok(t, e2e.StartAndWaitReady(m))
 
 	dir := filepath.Join(e.SharedDir(), "tmp")
@@ -736,7 +736,7 @@ func TestSidecarStorePushdown(t *testing.T) {
 	testutil.Ok(t, err)
 
 	l := log.NewLogfmtLogger(os.Stdout)
-	bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test")
+	bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test")
 	testutil.Ok(t, err)
 	testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String()))
 
@@ -745,7 +745,7 @@ func TestSidecarStorePushdown(t *testing.T) {
 		"1",
 		client.BucketConfig{
 			Type:   client.S3,
-			Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), m.InternalDir()),
+			Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
 		},
 		"",
 		nil,

diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go
@@ -96,7 +96,7 @@ func TestRulesAPI_Fanout(t *testing.T) {
 	ruleAndAssert(t, ctx, q.Endpoint("http"), "", []*rulespb.RuleGroup{
 		{
 			Name: "example_abort",
-			File: "/shared/data/querier-query/rules/rules.yaml",
+			File: q.Dir() + "/rules/rules.yaml",
 			Rules: []*rulespb.Rule{
 				rulespb.NewAlertingRule(&rulespb.Alert{
 					Name:  "TestAlert_AbortOnPartialResponse",
@@ -112,7 +112,7 @@ func TestRulesAPI_Fanout(t *testing.T) {
 		},
 		{
 			Name: "example_abort",
-			File: "/shared/data/querier-query/thanos-rules/rules-0.yaml",
+			File: q.Dir() + "/thanos-rules/rules-0.yaml",
 			Rules: []*rulespb.Rule{
 				rulespb.NewAlertingRule(&rulespb.Alert{
 					Name:  "TestAlert_AbortOnPartialResponse",
@@ -127,7 +127,7 @@ func TestRulesAPI_Fanout(t *testing.T) {
 		},
 		{
 			Name: "example_warn",
-			File: "/shared/data/querier-query/thanos-rules/rules-1.yaml",
+			File: q.Dir() + "/thanos-rules/rules-1.yaml",
 			Rules: []*rulespb.Rule{
 				rulespb.NewAlertingRule(&rulespb.Alert{
 					Name:  "TestAlert_WarnOnPartialResponse",
@@ -142,7 +142,7 @@ func TestRulesAPI_Fanout(t *testing.T) {
 		},
 		{
 			Name:  "example_with_limit",
-			File:  "/shared/data/querier-query/thanos-rules/rules-with-limit.yaml",
+			File:  q.Dir() + "/thanos-rules/rules-with-limit.yaml",
 			Limit: 1,
 			Rules: []*rulespb.Rule{
 				rulespb.NewAlertingRule(&rulespb.Alert{