Merge branch 'main' into cortex-config-monitoring-cluster

.drone.yml

@@ -19,7 +19,7 @@ steps:
         -e _EXPERIMENTAL_DAGGER_CACHESERVICE_TOKEN=$_EXPERIMENTAL_DAGGER_CACHESERVICE_TOKEN \
         -e _EXPERIMENTAL_DAGGER_CACHESERVICE_URL=https://api.dagger.cloud/magicache \
         -v dagger-engine:/var/lib/dagger \
-        registry.dagger.io/engine:v0.8.3
+        registry.dagger.io/engine:v0.8.4
     volumes: &volumes
       - name: docker-sock
         path: /var/run/docker.sock

dagger/go.mod

@@ -3,7 +3,7 @@ module github.com/rancher/opni/dagger
 go 1.21
 
 require (
-	dagger.io/dagger v0.8.3
+	dagger.io/dagger v0.8.4
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/go-playground/validator/v10 v10.14.1

dagger/go.sum

@@ -1,5 +1,5 @@
-dagger.io/dagger v0.8.3 h1:FEUexo5Rx9ns7/YYFtEcceI/g62neasBTHB2d5axK+s=
-dagger.io/dagger v0.8.3/go.mod h1:Nwl7WI8YETaZhGjPJvkiOZnKLJXBaJOkSarp5m4+FxA=
+dagger.io/dagger v0.8.4 h1:2zNu40cBvPZc/CSgMnLRfayfQj5VtbJlDtWJyWGwGSQ=
+dagger.io/dagger v0.8.4/go.mod h1:Nwl7WI8YETaZhGjPJvkiOZnKLJXBaJOkSarp5m4+FxA=
 github.com/99designs/gqlgen v0.17.36 h1:u/o/rv2SZ9s5280dyUOOrkpIIkr/7kITMXYD3rkJ9go=
 github.com/99designs/gqlgen v0.17.36/go.mod h1:6RdyY8puhCoWAQVr2qzF2OMVfudQzc8ACxzpzluoQm4=
 github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=

go.mod

@@ -563,7 +563,7 @@ replace (
 	github.com/opensearch-project/opensearch-go/v2 => github.com/dbason/opensearch-go/v2 v2.0.0-20221202021211-6aec8f80bc41
 	github.com/rancher/charts-build-scripts => github.com/dbason/charts-build-scripts v0.3.4-0.20220429024555-807c076e8116
 	go.uber.org/zap => github.com/kralicky/zap v1.24.1-0.20230718165024-a2256218e4cc
-	opensearch.opster.io => github.com/dbason/opensearch-k8s-operator/opensearch-operator v0.0.0-20230809020457-b60fd235fc1d
+	opensearch.opster.io => github.com/dbason/opensearch-k8s-operator/opensearch-operator v0.0.0-20230911044558-c677eb01b26a
 )
 
 // Cortex replacements (copied from cortex go.mod)

go.sum

@@ -818,8 +818,8 @@ github.com/dbason/featureflags v0.0.0-20220803034705-b6242a8d72b2 h1:GPBUVhDuyLG
 github.com/dbason/featureflags v0.0.0-20220803034705-b6242a8d72b2/go.mod h1:ew+9wAV4nyz0ZKybKcM4cMosZwggyAkzU3BU/wZxHTA=
 github.com/dbason/opensearch-go/v2 v2.0.0-20221202021211-6aec8f80bc41 h1:YhfKUAl5zCycdjDw8/nFN8wodEbaXzovvtMmaFDmWYE=
 github.com/dbason/opensearch-go/v2 v2.0.0-20221202021211-6aec8f80bc41/go.mod h1:Ny7Xx3xtC8YRYICmolmndYzxD8sTDFhJfgs3UW/IIj4=
-github.com/dbason/opensearch-k8s-operator/opensearch-operator v0.0.0-20230809020457-b60fd235fc1d h1:/qRYb8EAviWyGXHe258/CjTxURgI0sUncg1Equry5kc=
-github.com/dbason/opensearch-k8s-operator/opensearch-operator v0.0.0-20230809020457-b60fd235fc1d/go.mod h1:86ddHdDQGnMvWTg/KLD5VMpMuCJTwZXIW0oB9pzGA9U=
+github.com/dbason/opensearch-k8s-operator/opensearch-operator v0.0.0-20230911044558-c677eb01b26a h1:YTccp0Y/fdW1ot/3KunED3fpM1jsYN79F3/DjiQRwjs=
+github.com/dbason/opensearch-k8s-operator/opensearch-operator v0.0.0-20230911044558-c677eb01b26a/go.mod h1:86ddHdDQGnMvWTg/KLD5VMpMuCJTwZXIW0oB9pzGA9U=
 github.com/dbason/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230704041503-6f0f267c2d92 h1:6WUcdWI9zLMuIjGXPuxU0XCsO2S28teXBHEoooBzJFs=
 github.com/dbason/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230704041503-6f0f267c2d92/go.mod h1:Ah++gPjqLE2u+jVoAplt+dySHThvdXAgWn1se/fDVxw=
 github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=

magefiles/targets/dagger.go

@@ -1,9 +1,15 @@
 package targets
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"fmt"
+	"go/build"
+	"io"
+	"net/http"
 	"os"
 	"os/exec"
+	"path"
 
 	"slices"
 
@@ -61,3 +67,54 @@ func (ns Dagger) Help() error {
 func (Dagger) Setup() error {
 	return sh.RunV(mg.GoCmd(), "run", string(dagger), "--setup")
 }
+
+// Installs or updates the Dagger CLI to ~/go/bin/dagger
+func (Dagger) Install() error {
+	modVersion, err := sh.Output(mg.GoCmd(), "list", "-m", "-f", "{{.Version}}", "dagger.io/dagger")
+	if err != nil {
+		return err
+	}
+
+	gopath, goos, goarch := build.Default.GOPATH, build.Default.GOOS, build.Default.GOARCH
+	url := fmt.Sprintf("https://github.com/dagger/dagger/releases/download/%[1]s/dagger_%[1]s_%s_%s.tar.gz", modVersion, goos, goarch)
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("could not download %s: %s", url, resp.Status)
+	}
+
+	gzReader, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer gzReader.Close()
+	tarReader := tar.NewReader(gzReader)
+	var header *tar.Header
+	for {
+		if header, err = tarReader.Next(); err != nil {
+			return err
+		}
+		if header.Name == "dagger" {
+			break
+		}
+	}
+	if header == nil {
+		return fmt.Errorf("could not find dagger binary in release archive")
+	}
+
+	outFilename := path.Join(gopath, "bin", "dagger")
+	f, err := os.OpenFile(outFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, header.FileInfo().Mode())
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := io.CopyN(f, tarReader, header.Size); err != nil {
+		return err
+	}
+
+	fmt.Printf("Installed dagger %s to %s\n", modVersion, outFilename)
+	return nil
+}

plugins/logging/pkg/errors/errors.go

@@ -11,6 +11,7 @@ import (
 var (
 	ErrInvalidList             = utilerrors.New(codes.FailedPrecondition, errors.New("list did not return exactly 1 result"))
 	ErrInvalidPersistence      = utilerrors.New(codes.InvalidArgument, errors.New("invalid persistence config"))
+	ErrInvalidDataPersistence  = utilerrors.New(codes.InvalidArgument, errors.New("minimum of 2 data nodes required if no persistent storage"))
 	ErrClusterIDMissing        = utilerrors.New(codes.InvalidArgument, errors.New("request does not include cluster ID"))
 	ErrOpensearchResponse      = utilerrors.New(codes.Unavailable, errors.New("opensearch request unsuccessful"))
 	ErrNoOpensearchClient      = utilerrors.New(codes.Unavailable, errors.New("opensearch client is not set"))

plugins/logging/pkg/gateway/admin_v2.go

@@ -102,7 +102,12 @@ func (m *LoggingManagerV2) CreateOrUpdateOpensearchCluster(ctx context.Context,
 	}
 	// Input should always have a data nodes field
 	if cluster.GetDataNodes() == nil {
-		return &emptypb.Empty{}, loggingerrors.ErrInvalidDuration
+		return &emptypb.Empty{}, loggingerrors.ErrMissingDataNode
+	}
+
+	// Don't allow a single data node with no persistence
+	if err := m.validateStorage(cluster.GetDataNodes()); err != nil {
+		return nil, err
 	}
 
 	go m.opensearchManager.SetClient(m.managementDriver.NewOpensearchClientForCluster)
@@ -259,6 +264,14 @@ func (m *LoggingManagerV2) validDurationString(duration string) bool {
 	return match
 }
 
+func (m *LoggingManagerV2) validateStorage(dataNodes *loggingadmin.DataDetails) error {
+	if dataNodes.GetReplicas() < 2 && !dataNodes.GetPersistence().GetEnabled() {
+		m.logger.Error("minimum of 2 data nodes required if no persistent storage")
+		return loggingerrors.ErrInvalidDataPersistence
+	}
+	return nil
+}
+
 func (m *LoggingManagerV2) opensearchClusterReady() bool {
 	absentRetriesMax := 3
 	ctx := context.TODO()