Skip to content

Commit

Permalink
Dispatch abstractions for 5 more APIs (vertica#424)
Browse files Browse the repository at this point in the history 
More work to move admintools commands behind the dispatch abstraction.
This is work needed for our vclusterOps integration. This PR adds the
following interfaces:
- stop DB
- db add node.
- db remove node
- remove subcluster
- add subcluster

---------

Co-authored-by: Roy Paulin <paulin.nguetsop@yahoo.com>
  • Loading branch information
@roypaulin
spilchen and roypaulin authored Jun 25, 2023
1 parent 06ebffb commit a6efce7
Show file tree
Hide file tree
Showing 38 changed files with 1,135 additions and 208 deletions.
85 changes: 31 additions & 54 deletions pkg/controllers/vdb/dbaddnode_reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package vdb
import (
"context"
"sort"
"strings"
"time"

"github.com/go-logr/logr"
Expand All @@ -27,24 +26,34 @@ import (
"github.com/vertica/vertica-kubernetes/pkg/controllers"
verrors "github.com/vertica/vertica-kubernetes/pkg/errors"
"github.com/vertica/vertica-kubernetes/pkg/events"
"github.com/vertica/vertica-kubernetes/pkg/names"
"github.com/vertica/vertica-kubernetes/pkg/vadmin"
"github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/addnode"
corev1 "k8s.io/api/core/v1"
ctrl "sigs.k8s.io/controller-runtime"
)

// DBAddNodeReconciler will ensure each pod is added to the database.
type DBAddNodeReconciler struct {
VRec *VerticaDBReconciler
Log logr.Logger
Vdb *vapi.VerticaDB // Vdb is the CRD we are acting on.
PRunner cmds.PodRunner
PFacts *PodFacts
VRec *VerticaDBReconciler
Log logr.Logger
Vdb *vapi.VerticaDB // Vdb is the CRD we are acting on.
PRunner cmds.PodRunner
PFacts *PodFacts
Dispatcher vadmin.Dispatcher
}

// MakeDBAddNodeReconciler will build a DBAddNodeReconciler object
func MakeDBAddNodeReconciler(vdbrecon *VerticaDBReconciler, log logr.Logger,
vdb *vapi.VerticaDB, prunner cmds.PodRunner, pfacts *PodFacts) controllers.ReconcileActor {
return &DBAddNodeReconciler{VRec: vdbrecon, Log: log, Vdb: vdb, PRunner: prunner, PFacts: pfacts}
vdb *vapi.VerticaDB, prunner cmds.PodRunner, pfacts *PodFacts, dispatcher vadmin.Dispatcher,
) controllers.ReconcileActor {
return &DBAddNodeReconciler{
VRec: vdbrecon,
Log: log,
Vdb: vdb,
PRunner: prunner,
PFacts: pfacts,
Dispatcher: dispatcher,
}
}

// Reconcile will ensure a DB exists and create one if it doesn't
Expand Down Expand Up @@ -121,7 +130,7 @@ func (d *DBAddNodeReconciler) reconcileSubcluster(ctx context.Context, sc *vapi.

// runAddNode will add nodes to the given subcluster
func (d *DBAddNodeReconciler) runAddNode(ctx context.Context, pods []*PodFact) (ctrl.Result, error) {
atPod, ok := d.PFacts.findPodToRunVsql(false, "")
initiatorPod, ok := d.PFacts.findPodToRunVsql(false, "")
if !ok {
d.Log.Info("No pod found to run vsql and admintools from. Requeue reconciliation.")
return ctrl.Result{Requeue: true}, nil
Expand All @@ -136,25 +145,17 @@ func (d *DBAddNodeReconciler) runAddNode(ctx context.Context, pods []*PodFact) (
}
}

if d.VRec.OpCfg.DevMode {
debugDumpAdmintoolsConf(ctx, d.PRunner, atPod.name)
}

if stdout, err := d.runAddNodeForPod(ctx, pods, atPod); err != nil {
if err := d.runAddNodeForPod(ctx, pods, initiatorPod); err != nil {
// If we reached the node limit according to the license, end this
// reconcile successfully. We don't want to fail and requeue because
// this isn't going to get fixed until someone manually adds a new
// license.
if isLicenseLimitError(stdout) {
if _, ok := err.(*addnode.LicenseLimitError); ok {
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}

if d.VRec.OpCfg.DevMode {
debugDumpAdmintoolsConf(ctx, d.PRunner, atPod.name)
}

// Invalidate the cached pod facts now that some pods have a DB now.
d.PFacts.Invalidate()

Expand All @@ -163,46 +164,22 @@ func (d *DBAddNodeReconciler) runAddNode(ctx context.Context, pods []*PodFact) (

// runAddNodeForPod will execute the command to add a single node to the cluster
// Returns the stdout from the command.
func (d *DBAddNodeReconciler) runAddNodeForPod(ctx context.Context, pods []*PodFact, atPod *PodFact) (string, error) {
func (d *DBAddNodeReconciler) runAddNodeForPod(ctx context.Context, pods []*PodFact, initiatorPod *PodFact) error {
podNames := genPodNames(pods)
d.VRec.Eventf(d.Vdb, corev1.EventTypeNormal, events.AddNodeStart,
"Calling 'admintools -t db_add_node' for pod(s) '%s'", podNames)
start := time.Now()
cmd := d.genAddNodeCommand(pods)
stdout, _, err := d.PRunner.ExecAdmintools(ctx, atPod.name, names.ServerContainer, cmd...)
opts := []addnode.Option{
addnode.WithInitiator(initiatorPod.name, initiatorPod.podIP),
addnode.WithSubcluster(pods[0].subclusterName),
}
for i := range pods {
opts = append(opts, addnode.WithHost(pods[i].dnsName))
}
err := d.Dispatcher.AddNode(ctx, opts...)
if err != nil {
switch {
case isLicenseLimitError(stdout):
d.VRec.Event(d.Vdb, corev1.EventTypeWarning, events.AddNodeLicenseFail,
"You cannot add more nodes to the database. You have reached the limit allowed by your license.")
default:
d.VRec.Eventf(d.Vdb, corev1.EventTypeWarning, events.AddNodeFailed,
"Failed when calling 'admintools -t db_add_node' for pod(s) '%s'", podNames)
}
} else {
d.VRec.Eventf(d.Vdb, corev1.EventTypeNormal, events.AddNodeSucceeded,
"Successfully called 'admintools -t db_add_node' and it took %s", time.Since(start))
}
return stdout, err
}

// isLicenseLimitError returns true if the stdout contains the error about not enough licenses
func isLicenseLimitError(stdout string) bool {
return strings.Contains(stdout, "Cannot create another node. The current license permits")
}

// genAddNodeCommand returns the command to run to add nodes to the cluster.
func (d *DBAddNodeReconciler) genAddNodeCommand(pods []*PodFact) []string {
hostNames := make([]string, 0, len(pods))
for _, pod := range pods {
hostNames = append(hostNames, pod.dnsName)
}

return []string{
"-t", "db_add_node",
"--hosts", strings.Join(hostNames, ","),
"--database", d.Vdb.Spec.DBName,
"--subcluster", pods[0].subclusterName,
"--noprompt",
}
return err
}
25 changes: 16 additions & 9 deletions pkg/controllers/vdb/dbaddnode_reconciler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ var _ = Describe("dbaddnode_reconcile", func() {

fpr := &cmds.FakePodRunner{}
pfacts := createPodFactsDefault(fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
lastCall := fpr.Histories[len(fpr.Histories)-1]
Expect(lastCall.Command).ShouldNot(ContainElements("/opt/vertica/bin/admintools", "db_add_node"))
Expand All @@ -53,7 +54,8 @@ var _ = Describe("dbaddnode_reconcile", func() {

fpr := &cmds.FakePodRunner{}
pfacts := createPodFactsWithNoDB(ctx, vdb, fpr, 3)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{Requeue: true}))
lastCall := fpr.Histories[len(fpr.Histories)-1]
Expect(lastCall.Command).ShouldNot(ContainElements("/opt/vertica/bin/admintools", "db_add_node"))
Expand All @@ -67,7 +69,8 @@ var _ = Describe("dbaddnode_reconcile", func() {

fpr := &cmds.FakePodRunner{}
pfacts := createPodFactsWithNoDB(ctx, vdb, fpr, 1)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
atCmd := fpr.FindCommands("db_add_node")
Expect(len(atCmd)).Should(Equal(1))
Expand All @@ -91,15 +94,15 @@ var _ = Describe("dbaddnode_reconcile", func() {
// so that it fails because we hit the node limit.
atPod := names.GenPodName(vdb, &vdb.Spec.Subclusters[0], 0)
fpr.Results[atPod] = []cmds.CmdResult{
{}, // Dump admintools.conf
{
Err: errors.New("admintools command failed"),
Stdout: "There was an error adding the nodes to the database: DB client operation \"create nodes\" failed during `ddl`: " +
"Severity: ROLLBACK, Message: Cannot create another node. The current license permits 3 node(s) and the database catalog " +
"already contains 3 node(s), Sqlstate: V2001",
},
}
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
lastCall := fpr.FindCommands("/opt/vertica/bin/admintools", "-t", "db_add_node")
Expect(len(lastCall)).Should(Equal(1))
Expand All @@ -113,7 +116,8 @@ var _ = Describe("dbaddnode_reconcile", func() {

fpr := &cmds.FakePodRunner{}
pfacts := createPodFactsWithNoDB(ctx, vdb, fpr, 1)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
atCmd := fpr.FindCommands("select rebalance_shards('defaultsubcluster')")
Expect(len(atCmd)).Should(Equal(0))
Expand All @@ -127,7 +131,8 @@ var _ = Describe("dbaddnode_reconcile", func() {

fpr := &cmds.FakePodRunner{}
pfacts := createPodFactsDefault(fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
atCmd := fpr.FindCommands("/opt/vertica/bin/admintools", "-t", "db_add_node")
Expect(len(atCmd)).Should(Equal(0))
Expand All @@ -150,7 +155,8 @@ var _ = Describe("dbaddnode_reconcile", func() {
pfacts.Detail[podWithNoDB].upNode = false
pfacts.Detail[podWithNoDB].isPodRunning = false
pfacts.Detail[podWithNoDB].isInstalled = false
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{Requeue: true}))
lastCall := fpr.FindCommands("/opt/vertica/bin/admintools", "-t", "db_add_node")
Expect(len(lastCall)).Should(Equal(0))
Expand All @@ -177,7 +183,8 @@ var _ = Describe("dbaddnode_reconcile", func() {
fpr := &cmds.FakePodRunner{Results: make(cmds.CmdResults)}
pfacts := createPodFactsWithNoDB(ctx, vdb, fpr, 2)
Expect(pfacts.Collect(ctx, vdb)).Should(Succeed())
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddNodeReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
lastCall := fpr.FindCommands("/opt/vertica/bin/admintools", "-t", "db_add_node")
Expect(len(lastCall)).Should(Equal(1))
Expand Down
56 changes: 26 additions & 30 deletions pkg/controllers/vdb/dbaddsubcluster_reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,26 +26,36 @@ import (
verrors "github.com/vertica/vertica-kubernetes/pkg/errors"
"github.com/vertica/vertica-kubernetes/pkg/events"
"github.com/vertica/vertica-kubernetes/pkg/names"
"github.com/vertica/vertica-kubernetes/pkg/vadmin"
"github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/addsc"
corev1 "k8s.io/api/core/v1"
ctrl "sigs.k8s.io/controller-runtime"
)

// DBAddSubclusterReconciler will create a new subcluster if necessary
type DBAddSubclusterReconciler struct {
VRec *VerticaDBReconciler
Log logr.Logger
Vdb *vapi.VerticaDB // Vdb is the CRD we are acting on.
PRunner cmds.PodRunner
PFacts *PodFacts
ATPod *PodFact // The pod that we run admintools from
VRec *VerticaDBReconciler
Log logr.Logger
Vdb *vapi.VerticaDB // Vdb is the CRD we are acting on.
PRunner cmds.PodRunner
PFacts *PodFacts
ATPod *PodFact // The pod that we run admintools from
Dispatcher vadmin.Dispatcher
}

type SubclustersSet map[string]bool

// MakeDBAddSubclusterReconciler will build a DBAddSubclusterReconciler object
func MakeDBAddSubclusterReconciler(vdbrecon *VerticaDBReconciler, log logr.Logger,
vdb *vapi.VerticaDB, prunner cmds.PodRunner, pfacts *PodFacts) controllers.ReconcileActor {
return &DBAddSubclusterReconciler{VRec: vdbrecon, Log: log, Vdb: vdb, PRunner: prunner, PFacts: pfacts}
vdb *vapi.VerticaDB, prunner cmds.PodRunner, pfacts *PodFacts, dispatcher vadmin.Dispatcher) controllers.ReconcileActor {
return &DBAddSubclusterReconciler{
VRec: vdbrecon,
Log: log,
Vdb: vdb,
PRunner: prunner,
PFacts: pfacts,
Dispatcher: dispatcher,
}
}

// Reconcile will ensure a subcluster exists for each one defined in the vdb.
Expand Down Expand Up @@ -125,29 +135,15 @@ func (d *DBAddSubclusterReconciler) parseFetchSubclusterVsql(stdout string) Subc

// createSubcluster will create the given subcluster
func (d *DBAddSubclusterReconciler) createSubcluster(ctx context.Context, sc *vapi.Subcluster) error {
cmd := []string{
"-t", "db_add_subcluster",
"--database", d.Vdb.Spec.DBName,
"--subcluster", sc.Name,
}

// In v11, when adding a subcluster it defaults to a secondary. Prior
// versions default to a primary. Use the correct switch, depending on what
// version we are using.
vinf, ok := d.Vdb.MakeVersionInfo()
const DefaultSecondarySubclusterCreationVersion = "v11.0.0"
if ok && vinf.IsEqualOrNewer(DefaultSecondarySubclusterCreationVersion) {
if sc.IsPrimary {
cmd = append(cmd, "--is-primary")
}
} else {
if !sc.IsPrimary {
cmd = append(cmd, "--is-secondary")
}
err := d.Dispatcher.AddSubcluster(ctx,
addsc.WithInitiator(d.ATPod.name, d.ATPod.podIP),
addsc.WithSubcluster(sc.Name),
addsc.WithIsPrimary(sc.IsPrimary),
)
if err != nil {
return err
}

_, _, err := d.PRunner.ExecAdmintools(ctx, d.ATPod.name, names.ServerContainer, cmd...)
d.VRec.Eventf(d.Vdb, corev1.EventTypeNormal, events.SubclusterAdded,
"Added new subcluster '%s'", sc.Name)
return err
return nil
}
17 changes: 10 additions & 7 deletions pkg/controllers/vdb/dbaddsubcluster_reconciler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ var _ = Describe("dbaddsubcluster_reconcile", func() {

fpr := &cmds.FakePodRunner{}
pfacts := MakePodFacts(vdbRec, fpr)
a := MakeDBAddSubclusterReconciler(vdbRec, logger, vdb, fpr, &pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
a := MakeDBAddSubclusterReconciler(vdbRec, logger, vdb, fpr, &pfacts, dispatcher)
r := a.(*DBAddSubclusterReconciler)
subclusters := r.parseFetchSubclusterVsql(
" sc1\n" +
Expand Down Expand Up @@ -76,7 +77,8 @@ var _ = Describe("dbaddsubcluster_reconcile", func() {
{Stdout: " sc1\n"},
},
}
r := MakeDBAddSubclusterReconciler(vdbRec, logger, vdb, fpr, pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddSubclusterReconciler(vdbRec, logger, vdb, fpr, pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
// Last command should be AT -t db_add_subcluster
atCmdHistory := fpr.Histories[len(fpr.Histories)-1]
Expand All @@ -85,15 +87,15 @@ var _ = Describe("dbaddsubcluster_reconcile", func() {
))
})

It("should use the proper subcluster type switch for v10.1.1 versions", func() {
It("should use the proper subcluster type switch", func() {
vdb := vapi.MakeVDB()
vdb.ObjectMeta.Annotations[vapi.VersionAnnotation] = "v10.1.1-0"
test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
defer test.DeletePods(ctx, k8sClient, vdb)

fpr := &cmds.FakePodRunner{}
pfacts := MakePodFacts(vdbRec, fpr)
act := MakeDBAddSubclusterReconciler(vdbRec, logger, vdb, fpr, &pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
act := MakeDBAddSubclusterReconciler(vdbRec, logger, vdb, fpr, &pfacts, dispatcher)
r := act.(*DBAddSubclusterReconciler)
Expect(pfacts.Collect(ctx, vdb)).Should(Succeed())
r.ATPod = pfacts.Detail[names.GenPodName(vdb, &vdb.Spec.Subclusters[0], 0)]
Expand All @@ -109,7 +111,7 @@ var _ = Describe("dbaddsubcluster_reconcile", func() {
Expect(r.createSubcluster(ctx, &vdb.Spec.Subclusters[0])).Should(Succeed())
hists = fpr.FindCommands("db_add_subcluster")
Expect(len(hists)).Should(Equal(1))
Expect(hists[0].Command).ShouldNot(ContainElement("--is-primary"))
Expect(hists[0].Command).Should(ContainElement("--is-primary"))
})

It("should exit without error if not using an EON database", func() {
Expand All @@ -121,7 +123,8 @@ var _ = Describe("dbaddsubcluster_reconcile", func() {
Expect(vdb.IsEON()).Should(BeFalse())
fpr := &cmds.FakePodRunner{}
pfacts := MakePodFacts(vdbRec, fpr)
r := MakeDBAddSubclusterReconciler(vdbRec, logger, vdb, fpr, &pfacts)
dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr)
r := MakeDBAddSubclusterReconciler(vdbRec, logger, vdb, fpr, &pfacts, dispatcher)
Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
})
})
Loading

0 comments on commit a6efce7

Please sign in to comment.