Skip to content

Commit

Permalink
migrate: Make rollback more stylish
Browse files Browse the repository at this point in the history
Co-authored-by: Trong Nguyen <trong.huu.nguyen@nav.no>
  • Loading branch information
mortenlj and tronghn committed Oct 4, 2024
1 parent 0ed9567 commit df1a140
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 27 deletions.
35 changes: 29 additions & 6 deletions pkg/postgres/migrate/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,15 @@ func (m *Migrator) Create(ctx context.Context, obj ctrl.Object) error {
return m.client.Create(ctx, obj)
}

func (m *Migrator) Delete(ctx context.Context, obj ctrl.Object) error {
if m.dryRun {
v := reflect.Indirect(reflect.ValueOf(obj))
fmt.Printf("Dry run: Skipping deletion of %s: %s\n", v.Type().Name(), obj.GetName())
return nil
}
return m.client.Delete(ctx, obj)
}

func (m *Migrator) doNaisJob(ctx context.Context, cfgMap *corev1.ConfigMap, command Command) (string, error) {
pterm.Println("Creating NaisJob ...")
imageTag, err := getLatestImageTag()
Expand All @@ -84,7 +93,7 @@ func (m *Migrator) kubectlLabelSelector(command Command) string {
}

func (m *Migrator) deleteMigrationConfig(ctx context.Context) error {
err := ctrl.IgnoreNotFound(m.client.Delete(ctx, m.cfg.GetConfigMap()))
err := ctrl.IgnoreNotFound(m.Delete(ctx, m.cfg.GetConfigMap()))
if err != nil {
return fmt.Errorf("failed to delete ConfigMap: %w", err)
}
Expand All @@ -105,6 +114,14 @@ func (m *Migrator) LookupGcpProjectId(ctx context.Context) (string, error) {
}

func (m *Migrator) waitForJobCompletion(ctx context.Context, jobName string, command Command) error {
spinner, _ := pterm.DefaultSpinner.Start("Waiting for job to complete ...")

if m.dryRun {
pterm.Printf("Dry run: Artificial waiting for job %s/%s to complete, 5 seconds\n", m.cfg.Namespace, jobName)
time.Sleep(5 * time.Second)
spinner.Success("Job completed")
return nil
}
listOptions := []ctrl.ListOption{
ctrl.InNamespace(m.cfg.Namespace),
ctrl.MatchingLabels{
Expand All @@ -115,29 +132,35 @@ func (m *Migrator) waitForJobCompletion(ctx context.Context, jobName string, com

b := retry.NewConstant(20 * time.Second)
b = retry.WithMaxDuration(15*time.Minute, b)
return retry.Do(ctx, b, func(ctx context.Context) error {
err := retry.Do(ctx, b, func(ctx context.Context) error {
jobs := &batchv1.JobList{}
err := m.client.List(ctx, jobs, listOptions...)
if err != nil {
fmt.Printf("Error getting jobs %s/%s, retrying: %v\n", m.cfg.Namespace, jobName, err)
pterm.Warning.Printf("Error getting jobs %s/%s, retrying: %v\n", m.cfg.Namespace, jobName, err)
return retry.RetryableError(err)
}
if len(jobs.Items) < 1 {
fmt.Printf("No jobs found %s/%s, retrying\n", m.cfg.Namespace, jobName)
pterm.Printf("No jobs found %s/%s, retrying\n", m.cfg.Namespace, jobName)
return retry.RetryableError(fmt.Errorf("no jobs found"))
}
if len(jobs.Items) > 1 {
fmt.Printf("Multiple jobs found %s/%s! This should not happen, contact the nais team!\n", m.cfg.Namespace, jobName)
pterm.Error.Printf("Multiple jobs found %s/%s! This should not happen, contact the nais team!\n", m.cfg.Namespace, jobName)
return fmt.Errorf("multiple jobs found")
}
for _, job := range jobs.Items {
if job.Status.Succeeded == 1 {
return nil
}
}
fmt.Printf("Job %s/%s has not completed yet, retrying\n", m.cfg.Namespace, jobName)
pterm.Printf("Job %s/%s has not completed yet, retrying\n", m.cfg.Namespace, jobName)
return retry.RetryableError(fmt.Errorf("job %s/%s has not completed yet", m.cfg.Namespace, jobName))
})
if err != nil {
spinner.Fail()
return err
}
spinner.Success("Job completed")
return nil
}

func (m *Migrator) printConfig() {
Expand Down
34 changes: 13 additions & 21 deletions pkg/postgres/migrate/rollback.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,9 @@ package migrate

import (
"context"
"fmt"
"github.com/pterm/pterm"
)

const RollbackStartedMessage = `
Rollback has been started successfully.
To monitor the rollback, run the following command in a separate terminal:
kubectl logs -f -l %s -n %s
Pausing to wait for rollback job to complete in order to do final finalize actions ...
`

const RollbackSuccessMessage = `
Rollback has completed successfully.
Your application should be up and running with the original database instance.
The new instance has been deleted and the migration is stopped.
You are now free to start another attempt if you wish.
`

func (m *Migrator) Rollback(ctx context.Context) error {
pterm.Println("Resolving config ...")
cfgMap, err := m.cfg.PopulateFromConfigMap(ctx, m.client)
Expand All @@ -45,7 +26,12 @@ func (m *Migrator) Rollback(ctx context.Context) error {
}

label := m.kubectlLabelSelector(CommandRollback)
fmt.Printf(RollbackStartedMessage, label, m.cfg.Namespace)
pterm.DefaultHeader.Println("Rollback has been started successfully")
pterm.Println()
pterm.Println("To monitor the rollback, run the following command in a separate terminal:")
cmdStyle.Printfln("\tkubectl logs -f -l %s -n %s", label, m.cfg.Namespace)
pterm.Println()
pterm.Println("Pausing to wait for rollback job to complete in order to do final finalize actions ...")

err = m.waitForJobCompletion(ctx, jobName, CommandRollback)
if err != nil {
Expand All @@ -57,6 +43,12 @@ func (m *Migrator) Rollback(ctx context.Context) error {
return err
}

fmt.Print(RollbackSuccessMessage)
pterm.DefaultHeader.Println("Rollback has completed successfully")
pterm.Println()
pterm.Println("Your application should be up and running with the original database instance.")
pterm.Println("The new instance has been deleted and the migration is stopped.")
pterm.Println()
pterm.Println("You are now free to start another attempt if you wish.")

return nil
}

0 comments on commit df1a140

Please sign in to comment.