feat: wait until resources are deleted argoproj#6085 (argoproj#16733)

* feat: wait until resources are deleted

Signed-off-by: MichaelMorris <michael.morris@est.tech>

* Added unit and e2e test

Signed-off-by: MichaelMorris <michael.morris@est.tech>

---------

Signed-off-by: MichaelMorris <michael.morris@est.tech>
Signed-off-by: Kevin Lyda <kevin@lyda.ie>

cmd/argocd/commands/app.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	k8swatch "k8s.io/apimachinery/pkg/watch"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/yaml"
 
@@ -101,6 +102,7 @@ type watchOpts struct {
 	operation bool
 	suspended bool
 	degraded  bool
+	delete    bool
 }
 
 // NewApplicationCreateCommand returns a new instance of an `argocd app create` command
@@ -1277,6 +1279,7 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra.
 		noPrompt          bool
 		propagationPolicy string
 		selector          string
+		wait              bool
 	)
 	var command = &cobra.Command{
 		Use:   "delete APPNAME",
@@ -1300,7 +1303,8 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra.
 				c.HelpFunc()(c, args)
 				os.Exit(1)
 			}
-			conn, appIf := headless.NewClientOrDie(clientOpts, c).NewApplicationClientOrDie()
+			acdClient := headless.NewClientOrDie(clientOpts, c)
+			conn, appIf := acdClient.NewApplicationClientOrDie()
 			defer argoio.Close(conn)
 			var isTerminal bool = isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())
 			var isConfirmAll bool = false
@@ -1347,13 +1351,20 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra.
 					if lowercaseAnswer == "y" {
 						_, err := appIf.Delete(ctx, &appDeleteReq)
 						errors.CheckError(err)
+						if wait {
+							checkForDeleteEvent(ctx, acdClient, appFullName)
+						}
 						fmt.Printf("application '%s' deleted\n", appFullName)
 					} else {
 						fmt.Println("The command to delete '" + appFullName + "' was cancelled.")
 					}
 				} else {
 					_, err := appIf.Delete(ctx, &appDeleteReq)
 					errors.CheckError(err)
+
+					if wait {
+						checkForDeleteEvent(ctx, acdClient, appFullName)
+					}
 				}
 			}
 		},
@@ -1362,9 +1373,19 @@ func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra.
 	command.Flags().StringVarP(&propagationPolicy, "propagation-policy", "p", "foreground", "Specify propagation policy for deletion of application's resources. One of: foreground|background")
 	command.Flags().BoolVarP(&noPrompt, "yes", "y", false, "Turn off prompting to confirm cascaded deletion of application resources")
 	command.Flags().StringVarP(&selector, "selector", "l", "", "Delete all apps with matching label. Supports '=', '==', '!=', in, notin, exists & not exists. Matching apps must satisfy all of the specified label constraints.")
+	command.Flags().BoolVar(&wait, "wait", false, "Wait until deletion of the application(s) completes")
 	return command
 }
 
+func checkForDeleteEvent(ctx context.Context, acdClient argocdclient.Client, appFullName string) {
+	appEventCh := acdClient.WatchApplicationWithRetry(ctx, appFullName, "")
+	for appEvent := range appEventCh {
+		if appEvent.Type == k8swatch.Deleted {
+			return
+		}
+	}
+}
+
 // Print simple list of application names
 func printApplicationNames(apps []argoappv1.Application) {
 	for _, app := range apps {
@@ -1638,6 +1659,7 @@ func NewApplicationWaitCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co
 	command.Flags().BoolVar(&watch.health, "health", false, "Wait for health")
 	command.Flags().BoolVar(&watch.suspended, "suspended", false, "Wait for suspended")
 	command.Flags().BoolVar(&watch.degraded, "degraded", false, "Wait for degraded")
+	command.Flags().BoolVar(&watch.delete, "delete", false, "Wait for delete")
 	command.Flags().StringVarP(&selector, "selector", "l", "", "Wait for apps by label. Supports '=', '==', '!=', in, notin, exists & not exists. Matching apps must satisfy all of the specified label constraints.")
 	command.Flags().StringArrayVar(&resources, "resource", []string{}, fmt.Sprintf("Sync only specific resources as GROUP%[1]sKIND%[1]sNAME or %[2]sGROUP%[1]sKIND%[1]sNAME. Fields may be blank and '*' can be used. This option may be specified repeatedly", resourceFieldDelimiter, resourceExcludeIndicator))
 	command.Flags().BoolVar(&watch.operation, "operation", false, "Wait for pending operations")
@@ -2132,6 +2154,9 @@ func groupResourceStates(app *argoappv1.Application, selectedResources []*argoap
 
 // check if resource health, sync and operation statuses matches watch options
 func checkResourceStatus(watch watchOpts, healthStatus string, syncStatus string, operationStatus *argoappv1.Operation) bool {
+	if watch.delete {
+		return false
+	}
 	healthCheckPassed := true
 
 	if watch.suspended && watch.health && watch.degraded {
@@ -2284,6 +2309,12 @@ func waitOnApplicationStatus(ctx context.Context, acdClient argocdclient.Client,
 
 		finalOperationState = app.Status.OperationState
 		operationInProgress := false
+
+		if watch.delete && appEvent.Type == k8swatch.Deleted {
+			fmt.Printf("Application '%s' deleted\n", app.QualifiedName())
+			return nil, nil, nil
+		}
+
 		// consider the operation is in progress
 		if app.Operation != nil {
 			// if it just got requested

cmd/argocd/commands/app_test.go

@@ -1,23 +1,43 @@
 package commands
 
 import (
+	"context"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"testing"
 	"time"
 
 	argocdclient "github.com/argoproj/argo-cd/v2/pkg/apiclient"
+	accountpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/account"
+	applicationpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/application"
+	applicationsetpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/applicationset"
+	certificatepkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/certificate"
+	clusterpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/cluster"
+	gpgkeypkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/gpgkey"
+	notificationpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/notification"
+	projectpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/project"
+	repocredspkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/repocreds"
+	repositorypkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/repository"
+	sessionpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/session"
+	settingspkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/settings"
+	versionpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/version"
 	"github.com/argoproj/argo-cd/v2/pkg/apis/application"
 	"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
+
 	"github.com/argoproj/gitops-engine/pkg/health"
 	"github.com/argoproj/gitops-engine/pkg/utils/kube"
+	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/stretchr/testify/assert"
+	"golang.org/x/oauth2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	k8swatch "k8s.io/apimachinery/pkg/watch"
 )
 
 func Test_getInfos(t *testing.T) {
@@ -806,6 +826,14 @@ func TestTargetObjects_invalid(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestCheckForDeleteEvent(t *testing.T) {
+
+	ctx := context.Background()
+	fakeClient := new(fakeAcdClient)
+
+	checkForDeleteEvent(ctx, fakeClient, "testApp")
+}
+
 func TestPrintApplicationNames(t *testing.T) {
 	output, _ := captureOutput(func() error {
 		app := &v1alpha1.Application{
@@ -1599,3 +1627,104 @@ func testApp(name, project string, labels map[string]string, annotations map[str
 		},
 	}
 }
+
+type fakeAcdClient struct{}
+
+func (c *fakeAcdClient) ClientOptions() argocdclient.ClientOptions {
+	return argocdclient.ClientOptions{}
+}
+func (c *fakeAcdClient) HTTPClient() (*http.Client, error) { return nil, nil }
+func (c *fakeAcdClient) OIDCConfig(context.Context, *settingspkg.Settings) (*oauth2.Config, *oidc.Provider, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewRepoClient() (io.Closer, repositorypkg.RepositoryServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewRepoClientOrDie() (io.Closer, repositorypkg.RepositoryServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewRepoCredsClient() (io.Closer, repocredspkg.RepoCredsServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewRepoCredsClientOrDie() (io.Closer, repocredspkg.RepoCredsServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewCertClient() (io.Closer, certificatepkg.CertificateServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewCertClientOrDie() (io.Closer, certificatepkg.CertificateServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewClusterClient() (io.Closer, clusterpkg.ClusterServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewClusterClientOrDie() (io.Closer, clusterpkg.ClusterServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewGPGKeyClient() (io.Closer, gpgkeypkg.GPGKeyServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewGPGKeyClientOrDie() (io.Closer, gpgkeypkg.GPGKeyServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewApplicationClient() (io.Closer, applicationpkg.ApplicationServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewApplicationSetClient() (io.Closer, applicationsetpkg.ApplicationSetServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewApplicationClientOrDie() (io.Closer, applicationpkg.ApplicationServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewApplicationSetClientOrDie() (io.Closer, applicationsetpkg.ApplicationSetServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewNotificationClient() (io.Closer, notificationpkg.NotificationServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewNotificationClientOrDie() (io.Closer, notificationpkg.NotificationServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewSessionClient() (io.Closer, sessionpkg.SessionServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewSessionClientOrDie() (io.Closer, sessionpkg.SessionServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewSettingsClient() (io.Closer, settingspkg.SettingsServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewSettingsClientOrDie() (io.Closer, settingspkg.SettingsServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewVersionClient() (io.Closer, versionpkg.VersionServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewVersionClientOrDie() (io.Closer, versionpkg.VersionServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewProjectClient() (io.Closer, projectpkg.ProjectServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewProjectClientOrDie() (io.Closer, projectpkg.ProjectServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) NewAccountClient() (io.Closer, accountpkg.AccountServiceClient, error) {
+	return nil, nil, nil
+}
+func (c *fakeAcdClient) NewAccountClientOrDie() (io.Closer, accountpkg.AccountServiceClient) {
+	return nil, nil
+}
+func (c *fakeAcdClient) WatchApplicationWithRetry(ctx context.Context, appName string, revision string) chan *v1alpha1.ApplicationWatchEvent {
+	appEventsCh := make(chan *v1alpha1.ApplicationWatchEvent)
+
+	go func() {
+		modifiedEvent := new(v1alpha1.ApplicationWatchEvent)
+		modifiedEvent.Type = k8swatch.Modified
+		appEventsCh <- modifiedEvent
+		deletedEvent := new(v1alpha1.ApplicationWatchEvent)
+		deletedEvent.Type = k8swatch.Deleted
+		appEventsCh <- deletedEvent
+	}()
+	return appEventsCh
+}

docs/user-guide/commands/argocd_app_delete.md

@@ -32,6 +32,7 @@ argocd app delete APPNAME [flags]
   -h, --help                        help for delete
   -p, --propagation-policy string   Specify propagation policy for deletion of application's resources. One of: foreground|background (default "foreground")
   -l, --selector string             Delete all apps with matching label. Supports '=', '==', '!=', in, notin, exists & not exists. Matching apps must satisfy all of the specified label constraints.
+      --wait                        Wait until deletion of the application(s) completes
   -y, --yes                         Turn off prompting to confirm cascaded deletion of application resources
 ```
 

docs/user-guide/commands/argocd_app_wait.md

@@ -39,6 +39,7 @@ argocd app wait [APPNAME.. | -l selector] [flags]
 
 ```
       --degraded               Wait for degraded
+      --delete                 Wait for delete
       --health                 Wait for health
   -h, --help                   help for wait
       --operation              Wait for pending operations

test/e2e/app_deletion_test.go

@@ -67,3 +67,18 @@ func TestDeletingAppByLabel(t *testing.T) {
 		// delete is successful
 		Expect(DoesNotExist())
 }
+
+func TestDeletingAppByLabelWait(t *testing.T) {
+	Given(t).
+		Path(guestbookPath).
+		When().
+		CreateApp("--label=foo=bar").
+		Sync().
+		Then().
+		Expect(SyncStatusIs(SyncStatusCode(SyncStatusCodeSynced))).
+		When().
+		DeleteBySelectorWithWait("foo=bar").
+		Then().
+		// delete is successful
+		Expect(DoesNotExistNow())
+}

test/e2e/fixture/app/actions.go

@@ -417,6 +417,12 @@ func (a *Actions) DeleteBySelector(selector string) *Actions {
 	return a
 }
 
+func (a *Actions) DeleteBySelectorWithWait(selector string) *Actions {
+	a.context.t.Helper()
+	a.runCli("app", "delete", fmt.Sprintf("--selector=%s", selector), "--yes", "--wait")
+	return a
+}
+
 func (a *Actions) Wait(args ...string) *Actions {
 	a.context.t.Helper()
 	args = append([]string{"app", "wait"}, args...)

test/e2e/fixture/app/expectation.go

@@ -216,6 +216,19 @@ func DoesNotExist() Expectation {
 	}
 }
 
+func DoesNotExistNow() Expectation {
+	return func(c *Consequences) (state, string) {
+		_, err := c.get()
+		if err != nil {
+			if apierr.IsNotFound(err) {
+				return succeeded, "app does not exist"
+			}
+			return failed, err.Error()
+		}
+		return failed, "app should not exist"
+	}
+}
+
 func Pod(predicate func(p v1.Pod) bool) Expectation {
 	return func(c *Consequences) (state, string) {
 		pods, err := pods()