ROX-13838: deprovision disgraced centrals (#1250)

stackrox · Nov 27, 2023 · c85d7df · c85d7df
1 parent 723d081
commit c85d7df
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 39 deletions.
diff --git a/internal/dinosaur/pkg/services/dinosaur.go b/internal/dinosaur/pkg/services/dinosaur.go
@@ -6,30 +6,27 @@ import (
 	"sync"
 	"time"
 
-	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/rhsso"
-	"github.com/stackrox/acs-fleet-manager/pkg/client/iam"
-	dynamicClientAPI "github.com/stackrox/acs-fleet-manager/pkg/client/redhatsso/api"
-	"github.com/stackrox/acs-fleet-manager/pkg/client/redhatsso/dynamicclients"
-	"github.com/stackrox/acs-fleet-manager/pkg/environments"
-
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/golang/glog"
 	dinosaurConstants "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/dbapi"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/config"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/dinosaurs/types"
-	"github.com/stackrox/acs-fleet-manager/pkg/services"
-	coreServices "github.com/stackrox/acs-fleet-manager/pkg/services/queryparser"
-
-	"github.com/golang/glog"
-
-	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/rhsso"
 	"github.com/stackrox/acs-fleet-manager/pkg/api"
 	"github.com/stackrox/acs-fleet-manager/pkg/auth"
 	"github.com/stackrox/acs-fleet-manager/pkg/client/aws"
+	"github.com/stackrox/acs-fleet-manager/pkg/client/iam"
 	"github.com/stackrox/acs-fleet-manager/pkg/client/ocm"
+	dynamicClientAPI "github.com/stackrox/acs-fleet-manager/pkg/client/redhatsso/api"
+	"github.com/stackrox/acs-fleet-manager/pkg/client/redhatsso/dynamicclients"
 	"github.com/stackrox/acs-fleet-manager/pkg/db"
+	"github.com/stackrox/acs-fleet-manager/pkg/environments"
 	"github.com/stackrox/acs-fleet-manager/pkg/errors"
 	"github.com/stackrox/acs-fleet-manager/pkg/logger"
 	"github.com/stackrox/acs-fleet-manager/pkg/metrics"
+	"github.com/stackrox/acs-fleet-manager/pkg/services"
+	coreServices "github.com/stackrox/acs-fleet-manager/pkg/services/queryparser"
 )
 
 var (
@@ -55,6 +52,8 @@ const DinosaurRoutesActionCreate DinosaurRoutesAction = "CREATE"
 // DinosaurRoutesActionDelete ...
 const DinosaurRoutesActionDelete DinosaurRoutesAction = "DELETE"
 
+const gracePeriod = 14 * 24 * time.Hour
+
 // CNameRecordStatus ...
 type CNameRecordStatus struct {
 	ID     *string
@@ -100,7 +99,7 @@ type DinosaurService interface {
 	RegisterDinosaurDeprovisionJob(ctx context.Context, id string) *errors.ServiceError
 	// DeprovisionDinosaurForUsers registers all dinosaurs for deprovisioning given the list of owners
 	DeprovisionDinosaurForUsers(users []string) *errors.ServiceError
-	DeprovisionExpiredDinosaurs(dinosaurAgeInHours int) *errors.ServiceError
+	DeprovisionExpiredDinosaurs() *errors.ServiceError
 	CountByStatus(status []dinosaurConstants.CentralStatus) ([]DinosaurStatusCount, error)
 	CountByRegionAndInstanceType() ([]DinosaurRegionCount, error)
 	ListDinosaursWithRoutesNotCreated() ([]*dbapi.CentralRequest, *errors.ServiceError)
@@ -518,13 +517,19 @@ func (k *dinosaurService) DeprovisionDinosaurForUsers(users []string) *errors.Se
 }
 
 // DeprovisionExpiredDinosaurs cleaning up expired dinosaurs
-func (k *dinosaurService) DeprovisionExpiredDinosaurs(dinosaurAgeInHours int) *errors.ServiceError {
+func (k *dinosaurService) DeprovisionExpiredDinosaurs() *errors.ServiceError {
 	now := time.Now()
-	dbConn := k.connectionFactory.New().
-		Model(&dbapi.CentralRequest{}).
-		Where("instance_type = ?", types.EVAL.String()).
-		Where("created_at  <=  ?", now.Add(-1*time.Duration(dinosaurAgeInHours)*time.Hour)).
-		Where("status NOT IN (?)", dinosaurDeletionStatuses)
+	dbConn := k.connectionFactory.New().Model(&dbapi.CentralRequest{}).
+		Where("expired_at IS NOT NULL").Where("expired_at < ?", now.Add(-gracePeriod))
+
+	if k.dinosaurConfig.CentralLifespan.EnableDeletionOfExpiredCentral {
+		dbConn = dbConn.Where(dbConn.
+			Or("instance_type = ?", types.EVAL.String()).
+			Where("created_at <= ?", now.Add(
+				-time.Duration(k.dinosaurConfig.CentralLifespan.CentralLifespanInHours)*time.Hour)))
+	}
+
+	dbConn = dbConn.Where("status NOT IN (?)", dinosaurDeletionStatuses)
 
 	db := dbConn.Updates(map[string]interface{}{
 		"status":             dinosaurConstants.CentralRequestStatusDeprovision,
@@ -536,7 +541,7 @@ func (k *dinosaurService) DeprovisionExpiredDinosaurs(dinosaurAgeInHours int) *e
 	}
 
 	if db.RowsAffected >= 1 {
-		glog.Infof("%v central_request's lifespans are over %d hours and have had their status updated to deprovisioning", db.RowsAffected, dinosaurAgeInHours)
+		glog.Infof("%v central_request's have had their status updated to deprovisioning", db.RowsAffected)
 		var counter int64
 		for ; counter < db.RowsAffected; counter++ {
 			metrics.IncreaseCentralTotalOperationsCountMetric(dinosaurConstants.CentralOperationDeprovision)

diff --git a/internal/dinosaur/pkg/services/dinosaur_test.go b/internal/dinosaur/pkg/services/dinosaur_test.go
@@ -7,10 +7,12 @@ import (
 
 	mocket "github.com/selvatico/go-mocket"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/dbapi"
+	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/config"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/converters"
 	"github.com/stackrox/acs-fleet-manager/pkg/api"
 	"github.com/stackrox/acs-fleet-manager/pkg/auth"
 	"github.com/stackrox/acs-fleet-manager/pkg/db"
+	"github.com/stretchr/testify/assert"
 	"gorm.io/gorm"
 )
 
@@ -172,3 +174,32 @@ func Test_dinosaurService_Get(t *testing.T) {
 		})
 	}
 }
+
+func Test_dinosaurService_DeprovisionExpiredDinosaursQuery(t *testing.T) {
+	k := &dinosaurService{
+		connectionFactory: db.NewMockConnectionFactory(nil),
+		dinosaurConfig: &config.CentralConfig{
+			CentralLifespan: config.NewCentralLifespanConfig(),
+		},
+	}
+
+	m := mocket.Catcher.Reset().NewMock().WithQuery(`UPDATE "central_requests" ` +
+		`SET "deletion_timestamp"=$1,"status"=$2,"updated_at"=$3 WHERE ` +
+		`(expired_at IS NOT NULL AND expired_at < $4 OR instance_type = $5 AND created_at <= $6) ` +
+		`AND status NOT IN ($7,$8) AND "central_requests"."deleted_at" IS NULL`).
+		OneTime()
+
+	svcErr := k.DeprovisionExpiredDinosaurs()
+	assert.Nil(t, svcErr)
+	assert.True(t, m.Triggered)
+
+	m = mocket.Catcher.Reset().NewMock().WithQuery(`UPDATE "central_requests" ` +
+		`SET "deletion_timestamp"=$1,"status"=$2,"updated_at"=$3 WHERE ` +
+		`expired_at IS NOT NULL AND expired_at < $4 ` +
+		`AND status NOT IN ($5,$6) AND "central_requests"."deleted_at" IS NULL`).
+		OneTime()
+	k.dinosaurConfig.CentralLifespan.EnableDeletionOfExpiredCentral = false
+	svcErr = k.DeprovisionExpiredDinosaurs()
+	assert.Nil(t, svcErr)
+	assert.True(t, m.Triggered)
+}
diff --git a/internal/dinosaur/pkg/services/dinosaurservice_moq.go b/internal/dinosaur/pkg/services/dinosaurservice_moq.go
diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_mgr.go
@@ -82,13 +82,9 @@ func (k *DinosaurManager) Reconcile() []error {
 	}
 
 	// cleaning up expired dinosaurs
-	dinosaurConfig := k.dinosaurConfig
-	if dinosaurConfig.CentralLifespan.EnableDeletionOfExpiredCentral {
-		expiredDinosaursError := k.dinosaurService.DeprovisionExpiredDinosaurs(dinosaurConfig.CentralLifespan.CentralLifespanInHours)
-		if expiredDinosaursError != nil {
-			wrappedError := errors.Wrap(expiredDinosaursError, "failed to deprovision expired Central instances")
-			encounteredErrors = append(encounteredErrors, wrappedError)
-		}
+	if svcErr := k.dinosaurService.DeprovisionExpiredDinosaurs(); svcErr != nil {
+		encounteredErrors = append(encounteredErrors,
+			errors.Wrap(svcErr, "failed to deprovision expired Central instances"))
 	}
 
 	return encounteredErrors