-
Notifications
You must be signed in to change notification settings - Fork 40
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add a prune package to handle cleanup of pods and job resources (#75)
* add initial prune logic * add unit test, fix lint errors * update test * add prune job test * cleanup prune test * cleanup a few things * address review comments Co-authored-by: churromechanic <info@churrodata.com>
- Loading branch information
1 parent
d4e70d9
commit 68f5b7b
Showing
7 changed files
with
786 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"time" | ||
) | ||
|
||
// maxAge looks for and prunes resources, currently jobs and pods, | ||
// that exceed a user specified age (e.g. 3d) | ||
func pruneByMaxAge(ctx context.Context, config Config, resources []ResourceInfo) (err error) { | ||
config.log.V(1).Info("maxAge running", "setting", config.Strategy.MaxAgeSetting) | ||
|
||
maxAgeDuration, _ := time.ParseDuration(config.Strategy.MaxAgeSetting) | ||
maxAgeTime := time.Now().Add(-maxAgeDuration) | ||
|
||
for i := 0; i < len(resources); i++ { | ||
config.log.V(1).Info("age of pod ", "age", time.Since(resources[i].StartTime), "maxage", maxAgeTime) | ||
if resources[i].StartTime.Before(maxAgeTime) { | ||
config.log.V(1).Info("pruning ", "kind", resources[i].GVK, "name", resources[i].Name) | ||
|
||
err := config.removeResource(ctx, resources[i]) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
} | ||
|
||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"time" | ||
) | ||
|
||
// pruneByMaxCount looks for and prunes resources, currently jobs and pods, | ||
// that exceed a user specified count (e.g. 3), the oldest resources | ||
// are pruned | ||
func pruneByMaxCount(ctx context.Context, config Config, resources []ResourceInfo) (err error) { | ||
config.log.V(1).Info("pruneByMaxCount running ", "max count", config.Strategy.MaxCountSetting, "resource count", len(resources)) | ||
|
||
if len(resources) > config.Strategy.MaxCountSetting { | ||
removeCount := len(resources) - config.Strategy.MaxCountSetting | ||
for i := len(resources) - 1; i >= 0; i-- { | ||
config.log.V(1).Info("pruning pod ", "pod name", resources[i].Name, "age", time.Since(resources[i].StartTime)) | ||
|
||
err := config.removeResource(ctx, resources[i]) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
removeCount-- | ||
if removeCount == 0 { | ||
break | ||
} | ||
} | ||
} | ||
|
||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,176 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"time" | ||
|
||
"github.com/go-logr/logr" | ||
"k8s.io/apimachinery/pkg/labels" | ||
"k8s.io/apimachinery/pkg/runtime/schema" | ||
"k8s.io/client-go/kubernetes" | ||
) | ||
|
||
// ResourceStatus describes the Kubernetes resource status we are evaluating | ||
type ResourceStatus string | ||
|
||
// Strategy describes the pruning strategy we want to employ | ||
type Strategy string | ||
|
||
const ( | ||
// CustomStrategy maximum age of a resource that is desired, Duration | ||
CustomStrategy Strategy = "Custom" | ||
// MaxAgeStrategy maximum age of a resource that is desired, Duration | ||
MaxAgeStrategy Strategy = "MaxAge" | ||
// MaxCountStrategy maximum number of a resource that is desired, int | ||
MaxCountStrategy Strategy = "MaxCount" | ||
// JobKind equates to a Kube Job resource kind | ||
JobKind string = "Job" | ||
// PodKind equates to a Kube Pod resource kind | ||
PodKind string = "Pod" | ||
) | ||
|
||
// StrategyConfig holds settings unique to each pruning mode | ||
type StrategyConfig struct { | ||
Mode Strategy | ||
MaxAgeSetting string | ||
MaxCountSetting int | ||
CustomSettings map[string]interface{} | ||
} | ||
|
||
// StrategyFunc function allows a means to specify | ||
// custom prune strategies | ||
type StrategyFunc func(cfg Config, resources []ResourceInfo) error | ||
|
||
// PreDelete function is called before a resource is pruned | ||
type PreDelete func(cfg Config, something ResourceInfo) error | ||
|
||
// Config defines a pruning configuration and ultimately | ||
// determines what will get pruned | ||
type Config struct { | ||
Clientset kubernetes.Interface // kube client used by pruning | ||
LabelSelector string //selector resources to prune | ||
DryRun bool //true only performs a check, not removals | ||
Resources []schema.GroupVersionKind //pods, jobs are supported | ||
Namespaces []string //empty means all namespaces | ||
Strategy StrategyConfig //strategy for pruning, either age or max | ||
CustomStrategy StrategyFunc //custom strategy | ||
PreDeleteHook PreDelete //called before resource is deleteds | ||
log logr.Logger | ||
} | ||
|
||
// Execute causes the pruning work to be executed based on its configuration | ||
func (config Config) Execute(ctx context.Context) error { | ||
|
||
config.log.V(1).Info("Execute Prune") | ||
|
||
err := config.validate() | ||
if err != nil { | ||
return err | ||
} | ||
|
||
for i := 0; i < len(config.Resources); i++ { | ||
var resourceList []ResourceInfo | ||
var err error | ||
|
||
if config.Resources[i].Kind == PodKind { | ||
resourceList, err = config.getSucceededPods(ctx) | ||
if err != nil { | ||
return err | ||
} | ||
config.log.V(1).Info("pods ", "count", len(resourceList)) | ||
} else if config.Resources[i].Kind == JobKind { | ||
resourceList, err = config.getCompletedJobs(ctx) | ||
if err != nil { | ||
return err | ||
} | ||
config.log.V(1).Info("jobs ", "count", len(resourceList)) | ||
} | ||
|
||
switch config.Strategy.Mode { | ||
case MaxAgeStrategy: | ||
err = pruneByMaxAge(ctx, config, resourceList) | ||
case MaxCountStrategy: | ||
err = pruneByMaxCount(ctx, config, resourceList) | ||
case CustomStrategy: | ||
err = config.CustomStrategy(config, resourceList) | ||
default: | ||
return fmt.Errorf("unknown strategy") | ||
} | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
|
||
config.log.V(1).Info("Prune completed") | ||
|
||
return nil | ||
} | ||
|
||
// containsString checks if a string is present in a slice | ||
func containsString(s []string, str string) bool { | ||
for _, v := range s { | ||
if v == str { | ||
return true | ||
} | ||
} | ||
|
||
return false | ||
} | ||
|
||
// containsName checks if a string is present in a ResourceInfo slice | ||
func containsName(s []ResourceInfo, str string) bool { | ||
for _, v := range s { | ||
if v.Name == str { | ||
return true | ||
} | ||
} | ||
|
||
return false | ||
} | ||
func (config Config) validate() (err error) { | ||
|
||
if config.CustomStrategy == nil && config.Strategy.Mode == CustomStrategy { | ||
return fmt.Errorf("custom strategies require a strategy function to be specified") | ||
} | ||
|
||
if len(config.Namespaces) == 0 { | ||
return fmt.Errorf("namespaces are required") | ||
} | ||
|
||
if containsString(config.Namespaces, "") { | ||
return fmt.Errorf("empty namespace value not supported") | ||
} | ||
|
||
_, err = labels.Parse(config.LabelSelector) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if config.Strategy.Mode == MaxAgeStrategy { | ||
_, err = time.ParseDuration(config.Strategy.MaxAgeSetting) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
if config.Strategy.Mode == MaxCountStrategy { | ||
if config.Strategy.MaxCountSetting < 0 { | ||
return fmt.Errorf("max count is required to be greater than or equal to 0") | ||
} | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"testing" | ||
|
||
. "github.com/onsi/ginkgo" | ||
. "github.com/onsi/gomega" | ||
) | ||
|
||
func TestPrune(t *testing.T) { | ||
RegisterFailHandler(Fail) | ||
RunSpecs(t, "Prune Suite") | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
|
||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
) | ||
|
||
func (config Config) removeResource(ctx context.Context, resource ResourceInfo) (err error) { | ||
|
||
if config.DryRun { | ||
return nil | ||
} | ||
|
||
if config.PreDeleteHook != nil { | ||
err = config.PreDeleteHook(config, resource) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
|
||
switch resource.GVK.Kind { | ||
case PodKind: | ||
err := config.Clientset.CoreV1().Pods(resource.Namespace).Delete(ctx, resource.Name, metav1.DeleteOptions{}) | ||
if err != nil { | ||
return err | ||
} | ||
case JobKind: | ||
err := config.Clientset.BatchV1().Jobs(resource.Namespace).Delete(ctx, resource.Name, metav1.DeleteOptions{}) | ||
if err != nil { | ||
return err | ||
} | ||
default: | ||
return fmt.Errorf("unsupported resource kind") | ||
} | ||
|
||
return nil | ||
} |
Oops, something went wrong.