-
Notifications
You must be signed in to change notification settings - Fork 40
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
add a prune package to handle cleanup of pods and job resources #75
Merged
openshift-merge-robot
merged 7 commits into
operator-framework:main
from
jmccormick2001:main
Oct 28, 2021
Merged
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
b4e944c
add initial prune logic
churrodata f436940
add unit test, fix lint errors
churrodata d1c39e2
update test
churrodata 68305e5
add prune job test
churrodata 07a95fa
cleanup prune test
churrodata 199a3ce
cleanup a few things
churrodata f6dd55c
address review comments
churrodata File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"time" | ||
) | ||
|
||
// maxAge looks for and prunes resources, currently jobs and pods, | ||
// that exceed a user specified age (e.g. 3d) | ||
func pruneByMaxAge(ctx context.Context, config Config, resources []ResourceInfo) (err error) { | ||
config.log.V(1).Info("maxAge running", "setting", config.Strategy.MaxAgeSetting) | ||
|
||
maxAgeDuration, _ := time.ParseDuration(config.Strategy.MaxAgeSetting) | ||
maxAgeTime := time.Now().Add(-maxAgeDuration) | ||
|
||
for i := 0; i < len(resources); i++ { | ||
config.log.V(1).Info("age of pod ", "age", time.Since(resources[i].StartTime), "maxage", maxAgeTime) | ||
if resources[i].StartTime.Before(maxAgeTime) { | ||
config.log.V(1).Info("pruning ", "kind", resources[i].GVK, "name", resources[i].Name) | ||
|
||
err := config.removeResource(ctx, resources[i]) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
} | ||
|
||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"time" | ||
) | ||
|
||
// pruneByMaxCount looks for and prunes resources, currently jobs and pods, | ||
// that exceed a user specified count (e.g. 3), the oldest resources | ||
// are pruned | ||
func pruneByMaxCount(ctx context.Context, config Config, resources []ResourceInfo) (err error) { | ||
config.log.V(1).Info("pruneByMaxCount running ", "max count", config.Strategy.MaxCountSetting, "resource count", len(resources)) | ||
|
||
if len(resources) > config.Strategy.MaxCountSetting { | ||
removeCount := len(resources) - config.Strategy.MaxCountSetting | ||
for i := len(resources) - 1; i >= 0; i-- { | ||
config.log.V(1).Info("pruning pod ", "pod name", resources[i].Name, "age", time.Since(resources[i].StartTime)) | ||
|
||
err := config.removeResource(ctx, resources[i]) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
removeCount-- | ||
if removeCount == 0 { | ||
break | ||
} | ||
} | ||
} | ||
|
||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,176 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"time" | ||
|
||
"github.com/go-logr/logr" | ||
"k8s.io/apimachinery/pkg/labels" | ||
"k8s.io/apimachinery/pkg/runtime/schema" | ||
"k8s.io/client-go/kubernetes" | ||
) | ||
|
||
// ResourceStatus describes the Kubernetes resource status we are evaluating | ||
type ResourceStatus string | ||
|
||
// Strategy describes the pruning strategy we want to employ | ||
type Strategy string | ||
|
||
const ( | ||
// CustomStrategy maximum age of a resource that is desired, Duration | ||
CustomStrategy Strategy = "Custom" | ||
// MaxAgeStrategy maximum age of a resource that is desired, Duration | ||
MaxAgeStrategy Strategy = "MaxAge" | ||
// MaxCountStrategy maximum number of a resource that is desired, int | ||
MaxCountStrategy Strategy = "MaxCount" | ||
// JobKind equates to a Kube Job resource kind | ||
JobKind string = "Job" | ||
// PodKind equates to a Kube Pod resource kind | ||
PodKind string = "Pod" | ||
) | ||
|
||
// StrategyConfig holds settings unique to each pruning mode | ||
type StrategyConfig struct { | ||
Mode Strategy | ||
MaxAgeSetting string | ||
MaxCountSetting int | ||
CustomSettings map[string]interface{} | ||
} | ||
|
||
// StrategyFunc function allows a means to specify | ||
// custom prune strategies | ||
type StrategyFunc func(cfg Config, resources []ResourceInfo) error | ||
|
||
// PreDelete function is called before a resource is pruned | ||
type PreDelete func(cfg Config, something ResourceInfo) error | ||
|
||
// Config defines a pruning configuration and ultimately | ||
// determines what will get pruned | ||
type Config struct { | ||
Clientset kubernetes.Interface // kube client used by pruning | ||
LabelSelector string //selector resources to prune | ||
DryRun bool //true only performs a check, not removals | ||
Resources []schema.GroupVersionKind //pods, jobs are supported | ||
Namespaces []string //empty means all namespaces | ||
Strategy StrategyConfig //strategy for pruning, either age or max | ||
CustomStrategy StrategyFunc //custom strategy | ||
PreDeleteHook PreDelete //called before resource is deleteds | ||
log logr.Logger | ||
} | ||
|
||
// Execute causes the pruning work to be executed based on its configuration | ||
func (config Config) Execute(ctx context.Context) error { | ||
|
||
config.log.V(1).Info("Execute Prune") | ||
|
||
err := config.validate() | ||
if err != nil { | ||
return err | ||
} | ||
|
||
for i := 0; i < len(config.Resources); i++ { | ||
var resourceList []ResourceInfo | ||
var err error | ||
|
||
if config.Resources[i].Kind == PodKind { | ||
resourceList, err = config.getSucceededPods(ctx) | ||
if err != nil { | ||
return err | ||
} | ||
config.log.V(1).Info("pods ", "count", len(resourceList)) | ||
} else if config.Resources[i].Kind == JobKind { | ||
resourceList, err = config.getCompletedJobs(ctx) | ||
if err != nil { | ||
return err | ||
} | ||
config.log.V(1).Info("jobs ", "count", len(resourceList)) | ||
} | ||
|
||
switch config.Strategy.Mode { | ||
case MaxAgeStrategy: | ||
err = pruneByMaxAge(ctx, config, resourceList) | ||
case MaxCountStrategy: | ||
err = pruneByMaxCount(ctx, config, resourceList) | ||
case CustomStrategy: | ||
err = config.CustomStrategy(config, resourceList) | ||
default: | ||
return fmt.Errorf("unknown strategy") | ||
} | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
|
||
config.log.V(1).Info("Prune completed") | ||
|
||
return nil | ||
} | ||
|
||
// containsString checks if a string is present in a slice | ||
func containsString(s []string, str string) bool { | ||
for _, v := range s { | ||
if v == str { | ||
return true | ||
} | ||
} | ||
|
||
return false | ||
} | ||
|
||
// containsName checks if a string is present in a ResourceInfo slice | ||
func containsName(s []ResourceInfo, str string) bool { | ||
for _, v := range s { | ||
if v.Name == str { | ||
return true | ||
} | ||
} | ||
|
||
return false | ||
} | ||
func (config Config) validate() (err error) { | ||
|
||
if config.CustomStrategy == nil && config.Strategy.Mode == CustomStrategy { | ||
return fmt.Errorf("custom strategies require a strategy function to be specified") | ||
} | ||
|
||
if len(config.Namespaces) == 0 { | ||
return fmt.Errorf("namespaces are required") | ||
} | ||
|
||
if containsString(config.Namespaces, "") { | ||
return fmt.Errorf("empty namespace value not supported") | ||
} | ||
|
||
_, err = labels.Parse(config.LabelSelector) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if config.Strategy.Mode == MaxAgeStrategy { | ||
_, err = time.ParseDuration(config.Strategy.MaxAgeSetting) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
if config.Strategy.Mode == MaxCountStrategy { | ||
if config.Strategy.MaxCountSetting < 0 { | ||
return fmt.Errorf("max count is required to be greater than or equal to 0") | ||
} | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"testing" | ||
|
||
. "github.com/onsi/ginkgo" | ||
. "github.com/onsi/gomega" | ||
) | ||
|
||
func TestPrune(t *testing.T) { | ||
RegisterFailHandler(Fail) | ||
RunSpecs(t, "Prune Suite") | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
|
||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
) | ||
|
||
func (config Config) removeResource(ctx context.Context, resource ResourceInfo) (err error) { | ||
|
||
if config.DryRun { | ||
return nil | ||
} | ||
|
||
if config.PreDeleteHook != nil { | ||
err = config.PreDeleteHook(config, resource) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
|
||
switch resource.GVK.Kind { | ||
case PodKind: | ||
err := config.Clientset.CoreV1().Pods(resource.Namespace).Delete(ctx, resource.Name, metav1.DeleteOptions{}) | ||
if err != nil { | ||
return err | ||
} | ||
case JobKind: | ||
err := config.Clientset.BatchV1().Jobs(resource.Namespace).Delete(ctx, resource.Name, metav1.DeleteOptions{}) | ||
if err != nil { | ||
return err | ||
} | ||
default: | ||
return fmt.Errorf("unsupported resource kind") | ||
} | ||
|
||
return nil | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we want to return some sort of error if we attempt to delete an unsupported resource?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
added in the error return. thanks for catching that.