-
Notifications
You must be signed in to change notification settings - Fork 40
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
add a prune package to handle cleanup of pods and job resources #75
Changes from 6 commits
b4e944c
f436940
d1c39e2
68305e5
07a95fa
199a3ce
f6dd55c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"time" | ||
) | ||
|
||
// maxAge looks for and prunes resources, currently jobs and pods, | ||
// that exceed a user specified age (e.g. 3d) | ||
func pruneByMaxAge(config Config, resources []ResourceInfo) (err error) { | ||
log.V(1).Info("maxAge running", "setting", config.Strategy.MaxAgeSetting) | ||
|
||
maxAgeDuration, _ := time.ParseDuration(config.Strategy.MaxAgeSetting) | ||
maxAgeTime := time.Now().Add(-maxAgeDuration) | ||
|
||
for i := 0; i < len(resources); i++ { | ||
log.V(1).Info("age of pod ", "age", time.Since(resources[i].StartTime), "maxage", maxAgeTime) | ||
if resources[i].StartTime.Before(maxAgeTime) { | ||
log.V(1).Info("pruning ", "kind", resources[i].Kind, "name", resources[i].Name) | ||
if !config.DryRun { | ||
err := config.removeResource(resources[i]) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
} | ||
} | ||
|
||
return nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"time" | ||
) | ||
|
||
// pruneByMaxCount looks for and prunes resources, currently jobs and pods, | ||
// that exceed a user specified count (e.g. 3), the oldest resources | ||
// are pruned | ||
func pruneByMaxCount(config Config, resources []ResourceInfo) (err error) { | ||
log.V(1).Info("pruneByMaxCount running ", "max count", config.Strategy.MaxCountSetting, "resource count", len(resources)) | ||
|
||
if len(resources) > config.Strategy.MaxCountSetting { | ||
removeCount := len(resources) - config.Strategy.MaxCountSetting | ||
for i := len(resources) - 1; i >= 0; i-- { | ||
log.V(1).Info("pruning pod ", "pod name", resources[i].Name, "age", time.Since(resources[i].StartTime)) | ||
if !config.DryRun { | ||
err := config.removeResource(resources[i]) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
removeCount-- | ||
if removeCount == 0 { | ||
break | ||
} | ||
} | ||
} | ||
|
||
return nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,187 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"time" | ||
|
||
logf "sigs.k8s.io/controller-runtime/pkg/log" | ||
|
||
"k8s.io/apimachinery/pkg/labels" | ||
"k8s.io/client-go/kubernetes" | ||
) | ||
|
||
// ResourceKind describes the Kubernetes Kind we are wanting to prune | ||
type ResourceKind string | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we reuse a type from the Kubernetes library here like There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. latest commit uses the apimachinery schema.GroupVersionKind to hold that value which should be better. |
||
|
||
// ResourceStatus describes the Kubernetes resource status we are evaluating | ||
type ResourceStatus string | ||
|
||
// Strategy describes the pruning strategy we want to employ | ||
type Strategy string | ||
|
||
const ( | ||
// CustomStrategy maximum age of a resource that is desired, Duration | ||
CustomStrategy Strategy = "Custom" | ||
// MaxAgeStrategy maximum age of a resource that is desired, Duration | ||
MaxAgeStrategy Strategy = "MaxAge" | ||
// MaxCountStrategy maximum number of a resource that is desired, int | ||
MaxCountStrategy Strategy = "MaxCount" | ||
// JobKind equates to a Kube Job resource kind | ||
JobKind ResourceKind = "job" | ||
// PodKind equates to a Kube Pod resource kind | ||
PodKind ResourceKind = "pod" | ||
) | ||
|
||
// StrategyConfig holds settings unique to each pruning mode | ||
type StrategyConfig struct { | ||
Mode Strategy | ||
MaxAgeSetting string | ||
MaxCountSetting int | ||
CustomSettings map[string]interface{} | ||
} | ||
|
||
// StrategyImplementation function allows a means to specify | ||
// custom prune strategies | ||
type StrategyImplementation func(cfg Config, resources []ResourceInfo) error | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: since its a function type, not a struct, would There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. better name, latest commit uses that name. |
||
|
||
// PreDelete function is called before a resource is pruned | ||
type PreDelete func(cfg Config, something ResourceInfo) error | ||
|
||
// Config defines a pruning configuration and ultimately | ||
// determines what will get pruned | ||
type Config struct { | ||
Ctx context.Context //context used by pruning | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Its considered bad practice by the go standard library to use a context as a struct member.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ok, I moved it outside the Config with this latest commit. |
||
Clientset kubernetes.Interface // kube client used by pruning | ||
LabelSelector string //selector resources to prune | ||
DryRun bool //true only performs a check, not removals | ||
Resources []ResourceKind //pods, jobs are supported | ||
Namespaces []string //empty means all namespaces | ||
Strategy StrategyConfig //strategy for pruning, either age or max | ||
CustomStrategy StrategyImplementation //custom strategy | ||
PreDeleteHook PreDelete //called before resource is deleteds | ||
} | ||
|
||
var log = logf.Log.WithName("prune") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we move this to the config struct? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. moved the log into the Config with latest commit. |
||
|
||
// Execute causes the pruning work to be executed based on its configuration | ||
func (config Config) Execute() error { | ||
|
||
log.V(1).Info("Execute Prune") | ||
|
||
err := config.validate() | ||
if err != nil { | ||
return err | ||
} | ||
|
||
for i := 0; i < len(config.Resources); i++ { | ||
var resourceList []ResourceInfo | ||
var err error | ||
|
||
if config.Resources[i] == PodKind { | ||
resourceList, err = config.getSucceededPods() | ||
if err != nil { | ||
return err | ||
} | ||
log.V(1).Info("pods ", "count", len(resourceList)) | ||
} else if config.Resources[i] == JobKind { | ||
resourceList, err = config.getCompletedJobs() | ||
if err != nil { | ||
return err | ||
} | ||
log.V(1).Info("jobs ", "count", len(resourceList)) | ||
} | ||
|
||
switch config.Strategy.Mode { | ||
case MaxAgeStrategy: | ||
err = pruneByMaxAge(config, resourceList) | ||
if err != nil { | ||
return err | ||
} | ||
case MaxCountStrategy: | ||
err = pruneByMaxCount(config, resourceList) | ||
if err != nil { | ||
return err | ||
} | ||
case CustomStrategy: | ||
err = config.CustomStrategy(config, resourceList) | ||
if err != nil { | ||
return err | ||
} | ||
default: | ||
return fmt.Errorf("unknown strategy") | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: you can just check There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. doh! thanks, new version corrects that. |
||
} | ||
|
||
log.V(1).Info("Prune completed") | ||
|
||
return nil | ||
} | ||
|
||
// containsString checks if a string is present in a slice | ||
func containsString(s []string, str string) bool { | ||
for _, v := range s { | ||
if v == str { | ||
return true | ||
} | ||
} | ||
|
||
return false | ||
} | ||
|
||
// containsName checks if a string is present in a ResourceInfo slice | ||
func containsName(s []ResourceInfo, str string) bool { | ||
for _, v := range s { | ||
if v.Name == str { | ||
return true | ||
} | ||
} | ||
|
||
return false | ||
} | ||
func (config Config) validate() (err error) { | ||
|
||
if config.CustomStrategy == nil && config.Strategy.Mode == CustomStrategy { | ||
return fmt.Errorf("custom strategies require a strategy function to be specified") | ||
} | ||
|
||
if len(config.Namespaces) == 0 { | ||
return fmt.Errorf("namespaces are required") | ||
} | ||
|
||
if containsString(config.Namespaces, "") { | ||
return fmt.Errorf("empty namespace value not supported") | ||
} | ||
|
||
_, err = labels.Parse(config.LabelSelector) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if config.Strategy.Mode == MaxAgeStrategy { | ||
_, err = time.ParseDuration(config.Strategy.MaxAgeSetting) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
if config.Strategy.Mode == MaxCountStrategy { | ||
if config.Strategy.MaxCountSetting < 0 { | ||
return fmt.Errorf("max count is required to be greater than or equal to 0") | ||
} | ||
} | ||
return nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
"testing" | ||
|
||
. "github.com/onsi/ginkgo" | ||
. "github.com/onsi/gomega" | ||
) | ||
|
||
func TestPrune(t *testing.T) { | ||
RegisterFailHandler(Fail) | ||
RunSpecs(t, "Prune Suite") | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
// Copyright 2021 The Operator-SDK Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package prune | ||
|
||
import ( | ||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
) | ||
|
||
func (config Config) removeResource(resource ResourceInfo) (err error) { | ||
|
||
if config.PreDeleteHook != nil { | ||
err = config.PreDeleteHook(config, resource) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
|
||
switch resource.Kind { | ||
case PodKind: | ||
err := config.Clientset.CoreV1().Pods(resource.Namespace).Delete(config.Ctx, resource.Name, metav1.DeleteOptions{}) | ||
if err != nil { | ||
return err | ||
} | ||
case JobKind: | ||
err := config.Clientset.BatchV1().Jobs(resource.Namespace).Delete(config.Ctx, resource.Name, metav1.DeleteOptions{}) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we want to return some sort of error if we attempt to delete an unsupported resource? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. added in the error return. thanks for catching that. |
||
|
||
return nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we move this code to a separate function that is called outside the strategy since it's universal across all strategies? Then we can make the strategy function signature
func(config Config, resources []ResourceInfo) (toPrune []ResourceInfo, err error)
and remove the deletion responsibility from the strategy implementation.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I moved the config.DryRun check into the removeResource func to make it cleaner.