From 58b4b2a7a5b7bd1b6c9a7ebaba61b9ccb7c8cc13 Mon Sep 17 00:00:00 2001 From: Brian Akins Date: Fri, 20 Jul 2018 08:46:56 -0400 Subject: [PATCH] Add minimum age check for pod candidates --- chaoskube/chaoskube.go | 27 ++++++++- chaoskube/chaoskube_test.go | 116 +++++++++++++++++++++++++++++++++++- main.go | 4 ++ 3 files changed, 144 insertions(+), 3 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index 645b6e89..b5e40974 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -35,6 +35,8 @@ type Chaoskube struct { ExcludedDaysOfYear []time.Time // the timezone to apply when detecting the current weekday Timezone *time.Location + // minimum age of pods to consider + MinimumAge time.Duration // an instance of logrus.StdLogger to write log messages to Logger log.FieldLogger // dry run will not allow any pod terminations @@ -63,7 +65,7 @@ var ( // * a time zone to apply to the aforementioned time-based filters // * a logger implementing logrus.FieldLogger to send log output to // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, logger log.FieldLogger, dryRun bool) *Chaoskube { +func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool) *Chaoskube { return &Chaoskube{ Client: client, Labels: labels, @@ -73,6 +75,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces labels.Sel ExcludedTimesOfDay: excludedTimesOfDay, ExcludedDaysOfYear: excludedDaysOfYear, Timezone: timezone, + MinimumAge: minimumAge, Logger: logger, DryRun: dryRun, Now: time.Now, @@ -156,6 +159,8 @@ func (c *Chaoskube) Candidates() ([]v1.Pod, error) { return nil, err } + pods = filterByMinimumAge(pods, c.MinimumAge, c.Now()) + return pods, nil } @@ -251,3 +256,23 @@ func filterByAnnotations(pods []v1.Pod, annotations labels.Selector) ([]v1.Pod, return filteredList, nil } + +// filterByMinimumAge filters pods by creation time. Only pods +// older than minimumAge are returned +func filterByMinimumAge(pods []v1.Pod, minimumAge time.Duration, now time.Time) []v1.Pod { + if minimumAge <= time.Duration(0) { + return pods + } + + creationTime := now.Add(-minimumAge) + + filteredList := []v1.Pod{} + + for _, pod := range pods { + if pod.ObjectMeta.CreationTimestamp.Time.Before(creationTime) { + filteredList = append(filteredList, pod) + } + } + + return filteredList +} diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 9104c8f0..3ec36320 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -9,6 +9,7 @@ import ( "github.com/sirupsen/logrus/hooks/test" "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes/fake" @@ -40,6 +41,7 @@ func (suite *Suite) TestNew() { excludedWeekdays = []time.Weekday{time.Friday} excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}} excludedDaysOfYear = []time.Time{time.Now()} + minimumAge = time.Duration(42) ) chaoskube := New( @@ -51,6 +53,7 @@ func (suite *Suite) TestNew() { excludedTimesOfDay, excludedDaysOfYear, time.UTC, + minimumAge, logger, false, ) @@ -66,6 +69,7 @@ func (suite *Suite) TestNew() { suite.Equal(time.UTC, chaoskube.Timezone) suite.Equal(logger, chaoskube.Logger) suite.Equal(false, chaoskube.DryRun) + suite.Equal(minimumAge, chaoskube.MinimumAge) } func (suite *Suite) TestCandidates() { @@ -108,6 +112,7 @@ func (suite *Suite) TestCandidates() { []time.Time{}, time.UTC, false, + time.Duration(42), ) suite.assertCandidates(chaoskube, tt.pods) @@ -141,6 +146,7 @@ func (suite *Suite) TestVictim() { []time.Time{}, time.UTC, false, + time.Duration(42), ) suite.assertVictim(chaoskube, tt.victim) @@ -157,6 +163,7 @@ func (suite *Suite) TestNoVictimReturnsError() { []util.TimePeriod{}, []time.Time{}, time.UTC, + time.Duration(42), false, ) @@ -185,6 +192,7 @@ func (suite *Suite) TestDeletePod() { []time.Time{}, time.UTC, tt.dryRun, + time.Duration(42), ) victim := util.NewPod("default", "foo") @@ -414,6 +422,7 @@ func (suite *Suite) TestTerminateVictim() { tt.excludedDaysOfYear, tt.timezone, false, + time.Duration(42), ) chaoskube.Now = tt.now @@ -437,6 +446,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() { []util.TimePeriod{}, []time.Time{}, time.UTC, + time.Duration(42), false, ) @@ -486,7 +496,7 @@ func (suite *Suite) assertLog(level log.Level, msg string, fields log.Fields) { } } -func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool) *Chaoskube { +func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool, minimumAge time.Duration) *Chaoskube { chaoskube := suite.setup( labelSelector, annotations, @@ -495,6 +505,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab excludedTimesOfDay, excludedDaysOfYear, timezone, + minimumAge, dryRun, ) @@ -511,7 +522,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab return chaoskube } -func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool) *Chaoskube { +func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool) *Chaoskube { logOutput.Reset() return New( @@ -523,6 +534,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele excludedTimesOfDay, excludedDaysOfYear, timezone, + minimumAge, logger, dryRun, ) @@ -540,3 +552,103 @@ func (t ThankGodItsFriday) Now() time.Time { blackFriday, _ := time.Parse(time.RFC1123, "Fri, 24 Sep 1869 15:04:05 UTC") return blackFriday } + +func (suite *Suite) TestMinimumAge() { + type pod struct { + name string + namespace string + creationTime time.Time + } + + for _, tt := range []struct { + minimumAge time.Duration + now func() time.Time + pods []pod + candidates int + }{ + // no minimum age set + { + time.Duration(0), + func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, + []pod{ + { + name: "test1", + namespace: "test", + creationTime: time.Date(0, 10, 24, 9, 00, 00, 00, time.UTC), + }, + }, + 1, + }, + // minimum age set, but pod is too young + { + time.Hour * 1, + func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, + []pod{ + { + name: "test1", + namespace: "test", + creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC), + }, + }, + 0, + }, + // one pod is too young, one matches + { + time.Hour * 1, + func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, + []pod{ + // too young + { + name: "test1", + namespace: "test", + creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC), + }, + // matches + { + name: "test2", + namespace: "test", + creationTime: time.Date(0, 10, 23, 8, 00, 00, 00, time.UTC), + }, + }, + 1, + }, + // exact time - should not match + { + time.Hour * 1, + func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, + []pod{ + { + name: "test1", + namespace: "test", + creationTime: time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC), + }, + }, + 0, + }, + } { + chaoskube := suite.setup( + labels.Everything(), + labels.Everything(), + labels.Everything(), + []time.Weekday{}, + []util.TimePeriod{}, + []time.Time{}, + time.UTC, + tt.minimumAge, + false, + ) + chaoskube.Now = tt.now + + for _, p := range tt.pods { + pod := util.NewPod(p.namespace, p.name) + pod.ObjectMeta.CreationTimestamp = metav1.Time{Time: p.creationTime} + _, err := chaoskube.Client.Core().Pods(pod.Namespace).Create(&pod) + suite.Require().NoError(err) + } + + pods, err := chaoskube.Candidates() + suite.Require().NoError(err) + + suite.Len(pods, tt.candidates) + } +} diff --git a/main.go b/main.go index d527151a..4977ee3f 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ var ( excludedTimesOfDay string excludedDaysOfYear string timezone string + minimumAge time.Duration master string kubeconfig string interval time.Duration @@ -46,6 +47,7 @@ func init() { kingpin.Flag("excluded-times-of-day", "A list of time periods of a day when termination is suspended, e.g. 22:00-08:00").StringVar(&excludedTimesOfDay) kingpin.Flag("excluded-days-of-year", "A list of days of a year when termination is suspended, e.g. Apr1,Dec24").StringVar(&excludedDaysOfYear) kingpin.Flag("timezone", "The timezone by which to interpret the excluded weekdays and times of day, e.g. UTC, Local, Europe/Berlin. Defaults to UTC.").Default("UTC").StringVar(&timezone) + kingpin.Flag("minimum-age", "Minimum age of pods to consider for termination").Default("0s").DurationVar(&minimumAge) kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master) kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig) kingpin.Flag("interval", "Interval between Pod terminations").Default("10m").DurationVar(&interval) @@ -69,6 +71,7 @@ func main() { "excludedTimesOfDay": excludedTimesOfDay, "excludedDaysOfYear": excludedDaysOfYear, "timezone": timezone, + "minimumAge": minimumAge, "master": master, "kubeconfig": kubeconfig, "interval": interval, @@ -145,6 +148,7 @@ func main() { parsedTimesOfDay, parsedDaysOfYear, parsedTimezone, + minimumAge, log.StandardLogger(), dryRun, )