Merge pull request #47 from deliveroo/CPR-569/add-cleanup-pods-command

CPR-569: New command to clean up pods

VERSION

@@ -1 +1 @@
-0.21.3
+0.22.0

cli/cleanup/cmd.go

@@ -0,0 +1,16 @@
+package cleanup
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// CleanUpCmd represents the clean up command
+var CleanUpCmd = &cobra.Command{
+	Use:   "cleanup",
+	Short: "Clean up resources",
+	Long:  "Cleans up all resources that are in finished state or not needed",
+}
+
+func init() {
+	CleanUpCmd.AddCommand(podsCmd)
+}

cli/cleanup/pods.go

@@ -0,0 +1,76 @@
+package cleanup
+
+import (
+	"log"
+	"time"
+
+	"github.com/deliveroo/paddle/cli/pipeline"
+	"github.com/spf13/cobra"
+	v12 "k8s.io/api/core/v1"
+	k8errors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
+)
+
+var clientset kubernetes.Interface
+var logFatalf = log.Fatalf
+
+var podsCmd = &cobra.Command{
+	Use:   "pods",
+	Short: "Clean up all finished pods",
+	Long: `Fetch all pods and delete the ones in complete state
+
+Example:
+
+$ paddle cleanup pods
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		runPodsCleanup()
+	},
+}
+
+func init() {
+	config, err := pipeline.GetKubernetesConfig()
+	if err != nil {
+		panic(err.Error())
+	}
+	clientset, err = kubernetes.NewForConfig(config)
+	if err != nil {
+		panic(err.Error())
+	}
+}
+
+func runPodsCleanup() {
+	pods := clientset.CoreV1().Pods("modeltraining")
+	podList, err := pods.List(metav1.ListOptions{})
+	if err != nil {
+		logFatalf("[paddle] error fetching list of pods: %s", err.Error())
+	}
+
+	for _, pod := range podList.Items {
+		switch string(pod.Status.Phase) {
+		case "Succeeded":
+			deletePod(pods, pod)
+		case "Failed":
+			diff := time.Now().UTC().Sub(pod.CreationTimestamp.UTC())
+			if (diff.Hours() / 24) >= 1 {
+				deletePod(pods, pod)
+			}
+		default:
+			continue
+		}
+	}
+}
+
+func deletePod(podInterface v1.PodInterface, pod v12.Pod) {
+	err := podInterface.Delete(pod.Name, &metav1.DeleteOptions{})
+	if err != nil {
+		if k8errors.IsNotFound(err) {
+			log.Printf("[paddle] deleted pod %s", pod.Name)
+		} else {
+			log.Printf("[paddle] error deleting pod %s", pod.Name)
+		}
+	}
+	log.Printf("[paddle] deleted pod with pod name: %s, pod status: %s", pod.Name, string(pod.Status.Phase))
+}

cli/pipeline/template.go

@@ -3,8 +3,10 @@ package pipeline
 import (
 	"bytes"
 	"fmt"
+	"strconv"
 	"strings"
 	"text/template"
+	"time"
 )
 
 type PodSecret struct {
@@ -191,7 +193,8 @@ func NewPodDefinition(pipelineDefinition *PipelineDefinition, pipelineDefinition
 	stepName := sanitizeName(pipelineDefinitionStep.Step)
 	branchName := sanitizeName(pipelineDefinitionStep.Branch)
 	stepVersion := sanitizeName(pipelineDefinitionStep.Version)
-	podName := fmt.Sprintf("%s-%s-%s-%s", sanitizeName(pipelineDefinition.Pipeline), sanitizeName(pipelineDefinitionStep.Version), stepName, branchName)
+	timestamp := strconv.Itoa(int(time.Now().UTC().Unix()))
+	podName := fmt.Sprintf("%s-%s-%s-%s-%s", sanitizeName(pipelineDefinition.Pipeline), sanitizeName(pipelineDefinitionStep.Version), stepName, branchName, timestamp[len(timestamp)-4:])
 
 	return &PodDefinition{
 		PodName:         podName,

cli/root.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/deliveroo/paddle/cli/cleanup"
 	"github.com/deliveroo/paddle/cli/data"
 	"github.com/deliveroo/paddle/cli/pipeline"
 	"github.com/deliveroo/paddle/cli/steps"
@@ -58,6 +59,7 @@ func init() {
 	RootCmd.AddCommand(data.DataCmd)
 	RootCmd.AddCommand(pipeline.PipelineCmd)
 	RootCmd.AddCommand(steps.StepsCmd)
+	RootCmd.AddCommand(cleanup.CleanUpCmd)
 }
 
 // initConfig reads in config file and ENV variables if set.