From 2ff36746e8125834273145f638ec767a77b3d9d1 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Thu, 7 Nov 2019 20:09:41 +0800 Subject: [PATCH 01/34] add ssh-key-file-path to allow users specify ssh key files path --- pkg/controllers/job/plugins/ssh/ssh.go | 28 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/pkg/controllers/job/plugins/ssh/ssh.go b/pkg/controllers/job/plugins/ssh/ssh.go index 1c7e7ff023..4ca5216e06 100644 --- a/pkg/controllers/job/plugins/ssh/ssh.go +++ b/pkg/controllers/job/plugins/ssh/ssh.go @@ -23,10 +23,10 @@ import ( "encoding/pem" "flag" "fmt" - - "golang.org/x/crypto/ssh" + "strings" "github.com/golang/glog" + "golang.org/x/crypto/ssh" "k8s.io/api/core/v1" @@ -44,14 +44,23 @@ type sshPlugin struct { Clientset pluginsinterface.PluginClientset // flag parse args - noRoot bool + noRoot bool + sshKeyFilePath string } // New creates ssh plugin func New(client pluginsinterface.PluginClientset, arguments []string) pluginsinterface.PluginInterface { - sshPlugin := sshPlugin{pluginArguments: arguments, Clientset: client} + sshPlugin := sshPlugin{ + pluginArguments: arguments, + Clientset: client, + sshKeyFilePath: SSHAbsolutePath, + } sshPlugin.addFlags() + // if not set ssh key files path, use the default. + if sshPlugin.noRoot && sshPlugin.sshKeyFilePath == SSHAbsolutePath { + sshPlugin.sshKeyFilePath = env.ConfigMapMountPath + "/" + SSHRelativePath + } return &sshPlugin } @@ -94,10 +103,6 @@ func (sp *sshPlugin) OnJobDelete(job *batch.Job) error { } func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) { - sshPath := SSHAbsolutePath - if sp.noRoot { - sshPath = env.ConfigMapMountPath + "/" + SSHRelativePath - } cmName := sp.cmName(job) sshVolume := v1.Volume{ @@ -129,7 +134,7 @@ func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) { DefaultMode: &mode, } - if sshPath != SSHAbsolutePath { + if sp.sshKeyFilePath != SSHAbsolutePath { var noRootMode int32 = 0755 sshVolume.ConfigMap.DefaultMode = &noRootMode } @@ -138,7 +143,7 @@ func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) { for i, c := range pod.Spec.Containers { vm := v1.VolumeMount{ - MountPath: sshPath, + MountPath: strings.TrimSuffix(sp.sshKeyFilePath, "/"+SSHRelativePath), SubPath: SSHRelativePath, Name: cmName, } @@ -187,7 +192,10 @@ func (sp *sshPlugin) cmName(job *batch.Job) string { func (sp *sshPlugin) addFlags() { flagSet := flag.NewFlagSet(sp.Name(), flag.ContinueOnError) + // TODO: deprecate no-root flagSet.BoolVar(&sp.noRoot, "no-root", sp.noRoot, "The ssh user, --no-root is common user") + flagSet.StringVar(&sp.sshKeyFilePath, "ssh-key-file-path", sp.sshKeyFilePath, "The path used to store "+ + "ssh private and public keys, it is `/root/.ssh` by default.") if err := flagSet.Parse(sp.pluginArguments); err != nil { glog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) From f7e7ed84eace1a05b80b3eb85d37945dc495c9ad Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Thu, 7 Nov 2019 20:24:16 +0800 Subject: [PATCH 02/34] Add ssh plugin ut --- pkg/controllers/job/plugins/ssh/ssh_test.go | 72 +++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 pkg/controllers/job/plugins/ssh/ssh_test.go diff --git a/pkg/controllers/job/plugins/ssh/ssh_test.go b/pkg/controllers/job/plugins/ssh/ssh_test.go new file mode 100644 index 0000000000..4017c81fd1 --- /dev/null +++ b/pkg/controllers/job/plugins/ssh/ssh_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ssh + +import ( + "testing" + + "volcano.sh/volcano/pkg/controllers/job/plugins/env" + "volcano.sh/volcano/pkg/controllers/job/plugins/interface" +) + +func TestSSHPlugin(t *testing.T) { + + tests := []struct { + name string + params []string + noRoot bool + sshKeyFilePath string + }{ + { + name: "no params specified", + noRoot: false, + sshKeyFilePath: SSHAbsolutePath, + }, + { + name: "--no-root=true, ssh-key-file-path empty", + params: []string{"--no-root=true"}, + noRoot: true, + sshKeyFilePath: env.ConfigMapMountPath + "/" + SSHRelativePath, + }, + { + name: "--no-root=true, --ssh-key-file-path=/a/b", + params: []string{"--no-root=true", "--ssh-key-file-path=/a/b"}, + noRoot: true, + sshKeyFilePath: "/a/b", + }, + { + name: "--ssh-key-file-path=/a/b", + params: []string{"--ssh-key-file-path=/a/b"}, + noRoot: false, + sshKeyFilePath: "/a/b", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pluginInterface := New(pluginsinterface.PluginClientset{}, test.params) + plugin := pluginInterface.(*sshPlugin) + if plugin.noRoot != test.noRoot { + t.Errorf("Expected noRoot=%v, got %v", test.noRoot, plugin.noRoot) + } + + if plugin.sshKeyFilePath != test.sshKeyFilePath { + t.Errorf("Expected sshKeyFilePath=%s, got %s", test.sshKeyFilePath, plugin.sshKeyFilePath) + } + }) + } +} From bf267143b61be605fbeab1308f6040010a148f7a Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Mon, 11 Nov 2019 10:32:40 +0800 Subject: [PATCH 03/34] Fix key files mode --- pkg/controllers/job/plugins/ssh/ssh.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/controllers/job/plugins/ssh/ssh.go b/pkg/controllers/job/plugins/ssh/ssh.go index 4ca5216e06..1bad52b97e 100644 --- a/pkg/controllers/job/plugins/ssh/ssh.go +++ b/pkg/controllers/job/plugins/ssh/ssh.go @@ -23,8 +23,6 @@ import ( "encoding/pem" "flag" "fmt" - "strings" - "github.com/golang/glog" "golang.org/x/crypto/ssh" @@ -143,7 +141,7 @@ func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) { for i, c := range pod.Spec.Containers { vm := v1.VolumeMount{ - MountPath: strings.TrimSuffix(sp.sshKeyFilePath, "/"+SSHRelativePath), + MountPath: sp.sshKeyFilePath, SubPath: SSHRelativePath, Name: cmName, } From 450fbf7950184a6f01ac0b3485825be5c7fd09a9 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Thu, 21 Nov 2019 11:23:32 +0800 Subject: [PATCH 04/34] Simplify job pvc create process --- pkg/controllers/job/helpers/helpers.go | 10 +++--- pkg/controllers/job/job_controller_actions.go | 31 ++++++------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/pkg/controllers/job/helpers/helpers.go b/pkg/controllers/job/helpers/helpers.go index 86583214e4..207c164c45 100644 --- a/pkg/controllers/job/helpers/helpers.go +++ b/pkg/controllers/job/helpers/helpers.go @@ -30,8 +30,8 @@ import ( const ( // PodNameFmt pod name format PodNameFmt = "%s-%s-%d" - // VolumeClaimFmt volume claim name format - VolumeClaimFmt = "%s-volume-%s" + // persistentVolumeClaimFmt represents persistent volume claim name format + persistentVolumeClaimFmt = "%s-pvc-%s" ) // GetTaskIndex returns task Index @@ -61,9 +61,9 @@ func GenRandomStr(l int) string { return string(result) } -// MakeVolumeClaimName creates volume claim name -func MakeVolumeClaimName(jobName string) string { - return fmt.Sprintf(VolumeClaimFmt, jobName, GenRandomStr(12)) +// GenPVCName generates pvc name with job name +func GenPVCName(jobName string) string { + return fmt.Sprintf(persistentVolumeClaimFmt, jobName, GenRandomStr(12)) } // GetJobKeyByReq gets the key for the job request diff --git a/pkg/controllers/job/job_controller_actions.go b/pkg/controllers/job/job_controller_actions.go index 0b3f7abe87..13e389b506 100644 --- a/pkg/controllers/job/job_controller_actions.go +++ b/pkg/controllers/job/job_controller_actions.go @@ -17,7 +17,6 @@ limitations under the License. package job import ( - "errors" "fmt" "sort" "sync" @@ -338,16 +337,15 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt func (cc *Controller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) { // If PVC does not exist, create them for Job. var needUpdate bool - volumes := job.Spec.Volumes if job.Status.ControlledResources == nil { job.Status.ControlledResources = make(map[string]string) } - for index, volume := range volumes { + for index, volume := range job.Spec.Volumes { vcName := volume.VolumeClaimName if len(vcName) == 0 { - //NOTE(k82cn): Ensure never have duplicated generated names. + // NOTE(k82cn): Ensure never have duplicated generated names. for { - vcName = jobhelpers.MakeVolumeClaimName(job.Name) + vcName = jobhelpers.GenPVCName(job.Name) exist, err := cc.checkPVCExist(job, vcName) if err != nil { return job, err @@ -363,26 +361,17 @@ func (cc *Controller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) if err := cc.createPVC(job, vcName, volume.VolumeClaim); err != nil { return job, err } - job.Status.ControlledResources["volume-pvc-"+vcName] = vcName - } else { - job.Status.ControlledResources["volume-emptyDir-"+vcName] = vcName } } else { - if job.Status.ControlledResources["volume-emptyDir-"+vcName] == vcName || job.Status.ControlledResources["volume-pvc-"+vcName] == vcName { - continue - } exist, err := cc.checkPVCExist(job, vcName) if err != nil { return job, err } - if exist { - job.Status.ControlledResources["volume-pvc-"+vcName] = vcName - } else { - msg := fmt.Sprintf("pvc %s is not found, the job will be in the Pending state until the PVC is created", vcName) - glog.Error(msg) - return job, errors.New(msg) + if !exist { + return job, fmt.Errorf("pvc %s is not found, the job will be in the Pending state until the PVC is created", vcName) } } + job.Status.ControlledResources["volume-pvc-"+vcName] = vcName } if needUpdate { newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(job) @@ -398,13 +387,13 @@ func (cc *Controller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) return job, nil } -func (cc *Controller) checkPVCExist(job *batch.Job, vcName string) (bool, error) { - if _, err := cc.pvcLister.PersistentVolumeClaims(job.Namespace).Get(vcName); err != nil { +func (cc *Controller) checkPVCExist(job *batch.Job, pvc string) (bool, error) { + if _, err := cc.pvcLister.PersistentVolumeClaims(job.Namespace).Get(pvc); err != nil { if apierrors.IsNotFound(err) { return false, nil } - glog.V(3).Infof("Failed to get PVC for job <%s/%s>: %v", - job.Namespace, job.Name, err) + glog.V(3).Infof("Failed to get PVC %s for job <%s/%s>: %v", + pvc, job.Namespace, job.Name, err) return false, err } return true, nil From 3daeba17037702d4e8b4e7b8f8f5c8bec1b655fd Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Thu, 21 Nov 2019 14:47:25 +0800 Subject: [PATCH 05/34] Must specify VolumeSpec.VolumeClaimName or VolumeSpec.VolumeClaim --- pkg/admission/admission_controller.go | 19 ++++++++++------ pkg/admission/admit_job.go | 4 ++-- pkg/controllers/job/job_controller_util.go | 25 +++++++++------------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/admission/admission_controller.go b/pkg/admission/admission_controller.go index 22a370767d..8002345952 100644 --- a/pkg/admission/admission_controller.go +++ b/pkg/admission/admission_controller.go @@ -232,25 +232,30 @@ func getValidActions() []batchv1alpha1.Action { return actions } -// ValidateIO validate IO configuration -func ValidateIO(volumes []batchv1alpha1.VolumeSpec) (string, bool) { +// validateIO validates IO configuration +func validateIO(volumes []batchv1alpha1.VolumeSpec) error { volumeMap := map[string]bool{} for _, volume := range volumes { if len(volume.MountPath) == 0 { - return " mountPath is required;", true + return fmt.Errorf(" mountPath is required;") } if _, found := volumeMap[volume.MountPath]; found { - return fmt.Sprintf(" duplicated mountPath: %s;", volume.MountPath), true + return fmt.Errorf(" duplicated mountPath: %s;", volume.MountPath) + } + if volume.VolumeClaim == nil && volume.VolumeClaimName == "" { + return fmt.Errorf(" either VolumeClaim or VolumeClaimName must be specified;") } if len(volume.VolumeClaimName) != 0 { if volume.VolumeClaim != nil { - return fmt.Sprintf("Confilct: If you want to use an existing PVC, just specify VolumeClaimName. If you want to create a new PVC, you do not need to specify VolumeClaimName."), true + return fmt.Errorf("confilct: If you want to use an existing PVC, just specify VolumeClaimName." + + "If you want to create a new PVC, you do not need to specify VolumeClaimName") } if errMsgs := validation.ValidatePersistentVolumeName(volume.VolumeClaimName, false); len(errMsgs) > 0 { - return fmt.Sprintf("Illegal VolumeClaimName %s : %v", volume.VolumeClaimName, errMsgs), true + return fmt.Errorf("invalid VolumeClaimName %s : %v", volume.VolumeClaimName, errMsgs) } } + volumeMap[volume.MountPath] = true } - return "", false + return nil } diff --git a/pkg/admission/admit_job.go b/pkg/admission/admit_job.go index 1bfc9784ab..7136a42235 100644 --- a/pkg/admission/admit_job.go +++ b/pkg/admission/admit_job.go @@ -147,8 +147,8 @@ func validateJob(job v1alpha1.Job, reviewResponse *v1beta1.AdmissionResponse) st } } - if validateInfo, ok := ValidateIO(job.Spec.Volumes); ok { - msg = msg + validateInfo + if err := validateIO(job.Spec.Volumes); err != nil { + msg = msg + err.Error() } // Check whether Queue already present or not diff --git a/pkg/controllers/job/job_controller_util.go b/pkg/controllers/job/job_controller_util.go index df37c349d4..0b5d8deca2 100644 --- a/pkg/controllers/job/job_controller_util.go +++ b/pkg/controllers/job/job_controller_util.go @@ -63,24 +63,19 @@ func createJobPod(job *batch.Job, template *v1.PodTemplateSpec, ix int) *v1.Pod vcName := volume.VolumeClaimName name := fmt.Sprintf("%s-%s", job.Name, jobhelpers.GenRandomStr(12)) if _, ok := volumeMap[vcName]; !ok { - if _, ok := job.Status.ControlledResources["volume-emptyDir-"+vcName]; ok { - volume := v1.Volume{ - Name: name, - } - volume.EmptyDir = &v1.EmptyDirVolumeSource{} - pod.Spec.Volumes = append(pod.Spec.Volumes, volume) - } else { - volume := v1.Volume{ - Name: name, - } - volume.PersistentVolumeClaim = &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: vcName, - } - pod.Spec.Volumes = append(pod.Spec.Volumes, volume) + volume := v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: vcName, + }, + }, } + pod.Spec.Volumes = append(pod.Spec.Volumes, volume) volumeMap[vcName] = name } else { - name = volumeMap[vcName] + // duplicate volumes, should be prevented + continue } for i, c := range pod.Spec.Containers { From 43853298da6f0c0ca1ab6c565be61f6a113e9a56 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Thu, 21 Nov 2019 15:04:34 +0800 Subject: [PATCH 06/34] fix ut --- .../job/job_controller_util_test.go | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/pkg/controllers/job/job_controller_util_test.go b/pkg/controllers/job/job_controller_util_test.go index 624e081fff..da5cf11576 100644 --- a/pkg/controllers/job/job_controller_util_test.go +++ b/pkg/controllers/job/job_controller_util_test.go @@ -213,11 +213,6 @@ func TestCreateJobPod(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, PodTemplate: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -292,11 +287,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ Action: v1alpha1.EnqueueAction, @@ -332,11 +322,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ Event: v1alpha1.OutOfSyncEvent, @@ -372,12 +357,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - Version: 2, - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ JobVersion: 1, @@ -420,11 +399,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ TaskName: "task1", @@ -466,11 +440,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ TaskName: "task1", @@ -507,11 +476,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ TaskName: "task1", @@ -554,11 +518,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ Event: v1alpha1.CommandIssuedEvent, @@ -601,11 +560,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{}, ReturnVal: v1alpha1.SyncJobAction, From 1fd5d971ebc383c4741d4d655048eba0f26b106c Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Thu, 21 Nov 2019 15:18:54 +0800 Subject: [PATCH 07/34] Add test for invalid volumes definition --- pkg/admission/admit_job_test.go | 53 ++++++++++++++++++++++++++++- test/e2e/job_controlled_resource.go | 41 ---------------------- 2 files changed, 52 insertions(+), 42 deletions(-) diff --git a/pkg/admission/admit_job_test.go b/pkg/admission/admit_job_test.go index 88896d8ff3..f1e8243925 100644 --- a/pkg/admission/admit_job_test.go +++ b/pkg/admission/admit_job_test.go @@ -820,6 +820,57 @@ func TestValidateExecution(t *testing.T) { Name: "duplicate-mount-volume", Namespace: namespace, }, + Spec: v1alpha1.JobSpec{ + MinAvailable: 1, + Queue: "default", + Tasks: []v1alpha1.TaskSpec{ + { + Name: "task-1", + Replicas: 1, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"name": "test"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fake-name", + Image: "busybox:1.24", + }, + }, + }, + }, + }, + }, + Policies: []v1alpha1.LifecyclePolicy{ + { + Event: v1alpha1.AnyEvent, + Action: v1alpha1.AbortJobAction, + }, + }, + Volumes: []v1alpha1.VolumeSpec{ + { + MountPath: "/var", + VolumeClaimName: "pvc1", + }, + { + MountPath: "/var", + VolumeClaimName: "pvc2", + }, + }, + }, + }, + reviewResponse: v1beta1.AdmissionResponse{Allowed: true}, + ret: " duplicated mountPath: /var;", + ExpectErr: true, + }, + { + Name: "volume without VolumeClaimName and VolumeClaim", + Job: v1alpha1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-volume", + Namespace: namespace, + }, Spec: v1alpha1.JobSpec{ MinAvailable: 1, Queue: "default", @@ -859,7 +910,7 @@ func TestValidateExecution(t *testing.T) { }, }, reviewResponse: v1beta1.AdmissionResponse{Allowed: true}, - ret: " duplicated mountPath: /var;", + ret: " either VolumeClaim or VolumeClaimName must be specified;", ExpectErr: true, }, // task Policy with any event and other events diff --git a/test/e2e/job_controlled_resource.go b/test/e2e/job_controlled_resource.go index 4f173719a3..c19e01755c 100644 --- a/test/e2e/job_controlled_resource.go +++ b/test/e2e/job_controlled_resource.go @@ -26,47 +26,6 @@ import ( ) var _ = Describe("Job E2E Test: Test Job PVCs", func() { - It("Generate PVC name if not specified", func() { - jobName := "job-pvc-name-empty" - namespace := "test" - taskName := "task" - pvcName := "specifiedpvcname" - context := initTestContext() - defer cleanupTestContext(context) - - job := createJob(context, &jobSpec{ - namespace: namespace, - name: jobName, - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: oneCPU, - min: 1, - rep: 1, - name: taskName, - }, - }, - volumes: []v1alpha1.VolumeSpec{ - { - MountPath: "/mounttwo", - }, - }, - }) - - err := waitJobReady(context, job) - Expect(err).NotTo(HaveOccurred()) - - job, err = context.vcclient.BatchV1alpha1().Jobs(namespace).Get(jobName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - - Expect(len(job.Spec.Volumes)).To(Equal(1), - "Two volumes should be created") - for _, volume := range job.Spec.Volumes { - Expect(volume.VolumeClaimName).Should(Or(ContainSubstring(jobName), Equal(pvcName)), - "PVC name should be generated for manually specified.") - } - }) - It("use exisisting PVC in job", func() { jobName := "job-pvc-name-exist" namespace := "test" From 913fb8097fe5616acd2473a0b35662186db2c759 Mon Sep 17 00:00:00 2001 From: Zhang Jinghui Date: Sat, 23 Nov 2019 19:18:00 +0800 Subject: [PATCH 08/34] modify comments on OnPodCreate function of svc plugin --- pkg/controllers/job/plugins/svc/svc.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/job/plugins/svc/svc.go b/pkg/controllers/job/plugins/svc/svc.go index c80070deee..b5a9ea1801 100644 --- a/pkg/controllers/job/plugins/svc/svc.go +++ b/pkg/controllers/job/plugins/svc/svc.go @@ -69,7 +69,17 @@ func (sp *servicePlugin) addFlags() { } func (sp *servicePlugin) OnPodCreate(pod *v1.Pod, job *batch.Job) error { - // use podName.serviceName as default pod DNS domain + // Add `hostname` and `subdomain` for pod, mount service config for pod. + // A pod with `hostname` and `subdomain` will have the fully qualified domain name(FQDN) + // `hostname.subdomain.namespace.svc.cluster-domain.example`. + // If there exists a headless service in the same namespace as the pod and with the + // same name as the `subdomain`, the cluster's KubeDNS Server will returns an A record for + // the Pods's fully qualified hostname, pointing to the Pod’s IP. + // `hostname.subdomain` will be used as address of the pod. + // By default, a client Pod’s DNS search list will include the Pod’s own namespace and + // the cluster’s default domain, so the pod can be accessed by pods in the same namespace + // through the address of pod. + // More info: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service if len(pod.Spec.Hostname) == 0 { pod.Spec.Hostname = pod.Name } From 9e2c1c5e9537b86fe407e3eb1f081d607e24b080 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Tue, 26 Nov 2019 15:17:27 +0800 Subject: [PATCH 09/34] Limit pods ingress access by networkpolicy --- pkg/controllers/job/plugins/svc/svc.go | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/pkg/controllers/job/plugins/svc/svc.go b/pkg/controllers/job/plugins/svc/svc.go index c80070deee..b800365267 100644 --- a/pkg/controllers/job/plugins/svc/svc.go +++ b/pkg/controllers/job/plugins/svc/svc.go @@ -24,6 +24,7 @@ import ( "github.com/golang/glog" "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -97,6 +98,11 @@ func (sp *servicePlugin) OnJobAdd(job *batch.Job) error { return err } + // TODO: maybe add a flag + if err := sp.createNetworkPolicyIfNotExist(job); err != nil { + return err + } + job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name() return nil @@ -188,6 +194,56 @@ func (sp *servicePlugin) createServiceIfNotExist(job *batch.Job) error { return nil } +// Limit pods can be accessible only by pods belong to the job. +func (sp *servicePlugin) createNetworkPolicyIfNotExist(job *batch.Job) error { + // If network policy does not exist, create one for Job. + if _, err := sp.Clientset.KubeClients.NetworkingV1().NetworkPolicies(job.Namespace).Get(job.Name, metav1.GetOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + glog.V(3).Infof("Failed to get NetworkPolicy for Job <%s/%s>: %v", + job.Namespace, job.Name, err) + return err + } + + networkpolicy := &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: job.Namespace, + Name: job.Name, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(job, helpers.JobKind), + }, + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + batch.JobNameKey: job.Name, + batch.JobNamespaceKey: job.Namespace, + }, + }, + Ingress: []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{ + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + batch.JobNameKey: job.Name, + batch.JobNamespaceKey: job.Namespace, + }, + }, + }}, + }}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + }, + } + + if _, e := sp.Clientset.KubeClients.NetworkingV1().NetworkPolicies(job.Namespace).Create(networkpolicy); e != nil { + glog.V(3).Infof("Failed to create Service for Job <%s/%s>: %v", job.Namespace, job.Name, e) + return e + } + job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name() + + } + + return nil +} + func (sp *servicePlugin) cmName(job *batch.Job) string { return fmt.Sprintf("%s-%s", job.Name, sp.Name()) } From 2791e8b28d3694b578414e518fc10f13df117edc Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Tue, 26 Nov 2019 17:43:35 +0800 Subject: [PATCH 10/34] Move myself to controller owner --- pkg/controllers/OWNERS | 2 ++ pkg/scheduler/OWNERS | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/OWNERS b/pkg/controllers/OWNERS index 16643b106d..f206ebbb72 100644 --- a/pkg/controllers/OWNERS +++ b/pkg/controllers/OWNERS @@ -1,4 +1,6 @@ reviewers: + - hzxuzhonghu - TommyLike approvers: + - hzxuzhonghu - TommyLike diff --git a/pkg/scheduler/OWNERS b/pkg/scheduler/OWNERS index 896e4cc053..caf70c72d5 100644 --- a/pkg/scheduler/OWNERS +++ b/pkg/scheduler/OWNERS @@ -2,9 +2,7 @@ approvers: - k82cn - animeshsingh - hex108 - - hzxuzhonghu reviewers: - k82cn - animeshsingh - - hex108 - - hzxuzhonghu + - hex108 \ No newline at end of file From ba3f278afa9cecc21bb4589594393b6ca1f19a01 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Wed, 27 Nov 2019 15:23:28 +0800 Subject: [PATCH 11/34] Update to klog. Signed-off-by: Klaus Ma --- Gopkg.lock | 9 - Gopkg.toml | 4 - Makefile | 1 + cmd/admission/app/options/options.go | 11 +- cmd/admission/app/server.go | 14 +- cmd/admission/main.go | 15 +- cmd/cli/vcctl.go | 10 +- cmd/controllers/app/server.go | 4 +- cmd/controllers/main.go | 9 +- cmd/scheduler/app/server.go | 6 +- cmd/scheduler/main.go | 8 +- pkg/admission/admission_controller.go | 6 +- pkg/admission/admit_job.go | 5 +- pkg/admission/admit_pod.go | 7 +- pkg/admission/mutate_job.go | 7 +- pkg/admission/server.go | 11 +- pkg/apis/helpers/helpers.go | 17 +- pkg/controllers/cache/cache.go | 9 +- .../garbagecollector/garbagecollector.go | 27 +- pkg/controllers/job/job_controller.go | 25 +- pkg/controllers/job/job_controller_actions.go | 61 +- pkg/controllers/job/job_controller_handler.go | 75 +- pkg/controllers/job/job_controller_plugins.go | 21 +- pkg/controllers/job/job_controller_resync.go | 10 +- pkg/controllers/job/job_controller_util.go | 5 +- pkg/controllers/job/plugins/ssh/ssh.go | 9 +- pkg/controllers/job/plugins/svc/svc.go | 10 +- pkg/controllers/podgroup/pg_controller.go | 11 +- .../podgroup/pg_controller_handler.go | 13 +- pkg/controllers/queue/queue_controller.go | 22 +- pkg/scheduler/actions/allocate/allocate.go | 38 +- pkg/scheduler/actions/backfill/backfill.go | 14 +- pkg/scheduler/actions/enqueue/enqueue.go | 16 +- pkg/scheduler/actions/preempt/preempt.go | 26 +- pkg/scheduler/actions/reclaim/reclaim.go | 30 +- pkg/scheduler/api/namespace_info.go | 5 +- pkg/scheduler/api/node_info.go | 5 +- pkg/scheduler/cache/cache.go | 55 +- pkg/scheduler/cache/event_handlers.go | 165 ++- pkg/scheduler/framework/arguments.go | 6 +- pkg/scheduler/framework/framework.go | 4 +- pkg/scheduler/framework/job_updater.go | 5 +- pkg/scheduler/framework/session.go | 41 +- pkg/scheduler/framework/statement.go | 58 +- pkg/scheduler/plugins/binpack/binpack.go | 17 +- pkg/scheduler/plugins/drf/drf.go | 12 +- pkg/scheduler/plugins/gang/gang.go | 11 +- pkg/scheduler/plugins/nodeorder/nodeorder.go | 25 +- .../plugins/predicates/predicates.go | 33 +- pkg/scheduler/plugins/priority/priority.go | 6 +- .../plugins/proportion/proportion.go | 28 +- pkg/scheduler/plugins/util/util.go | 5 +- pkg/scheduler/scheduler.go | 9 +- pkg/scheduler/util/assert/assert.go | 4 +- pkg/scheduler/util/scheduler_helper.go | 12 +- vendor/github.com/golang/glog/LICENSE | 191 --- vendor/github.com/golang/glog/README | 44 - vendor/github.com/golang/glog/glog.go | 1180 ----------------- vendor/github.com/golang/glog/glog_file.go | 124 -- 59 files changed, 519 insertions(+), 2092 deletions(-) delete mode 100644 vendor/github.com/golang/glog/LICENSE delete mode 100644 vendor/github.com/golang/glog/README delete mode 100644 vendor/github.com/golang/glog/glog.go delete mode 100644 vendor/github.com/golang/glog/glog_file.go diff --git a/Gopkg.lock b/Gopkg.lock index c8127ee1d3..fa702ea903 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -73,14 +73,6 @@ revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02" version = "v0.5" -[[projects]] - branch = "master" - digest = "1:1ba1d79f2810270045c328ae5d674321db34e3aae468eb4233883b473c5c0467" - name = "github.com/golang/glog" - packages = ["."] - pruneopts = "UT" - revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" - [[projects]] digest = "1:7672c206322f45b33fac1ae2cb899263533ce0adcc6481d207725560208ec84e" name = "github.com/golang/groupcache" @@ -1031,7 +1023,6 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ - "github.com/golang/glog", "github.com/hashicorp/go-multierror", "github.com/onsi/ginkgo", "github.com/onsi/gomega", diff --git a/Gopkg.toml b/Gopkg.toml index 6d42725fd1..82100b6136 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -32,10 +32,6 @@ required = [ "k8s.io/code-generator/cmd/defaulter-gen", ] -[[constraint]] - branch = "master" - name = "github.com/golang/glog" - [[constraint]] name = "github.com/onsi/ginkgo" version = "1.7.0" diff --git a/Makefile b/Makefile index ca3461ded1..d6f01a608b 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,7 @@ generate-code: ./hack/update-gencode.sh unit-test: + go clean -testcache go list ./... | grep -v e2e | xargs go test -v -race e2e-test-kind: diff --git a/cmd/admission/app/options/options.go b/cmd/admission/app/options/options.go index 13227fbee9..5ca24216a8 100644 --- a/cmd/admission/app/options/options.go +++ b/cmd/admission/app/options/options.go @@ -20,13 +20,12 @@ import ( "flag" "fmt" - "github.com/golang/glog" - "k8s.io/api/admissionregistration/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" admissionregistrationv1beta1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1" + "k8s.io/klog" ) const ( @@ -235,13 +234,13 @@ func registerMutateWebhook(client admissionregistrationv1beta1.MutatingWebhookCo return err } if err == nil && existing != nil { - glog.Infof("Updating MutatingWebhookConfiguration %v", hook) + klog.Infof("Updating MutatingWebhookConfiguration %v", hook) existing.Webhooks = hook.Webhooks if _, err := client.Update(existing); err != nil { return err } } else { - glog.Infof("Creating MutatingWebhookConfiguration %v", hook) + klog.Infof("Creating MutatingWebhookConfiguration %v", hook) if _, err := client.Create(&hook); err != nil { return err } @@ -259,12 +258,12 @@ func registerValidateWebhook(client admissionregistrationv1beta1.ValidatingWebho } if err == nil && existing != nil { existing.Webhooks = hook.Webhooks - glog.Infof("Updating ValidatingWebhookConfiguration %v", hook) + klog.Infof("Updating ValidatingWebhookConfiguration %v", hook) if _, err := client.Update(existing); err != nil { return err } } else { - glog.Infof("Creating ValidatingWebhookConfiguration %v", hook) + klog.Infof("Creating ValidatingWebhookConfiguration %v", hook) if _, err := client.Create(&hook); err != nil { return err } diff --git a/cmd/admission/app/server.go b/cmd/admission/app/server.go index 41a1389a81..14e4b045c1 100644 --- a/cmd/admission/app/server.go +++ b/cmd/admission/app/server.go @@ -19,10 +19,10 @@ package app import ( "crypto/tls" - "github.com/golang/glog" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/klog" + "volcano.sh/volcano/cmd/admission/app/options" "volcano.sh/volcano/pkg/client/clientset/versioned" ) @@ -31,7 +31,7 @@ import ( func GetClient(restConfig *rest.Config) *kubernetes.Clientset { clientset, err := kubernetes.NewForConfig(restConfig) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return clientset } @@ -40,7 +40,7 @@ func GetClient(restConfig *rest.Config) *kubernetes.Clientset { func GetVolcanoClient(restConfig *rest.Config) *versioned.Clientset { clientset, err := versioned.NewForConfig(restConfig) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return clientset } @@ -52,7 +52,7 @@ func ConfigTLS(config *options.Config, restConfig *rest.Config) *tls.Config { if len(config.CertFile) != 0 && len(config.KeyFile) != 0 { sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return &tls.Config{ @@ -63,7 +63,7 @@ func ConfigTLS(config *options.Config, restConfig *rest.Config) *tls.Config { if len(restConfig.CertData) != 0 && len(restConfig.KeyData) != 0 { sCert, err := tls.X509KeyPair(restConfig.CertData, restConfig.KeyData) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return &tls.Config{ @@ -71,6 +71,6 @@ func ConfigTLS(config *options.Config, restConfig *rest.Config) *tls.Config { } } - glog.Fatal("tls: failed to find any tls config data") + klog.Fatal("tls: failed to find any tls config data") return &tls.Config{} } diff --git a/cmd/admission/main.go b/cmd/admission/main.go index c5d3789c41..60e5e5313c 100644 --- a/cmd/admission/main.go +++ b/cmd/admission/main.go @@ -24,9 +24,8 @@ import ( "strconv" "syscall" - "github.com/golang/glog" - "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" "volcano.sh/volcano/cmd/admission/app" "volcano.sh/volcano/cmd/admission/app/options" @@ -55,13 +54,13 @@ func main() { http.HandleFunc(admission.MutateJobPath, serveMutateJobs) if err := config.CheckPortOrDie(); err != nil { - glog.Fatalf("Configured port is invalid: %v", err) + klog.Fatalf("Configured port is invalid: %v", err) } addr := ":" + strconv.Itoa(config.Port) restConfig, err := clientcmd.BuildConfigFromFlags(config.Master, config.Kubeconfig) if err != nil { - glog.Fatalf("Unable to build k8s config: %v", err) + klog.Fatalf("Unable to build k8s config: %v", err) } admission.VolcanoClientSet = app.GetVolcanoClient(restConfig) @@ -70,12 +69,12 @@ func main() { caBundle, err := ioutil.ReadFile(config.CaCertFile) if err != nil { - glog.Fatalf("Unable to read cacert file: %v", err) + klog.Fatalf("Unable to read cacert file: %v", err) } err = options.RegisterWebhooks(config, app.GetClient(restConfig), caBundle) if err != nil { - glog.Fatalf("Unable to register webhook configs: %v", err) + klog.Fatalf("Unable to register webhook configs: %v", err) } stopChannel := make(chan os.Signal) @@ -89,7 +88,7 @@ func main() { go func() { err = server.ListenAndServeTLS("", "") if err != nil && err != http.ErrServerClosed { - glog.Fatalf("ListenAndServeTLS for admission webhook failed: %v", err) + klog.Fatalf("ListenAndServeTLS for admission webhook failed: %v", err) close(webhookServeError) } }() @@ -97,7 +96,7 @@ func main() { select { case <-stopChannel: if err := server.Close(); err != nil { - glog.Fatalf("Close admission server failed: %v", err) + klog.Fatalf("Close admission server failed: %v", err) } return case <-webhookServeError: diff --git a/cmd/cli/vcctl.go b/cmd/cli/vcctl.go index 7d32886659..42197f88a8 100644 --- a/cmd/cli/vcctl.go +++ b/cmd/cli/vcctl.go @@ -20,11 +20,11 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" "volcano.sh/volcano/pkg/version" ) @@ -34,9 +34,11 @@ var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum func main() { // flag.InitFlags() - // The default glog flush interval is 30 seconds, which is frighteningly long. - go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop) - defer glog.Flush() + klog.InitFlags(nil) + + // The default klog flush interval is 30 seconds, which is frighteningly long. + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) + defer klog.Flush() rootCmd := cobra.Command{ Use: "vcctl", diff --git a/cmd/controllers/app/server.go b/cmd/controllers/app/server.go index b210f0059e..5f5496dd61 100644 --- a/cmd/controllers/app/server.go +++ b/cmd/controllers/app/server.go @@ -22,7 +22,7 @@ import ( "os" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -129,7 +129,7 @@ func Run(opt *options.ServerOption) error { Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { - glog.Fatalf("leaderelection lost") + klog.Fatalf("leaderelection lost") }, }, }) diff --git a/cmd/controllers/main.go b/cmd/controllers/main.go index 64ee534b34..44623c6800 100644 --- a/cmd/controllers/main.go +++ b/cmd/controllers/main.go @@ -20,11 +20,11 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/util/flag" + "k8s.io/klog" "volcano.sh/volcano/cmd/controllers/app" "volcano.sh/volcano/cmd/controllers/app/options" @@ -38,6 +38,7 @@ func main() { s.AddFlags(pflag.CommandLine) flag.InitFlags() + klog.InitFlags(nil) if s.PrintVersion { version.PrintVersionAndExit() @@ -46,9 +47,9 @@ func main() { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } - // The default glog flush interval is 30 seconds, which is frighteningly long. - go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop) - defer glog.Flush() + // The default klog flush interval is 30 seconds, which is frighteningly long. + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) + defer klog.Flush() if err := app.Run(s); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) diff --git a/cmd/scheduler/app/server.go b/cmd/scheduler/app/server.go index d7c1784d7e..3010bbd70b 100644 --- a/cmd/scheduler/app/server.go +++ b/cmd/scheduler/app/server.go @@ -23,7 +23,6 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus/promhttp" "volcano.sh/volcano/cmd/scheduler/app/options" @@ -36,6 +35,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/klog" // Register gcp auth _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -95,7 +95,7 @@ func Run(opt *options.ServerOption) error { go func() { http.Handle("/metrics", promhttp.Handler()) - glog.Fatalf("Prometheus Http Server failed %s", http.ListenAndServe(opt.ListenAddress, nil)) + klog.Fatalf("Prometheus Http Server failed %s", http.ListenAndServe(opt.ListenAddress, nil)) }() if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-scheduler"); err != nil { @@ -149,7 +149,7 @@ func Run(opt *options.ServerOption) error { Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { - glog.Fatalf("leaderelection lost") + klog.Fatalf("leaderelection lost") }, }, }) diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go index 63ea46e1b3..344a99c380 100644 --- a/cmd/scheduler/main.go +++ b/cmd/scheduler/main.go @@ -24,11 +24,11 @@ import ( // init pprof server _ "net/http/pprof" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/util/flag" + "k8s.io/klog" "volcano.sh/volcano/cmd/scheduler/app" "volcano.sh/volcano/cmd/scheduler/app/options" @@ -46,6 +46,8 @@ var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum func main() { runtime.GOMAXPROCS(runtime.NumCPU()) + klog.InitFlags(nil) + s := options.NewServerOption() s.AddFlags(pflag.CommandLine) s.RegisterOptions() @@ -56,8 +58,8 @@ func main() { os.Exit(1) } - go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop) - defer glog.Flush() + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) + defer klog.Flush() if err := app.Run(s); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) diff --git a/pkg/admission/admission_controller.go b/pkg/admission/admission_controller.go index 8002345952..c7f9794607 100644 --- a/pkg/admission/admission_controller.go +++ b/pkg/admission/admission_controller.go @@ -19,7 +19,6 @@ package admission import ( "fmt" - "github.com/golang/glog" "github.com/hashicorp/go-multierror" "k8s.io/api/admission/v1beta1" @@ -29,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog" "k8s.io/kubernetes/pkg/apis/core/validation" batchv1alpha1 "volcano.sh/volcano/pkg/apis/batch/v1alpha1" @@ -97,7 +97,7 @@ func addToScheme(scheme *runtime.Scheme) { //ToAdmissionResponse updates the admission response with the input error func ToAdmissionResponse(err error) *v1beta1.AdmissionResponse { - glog.Error(err) + klog.Error(err) return &v1beta1.AdmissionResponse{ Result: &metav1.Status{ Message: err.Error(), @@ -120,7 +120,7 @@ func DecodeJob(object runtime.RawExtension, resource metav1.GroupVersionResource if _, _, err := deserializer.Decode(raw, nil, &job); err != nil { return job, err } - glog.V(3).Infof("the job struct is %+v", job) + klog.V(3).Infof("the job struct is %+v", job) return job, nil } diff --git a/pkg/admission/admit_job.go b/pkg/admission/admit_job.go index 7136a42235..a34e8a3bf3 100644 --- a/pkg/admission/admit_job.go +++ b/pkg/admission/admit_job.go @@ -20,13 +20,12 @@ import ( "fmt" "strings" - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog" k8score "k8s.io/kubernetes/pkg/apis/core" k8scorev1 "k8s.io/kubernetes/pkg/apis/core/v1" k8scorevalid "k8s.io/kubernetes/pkg/apis/core/validation" @@ -43,7 +42,7 @@ var VolcanoClientSet vcclientset.Interface // AdmitJobs is to admit jobs and return response func AdmitJobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(3).Infof("admitting jobs -- %s", ar.Request.Operation) + klog.V(3).Infof("admitting jobs -- %s", ar.Request.Operation) job, err := DecodeJob(ar.Request.Object, ar.Request.Resource) if err != nil { diff --git a/pkg/admission/admit_pod.go b/pkg/admission/admit_pod.go index 43395ff040..09d891f498 100644 --- a/pkg/admission/admit_pod.go +++ b/pkg/admission/admit_pod.go @@ -21,13 +21,12 @@ import ( "net/http" "strings" - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/helpers" "volcano.sh/volcano/pkg/apis/scheduling/v1alpha1" @@ -42,7 +41,7 @@ func (c *Controller) ServerPods(w http.ResponseWriter, r *http.Request) { // AdmitPods is to admit pods and return response func (c *Controller) AdmitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(3).Infof("admitting pods -- %s", ar.Request.Operation) + klog.V(3).Infof("admitting pods -- %s", ar.Request.Operation) pod, err := decodePod(ar.Request.Object, ar.Request.Resource) if err != nil { @@ -82,7 +81,7 @@ func decodePod(object runtime.RawExtension, resource metav1.GroupVersionResource if _, _, err := deserializer.Decode(raw, nil, &pod); err != nil { return pod, err } - glog.V(3).Infof("the pod struct is %+v", pod) + klog.V(3).Infof("the pod struct is %+v", pod) return pod, nil } diff --git a/pkg/admission/mutate_job.go b/pkg/admission/mutate_job.go index 6c08115215..8fba26f7e0 100644 --- a/pkg/admission/mutate_job.go +++ b/pkg/admission/mutate_job.go @@ -21,10 +21,9 @@ import ( "fmt" "strconv" - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/batch/v1alpha1" ) @@ -42,7 +41,7 @@ type patchOperation struct { // MutateJobs mutate jobs func MutateJobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(3).Infof("mutating jobs") + klog.V(3).Infof("mutating jobs") job, err := DecodeJob(ar.Request.Object, ar.Request.Resource) if err != nil { @@ -66,7 +65,7 @@ func MutateJobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { reviewResponse.Result = &metav1.Status{Message: err.Error()} return &reviewResponse } - glog.V(3).Infof("AdmissionResponse: patch=%v\n", string(patchBytes)) + klog.V(3).Infof("AdmissionResponse: patch=%v\n", string(patchBytes)) reviewResponse.Patch = patchBytes pt := v1beta1.PatchTypeJSONPatch reviewResponse.PatchType = &pt diff --git a/pkg/admission/server.go b/pkg/admission/server.go index 4c3fa15e88..f61a8d9d66 100644 --- a/pkg/admission/server.go +++ b/pkg/admission/server.go @@ -21,10 +21,9 @@ import ( "io/ioutil" "net/http" - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog" ) // Serve the http request @@ -39,7 +38,7 @@ func Serve(w http.ResponseWriter, r *http.Request, admit AdmitFunc) { // verify the content type is accurate contentType := r.Header.Get(CONTENTTYPE) if contentType != APPLICATIONJSON { - glog.Errorf("contentType=%s, expect application/json", contentType) + klog.Errorf("contentType=%s, expect application/json", contentType) return } @@ -51,15 +50,15 @@ func Serve(w http.ResponseWriter, r *http.Request, admit AdmitFunc) { } else { reviewResponse = admit(ar) } - glog.V(3).Infof("sending response: %v", reviewResponse) + klog.V(3).Infof("sending response: %v", reviewResponse) response := createResponse(reviewResponse, &ar) resp, err := json.Marshal(response) if err != nil { - glog.Error(err) + klog.Error(err) } if _, err := w.Write(resp); err != nil { - glog.Error(err) + klog.Error(err) } } diff --git a/pkg/apis/helpers/helpers.go b/pkg/apis/helpers/helpers.go index 130fcb6c1e..c42601a891 100644 --- a/pkg/apis/helpers/helpers.go +++ b/pkg/apis/helpers/helpers.go @@ -26,8 +26,6 @@ import ( "syscall" "time" - "github.com/golang/glog" - "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -38,6 +36,7 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/mux" "k8s.io/client-go/kubernetes" + "k8s.io/klog" vcbatch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" vcbus "volcano.sh/volcano/pkg/apis/bus/v1alpha1" @@ -85,7 +84,7 @@ func CreateConfigMapIfNotExist(job *vcbatch.Job, kubeClients kubernetes.Interfac cmOld, err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Get(cmName, metav1.GetOptions{}) if err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get Configmap for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get Configmap for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -102,7 +101,7 @@ func CreateConfigMapIfNotExist(job *vcbatch.Job, kubeClients kubernetes.Interfac } if _, err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Create(cm); err != nil { - glog.V(3).Infof("Failed to create ConfigMap for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to create ConfigMap for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -111,7 +110,7 @@ func CreateConfigMapIfNotExist(job *vcbatch.Job, kubeClients kubernetes.Interfac cmOld.Data = data if _, err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Update(cmOld); err != nil { - glog.V(3).Infof("Failed to update ConfigMap for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to update ConfigMap for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -123,7 +122,7 @@ func CreateConfigMapIfNotExist(job *vcbatch.Job, kubeClients kubernetes.Interfac func DeleteConfigmap(job *vcbatch.Job, kubeClients kubernetes.Interface, cmName string) error { if _, err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Get(cmName, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get Configmap for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get Configmap for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -133,7 +132,7 @@ func DeleteConfigmap(job *vcbatch.Job, kubeClients kubernetes.Interface, cmName if err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Delete(cmName, nil); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete Configmap of Job %v/%v: %v", + klog.Errorf("Failed to delete Configmap of Job %v/%v: %v", job.Namespace, job.Name, err) return err } @@ -204,9 +203,9 @@ func runServer(server *http.Server, ln net.Listener) error { msg := fmt.Sprintf("Stopped listening on %s", listener.Addr().String()) select { case <-stopCh: - glog.Info(msg) + klog.Info(msg) default: - glog.Fatalf("%s due to error: %v", msg, err) + klog.Fatalf("%s due to error: %v", msg, err) } }() diff --git a/pkg/controllers/cache/cache.go b/pkg/controllers/cache/cache.go index cb76ce88fc..c2d45eaa76 100644 --- a/pkg/controllers/cache/cache.go +++ b/pkg/controllers/cache/cache.go @@ -21,13 +21,12 @@ import ( "sync" "time" - "github.com/golang/glog" - "golang.org/x/time/rate" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/batch/v1alpha1" "volcano.sh/volcano/pkg/controllers/apis" @@ -295,7 +294,7 @@ func (jc *jobCache) processCleanupJob() bool { job, ok := obj.(*apis.JobInfo) if !ok { - glog.Errorf("failed to convert %v to *apis.JobInfo", obj) + klog.Errorf("failed to convert %v to *apis.JobInfo", obj) return true } @@ -306,7 +305,7 @@ func (jc *jobCache) processCleanupJob() bool { jc.deletedJobs.Forget(obj) key := keyFn(job.Namespace, job.Name) delete(jc.jobs, key) - glog.V(3).Infof("Job <%s> was deleted.", key) + klog.V(3).Infof("Job <%s> was deleted.", key) } else { // Retry jc.deleteJob(job) @@ -315,7 +314,7 @@ func (jc *jobCache) processCleanupJob() bool { } func (jc *jobCache) deleteJob(job *apis.JobInfo) { - glog.V(3).Infof("Try to delete Job <%v/%v>", + klog.V(3).Infof("Try to delete Job <%v/%v>", job.Namespace, job.Name) jc.deletedJobs.AddRateLimited(job) diff --git a/pkg/controllers/garbagecollector/garbagecollector.go b/pkg/controllers/garbagecollector/garbagecollector.go index 160bd9cc4a..5bdd8fc2ea 100644 --- a/pkg/controllers/garbagecollector/garbagecollector.go +++ b/pkg/controllers/garbagecollector/garbagecollector.go @@ -20,13 +20,12 @@ import ( "fmt" "time" - "github.com/golang/glog" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" "volcano.sh/volcano/pkg/apis/batch/v1alpha1" @@ -80,8 +79,8 @@ func NewGarbageCollector(vkClient vcclientset.Interface) *GarbageCollector { func (gb *GarbageCollector) Run(stopCh <-chan struct{}) { defer gb.queue.ShutDown() - glog.Infof("Starting garbage collector") - defer glog.Infof("Shutting down garbage collector") + klog.Infof("Starting garbage collector") + defer klog.Infof("Shutting down garbage collector") go gb.jobInformer.Informer().Run(stopCh) if !controller.WaitForCacheSync("garbage collector", stopCh, gb.jobSynced) { @@ -95,7 +94,7 @@ func (gb *GarbageCollector) Run(stopCh <-chan struct{}) { func (gb *GarbageCollector) addJob(obj interface{}) { job := obj.(*v1alpha1.Job) - glog.V(4).Infof("Adding job %s/%s", job.Namespace, job.Name) + klog.V(4).Infof("Adding job %s/%s", job.Namespace, job.Name) if job.DeletionTimestamp == nil && needsCleanup(job) { gb.enqueue(job) @@ -104,7 +103,7 @@ func (gb *GarbageCollector) addJob(obj interface{}) { func (gb *GarbageCollector) updateJob(old, cur interface{}) { job := cur.(*v1alpha1.Job) - glog.V(4).Infof("Updating job %s/%s", job.Namespace, job.Name) + klog.V(4).Infof("Updating job %s/%s", job.Namespace, job.Name) if job.DeletionTimestamp == nil && needsCleanup(job) { gb.enqueue(job) @@ -112,10 +111,10 @@ func (gb *GarbageCollector) updateJob(old, cur interface{}) { } func (gb *GarbageCollector) enqueue(job *v1alpha1.Job) { - glog.V(4).Infof("Add job %s/%s to cleanup", job.Namespace, job.Name) + klog.V(4).Infof("Add job %s/%s to cleanup", job.Namespace, job.Name) key, err := controller.KeyFunc(job) if err != nil { - glog.Errorf("couldn't get key for object %#v: %v", job, err) + klog.Errorf("couldn't get key for object %#v: %v", job, err) return } @@ -125,7 +124,7 @@ func (gb *GarbageCollector) enqueue(job *v1alpha1.Job) { func (gb *GarbageCollector) enqueueAfter(job *v1alpha1.Job, after time.Duration) { key, err := controller.KeyFunc(job) if err != nil { - glog.Errorf("couldn't get key for object %#v: %v", job, err) + klog.Errorf("couldn't get key for object %#v: %v", job, err) return } @@ -156,7 +155,7 @@ func (gb *GarbageCollector) handleErr(err error, key interface{}) { return } - glog.Errorf("error cleaning up Job %v, will retry: %v", key, err) + klog.Errorf("error cleaning up Job %v, will retry: %v", key, err) gb.queue.AddRateLimited(key) } @@ -171,7 +170,7 @@ func (gb *GarbageCollector) processJob(key string) error { return err } - glog.V(4).Infof("Checking if Job %s/%s is ready for cleanup", namespace, name) + klog.V(4).Infof("Checking if Job %s/%s is ready for cleanup", namespace, name) // Ignore the Jobs that are already deleted or being deleted, or the ones that don't need clean up. job, err := gb.jobLister.Jobs(namespace).Get(name) if errors.IsNotFound(err) { @@ -210,7 +209,7 @@ func (gb *GarbageCollector) processJob(key string) error { PropagationPolicy: &policy, Preconditions: &metav1.Preconditions{UID: &fresh.UID}, } - glog.V(4).Infof("Cleaning up Job %s/%s", namespace, name) + klog.V(4).Infof("Cleaning up Job %s/%s", namespace, name) return gb.vcClient.BatchV1alpha1().Jobs(fresh.Namespace).Delete(fresh.Name, options) } @@ -267,10 +266,10 @@ func timeLeft(j *v1alpha1.Job, since *time.Time) (*time.Duration, error) { return nil, err } if finishAt.UTC().After(since.UTC()) { - glog.Warningf("Warning: Found Job %s/%s finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.", j.Namespace, j.Name) + klog.Warningf("Warning: Found Job %s/%s finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.", j.Namespace, j.Name) } remaining := expireAt.UTC().Sub(since.UTC()) - glog.V(4).Infof("Found Job %s/%s finished at %v, remaining TTL %v since %v, TTL will expire at %v", j.Namespace, j.Name, finishAt.UTC(), remaining, since.UTC(), expireAt.UTC()) + klog.V(4).Infof("Found Job %s/%s finished at %v, remaining TTL %v since %v, TTL will expire at %v", j.Namespace, j.Name, finishAt.UTC(), remaining, since.UTC(), expireAt.UTC()) return &remaining, nil } diff --git a/pkg/controllers/job/job_controller.go b/pkg/controllers/job/job_controller.go index 058512136e..2c3cdec1a3 100644 --- a/pkg/controllers/job/job_controller.go +++ b/pkg/controllers/job/job_controller.go @@ -23,8 +23,6 @@ import ( "sync" "time" - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/api/scheduling/v1beta1" "k8s.io/apimachinery/pkg/util/wait" @@ -38,6 +36,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" batchv1alpha1 "volcano.sh/volcano/pkg/apis/batch/v1alpha1" busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1" @@ -125,7 +124,7 @@ func NewJobController( //Initialize event client eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(vcscheme.Scheme, v1.EventSource{Component: "vc-controllers"}) @@ -253,11 +252,11 @@ func (cc *Controller) Run(stopCh <-chan struct{}) { // Re-sync error tasks. go wait.Until(cc.processResyncTask, 0, stopCh) - glog.Infof("JobController is running ...... ") + klog.Infof("JobController is running ...... ") } func (cc *Controller) worker(i uint32) { - glog.Infof("worker %d start ...... ", i) + klog.Infof("worker %d start ...... ", i) for cc.processNextReq(i) { } @@ -297,7 +296,7 @@ func (cc *Controller) processNextReq(count uint32) bool { queue := cc.queueList[count] obj, shutdown := queue.Get() if shutdown { - glog.Errorf("Fail to pop item from queue") + klog.Errorf("Fail to pop item from queue") return false } @@ -306,30 +305,30 @@ func (cc *Controller) processNextReq(count uint32) bool { key := jobcache.JobKeyByReq(&req) if !cc.belongsToThisRoutine(key, count) { - glog.Errorf("should not occur The job does not belongs to this routine key:%s, worker:%d...... ", key, count) + klog.Errorf("should not occur The job does not belongs to this routine key:%s, worker:%d...... ", key, count) queueLocal := cc.getWorkerQueue(key) queueLocal.Add(req) return true } - glog.V(3).Infof("Try to handle request <%v>", req) + klog.V(3).Infof("Try to handle request <%v>", req) jobInfo, err := cc.cache.Get(jobcache.JobKeyByReq(&req)) if err != nil { // TODO(k82cn): ignore not-ready error. - glog.Errorf("Failed to get job by <%v> from cache: %v", req, err) + klog.Errorf("Failed to get job by <%v> from cache: %v", req, err) return true } st := state.NewState(jobInfo) if st == nil { - glog.Errorf("Invalid state <%s> of Job <%v/%v>", + klog.Errorf("Invalid state <%s> of Job <%v/%v>", jobInfo.Job.Status.State, jobInfo.Job.Namespace, jobInfo.Job.Name) return true } action := applyPolicies(jobInfo.Job, &req) - glog.V(3).Infof("Execute <%v> on Job <%s/%s> in <%s> by <%T>.", + klog.V(3).Infof("Execute <%v> on Job <%s/%s> in <%s> by <%T>.", action, req.Namespace, req.JobName, jobInfo.Job.Status.State.Phase, st) if action != batchv1alpha1.SyncJobAction { @@ -339,7 +338,7 @@ func (cc *Controller) processNextReq(count uint32) bool { if err := st.Execute(action); err != nil { if queue.NumRequeues(req) < maxRetries { - glog.V(2).Infof("Failed to handle Job <%s/%s>: %v", + klog.V(2).Infof("Failed to handle Job <%s/%s>: %v", jobInfo.Job.Namespace, jobInfo.Job.Name, err) // If any error, requeue it. queue.AddRateLimited(req) @@ -347,7 +346,7 @@ func (cc *Controller) processNextReq(count uint32) bool { } cc.recordJobEvent(jobInfo.Job.Namespace, jobInfo.Job.Name, batchv1alpha1.ExecuteAction, fmt.Sprintf( "Job failed on action %s for retry limit reached", action)) - glog.Warningf("Dropping job<%s/%s> out of the queue: %v because max retries has reached", jobInfo.Job.Namespace, jobInfo.Job.Name, err) + klog.Warningf("Dropping job<%s/%s> out of the queue: %v because max retries has reached", jobInfo.Job.Namespace, jobInfo.Job.Name, err) } // If no error, forget it. diff --git a/pkg/controllers/job/job_controller_actions.go b/pkg/controllers/job/job_controller_actions.go index 13e389b506..cf95229018 100644 --- a/pkg/controllers/job/job_controller_actions.go +++ b/pkg/controllers/job/job_controller_actions.go @@ -22,11 +22,10 @@ import ( "sync" "sync/atomic" - "github.com/golang/glog" - "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" k8scontroller "k8s.io/kubernetes/pkg/controller" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" @@ -38,13 +37,13 @@ import ( ) func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseMap, updateStatus state.UpdateStatusFn) error { - glog.V(3).Infof("Killing Job <%s/%s>", jobInfo.Job.Namespace, jobInfo.Job.Name) - defer glog.V(3).Infof("Finished Job <%s/%s> killing", jobInfo.Job.Namespace, jobInfo.Job.Name) + klog.V(3).Infof("Killing Job <%s/%s>", jobInfo.Job.Namespace, jobInfo.Job.Name) + defer klog.V(3).Infof("Finished Job <%s/%s> killing", jobInfo.Job.Namespace, jobInfo.Job.Name) job := jobInfo.Job - glog.Infof("Current Version is: %d of job: %s/%s", job.Status.Version, job.Namespace, job.Name) + klog.Infof("Current Version is: %d of job: %s/%s", job.Status.Version, job.Namespace, job.Name) if job.DeletionTimestamp != nil { - glog.Infof("Job <%s/%s> is terminating, skip management process.", + klog.Infof("Job <%s/%s> is terminating, skip management process.", job.Namespace, job.Name) return nil } @@ -59,7 +58,7 @@ func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseM total++ if pod.DeletionTimestamp != nil { - glog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name) + klog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name) terminating++ continue } @@ -82,7 +81,7 @@ func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseM } if len(errs) != 0 { - glog.Errorf("failed to kill pods for job %s/%s, with err %+v", job.Namespace, job.Name, errs) + klog.Errorf("failed to kill pods for job %s/%s, with err %+v", job.Namespace, job.Name, errs) cc.recorder.Event(job, v1.EventTypeWarning, k8scontroller.FailedDeletePodReason, fmt.Sprintf("Error deleting pods: %+v", errs)) return fmt.Errorf("failed to kill %d pods of %d", len(errs), total) @@ -115,12 +114,12 @@ func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseM // Update Job status newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(job) if err != nil { - glog.Errorf("Failed to update status of Job %v/%v: %v", + klog.Errorf("Failed to update status of Job %v/%v: %v", job.Namespace, job.Name, err) return err } if e := cc.cache.Update(newJob); e != nil { - glog.Errorf("KillJob - Failed to update Job %v/%v in cache: %v", + klog.Errorf("KillJob - Failed to update Job %v/%v in cache: %v", newJob.Namespace, newJob.Name, e) return e } @@ -128,7 +127,7 @@ func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseM // Delete PodGroup if err := cc.vcClient.SchedulingV1alpha2().PodGroups(job.Namespace).Delete(job.Name, nil); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete PodGroup of Job %v/%v: %v", + klog.Errorf("Failed to delete PodGroup of Job %v/%v: %v", job.Namespace, job.Name, err) return err } @@ -174,14 +173,14 @@ func (cc *Controller) createJob(job *batch.Job) (*batch.Job, error) { } func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateStatusFn) error { - glog.V(3).Infof("Starting to sync up Job <%s/%s>", jobInfo.Job.Namespace, jobInfo.Job.Name) - defer glog.V(3).Infof("Finished Job <%s/%s> sync up", jobInfo.Job.Namespace, jobInfo.Job.Name) + klog.V(3).Infof("Starting to sync up Job <%s/%s>", jobInfo.Job.Namespace, jobInfo.Job.Name) + defer klog.V(3).Infof("Finished Job <%s/%s> sync up", jobInfo.Job.Namespace, jobInfo.Job.Name) job := jobInfo.Job.DeepCopy() - glog.Infof("Current Version is: %d of job: %s/%s", job.Status.Version, job.Namespace, job.Name) + klog.Infof("Current Version is: %d of job: %s/%s", job.Status.Version, job.Namespace, job.Name) if job.DeletionTimestamp != nil { - glog.Infof("Job <%s/%s> is terminating, skip management process.", + klog.Infof("Job <%s/%s> is terminating, skip management process.", job.Namespace, job.Name) return nil } @@ -226,7 +225,7 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt } else { delete(pods, podName) if pod.DeletionTimestamp != nil { - glog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name) + klog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name) atomic.AddInt32(&terminating, 1) continue } @@ -250,12 +249,12 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt // Failed to create Pod, waitCreationGroup a moment and then create it again // This is to ensure all podsMap under the same Job created // So gang-scheduling could schedule the Job successfully - glog.Errorf("Failed to create pod %s for Job %s, err %#v", + klog.Errorf("Failed to create pod %s for Job %s, err %#v", pod.Name, job.Name, err) appendError(&creationErrs, fmt.Errorf("failed to create pod %s, err: %#v", pod.Name, err)) } else { classifyAndAddUpPodBaseOnPhase(newPod, &pending, &running, &succeeded, &failed, &unknown) - glog.V(3).Infof("Created Task <%s> of Job <%s/%s>", + klog.V(3).Infof("Created Task <%s> of Job <%s/%s>", pod.Name, job.Namespace, job.Name) } }(pod) @@ -280,12 +279,12 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt // Failed to delete Pod, waitCreationGroup a moment and then create it again // This is to ensure all podsMap under the same Job created // So gang-scheduling could schedule the Job successfully - glog.Errorf("Failed to delete pod %s for Job %s, err %#v", + klog.Errorf("Failed to delete pod %s for Job %s, err %#v", pod.Name, job.Name, err) appendError(&deletionErrs, err) cc.resyncTask(pod) } else { - glog.V(3).Infof("Deleted Task <%s> of Job <%s/%s>", + klog.V(3).Infof("Deleted Task <%s> of Job <%s/%s>", pod.Name, job.Namespace, job.Name) atomic.AddInt32(&terminating, 1) } @@ -321,12 +320,12 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt } newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(job) if err != nil { - glog.Errorf("Failed to update status of Job %v/%v: %v", + klog.Errorf("Failed to update status of Job %v/%v: %v", job.Namespace, job.Name, err) return err } if e := cc.cache.Update(newJob); e != nil { - glog.Errorf("SyncJob - Failed to update Job %v/%v in cache: %v", + klog.Errorf("SyncJob - Failed to update Job %v/%v in cache: %v", newJob.Namespace, newJob.Name, e) return e } @@ -376,7 +375,7 @@ func (cc *Controller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) if needUpdate { newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(job) if err != nil { - glog.Errorf("Failed to update Job %v/%v for volume claim name: %v ", + klog.Errorf("Failed to update Job %v/%v for volume claim name: %v ", job.Namespace, job.Name, err) return job, err } @@ -392,7 +391,7 @@ func (cc *Controller) checkPVCExist(job *batch.Job, pvc string) (bool, error) { if apierrors.IsNotFound(err) { return false, nil } - glog.V(3).Infof("Failed to get PVC %s for job <%s/%s>: %v", + klog.V(3).Infof("Failed to get PVC %s for job <%s/%s>: %v", pvc, job.Namespace, job.Name, err) return false, err } @@ -411,10 +410,10 @@ func (cc *Controller) createPVC(job *batch.Job, vcName string, volumeClaim *v1.P Spec: *volumeClaim, } - glog.V(3).Infof("Try to create PVC: %v", pvc) + klog.V(3).Infof("Try to create PVC: %v", pvc) if _, e := cc.kubeClient.CoreV1().PersistentVolumeClaims(job.Namespace).Create(pvc); e != nil { - glog.V(3).Infof("Failed to create PVC for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to create PVC for Job <%s/%s>: %v", job.Namespace, job.Name, e) return e } @@ -425,7 +424,7 @@ func (cc *Controller) createPodGroupIfNotExist(job *batch.Job) error { // If PodGroup does not exist, create one for Job. if _, err := cc.pgLister.PodGroups(job.Namespace).Get(job.Name); err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get PodGroup for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get PodGroup for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -448,7 +447,7 @@ func (cc *Controller) createPodGroupIfNotExist(job *batch.Job) error { if _, err = cc.vcClient.SchedulingV1alpha2().PodGroups(job.Namespace).Create(pg); err != nil { if !apierrors.IsAlreadyExists(err) { - glog.V(3).Infof("Failed to create PodGroup for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to create PodGroup for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -461,7 +460,7 @@ func (cc *Controller) createPodGroupIfNotExist(job *batch.Job) error { func (cc *Controller) deleteJobPod(jobName string, pod *v1.Pod) error { err := cc.kubeClient.CoreV1().Pods(pod.Namespace).Delete(pod.Name, nil) if err != nil && !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete pod %s/%s for Job %s, err %#v", + klog.Errorf("Failed to delete pod %s/%s for Job %s, err %#v", pod.Namespace, pod.Name, jobName, err) return fmt.Errorf("failed to delete pod %s, err %#v", pod.Name, err) @@ -513,12 +512,12 @@ func (cc *Controller) initJobStatus(job *batch.Job) (*batch.Job, error) { job.Status.MinAvailable = int32(job.Spec.MinAvailable) newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(job) if err != nil { - glog.Errorf("Failed to update status of Job %v/%v: %v", + klog.Errorf("Failed to update status of Job %v/%v: %v", job.Namespace, job.Name, err) return nil, err } if err := cc.cache.Update(newJob); err != nil { - glog.Errorf("CreateJob - Failed to update Job %v/%v in cache: %v", + klog.Errorf("CreateJob - Failed to update Job %v/%v in cache: %v", newJob.Namespace, newJob.Name, err) return nil, err } diff --git a/pkg/controllers/job/job_controller_handler.go b/pkg/controllers/job/job_controller_handler.go index 749620113d..335f842a08 100644 --- a/pkg/controllers/job/job_controller_handler.go +++ b/pkg/controllers/job/job_controller_handler.go @@ -21,12 +21,11 @@ import ( "reflect" "strconv" - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/api/scheduling/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/cache" + "k8s.io/klog" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" bus "volcano.sh/volcano/pkg/apis/bus/v1alpha1" @@ -40,7 +39,7 @@ import ( func (cc *Controller) addCommand(obj interface{}) { cmd, ok := obj.(*bus.Command) if !ok { - glog.Errorf("obj is not Command") + klog.Errorf("obj is not Command") return } @@ -50,7 +49,7 @@ func (cc *Controller) addCommand(obj interface{}) { func (cc *Controller) addJob(obj interface{}) { job, ok := obj.(*batch.Job) if !ok { - glog.Errorf("obj is not Job") + klog.Errorf("obj is not Job") return } @@ -63,7 +62,7 @@ func (cc *Controller) addJob(obj interface{}) { // TODO(k82cn): if failed to add job, the cache should be refresh if err := cc.cache.Add(job); err != nil { - glog.Errorf("Failed to add job <%s/%s>: %v in cache", + klog.Errorf("Failed to add job <%s/%s>: %v in cache", job.Namespace, job.Name, err) } key := jobhelpers.GetJobKeyByReq(&req) @@ -74,25 +73,25 @@ func (cc *Controller) addJob(obj interface{}) { func (cc *Controller) updateJob(oldObj, newObj interface{}) { newJob, ok := newObj.(*batch.Job) if !ok { - glog.Errorf("newObj is not Job") + klog.Errorf("newObj is not Job") return } oldJob, ok := oldObj.(*batch.Job) if !ok { - glog.Errorf("oldJob is not Job") + klog.Errorf("oldJob is not Job") return } // NOTE: Since we only reconcile job based on Spec, we will ignore other attributes // For Job status, it's used internally and always been updated via our controller. if reflect.DeepEqual(newJob.Spec, oldJob.Spec) && newJob.Status.State.Phase == oldJob.Status.State.Phase { - glog.Infof("Job update event is ignored since no update in 'Spec'.") + klog.Infof("Job update event is ignored since no update in 'Spec'.") return } if err := cc.cache.Update(newJob); err != nil { - glog.Errorf("UpdateJob - Failed to update job <%s/%s>: %v in cache", + klog.Errorf("UpdateJob - Failed to update job <%s/%s>: %v in cache", newJob.Namespace, newJob.Name, err) } @@ -114,18 +113,18 @@ func (cc *Controller) deleteJob(obj interface{}) { // If we reached here it means the Job was deleted but its final state is unrecorded. tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } job, ok = tombstone.Obj.(*batch.Job) if !ok { - glog.Errorf("Tombstone contained object that is not a volcano Job: %#v", obj) + klog.Errorf("Tombstone contained object that is not a volcano Job: %#v", obj) return } } if err := cc.cache.Delete(job); err != nil { - glog.Errorf("Failed to delete job <%s/%s>: %v in cache", + klog.Errorf("Failed to delete job <%s/%s>: %v in cache", job.Namespace, job.Name, err) } } @@ -133,7 +132,7 @@ func (cc *Controller) deleteJob(obj interface{}) { func (cc *Controller) addPod(obj interface{}) { pod, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("Failed to convert %v to v1.Pod", obj) + klog.Errorf("Failed to convert %v to v1.Pod", obj) return } // Filter out pods that are not created from volcano job @@ -143,21 +142,21 @@ func (cc *Controller) addPod(obj interface{}) { jobName, found := pod.Annotations[batch.JobNameKey] if !found { - glog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } version, found := pod.Annotations[batch.JobVersion] if !found { - glog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } dVersion, err := strconv.Atoi(version) if err != nil { - glog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping", + klog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping", pod.Namespace, pod.Name) return } @@ -176,7 +175,7 @@ func (cc *Controller) addPod(obj interface{}) { } if err := cc.cache.AddPod(pod); err != nil { - glog.Errorf("Failed to add Pod <%s/%s>: %v to cache", + klog.Errorf("Failed to add Pod <%s/%s>: %v to cache", pod.Namespace, pod.Name, err) } key := jobhelpers.GetJobKeyByReq(&req) @@ -187,13 +186,13 @@ func (cc *Controller) addPod(obj interface{}) { func (cc *Controller) updatePod(oldObj, newObj interface{}) { oldPod, ok := oldObj.(*v1.Pod) if !ok { - glog.Errorf("Failed to convert %v to v1.Pod", oldObj) + klog.Errorf("Failed to convert %v to v1.Pod", oldObj) return } newPod, ok := newObj.(*v1.Pod) if !ok { - glog.Errorf("Failed to convert %v to v1.Pod", newObj) + klog.Errorf("Failed to convert %v to v1.Pod", newObj) return } @@ -213,34 +212,34 @@ func (cc *Controller) updatePod(oldObj, newObj interface{}) { taskName, found := newPod.Annotations[batch.TaskSpecKey] if !found { - glog.Infof("Failed to find taskName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find taskName of Pod <%s/%s>, skipping", newPod.Namespace, newPod.Name) return } jobName, found := newPod.Annotations[batch.JobNameKey] if !found { - glog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", newPod.Namespace, newPod.Name) return } version, found := newPod.Annotations[batch.JobVersion] if !found { - glog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", newPod.Namespace, newPod.Name) return } dVersion, err := strconv.Atoi(version) if err != nil { - glog.Infof("Failed to convert jobVersion of Pod into number <%s/%s>, skipping", + klog.Infof("Failed to convert jobVersion of Pod into number <%s/%s>, skipping", newPod.Namespace, newPod.Name) return } if err := cc.cache.UpdatePod(newPod); err != nil { - glog.Errorf("Failed to update Pod <%s/%s>: %v in cache", + klog.Errorf("Failed to update Pod <%s/%s>: %v in cache", newPod.Namespace, newPod.Name, err) } @@ -284,12 +283,12 @@ func (cc *Controller) deletePod(obj interface{}) { // If we reached here it means the pod was deleted but its final state is unrecorded. tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } pod, ok = tombstone.Obj.(*v1.Pod) if !ok { - glog.Errorf("Tombstone contained object that is not a Pod: %#v", obj) + klog.Errorf("Tombstone contained object that is not a Pod: %#v", obj) return } } @@ -301,28 +300,28 @@ func (cc *Controller) deletePod(obj interface{}) { taskName, found := pod.Annotations[batch.TaskSpecKey] if !found { - glog.Infof("Failed to find taskName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find taskName of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } jobName, found := pod.Annotations[batch.JobNameKey] if !found { - glog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } version, found := pod.Annotations[batch.JobVersion] if !found { - glog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } dVersion, err := strconv.Atoi(version) if err != nil { - glog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping", + klog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping", pod.Namespace, pod.Name) return } @@ -337,7 +336,7 @@ func (cc *Controller) deletePod(obj interface{}) { } if err := cc.cache.DeletePod(pod); err != nil { - glog.Errorf("Failed to delete Pod <%s/%s>: %v in cache", + klog.Errorf("Failed to delete Pod <%s/%s>: %v in cache", pod.Namespace, pod.Name, err) } @@ -349,7 +348,7 @@ func (cc *Controller) deletePod(obj interface{}) { func (cc *Controller) recordJobEvent(namespace, name string, event batch.JobEvent, message string) { job, err := cc.cache.Get(jobcache.JobKeyByName(namespace, name)) if err != nil { - glog.Warningf("Failed to find job in cache when reporting job event <%s/%s>: %v", + klog.Warningf("Failed to find job in cache when reporting job event <%s/%s>: %v", namespace, name, err) return } @@ -372,7 +371,7 @@ func (cc *Controller) processNextCommand() bool { if err := cc.vcClient.BusV1alpha1().Commands(cmd.Namespace).Delete(cmd.Name, nil); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete Command <%s/%s>.", cmd.Namespace, cmd.Name) + klog.Errorf("Failed to delete Command <%s/%s>.", cmd.Namespace, cmd.Name) cc.commandQueue.AddRateLimited(cmd) } return true @@ -398,19 +397,19 @@ func (cc *Controller) processNextCommand() bool { func (cc *Controller) updatePodGroup(oldObj, newObj interface{}) { oldPG, ok := oldObj.(*scheduling.PodGroup) if !ok { - glog.Errorf("Failed to convert %v to PodGroup", newObj) + klog.Errorf("Failed to convert %v to PodGroup", newObj) return } newPG, ok := newObj.(*scheduling.PodGroup) if !ok { - glog.Errorf("Failed to convert %v to PodGroup", newObj) + klog.Errorf("Failed to convert %v to PodGroup", newObj) return } _, err := cc.cache.Get(jobcache.JobKeyByName(newPG.Namespace, newPG.Name)) if err != nil && newPG.Annotations != nil { - glog.Warningf( + klog.Warningf( "Failed to find job in cache by PodGroup, this may not be a PodGroup for volcano job.") } @@ -466,11 +465,11 @@ func convert2PriorityClass(obj interface{}) *v1beta1.PriorityClass { var ok bool pc, ok = t.Obj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) return nil } default: - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) return nil } diff --git a/pkg/controllers/job/job_controller_plugins.go b/pkg/controllers/job/job_controller_plugins.go index 861a1e829b..b410fc9d08 100644 --- a/pkg/controllers/job/job_controller_plugins.go +++ b/pkg/controllers/job/job_controller_plugins.go @@ -19,9 +19,8 @@ package job import ( "fmt" - "github.com/golang/glog" - "k8s.io/api/core/v1" + "k8s.io/klog" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" "volcano.sh/volcano/pkg/controllers/job/plugins" @@ -34,12 +33,12 @@ func (cc *Controller) pluginOnPodCreate(job *batch.Job, pod *v1.Pod) error { pb, found := plugins.GetPluginBuilder(name) if !found { err := fmt.Errorf("failed to get plugin %s", name) - glog.Error(err) + klog.Error(err) return err } - glog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) + klog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) if err := pb(client, args).OnPodCreate(pod, job); err != nil { - glog.Errorf("Failed to process on pod create plugin %s, err %v.", name, err) + klog.Errorf("Failed to process on pod create plugin %s, err %v.", name, err) return err } @@ -56,12 +55,12 @@ func (cc *Controller) pluginOnJobAdd(job *batch.Job) error { pb, found := plugins.GetPluginBuilder(name) if !found { err := fmt.Errorf("failed to get plugin %s", name) - glog.Error(err) + klog.Error(err) return err } - glog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) + klog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) if err := pb(client, args).OnJobAdd(job); err != nil { - glog.Errorf("Failed to process on job add plugin %s, err %v.", name, err) + klog.Errorf("Failed to process on job add plugin %s, err %v.", name, err) return err } @@ -76,12 +75,12 @@ func (cc *Controller) pluginOnJobDelete(job *batch.Job) error { pb, found := plugins.GetPluginBuilder(name) if !found { err := fmt.Errorf("failed to get plugin %s", name) - glog.Error(err) + klog.Error(err) return err } - glog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) + klog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) if err := pb(client, args).OnJobDelete(job); err != nil { - glog.Errorf("failed to process on job delete plugin %s, err %v.", name, err) + klog.Errorf("failed to process on job delete plugin %s, err %v.", name, err) return err } diff --git a/pkg/controllers/job/job_controller_resync.go b/pkg/controllers/job/job_controller_resync.go index 888cf250a7..66e28d9ed5 100644 --- a/pkg/controllers/job/job_controller_resync.go +++ b/pkg/controllers/job/job_controller_resync.go @@ -20,13 +20,13 @@ import ( "fmt" "time" - "github.com/golang/glog" "golang.org/x/time/rate" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" ) func newRateLimitingQueue() workqueue.RateLimitingInterface { @@ -53,12 +53,12 @@ func (cc *Controller) processResyncTask() { task, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("failed to convert %v to *v1.Pod", obj) + klog.Errorf("failed to convert %v to *v1.Pod", obj) return } if err := cc.syncTask(task); err != nil { - glog.Errorf("Failed to sync pod <%v/%v>, retry it, err %v", task.Namespace, task.Name, err) + klog.Errorf("Failed to sync pod <%v/%v>, retry it, err %v", task.Namespace, task.Name, err) cc.resyncTask(task) } } @@ -71,10 +71,10 @@ func (cc *Controller) syncTask(oldTask *v1.Pod) error { if err != nil { if errors.IsNotFound(err) { if err := cc.cache.DeletePod(oldTask); err != nil { - glog.Errorf("failed to delete cache pod <%v/%v>, err %v.", oldTask.Namespace, oldTask.Name, err) + klog.Errorf("failed to delete cache pod <%v/%v>, err %v.", oldTask.Namespace, oldTask.Name, err) return err } - glog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name) + klog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name) return nil } diff --git a/pkg/controllers/job/job_controller_util.go b/pkg/controllers/job/job_controller_util.go index 0b5d8deca2..39d8c06f00 100644 --- a/pkg/controllers/job/job_controller_util.go +++ b/pkg/controllers/job/job_controller_util.go @@ -19,11 +19,10 @@ package job import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" "volcano.sh/volcano/pkg/apis/helpers" @@ -132,7 +131,7 @@ func applyPolicies(job *batch.Job, req *apis.Request) batch.Action { // For all the requests triggered from discarded job resources will perform sync action instead if req.JobVersion < job.Status.Version { - glog.Infof("Request %s is outdated, will perform sync instead.", req) + klog.Infof("Request %s is outdated, will perform sync instead.", req) return batch.SyncJobAction } diff --git a/pkg/controllers/job/plugins/ssh/ssh.go b/pkg/controllers/job/plugins/ssh/ssh.go index 1bad52b97e..bef4c57fcf 100644 --- a/pkg/controllers/job/plugins/ssh/ssh.go +++ b/pkg/controllers/job/plugins/ssh/ssh.go @@ -23,10 +23,11 @@ import ( "encoding/pem" "flag" "fmt" - "github.com/golang/glog" + "golang.org/x/crypto/ssh" "k8s.io/api/core/v1" + "k8s.io/klog" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" "volcano.sh/volcano/pkg/apis/helpers" @@ -157,7 +158,7 @@ func generateRsaKey(job *batch.Job) (map[string]string, error) { privateKey, err := rsa.GenerateKey(rand.Reader, bitSize) if err != nil { - glog.Errorf("rsa generateKey err: %v", err) + klog.Errorf("rsa generateKey err: %v", err) return nil, err } @@ -171,7 +172,7 @@ func generateRsaKey(job *batch.Job) (map[string]string, error) { // id_rsa.pub publicRsaKey, err := ssh.NewPublicKey(&privateKey.PublicKey) if err != nil { - glog.Errorf("ssh newPublicKey err: %v", err) + klog.Errorf("ssh newPublicKey err: %v", err) return nil, err } publicKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey) @@ -196,7 +197,7 @@ func (sp *sshPlugin) addFlags() { "ssh private and public keys, it is `/root/.ssh` by default.") if err := flagSet.Parse(sp.pluginArguments); err != nil { - glog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) + klog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) } return } diff --git a/pkg/controllers/job/plugins/svc/svc.go b/pkg/controllers/job/plugins/svc/svc.go index b5a9ea1801..52055ef3b0 100644 --- a/pkg/controllers/job/plugins/svc/svc.go +++ b/pkg/controllers/job/plugins/svc/svc.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -63,7 +63,7 @@ func (sp *servicePlugin) addFlags() { "set publishNotReadyAddresses of svc to true") if err := flagSet.Parse(sp.pluginArguments); err != nil { - glog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) + klog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) } return } @@ -119,7 +119,7 @@ func (sp *servicePlugin) OnJobDelete(job *batch.Job) error { if err := sp.Clientset.KubeClients.CoreV1().Services(job.Namespace).Delete(job.Name, nil); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete Service of Job %v/%v: %v", job.Namespace, job.Name, err) + klog.Errorf("Failed to delete Service of Job %v/%v: %v", job.Namespace, job.Name, err) return err } } @@ -156,7 +156,7 @@ func (sp *servicePlugin) createServiceIfNotExist(job *batch.Job) error { // If Service does not exist, create one for Job. if _, err := sp.Clientset.KubeClients.CoreV1().Services(job.Namespace).Get(job.Name, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get Service for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get Service for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -188,7 +188,7 @@ func (sp *servicePlugin) createServiceIfNotExist(job *batch.Job) error { } if _, e := sp.Clientset.KubeClients.CoreV1().Services(job.Namespace).Create(svc); e != nil { - glog.V(3).Infof("Failed to create Service for Job <%s/%s>: %v", job.Namespace, job.Name, e) + klog.V(3).Infof("Failed to create Service for Job <%s/%s>: %v", job.Namespace, job.Name, e) return e } job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name() diff --git a/pkg/controllers/podgroup/pg_controller.go b/pkg/controllers/podgroup/pg_controller.go index 82159b5c4e..dc5a031031 100644 --- a/pkg/controllers/podgroup/pg_controller.go +++ b/pkg/controllers/podgroup/pg_controller.go @@ -17,8 +17,6 @@ limitations under the License. package podgroup import ( - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" @@ -27,6 +25,7 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" scheduling "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" vcclientset "volcano.sh/volcano/pkg/client/clientset/versioned" @@ -107,7 +106,7 @@ func (cc *Controller) Run(stopCh <-chan struct{}) { go wait.Until(cc.worker, 0, stopCh) - glog.Infof("PodgroupController is running ...... ") + klog.Infof("PodgroupController is running ...... ") } func (cc *Controller) worker() { @@ -118,7 +117,7 @@ func (cc *Controller) worker() { func (cc *Controller) processNextReq() bool { obj, shutdown := cc.queue.Get() if shutdown { - glog.Errorf("Fail to pop item from queue") + klog.Errorf("Fail to pop item from queue") return false } @@ -127,13 +126,13 @@ func (cc *Controller) processNextReq() bool { pod, err := cc.podLister.Pods(req.podNamespace).Get(req.podName) if err != nil { - glog.Errorf("Failed to get pod by <%v> from cache: %v", req, err) + klog.Errorf("Failed to get pod by <%v> from cache: %v", req, err) return true } // normal pod use volcano if err := cc.createNormalPodPGIfNotExist(pod); err != nil { - glog.Errorf("Failed to handle Pod <%s/%s>: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Failed to handle Pod <%s/%s>: %v", pod.Namespace, pod.Name, err) cc.queue.AddRateLimited(req) return true } diff --git a/pkg/controllers/podgroup/pg_controller_handler.go b/pkg/controllers/podgroup/pg_controller_handler.go index 799e9719f2..781559f4b3 100644 --- a/pkg/controllers/podgroup/pg_controller_handler.go +++ b/pkg/controllers/podgroup/pg_controller_handler.go @@ -17,11 +17,10 @@ limitations under the License. package podgroup import ( - "github.com/golang/glog" - "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/helpers" scheduling "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" @@ -35,7 +34,7 @@ type podRequest struct { func (cc *Controller) addPod(obj interface{}) { pod, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("Failed to convert %v to v1.Pod", obj) + klog.Errorf("Failed to convert %v to v1.Pod", obj) return } @@ -55,14 +54,14 @@ func (cc *Controller) updatePodAnnotations(pod *v1.Pod, pgName string) error { pod.Annotations[scheduling.GroupNameAnnotationKey] = pgName } else { if pod.Annotations[scheduling.GroupNameAnnotationKey] != pgName { - glog.Errorf("normal pod %s/%s annotations %s value is not %s, but %s", pod.Namespace, pod.Name, + klog.Errorf("normal pod %s/%s annotations %s value is not %s, but %s", pod.Namespace, pod.Name, scheduling.GroupNameAnnotationKey, pgName, pod.Annotations[scheduling.GroupNameAnnotationKey]) } return nil } if _, err := cc.kubeClient.CoreV1().Pods(pod.Namespace).Update(pod); err != nil { - glog.Errorf("Failed to update pod <%s/%s>: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Failed to update pod <%s/%s>: %v", pod.Namespace, pod.Name, err) return err } @@ -74,7 +73,7 @@ func (cc *Controller) createNormalPodPGIfNotExist(pod *v1.Pod) error { if _, err := cc.pgLister.PodGroups(pod.Namespace).Get(pgName); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to get normal PodGroup for Pod <%s/%s>: %v", + klog.Errorf("Failed to get normal PodGroup for Pod <%s/%s>: %v", pod.Namespace, pod.Name, err) return err } @@ -92,7 +91,7 @@ func (cc *Controller) createNormalPodPGIfNotExist(pod *v1.Pod) error { } if _, err := cc.vcClient.SchedulingV1alpha2().PodGroups(pod.Namespace).Create(pg); err != nil { - glog.Errorf("Failed to create normal PodGroup for Pod <%s/%s>: %v", + klog.Errorf("Failed to create normal PodGroup for Pod <%s/%s>: %v", pod.Namespace, pod.Name, err) return err } diff --git a/pkg/controllers/queue/queue_controller.go b/pkg/controllers/queue/queue_controller.go index e0b9007706..5c8dadbb49 100644 --- a/pkg/controllers/queue/queue_controller.go +++ b/pkg/controllers/queue/queue_controller.go @@ -21,12 +21,12 @@ import ( "reflect" "sync" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" vcclientset "volcano.sh/volcano/pkg/client/clientset/versioned" @@ -106,12 +106,12 @@ func (c *Controller) Run(stopCh <-chan struct{}) { go c.pgInformer.Informer().Run(stopCh) if !cache.WaitForCacheSync(stopCh, c.queueSynced, c.pgSynced) { - glog.Errorf("unable to sync caches for queue controller") + klog.Errorf("unable to sync caches for queue controller") return } go wait.Until(c.worker, 0, stopCh) - glog.Infof("QueueController is running ...... ") + klog.Infof("QueueController is running ...... ") } // worker runs a worker thread that just dequeues items, processes them, and @@ -131,7 +131,7 @@ func (c *Controller) processNextWorkItem() bool { defer c.queue.Done(eKey) if err := c.syncQueue(eKey.(string)); err != nil { - glog.V(2).Infof("Error syncing queues %q, retrying. Error: %v", eKey, err) + klog.V(2).Infof("Error syncing queues %q, retrying. Error: %v", eKey, err) c.queue.AddRateLimited(eKey) return true } @@ -156,7 +156,7 @@ func (c *Controller) getPodGroups(key string) ([]string, error) { } func (c *Controller) syncQueue(key string) error { - glog.V(4).Infof("Begin sync queue %s", key) + klog.V(4).Infof("Begin sync queue %s", key) podGroups, err := c.getPodGroups(key) if err != nil { @@ -190,7 +190,7 @@ func (c *Controller) syncQueue(key string) error { queue, err := c.queueLister.Get(key) if err != nil { if errors.IsNotFound(err) { - glog.V(2).Infof("queue %s has been deleted", key) + klog.V(2).Infof("queue %s has been deleted", key) return nil } // TODO: do not retry to syncQueue for this error @@ -206,7 +206,7 @@ func (c *Controller) syncQueue(key string) error { newQueue.Status = queueStatus if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { - glog.Errorf("Failed to update status of Queue %s: %v", newQueue.Name, err) + klog.Errorf("Failed to update status of Queue %s: %v", newQueue.Name, err) return err } @@ -223,12 +223,12 @@ func (c *Controller) deleteQueue(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } queue, ok = tombstone.Obj.(*schedulingv1alpha2.Queue) if !ok { - glog.Errorf("Tombstone contained object that is not a Queue: %#v", obj) + klog.Errorf("Tombstone contained object that is not a Queue: %#v", obj) return } } @@ -270,12 +270,12 @@ func (c *Controller) deletePodGroup(obj interface{}) { if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } pg, ok = tombstone.Obj.(*schedulingv1alpha2.PodGroup) if !ok { - glog.Errorf("Tombstone contained object that is not a PodGroup: %#v", obj) + klog.Errorf("Tombstone contained object that is not a PodGroup: %#v", obj) return } } diff --git a/pkg/scheduler/actions/allocate/allocate.go b/pkg/scheduler/actions/allocate/allocate.go index 80a1ee13e2..5277f67b62 100644 --- a/pkg/scheduler/actions/allocate/allocate.go +++ b/pkg/scheduler/actions/allocate/allocate.go @@ -17,7 +17,7 @@ limitations under the License. package allocate import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -40,8 +40,8 @@ func (alloc *allocateAction) Name() string { func (alloc *allocateAction) Initialize() {} func (alloc *allocateAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Allocate ...") - defer glog.V(3).Infof("Leaving Allocate ...") + klog.V(3).Infof("Enter Allocate ...") + defer klog.V(3).Infof("Leaving Allocate ...") // the allocation for pod may have many stages // 1. pick a namespace named N (using ssn.NamespaceOrderFn) @@ -62,12 +62,12 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { continue } if vr := ssn.JobValid(job); vr != nil && !vr.Pass { - glog.V(4).Infof("Job <%s/%s> Queue <%s> skip allocate, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) + klog.V(4).Infof("Job <%s/%s> Queue <%s> skip allocate, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) continue } if _, found := ssn.Queues[job.Queue]; !found { - glog.Warningf("Skip adding Job <%s/%s> because its queue %s is not found", + klog.Warningf("Skip adding Job <%s/%s> because its queue %s is not found", job.Namespace, job.Name, job.Queue) continue } @@ -87,11 +87,11 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { queueMap[job.Queue] = jobs } - glog.V(4).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue) + klog.V(4).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue) jobs.Push(job) } - glog.V(3).Infof("Try to allocate resource to %d Namespaces", len(jobsMap)) + klog.V(3).Infof("Try to allocate resource to %d Namespaces", len(jobsMap)) pendingTasks := map[api.JobID]*util.PriorityQueue{} @@ -135,7 +135,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { for queueId := range queueInNamespace { currentQueue := ssn.Queues[queueId] if ssn.Overused(currentQueue) { - glog.V(3).Infof("Namespace <%s> Queue <%s> is overused, ignore it.", namespace, currentQueue.Name) + klog.V(3).Infof("Namespace <%s> Queue <%s> is overused, ignore it.", namespace, currentQueue.Name) delete(queueInNamespace, queueId) continue } @@ -146,15 +146,15 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { } if queue == nil { - glog.V(3).Infof("Namespace <%s> have no queue, skip it", namespace) + klog.V(3).Infof("Namespace <%s> have no queue, skip it", namespace) continue } - glog.V(3).Infof("Try to allocate resource to Jobs in Namespace <%s> Queue <%v>", namespace, queue.Name) + klog.V(3).Infof("Try to allocate resource to Jobs in Namespace <%s> Queue <%v>", namespace, queue.Name) jobs, found := queueInNamespace[queue.UID] if !found || jobs.Empty() { - glog.V(4).Infof("Can not find jobs for queue %s.", queue.Name) + klog.V(4).Infof("Can not find jobs for queue %s.", queue.Name) continue } @@ -164,7 +164,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { for _, task := range job.TaskStatusIndex[api.Pending] { // Skip BestEffort task in 'allocate' action. if task.Resreq.IsEmpty() { - glog.V(4).Infof("Task <%v/%v> is BestEffort task, skip it.", + klog.V(4).Infof("Task <%v/%v> is BestEffort task, skip it.", task.Namespace, task.Name) continue } @@ -175,7 +175,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { } tasks := pendingTasks[job.UID] - glog.V(3).Infof("Try to allocate resource to %d tasks of Job <%v/%v>", + klog.V(3).Infof("Try to allocate resource to %d tasks of Job <%v/%v>", tasks.Len(), job.Namespace, job.Name) stmt := ssn.Statement() @@ -183,7 +183,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { for !tasks.Empty() { task := tasks.Pop().(*api.TaskInfo) - glog.V(3).Infof("There are <%d> nodes for Job <%v/%v>", + klog.V(3).Infof("There are <%d> nodes for Job <%v/%v>", len(ssn.Nodes), job.Namespace, job.Name) //any task that doesn't fit will be the last processed @@ -205,25 +205,25 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { node := util.SelectBestNode(nodeScores) // Allocate idle resource to the task. if task.InitResreq.LessEqual(node.Idle) { - glog.V(3).Infof("Binding Task <%v/%v> to node <%v>", + klog.V(3).Infof("Binding Task <%v/%v> to node <%v>", task.Namespace, task.Name, node.Name) if err := stmt.Allocate(task, node.Name); err != nil { - glog.Errorf("Failed to bind Task %v on %v in Session %v, err: %v", + klog.Errorf("Failed to bind Task %v on %v in Session %v, err: %v", task.UID, node.Name, ssn.UID, err) } } else { //store information about missing resources job.NodesFitDelta[node.Name] = node.Idle.Clone() job.NodesFitDelta[node.Name].FitDelta(task.InitResreq) - glog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s> with limited resources", + klog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s> with limited resources", task.Namespace, task.Name, node.Name) // Allocate releasing resource to the task if any. if task.InitResreq.LessEqual(node.Releasing) { - glog.V(3).Infof("Pipelining Task <%v/%v> to node <%v> for <%v> on <%v>", + klog.V(3).Infof("Pipelining Task <%v/%v> to node <%v> for <%v> on <%v>", task.Namespace, task.Name, node.Name, task.InitResreq, node.Releasing) if err := stmt.Pipeline(task, node.Name); err != nil { - glog.Errorf("Failed to pipeline Task %v on %v", + klog.Errorf("Failed to pipeline Task %v on %v", task.UID, node.Name) } } diff --git a/pkg/scheduler/actions/backfill/backfill.go b/pkg/scheduler/actions/backfill/backfill.go index c4e96364b9..20fdfdf47e 100644 --- a/pkg/scheduler/actions/backfill/backfill.go +++ b/pkg/scheduler/actions/backfill/backfill.go @@ -17,7 +17,7 @@ limitations under the License. package backfill import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -39,8 +39,8 @@ func (alloc *backfillAction) Name() string { func (alloc *backfillAction) Initialize() {} func (alloc *backfillAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Backfill ...") - defer glog.V(3).Infof("Leaving Backfill ...") + klog.V(3).Infof("Enter Backfill ...") + defer klog.V(3).Infof("Leaving Backfill ...") // TODO (k82cn): When backfill, it's also need to balance between Queues. for _, job := range ssn.Jobs { @@ -48,7 +48,7 @@ func (alloc *backfillAction) Execute(ssn *framework.Session) { continue } if vr := ssn.JobValid(job); vr != nil && !vr.Pass { - glog.V(4).Infof("Job <%s/%s> Queue <%s> skip backfill, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) + klog.V(4).Infof("Job <%s/%s> Queue <%s> skip backfill, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) continue } @@ -63,15 +63,15 @@ func (alloc *backfillAction) Execute(ssn *framework.Session) { // TODO (k82cn): predicates did not consider pod number for now, there'll // be ping-pong case here. if err := ssn.PredicateFn(task, node); err != nil { - glog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v", + klog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v", task.Namespace, task.Name, node.Name, err) fe.SetNodeError(node.Name, err) continue } - glog.V(3).Infof("Binding Task <%v/%v> to node <%v>", task.Namespace, task.Name, node.Name) + klog.V(3).Infof("Binding Task <%v/%v> to node <%v>", task.Namespace, task.Name, node.Name) if err := ssn.Allocate(task, node.Name); err != nil { - glog.Errorf("Failed to bind Task %v on %v in Session %v", task.UID, node.Name, ssn.UID) + klog.Errorf("Failed to bind Task %v on %v in Session %v", task.UID, node.Name, ssn.UID) fe.SetNodeError(node.Name, err) continue } diff --git a/pkg/scheduler/actions/enqueue/enqueue.go b/pkg/scheduler/actions/enqueue/enqueue.go index 2c01e7ec37..11c7765a79 100644 --- a/pkg/scheduler/actions/enqueue/enqueue.go +++ b/pkg/scheduler/actions/enqueue/enqueue.go @@ -17,7 +17,7 @@ limitations under the License. package enqueue import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -40,8 +40,8 @@ func (enqueue *enqueueAction) Name() string { func (enqueue *enqueueAction) Initialize() {} func (enqueue *enqueueAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Enqueue ...") - defer glog.V(3).Infof("Leaving Enqueue ...") + klog.V(3).Infof("Enter Enqueue ...") + defer klog.V(3).Infof("Leaving Enqueue ...") queues := util.NewPriorityQueue(ssn.QueueOrderFn) queueMap := map[api.QueueID]*api.QueueInfo{} @@ -50,12 +50,12 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { for _, job := range ssn.Jobs { if queue, found := ssn.Queues[job.Queue]; !found { - glog.Errorf("Failed to find Queue <%s> for Job <%s/%s>", + klog.Errorf("Failed to find Queue <%s> for Job <%s/%s>", job.Queue, job.Namespace, job.Name) continue } else { if _, existed := queueMap[queue.UID]; !existed { - glog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", + klog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", queue.Name, job.Namespace, job.Name) queueMap[queue.UID] = queue @@ -67,12 +67,12 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { if _, found := jobsMap[job.Queue]; !found { jobsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn) } - glog.V(3).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue) + klog.V(3).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue) jobsMap[job.Queue].Push(job) } } - glog.V(3).Infof("Try to enqueue PodGroup to %d Queues", len(jobsMap)) + klog.V(3).Infof("Try to enqueue PodGroup to %d Queues", len(jobsMap)) emptyRes := api.EmptyResource() nodesIdleRes := api.EmptyResource() @@ -86,7 +86,7 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { } if nodesIdleRes.Less(emptyRes) { - glog.V(3).Infof("Node idle resource is overused, ignore it.") + klog.V(3).Infof("Node idle resource is overused, ignore it.") break } diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index 923d5f0574..36e1059c7f 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -19,7 +19,7 @@ package preempt import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -43,8 +43,8 @@ func (alloc *preemptAction) Name() string { func (alloc *preemptAction) Initialize() {} func (alloc *preemptAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Preempt ...") - defer glog.V(3).Infof("Leaving Preempt ...") + klog.V(3).Infof("Enter Preempt ...") + defer klog.V(3).Infof("Leaving Preempt ...") preemptorsMap := map[api.QueueID]*util.PriorityQueue{} preemptorTasks := map[api.JobID]*util.PriorityQueue{} @@ -57,14 +57,14 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { continue } if vr := ssn.JobValid(job); vr != nil && !vr.Pass { - glog.V(4).Infof("Job <%s/%s> Queue <%s> skip preemption, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) + klog.V(4).Infof("Job <%s/%s> Queue <%s> skip preemption, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) continue } if queue, found := ssn.Queues[job.Queue]; !found { continue } else if _, existed := queues[queue.UID]; !existed { - glog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", + klog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", queue.Name, job.Namespace, job.Name) queues[queue.UID] = queue } @@ -89,7 +89,7 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { // If no preemptors, no preemption. if preemptors == nil || preemptors.Empty() { - glog.V(4).Infof("No preemptors in Queue <%s>, break.", queue.Name) + klog.V(4).Infof("No preemptors in Queue <%s>, break.", queue.Name) break } @@ -100,7 +100,7 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { for { // If not preemptor tasks, next job. if preemptorTasks[preemptorJob.UID].Empty() { - glog.V(3).Infof("No preemptor task in job <%s/%s>.", + klog.V(3).Infof("No preemptor task in job <%s/%s>.", preemptorJob.Namespace, preemptorJob.Name) break } @@ -194,7 +194,7 @@ func preempt( selectedNodes := util.SortNodes(nodeScores) for _, node := range selectedNodes { - glog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", + klog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", preemptor.Namespace, preemptor.Name, node.Name) var preemptees []*api.TaskInfo @@ -212,7 +212,7 @@ func preempt( metrics.UpdatePreemptionVictimsCount(len(victims)) if err := validateVictims(victims, resreq); err != nil { - glog.V(3).Infof("No validated victims on Node <%s>: %v", node.Name, err) + klog.V(3).Infof("No validated victims on Node <%s>: %v", node.Name, err) continue } @@ -225,10 +225,10 @@ func preempt( // Preempt victims for tasks, pick lowest priority task first. for !victimsQueue.Empty() { preemptee := victimsQueue.Pop().(*api.TaskInfo) - glog.Errorf("Try to preempt Task <%s/%s> for Tasks <%s/%s>", + klog.Errorf("Try to preempt Task <%s/%s> for Tasks <%s/%s>", preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name) if err := stmt.Evict(preemptee, "preempt"); err != nil { - glog.Errorf("Failed to preempt Task <%s/%s> for Tasks <%s/%s>: %v", + klog.Errorf("Failed to preempt Task <%s/%s> for Tasks <%s/%s>: %v", preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name, err) continue } @@ -240,12 +240,12 @@ func preempt( } metrics.RegisterPreemptionAttempts() - glog.V(3).Infof("Preempted <%v> for task <%s/%s> requested <%v>.", + klog.V(3).Infof("Preempted <%v> for task <%s/%s> requested <%v>.", preempted, preemptor.Namespace, preemptor.Name, preemptor.InitResreq) if preemptor.InitResreq.LessEqual(preempted) { if err := stmt.Pipeline(preemptor, node.Name); err != nil { - glog.Errorf("Failed to pipline Task <%s/%s> on Node <%s>", + klog.Errorf("Failed to pipline Task <%s/%s> on Node <%s>", preemptor.Namespace, preemptor.Name, node.Name) } diff --git a/pkg/scheduler/actions/reclaim/reclaim.go b/pkg/scheduler/actions/reclaim/reclaim.go index ebc041804c..a644f1af79 100644 --- a/pkg/scheduler/actions/reclaim/reclaim.go +++ b/pkg/scheduler/actions/reclaim/reclaim.go @@ -17,7 +17,7 @@ limitations under the License. package reclaim import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -40,8 +40,8 @@ func (alloc *reclaimAction) Name() string { func (alloc *reclaimAction) Initialize() {} func (alloc *reclaimAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Reclaim ...") - defer glog.V(3).Infof("Leaving Reclaim ...") + klog.V(3).Infof("Enter Reclaim ...") + defer klog.V(3).Infof("Leaving Reclaim ...") queues := util.NewPriorityQueue(ssn.QueueOrderFn) queueMap := map[api.QueueID]*api.QueueInfo{} @@ -49,7 +49,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { preemptorsMap := map[api.QueueID]*util.PriorityQueue{} preemptorTasks := map[api.JobID]*util.PriorityQueue{} - glog.V(3).Infof("There are <%d> Jobs and <%d> Queues in total for scheduling.", + klog.V(3).Infof("There are <%d> Jobs and <%d> Queues in total for scheduling.", len(ssn.Jobs), len(ssn.Queues)) var underRequest []*api.JobInfo @@ -58,17 +58,17 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { continue } if vr := ssn.JobValid(job); vr != nil && !vr.Pass { - glog.V(4).Infof("Job <%s/%s> Queue <%s> skip reclaim, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) + klog.V(4).Infof("Job <%s/%s> Queue <%s> skip reclaim, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) continue } if queue, found := ssn.Queues[job.Queue]; !found { - glog.Errorf("Failed to find Queue <%s> for Job <%s/%s>", + klog.Errorf("Failed to find Queue <%s> for Job <%s/%s>", job.Queue, job.Namespace, job.Name) continue } else { if _, existed := queueMap[queue.UID]; !existed { - glog.V(4).Infof("Added Queue <%s> for Job <%s/%s>", + klog.V(4).Infof("Added Queue <%s> for Job <%s/%s>", queue.Name, job.Namespace, job.Name) queueMap[queue.UID] = queue @@ -100,7 +100,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { queue := queues.Pop().(*api.QueueInfo) if ssn.Overused(queue) { - glog.V(3).Infof("Queue <%s> is overused, ignore it.", queue.Name) + klog.V(3).Infof("Queue <%s> is overused, ignore it.", queue.Name) continue } @@ -128,7 +128,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { resreq := task.InitResreq.Clone() reclaimed := api.EmptyResource() - glog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", + klog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", task.Namespace, task.Name, n.Name) var reclaimees []*api.TaskInfo @@ -148,7 +148,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { victims := ssn.Reclaimable(task, reclaimees) if len(victims) == 0 { - glog.V(3).Infof("No victims on Node <%s>.", n.Name) + klog.V(3).Infof("No victims on Node <%s>.", n.Name) continue } @@ -158,16 +158,16 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { allRes.Add(v.Resreq) } if allRes.Less(resreq) { - glog.V(3).Infof("Not enough resource from victims on Node <%s>.", n.Name) + klog.V(3).Infof("Not enough resource from victims on Node <%s>.", n.Name) continue } // Reclaim victims for tasks. for _, reclaimee := range victims { - glog.Errorf("Try to reclaim Task <%s/%s> for Tasks <%s/%s>", + klog.Errorf("Try to reclaim Task <%s/%s> for Tasks <%s/%s>", reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name) if err := ssn.Evict(reclaimee, "reclaim"); err != nil { - glog.Errorf("Failed to reclaim Task <%s/%s> for Tasks <%s/%s>: %v", + klog.Errorf("Failed to reclaim Task <%s/%s> for Tasks <%s/%s>: %v", reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name, err) continue } @@ -178,12 +178,12 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { } } - glog.V(3).Infof("Reclaimed <%v> for task <%s/%s> requested <%v>.", + klog.V(3).Infof("Reclaimed <%v> for task <%s/%s> requested <%v>.", reclaimed, task.Namespace, task.Name, task.InitResreq) if task.InitResreq.LessEqual(reclaimed) { if err := ssn.Pipeline(task, n.Name); err != nil { - glog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>", + klog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>", task.Namespace, task.Name, n.Name) } diff --git a/pkg/scheduler/api/namespace_info.go b/pkg/scheduler/api/namespace_info.go index 80ce552a19..2642b4fb27 100644 --- a/pkg/scheduler/api/namespace_info.go +++ b/pkg/scheduler/api/namespace_info.go @@ -19,10 +19,9 @@ package api import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog" ) // NamespaceName is name of namespace @@ -132,7 +131,7 @@ func (n *NamespaceCollection) Snapshot() *NamespaceInfo { obj, err := n.quotaWeight.Pop() if err != nil { - glog.Warningf("namespace %s, quota weight meets error %v when pop", n.Name, err) + klog.Warningf("namespace %s, quota weight meets error %v when pop", n.Name, err) } else { item := obj.(*quotaItem) weight = item.weight diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 9bb8924288..a62ce98dbd 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -19,9 +19,8 @@ package api import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" + "k8s.io/klog" ) // NodeInfo is node level aggregated information. @@ -149,7 +148,7 @@ func (ni *NodeInfo) SetNode(node *v1.Node) { ni.setNodeState(node) if !ni.Ready() { - glog.Warningf("Failed to set node info, phase: %s, reason: %s", + klog.Warningf("Failed to set node info, phase: %s, reason: %s", ni.State.Phase, ni.State.Reason) return } diff --git a/pkg/scheduler/cache/cache.go b/pkg/scheduler/cache/cache.go index 37ca774182..cd813303a3 100644 --- a/pkg/scheduler/cache/cache.go +++ b/pkg/scheduler/cache/cache.go @@ -21,8 +21,6 @@ import ( "sync" "time" - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/api/scheduling/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -41,6 +39,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/scheduler/volumebinder" @@ -128,7 +127,7 @@ func (db *defaultBinder) Bind(p *v1.Pod, hostname string) error { Name: hostname, }, }); err != nil { - glog.Errorf("Failed to bind pod <%v/%v>: %#v", p.Namespace, p.Name, err) + klog.Errorf("Failed to bind pod <%v/%v>: %#v", p.Namespace, p.Name, err) return err } return nil @@ -140,10 +139,10 @@ type defaultEvictor struct { //Evict will send delete pod request to api server func (de *defaultEvictor) Evict(p *v1.Pod) error { - glog.V(3).Infof("Evicting pod %v/%v", p.Namespace, p.Name) + klog.V(3).Infof("Evicting pod %v/%v", p.Namespace, p.Name) if err := de.kubeclient.CoreV1().Pods(p.Namespace).Delete(p.Name, nil); err != nil { - glog.Errorf("Failed to evict pod <%v/%v>: %#v", p.Namespace, p.Name, err) + klog.Errorf("Failed to evict pod <%v/%v>: %#v", p.Namespace, p.Name, err) return err } return nil @@ -182,7 +181,7 @@ func podConditionHaveUpdate(status *v1.PodStatus, condition *v1.PodCondition) bo // UpdatePodCondition will Update pod with podCondition func (su *defaultStatusUpdater) UpdatePodCondition(pod *v1.Pod, condition *v1.PodCondition) (*v1.Pod, error) { - glog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s)", pod.Namespace, pod.Name, condition.Type, condition.Status) + klog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s)", pod.Namespace, pod.Name, condition.Type, condition.Status) if podutil.UpdatePodCondition(&pod.Status, condition) { return su.kubeclient.CoreV1().Pods(pod.Namespace).UpdateStatus(pod) } @@ -194,19 +193,19 @@ func (su *defaultStatusUpdater) UpdatePodGroup(pg *schedulingapi.PodGroup) (*sch if pg.Version == schedulingapi.PodGroupVersionV1Alpha1 { podgroup := &v1alpha1.PodGroup{} if err := schedulingscheme.Scheme.Convert(&pg.PodGroup, podgroup, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) return nil, err } updated, err := su.vcclient.SchedulingV1alpha1().PodGroups(podgroup.Namespace).Update(podgroup) if err != nil { - glog.Errorf("Error while updating PodGroup with error: %v", err) + klog.Errorf("Error while updating PodGroup with error: %v", err) return nil, err } podGroupInfo := &schedulingapi.PodGroup{Version: schedulingapi.PodGroupVersionV1Alpha1} if err := schedulingscheme.Scheme.Convert(updated, &podGroupInfo.PodGroup, nil); err != nil { - glog.Errorf("Error while converting v1alpha.PodGroup to api.PodGroup with error: %v", err) + klog.Errorf("Error while converting v1alpha.PodGroup to api.PodGroup with error: %v", err) return nil, err } @@ -216,18 +215,18 @@ func (su *defaultStatusUpdater) UpdatePodGroup(pg *schedulingapi.PodGroup) (*sch if pg.Version == schedulingapi.PodGroupVersionV1Alpha2 { podgroup := &v1alpha2.PodGroup{} if err := schedulingscheme.Scheme.Convert(&pg.PodGroup, podgroup, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) return nil, err } updated, err := su.vcclient.SchedulingV1alpha2().PodGroups(podgroup.Namespace).Update(podgroup) if err != nil { - glog.Errorf("Error while updating PodGroup with error: %v", err) + klog.Errorf("Error while updating PodGroup with error: %v", err) } podGroupInfo := &schedulingapi.PodGroup{Version: schedulingapi.PodGroupVersionV1Alpha2} if err := schedulingscheme.Scheme.Convert(updated, &podGroupInfo.PodGroup, nil); err != nil { - glog.Errorf("Error While converting v2alpha.PodGroup to api.PodGroup with error: %v", err) + klog.Errorf("Error While converting v2alpha.PodGroup to api.PodGroup with error: %v", err) return nil, err } @@ -534,7 +533,7 @@ func (sc *SchedulerCache) Evict(taskInfo *schedulingapi.TaskInfo, reason string) if job.PodGroup.Version == schedulingapi.PodGroupVersionV1Alpha1 { podgroup := &v1alpha1.PodGroup{} if err := schedulingscheme.Scheme.Convert(&job.PodGroup.PodGroup, podgroup, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) return err } sc.Recorder.Eventf(podgroup, v1.EventTypeNormal, "Evict", reason) @@ -544,7 +543,7 @@ func (sc *SchedulerCache) Evict(taskInfo *schedulingapi.TaskInfo, reason string) if job.PodGroup.Version == schedulingapi.PodGroupVersionV1Alpha2 { podgroup := &v1alpha2.PodGroup{} if err := schedulingscheme.Scheme.Convert(&job.PodGroup.PodGroup, podgroup, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) return err } sc.Recorder.Eventf(podgroup, v1.EventTypeNormal, "Evict", reason) @@ -647,14 +646,14 @@ func (sc *SchedulerCache) taskUnschedulable(task *schedulingapi.TaskInfo, messag return err } } else { - glog.V(4).Infof("task unscheduleable %s/%s, message: %s, skip by no condition update", pod.Namespace, pod.Name, message) + klog.V(4).Infof("task unscheduleable %s/%s, message: %s, skip by no condition update", pod.Namespace, pod.Name, message) } return nil } func (sc *SchedulerCache) deleteJob(job *schedulingapi.JobInfo) { - glog.V(3).Infof("Try to delete Job <%v:%v/%v>", job.UID, job.Namespace, job.Name) + klog.V(3).Infof("Try to delete Job <%v:%v/%v>", job.UID, job.Namespace, job.Name) sc.deletedJobs.AddRateLimited(job) } @@ -669,7 +668,7 @@ func (sc *SchedulerCache) processCleanupJob() { job, found := obj.(*schedulingapi.JobInfo) if !found { - glog.Errorf("Failed to convert <%v> to *JobInfo", obj) + klog.Errorf("Failed to convert <%v> to *JobInfo", obj) return } @@ -678,7 +677,7 @@ func (sc *SchedulerCache) processCleanupJob() { if schedulingapi.JobTerminated(job) { delete(sc.Jobs, job.UID) - glog.V(3).Infof("Job <%v:%v/%v> was deleted.", job.UID, job.Namespace, job.Name) + klog.V(3).Infof("Job <%v:%v/%v> was deleted.", job.UID, job.Namespace, job.Name) } else { // Retry sc.deleteJob(job) @@ -699,12 +698,12 @@ func (sc *SchedulerCache) processResyncTask() { task, ok := obj.(*schedulingapi.TaskInfo) if !ok { - glog.Errorf("failed to convert %v to *v1.Pod", obj) + klog.Errorf("failed to convert %v to *v1.Pod", obj) return } if err := sc.syncTask(task); err != nil { - glog.Errorf("Failed to sync pod <%v/%v>, retry it.", task.Namespace, task.Name) + klog.Errorf("Failed to sync pod <%v/%v>, retry it.", task.Namespace, task.Name) sc.resyncTask(task) } } @@ -745,7 +744,7 @@ func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo { value.Priority = priorityClass.Value } - glog.V(4).Infof("The priority of job <%s/%s> is <%s/%d>", + klog.V(4).Infof("The priority of job <%s/%s> is <%s/%d>", value.Namespace, value.Name, priName, value.Priority) } @@ -760,21 +759,21 @@ func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo { for _, value := range sc.NamespaceCollection { info := value.Snapshot() snapshot.NamespaceInfo[info.Name] = info - glog.V(4).Infof("Namespace %s has weight %v", + klog.V(4).Infof("Namespace %s has weight %v", value.Name, info.GetWeight()) } for _, value := range sc.Jobs { // If no scheduling spec, does not handle it. if value.PodGroup == nil && value.PDB == nil { - glog.V(4).Infof("The scheduling spec of Job <%v:%s/%s> is nil, ignore it.", + klog.V(4).Infof("The scheduling spec of Job <%v:%s/%s> is nil, ignore it.", value.UID, value.Namespace, value.Name) continue } if _, found := snapshot.Queues[value.Queue]; !found { - glog.V(3).Infof("The Queue <%v> of Job <%v/%v> does not exist, ignore it.", + klog.V(3).Infof("The Queue <%v> of Job <%v/%v> does not exist, ignore it.", value.Queue, value.Namespace, value.Name) continue } @@ -784,7 +783,7 @@ func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo { } wg.Wait() - glog.V(3).Infof("There are <%d> Jobs, <%d> Queues and <%d> Nodes in total for scheduling.", + klog.V(3).Infof("There are <%d> Jobs, <%d> Queues and <%d> Nodes in total for scheduling.", len(snapshot.Jobs), len(snapshot.Queues), len(snapshot.Nodes)) return snapshot @@ -861,7 +860,7 @@ func (sc *SchedulerCache) RecordJobStatusEvent(job *schedulingapi.JobInfo) { msg = fitError.Error() } if err := sc.taskUnschedulable(taskInfo, msg); err != nil { - glog.Errorf("Failed to update unschedulable task status <%s/%s>: %v", + klog.Errorf("Failed to update unschedulable task status <%s/%s>: %v", taskInfo.Namespace, taskInfo.Name, err) } } @@ -891,7 +890,7 @@ func (sc *SchedulerCache) recordPodGroupEvent(podGroup *schedulingapi.PodGroup, if podGroup.Version == schedulingapi.PodGroupVersionV1Alpha1 { pg := &v1alpha1.PodGroup{} if err := schedulingscheme.Scheme.Convert(&podGroup.PodGroup, pg, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) return } @@ -901,7 +900,7 @@ func (sc *SchedulerCache) recordPodGroupEvent(podGroup *schedulingapi.PodGroup, if podGroup.Version == schedulingapi.PodGroupVersionV1Alpha2 { pg := &v1alpha2.PodGroup{} if err := schedulingscheme.Scheme.Convert(&podGroup.PodGroup, pg, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) return } diff --git a/pkg/scheduler/cache/event_handlers.go b/pkg/scheduler/cache/event_handlers.go index 03d65d84d0..dfe0c62f91 100644 --- a/pkg/scheduler/cache/event_handlers.go +++ b/pkg/scheduler/cache/event_handlers.go @@ -19,14 +19,13 @@ package cache import ( "fmt" - "github.com/golang/glog" - "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1beta1" "k8s.io/api/scheduling/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/apis/scheduling/scheme" @@ -45,7 +44,7 @@ func isTerminated(status schedulingapi.TaskStatus) bool { func (sc *SchedulerCache) getOrCreateJob(pi *schedulingapi.TaskInfo) *schedulingapi.JobInfo { if len(pi.Job) == 0 { if pi.Pod.Spec.SchedulerName != sc.schedulerName { - glog.V(4).Infof("Pod %s/%s will not not scheduled by %s, skip creating PodGroup and Job for it", + klog.V(4).Infof("Pod %s/%s will not not scheduled by %s, skip creating PodGroup and Job for it", pi.Pod.Namespace, pi.Pod.Name, sc.schedulerName) } return nil @@ -90,7 +89,7 @@ func (sc *SchedulerCache) syncTask(oldTask *schedulingapi.TaskInfo) error { if err != nil { if errors.IsNotFound(err) { sc.deleteTask(oldTask) - glog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name) + klog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name) return nil } @@ -173,7 +172,7 @@ func (sc *SchedulerCache) deletePod(pod *v1.Pod) error { func (sc *SchedulerCache) AddPod(obj interface{}) { pod, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("Cannot convert to *v1.Pod: %v", obj) + klog.Errorf("Cannot convert to *v1.Pod: %v", obj) return } @@ -182,11 +181,11 @@ func (sc *SchedulerCache) AddPod(obj interface{}) { err := sc.addPod(pod) if err != nil { - glog.Errorf("Failed to add pod <%s/%s> into cache: %v", + klog.Errorf("Failed to add pod <%s/%s> into cache: %v", pod.Namespace, pod.Name, err) return } - glog.V(3).Infof("Added pod <%s/%v> into cache.", pod.Namespace, pod.Name) + klog.V(3).Infof("Added pod <%s/%v> into cache.", pod.Namespace, pod.Name) return } @@ -194,12 +193,12 @@ func (sc *SchedulerCache) AddPod(obj interface{}) { func (sc *SchedulerCache) UpdatePod(oldObj, newObj interface{}) { oldPod, ok := oldObj.(*v1.Pod) if !ok { - glog.Errorf("Cannot convert oldObj to *v1.Pod: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *v1.Pod: %v", oldObj) return } newPod, ok := newObj.(*v1.Pod) if !ok { - glog.Errorf("Cannot convert newObj to *v1.Pod: %v", newObj) + klog.Errorf("Cannot convert newObj to *v1.Pod: %v", newObj) return } @@ -208,11 +207,11 @@ func (sc *SchedulerCache) UpdatePod(oldObj, newObj interface{}) { err := sc.updatePod(oldPod, newPod) if err != nil { - glog.Errorf("Failed to update pod %v in cache: %v", oldPod.Name, err) + klog.Errorf("Failed to update pod %v in cache: %v", oldPod.Name, err) return } - glog.V(3).Infof("Updated pod <%s/%v> in cache.", oldPod.Namespace, oldPod.Name) + klog.V(3).Infof("Updated pod <%s/%v> in cache.", oldPod.Namespace, oldPod.Name) return } @@ -227,11 +226,11 @@ func (sc *SchedulerCache) DeletePod(obj interface{}) { var ok bool pod, ok = t.Obj.(*v1.Pod) if !ok { - glog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj) + klog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1.Pod: %v", t) + klog.Errorf("Cannot convert to *v1.Pod: %v", t) return } @@ -240,11 +239,11 @@ func (sc *SchedulerCache) DeletePod(obj interface{}) { err := sc.deletePod(pod) if err != nil { - glog.Errorf("Failed to delete pod %v from cache: %v", pod.Name, err) + klog.Errorf("Failed to delete pod %v from cache: %v", pod.Name, err) return } - glog.V(3).Infof("Deleted pod <%s/%v> from cache.", pod.Namespace, pod.Name) + klog.V(3).Infof("Deleted pod <%s/%v> from cache.", pod.Namespace, pod.Name) return } @@ -282,7 +281,7 @@ func (sc *SchedulerCache) deleteNode(node *v1.Node) error { func (sc *SchedulerCache) AddNode(obj interface{}) { node, ok := obj.(*v1.Node) if !ok { - glog.Errorf("Cannot convert to *v1.Node: %v", obj) + klog.Errorf("Cannot convert to *v1.Node: %v", obj) return } @@ -291,7 +290,7 @@ func (sc *SchedulerCache) AddNode(obj interface{}) { err := sc.addNode(node) if err != nil { - glog.Errorf("Failed to add node %s into cache: %v", node.Name, err) + klog.Errorf("Failed to add node %s into cache: %v", node.Name, err) return } return @@ -301,12 +300,12 @@ func (sc *SchedulerCache) AddNode(obj interface{}) { func (sc *SchedulerCache) UpdateNode(oldObj, newObj interface{}) { oldNode, ok := oldObj.(*v1.Node) if !ok { - glog.Errorf("Cannot convert oldObj to *v1.Node: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *v1.Node: %v", oldObj) return } newNode, ok := newObj.(*v1.Node) if !ok { - glog.Errorf("Cannot convert newObj to *v1.Node: %v", newObj) + klog.Errorf("Cannot convert newObj to *v1.Node: %v", newObj) return } @@ -315,7 +314,7 @@ func (sc *SchedulerCache) UpdateNode(oldObj, newObj interface{}) { err := sc.updateNode(oldNode, newNode) if err != nil { - glog.Errorf("Failed to update node %v in cache: %v", oldNode.Name, err) + klog.Errorf("Failed to update node %v in cache: %v", oldNode.Name, err) return } return @@ -331,11 +330,11 @@ func (sc *SchedulerCache) DeleteNode(obj interface{}) { var ok bool node, ok = t.Obj.(*v1.Node) if !ok { - glog.Errorf("Cannot convert to *v1.Node: %v", t.Obj) + klog.Errorf("Cannot convert to *v1.Node: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1.Node: %v", t) + klog.Errorf("Cannot convert to *v1.Node: %v", t) return } @@ -344,7 +343,7 @@ func (sc *SchedulerCache) DeleteNode(obj interface{}) { err := sc.deleteNode(node) if err != nil { - glog.Errorf("Failed to delete node %s from cache: %v", node.Name, err) + klog.Errorf("Failed to delete node %s from cache: %v", node.Name, err) return } return @@ -395,24 +394,24 @@ func (sc *SchedulerCache) deletePodGroup(id schedulingapi.JobID) error { func (sc *SchedulerCache) AddPodGroupV1alpha1(obj interface{}) { ss, ok := obj.(*schedulingv1.PodGroup) if !ok { - glog.Errorf("Cannot convert to *schedulingv1.PodGroup: %v", obj) + klog.Errorf("Cannot convert to *schedulingv1.PodGroup: %v", obj) return } podgroup := scheduling.PodGroup{} if err := scheme.Scheme.Convert(ss, &podgroup, nil); err != nil { - glog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup) + klog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup) return } pg := &schedulingapi.PodGroup{podgroup, schedulingapi.PodGroupVersionV1Alpha1} - glog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec) + klog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec) sc.Mutex.Lock() defer sc.Mutex.Unlock() if err := sc.setPodGroup(pg); err != nil { - glog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err) + klog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err) return } @@ -423,24 +422,24 @@ func (sc *SchedulerCache) AddPodGroupV1alpha1(obj interface{}) { func (sc *SchedulerCache) AddPodGroupV1alpha2(obj interface{}) { ss, ok := obj.(*schedulingv2.PodGroup) if !ok { - glog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", obj) + klog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", obj) return } podgroup := scheduling.PodGroup{} if err := scheme.Scheme.Convert(ss, &podgroup, nil); err != nil { - glog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup) + klog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup) return } pg := &schedulingapi.PodGroup{podgroup, schedulingapi.PodGroupVersionV1Alpha2} - glog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec) + klog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec) sc.Mutex.Lock() defer sc.Mutex.Unlock() if err := sc.setPodGroup(pg); err != nil { - glog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err) + klog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err) return } return @@ -450,12 +449,12 @@ func (sc *SchedulerCache) AddPodGroupV1alpha2(obj interface{}) { func (sc *SchedulerCache) UpdatePodGroupV1alpha1(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*schedulingv1.PodGroup) if !ok { - glog.Errorf("Cannot convert oldObj to *schedulingv1.SchedulingSpec: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *schedulingv1.SchedulingSpec: %v", oldObj) return } newSS, ok := newObj.(*schedulingv1.PodGroup) if !ok { - glog.Errorf("Cannot convert newObj to *schedulingv1.SchedulingSpec: %v", newObj) + klog.Errorf("Cannot convert newObj to *schedulingv1.SchedulingSpec: %v", newObj) return } @@ -465,7 +464,7 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha1(oldObj, newObj interface{}) { podgroup := scheduling.PodGroup{} if err := scheme.Scheme.Convert(newSS, &podgroup, nil); err != nil { - glog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup) + klog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup) return } @@ -475,7 +474,7 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha1(oldObj, newObj interface{}) { defer sc.Mutex.Unlock() if err := sc.updatePodGroup(pg); err != nil { - glog.Errorf("Failed to update SchedulingSpec %s into cache: %v", pg.Name, err) + klog.Errorf("Failed to update SchedulingSpec %s into cache: %v", pg.Name, err) return } return @@ -485,12 +484,12 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha1(oldObj, newObj interface{}) { func (sc *SchedulerCache) UpdatePodGroupV1alpha2(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*schedulingv2.PodGroup) if !ok { - glog.Errorf("Cannot convert oldObj to *schedulingv2.SchedulingSpec: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *schedulingv2.SchedulingSpec: %v", oldObj) return } newSS, ok := newObj.(*schedulingv2.PodGroup) if !ok { - glog.Errorf("Cannot convert newObj to *schedulingv2.SchedulingSpec: %v", newObj) + klog.Errorf("Cannot convert newObj to *schedulingv2.SchedulingSpec: %v", newObj) return } @@ -500,7 +499,7 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha2(oldObj, newObj interface{}) { podgroup := scheduling.PodGroup{} if err := scheme.Scheme.Convert(newSS, &podgroup, nil); err != nil { - glog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup) + klog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup) return } @@ -510,7 +509,7 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha2(oldObj, newObj interface{}) { defer sc.Mutex.Unlock() if err := sc.updatePodGroup(pg); err != nil { - glog.Errorf("Failed to update podgroup %s into cache: %v", pg.Name, err) + klog.Errorf("Failed to update podgroup %s into cache: %v", pg.Name, err) return } } @@ -525,11 +524,11 @@ func (sc *SchedulerCache) DeletePodGroupV1alpha1(obj interface{}) { var ok bool ss, ok = t.Obj.(*schedulingv1.PodGroup) if !ok { - glog.Errorf("Cannot convert to podgroup: %v", t.Obj) + klog.Errorf("Cannot convert to podgroup: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to podgroup: %v", t) + klog.Errorf("Cannot convert to podgroup: %v", t) return } @@ -539,7 +538,7 @@ func (sc *SchedulerCache) DeletePodGroupV1alpha1(obj interface{}) { defer sc.Mutex.Unlock() if err := sc.deletePodGroup(jobID); err != nil { - glog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err) + klog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err) return } return @@ -555,11 +554,11 @@ func (sc *SchedulerCache) DeletePodGroupV1alpha2(obj interface{}) { var ok bool ss, ok = t.Obj.(*schedulingv2.PodGroup) if !ok { - glog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", t.Obj) + klog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", t) + klog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", t) return } @@ -569,7 +568,7 @@ func (sc *SchedulerCache) DeletePodGroupV1alpha2(obj interface{}) { defer sc.Mutex.Unlock() if err := sc.deletePodGroup(jobID); err != nil { - glog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err) + klog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err) return } @@ -621,7 +620,7 @@ func (sc *SchedulerCache) deletePDB(pdb *policyv1.PodDisruptionBudget) error { func (sc *SchedulerCache) AddPDB(obj interface{}) { pdb, ok := obj.(*policyv1.PodDisruptionBudget) if !ok { - glog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", obj) + klog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", obj) return } @@ -630,7 +629,7 @@ func (sc *SchedulerCache) AddPDB(obj interface{}) { err := sc.setPDB(pdb) if err != nil { - glog.Errorf("Failed to add PodDisruptionBudget %s into cache: %v", pdb.Name, err) + klog.Errorf("Failed to add PodDisruptionBudget %s into cache: %v", pdb.Name, err) return } return @@ -640,12 +639,12 @@ func (sc *SchedulerCache) AddPDB(obj interface{}) { func (sc *SchedulerCache) UpdatePDB(oldObj, newObj interface{}) { oldPDB, ok := oldObj.(*policyv1.PodDisruptionBudget) if !ok { - glog.Errorf("Cannot convert oldObj to *policyv1.PodDisruptionBudget: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *policyv1.PodDisruptionBudget: %v", oldObj) return } newPDB, ok := newObj.(*policyv1.PodDisruptionBudget) if !ok { - glog.Errorf("Cannot convert newObj to *policyv1.PodDisruptionBudget: %v", newObj) + klog.Errorf("Cannot convert newObj to *policyv1.PodDisruptionBudget: %v", newObj) return } @@ -654,7 +653,7 @@ func (sc *SchedulerCache) UpdatePDB(oldObj, newObj interface{}) { err := sc.updatePDB(oldPDB, newPDB) if err != nil { - glog.Errorf("Failed to update PodDisruptionBudget %s into cache: %v", oldPDB.Name, err) + klog.Errorf("Failed to update PodDisruptionBudget %s into cache: %v", oldPDB.Name, err) return } return @@ -670,11 +669,11 @@ func (sc *SchedulerCache) DeletePDB(obj interface{}) { var ok bool pdb, ok = t.Obj.(*policyv1.PodDisruptionBudget) if !ok { - glog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", t.Obj) + klog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", t) + klog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", t) return } @@ -683,7 +682,7 @@ func (sc *SchedulerCache) DeletePDB(obj interface{}) { err := sc.deletePDB(pdb) if err != nil { - glog.Errorf("Failed to delete PodDisruptionBudget %s from cache: %v", pdb.Name, err) + klog.Errorf("Failed to delete PodDisruptionBudget %s from cache: %v", pdb.Name, err) return } return @@ -693,20 +692,20 @@ func (sc *SchedulerCache) DeletePDB(obj interface{}) { func (sc *SchedulerCache) AddQueueV1alpha1(obj interface{}) { ss, ok := obj.(*schedulingv1.Queue) if !ok { - glog.Errorf("Cannot convert to *schedulingv1.Queue: %v", obj) + klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", obj) return } queue := &scheduling.Queue{} if err := scheme.Scheme.Convert(ss, queue, nil); err != nil { - glog.Errorf("Failed to convert queue from %T to %T", ss, queue) + klog.Errorf("Failed to convert queue from %T to %T", ss, queue) return } sc.Mutex.Lock() defer sc.Mutex.Unlock() - glog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec) + klog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec) sc.addQueue(queue) return @@ -716,17 +715,17 @@ func (sc *SchedulerCache) AddQueueV1alpha1(obj interface{}) { func (sc *SchedulerCache) AddQueueV1alpha2(obj interface{}) { ss, ok := obj.(*schedulingv2.Queue) if !ok { - glog.Errorf("Cannot convert to *schedulingv2.Queue: %v", obj) + klog.Errorf("Cannot convert to *schedulingv2.Queue: %v", obj) return } queue := &scheduling.Queue{} if err := scheme.Scheme.Convert(ss, queue, nil); err != nil { - glog.Errorf("Failed to convert queue from %T to %T", ss, queue) + klog.Errorf("Failed to convert queue from %T to %T", ss, queue) return } - glog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec) + klog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec) sc.Mutex.Lock() defer sc.Mutex.Unlock() @@ -739,12 +738,12 @@ func (sc *SchedulerCache) AddQueueV1alpha2(obj interface{}) { func (sc *SchedulerCache) UpdateQueueV1alpha1(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*schedulingv1.Queue) if !ok { - glog.Errorf("Cannot convert oldObj to *schedulingv1.Queue: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *schedulingv1.Queue: %v", oldObj) return } newSS, ok := newObj.(*schedulingv1.Queue) if !ok { - glog.Errorf("Cannot convert newObj to *schedulingv1.Queue: %v", newObj) + klog.Errorf("Cannot convert newObj to *schedulingv1.Queue: %v", newObj) return } @@ -754,7 +753,7 @@ func (sc *SchedulerCache) UpdateQueueV1alpha1(oldObj, newObj interface{}) { newQueue := &scheduling.Queue{} if err := scheme.Scheme.Convert(newSS, newQueue, nil); err != nil { - glog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue) + klog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue) return } @@ -769,12 +768,12 @@ func (sc *SchedulerCache) UpdateQueueV1alpha1(oldObj, newObj interface{}) { func (sc *SchedulerCache) UpdateQueueV1alpha2(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*schedulingv2.Queue) if !ok { - glog.Errorf("Cannot convert oldObj to *schedulingv2.Queue: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *schedulingv2.Queue: %v", oldObj) return } newSS, ok := newObj.(*schedulingv2.Queue) if !ok { - glog.Errorf("Cannot convert newObj to *schedulingv2.Queue: %v", newObj) + klog.Errorf("Cannot convert newObj to *schedulingv2.Queue: %v", newObj) return } @@ -784,7 +783,7 @@ func (sc *SchedulerCache) UpdateQueueV1alpha2(oldObj, newObj interface{}) { newQueue := &scheduling.Queue{} if err := scheme.Scheme.Convert(newSS, newQueue, nil); err != nil { - glog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue) + klog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue) return } @@ -805,11 +804,11 @@ func (sc *SchedulerCache) DeleteQueueV1alpha1(obj interface{}) { var ok bool ss, ok = t.Obj.(*schedulingv1.Queue) if !ok { - glog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t.Obj) + klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t) + klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t) return } @@ -830,11 +829,11 @@ func (sc *SchedulerCache) DeleteQueueV1alpha2(obj interface{}) { var ok bool ss, ok = t.Obj.(*schedulingv2.Queue) if !ok { - glog.Errorf("Cannot convert to *schedulingv2.Queue: %v", t.Obj) + klog.Errorf("Cannot convert to *schedulingv2.Queue: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t) + klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t) return } @@ -868,11 +867,11 @@ func (sc *SchedulerCache) DeletePriorityClass(obj interface{}) { var ok bool ss, ok = t.Obj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) return } @@ -886,7 +885,7 @@ func (sc *SchedulerCache) DeletePriorityClass(obj interface{}) { func (sc *SchedulerCache) UpdatePriorityClass(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert oldObj to *v1beta1.PriorityClass: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *v1beta1.PriorityClass: %v", oldObj) return @@ -894,7 +893,7 @@ func (sc *SchedulerCache) UpdatePriorityClass(oldObj, newObj interface{}) { newSS, ok := newObj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert newObj to *v1beta1.PriorityClass: %v", newObj) + klog.Errorf("Cannot convert newObj to *v1beta1.PriorityClass: %v", newObj) return @@ -917,11 +916,11 @@ func (sc *SchedulerCache) AddPriorityClass(obj interface{}) { var ok bool ss, ok = t.Obj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) return } @@ -944,7 +943,7 @@ func (sc *SchedulerCache) deletePriorityClass(pc *v1beta1.PriorityClass) { func (sc *SchedulerCache) addPriorityClass(pc *v1beta1.PriorityClass) { if pc.GlobalDefault { if sc.defaultPriorityClass != nil { - glog.Errorf("Updated default priority class from <%s> to <%s> forcefully.", + klog.Errorf("Updated default priority class from <%s> to <%s> forcefully.", sc.defaultPriorityClass.Name, pc.Name) } @@ -984,18 +983,18 @@ func (sc *SchedulerCache) DeleteResourceQuota(obj interface{}) { var ok bool r, ok = t.Obj.(*v1.ResourceQuota) if !ok { - glog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t.Obj) + klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t) + klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t) return } sc.Mutex.Lock() defer sc.Mutex.Unlock() - glog.V(3).Infof("Delete ResourceQuota <%s/%v> in cache", r.Namespace, r.Name) + klog.V(3).Infof("Delete ResourceQuota <%s/%v> in cache", r.Namespace, r.Name) sc.deleteResourceQuota(r) } @@ -1003,14 +1002,14 @@ func (sc *SchedulerCache) DeleteResourceQuota(obj interface{}) { func (sc *SchedulerCache) UpdateResourceQuota(oldObj, newObj interface{}) { newR, ok := newObj.(*v1.ResourceQuota) if !ok { - glog.Errorf("Cannot convert newObj to *v1.ResourceQuota: %v", newObj) + klog.Errorf("Cannot convert newObj to *v1.ResourceQuota: %v", newObj) return } sc.Mutex.Lock() defer sc.Mutex.Unlock() - glog.V(3).Infof("Update ResourceQuota <%s/%v> in cache, with spec: %v.", newR.Namespace, newR.Name, newR.Spec.Hard) + klog.V(3).Infof("Update ResourceQuota <%s/%v> in cache, with spec: %v.", newR.Namespace, newR.Name, newR.Spec.Hard) sc.updateResourceQuota(newR) } @@ -1021,13 +1020,13 @@ func (sc *SchedulerCache) AddResourceQuota(obj interface{}) { case *v1.ResourceQuota: r = t default: - glog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t) + klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t) return } sc.Mutex.Lock() defer sc.Mutex.Unlock() - glog.V(3).Infof("Add ResourceQuota <%s/%v> in cache, with spec: %v.", r.Namespace, r.Name, r.Spec.Hard) + klog.V(3).Infof("Add ResourceQuota <%s/%v> in cache, with spec: %v.", r.Namespace, r.Name, r.Spec.Hard) sc.updateResourceQuota(r) } diff --git a/pkg/scheduler/framework/arguments.go b/pkg/scheduler/framework/arguments.go index 5d803b67e7..b94462e4e2 100644 --- a/pkg/scheduler/framework/arguments.go +++ b/pkg/scheduler/framework/arguments.go @@ -19,7 +19,7 @@ package framework import ( "strconv" - "github.com/golang/glog" + "k8s.io/klog" ) // Arguments map @@ -38,7 +38,7 @@ func (a Arguments) GetInt(ptr *int, key string) { value, err := strconv.Atoi(argv) if err != nil { - glog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) + klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) return } @@ -58,7 +58,7 @@ func (a Arguments) GetBool(ptr *bool, key string) { value, err := strconv.ParseBool(argv) if err != nil { - glog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) + klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) return } diff --git a/pkg/scheduler/framework/framework.go b/pkg/scheduler/framework/framework.go index 379b7c7b8d..dfe4a43234 100644 --- a/pkg/scheduler/framework/framework.go +++ b/pkg/scheduler/framework/framework.go @@ -19,7 +19,7 @@ package framework import ( "time" - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" @@ -34,7 +34,7 @@ func OpenSession(cache cache.Cache, tiers []conf.Tier) *Session { for _, tier := range tiers { for _, plugin := range tier.Plugins { if pb, found := GetPluginBuilder(plugin.Name); !found { - glog.Errorf("Failed to get plugin %s.", plugin.Name) + klog.Errorf("Failed to get plugin %s.", plugin.Name) } else { plugin := pb(plugin.Arguments) ssn.plugins[plugin.Name()] = plugin diff --git a/pkg/scheduler/framework/job_updater.go b/pkg/scheduler/framework/job_updater.go index 2c00614287..98454fbc39 100644 --- a/pkg/scheduler/framework/job_updater.go +++ b/pkg/scheduler/framework/job_updater.go @@ -6,9 +6,8 @@ import ( "reflect" "time" - "github.com/golang/glog" - "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -116,7 +115,7 @@ func (ju *jobUpdater) updateJob(index int) { updatePG := !found || isPodGroupStatusUpdated(&job.PodGroup.Status, oldStatus) if _, err := ssn.cache.UpdateJobStatus(job, updatePG); err != nil { - glog.Errorf("Failed to update job <%s/%s>: %v", + klog.Errorf("Failed to update job <%s/%s>: %v", job.Namespace, job.Name, err) } } diff --git a/pkg/scheduler/framework/session.go b/pkg/scheduler/framework/session.go index 1397fd21c7..752ed38806 100644 --- a/pkg/scheduler/framework/session.go +++ b/pkg/scheduler/framework/session.go @@ -19,12 +19,11 @@ package framework import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -120,7 +119,7 @@ func openSession(cache cache.Cache) *Session { } if err := ssn.UpdateJobCondition(job, jc); err != nil { - glog.Errorf("Failed to update job condition: %v", err) + klog.Errorf("Failed to update job condition: %v", err) } } @@ -132,7 +131,7 @@ func openSession(cache cache.Cache) *Session { ssn.Queues = snapshot.Queues ssn.NamespaceInfo = snapshot.NamespaceInfo - glog.V(3).Infof("Open Session %v with <%d> Job and <%d> Queues", + klog.V(3).Infof("Open Session %v with <%d> Job and <%d> Queues", ssn.UID, len(ssn.Jobs), len(ssn.Queues)) return ssn @@ -151,7 +150,7 @@ func closeSession(ssn *Session) { ssn.namespaceOrderFns = nil ssn.queueOrderFns = nil - glog.V(3).Infof("Close Session %v", ssn.UID) + klog.V(3).Infof("Close Session %v", ssn.UID) } func jobStatus(ssn *Session, jobInfo *api.JobInfo) scheduling.PodGroupStatus { @@ -207,12 +206,12 @@ func (ssn *Session) Pipeline(task *api.TaskInfo, hostname string) error { job, found := ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Pipelined); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Pipelined, ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, ssn.UID) return fmt.Errorf("failed to find job %s when binding", task.Job) } @@ -221,14 +220,14 @@ func (ssn *Session) Pipeline(task *api.TaskInfo, hostname string) error { if node, found := ssn.Nodes[hostname]; found { if err := node.AddTask(task); err != nil { - glog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, ssn.UID, err) return err } - glog.V(3).Infof("After added Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After added Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, ssn.UID) return fmt.Errorf("failed to find node %s", hostname) } @@ -254,12 +253,12 @@ func (ssn *Session) Allocate(task *api.TaskInfo, hostname string) error { job, found := ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Allocated); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Allocated, ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, ssn.UID) return fmt.Errorf("failed to find job %s", task.Job) } @@ -268,14 +267,14 @@ func (ssn *Session) Allocate(task *api.TaskInfo, hostname string) error { if node, found := ssn.Nodes[hostname]; found { if err := node.AddTask(task); err != nil { - glog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, ssn.UID, err) return err } - glog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, ssn.UID) return fmt.Errorf("failed to find node %s", hostname) } @@ -292,7 +291,7 @@ func (ssn *Session) Allocate(task *api.TaskInfo, hostname string) error { if ssn.JobReady(job) { for _, task := range job.TaskStatusIndex[api.Allocated] { if err := ssn.dispatch(task); err != nil { - glog.Errorf("Failed to dispatch task <%v/%v>: %v", + klog.Errorf("Failed to dispatch task <%v/%v>: %v", task.Namespace, task.Name, err) return err } @@ -314,12 +313,12 @@ func (ssn *Session) dispatch(task *api.TaskInfo) error { // Update status in session if job, found := ssn.Jobs[task.Job]; found { if err := job.UpdateTaskStatus(task, api.Binding); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Binding, ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, ssn.UID) return fmt.Errorf("failed to find job %s", task.Job) } @@ -338,12 +337,12 @@ func (ssn *Session) Evict(reclaimee *api.TaskInfo, reason string) error { job, found := ssn.Jobs[reclaimee.Job] if found { if err := job.UpdateTaskStatus(reclaimee, api.Releasing); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", reclaimee.Namespace, reclaimee.Name, api.Releasing, ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", reclaimee.Job, ssn.UID) return fmt.Errorf("failed to find job %s", reclaimee.Job) } @@ -351,7 +350,7 @@ func (ssn *Session) Evict(reclaimee *api.TaskInfo, reason string) error { // Update task in node. if node, found := ssn.Nodes[reclaimee.NodeName]; found { if err := node.UpdateTask(reclaimee); err != nil { - glog.Errorf("Failed to update task <%v/%v> in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> in Session <%v>: %v", reclaimee.Namespace, reclaimee.Name, ssn.UID, err) return err } diff --git a/pkg/scheduler/framework/statement.go b/pkg/scheduler/framework/statement.go index 6fa2682dcb..d568c24241 100644 --- a/pkg/scheduler/framework/statement.go +++ b/pkg/scheduler/framework/statement.go @@ -19,7 +19,7 @@ package framework import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/metrics" @@ -42,11 +42,11 @@ func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) error { job, found := s.ssn.Jobs[reclaimee.Job] if found { if err := job.UpdateTaskStatus(reclaimee, api.Releasing); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", reclaimee.Namespace, reclaimee.Name, api.Releasing, s.ssn.UID, err) } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", reclaimee.Job, s.ssn.UID) } @@ -74,7 +74,7 @@ func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) error { func (s *Statement) evict(reclaimee *api.TaskInfo, reason string) error { if err := s.ssn.cache.Evict(reclaimee, reason); err != nil { if e := s.unevict(reclaimee, reason); err != nil { - glog.Errorf("Faled to unevict task <%v/%v>: %v.", + klog.Errorf("Faled to unevict task <%v/%v>: %v.", reclaimee.Namespace, reclaimee.Name, e) } return err @@ -88,11 +88,11 @@ func (s *Statement) unevict(reclaimee *api.TaskInfo, reason string) error { job, found := s.ssn.Jobs[reclaimee.Job] if found { if err := job.UpdateTaskStatus(reclaimee, api.Running); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", reclaimee.Namespace, reclaimee.Name, api.Releasing, s.ssn.UID, err) } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", reclaimee.Job, s.ssn.UID) } @@ -117,11 +117,11 @@ func (s *Statement) Pipeline(task *api.TaskInfo, hostname string) error { job, found := s.ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Pipelined); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Pipelined, s.ssn.UID, err) } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, s.ssn.UID) } @@ -129,13 +129,13 @@ func (s *Statement) Pipeline(task *api.TaskInfo, hostname string) error { if node, found := s.ssn.Nodes[hostname]; found { if err := node.AddTask(task); err != nil { - glog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, s.ssn.UID, err) } - glog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, s.ssn.UID) } @@ -162,11 +162,11 @@ func (s *Statement) unpipeline(task *api.TaskInfo) error { job, found := s.ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Pending); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Pipelined, s.ssn.UID, err) } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, s.ssn.UID) } @@ -174,13 +174,13 @@ func (s *Statement) unpipeline(task *api.TaskInfo) error { if node, found := s.ssn.Nodes[hostname]; found { if err := node.RemoveTask(task); err != nil { - glog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, s.ssn.UID, err) } - glog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, s.ssn.UID) } @@ -205,12 +205,12 @@ func (s *Statement) Allocate(task *api.TaskInfo, hostname string) error { job, found := s.ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Allocated); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Allocated, s.ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, s.ssn.UID) return fmt.Errorf("failed to find job %s", task.Job) } @@ -219,14 +219,14 @@ func (s *Statement) Allocate(task *api.TaskInfo, hostname string) error { if node, found := s.ssn.Nodes[hostname]; found { if err := node.AddTask(task); err != nil { - glog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, s.ssn.UID, err) return err } - glog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, s.ssn.UID) return fmt.Errorf("failed to find node %s", hostname) } @@ -241,7 +241,7 @@ func (s *Statement) Allocate(task *api.TaskInfo, hostname string) error { } // Update status in session - glog.V(3).Info("Allocating operations ...") + klog.V(3).Info("Allocating operations ...") s.operations = append(s.operations, operation{ name: "allocate", args: []interface{}{task, hostname}, @@ -262,12 +262,12 @@ func (s *Statement) allocate(task *api.TaskInfo, hostname string) error { // Update status in session if job, found := s.ssn.Jobs[task.Job]; found { if err := job.UpdateTaskStatus(task, api.Binding); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Binding, s.ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, s.ssn.UID) return fmt.Errorf("failed to find job %s", task.Job) } @@ -282,16 +282,16 @@ func (s *Statement) unallocate(task *api.TaskInfo, reason string) error { job, found := s.ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Pending); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Pending, s.ssn.UID, err) } } else { - glog.Errorf("Failed to find Job <%s> in Session <%s> index when unallocating.", + klog.Errorf("Failed to find Job <%s> in Session <%s> index when unallocating.", task.Job, s.ssn.UID) } if node, found := s.ssn.Nodes[task.NodeName]; found { - glog.V(3).Infof("Remove Task <%v> on node <%v>", task.Name, task.NodeName) + klog.V(3).Infof("Remove Task <%v> on node <%v>", task.Name, task.NodeName) node.RemoveTask(task) } @@ -307,7 +307,7 @@ func (s *Statement) unallocate(task *api.TaskInfo, reason string) error { // Discard operation for evict, pipeline and allocate func (s *Statement) Discard() { - glog.V(3).Info("Discarding operations ...") + klog.V(3).Info("Discarding operations ...") for i := len(s.operations) - 1; i >= 0; i-- { op := s.operations[i] switch op.name { @@ -323,7 +323,7 @@ func (s *Statement) Discard() { // Commit operation for evict and pipeline func (s *Statement) Commit() { - glog.V(3).Info("Committing operations ...") + klog.V(3).Info("Committing operations ...") for _, op := range s.operations { switch op.name { case "evict": diff --git a/pkg/scheduler/plugins/binpack/binpack.go b/pkg/scheduler/plugins/binpack/binpack.go index 108c4d06f4..f54709c32e 100644 --- a/pkg/scheduler/plugins/binpack/binpack.go +++ b/pkg/scheduler/plugins/binpack/binpack.go @@ -20,9 +20,8 @@ import ( "fmt" "strings" - "github.com/golang/glog" - "k8s.io/api/core/v1" + "k8s.io/klog" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api" @@ -156,10 +155,10 @@ func (bp *binpackPlugin) Name() string { } func (bp *binpackPlugin) OnSessionOpen(ssn *framework.Session) { - glog.V(4).Infof("Enter binpack plugin ...") - if glog.V(4) { + klog.V(4).Infof("Enter binpack plugin ...") + if klog.V(4) { defer func() { - glog.V(4).Infof("Leaving binpack plugin. %s ...", bp.weight.String()) + klog.V(4).Infof("Leaving binpack plugin. %s ...", bp.weight.String()) }() notFoundResource := []string{} @@ -175,19 +174,19 @@ func (bp *binpackPlugin) OnSessionOpen(ssn *framework.Session) { notFoundResource = append(notFoundResource, string(resource)) } } - glog.V(4).Infof("resources [%s] record in weight but not found on any node", strings.Join(notFoundResource, ", ")) + klog.V(4).Infof("resources [%s] record in weight but not found on any node", strings.Join(notFoundResource, ", ")) } nodeOrderFn := func(task *api.TaskInfo, node *api.NodeInfo) (float64, error) { binPackingScore := BinPackingScore(task, node, bp.weight) - glog.V(4).Infof("Binpack score for Task %s/%s on node %s is: %v", task.Namespace, task.Name, node.Name, binPackingScore) + klog.V(4).Infof("Binpack score for Task %s/%s on node %s is: %v", task.Namespace, task.Name, node.Name, binPackingScore) return binPackingScore, nil } if bp.weight.BinPackingWeight != 0 { ssn.AddNodeOrderFn(bp.Name(), nodeOrderFn) } else { - glog.Infof("binpack weight is zero, skip node order function") + klog.Infof("binpack weight is zero, skip node order function") } } @@ -230,7 +229,7 @@ func BinPackingScore(task *api.TaskInfo, node *api.NodeInfo, weight priorityWeig } resourceScore := ResourceBinPackingScore(request, allocate, nodeUsed, resourceWeight) - glog.V(5).Infof("task %s/%s on node %s resource %s, need %f, used %f, allocatable %f, weight %d, score %f", task.Namespace, task.Name, node.Name, resource, request, nodeUsed, allocate, resourceWeight, resourceScore) + klog.V(5).Infof("task %s/%s on node %s resource %s, need %f, used %f, allocatable %f, weight %d, score %f", task.Namespace, task.Name, node.Name, resource, request, nodeUsed, allocate, resourceWeight, resourceScore) score += resourceScore weightSum += resourceWeight diff --git a/pkg/scheduler/plugins/drf/drf.go b/pkg/scheduler/plugins/drf/drf.go index e7adc0d4d4..cd96157cd7 100644 --- a/pkg/scheduler/plugins/drf/drf.go +++ b/pkg/scheduler/plugins/drf/drf.go @@ -19,7 +19,7 @@ package drf import ( "math" - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api/helpers" @@ -193,7 +193,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { } } - glog.V(4).Infof("Victims from DRF plugins are %+v", victims) + klog.V(4).Infof("Victims from DRF plugins are %+v", victims) return victims } @@ -204,7 +204,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { lv := l.(*api.JobInfo) rv := r.(*api.JobInfo) - glog.V(4).Infof("DRF JobOrderFn: <%v/%v> share state: %v, <%v/%v> share state: %v", + klog.V(4).Infof("DRF JobOrderFn: <%v/%v> share state: %v, <%v/%v> share state: %v", lv.Namespace, lv.Name, drf.jobAttrs[lv.UID].share, rv.Namespace, rv.Name, drf.jobAttrs[rv.UID].share) if drf.jobAttrs[lv.UID].share == drf.jobAttrs[rv.UID].share { @@ -230,7 +230,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { lWeight := ssn.NamespaceInfo[lv].GetWeight() rWeight := ssn.NamespaceInfo[rv].GetWeight() - glog.V(4).Infof("DRF NamespaceOrderFn: <%v> share state: %f, weight %v, <%v> share state: %f, weight %v", + klog.V(4).Infof("DRF NamespaceOrderFn: <%v> share state: %f, weight %v, <%v> share state: %f, weight %v", lv, lOpt.share, lWeight, rv, rOpt.share, rWeight) lWeightedShare := lOpt.share / float64(lWeight) @@ -268,7 +268,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { nsShare = nsOpt.share } - glog.V(4).Infof("DRF AllocateFunc: task <%v/%v>, resreq <%v>, share <%v>, namespace share <%v>", + klog.V(4).Infof("DRF AllocateFunc: task <%v/%v>, resreq <%v>, share <%v>, namespace share <%v>", event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share, nsShare) }, DeallocateFunc: func(event *framework.Event) { @@ -286,7 +286,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { nsShare = nsOpt.share } - glog.V(4).Infof("DRF EvictFunc: task <%v/%v>, resreq <%v>, share <%v>, namespace share <%v>", + klog.V(4).Infof("DRF EvictFunc: task <%v/%v>, resreq <%v>, share <%v>, namespace share <%v>", event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share, nsShare) }, }) diff --git a/pkg/scheduler/plugins/gang/gang.go b/pkg/scheduler/plugins/gang/gang.go index 36e180f035..d27634d9d6 100644 --- a/pkg/scheduler/plugins/gang/gang.go +++ b/pkg/scheduler/plugins/gang/gang.go @@ -19,10 +19,9 @@ package gang import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/apis/scheduling/v1alpha1" @@ -82,14 +81,14 @@ func (gp *gangPlugin) OnSessionOpen(ssn *framework.Session) { preemptable := job.MinAvailable <= occupid-1 || job.MinAvailable == 1 if !preemptable { - glog.V(3).Infof("Can not preempt task <%v/%v> because of gang-scheduling", + klog.V(3).Infof("Can not preempt task <%v/%v> because of gang-scheduling", preemptee.Namespace, preemptee.Name) } else { victims = append(victims, preemptee) } } - glog.V(3).Infof("Victims from Gang plugins are %+v", victims) + klog.V(3).Infof("Victims from Gang plugins are %+v", victims) return victims } @@ -105,7 +104,7 @@ func (gp *gangPlugin) OnSessionOpen(ssn *framework.Session) { lReady := lv.Ready() rReady := rv.Ready() - glog.V(4).Infof("Gang JobOrderFn: <%v/%v> is ready: %t, <%v/%v> is ready: %t", + klog.V(4).Infof("Gang JobOrderFn: <%v/%v> is ready: %t, <%v/%v> is ready: %t", lv.Namespace, lv.Name, lReady, rv.Namespace, rv.Name, rReady) if lReady && rReady { @@ -158,7 +157,7 @@ func (gp *gangPlugin) OnSessionClose(ssn *framework.Session) { } if err := ssn.UpdateJobCondition(job, jc); err != nil { - glog.Errorf("Failed to update job <%s/%s> condition: %v", + klog.Errorf("Failed to update job <%s/%s> condition: %v", job.Namespace, job.Name, err) } diff --git a/pkg/scheduler/plugins/nodeorder/nodeorder.go b/pkg/scheduler/plugins/nodeorder/nodeorder.go index 5cda0a8865..c5f0603f95 100644 --- a/pkg/scheduler/plugins/nodeorder/nodeorder.go +++ b/pkg/scheduler/plugins/nodeorder/nodeorder.go @@ -19,9 +19,8 @@ package nodeorder import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "k8s.io/kubernetes/pkg/scheduler/cache" @@ -138,10 +137,10 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] if !found { - glog.Warningf("node order, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + klog.Warningf("node order, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) } else { node.AddPod(pod) - glog.V(4).Infof("node order, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + klog.V(4).Infof("node order, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) } }, DeallocateFunc: func(event *framework.Event) { @@ -150,10 +149,10 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] if !found { - glog.Warningf("node order, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + klog.Warningf("node order, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) } else { node.RemovePod(pod) - glog.V(4).Infof("node order, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) + klog.V(4).Infof("node order, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) } }, }) @@ -163,7 +162,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { if !found { nodeInfo = cache.NewNodeInfo(node.Pods()...) nodeInfo.SetNode(node.Node) - glog.Warningf("node order, generate node info for %s at NodeOrderFn is unexpected", node.Name) + klog.Warningf("node order, generate node info for %s at NodeOrderFn is unexpected", node.Name) } var score = 0.0 @@ -172,7 +171,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { host, err := priorities.LeastRequestedPriorityMap(task.Pod, nil, nodeInfo) if err != nil { - glog.Warningf("Least Requested Priority Failed because of Error: %v", err) + klog.Warningf("Least Requested Priority Failed because of Error: %v", err) return 0, err } // If leastReqWeight in provided, host.Score is multiplied with weight, if not, host.Score is added to total score. @@ -180,7 +179,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { host, err = priorities.BalancedResourceAllocationMap(task.Pod, nil, nodeInfo) if err != nil { - glog.Warningf("Balanced Resource Allocation Priority Failed because of Error: %v", err) + klog.Warningf("Balanced Resource Allocation Priority Failed because of Error: %v", err) return 0, err } // If balancedRescourceWeight in provided, host.Score is multiplied with weight, if not, host.Score is added to total score. @@ -188,13 +187,13 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { host, err = priorities.CalculateNodeAffinityPriorityMap(task.Pod, nil, nodeInfo) if err != nil { - glog.Warningf("Calculate Node Affinity Priority Failed because of Error: %v", err) + klog.Warningf("Calculate Node Affinity Priority Failed because of Error: %v", err) return 0, err } // If nodeAffinityWeight in provided, host.Score is multiplied with weight, if not, host.Score is added to total score. score = score + float64(host.Score*weight.nodeAffinityWeight) - glog.V(4).Infof("Total Score for task %s/%s on node %s is: %f", task.Namespace, task.Name, node.Name, score) + klog.V(4).Infof("Total Score for task %s/%s on node %s is: %f", task.Namespace, task.Name, node.Name, score) return score, nil } ssn.AddNodeOrderFn(pp.Name(), nodeOrderFn) @@ -205,7 +204,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { mapFn := priorities.NewInterPodAffinityPriority(cn, nl, pl, v1.DefaultHardPodAffinitySymmetricWeight) interPodAffinityScore, err := mapFn(task.Pod, nodeMap, nodeSlice) if err != nil { - glog.Warningf("Calculate Inter Pod Affinity Priority Failed because of Error: %v", err) + klog.Warningf("Calculate Inter Pod Affinity Priority Failed because of Error: %v", err) return nil, err } @@ -214,7 +213,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { score[host.Host] = float64(host.Score) * float64(weight.podAffinityWeight) } - glog.V(4).Infof("Batch Total Score for task %s/%s is: %v", task.Namespace, task.Name, score) + klog.V(4).Infof("Batch Total Score for task %s/%s is: %v", task.Namespace, task.Name, score) return score, nil } ssn.AddBatchNodeOrderFn(pp.Name(), batchNodeOrderFn) diff --git a/pkg/scheduler/plugins/predicates/predicates.go b/pkg/scheduler/plugins/predicates/predicates.go index 1a8503c9b6..c22a69e9ef 100644 --- a/pkg/scheduler/plugins/predicates/predicates.go +++ b/pkg/scheduler/plugins/predicates/predicates.go @@ -20,8 +20,7 @@ import ( "fmt" "strings" - "github.com/golang/glog" - + "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" "k8s.io/kubernetes/pkg/scheduler/cache" @@ -128,10 +127,10 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] if !found { - glog.Warningf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + klog.Warningf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) } else { node.AddPod(pod) - glog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) } }, DeallocateFunc: func(event *framework.Event) { @@ -140,10 +139,10 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] if !found { - glog.Warningf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + klog.Warningf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) } else { node.RemovePod(pod) - glog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) + klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) } }, }) @@ -159,11 +158,11 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { if !found { nodeInfo = cache.NewNodeInfo(node.Pods()...) nodeInfo.SetNode(node.Node) - glog.Warningf("predicates, generate node info for %s at PredicateFn is unexpected", node.Name) + klog.Warningf("predicates, generate node info for %s at PredicateFn is unexpected", node.Name) } if node.Allocatable.MaxTaskNum <= len(nodeInfo.Pods()) { - glog.V(4).Infof("NodePodNumber predicates Task <%s/%s> on Node <%s> failed", + klog.V(4).Infof("NodePodNumber predicates Task <%s/%s> on Node <%s> failed", task.Namespace, task.Name, node.Name) return api.NewFitError(task, node, api.NodePodNumberExceeded) } @@ -174,7 +173,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodeCondition predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodeCondition predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -187,7 +186,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodeUnschedulable Predicate Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodeUnschedulable Predicate Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -200,7 +199,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("NodeSelect predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("NodeSelect predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -213,7 +212,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("HostPorts predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("HostPorts predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -226,7 +225,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("Toleration/Taint predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("Toleration/Taint predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -240,7 +239,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodeMemoryPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodeMemoryPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -255,7 +254,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodeDiskPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodeDiskPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -270,7 +269,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodePIDPressurePredicate predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodePIDPressurePredicate predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -291,7 +290,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("Pod Affinity/Anti-Affinity predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("Pod Affinity/Anti-Affinity predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { diff --git a/pkg/scheduler/plugins/priority/priority.go b/pkg/scheduler/plugins/priority/priority.go index f58108e906..979a4b312a 100644 --- a/pkg/scheduler/plugins/priority/priority.go +++ b/pkg/scheduler/plugins/priority/priority.go @@ -17,7 +17,7 @@ limitations under the License. package priority import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/framework" @@ -45,7 +45,7 @@ func (pp *priorityPlugin) OnSessionOpen(ssn *framework.Session) { lv := l.(*api.TaskInfo) rv := r.(*api.TaskInfo) - glog.V(4).Infof("Priority TaskOrder: <%v/%v> priority is %v, <%v/%v> priority is %v", + klog.V(4).Infof("Priority TaskOrder: <%v/%v> priority is %v, <%v/%v> priority is %v", lv.Namespace, lv.Name, lv.Priority, rv.Namespace, rv.Name, rv.Priority) if lv.Priority == rv.Priority { @@ -66,7 +66,7 @@ func (pp *priorityPlugin) OnSessionOpen(ssn *framework.Session) { lv := l.(*api.JobInfo) rv := r.(*api.JobInfo) - glog.V(4).Infof("Priority JobOrderFn: <%v/%v> priority: %d, <%v/%v> priority: %d", + klog.V(4).Infof("Priority JobOrderFn: <%v/%v> priority: %d, <%v/%v> priority: %d", lv.Namespace, lv.Name, lv.Priority, rv.Namespace, rv.Name, rv.Priority) if lv.Priority > rv.Priority { diff --git a/pkg/scheduler/plugins/proportion/proportion.go b/pkg/scheduler/plugins/proportion/proportion.go index 78b4778f92..cca2311a9f 100644 --- a/pkg/scheduler/plugins/proportion/proportion.go +++ b/pkg/scheduler/plugins/proportion/proportion.go @@ -17,7 +17,7 @@ limitations under the License. package proportion import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api/helpers" @@ -64,11 +64,11 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { pp.totalResource.Add(n.Allocatable) } - glog.V(4).Infof("The total resource is <%v>", pp.totalResource) + klog.V(4).Infof("The total resource is <%v>", pp.totalResource) // Build attributes for Queues. for _, job := range ssn.Jobs { - glog.V(4).Infof("Considering Job <%s/%s>.", job.Namespace, job.Name) + klog.V(4).Infof("Considering Job <%s/%s>.", job.Namespace, job.Name) if _, found := pp.queueOpts[job.Queue]; !found { queue := ssn.Queues[job.Queue] @@ -82,7 +82,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { request: api.EmptyResource(), } pp.queueOpts[job.Queue] = attr - glog.V(4).Infof("Added Queue <%s> attributes.", job.Queue) + klog.V(4).Infof("Added Queue <%s> attributes.", job.Queue) } for status, tasks := range job.TaskStatusIndex { @@ -114,7 +114,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { // If no queues, break if totalWeight == 0 { - glog.V(4).Infof("Exiting when total weight is 0") + klog.V(4).Infof("Exiting when total weight is 0") break } @@ -124,7 +124,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { increasedDeserved := api.EmptyResource() decreasedDeserved := api.EmptyResource() for _, attr := range pp.queueOpts { - glog.V(4).Infof("Considering Queue <%s>: weight <%d>, total weight <%d>.", + klog.V(4).Infof("Considering Queue <%s>: weight <%d>, total weight <%d>.", attr.name, attr.weight, totalWeight) if _, found := meet[attr.queueID]; found { continue @@ -136,12 +136,12 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { if attr.request.Less(attr.deserved) { attr.deserved = helpers.Min(attr.deserved, attr.request) meet[attr.queueID] = struct{}{} - glog.V(4).Infof("queue <%s> is meet", attr.name) + klog.V(4).Infof("queue <%s> is meet", attr.name) } pp.updateShare(attr) - glog.V(4).Infof("The attributes of queue <%s> in proportion: deserved <%v>, allocate <%v>, request <%v>, share <%0.2f>", + klog.V(4).Infof("The attributes of queue <%s> in proportion: deserved <%v>, allocate <%v>, request <%v>, share <%0.2f>", attr.name, attr.deserved, attr.allocated, attr.request, attr.share) increased, decreased := attr.deserved.Diff(oldDeserved) @@ -151,7 +151,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { remaining.Sub(increasedDeserved).Add(decreasedDeserved) if remaining.IsEmpty() { - glog.V(4).Infof("Exiting when remaining is empty: <%v>", remaining) + klog.V(4).Infof("Exiting when remaining is empty: <%v>", remaining) break } } @@ -184,7 +184,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { } allocated := allocations[job.Queue] if allocated.Less(reclaimee.Resreq) { - glog.V(3).Infof("Failed to allocate resource for Task <%s/%s> in Queue <%s>, not enough resource.", + klog.V(3).Infof("Failed to allocate resource for Task <%s/%s> in Queue <%s>, not enough resource.", reclaimee.Namespace, reclaimee.Name, job.Queue) continue } @@ -204,7 +204,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { overused := !attr.allocated.LessEqual(attr.deserved) if overused { - glog.V(3).Infof("Queue <%v>: deserved <%v>, allocated <%v>, share <%v>", + klog.V(3).Infof("Queue <%v>: deserved <%v>, allocated <%v>, share <%v>", queue.Name, attr.deserved, attr.allocated, attr.share) } @@ -224,7 +224,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { pgResource := api.NewResource(*job.PodGroup.Spec.MinResources) if len(queue.Queue.Spec.Capability) == 0 { - glog.V(4).Infof("Capability of queue <%s> was not set, allow job <%s/%s> to Inqueue.", + klog.V(4).Infof("Capability of queue <%s> was not set, allow job <%s/%s> to Inqueue.", queue.Name, job.Namespace, job.Name) return true } @@ -244,7 +244,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { pp.updateShare(attr) - glog.V(4).Infof("Proportion AllocateFunc: task <%v/%v>, resreq <%v>, share <%v>", + klog.V(4).Infof("Proportion AllocateFunc: task <%v/%v>, resreq <%v>, share <%v>", event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share) }, DeallocateFunc: func(event *framework.Event) { @@ -254,7 +254,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { pp.updateShare(attr) - glog.V(4).Infof("Proportion EvictFunc: task <%v/%v>, resreq <%v>, share <%v>", + klog.V(4).Infof("Proportion EvictFunc: task <%v/%v>, resreq <%v>, share <%v>", event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share) }, }) diff --git a/pkg/scheduler/plugins/util/util.go b/pkg/scheduler/plugins/util/util.go index e82025f923..bd6c7cc354 100644 --- a/pkg/scheduler/plugins/util/util.go +++ b/pkg/scheduler/plugins/util/util.go @@ -19,10 +19,9 @@ package util import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/cache" @@ -98,7 +97,7 @@ func (pl *PodLister) GetPod(task *api.TaskInfo) *v1.Pod { if !found { // we could not write the copied pod back into cache for read only pod = pl.copyTaskPod(task) - glog.Warningf("DeepCopy for pod %s/%s at PodLister.GetPod is unexpected", pod.Namespace, pod.Name) + klog.Warningf("DeepCopy for pod %s/%s at PodLister.GetPod is unexpected", pod.Namespace, pod.Name) } return pod } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index db273e2fdc..4808f39c43 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -19,10 +19,9 @@ package scheduler import ( "time" - "github.com/golang/glog" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" + "k8s.io/klog" schedcache "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" @@ -69,9 +68,9 @@ func (pc *Scheduler) Run(stopCh <-chan struct{}) { } func (pc *Scheduler) runOnce() { - glog.V(4).Infof("Start scheduling ...") + klog.V(4).Infof("Start scheduling ...") scheduleStartTime := time.Now() - defer glog.V(4).Infof("End scheduling ...") + defer klog.V(4).Infof("End scheduling ...") defer metrics.UpdateE2eDuration(metrics.Duration(scheduleStartTime)) pc.loadSchedulerConf() @@ -93,7 +92,7 @@ func (pc *Scheduler) loadSchedulerConf() { schedConf := defaultSchedulerConf if len(pc.schedulerConf) != 0 { if schedConf, err = readSchedulerConf(pc.schedulerConf); err != nil { - glog.Errorf("Failed to read scheduler configuration '%s', using default configuration: %v", + klog.Errorf("Failed to read scheduler configuration '%s', using default configuration: %v", pc.schedulerConf, err) schedConf = defaultSchedulerConf } diff --git a/pkg/scheduler/util/assert/assert.go b/pkg/scheduler/util/assert/assert.go index 0ff84f0db0..f16d8078cb 100644 --- a/pkg/scheduler/util/assert/assert.go +++ b/pkg/scheduler/util/assert/assert.go @@ -5,7 +5,7 @@ import ( "os" "runtime/debug" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -32,7 +32,7 @@ func Assert(condition bool, message string) { if panicOnError { panic(message) } - glog.Errorf("%s, %s", message, debug.Stack()) + klog.Errorf("%s, %s", message, debug.Stack()) } // Assertf check condition, if condition is false, print message using Assert diff --git a/pkg/scheduler/util/scheduler_helper.go b/pkg/scheduler/util/scheduler_helper.go index 8d2e7122d6..5cfce5cac4 100644 --- a/pkg/scheduler/util/scheduler_helper.go +++ b/pkg/scheduler/util/scheduler_helper.go @@ -24,7 +24,7 @@ import ( "sync" "sync/atomic" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/client-go/util/workqueue" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" @@ -84,12 +84,12 @@ func PredicateNodes(task *api.TaskInfo, nodes []*api.NodeInfo, fn api.PredicateF // to make sure all nodes have the same chance of being examined across pods. node := nodes[(lastProcessedNodeIndex+index)%allNodes] atomic.AddInt32(&processedNodes, 1) - glog.V(3).Infof("Considering Task <%v/%v> on node <%v>: <%v> vs. <%v>", + klog.V(3).Infof("Considering Task <%v/%v> on node <%v>: <%v> vs. <%v>", task.Namespace, task.Name, node.Name, task.Resreq, node.Idle) // TODO (k82cn): Enable eCache for performance improvement. if err := fn(task, node); err != nil { - glog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v", + klog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v", task.Namespace, task.Name, node.Name, err) errorLock.Lock() fe.SetNodeError(node.Name, err) @@ -126,7 +126,7 @@ func PrioritizeNodes(task *api.TaskInfo, nodes []*api.NodeInfo, batchFn api.Batc node := nodes[index] mapScores, orderScore, err := mapFn(task, node) if err != nil { - glog.Errorf("Error in Calculating Priority for the node:%v", err) + klog.Errorf("Error in Calculating Priority for the node:%v", err) return } @@ -147,13 +147,13 @@ func PrioritizeNodes(task *api.TaskInfo, nodes []*api.NodeInfo, batchFn api.Batc workqueue.ParallelizeUntil(context.TODO(), 16, len(nodes), scoreNode) reduceScores, err := reduceFn(task, pluginNodeScoreMap) if err != nil { - glog.Errorf("Error in Calculating Priority for the node:%v", err) + klog.Errorf("Error in Calculating Priority for the node:%v", err) return nodeScores } batchNodeScore, err := batchFn(task, nodes) if err != nil { - glog.Errorf("Error in Calculating batch Priority for the node, err %v", err) + klog.Errorf("Error in Calculating batch Priority for the node, err %v", err) return nodeScores } diff --git a/vendor/github.com/golang/glog/LICENSE b/vendor/github.com/golang/glog/LICENSE deleted file mode 100644 index 37ec93a14f..0000000000 --- a/vendor/github.com/golang/glog/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/golang/glog/README b/vendor/github.com/golang/glog/README deleted file mode 100644 index 387b4eb689..0000000000 --- a/vendor/github.com/golang/glog/README +++ /dev/null @@ -1,44 +0,0 @@ -glog -==== - -Leveled execution logs for Go. - -This is an efficient pure Go implementation of leveled logs in the -manner of the open source C++ package - https://github.com/google/glog - -By binding methods to booleans it is possible to use the log package -without paying the expense of evaluating the arguments to the log. -Through the -vmodule flag, the package also provides fine-grained -control over logging at the file level. - -The comment from glog.go introduces the ideas: - - Package glog implements logging analogous to the Google-internal - C++ INFO/ERROR/V setup. It provides functions Info, Warning, - Error, Fatal, plus formatting variants such as Infof. It - also provides V-style logging controlled by the -v and - -vmodule=file=2 flags. - - Basic examples: - - glog.Info("Prepare to repel boarders") - - glog.Fatalf("Initialization failed: %s", err) - - See the documentation for the V function for an explanation - of these examples: - - if glog.V(2) { - glog.Info("Starting transaction...") - } - - glog.V(2).Infoln("Processed", nItems, "elements") - - -The repository contains an open source version of the log package -used inside Google. The master copy of the source lives inside -Google, not here. The code in this repo is for export only and is not itself -under development. Feature requests will be ignored. - -Send bug reports to golang-nuts@googlegroups.com. diff --git a/vendor/github.com/golang/glog/glog.go b/vendor/github.com/golang/glog/glog.go deleted file mode 100644 index 54bd7afdca..0000000000 --- a/vendor/github.com/golang/glog/glog.go +++ /dev/null @@ -1,1180 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package glog implements logging analogous to the Google-internal C++ INFO/ERROR/V setup. -// It provides functions Info, Warning, Error, Fatal, plus formatting variants such as -// Infof. It also provides V-style logging controlled by the -v and -vmodule=file=2 flags. -// -// Basic examples: -// -// glog.Info("Prepare to repel boarders") -// -// glog.Fatalf("Initialization failed: %s", err) -// -// See the documentation for the V function for an explanation of these examples: -// -// if glog.V(2) { -// glog.Info("Starting transaction...") -// } -// -// glog.V(2).Infoln("Processed", nItems, "elements") -// -// Log output is buffered and written periodically using Flush. Programs -// should call Flush before exiting to guarantee all log output is written. -// -// By default, all log statements write to files in a temporary directory. -// This package provides several flags that modify this behavior. -// As a result, flag.Parse must be called before any logging is done. -// -// -logtostderr=false -// Logs are written to standard error instead of to files. -// -alsologtostderr=false -// Logs are written to standard error as well as to files. -// -stderrthreshold=ERROR -// Log events at or above this severity are logged to standard -// error as well as to files. -// -log_dir="" -// Log files will be written to this directory instead of the -// default temporary directory. -// -// Other flags provide aids to debugging. -// -// -log_backtrace_at="" -// When set to a file and line number holding a logging statement, -// such as -// -log_backtrace_at=gopherflakes.go:234 -// a stack trace will be written to the Info log whenever execution -// hits that statement. (Unlike with -vmodule, the ".go" must be -// present.) -// -v=0 -// Enable V-leveled logging at the specified level. -// -vmodule="" -// The syntax of the argument is a comma-separated list of pattern=N, -// where pattern is a literal file name (minus the ".go" suffix) or -// "glob" pattern and N is a V level. For instance, -// -vmodule=gopher*=3 -// sets the V level to 3 in all Go files whose names begin "gopher". -// -package glog - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "io" - stdLog "log" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" -) - -// severity identifies the sort of log: info, warning etc. It also implements -// the flag.Value interface. The -stderrthreshold flag is of type severity and -// should be modified only through the flag.Value interface. The values match -// the corresponding constants in C++. -type severity int32 // sync/atomic int32 - -// These constants identify the log levels in order of increasing severity. -// A message written to a high-severity log file is also written to each -// lower-severity log file. -const ( - infoLog severity = iota - warningLog - errorLog - fatalLog - numSeverity = 4 -) - -const severityChar = "IWEF" - -var severityName = []string{ - infoLog: "INFO", - warningLog: "WARNING", - errorLog: "ERROR", - fatalLog: "FATAL", -} - -// get returns the value of the severity. -func (s *severity) get() severity { - return severity(atomic.LoadInt32((*int32)(s))) -} - -// set sets the value of the severity. -func (s *severity) set(val severity) { - atomic.StoreInt32((*int32)(s), int32(val)) -} - -// String is part of the flag.Value interface. -func (s *severity) String() string { - return strconv.FormatInt(int64(*s), 10) -} - -// Get is part of the flag.Value interface. -func (s *severity) Get() interface{} { - return *s -} - -// Set is part of the flag.Value interface. -func (s *severity) Set(value string) error { - var threshold severity - // Is it a known name? - if v, ok := severityByName(value); ok { - threshold = v - } else { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - threshold = severity(v) - } - logging.stderrThreshold.set(threshold) - return nil -} - -func severityByName(s string) (severity, bool) { - s = strings.ToUpper(s) - for i, name := range severityName { - if name == s { - return severity(i), true - } - } - return 0, false -} - -// OutputStats tracks the number of output lines and bytes written. -type OutputStats struct { - lines int64 - bytes int64 -} - -// Lines returns the number of lines written. -func (s *OutputStats) Lines() int64 { - return atomic.LoadInt64(&s.lines) -} - -// Bytes returns the number of bytes written. -func (s *OutputStats) Bytes() int64 { - return atomic.LoadInt64(&s.bytes) -} - -// Stats tracks the number of lines of output and number of bytes -// per severity level. Values must be read with atomic.LoadInt64. -var Stats struct { - Info, Warning, Error OutputStats -} - -var severityStats = [numSeverity]*OutputStats{ - infoLog: &Stats.Info, - warningLog: &Stats.Warning, - errorLog: &Stats.Error, -} - -// Level is exported because it appears in the arguments to V and is -// the type of the v flag, which can be set programmatically. -// It's a distinct type because we want to discriminate it from logType. -// Variables of type level are only changed under logging.mu. -// The -v flag is read only with atomic ops, so the state of the logging -// module is consistent. - -// Level is treated as a sync/atomic int32. - -// Level specifies a level of verbosity for V logs. *Level implements -// flag.Value; the -v flag is of type Level and should be modified -// only through the flag.Value interface. -type Level int32 - -// get returns the value of the Level. -func (l *Level) get() Level { - return Level(atomic.LoadInt32((*int32)(l))) -} - -// set sets the value of the Level. -func (l *Level) set(val Level) { - atomic.StoreInt32((*int32)(l), int32(val)) -} - -// String is part of the flag.Value interface. -func (l *Level) String() string { - return strconv.FormatInt(int64(*l), 10) -} - -// Get is part of the flag.Value interface. -func (l *Level) Get() interface{} { - return *l -} - -// Set is part of the flag.Value interface. -func (l *Level) Set(value string) error { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(Level(v), logging.vmodule.filter, false) - return nil -} - -// moduleSpec represents the setting of the -vmodule flag. -type moduleSpec struct { - filter []modulePat -} - -// modulePat contains a filter for the -vmodule flag. -// It holds a verbosity level and a file pattern to match. -type modulePat struct { - pattern string - literal bool // The pattern is a literal string - level Level -} - -// match reports whether the file matches the pattern. It uses a string -// comparison if the pattern contains no metacharacters. -func (m *modulePat) match(file string) bool { - if m.literal { - return file == m.pattern - } - match, _ := filepath.Match(m.pattern, file) - return match -} - -func (m *moduleSpec) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - var b bytes.Buffer - for i, f := range m.filter { - if i > 0 { - b.WriteRune(',') - } - fmt.Fprintf(&b, "%s=%d", f.pattern, f.level) - } - return b.String() -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported. -func (m *moduleSpec) Get() interface{} { - return nil -} - -var errVmoduleSyntax = errors.New("syntax error: expect comma-separated list of filename=N") - -// Syntax: -vmodule=recordio=2,file=1,gfs*=3 -func (m *moduleSpec) Set(value string) error { - var filter []modulePat - for _, pat := range strings.Split(value, ",") { - if len(pat) == 0 { - // Empty strings such as from a trailing comma can be ignored. - continue - } - patLev := strings.Split(pat, "=") - if len(patLev) != 2 || len(patLev[0]) == 0 || len(patLev[1]) == 0 { - return errVmoduleSyntax - } - pattern := patLev[0] - v, err := strconv.Atoi(patLev[1]) - if err != nil { - return errors.New("syntax error: expect comma-separated list of filename=N") - } - if v < 0 { - return errors.New("negative value for vmodule level") - } - if v == 0 { - continue // Ignore. It's harmless but no point in paying the overhead. - } - // TODO: check syntax of filter? - filter = append(filter, modulePat{pattern, isLiteral(pattern), Level(v)}) - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(logging.verbosity, filter, true) - return nil -} - -// isLiteral reports whether the pattern is a literal string, that is, has no metacharacters -// that require filepath.Match to be called to match the pattern. -func isLiteral(pattern string) bool { - return !strings.ContainsAny(pattern, `\*?[]`) -} - -// traceLocation represents the setting of the -log_backtrace_at flag. -type traceLocation struct { - file string - line int -} - -// isSet reports whether the trace location has been specified. -// logging.mu is held. -func (t *traceLocation) isSet() bool { - return t.line > 0 -} - -// match reports whether the specified file and line matches the trace location. -// The argument file name is the full path, not the basename specified in the flag. -// logging.mu is held. -func (t *traceLocation) match(file string, line int) bool { - if t.line != line { - return false - } - if i := strings.LastIndex(file, "/"); i >= 0 { - file = file[i+1:] - } - return t.file == file -} - -func (t *traceLocation) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - return fmt.Sprintf("%s:%d", t.file, t.line) -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported -func (t *traceLocation) Get() interface{} { - return nil -} - -var errTraceSyntax = errors.New("syntax error: expect file.go:234") - -// Syntax: -log_backtrace_at=gopherflakes.go:234 -// Note that unlike vmodule the file extension is included here. -func (t *traceLocation) Set(value string) error { - if value == "" { - // Unset. - t.line = 0 - t.file = "" - } - fields := strings.Split(value, ":") - if len(fields) != 2 { - return errTraceSyntax - } - file, line := fields[0], fields[1] - if !strings.Contains(file, ".") { - return errTraceSyntax - } - v, err := strconv.Atoi(line) - if err != nil { - return errTraceSyntax - } - if v <= 0 { - return errors.New("negative or zero value for level") - } - logging.mu.Lock() - defer logging.mu.Unlock() - t.line = v - t.file = file - return nil -} - -// flushSyncWriter is the interface satisfied by logging destinations. -type flushSyncWriter interface { - Flush() error - Sync() error - io.Writer -} - -func init() { - flag.BoolVar(&logging.toStderr, "logtostderr", false, "log to standard error instead of files") - flag.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") - flag.Var(&logging.verbosity, "v", "log level for V logs") - flag.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") - flag.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") - flag.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") - - // Default stderrThreshold is ERROR. - logging.stderrThreshold = errorLog - - logging.setVState(0, nil, false) - go logging.flushDaemon() -} - -// Flush flushes all pending log I/O. -func Flush() { - logging.lockAndFlushAll() -} - -// loggingT collects all the global state of the logging setup. -type loggingT struct { - // Boolean flags. Not handled atomically because the flag.Value interface - // does not let us avoid the =true, and that shorthand is necessary for - // compatibility. TODO: does this matter enough to fix? Seems unlikely. - toStderr bool // The -logtostderr flag. - alsoToStderr bool // The -alsologtostderr flag. - - // Level flag. Handled atomically. - stderrThreshold severity // The -stderrthreshold flag. - - // freeList is a list of byte buffers, maintained under freeListMu. - freeList *buffer - // freeListMu maintains the free list. It is separate from the main mutex - // so buffers can be grabbed and printed to without holding the main lock, - // for better parallelization. - freeListMu sync.Mutex - - // mu protects the remaining elements of this structure and is - // used to synchronize logging. - mu sync.Mutex - // file holds writer for each of the log types. - file [numSeverity]flushSyncWriter - // pcs is used in V to avoid an allocation when computing the caller's PC. - pcs [1]uintptr - // vmap is a cache of the V Level for each V() call site, identified by PC. - // It is wiped whenever the vmodule flag changes state. - vmap map[uintptr]Level - // filterLength stores the length of the vmodule filter chain. If greater - // than zero, it means vmodule is enabled. It may be read safely - // using sync.LoadInt32, but is only modified under mu. - filterLength int32 - // traceLocation is the state of the -log_backtrace_at flag. - traceLocation traceLocation - // These flags are modified only under lock, although verbosity may be fetched - // safely using atomic.LoadInt32. - vmodule moduleSpec // The state of the -vmodule flag. - verbosity Level // V logging level, the value of the -v flag/ -} - -// buffer holds a byte Buffer for reuse. The zero value is ready for use. -type buffer struct { - bytes.Buffer - tmp [64]byte // temporary byte array for creating headers. - next *buffer -} - -var logging loggingT - -// setVState sets a consistent state for V logging. -// l.mu is held. -func (l *loggingT) setVState(verbosity Level, filter []modulePat, setFilter bool) { - // Turn verbosity off so V will not fire while we are in transition. - logging.verbosity.set(0) - // Ditto for filter length. - atomic.StoreInt32(&logging.filterLength, 0) - - // Set the new filters and wipe the pc->Level map if the filter has changed. - if setFilter { - logging.vmodule.filter = filter - logging.vmap = make(map[uintptr]Level) - } - - // Things are consistent now, so enable filtering and verbosity. - // They are enabled in order opposite to that in V. - atomic.StoreInt32(&logging.filterLength, int32(len(filter))) - logging.verbosity.set(verbosity) -} - -// getBuffer returns a new, ready-to-use buffer. -func (l *loggingT) getBuffer() *buffer { - l.freeListMu.Lock() - b := l.freeList - if b != nil { - l.freeList = b.next - } - l.freeListMu.Unlock() - if b == nil { - b = new(buffer) - } else { - b.next = nil - b.Reset() - } - return b -} - -// putBuffer returns a buffer to the free list. -func (l *loggingT) putBuffer(b *buffer) { - if b.Len() >= 256 { - // Let big buffers die a natural death. - return - } - l.freeListMu.Lock() - b.next = l.freeList - l.freeList = b - l.freeListMu.Unlock() -} - -var timeNow = time.Now // Stubbed out for testing. - -/* -header formats a log header as defined by the C++ implementation. -It returns a buffer containing the formatted header and the user's file and line number. -The depth specifies how many stack frames above lives the source line to be identified in the log message. - -Log lines have this form: - Lmmdd hh:mm:ss.uuuuuu threadid file:line] msg... -where the fields are defined as follows: - L A single character, representing the log level (eg 'I' for INFO) - mm The month (zero padded; ie May is '05') - dd The day (zero padded) - hh:mm:ss.uuuuuu Time in hours, minutes and fractional seconds - threadid The space-padded thread ID as returned by GetTID() - file The file name - line The line number - msg The user-supplied message -*/ -func (l *loggingT) header(s severity, depth int) (*buffer, string, int) { - _, file, line, ok := runtime.Caller(3 + depth) - if !ok { - file = "???" - line = 1 - } else { - slash := strings.LastIndex(file, "/") - if slash >= 0 { - file = file[slash+1:] - } - } - return l.formatHeader(s, file, line), file, line -} - -// formatHeader formats a log header using the provided file name and line number. -func (l *loggingT) formatHeader(s severity, file string, line int) *buffer { - now := timeNow() - if line < 0 { - line = 0 // not a real line number, but acceptable to someDigits - } - if s > fatalLog { - s = infoLog // for safety. - } - buf := l.getBuffer() - - // Avoid Fprintf, for speed. The format is so simple that we can do it quickly by hand. - // It's worth about 3X. Fprintf is hard. - _, month, day := now.Date() - hour, minute, second := now.Clock() - // Lmmdd hh:mm:ss.uuuuuu threadid file:line] - buf.tmp[0] = severityChar[s] - buf.twoDigits(1, int(month)) - buf.twoDigits(3, day) - buf.tmp[5] = ' ' - buf.twoDigits(6, hour) - buf.tmp[8] = ':' - buf.twoDigits(9, minute) - buf.tmp[11] = ':' - buf.twoDigits(12, second) - buf.tmp[14] = '.' - buf.nDigits(6, 15, now.Nanosecond()/1000, '0') - buf.tmp[21] = ' ' - buf.nDigits(7, 22, pid, ' ') // TODO: should be TID - buf.tmp[29] = ' ' - buf.Write(buf.tmp[:30]) - buf.WriteString(file) - buf.tmp[0] = ':' - n := buf.someDigits(1, line) - buf.tmp[n+1] = ']' - buf.tmp[n+2] = ' ' - buf.Write(buf.tmp[:n+3]) - return buf -} - -// Some custom tiny helper functions to print the log header efficiently. - -const digits = "0123456789" - -// twoDigits formats a zero-prefixed two-digit integer at buf.tmp[i]. -func (buf *buffer) twoDigits(i, d int) { - buf.tmp[i+1] = digits[d%10] - d /= 10 - buf.tmp[i] = digits[d%10] -} - -// nDigits formats an n-digit integer at buf.tmp[i], -// padding with pad on the left. -// It assumes d >= 0. -func (buf *buffer) nDigits(n, i, d int, pad byte) { - j := n - 1 - for ; j >= 0 && d > 0; j-- { - buf.tmp[i+j] = digits[d%10] - d /= 10 - } - for ; j >= 0; j-- { - buf.tmp[i+j] = pad - } -} - -// someDigits formats a zero-prefixed variable-width integer at buf.tmp[i]. -func (buf *buffer) someDigits(i, d int) int { - // Print into the top, then copy down. We know there's space for at least - // a 10-digit number. - j := len(buf.tmp) - for { - j-- - buf.tmp[j] = digits[d%10] - d /= 10 - if d == 0 { - break - } - } - return copy(buf.tmp[i:], buf.tmp[j:]) -} - -func (l *loggingT) println(s severity, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintln(buf, args...) - l.output(s, buf, file, line, false) -} - -func (l *loggingT) print(s severity, args ...interface{}) { - l.printDepth(s, 1, args...) -} - -func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) { - buf, file, line := l.header(s, depth) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -func (l *loggingT) printf(s severity, format string, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintf(buf, format, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -// printWithFileLine behaves like print but uses the provided file and line number. If -// alsoLogToStderr is true, the log message always appears on standard error; it -// will also appear in the log file unless --logtostderr is set. -func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToStderr bool, args ...interface{}) { - buf := l.formatHeader(s, file, line) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, alsoToStderr) -} - -// output writes the data to the log files and releases the buffer. -func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoToStderr bool) { - l.mu.Lock() - if l.traceLocation.isSet() { - if l.traceLocation.match(file, line) { - buf.Write(stacks(false)) - } - } - data := buf.Bytes() - if !flag.Parsed() { - os.Stderr.Write([]byte("ERROR: logging before flag.Parse: ")) - os.Stderr.Write(data) - } else if l.toStderr { - os.Stderr.Write(data) - } else { - if alsoToStderr || l.alsoToStderr || s >= l.stderrThreshold.get() { - os.Stderr.Write(data) - } - if l.file[s] == nil { - if err := l.createFiles(s); err != nil { - os.Stderr.Write(data) // Make sure the message appears somewhere. - l.exit(err) - } - } - switch s { - case fatalLog: - l.file[fatalLog].Write(data) - fallthrough - case errorLog: - l.file[errorLog].Write(data) - fallthrough - case warningLog: - l.file[warningLog].Write(data) - fallthrough - case infoLog: - l.file[infoLog].Write(data) - } - } - if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - // First, make sure we see the trace for the current goroutine on standard error. - // If -logtostderr has been specified, the loop below will do that anyway - // as the first stack in the full dump. - if !l.toStderr { - os.Stderr.Write(stacks(false)) - } - // Write the stack trace for all goroutines to the files. - trace := stacks(true) - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. - } - l.putBuffer(buf) - l.mu.Unlock() - if stats := severityStats[s]; stats != nil { - atomic.AddInt64(&stats.lines, 1) - atomic.AddInt64(&stats.bytes, int64(len(data))) - } -} - -// timeoutFlush calls Flush and returns when it completes or after timeout -// elapses, whichever happens first. This is needed because the hooks invoked -// by Flush may deadlock when glog.Fatal is called from a hook that holds -// a lock. -func timeoutFlush(timeout time.Duration) { - done := make(chan bool, 1) - go func() { - Flush() // calls logging.lockAndFlushAll() - done <- true - }() - select { - case <-done: - case <-time.After(timeout): - fmt.Fprintln(os.Stderr, "glog: Flush took longer than", timeout) - } -} - -// stacks is a wrapper for runtime.Stack that attempts to recover the data for all goroutines. -func stacks(all bool) []byte { - // We don't know how big the traces are, so grow a few times if they don't fit. Start large, though. - n := 10000 - if all { - n = 100000 - } - var trace []byte - for i := 0; i < 5; i++ { - trace = make([]byte, n) - nbytes := runtime.Stack(trace, all) - if nbytes < len(trace) { - return trace[:nbytes] - } - n *= 2 - } - return trace -} - -// logExitFunc provides a simple mechanism to override the default behavior -// of exiting on error. Used in testing and to guarantee we reach a required exit -// for fatal logs. Instead, exit could be a function rather than a method but that -// would make its use clumsier. -var logExitFunc func(error) - -// exit is called if there is trouble creating or writing log files. -// It flushes the logs and exits the program; there's no point in hanging around. -// l.mu is held. -func (l *loggingT) exit(err error) { - fmt.Fprintf(os.Stderr, "log: exiting because of error: %s\n", err) - // If logExitFunc is set, we do that instead of exiting. - if logExitFunc != nil { - logExitFunc(err) - return - } - l.flushAll() - os.Exit(2) -} - -// syncBuffer joins a bufio.Writer to its underlying file, providing access to the -// file's Sync method and providing a wrapper for the Write method that provides log -// file rotation. There are conflicting methods, so the file cannot be embedded. -// l.mu is held for all its methods. -type syncBuffer struct { - logger *loggingT - *bufio.Writer - file *os.File - sev severity - nbytes uint64 // The number of bytes written to this file -} - -func (sb *syncBuffer) Sync() error { - return sb.file.Sync() -} - -func (sb *syncBuffer) Write(p []byte) (n int, err error) { - if sb.nbytes+uint64(len(p)) >= MaxSize { - if err := sb.rotateFile(time.Now()); err != nil { - sb.logger.exit(err) - } - } - n, err = sb.Writer.Write(p) - sb.nbytes += uint64(n) - if err != nil { - sb.logger.exit(err) - } - return -} - -// rotateFile closes the syncBuffer's file and starts a new one. -func (sb *syncBuffer) rotateFile(now time.Time) error { - if sb.file != nil { - sb.Flush() - sb.file.Close() - } - var err error - sb.file, _, err = create(severityName[sb.sev], now) - sb.nbytes = 0 - if err != nil { - return err - } - - sb.Writer = bufio.NewWriterSize(sb.file, bufferSize) - - // Write header. - var buf bytes.Buffer - fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05")) - fmt.Fprintf(&buf, "Running on machine: %s\n", host) - fmt.Fprintf(&buf, "Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH) - fmt.Fprintf(&buf, "Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg\n") - n, err := sb.file.Write(buf.Bytes()) - sb.nbytes += uint64(n) - return err -} - -// bufferSize sizes the buffer associated with each log file. It's large -// so that log records can accumulate without the logging thread blocking -// on disk I/O. The flushDaemon will block instead. -const bufferSize = 256 * 1024 - -// createFiles creates all the log files for severity from sev down to infoLog. -// l.mu is held. -func (l *loggingT) createFiles(sev severity) error { - now := time.Now() - // Files are created in decreasing severity order, so as soon as we find one - // has already been created, we can stop. - for s := sev; s >= infoLog && l.file[s] == nil; s-- { - sb := &syncBuffer{ - logger: l, - sev: s, - } - if err := sb.rotateFile(now); err != nil { - return err - } - l.file[s] = sb - } - return nil -} - -const flushInterval = 30 * time.Second - -// flushDaemon periodically flushes the log file buffers. -func (l *loggingT) flushDaemon() { - for _ = range time.NewTicker(flushInterval).C { - l.lockAndFlushAll() - } -} - -// lockAndFlushAll is like flushAll but locks l.mu first. -func (l *loggingT) lockAndFlushAll() { - l.mu.Lock() - l.flushAll() - l.mu.Unlock() -} - -// flushAll flushes all the logs and attempts to "sync" their data to disk. -// l.mu is held. -func (l *loggingT) flushAll() { - // Flush from fatal down, in case there's trouble flushing. - for s := fatalLog; s >= infoLog; s-- { - file := l.file[s] - if file != nil { - file.Flush() // ignore error - file.Sync() // ignore error - } - } -} - -// CopyStandardLogTo arranges for messages written to the Go "log" package's -// default logs to also appear in the Google logs for the named and lower -// severities. Subsequent changes to the standard log's default output location -// or format may break this behavior. -// -// Valid names are "INFO", "WARNING", "ERROR", and "FATAL". If the name is not -// recognized, CopyStandardLogTo panics. -func CopyStandardLogTo(name string) { - sev, ok := severityByName(name) - if !ok { - panic(fmt.Sprintf("log.CopyStandardLogTo(%q): unrecognized severity name", name)) - } - // Set a log format that captures the user's file and line: - // d.go:23: message - stdLog.SetFlags(stdLog.Lshortfile) - stdLog.SetOutput(logBridge(sev)) -} - -// logBridge provides the Write method that enables CopyStandardLogTo to connect -// Go's standard logs to the logs provided by this package. -type logBridge severity - -// Write parses the standard logging line and passes its components to the -// logger for severity(lb). -func (lb logBridge) Write(b []byte) (n int, err error) { - var ( - file = "???" - line = 1 - text string - ) - // Split "d.go:23: message" into "d.go", "23", and "message". - if parts := bytes.SplitN(b, []byte{':'}, 3); len(parts) != 3 || len(parts[0]) < 1 || len(parts[2]) < 1 { - text = fmt.Sprintf("bad log format: %s", b) - } else { - file = string(parts[0]) - text = string(parts[2][1:]) // skip leading space - line, err = strconv.Atoi(string(parts[1])) - if err != nil { - text = fmt.Sprintf("bad line number: %s", b) - line = 1 - } - } - // printWithFileLine with alsoToStderr=true, so standard log messages - // always appear on standard error. - logging.printWithFileLine(severity(lb), file, line, true, text) - return len(b), nil -} - -// setV computes and remembers the V level for a given PC -// when vmodule is enabled. -// File pattern matching takes the basename of the file, stripped -// of its .go suffix, and uses filepath.Match, which is a little more -// general than the *? matching used in C++. -// l.mu is held. -func (l *loggingT) setV(pc uintptr) Level { - fn := runtime.FuncForPC(pc) - file, _ := fn.FileLine(pc) - // The file is something like /a/b/c/d.go. We want just the d. - if strings.HasSuffix(file, ".go") { - file = file[:len(file)-3] - } - if slash := strings.LastIndex(file, "/"); slash >= 0 { - file = file[slash+1:] - } - for _, filter := range l.vmodule.filter { - if filter.match(file) { - l.vmap[pc] = filter.level - return filter.level - } - } - l.vmap[pc] = 0 - return 0 -} - -// Verbose is a boolean type that implements Infof (like Printf) etc. -// See the documentation of V for more information. -type Verbose bool - -// V reports whether verbosity at the call site is at least the requested level. -// The returned value is a boolean of type Verbose, which implements Info, Infoln -// and Infof. These methods will write to the Info log if called. -// Thus, one may write either -// if glog.V(2) { glog.Info("log this") } -// or -// glog.V(2).Info("log this") -// The second form is shorter but the first is cheaper if logging is off because it does -// not evaluate its arguments. -// -// Whether an individual call to V generates a log record depends on the setting of -// the -v and --vmodule flags; both are off by default. If the level in the call to -// V is at least the value of -v, or of -vmodule for the source file containing the -// call, the V call will log. -func V(level Level) Verbose { - // This function tries hard to be cheap unless there's work to do. - // The fast path is two atomic loads and compares. - - // Here is a cheap but safe test to see if V logging is enabled globally. - if logging.verbosity.get() >= level { - return Verbose(true) - } - - // It's off globally but it vmodule may still be set. - // Here is another cheap but safe test to see if vmodule is enabled. - if atomic.LoadInt32(&logging.filterLength) > 0 { - // Now we need a proper lock to use the logging structure. The pcs field - // is shared so we must lock before accessing it. This is fairly expensive, - // but if V logging is enabled we're slow anyway. - logging.mu.Lock() - defer logging.mu.Unlock() - if runtime.Callers(2, logging.pcs[:]) == 0 { - return Verbose(false) - } - v, ok := logging.vmap[logging.pcs[0]] - if !ok { - v = logging.setV(logging.pcs[0]) - } - return Verbose(v >= level) - } - return Verbose(false) -} - -// Info is equivalent to the global Info function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Info(args ...interface{}) { - if v { - logging.print(infoLog, args...) - } -} - -// Infoln is equivalent to the global Infoln function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infoln(args ...interface{}) { - if v { - logging.println(infoLog, args...) - } -} - -// Infof is equivalent to the global Infof function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infof(format string, args ...interface{}) { - if v { - logging.printf(infoLog, format, args...) - } -} - -// Info logs to the INFO log. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Info(args ...interface{}) { - logging.print(infoLog, args...) -} - -// InfoDepth acts as Info but uses depth to determine which call frame to log. -// InfoDepth(0, "msg") is the same as Info("msg"). -func InfoDepth(depth int, args ...interface{}) { - logging.printDepth(infoLog, depth, args...) -} - -// Infoln logs to the INFO log. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Infoln(args ...interface{}) { - logging.println(infoLog, args...) -} - -// Infof logs to the INFO log. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Infof(format string, args ...interface{}) { - logging.printf(infoLog, format, args...) -} - -// Warning logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Warning(args ...interface{}) { - logging.print(warningLog, args...) -} - -// WarningDepth acts as Warning but uses depth to determine which call frame to log. -// WarningDepth(0, "msg") is the same as Warning("msg"). -func WarningDepth(depth int, args ...interface{}) { - logging.printDepth(warningLog, depth, args...) -} - -// Warningln logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Warningln(args ...interface{}) { - logging.println(warningLog, args...) -} - -// Warningf logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Warningf(format string, args ...interface{}) { - logging.printf(warningLog, format, args...) -} - -// Error logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Error(args ...interface{}) { - logging.print(errorLog, args...) -} - -// ErrorDepth acts as Error but uses depth to determine which call frame to log. -// ErrorDepth(0, "msg") is the same as Error("msg"). -func ErrorDepth(depth int, args ...interface{}) { - logging.printDepth(errorLog, depth, args...) -} - -// Errorln logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Errorln(args ...interface{}) { - logging.println(errorLog, args...) -} - -// Errorf logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Errorf(format string, args ...interface{}) { - logging.printf(errorLog, format, args...) -} - -// Fatal logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Fatal(args ...interface{}) { - logging.print(fatalLog, args...) -} - -// FatalDepth acts as Fatal but uses depth to determine which call frame to log. -// FatalDepth(0, "msg") is the same as Fatal("msg"). -func FatalDepth(depth int, args ...interface{}) { - logging.printDepth(fatalLog, depth, args...) -} - -// Fatalln logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Fatalln(args ...interface{}) { - logging.println(fatalLog, args...) -} - -// Fatalf logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Fatalf(format string, args ...interface{}) { - logging.printf(fatalLog, format, args...) -} - -// fatalNoStacks is non-zero if we are to exit without dumping goroutine stacks. -// It allows Exit and relatives to use the Fatal logs. -var fatalNoStacks uint32 - -// Exit logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Exit(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.print(fatalLog, args...) -} - -// ExitDepth acts as Exit but uses depth to determine which call frame to log. -// ExitDepth(0, "msg") is the same as Exit("msg"). -func ExitDepth(depth int, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printDepth(fatalLog, depth, args...) -} - -// Exitln logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -func Exitln(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.println(fatalLog, args...) -} - -// Exitf logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Exitf(format string, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printf(fatalLog, format, args...) -} diff --git a/vendor/github.com/golang/glog/glog_file.go b/vendor/github.com/golang/glog/glog_file.go deleted file mode 100644 index 65075d2811..0000000000 --- a/vendor/github.com/golang/glog/glog_file.go +++ /dev/null @@ -1,124 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// File I/O for logs. - -package glog - -import ( - "errors" - "flag" - "fmt" - "os" - "os/user" - "path/filepath" - "strings" - "sync" - "time" -) - -// MaxSize is the maximum size of a log file in bytes. -var MaxSize uint64 = 1024 * 1024 * 1800 - -// logDirs lists the candidate directories for new log files. -var logDirs []string - -// If non-empty, overrides the choice of directory in which to write logs. -// See createLogDirs for the full list of possible destinations. -var logDir = flag.String("log_dir", "", "If non-empty, write log files in this directory") - -func createLogDirs() { - if *logDir != "" { - logDirs = append(logDirs, *logDir) - } - logDirs = append(logDirs, os.TempDir()) -} - -var ( - pid = os.Getpid() - program = filepath.Base(os.Args[0]) - host = "unknownhost" - userName = "unknownuser" -) - -func init() { - h, err := os.Hostname() - if err == nil { - host = shortHostname(h) - } - - current, err := user.Current() - if err == nil { - userName = current.Username - } - - // Sanitize userName since it may contain filepath separators on Windows. - userName = strings.Replace(userName, `\`, "_", -1) -} - -// shortHostname returns its argument, truncating at the first period. -// For instance, given "www.google.com" it returns "www". -func shortHostname(hostname string) string { - if i := strings.Index(hostname, "."); i >= 0 { - return hostname[:i] - } - return hostname -} - -// logName returns a new log file name containing tag, with start time t, and -// the name for the symlink for tag. -func logName(tag string, t time.Time) (name, link string) { - name = fmt.Sprintf("%s.%s.%s.log.%s.%04d%02d%02d-%02d%02d%02d.%d", - program, - host, - userName, - tag, - t.Year(), - t.Month(), - t.Day(), - t.Hour(), - t.Minute(), - t.Second(), - pid) - return name, program + "." + tag -} - -var onceLogDirs sync.Once - -// create creates a new log file and returns the file and its filename, which -// contains tag ("INFO", "FATAL", etc.) and t. If the file is created -// successfully, create also attempts to update the symlink for that tag, ignoring -// errors. -func create(tag string, t time.Time) (f *os.File, filename string, err error) { - onceLogDirs.Do(createLogDirs) - if len(logDirs) == 0 { - return nil, "", errors.New("log: no log dirs") - } - name, link := logName(tag, t) - var lastErr error - for _, dir := range logDirs { - fname := filepath.Join(dir, name) - f, err := os.Create(fname) - if err == nil { - symlink := filepath.Join(dir, link) - os.Remove(symlink) // ignore err - os.Symlink(name, symlink) // ignore err - return f, fname, nil - } - lastErr = err - } - return nil, "", fmt.Errorf("log: cannot create log: %v", lastErr) -} From 85939b9a82ba616216a75cf4bd38ce00d93f2233 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Wed, 27 Nov 2019 17:02:32 +0800 Subject: [PATCH 12/34] Fixed build error. Signed-off-by: Klaus Ma --- pkg/controllers/job/plugins/svc/svc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/job/plugins/svc/svc.go b/pkg/controllers/job/plugins/svc/svc.go index 63b484cea6..54c79a47a8 100644 --- a/pkg/controllers/job/plugins/svc/svc.go +++ b/pkg/controllers/job/plugins/svc/svc.go @@ -209,7 +209,7 @@ func (sp *servicePlugin) createNetworkPolicyIfNotExist(job *batch.Job) error { // If network policy does not exist, create one for Job. if _, err := sp.Clientset.KubeClients.NetworkingV1().NetworkPolicies(job.Namespace).Get(job.Name, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get NetworkPolicy for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get NetworkPolicy for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -244,7 +244,7 @@ func (sp *servicePlugin) createNetworkPolicyIfNotExist(job *batch.Job) error { } if _, e := sp.Clientset.KubeClients.NetworkingV1().NetworkPolicies(job.Namespace).Create(networkpolicy); e != nil { - glog.V(3).Infof("Failed to create Service for Job <%s/%s>: %v", job.Namespace, job.Name, e) + klog.V(3).Infof("Failed to create Service for Job <%s/%s>: %v", job.Namespace, job.Name, e) return e } job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name() From 8bae01b0b6ab56d741ab3cf2089003c90d16c5f8 Mon Sep 17 00:00:00 2001 From: wuziyang Date: Wed, 27 Nov 2019 17:09:43 +0800 Subject: [PATCH 13/34] Fix wrong condition for reclaim action --- pkg/scheduler/actions/reclaim/reclaim.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/actions/reclaim/reclaim.go b/pkg/scheduler/actions/reclaim/reclaim.go index a644f1af79..9891497657 100644 --- a/pkg/scheduler/actions/reclaim/reclaim.go +++ b/pkg/scheduler/actions/reclaim/reclaim.go @@ -157,7 +157,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { for _, v := range victims { allRes.Add(v.Resreq) } - if allRes.Less(resreq) { + if !resreq.LessEqual(allRes) { klog.V(3).Infof("Not enough resource from victims on Node <%s>.", n.Name) continue } From 48850408c1a2849662615899d98091837e52267e Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Wed, 27 Nov 2019 18:17:50 +0800 Subject: [PATCH 14/34] Update admission to use pflag. Signed-off-by: Klaus Ma --- cmd/admission/app/options/options.go | 33 ++++++++++++++-------------- cmd/admission/main.go | 20 ++++++++++++++--- cmd/controllers/main.go | 5 ++++- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/cmd/admission/app/options/options.go b/cmd/admission/app/options/options.go index 5ca24216a8..98dd1bd8f0 100644 --- a/cmd/admission/app/options/options.go +++ b/cmd/admission/app/options/options.go @@ -17,9 +17,10 @@ limitations under the License. package options import ( - "flag" "fmt" + "github.com/spf13/pflag" + "k8s.io/api/admissionregistration/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,27 +58,27 @@ func NewConfig() *Config { } // AddFlags add flags -func (c *Config) AddFlags() { - flag.StringVar(&c.Master, "master", c.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - flag.StringVar(&c.Kubeconfig, "kubeconfig", c.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") - flag.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, ""+ +func (c *Config) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&c.Master, "master", c.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") + fs.StringVar(&c.Kubeconfig, "kubeconfig", c.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") + fs.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, ""+ "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ "after server cert).") - flag.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") - flag.StringVar(&c.CaCertFile, "ca-cert-file", c.CaCertFile, "File containing the x509 Certificate for HTTPS.") - flag.IntVar(&c.Port, "port", 443, "the port used by admission-controller-server.") - flag.StringVar(&c.MutateWebhookConfigName, "mutate-webhook-config-name", "", + fs.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") + fs.StringVar(&c.CaCertFile, "ca-cert-file", c.CaCertFile, "File containing the x509 Certificate for HTTPS.") + fs.IntVar(&c.Port, "port", 443, "the port used by admission-controller-server.") + fs.StringVar(&c.MutateWebhookConfigName, "mutate-webhook-config-name", "", "Name of the mutatingwebhookconfiguration resource in Kubernetes [Deprecated]: it will be generated when not specified.") - flag.StringVar(&c.MutateWebhookName, "mutate-webhook-name", "", + fs.StringVar(&c.MutateWebhookName, "mutate-webhook-name", "", "Name of the webhook entry in the webhook config. [Deprecated]: it will be generated when not specified") - flag.StringVar(&c.ValidateWebhookConfigName, "validate-webhook-config-name", "", + fs.StringVar(&c.ValidateWebhookConfigName, "validate-webhook-config-name", "", "Name of the mutatingwebhookconfiguration resource in Kubernetes. [Deprecated]: it will be generated when not specified") - flag.StringVar(&c.ValidateWebhookName, "validate-webhook-name", "", + fs.StringVar(&c.ValidateWebhookName, "validate-webhook-name", "", "Name of the webhook entry in the webhook config. [Deprecated]: it will be generated when not specified") - flag.BoolVar(&c.PrintVersion, "version", false, "Show version and quit") - flag.StringVar(&c.AdmissionServiceNamespace, "webhook-namespace", "default", "The namespace of this webhook") - flag.StringVar(&c.AdmissionServiceName, "webhook-service-name", "admission-service", "The name of this admission service") - flag.StringVar(&c.SchedulerName, "scheduler-name", defaultSchedulerName, "Volcano will handle pods whose .spec.SchedulerName is same as scheduler-name") + fs.BoolVar(&c.PrintVersion, "version", false, "Show version and quit") + fs.StringVar(&c.AdmissionServiceNamespace, "webhook-namespace", "default", "The namespace of this webhook") + fs.StringVar(&c.AdmissionServiceName, "webhook-service-name", "admission-service", "The name of this admission service") + fs.StringVar(&c.SchedulerName, "scheduler-name", defaultSchedulerName, "Volcano will handle pods whose .spec.SchedulerName is same as scheduler-name") } const ( diff --git a/cmd/admission/main.go b/cmd/admission/main.go index 60e5e5313c..37bc9e2ff4 100644 --- a/cmd/admission/main.go +++ b/cmd/admission/main.go @@ -16,14 +16,19 @@ limitations under the License. package main import ( - "flag" "io/ioutil" "net/http" "os" "os/signal" + "runtime" "strconv" "syscall" + "time" + "github.com/spf13/pflag" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/util/flag" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" @@ -41,15 +46,24 @@ func serveMutateJobs(w http.ResponseWriter, r *http.Request) { admission.Serve(w, r, admission.MutateJobs) } +var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum number of seconds between log flushes") + func main() { + runtime.GOMAXPROCS(runtime.NumCPU()) + klog.InitFlags(nil) + config := options.NewConfig() - config.AddFlags() - flag.Parse() + config.AddFlags(pflag.CommandLine) + + flag.InitFlags() if config.PrintVersion { version.PrintVersionAndExit() } + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) + defer klog.Flush() + http.HandleFunc(admission.AdmitJobPath, serveJobs) http.HandleFunc(admission.MutateJobPath, serveMutateJobs) diff --git a/cmd/controllers/main.go b/cmd/controllers/main.go index 44623c6800..20df8642e3 100644 --- a/cmd/controllers/main.go +++ b/cmd/controllers/main.go @@ -18,6 +18,7 @@ package main import ( "fmt" "os" + "runtime" "time" "github.com/spf13/pflag" @@ -34,11 +35,13 @@ import ( var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum number of seconds between log flushes") func main() { + runtime.GOMAXPROCS(runtime.NumCPU()) + klog.InitFlags(nil) + s := options.NewServerOption() s.AddFlags(pflag.CommandLine) flag.InitFlags() - klog.InitFlags(nil) if s.PrintVersion { version.PrintVersionAndExit() From a3fc8fc7f3dad9755e1b72d388224def9dbf9142 Mon Sep 17 00:00:00 2001 From: wuziyang Date: Thu, 28 Nov 2019 10:00:04 +0800 Subject: [PATCH 15/34] fix validate victims check for preempt action --- pkg/scheduler/actions/preempt/preempt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index 36e1059c7f..c0aaadf222 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -269,7 +269,7 @@ func validateVictims(victims []*api.TaskInfo, resreq *api.Resource) error { for _, v := range victims { allRes.Add(v.Resreq) } - if allRes.Less(resreq) { + if !resreq.LessEqual(allRes) { return fmt.Errorf("not enough resources") } From e657bedcd7c4408c1a6f4fca78cfe4afc6016bb5 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Mon, 2 Dec 2019 15:20:09 +0800 Subject: [PATCH 16/34] Enable networkpolicy create/get --- installer/helm/chart/volcano/templates/controllers.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installer/helm/chart/volcano/templates/controllers.yaml b/installer/helm/chart/volcano/templates/controllers.yaml index c5f18f3238..afdee63f52 100644 --- a/installer/helm/chart/volcano/templates/controllers.yaml +++ b/installer/helm/chart/volcano/templates/controllers.yaml @@ -46,6 +46,9 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create"] --- kind: ClusterRoleBinding From 3b9f2fbaf33d93dc07befedf83a6ad81d0f4f935 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Mon, 2 Dec 2019 17:15:57 +0800 Subject: [PATCH 17/34] Fix ci verify --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d6f01a608b..e9b4a45d5d 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ clean: rm -rf _output/ rm -f *.log -verify: generate-code +verify: hack/verify-gofmt.sh hack/verify-golint.sh hack/verify-gencode.sh From edae53358d6fe1d6c785525a0a8b3217779d044d Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Mon, 2 Dec 2019 17:18:06 +0800 Subject: [PATCH 18/34] gen code --- .../v1alpha2/zz_generated.conversion.go | 34 +++++++++++++++++++ .../v1alpha2/zz_generated.deepcopy.go | 16 +++++++++ pkg/apis/scheduling/zz_generated.deepcopy.go | 16 +++++++++ 3 files changed, 66 insertions(+) diff --git a/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go b/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go index f3a738d6fe..468d5a6cc8 100644 --- a/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go @@ -106,6 +106,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*QueueRequest)(nil), (*scheduling.QueueRequest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_QueueRequest_To_scheduling_QueueRequest(a.(*QueueRequest), b.(*scheduling.QueueRequest), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.QueueRequest)(nil), (*QueueRequest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_QueueRequest_To_v1alpha2_QueueRequest(a.(*scheduling.QueueRequest), b.(*QueueRequest), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*QueueSpec)(nil), (*scheduling.QueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_QueueSpec_To_scheduling_QueueSpec(a.(*QueueSpec), b.(*scheduling.QueueSpec), scope) }); err != nil { @@ -321,6 +331,30 @@ func Convert_scheduling_QueueList_To_v1alpha2_QueueList(in *scheduling.QueueList return autoConvert_scheduling_QueueList_To_v1alpha2_QueueList(in, out, s) } +func autoConvert_v1alpha2_QueueRequest_To_scheduling_QueueRequest(in *QueueRequest, out *scheduling.QueueRequest, s conversion.Scope) error { + out.Name = in.Name + out.Event = scheduling.QueueEvent(in.Event) + out.Action = scheduling.QueueAction(in.Action) + return nil +} + +// Convert_v1alpha2_QueueRequest_To_scheduling_QueueRequest is an autogenerated conversion function. +func Convert_v1alpha2_QueueRequest_To_scheduling_QueueRequest(in *QueueRequest, out *scheduling.QueueRequest, s conversion.Scope) error { + return autoConvert_v1alpha2_QueueRequest_To_scheduling_QueueRequest(in, out, s) +} + +func autoConvert_scheduling_QueueRequest_To_v1alpha2_QueueRequest(in *scheduling.QueueRequest, out *QueueRequest, s conversion.Scope) error { + out.Name = in.Name + out.Event = QueueEvent(in.Event) + out.Action = QueueAction(in.Action) + return nil +} + +// Convert_scheduling_QueueRequest_To_v1alpha2_QueueRequest is an autogenerated conversion function. +func Convert_scheduling_QueueRequest_To_v1alpha2_QueueRequest(in *scheduling.QueueRequest, out *QueueRequest, s conversion.Scope) error { + return autoConvert_scheduling_QueueRequest_To_v1alpha2_QueueRequest(in, out, s) +} + func autoConvert_v1alpha2_QueueSpec_To_scheduling_QueueSpec(in *QueueSpec, out *scheduling.QueueSpec, s conversion.Scope) error { out.Weight = in.Weight out.Capability = *(*v1.ResourceList)(unsafe.Pointer(&in.Capability)) diff --git a/pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go index a66a4de4bc..7ae3636210 100644 --- a/pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go @@ -215,6 +215,22 @@ func (in *QueueList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueRequest) DeepCopyInto(out *QueueRequest) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueRequest. +func (in *QueueRequest) DeepCopy() *QueueRequest { + if in == nil { + return nil + } + out := new(QueueRequest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QueueSpec) DeepCopyInto(out *QueueSpec) { *out = *in diff --git a/pkg/apis/scheduling/zz_generated.deepcopy.go b/pkg/apis/scheduling/zz_generated.deepcopy.go index 0410d2658e..55f65dd2cb 100644 --- a/pkg/apis/scheduling/zz_generated.deepcopy.go +++ b/pkg/apis/scheduling/zz_generated.deepcopy.go @@ -215,6 +215,22 @@ func (in *QueueList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueRequest) DeepCopyInto(out *QueueRequest) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueRequest. +func (in *QueueRequest) DeepCopy() *QueueRequest { + if in == nil { + return nil + } + out := new(QueueRequest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QueueSpec) DeepCopyInto(out *QueueSpec) { *out = *in From 3e6cdfd72cd21ed59ddb47b102f60f65daf567fe Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Mon, 2 Dec 2019 17:24:05 +0800 Subject: [PATCH 19/34] generate yaml --- installer/volcano-development.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index c842ce3699..37eb33bfbe 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -321,6 +321,9 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create"] --- kind: ClusterRoleBinding From b4315f67d41f83144c57bcf69d8c0bbb4a1ee7aa Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Mon, 2 Dec 2019 17:38:43 +0800 Subject: [PATCH 20/34] Add explict info about what todo to update generated yaml --- hack/check-generated-yaml.sh | 3 ++- hack/generate-yaml.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hack/check-generated-yaml.sh b/hack/check-generated-yaml.sh index 212e352a77..adc37d3d38 100755 --- a/hack/check-generated-yaml.sh +++ b/hack/check-generated-yaml.sh @@ -26,7 +26,8 @@ if ! diff ${VK_ROOT}/installer/volcano-development.yaml ${RELEASE_FOLDER}/volcan { echo echo "The Generated yaml is different from the one in installer/volcano-development.yaml" - echo "Suggest to sync both the files" + echo "please run 'make generate-yaml TAG=latest RELEASE_DIR=installer \ + && mv ${RELEASE_FOLDER}/volcano-latest.yaml ${VK_ROOT}/installer/volcano-development.yaml' to update" echo } >&2 false diff --git a/hack/generate-yaml.sh b/hack/generate-yaml.sh index 4c489cd715..e978c4f7ac 100755 --- a/hack/generate-yaml.sh +++ b/hack/generate-yaml.sh @@ -59,7 +59,7 @@ if [[ ! -d ${RELEASE_FOLDER} ]];then fi DEPLOYMENT_FILE=${RELEASE_FOLDER}/${YAML_FILENAME} -echo "Generating volcano yaml file into ${DEPLOYMENT_FILE}}" +echo "Generating volcano yaml file into ${DEPLOYMENT_FILE}" if [[ -f ${DEPLOYMENT_FILE} ]];then rm ${DEPLOYMENT_FILE} From 61c31eb86623ed82ea2744588923836df8952a4e Mon Sep 17 00:00:00 2001 From: Yodar Shafrir Date: Mon, 2 Dec 2019 17:47:43 +0200 Subject: [PATCH 21/34] Change statement.go unevict method to call UpdateTask instead of AddTask. This is needed in order to 'undo' the changes that were created during the call to 'evict' method (currently the node.Releasing value is wrong and will have the value as if the job was evicted, even if 'unevict' was called). When AddTask is called an error is returned because the task already exists in this node, and the Releasing value isn't set to the right value as if the job was never evicted. --- pkg/scheduler/framework/statement.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/framework/statement.go b/pkg/scheduler/framework/statement.go index d568c24241..37229119cb 100644 --- a/pkg/scheduler/framework/statement.go +++ b/pkg/scheduler/framework/statement.go @@ -98,7 +98,7 @@ func (s *Statement) unevict(reclaimee *api.TaskInfo, reason string) error { // Update task in node. if node, found := s.ssn.Nodes[reclaimee.NodeName]; found { - node.AddTask(reclaimee) + node.UpdateTask(reclaimee) } for _, eh := range s.ssn.eventHandlers { From 6655fab459708c5f7fa747da101779fb6bb2ecfa Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 4 Dec 2019 08:35:37 +0800 Subject: [PATCH 22/34] change node notfound error --- pkg/scheduler/plugins/nodeorder/nodeorder.go | 4 ++-- pkg/scheduler/plugins/util/util.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/scheduler/plugins/nodeorder/nodeorder.go b/pkg/scheduler/plugins/nodeorder/nodeorder.go index c5f0603f95..5c4d443cc8 100644 --- a/pkg/scheduler/plugins/nodeorder/nodeorder.go +++ b/pkg/scheduler/plugins/nodeorder/nodeorder.go @@ -17,7 +17,7 @@ limitations under the License. package nodeorder import ( - "fmt" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/api/core/v1" "k8s.io/klog" @@ -237,7 +237,7 @@ func (c *cachedNodeInfo) GetNodeInfo(name string) (*v1.Node, error) { } } } - return nil, fmt.Errorf("failed to find node <%s>", name) + return nil, errors.NewNotFound(v1.Resource("node"), name) } return node.Node, nil diff --git a/pkg/scheduler/plugins/util/util.go b/pkg/scheduler/plugins/util/util.go index bd6c7cc354..7c2fe823c4 100644 --- a/pkg/scheduler/plugins/util/util.go +++ b/pkg/scheduler/plugins/util/util.go @@ -17,7 +17,7 @@ limitations under the License. package util import ( - "fmt" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" @@ -204,7 +204,8 @@ type CachedNodeInfo struct { func (c *CachedNodeInfo) GetNodeInfo(name string) (*v1.Node, error) { node, found := c.Session.Nodes[name] if !found { - return nil, fmt.Errorf("failed to find node <%s>", name) + + return nil, errors.NewNotFound(v1.Resource("node"), name) } return node.Node, nil From 27648212f5c15067ebbd891fdabfe0e08a3f43dd Mon Sep 17 00:00:00 2001 From: Zhang Jinghui Date: Thu, 7 Nov 2019 00:05:25 +0800 Subject: [PATCH 23/34] add queue controller about state --- pkg/controllers/queue/queue_controller.go | 299 ++++++++++-------- .../queue/queue_controller_action.go | 167 ++++++++++ .../queue/queue_controller_handler.go | 186 +++++++++++ .../queue/queue_controller_test.go | 2 +- .../queue/queue_controller_util.go | 40 +++ pkg/controllers/queue/state/closed.go | 58 ++++ pkg/controllers/queue/state/closing.go | 68 ++++ pkg/controllers/queue/state/factory.go | 58 ++++ pkg/controllers/queue/state/open.go | 68 ++++ pkg/controllers/queue/state/unknown.go | 66 ++++ 10 files changed, 882 insertions(+), 130 deletions(-) create mode 100644 pkg/controllers/queue/queue_controller_action.go create mode 100644 pkg/controllers/queue/queue_controller_handler.go create mode 100644 pkg/controllers/queue/queue_controller_util.go create mode 100644 pkg/controllers/queue/state/closed.go create mode 100644 pkg/controllers/queue/state/closing.go create mode 100644 pkg/controllers/queue/state/factory.go create mode 100644 pkg/controllers/queue/state/open.go create mode 100644 pkg/controllers/queue/state/unknown.go diff --git a/pkg/controllers/queue/queue_controller.go b/pkg/controllers/queue/queue_controller.go index 5c8dadbb49..bcf839e71a 100644 --- a/pkg/controllers/queue/queue_controller.go +++ b/pkg/controllers/queue/queue_controller.go @@ -18,21 +18,38 @@ package queue import ( "fmt" - "reflect" "sync" + "time" - "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog" + busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1" schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" vcclientset "volcano.sh/volcano/pkg/client/clientset/versioned" + versionedscheme "volcano.sh/volcano/pkg/client/clientset/versioned/scheme" informerfactory "volcano.sh/volcano/pkg/client/informers/externalversions" + busv1alpha1informer "volcano.sh/volcano/pkg/client/informers/externalversions/bus/v1alpha1" schedulinginformer "volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1alpha2" + busv1alpha1lister "volcano.sh/volcano/pkg/client/listers/bus/v1alpha1" schedulinglister "volcano.sh/volcano/pkg/client/listers/scheduling/v1alpha2" + queuestate "volcano.sh/volcano/pkg/controllers/queue/state" +) + +const ( + // maxRetries is the number of times a queue or command will be retried before it is dropped out of the queue. + // With the current rate-limiter in use (5ms*2^(maxRetries-1)) the following numbers represent the times + // a queue or command is going to be requeued: + // 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s + maxRetries = 15 ) // Controller manages queue status. @@ -52,12 +69,24 @@ type Controller struct { pgLister schedulinglister.PodGroupLister pgSynced cache.InformerSynced + cmdInformer busv1alpha1informer.CommandInformer + cmdLister busv1alpha1lister.CommandLister + cmdSynced cache.InformerSynced + // queues that need to be updated. - queue workqueue.RateLimitingInterface + queue workqueue.RateLimitingInterface + commandQueue workqueue.RateLimitingInterface pgMutex sync.RWMutex // queue name -> podgroup namespace/name podGroups map[string]map[string]struct{} + + syncHandler func(req *schedulingv1alpha2.QueueRequest) error + syncCommandHandler func(cmd *busv1alpha1.Command) error + + enqueueQueue func(req *schedulingv1alpha2.QueueRequest) + + recorder record.EventRecorder } // NewQueueController creates a QueueController @@ -68,6 +97,11 @@ func NewQueueController( factory := informerfactory.NewSharedInformerFactory(vcClient, 0) queueInformer := factory.Scheduling().V1alpha2().Queues() pgInformer := factory.Scheduling().V1alpha2().PodGroups() + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(klog.Infof) + eventBroadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) + c := &Controller{ kubeClient: kubeClient, vcClient: vcClient, @@ -81,12 +115,17 @@ func NewQueueController( pgLister: pgInformer.Lister(), pgSynced: pgInformer.Informer().HasSynced, - queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + commandQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + podGroups: make(map[string]map[string]struct{}), + + recorder: eventBroadcaster.NewRecorder(versionedscheme.Scheme, v1.EventSource{Component: "vc-controllers"}), } queueInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addQueue, + UpdateFunc: c.updateQueue, DeleteFunc: c.deleteQueue, }) @@ -96,22 +135,58 @@ func NewQueueController( DeleteFunc: c.deletePodGroup, }) + c.cmdInformer = informerfactory.NewSharedInformerFactory(c.vcClient, 0).Bus().V1alpha1().Commands() + c.cmdInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: func(obj interface{}) bool { + switch obj.(type) { + case *busv1alpha1.Command: + cmd := obj.(*busv1alpha1.Command) + return IsQueueReference(cmd.TargetObject) + default: + return false + } + }, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: c.addCommand, + }, + }) + c.cmdLister = c.cmdInformer.Lister() + c.cmdSynced = c.cmdInformer.Informer().HasSynced + + queuestate.SyncQueue = c.syncQueue + queuestate.OpenQueue = c.openQueue + queuestate.CloseQueue = c.closeQueue + + c.syncHandler = c.handleQueue + c.syncCommandHandler = c.handleCommand + + c.enqueueQueue = c.enqueue + return c } // Run starts QueueController func (c *Controller) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + defer c.commandQueue.ShutDown() + + klog.Infof("Starting queue controller.") + defer klog.Infof("Shutting down queue controller.") go c.queueInformer.Informer().Run(stopCh) go c.pgInformer.Informer().Run(stopCh) + go c.cmdInformer.Informer().Run(stopCh) - if !cache.WaitForCacheSync(stopCh, c.queueSynced, c.pgSynced) { - klog.Errorf("unable to sync caches for queue controller") + if !cache.WaitForCacheSync(stopCh, c.queueSynced, c.pgSynced, c.cmdSynced) { + klog.Errorf("unable to sync caches for queue controller.") return } go wait.Until(c.worker, 0, stopCh) - klog.Infof("QueueController is running ...... ") + go wait.Until(c.commandWorker, 0, stopCh) + + <-stopCh } // worker runs a worker thread that just dequeues items, processes them, and @@ -124,168 +199,134 @@ func (c *Controller) worker() { } func (c *Controller) processNextWorkItem() bool { - eKey, quit := c.queue.Get() - if quit { + obj, shutdown := c.queue.Get() + if shutdown { return false } - defer c.queue.Done(eKey) + defer c.queue.Done(obj) - if err := c.syncQueue(eKey.(string)); err != nil { - klog.V(2).Infof("Error syncing queues %q, retrying. Error: %v", eKey, err) - c.queue.AddRateLimited(eKey) + req, ok := obj.(*schedulingv1alpha2.QueueRequest) + if !ok { + klog.Errorf("%v is not a valid queue request struct.", obj) return true } - c.queue.Forget(eKey) - return true -} - -func (c *Controller) getPodGroups(key string) ([]string, error) { - c.pgMutex.RLock() - defer c.pgMutex.RUnlock() + err := c.syncHandler(req) + c.handleQueueErr(err, obj) - if c.podGroups[key] == nil { - return nil, fmt.Errorf("queue %s has not been seen or deleted", key) - } - podGroups := make([]string, 0, len(c.podGroups[key])) - for pgKey := range c.podGroups[key] { - podGroups = append(podGroups, pgKey) - } - - return podGroups, nil + return true } -func (c *Controller) syncQueue(key string) error { - klog.V(4).Infof("Begin sync queue %s", key) +func (c *Controller) handleQueue(req *schedulingv1alpha2.QueueRequest) error { + startTime := time.Now() + defer func() { + klog.V(4).Infof("Finished syncing queue %s (%v).", req.Name, time.Since(startTime)) + }() - podGroups, err := c.getPodGroups(key) + queue, err := c.queueLister.Get(req.Name) if err != nil { - return err - } - - queueStatus := schedulingv1alpha2.QueueStatus{} - - for _, pgKey := range podGroups { - // Ignore error here, tt can not occur. - ns, name, _ := cache.SplitMetaNamespaceKey(pgKey) - - // TODO: check NotFound error and sync local cache. - pg, err := c.pgLister.PodGroups(ns).Get(name) - if err != nil { - return err + if apierrors.IsNotFound(err) { + klog.V(4).Infof("Queue %s has been deleted.", req.Name) + return nil } - switch pg.Status.Phase { - case schedulingv1alpha2.PodGroupPending: - queueStatus.Pending++ - case schedulingv1alpha2.PodGroupRunning: - queueStatus.Running++ - case schedulingv1alpha2.PodGroupUnknown: - queueStatus.Unknown++ - case schedulingv1alpha2.PodGroupInqueue: - queueStatus.Inqueue++ - } + return fmt.Errorf("get queue %s failed for %v", req.Name, err) } - queue, err := c.queueLister.Get(key) - if err != nil { - if errors.IsNotFound(err) { - klog.V(2).Infof("queue %s has been deleted", key) - return nil - } - // TODO: do not retry to syncQueue for this error - return err + queueState := queuestate.NewState(queue) + if queueState == nil { + return fmt.Errorf("queue %s state %s is invalid", queue.Name, queue.Status.State) } - // ignore update when status does not change - if reflect.DeepEqual(queueStatus, queue.Status) { - return nil + if err := queueState.Execute(req.Action); err != nil { + return fmt.Errorf("sync queue %s failed for %v, event is %v, action is %s", + req.Name, err, req.Event, req.Action) } - newQueue := queue.DeepCopy() - newQueue.Status = queueStatus + return nil +} + +func (c *Controller) handleQueueErr(err error, obj interface{}) { + if err == nil { + c.queue.Forget(obj) + return + } - if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { - klog.Errorf("Failed to update status of Queue %s: %v", newQueue.Name, err) - return err + if c.queue.NumRequeues(obj) < maxRetries { + klog.V(4).Infof("Error syncing queue request %v for %v.", obj, err) + c.queue.AddRateLimited(obj) + return } - return nil + req, _ := obj.(*schedulingv1alpha2.QueueRequest) + c.recordEventsForQueue(req.Name, v1.EventTypeWarning, string(req.Action), + fmt.Sprintf("%v queue failed for %v", req.Action, err)) + klog.V(2).Infof("Dropping queue request %v out of the queue for %v.", obj, err) + c.queue.Forget(obj) } -func (c *Controller) addQueue(obj interface{}) { - queue := obj.(*schedulingv1alpha2.Queue) - c.queue.Add(queue.Name) +func (c *Controller) commandWorker() { + for c.processNextCommand() { + } } -func (c *Controller) deleteQueue(obj interface{}) { - queue, ok := obj.(*schedulingv1alpha2.Queue) +func (c *Controller) processNextCommand() bool { + obj, shutdown := c.commandQueue.Get() + if shutdown { + return false + } + defer c.commandQueue.Done(obj) + + cmd, ok := obj.(*busv1alpha1.Command) if !ok { - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - klog.Errorf("Couldn't get object from tombstone %#v", obj) - return - } - queue, ok = tombstone.Obj.(*schedulingv1alpha2.Queue) - if !ok { - klog.Errorf("Tombstone contained object that is not a Queue: %#v", obj) - return - } + klog.Errorf("%v is not a valid Command struct.", obj) + return true } - c.pgMutex.Lock() - defer c.pgMutex.Unlock() - delete(c.podGroups, queue.Name) + err := c.syncCommandHandler(cmd) + c.handleCommandErr(err, obj) + + return true } -func (c *Controller) addPodGroup(obj interface{}) { - pg := obj.(*schedulingv1alpha2.PodGroup) - key, _ := cache.MetaNamespaceKeyFunc(obj) +func (c *Controller) handleCommand(cmd *busv1alpha1.Command) error { + startTime := time.Now() + defer func() { + klog.V(4).Infof("Finished syncing command %s/%s (%v).", cmd.Namespace, cmd.Name, time.Since(startTime)) + }() - c.pgMutex.Lock() - defer c.pgMutex.Unlock() + err := c.vcClient.BusV1alpha1().Commands(cmd.Namespace).Delete(cmd.Name, nil) + if err != nil { + if true == apierrors.IsNotFound(err) { + return nil + } - if c.podGroups[pg.Spec.Queue] == nil { - c.podGroups[pg.Spec.Queue] = make(map[string]struct{}) + return fmt.Errorf("failed to delete command <%s/%s> for %v", cmd.Namespace, cmd.Name, err) } - c.podGroups[pg.Spec.Queue][key] = struct{}{} - // enqueue - c.queue.Add(pg.Spec.Queue) -} + req := &schedulingv1alpha2.QueueRequest{ + Name: cmd.TargetObject.Name, + Event: schedulingv1alpha2.QueueCommandIssuedEvent, + Action: schedulingv1alpha2.QueueAction(cmd.Action), + } -func (c *Controller) updatePodGroup(old, new interface{}) { - oldPG := old.(*schedulingv1alpha2.PodGroup) - newPG := new.(*schedulingv1alpha2.PodGroup) + c.enqueueQueue(req) - // Note: we have no use case update PodGroup.Spec.Queue - // So do not consider it here. - if oldPG.Status.Phase != newPG.Status.Phase { - c.queue.Add(newPG.Spec.Queue) - } + return nil } -func (c *Controller) deletePodGroup(obj interface{}) { - pg, ok := obj.(*schedulingv1alpha2.PodGroup) - if !ok { - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - klog.Errorf("Couldn't get object from tombstone %#v", obj) - return - } - pg, ok = tombstone.Obj.(*schedulingv1alpha2.PodGroup) - if !ok { - klog.Errorf("Tombstone contained object that is not a PodGroup: %#v", obj) - return - } +func (c *Controller) handleCommandErr(err error, obj interface{}) { + if err == nil { + c.commandQueue.Forget(obj) + return } - key, _ := cache.MetaNamespaceKeyFunc(obj) - - c.pgMutex.Lock() - defer c.pgMutex.Unlock() - - delete(c.podGroups[pg.Spec.Queue], key) + if c.commandQueue.NumRequeues(obj) < maxRetries { + klog.V(4).Infof("Error syncing command %v for %v.", obj, err) + c.commandQueue.AddRateLimited(obj) + return + } - c.queue.Add(pg.Spec.Queue) + klog.V(2).Infof("Dropping command %v out of the queue for %v.", obj, err) + c.commandQueue.Forget(obj) } diff --git a/pkg/controllers/queue/queue_controller_action.go b/pkg/controllers/queue/queue_controller_action.go new file mode 100644 index 0000000000..c405f58f48 --- /dev/null +++ b/pkg/controllers/queue/queue_controller_action.go @@ -0,0 +1,167 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "fmt" + "reflect" + + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + "volcano.sh/volcano/pkg/controllers/queue/state" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + + "k8s.io/klog" +) + +func (c *Controller) syncQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error { + klog.V(4).Infof("Begin to sync queue %s.", queue.Name) + + podGroups := c.getPodGroups(queue.Name) + queueStatus := schedulingv1alpha2.QueueStatus{} + + for _, pgKey := range podGroups { + // Ignore error here, tt can not occur. + ns, name, _ := cache.SplitMetaNamespaceKey(pgKey) + + // TODO: check NotFound error and sync local cache. + pg, err := c.pgLister.PodGroups(ns).Get(name) + if err != nil { + return err + } + + switch pg.Status.Phase { + case schedulingv1alpha2.PodGroupPending: + queueStatus.Pending++ + case schedulingv1alpha2.PodGroupRunning: + queueStatus.Running++ + case schedulingv1alpha2.PodGroupUnknown: + queueStatus.Unknown++ + case schedulingv1alpha2.PodGroupInqueue: + queueStatus.Inqueue++ + } + } + + if updateStateFn != nil { + updateStateFn(&queueStatus, podGroups) + } else { + queueStatus.State = queue.Status.State + } + + // ignore update when status does not change + if reflect.DeepEqual(queueStatus, queue.Status) { + return nil + } + + newQueue := queue.DeepCopy() + newQueue.Status = queueStatus + if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { + klog.Errorf("Failed to update status of Queue %s: %v.", newQueue.Name, err) + return err + } + + return nil +} + +func (c *Controller) openQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error { + klog.V(4).Infof("Begin to open queue %s.", queue.Name) + + newQueue := queue.DeepCopy() + newQueue.Spec.State = schedulingv1alpha2.QueueStateOpen + + if queue.Spec.State != newQueue.Spec.State { + if _, err := c.vcClient.SchedulingV1alpha2().Queues().Update(newQueue); err != nil { + c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.OpenQueueAction), + fmt.Sprintf("Open queue failed for %v", err)) + return err + } + + c.recorder.Event(newQueue, v1.EventTypeNormal, string(schedulingv1alpha2.OpenQueueAction), + fmt.Sprintf("Open queue succeed")) + } else { + return nil + } + + q, err := c.vcClient.SchedulingV1alpha2().Queues().Get(newQueue.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + newQueue = q.DeepCopy() + if updateStateFn != nil { + updateStateFn(&newQueue.Status, nil) + } else { + return fmt.Errorf("internal error, update state function should be provided") + } + + if queue.Status.State != newQueue.Status.State { + if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { + c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.OpenQueueAction), + fmt.Sprintf("Update queue status from %s to %s failed for %v", + queue.Status.State, newQueue.Status.State, err)) + return err + } + } + + return nil +} + +func (c *Controller) closeQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error { + klog.V(4).Infof("Begin to close queue %s.", queue.Name) + + newQueue := queue.DeepCopy() + newQueue.Spec.State = schedulingv1alpha2.QueueStateClosed + + if queue.Spec.State != newQueue.Spec.State { + if _, err := c.vcClient.SchedulingV1alpha2().Queues().Update(newQueue); err != nil { + c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.CloseQueueAction), + fmt.Sprintf("Close queue failed for %v", err)) + return err + } + + c.recorder.Event(newQueue, v1.EventTypeNormal, string(schedulingv1alpha2.CloseQueueAction), + fmt.Sprintf("Close queue succeed")) + } else { + return nil + } + + q, err := c.vcClient.SchedulingV1alpha2().Queues().Get(newQueue.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + newQueue = q.DeepCopy() + podGroups := c.getPodGroups(newQueue.Name) + if updateStateFn != nil { + updateStateFn(&newQueue.Status, podGroups) + } else { + return fmt.Errorf("internal error, update state function should be provided") + } + + if queue.Status.State != newQueue.Status.State { + if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { + c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.CloseQueueAction), + fmt.Sprintf("Update queue status from %s to %s failed for %v", + queue.Status.State, newQueue.Status.State, err)) + return err + } + } + + return nil +} diff --git a/pkg/controllers/queue/queue_controller_handler.go b/pkg/controllers/queue/queue_controller_handler.go new file mode 100644 index 0000000000..7165f8a215 --- /dev/null +++ b/pkg/controllers/queue/queue_controller_handler.go @@ -0,0 +1,186 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1" + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + + "k8s.io/client-go/tools/cache" + + "k8s.io/klog" +) + +func (c *Controller) enqueue(req *schedulingv1alpha2.QueueRequest) { + c.queue.Add(req) +} + +func (c *Controller) addQueue(obj interface{}) { + queue := obj.(*schedulingv1alpha2.Queue) + + req := &schedulingv1alpha2.QueueRequest{ + Name: queue.Name, + + Event: schedulingv1alpha2.QueueOutOfSyncEvent, + Action: schedulingv1alpha2.SyncQueueAction, + } + + c.enqueue(req) +} + +func (c *Controller) deleteQueue(obj interface{}) { + queue, ok := obj.(*schedulingv1alpha2.Queue) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("Couldn't get object from tombstone %#v.", obj) + return + } + queue, ok = tombstone.Obj.(*schedulingv1alpha2.Queue) + if !ok { + klog.Errorf("Tombstone contained object that is not a Queue: %#v.", obj) + return + } + } + + c.pgMutex.Lock() + defer c.pgMutex.Unlock() + delete(c.podGroups, queue.Name) +} + +func (c *Controller) updateQueue(old, new interface{}) { + oldQueue, ok := old.(*schedulingv1alpha2.Queue) + if !ok { + klog.Errorf("Can not covert old object %v to queues.scheduling.sigs.dev.", old) + return + } + + newQueue, ok := new.(*schedulingv1alpha2.Queue) + if !ok { + klog.Errorf("Can not covert new object %v to queues.scheduling.sigs.dev.", old) + return + } + + if oldQueue.ResourceVersion == newQueue.ResourceVersion { + return + } + + c.addQueue(newQueue) + + return +} + +func (c *Controller) addPodGroup(obj interface{}) { + pg := obj.(*schedulingv1alpha2.PodGroup) + key, _ := cache.MetaNamespaceKeyFunc(obj) + + c.pgMutex.Lock() + defer c.pgMutex.Unlock() + + if c.podGroups[pg.Spec.Queue] == nil { + c.podGroups[pg.Spec.Queue] = make(map[string]struct{}) + } + c.podGroups[pg.Spec.Queue][key] = struct{}{} + + req := &schedulingv1alpha2.QueueRequest{ + Name: pg.Spec.Queue, + + Event: schedulingv1alpha2.QueueOutOfSyncEvent, + Action: schedulingv1alpha2.SyncQueueAction, + } + + c.enqueue(req) +} + +func (c *Controller) updatePodGroup(old, new interface{}) { + oldPG := old.(*schedulingv1alpha2.PodGroup) + newPG := new.(*schedulingv1alpha2.PodGroup) + + // Note: we have no use case update PodGroup.Spec.Queue + // So do not consider it here. + if oldPG.Status.Phase != newPG.Status.Phase { + c.addPodGroup(newPG) + } +} + +func (c *Controller) deletePodGroup(obj interface{}) { + pg, ok := obj.(*schedulingv1alpha2.PodGroup) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("Couldn't get object from tombstone %#v.", obj) + return + } + pg, ok = tombstone.Obj.(*schedulingv1alpha2.PodGroup) + if !ok { + klog.Errorf("Tombstone contained object that is not a PodGroup: %#v.", obj) + return + } + } + + key, _ := cache.MetaNamespaceKeyFunc(obj) + + c.pgMutex.Lock() + defer c.pgMutex.Unlock() + + delete(c.podGroups[pg.Spec.Queue], key) + + req := &schedulingv1alpha2.QueueRequest{ + Name: pg.Spec.Queue, + + Event: schedulingv1alpha2.QueueOutOfSyncEvent, + Action: schedulingv1alpha2.SyncQueueAction, + } + + c.enqueue(req) +} + +func (c *Controller) addCommand(obj interface{}) { + cmd, ok := obj.(*busv1alpha1.Command) + if !ok { + klog.Errorf("Obj %v is not command.", obj) + return + } + + c.commandQueue.Add(cmd) +} + +func (c *Controller) getPodGroups(key string) []string { + c.pgMutex.RLock() + defer c.pgMutex.RUnlock() + + if c.podGroups[key] == nil { + return nil + } + podGroups := make([]string, 0, len(c.podGroups[key])) + for pgKey := range c.podGroups[key] { + podGroups = append(podGroups, pgKey) + } + + return podGroups +} + +func (c *Controller) recordEventsForQueue(name, eventType, reason, message string) { + queue, err := c.queueLister.Get(name) + if err != nil { + klog.Errorf("Get queue %s failed for %v.", name, err) + return + } + + c.recorder.Event(queue, eventType, reason, message) + return +} diff --git a/pkg/controllers/queue/queue_controller_test.go b/pkg/controllers/queue/queue_controller_test.go index e1e7806a61..9c4a068456 100644 --- a/pkg/controllers/queue/queue_controller_test.go +++ b/pkg/controllers/queue/queue_controller_test.go @@ -271,7 +271,7 @@ func TestSyncQueue(t *testing.T) { c.queueInformer.Informer().GetIndexer().Add(testcase.queue) c.vcClient.SchedulingV1alpha2().Queues().Create(testcase.queue) - err := c.syncQueue(testcase.queue.Name) + err := c.syncQueue(testcase.queue, nil) item, _ := c.vcClient.SchedulingV1alpha2().Queues().Get(testcase.queue.Name, metav1.GetOptions{}) if err != nil && testcase.ExpectValue != item.Status.Pending { t.Errorf("case %d (%s): expected: %v, got %v ", i, testcase.Name, testcase.ExpectValue, c.queue.Len()) diff --git a/pkg/controllers/queue/queue_controller_util.go b/pkg/controllers/queue/queue_controller_util.go new file mode 100644 index 0000000000..28ce7a261b --- /dev/null +++ b/pkg/controllers/queue/queue_controller_util.go @@ -0,0 +1,40 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IsQueueReference return if ownerReference is Queue Kind +func IsQueueReference(ref *metav1.OwnerReference) bool { + if ref == nil { + return false + } + + if ref.APIVersion != schedulingv1alpha2.SchemeGroupVersion.String() { + return false + } + + if ref.Kind != "Queue" { + return false + } + + return true +} diff --git a/pkg/controllers/queue/state/closed.go b/pkg/controllers/queue/state/closed.go new file mode 100644 index 0000000000..91e402d87c --- /dev/null +++ b/pkg/controllers/queue/state/closed.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +type closedState struct { + queue *v1alpha2.Queue +} + +func (cs *closedState) Execute(action v1alpha2.QueueAction) error { + switch action { + case v1alpha2.OpenQueueAction: + return OpenQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateOpen + return + }) + case v1alpha2.CloseQueueAction: + return SyncQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateClosed + return + }) + default: + return SyncQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + specState := cs.queue.Spec.State + if specState == v1alpha2.QueueStateOpen { + status.State = v1alpha2.QueueStateOpen + return + } + + if specState == v1alpha2.QueueStateClosed { + status.State = v1alpha2.QueueStateClosed + return + } + + status.State = v1alpha2.QueueStateUnknown + return + }) + } + + return nil +} diff --git a/pkg/controllers/queue/state/closing.go b/pkg/controllers/queue/state/closing.go new file mode 100644 index 0000000000..afcbdd6b37 --- /dev/null +++ b/pkg/controllers/queue/state/closing.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +type closingState struct { + queue *v1alpha2.Queue +} + +func (cs *closingState) Execute(action v1alpha2.QueueAction) error { + switch action { + case v1alpha2.OpenQueueAction: + return OpenQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateOpen + return + }) + case v1alpha2.CloseQueueAction: + return SyncQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + }) + default: + return SyncQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + specState := cs.queue.Spec.State + if specState == v1alpha2.QueueStateOpen { + status.State = v1alpha2.QueueStateOpen + return + } + + if specState == v1alpha2.QueueStateClosed { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + + status.State = v1alpha2.QueueStateClosing + return + } + + status.State = v1alpha2.QueueStateUnknown + return + }) + } + + return nil +} diff --git a/pkg/controllers/queue/state/factory.go b/pkg/controllers/queue/state/factory.go new file mode 100644 index 0000000000..07212a075e --- /dev/null +++ b/pkg/controllers/queue/state/factory.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +// State interface +type State interface { + // Execute executes the actions based on current state. + Execute(action v1alpha2.QueueAction) error +} + +// UpdateQueueStatusFn updates the queue status +type UpdateQueueStatusFn func(status *v1alpha2.QueueStatus, podGroupList []string) + +// QueueActionFn will open, close or sync queue. +type QueueActionFn func(queue *v1alpha2.Queue, fn UpdateQueueStatusFn) error + +var ( + // SyncQueue will sync queue status. + SyncQueue QueueActionFn + // OpenQueue will set state of queue to open + OpenQueue QueueActionFn + // CloseQueue will set state of queue to close + CloseQueue QueueActionFn +) + +// NewState gets the state from queue status +func NewState(queue *v1alpha2.Queue) State { + switch queue.Status.State { + case "", v1alpha2.QueueStateOpen: + return &openState{queue: queue} + case v1alpha2.QueueStateClosed: + return &closedState{queue: queue} + case v1alpha2.QueueStateClosing: + return &closingState{queue: queue} + case v1alpha2.QueueStateUnknown: + return &unknownState{queue: queue} + } + + return nil +} diff --git a/pkg/controllers/queue/state/open.go b/pkg/controllers/queue/state/open.go new file mode 100644 index 0000000000..4d7ad898f9 --- /dev/null +++ b/pkg/controllers/queue/state/open.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +type openState struct { + queue *v1alpha2.Queue +} + +func (os *openState) Execute(action v1alpha2.QueueAction) error { + switch action { + case v1alpha2.OpenQueueAction: + return SyncQueue(os.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateOpen + return + }) + case v1alpha2.CloseQueueAction: + return CloseQueue(os.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + }) + default: + return SyncQueue(os.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + specState := os.queue.Spec.State + if len(specState) == 0 || specState == v1alpha2.QueueStateOpen { + status.State = v1alpha2.QueueStateOpen + return + } + + if specState == v1alpha2.QueueStateClosed { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + } + + status.State = v1alpha2.QueueStateUnknown + return + }) + } + + return nil +} diff --git a/pkg/controllers/queue/state/unknown.go b/pkg/controllers/queue/state/unknown.go new file mode 100644 index 0000000000..89bf6bebd9 --- /dev/null +++ b/pkg/controllers/queue/state/unknown.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +type unknownState struct { + queue *v1alpha2.Queue +} + +func (us *unknownState) Execute(action v1alpha2.QueueAction) error { + switch action { + case v1alpha2.OpenQueueAction: + return OpenQueue(us.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateOpen + return + }) + case v1alpha2.CloseQueueAction: + return CloseQueue(us.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + }) + default: + return SyncQueue(us.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + specState := us.queue.Spec.State + if specState == v1alpha2.QueueStateOpen { + status.State = v1alpha2.QueueStateOpen + return + } + + if specState == v1alpha2.QueueStateClosed { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + } + + status.State = v1alpha2.QueueStateUnknown + return + }) + } +} From 1bc41aef9776d026be0e4f1b84aa0f4f9fe0a697 Mon Sep 17 00:00:00 2001 From: Zhang Jinghui Date: Thu, 5 Dec 2019 20:38:08 +0800 Subject: [PATCH 24/34] dep ensure --- Gopkg.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Gopkg.lock b/Gopkg.lock index fa702ea903..7e24d1e1ef 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1039,6 +1039,7 @@ "k8s.io/api/apps/v1", "k8s.io/api/batch/v1", "k8s.io/api/core/v1", + "k8s.io/api/networking/v1", "k8s.io/api/policy/v1beta1", "k8s.io/api/scheduling/v1beta1", "k8s.io/apimachinery/pkg/api/errors", @@ -1091,6 +1092,7 @@ "k8s.io/code-generator/cmd/defaulter-gen", "k8s.io/code-generator/cmd/informer-gen", "k8s.io/code-generator/cmd/lister-gen", + "k8s.io/klog", "k8s.io/kubernetes/pkg/api/v1/pod", "k8s.io/kubernetes/pkg/apis/core", "k8s.io/kubernetes/pkg/apis/core/v1", From b167ec8226b6f9dd66bb67467ceb90b4e5a53a19 Mon Sep 17 00:00:00 2001 From: Mateusz Date: Mon, 2 Dec 2019 16:10:33 -0800 Subject: [PATCH 25/34] use future idle resources when checking if task can fit node Future idle resources (current idle resources + resources being released) are considered for pipelining during allocate and preempt action. Fixes read-write race condition in FakeEvictor Fixes a bug when task was used as preemptor on the first iteration of the preempt loop despite task's job being "pipelined". Fixes issue #582 --- pkg/scheduler/actions/allocate/allocate.go | 10 +- pkg/scheduler/actions/preempt/preempt.go | 67 +++-- pkg/scheduler/actions/preempt/preempt_test.go | 246 ++++++++++++------ pkg/scheduler/actions/reclaim/reclaim_test.go | 5 +- pkg/scheduler/api/node_info.go | 42 ++- pkg/scheduler/api/node_info_test.go | 3 + pkg/scheduler/util/test_utils.go | 11 +- 7 files changed, 252 insertions(+), 132 deletions(-) diff --git a/pkg/scheduler/actions/allocate/allocate.go b/pkg/scheduler/actions/allocate/allocate.go index 5277f67b62..3e2fe6a4ec 100644 --- a/pkg/scheduler/actions/allocate/allocate.go +++ b/pkg/scheduler/actions/allocate/allocate.go @@ -99,13 +99,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { predicateFn := func(task *api.TaskInfo, node *api.NodeInfo) error { // Check for Resource Predicate - // TODO: We could not allocate resource to task from both node.Idle and node.Releasing now, - // after it is done, we could change the following compare to: - // clonedNode := node.Idle.Clone() - // if !task.InitResreq.LessEqual(clonedNode.Add(node.Releasing)) { - // ... - // } - if !task.InitResreq.LessEqual(node.Idle) && !task.InitResreq.LessEqual(node.Releasing) { + if !task.InitResreq.LessEqual(node.FutureIdle()) { return api.NewFitError(task, node, api.NodeResourceFitFailed) } @@ -219,7 +213,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { task.Namespace, task.Name, node.Name) // Allocate releasing resource to the task if any. - if task.InitResreq.LessEqual(node.Releasing) { + if task.InitResreq.LessEqual(node.FutureIdle()) { klog.V(3).Infof("Pipelining Task <%v/%v> to node <%v> for <%v> on <%v>", task.Namespace, task.Name, node.Name, task.InitResreq, node.Releasing) if err := stmt.Pipeline(task, node.Name); err != nil { diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index c0aaadf222..e61ee6b48f 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -69,7 +69,7 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { queues[queue.UID] = queue } - if len(job.TaskStatusIndex[api.Pending]) != 0 { + if len(job.TaskStatusIndex[api.Pending]) != 0 && !ssn.JobPipelined(job) { if _, found := preemptorsMap[job.Queue]; !found { preemptorsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn) } @@ -98,6 +98,11 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { stmt := ssn.Statement() assigned := false for { + // If job is pipelined, then stop preempting. + if ssn.JobPipelined(preemptorJob) { + break + } + // If not preemptor tasks, next job. if preemptorTasks[preemptorJob.UID].Empty() { klog.V(3).Infof("No preemptor task in job <%s/%s>.", @@ -107,7 +112,7 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { preemptor := preemptorTasks[preemptorJob.UID].Pop().(*api.TaskInfo) - if preempted, _ := preempt(ssn, stmt, preemptor, ssn.Nodes, func(task *api.TaskInfo) bool { + if preempted, _ := preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool { // Ignore non running task. if task.Status != api.Running { return false @@ -122,16 +127,12 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { }); preempted { assigned = true } - - // If job is not pipelined, keep preempting - if ssn.JobPipelined(preemptorJob) { - stmt.Commit() - break - } } - // If job is not pipelined after try all tasks, next job. - if !ssn.JobPipelined(preemptorJob) { + // Commit changes only if job is pipelined, otherwise try next job. + if ssn.JobPipelined(preemptorJob) { + stmt.Commit() + } else { stmt.Discard() continue } @@ -155,7 +156,7 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { preemptor := preemptorTasks[job.UID].Pop().(*api.TaskInfo) stmt := ssn.Statement() - assigned, _ := preempt(ssn, stmt, preemptor, ssn.Nodes, func(task *api.TaskInfo) bool { + assigned, _ := preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool { // Ignore non running task. if task.Status != api.Running { return false @@ -181,12 +182,11 @@ func preempt( ssn *framework.Session, stmt *framework.Statement, preemptor *api.TaskInfo, - nodes map[string]*api.NodeInfo, filter func(*api.TaskInfo) bool, ) (bool, error) { assigned := false - allNodes := util.GetNodeList(nodes) + allNodes := util.GetNodeList(ssn.Nodes) predicateNodes, _ := util.PredicateNodes(preemptor, allNodes, ssn.PredicateFn) @@ -198,9 +198,6 @@ func preempt( preemptor.Namespace, preemptor.Name, node.Name) var preemptees []*api.TaskInfo - preempted := api.EmptyResource() - resreq := preemptor.InitResreq.Clone() - for _, task := range node.Tasks { if filter == nil { preemptees = append(preemptees, task.Clone()) @@ -211,7 +208,7 @@ func preempt( victims := ssn.Preemptable(preemptor, preemptees) metrics.UpdatePreemptionVictimsCount(len(victims)) - if err := validateVictims(victims, resreq); err != nil { + if err := validateVictims(preemptor, node, victims); err != nil { klog.V(3).Infof("No validated victims on Node <%s>: %v", node.Name, err) continue } @@ -223,9 +220,15 @@ func preempt( victimsQueue.Push(victim) } // Preempt victims for tasks, pick lowest priority task first. + preempted := api.EmptyResource() + for !victimsQueue.Empty() { + // If reclaimed enough resources, break loop to avoid Sub panic. + if preemptor.InitResreq.LessEqual(node.FutureIdle()) { + break + } preemptee := victimsQueue.Pop().(*api.TaskInfo) - klog.Errorf("Try to preempt Task <%s/%s> for Tasks <%s/%s>", + klog.V(3).Infof("Try to preempt Task <%s/%s> for Tasks <%s/%s>", preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name) if err := stmt.Evict(preemptee, "preempt"); err != nil { klog.Errorf("Failed to preempt Task <%s/%s> for Tasks <%s/%s>: %v", @@ -233,19 +236,15 @@ func preempt( continue } preempted.Add(preemptee.Resreq) - // If reclaimed enough resources, break loop to avoid Sub panic. - if resreq.LessEqual(preempted) { - break - } } metrics.RegisterPreemptionAttempts() - klog.V(3).Infof("Preempted <%v> for task <%s/%s> requested <%v>.", + klog.V(3).Infof("Preempted <%v> for Task <%s/%s> requested <%v>.", preempted, preemptor.Namespace, preemptor.Name, preemptor.InitResreq) - if preemptor.InitResreq.LessEqual(preempted) { + if preemptor.InitResreq.LessEqual(node.FutureIdle()) { if err := stmt.Pipeline(preemptor, node.Name); err != nil { - klog.Errorf("Failed to pipline Task <%s/%s> on Node <%s>", + klog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>", preemptor.Namespace, preemptor.Name, node.Name) } @@ -259,19 +258,19 @@ func preempt( return assigned, nil } -func validateVictims(victims []*api.TaskInfo, resreq *api.Resource) error { +func validateVictims(preemptor *api.TaskInfo, node *api.NodeInfo, victims []*api.TaskInfo) error { if len(victims) == 0 { return fmt.Errorf("no victims") } - - // If not enough resource, continue - allRes := api.EmptyResource() - for _, v := range victims { - allRes.Add(v.Resreq) + futureIdle := node.FutureIdle() + for _, victim := range victims { + futureIdle.Add(victim.Resreq) } - if !resreq.LessEqual(allRes) { - return fmt.Errorf("not enough resources") + // Every resource of the preemptor needs to be less or equal than corresponding + // idle resource after preemption. + if !preemptor.InitResreq.LessEqual(futureIdle) { + return fmt.Errorf("not enough resources: requested <%v>, but future idle <%v>", + preemptor.InitResreq, futureIdle) } - return nil } diff --git a/pkg/scheduler/actions/preempt/preempt_test.go b/pkg/scheduler/actions/preempt/preempt_test.go index 9a46c675f8..99c02023e4 100644 --- a/pkg/scheduler/actions/preempt/preempt_test.go +++ b/pkg/scheduler/actions/preempt/preempt_test.go @@ -54,7 +54,7 @@ func TestPreempt(t *testing.T) { expected int }{ { - name: "one Job with two Pods on one node", + name: "do not preempt if there are enough idle resources", podGroups: []*schedulingv2.PodGroup{ { ObjectMeta: metav1.ObjectMeta{ @@ -62,7 +62,8 @@ func TestPreempt(t *testing.T) { Namespace: "c1", }, Spec: schedulingv2.PodGroupSpec{ - Queue: "q1", + MinMember: 3, + Queue: "q1", }, }, }, @@ -70,10 +71,10 @@ func TestPreempt(t *testing.T) { util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), util.BuildPod("c1", "preemptor1", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), - util.BuildPod("c1", "preemptor2", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), }, + // If there are enough idle resources on the node, then there is no need to preempt anything. nodes: []*v1.Node{ - util.BuildNode("n1", util.BuildResourceList("3", "3Gi"), make(map[string]string)), + util.BuildNode("n1", util.BuildResourceList("10", "10G"), make(map[string]string)), }, queues: []*schedulingv2.Queue{ { @@ -85,10 +86,10 @@ func TestPreempt(t *testing.T) { }, }, }, - expected: 1, + expected: 0, }, { - name: "two Jobs on one node", + name: "do not preempt if job is pipelined", podGroups: []*schedulingv2.PodGroup{ { ObjectMeta: metav1.ObjectMeta{ @@ -96,7 +97,8 @@ func TestPreempt(t *testing.T) { Namespace: "c1", }, Spec: schedulingv2.PodGroupSpec{ - Queue: "q1", + MinMember: 1, + Queue: "q1", }, }, { @@ -105,19 +107,63 @@ func TestPreempt(t *testing.T) { Namespace: "c1", }, Spec: schedulingv2.PodGroupSpec{ - Queue: "q1", + MinMember: 1, + Queue: "q1", + }, + }, + }, + // Both pg1 and pg2 jobs are pipelined, because enough pods are already running. + pods: []*v1.Pod{ + util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptee3", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptor2", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string)), + }, + // All resources on the node will be in use. + nodes: []*v1.Node{ + util.BuildNode("n1", util.BuildResourceList("3", "3G"), make(map[string]string)), + }, + queues: []*schedulingv2.Queue{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "q1", + }, + Spec: schedulingv2.QueueSpec{ + Weight: 1, + }, + }, + }, + expected: 0, + }, + { + name: "preempt one task of different job to fit both jobs on one node", + podGroups: []*schedulingv2.PodGroup{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pg1", + Namespace: "c1", + }, + Spec: schedulingv2.PodGroupSpec{ + MinMember: 1, + Queue: "q1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pg2", + Namespace: "c1", + }, + Spec: schedulingv2.PodGroupSpec{ + MinMember: 1, + Queue: "q1", }, }, }, pods: []*v1.Pod{ - // running pod with pg1, under c1 util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), - // running pod with pg1, under c1 util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), - // pending pod with pg2, under c1 util.BuildPod("c1", "preemptor1", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string)), - // pending pod with pg2, under c1 util.BuildPod("c1", "preemptor2", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string)), }, nodes: []*v1.Node{ @@ -133,76 +179,130 @@ func TestPreempt(t *testing.T) { }, }, }, + expected: 1, + }, + { + name: "preempt enough tasks to fit large task of different job", + podGroups: []*schedulingv2.PodGroup{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pg1", + Namespace: "c1", + }, + Spec: schedulingv2.PodGroupSpec{ + MinMember: 1, + Queue: "q1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pg2", + Namespace: "c1", + }, + Spec: schedulingv2.PodGroupSpec{ + MinMember: 1, + Queue: "q1", + }, + }, + }, + // There are 3 cpus and 3G of memory idle and 3 tasks running each consuming 1 cpu and 1G of memory. + // Big task requiring 5 cpus and 5G of memory should preempt 2 of 3 running tasks to fit into the node. + pods: []*v1.Pod{ + util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptee3", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptor1", "", v1.PodPending, util.BuildResourceList("5", "5G"), "pg2", make(map[string]string), make(map[string]string)), + }, + nodes: []*v1.Node{ + util.BuildNode("n1", util.BuildResourceList("6", "6G"), make(map[string]string)), + }, + queues: []*schedulingv2.Queue{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "q1", + }, + Spec: schedulingv2.QueueSpec{ + Weight: 1, + }, + }, + }, expected: 2, }, } - allocate := New() - - for i, test := range tests { - binder := &util.FakeBinder{ - Binds: map[string]string{}, - Channel: make(chan string), - } - evictor := &util.FakeEvictor{ - Evicts: make([]string, 0), - Channel: make(chan string), - } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - Evictor: evictor, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - - Recorder: record.NewFakeRecorder(100), - } - for _, node := range test.nodes { - schedulerCache.AddNode(node) - } - for _, pod := range test.pods { - schedulerCache.AddPod(pod) - } - - for _, ss := range test.podGroups { - schedulerCache.AddPodGroupV1alpha2(ss) - } - - for _, q := range test.queues { - schedulerCache.AddQueueV1alpha2(q) - } - - trueValue := true - ssn := framework.OpenSession(schedulerCache, []conf.Tier{ - { - Plugins: []conf.PluginOption{ - { - Name: "conformance", - EnabledPreemptable: &trueValue, - }, - { - Name: "gang", - EnabledPreemptable: &trueValue, + preempt := New() + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + binder := &util.FakeBinder{ + Binds: map[string]string{}, + Channel: make(chan string), + } + evictor := &util.FakeEvictor{ + Channel: make(chan string), + } + schedulerCache := &cache.SchedulerCache{ + Nodes: make(map[string]*api.NodeInfo), + Jobs: make(map[api.JobID]*api.JobInfo), + Queues: make(map[api.QueueID]*api.QueueInfo), + Binder: binder, + Evictor: evictor, + StatusUpdater: &util.FakeStatusUpdater{}, + VolumeBinder: &util.FakeVolumeBinder{}, + + Recorder: record.NewFakeRecorder(100), + } + for _, node := range test.nodes { + schedulerCache.AddNode(node) + } + for _, pod := range test.pods { + schedulerCache.AddPod(pod) + } + + for _, ss := range test.podGroups { + schedulerCache.AddPodGroupV1alpha2(ss) + } + + for _, q := range test.queues { + schedulerCache.AddQueueV1alpha2(q) + } + + trueValue := true + ssn := framework.OpenSession(schedulerCache, []conf.Tier{ + { + Plugins: []conf.PluginOption{ + { + Name: "conformance", + EnabledPreemptable: &trueValue, + }, + { + Name: "gang", + EnabledPreemptable: &trueValue, + EnabledJobPipelined: &trueValue, + }, }, }, - }, - }) - defer framework.CloseSession(ssn) + }) + defer framework.CloseSession(ssn) - allocate.Execute(ssn) + preempt.Execute(ssn) - for i := 0; i < test.expected; i++ { + for i := 0; i < test.expected; i++ { + select { + case <-evictor.Channel: + case <-time.After(time.Second): + t.Errorf("not enough evictions") + } + } select { - case <-evictor.Channel: - case <-time.After(3 * time.Second): - t.Errorf("Failed to get evicting request.") + case key, opened := <-evictor.Channel: + if opened { + t.Errorf("unexpected eviction: %s", key) + } + case <-time.After(50 * time.Millisecond): + // TODO: Active waiting here is not optimal, but there is no better way currently. + // Ideally we would like to wait for evict and bind request goroutines to finish first. } - } - - if test.expected != len(evictor.Evicts) { - t.Errorf("case %d (%s): expected: %v, got %v ", i, test.name, test.expected, len(evictor.Evicts)) - } + }) } } diff --git a/pkg/scheduler/actions/reclaim/reclaim_test.go b/pkg/scheduler/actions/reclaim/reclaim_test.go index 784526173a..ac63dda3fc 100644 --- a/pkg/scheduler/actions/reclaim/reclaim_test.go +++ b/pkg/scheduler/actions/reclaim/reclaim_test.go @@ -108,7 +108,6 @@ func TestReclaim(t *testing.T) { Channel: make(chan string), } evictor := &util.FakeEvictor{ - Evicts: make([]string, 0), Channel: make(chan string), } schedulerCache := &cache.SchedulerCache{ @@ -164,8 +163,8 @@ func TestReclaim(t *testing.T) { } } - if test.expected != len(evictor.Evicts) { - t.Errorf("case %d (%s): expected: %v, got %v ", i, test.name, test.expected, len(evictor.Evicts)) + if test.expected != len(evictor.Evicts()) { + t.Errorf("case %d (%s): expected: %v, got %v ", i, test.name, test.expected, len(evictor.Evicts())) } } } diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index a62ce98dbd..df97598154 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -33,6 +33,8 @@ type NodeInfo struct { // The releasing resource on that node Releasing *Resource + // The pipelined resource on that node + Pipelined *Resource // The idle resource on that node Idle *Resource // The used resource on that node, including running and terminating @@ -48,6 +50,13 @@ type NodeInfo struct { Others map[string]interface{} } +// FutureIdle returns resources that will be idle in the future: +// +// That is current idle resources plus released resources minus pipelined resources. +func (ni *NodeInfo) FutureIdle() *Resource { + return ni.Idle.Clone().Add(ni.Releasing).Sub(ni.Pipelined) +} + // NodeState defines the current state of node. type NodeState struct { Phase NodePhase @@ -61,6 +70,7 @@ func NewNodeInfo(node *v1.Node) *NodeInfo { if node == nil { ni = &NodeInfo{ Releasing: EmptyResource(), + Pipelined: EmptyResource(), Idle: EmptyResource(), Used: EmptyResource(), @@ -75,6 +85,7 @@ func NewNodeInfo(node *v1.Node) *NodeInfo { Node: node, Releasing: EmptyResource(), + Pipelined: EmptyResource(), Idle: NewResource(node.Status.Allocatable), Used: EmptyResource(), @@ -158,16 +169,23 @@ func (ni *NodeInfo) SetNode(node *v1.Node) { ni.Allocatable = NewResource(node.Status.Allocatable) ni.Capability = NewResource(node.Status.Capacity) + ni.Releasing = EmptyResource() + ni.Pipelined = EmptyResource() ni.Idle = NewResource(node.Status.Allocatable) ni.Used = EmptyResource() - for _, task := range ni.Tasks { - if task.Status == Releasing { - ni.Releasing.Add(task.Resreq) + for _, ti := range ni.Tasks { + switch ti.Status { + case Releasing: + ni.Idle.Sub(ti.Resreq) + ni.Releasing.Add(ti.Resreq) + ni.Used.Add(ti.Resreq) + case Pipelined: + ni.Pipelined.Add(ti.Resreq) + default: + ni.Idle.Sub(ti.Resreq) + ni.Used.Add(ti.Resreq) } - - ni.Idle.Sub(task.Resreq) - ni.Used.Add(task.Resreq) } } @@ -202,15 +220,15 @@ func (ni *NodeInfo) AddTask(task *TaskInfo) error { return err } ni.Releasing.Add(ti.Resreq) + ni.Used.Add(ti.Resreq) case Pipelined: - ni.Releasing.Sub(ti.Resreq) + ni.Pipelined.Add(ti.Resreq) default: if err := ni.allocateIdleResource(ti); err != nil { return err } + ni.Used.Add(ti.Resreq) } - - ni.Used.Add(ti.Resreq) } ni.Tasks[key] = ti @@ -233,13 +251,13 @@ func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error { case Releasing: ni.Releasing.Sub(task.Resreq) ni.Idle.Add(task.Resreq) + ni.Used.Sub(task.Resreq) case Pipelined: - ni.Releasing.Add(task.Resreq) + ni.Pipelined.Sub(task.Resreq) default: ni.Idle.Add(task.Resreq) + ni.Used.Sub(task.Resreq) } - - ni.Used.Sub(task.Resreq) } delete(ni.Tasks, key) diff --git a/pkg/scheduler/api/node_info_test.go b/pkg/scheduler/api/node_info_test.go index 4ccbfb9985..6de73c8b32 100644 --- a/pkg/scheduler/api/node_info_test.go +++ b/pkg/scheduler/api/node_info_test.go @@ -57,6 +57,7 @@ func TestNodeInfo_AddPod(t *testing.T) { Idle: buildResource("5000m", "7G"), Used: buildResource("3000m", "3G"), Releasing: EmptyResource(), + Pipelined: EmptyResource(), Allocatable: buildResource("8000m", "10G"), Capability: buildResource("8000m", "10G"), State: NodeState{Phase: Ready}, @@ -76,6 +77,7 @@ func TestNodeInfo_AddPod(t *testing.T) { Idle: buildResource("2000m", "1G"), Used: EmptyResource(), Releasing: EmptyResource(), + Pipelined: EmptyResource(), Allocatable: buildResource("2000m", "1G"), Capability: buildResource("2000m", "1G"), State: NodeState{Phase: NotReady, Reason: "OutOfSync"}, @@ -124,6 +126,7 @@ func TestNodeInfo_RemovePod(t *testing.T) { Idle: buildResource("4000m", "6G"), Used: buildResource("4000m", "4G"), Releasing: EmptyResource(), + Pipelined: EmptyResource(), Allocatable: buildResource("8000m", "10G"), Capability: buildResource("8000m", "10G"), State: NodeState{Phase: Ready}, diff --git a/pkg/scheduler/util/test_utils.go b/pkg/scheduler/util/test_utils.go index 9384c3f46a..6a14189d77 100644 --- a/pkg/scheduler/util/test_utils.go +++ b/pkg/scheduler/util/test_utils.go @@ -114,10 +114,17 @@ func (fb *FakeBinder) Bind(p *v1.Pod, hostname string) error { // FakeEvictor is used as fake evictor type FakeEvictor struct { sync.Mutex - Evicts []string + evicts []string Channel chan string } +// Evicts returns copy of evicted pods. +func (fe *FakeEvictor) Evicts() []string { + fe.Lock() + defer fe.Unlock() + return append([]string{}, fe.evicts...) +} + // Evict is used by fake evictor to evict pods func (fe *FakeEvictor) Evict(p *v1.Pod) error { fe.Lock() @@ -125,7 +132,7 @@ func (fe *FakeEvictor) Evict(p *v1.Pod) error { fmt.Println("PodName: ", p.Name) key := fmt.Sprintf("%v/%v", p.Namespace, p.Name) - fe.Evicts = append(fe.Evicts, key) + fe.evicts = append(fe.evicts, key) fe.Channel <- key From cbb0276b026e32583e1d25a5f1eae9ad512cb5bd Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Fri, 6 Dec 2019 15:48:36 +0800 Subject: [PATCH 26/34] Upgrade helm to v3.0.1 --- hack/lib/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/lib/install.sh b/hack/lib/install.sh index 13418acd84..dd95f0e008 100644 --- a/hack/lib/install.sh +++ b/hack/lib/install.sh @@ -60,7 +60,7 @@ function install-helm { HELM_TEMP_DIR=`mktemp -d` curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > ${HELM_TEMP_DIR}/get_helm.sh #TODO: There are some issue with helm's latest version, remove '--version' when it get fixed. - chmod 700 ${HELM_TEMP_DIR}/get_helm.sh && ${HELM_TEMP_DIR}/get_helm.sh --version v2.13.0 + chmod 700 ${HELM_TEMP_DIR}/get_helm.sh && ${HELM_TEMP_DIR}/get_helm.sh --version v3.0.1 else echo -n "found helm, version: " && helm version fi From 55f10312f9aa8e56b10c41f830e0fd6912c8293e Mon Sep 17 00:00:00 2001 From: Zhang Jinghui Date: Thu, 5 Dec 2019 22:15:23 +0800 Subject: [PATCH 27/34] add arguments for action --- .../actions/allocate/allocate_test.go | 2 +- pkg/scheduler/actions/enqueue/enqueue.go | 24 +++- pkg/scheduler/actions/enqueue/enqueue_test.go | 53 +++++++++ pkg/scheduler/actions/preempt/preempt_test.go | 2 +- pkg/scheduler/actions/reclaim/reclaim_test.go | 2 +- pkg/scheduler/conf/scheduler_conf.go | 10 ++ pkg/scheduler/framework/arguments.go | 37 +++++- pkg/scheduler/framework/arguments_test.go | 111 ++++++++++++++++++ pkg/scheduler/framework/framework.go | 3 +- pkg/scheduler/framework/session.go | 5 +- pkg/scheduler/plugins/binpack/binpack_test.go | 2 +- pkg/scheduler/scheduler.go | 5 +- pkg/scheduler/util.go | 8 +- pkg/scheduler/util_test.go | 21 +++- 14 files changed, 267 insertions(+), 18 deletions(-) create mode 100644 pkg/scheduler/actions/enqueue/enqueue_test.go diff --git a/pkg/scheduler/actions/allocate/allocate_test.go b/pkg/scheduler/actions/allocate/allocate_test.go index 63c173ef55..0a74be5e5e 100644 --- a/pkg/scheduler/actions/allocate/allocate_test.go +++ b/pkg/scheduler/actions/allocate/allocate_test.go @@ -202,7 +202,7 @@ func TestAllocate(t *testing.T) { }, }, }, - }) + }, nil) defer framework.CloseSession(ssn) allocate.Execute(ssn) diff --git a/pkg/scheduler/actions/enqueue/enqueue.go b/pkg/scheduler/actions/enqueue/enqueue.go index 11c7765a79..8008f5ba47 100644 --- a/pkg/scheduler/actions/enqueue/enqueue.go +++ b/pkg/scheduler/actions/enqueue/enqueue.go @@ -25,6 +25,18 @@ import ( "volcano.sh/volcano/pkg/scheduler/util" ) +const ( + // overCommitFactor is resource overCommit factor for enqueue action + // It determines the number of `pending` pods that the scheduler will tolerate + // when the resources of the cluster is insufficient + overCommitFactor = "overcommit-factor" +) + +var ( + // defaultOverCommitFactor defines the default overCommit resource factor for enqueue action + defaultOverCommitFactor = 1.2 +) + type enqueueAction struct { ssn *framework.Session } @@ -77,7 +89,7 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { emptyRes := api.EmptyResource() nodesIdleRes := api.EmptyResource() for _, node := range ssn.Nodes { - nodesIdleRes.Add(node.Allocatable.Clone().Multi(1.2).Sub(node.Used)) + nodesIdleRes.Add(node.Allocatable.Clone().Multi(enqueue.getOverCommitFactor(ssn)).Sub(node.Used)) } for { @@ -122,3 +134,13 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { } func (enqueue *enqueueAction) UnInitialize() {} + +func (enqueue *enqueueAction) getOverCommitFactor(ssn *framework.Session) float64 { + factor := defaultOverCommitFactor + arg := framework.GetArgOfActionFromConf(ssn.Configurations, enqueue.Name()) + if arg != nil { + arg.GetFloat64(&factor, overCommitFactor) + } + + return factor +} diff --git a/pkg/scheduler/actions/enqueue/enqueue_test.go b/pkg/scheduler/actions/enqueue/enqueue_test.go new file mode 100644 index 0000000000..5c3787e192 --- /dev/null +++ b/pkg/scheduler/actions/enqueue/enqueue_test.go @@ -0,0 +1,53 @@ +package enqueue + +import ( + "testing" + + "volcano.sh/volcano/pkg/scheduler/conf" + "volcano.sh/volcano/pkg/scheduler/framework" +) + +func TestGetOverCommitFactor(t *testing.T) { + cases := []struct { + name string + ssn *framework.Session + expectedValue float64 + }{ + { + name: "arguments of action not exist", + ssn: &framework.Session{ + Configurations: []conf.Configuration{ + { + Name: "allocate", + Arguments: map[string]string{ + "placeholder": "placeholder", + }, + }, + }, + }, + expectedValue: 1.2, + }, + { + name: "arguments of action exist", + ssn: &framework.Session{ + Configurations: []conf.Configuration{ + { + Name: "enqueue", + Arguments: map[string]string{ + overCommitFactor: "2", + }, + }, + }, + }, + expectedValue: 2, + }, + } + + enqueue := New() + for index, c := range cases { + factor := enqueue.getOverCommitFactor(c.ssn) + if factor != c.expectedValue { + t.Errorf("index %d, case %s, expected %v, but got %v", index, c.name, c.expectedValue, factor) + } + } +} diff --git a/pkg/scheduler/actions/preempt/preempt_test.go b/pkg/scheduler/actions/preempt/preempt_test.go index 99c02023e4..f8be6703d8 100644 --- a/pkg/scheduler/actions/preempt/preempt_test.go +++ b/pkg/scheduler/actions/preempt/preempt_test.go @@ -282,7 +282,7 @@ func TestPreempt(t *testing.T) { }, }, }, - }) + }, nil) defer framework.CloseSession(ssn) preempt.Execute(ssn) diff --git a/pkg/scheduler/actions/reclaim/reclaim_test.go b/pkg/scheduler/actions/reclaim/reclaim_test.go index ac63dda3fc..1d32721f05 100644 --- a/pkg/scheduler/actions/reclaim/reclaim_test.go +++ b/pkg/scheduler/actions/reclaim/reclaim_test.go @@ -150,7 +150,7 @@ func TestReclaim(t *testing.T) { }, }, }, - }) + }, nil) defer framework.CloseSession(ssn) reclaim.Execute(ssn) diff --git a/pkg/scheduler/conf/scheduler_conf.go b/pkg/scheduler/conf/scheduler_conf.go index a45c6e4697..55378fd77f 100644 --- a/pkg/scheduler/conf/scheduler_conf.go +++ b/pkg/scheduler/conf/scheduler_conf.go @@ -22,6 +22,8 @@ type SchedulerConfiguration struct { Actions string `yaml:"actions"` // Tiers defines plugins in different tiers Tiers []Tier `yaml:"tiers"` + // Configurations is configuration for actions + Configurations []Configuration `yaml:"configurations"` } // Tier defines plugin tier @@ -29,6 +31,14 @@ type Tier struct { Plugins []PluginOption `yaml:"plugins"` } +// Configuration is configuration of action +type Configuration struct { + // Name is name of action + Name string `yaml:"name"` + // Arguments defines the different arguments that can be given to specified action + Arguments map[string]string `yaml:"arguments"` +} + // PluginOption defines the options of plugin type PluginOption struct { // The name of Plugin diff --git a/pkg/scheduler/framework/arguments.go b/pkg/scheduler/framework/arguments.go index b94462e4e2..be31cc7805 100644 --- a/pkg/scheduler/framework/arguments.go +++ b/pkg/scheduler/framework/arguments.go @@ -19,13 +19,15 @@ package framework import ( "strconv" + "volcano.sh/volcano/pkg/scheduler/conf" + "k8s.io/klog" ) // Arguments map type Arguments map[string]string -//GetInt get the integer value from string +// GetInt get the integer value from string func (a Arguments) GetInt(ptr *int, key string) { if ptr == nil { return @@ -45,7 +47,27 @@ func (a Arguments) GetInt(ptr *int, key string) { *ptr = value } -//GetBool get the bool value from string +// GetFloat64 get the float64 value from string +func (a Arguments) GetFloat64(ptr *float64, key string) { + if ptr == nil { + return + } + + argv, ok := a[key] + if !ok || len(argv) == 0 { + return + } + + value, err := strconv.ParseFloat(argv, 64) + if err != nil { + klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) + return + } + + *ptr = value +} + +// GetBool get the bool value from string func (a Arguments) GetBool(ptr *bool, key string) { if ptr == nil { return @@ -64,3 +86,14 @@ func (a Arguments) GetBool(ptr *bool, key string) { *ptr = value } + +// GetArgOfActionFromConf return argument of action reading from configuration of schedule +func GetArgOfActionFromConf(configurations []conf.Configuration, actionName string) Arguments { + for _, c := range configurations { + if c.Name == actionName { + return c.Arguments + } + } + + return nil +} diff --git a/pkg/scheduler/framework/arguments_test.go b/pkg/scheduler/framework/arguments_test.go index f9d5f30a03..44559363df 100644 --- a/pkg/scheduler/framework/arguments_test.go +++ b/pkg/scheduler/framework/arguments_test.go @@ -17,7 +17,10 @@ limitations under the License. package framework import ( + "reflect" "testing" + + "volcano.sh/volcano/pkg/scheduler/conf" ) type GetIntTestCases struct { @@ -74,3 +77,111 @@ func TestArgumentsGetInt(t *testing.T) { } } } + +func TestArgumentsGetFloat64(t *testing.T) { + key1 := "float64key" + + cases := []struct { + name string + arg Arguments + key string + baseValue float64 + expectValue float64 + }{ + { + name: "key not exist", + arg: Arguments{ + "anotherKey": "12", + }, + key: key1, + baseValue: 1.2, + expectValue: 1.2, + }, + { + name: "key exist", + arg: Arguments{ + key1: "1.5", + }, + key: key1, + baseValue: 1.2, + expectValue: 1.5, + }, + { + name: "value of key invalid", + arg: Arguments{ + key1: "errorValue", + }, + key: key1, + baseValue: 1.2, + expectValue: 1.2, + }, + { + name: "value of key null", + arg: Arguments{ + key1: "", + }, + key: key1, + baseValue: 1.2, + expectValue: 1.2, + }, + } + + for index, c := range cases { + baseValue := c.baseValue + c.arg.GetFloat64(&baseValue, c.key) + if baseValue != c.expectValue { + t.Errorf("index %d, case %s, value should be %v, but not %v", index, c.name, c.expectValue, baseValue) + } + } +} + +func TestGetArgOfActionFromConf(t *testing.T) { + cases := []struct { + name string + configurations []conf.Configuration + action string + expectedArguments Arguments + }{ + { + name: "action exist in configurations", + configurations: []conf.Configuration{ + { + Name: "enqueue", + Arguments: map[string]string{ + "overCommitFactor": "1.5", + }, + }, + { + Name: "allocate", + Arguments: map[string]string{ + "placeholde": "placeholde", + }, + }, + }, + action: "enqueue", + expectedArguments: map[string]string{ + "overCommitFactor": "1.5", + }, + }, + { + name: "action not exist in configurations", + configurations: []conf.Configuration{ + { + Name: "enqueue", + Arguments: map[string]string{ + "overCommitFactor": "1.5", + }, + }, + }, + action: "allocate", + expectedArguments: nil, + }, + } + + for index, c := range cases { + arg := GetArgOfActionFromConf(c.configurations, c.action) + if false == reflect.DeepEqual(arg, c.expectedArguments) { + t.Errorf("index %d, case %s,expected %v, but got %v", index, c.name, c.expectedArguments, arg) + } + } +} diff --git a/pkg/scheduler/framework/framework.go b/pkg/scheduler/framework/framework.go index dfe4a43234..f66275769e 100644 --- a/pkg/scheduler/framework/framework.go +++ b/pkg/scheduler/framework/framework.go @@ -27,9 +27,10 @@ import ( ) // OpenSession start the session -func OpenSession(cache cache.Cache, tiers []conf.Tier) *Session { +func OpenSession(cache cache.Cache, tiers []conf.Tier, configurations []conf.Configuration) *Session { ssn := openSession(cache) ssn.Tiers = tiers + ssn.Configurations = configurations for _, tier := range tiers { for _, plugin := range tier.Plugins { diff --git a/pkg/scheduler/framework/session.go b/pkg/scheduler/framework/session.go index 752ed38806..b74c48781a 100644 --- a/pkg/scheduler/framework/session.go +++ b/pkg/scheduler/framework/session.go @@ -45,8 +45,9 @@ type Session struct { Queues map[api.QueueID]*api.QueueInfo NamespaceInfo map[api.NamespaceName]*api.NamespaceInfo - Backlog []*api.JobInfo - Tiers []conf.Tier + Backlog []*api.JobInfo + Tiers []conf.Tier + Configurations []conf.Configuration plugins map[string]Plugin eventHandlers []*EventHandler diff --git a/pkg/scheduler/plugins/binpack/binpack_test.go b/pkg/scheduler/plugins/binpack/binpack_test.go index 1253793c41..bcc80767ba 100644 --- a/pkg/scheduler/plugins/binpack/binpack_test.go +++ b/pkg/scheduler/plugins/binpack/binpack_test.go @@ -269,7 +269,7 @@ func TestNode(t *testing.T) { }, }, }, - }) + }, nil) defer framework.CloseSession(ssn) for _, job := range ssn.Jobs { diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 4808f39c43..56c49b3869 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -36,6 +36,7 @@ type Scheduler struct { config *rest.Config actions []framework.Action plugins []conf.Tier + configurations []conf.Configuration schedulerConf string schedulePeriod time.Duration } @@ -75,7 +76,7 @@ func (pc *Scheduler) runOnce() { pc.loadSchedulerConf() - ssn := framework.OpenSession(pc.cache, pc.plugins) + ssn := framework.OpenSession(pc.cache, pc.plugins, pc.configurations) defer framework.CloseSession(ssn) for _, action := range pc.actions { @@ -98,7 +99,7 @@ func (pc *Scheduler) loadSchedulerConf() { } } - pc.actions, pc.plugins, err = loadSchedulerConf(schedConf) + pc.actions, pc.plugins, pc.configurations, err = loadSchedulerConf(schedConf) if err != nil { panic(err) } diff --git a/pkg/scheduler/util.go b/pkg/scheduler/util.go index 5a57906606..a831414c33 100644 --- a/pkg/scheduler/util.go +++ b/pkg/scheduler/util.go @@ -41,7 +41,7 @@ tiers: - name: nodeorder ` -func loadSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, error) { +func loadSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, []conf.Configuration, error) { var actions []framework.Action schedulerConf := &conf.SchedulerConfiguration{} @@ -50,7 +50,7 @@ func loadSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, error) copy(buf, confStr) if err := yaml.Unmarshal(buf, schedulerConf); err != nil { - return nil, nil, err + return nil, nil, nil, err } // Set default settings for each plugin if not set @@ -65,11 +65,11 @@ func loadSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, error) if action, found := framework.GetAction(strings.TrimSpace(actionName)); found { actions = append(actions, action) } else { - return nil, nil, fmt.Errorf("failed to found Action %s, ignore it", actionName) + return nil, nil, nil, fmt.Errorf("failed to found Action %s, ignore it", actionName) } } - return actions, schedulerConf.Tiers, nil + return actions, schedulerConf.Tiers, schedulerConf.Configurations, nil } func readSchedulerConf(confPath string) (string, error) { diff --git a/pkg/scheduler/util_test.go b/pkg/scheduler/util_test.go index cd3829c9b4..a06e077b4b 100644 --- a/pkg/scheduler/util_test.go +++ b/pkg/scheduler/util_test.go @@ -26,7 +26,11 @@ import ( func TestLoadSchedulerConf(t *testing.T) { configuration := ` -actions: "allocate, backfill" +actions: "enqueue, allocate, backfill" +configurations: +- name: enqueue + arguments: + "overcommit-factor": 1.5 tiers: - plugins: - name: priority @@ -142,7 +146,16 @@ tiers: }, } - _, tiers, err := loadSchedulerConf(configuration) + expectedConfigurations := []conf.Configuration{ + { + Name: "enqueue", + Arguments: map[string]string{ + "overcommit-factor": "1.5", + }, + }, + } + + _, tiers, configurations, err := loadSchedulerConf(configuration) if err != nil { t.Errorf("Failed to load scheduler configuration: %v", err) } @@ -150,4 +163,8 @@ tiers: t.Errorf("Failed to set default settings for plugins, expected: %+v, got %+v", expectedTiers, tiers) } + if !reflect.DeepEqual(configurations, expectedConfigurations) { + t.Errorf("Wrong configuration, expected: %+v, got %+v", + expectedConfigurations, configurations) + } } From e3ab41666428421218b918aa61e60a0e36059bbf Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Fri, 6 Dec 2019 19:02:33 +0800 Subject: [PATCH 28/34] Fix helmv3 --- docs/design/queue/queue-state-management.md | 4 ---- hack/run-e2e-kind.sh | 2 +- .../helm/chart/volcano/templates/batch_v1alpha1_job.yaml | 2 -- .../chart/volcano/templates/bus_v1alpha1_command.yaml | 2 -- installer/helm/chart/volcano/templates/default-queue.yaml | 2 -- .../volcano/templates/scheduling_v1alpha1_podgroup.yaml | 2 -- .../volcano/templates/scheduling_v1alpha1_queue.yaml | 2 -- installer/volcano-development.yaml | 8 -------- 8 files changed, 1 insertion(+), 23 deletions(-) diff --git a/docs/design/queue/queue-state-management.md b/docs/design/queue/queue-state-management.md index 3a15e31bdb..8a708427e8 100644 --- a/docs/design/queue/queue-state-management.md +++ b/docs/design/queue/queue-state-management.md @@ -122,8 +122,6 @@ apiVersion: admissionregistration.k8s.io/v1beta1 kind: ValidatingWebhookConfiguration metadata: name: {{ .Release.Name }}-validate-queue - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-delete webhooks: - clientConfig: caBundle: "" @@ -171,8 +169,6 @@ apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingWebhookConfiguration metadata: name: {{ .Release.Name }}-mutate-queue - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-delete webhooks: - clientConfig: caBundle: "" diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index e875451c69..6554130967 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -41,7 +41,7 @@ function install-volcano { kind load docker-image ${TF_EXAMPLE_IMAGE} ${CLUSTER_CONTEXT} echo "Install volcano chart" - helm install installer/helm/chart/volcano --namespace kube-system --name ${CLUSTER_NAME} --kubeconfig ${KUBECONFIG} --set basic.image_tag_version=${TAG} --set basic.scheduler_config_file=config/volcano-scheduler-ci.conf --wait + helm install volcano installer/helm/chart/volcano --namespace kube-system --name ${CLUSTER_NAME} --kubeconfig ${KUBECONFIG} --set basic.image_tag_version=${TAG} --set basic.scheduler_config_file=config/volcano-scheduler-ci.conf --wait } function uninstall-volcano { diff --git a/installer/helm/chart/volcano/templates/batch_v1alpha1_job.yaml b/installer/helm/chart/volcano/templates/batch_v1alpha1_job.yaml index 13a52405f2..b9ebfabdfe 100644 --- a/installer/helm/chart/volcano/templates/batch_v1alpha1_job.yaml +++ b/installer/helm/chart/volcano/templates/batch_v1alpha1_job.yaml @@ -2,8 +2,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: jobs.batch.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: batch.volcano.sh names: diff --git a/installer/helm/chart/volcano/templates/bus_v1alpha1_command.yaml b/installer/helm/chart/volcano/templates/bus_v1alpha1_command.yaml index 111b55cb0a..1850904314 100644 --- a/installer/helm/chart/volcano/templates/bus_v1alpha1_command.yaml +++ b/installer/helm/chart/volcano/templates/bus_v1alpha1_command.yaml @@ -2,8 +2,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: commands.bus.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: bus.volcano.sh names: diff --git a/installer/helm/chart/volcano/templates/default-queue.yaml b/installer/helm/chart/volcano/templates/default-queue.yaml index 00475756eb..3aa233f5db 100644 --- a/installer/helm/chart/volcano/templates/default-queue.yaml +++ b/installer/helm/chart/volcano/templates/default-queue.yaml @@ -2,7 +2,5 @@ apiVersion: scheduling.incubator.k8s.io/v1alpha1 kind: Queue metadata: name: default - annotations: - "helm.sh/hook": post-install spec: weight: 1 diff --git a/installer/helm/chart/volcano/templates/scheduling_v1alpha1_podgroup.yaml b/installer/helm/chart/volcano/templates/scheduling_v1alpha1_podgroup.yaml index 3b60f2bb55..244459a520 100644 --- a/installer/helm/chart/volcano/templates/scheduling_v1alpha1_podgroup.yaml +++ b/installer/helm/chart/volcano/templates/scheduling_v1alpha1_podgroup.yaml @@ -2,8 +2,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: podgroups.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install spec: group: scheduling.incubator.k8s.io names: diff --git a/installer/helm/chart/volcano/templates/scheduling_v1alpha1_queue.yaml b/installer/helm/chart/volcano/templates/scheduling_v1alpha1_queue.yaml index 46bc0ade69..1398b51b8e 100644 --- a/installer/helm/chart/volcano/templates/scheduling_v1alpha1_queue.yaml +++ b/installer/helm/chart/volcano/templates/scheduling_v1alpha1_queue.yaml @@ -2,8 +2,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: queues.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install spec: group: scheduling.incubator.k8s.io names: diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index c842ce3699..334a1e648d 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -371,8 +371,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: jobs.batch.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: batch.volcano.sh names: @@ -573,8 +571,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: commands.bus.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: bus.volcano.sh names: @@ -623,8 +619,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: podgroups.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install spec: group: scheduling.incubator.k8s.io names: @@ -667,8 +661,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: queues.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install spec: group: scheduling.incubator.k8s.io names: From fa9ea66a534748625ff657a948a9b9e97271d7f4 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Sat, 7 Dec 2019 09:59:36 +0800 Subject: [PATCH 29/34] Add Vivo as adopter Add Vivo as adopter --- docs/community/adopters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/community/adopters.md b/docs/community/adopters.md index ce6df5f64d..6564288de7 100644 --- a/docs/community/adopters.md +++ b/docs/community/adopters.md @@ -14,4 +14,5 @@ please feel free to add yourself into the following list by a pull request. | [JD Retail Infrastructure Department](https://jd.com/) |[@yuanchen8911](https://github.com/yuanchen8911)| Evaluation | Spark on K8S | | [kt NexR](https://www.ktnexr.com) |[@minyk](https://github.com/minyk), [@dieselnexr](https://github.com/dieselnexr)| Evaluation | spark scheduler of our next cloud native product. | | [QTT Bigdata Infra](https://ir.qutoutiao.net/) |[@yuzhaojing](https://github.com/yuzhaojing) | Evaluation | Spark and AI on K8S. | +| [Vivo](https://www.vivo.com/) | [@zionwu](https://github.com/zionwu) | Evaluation | Scheduler for Vtraining (deep learning training platform) | From e56b18c700471a4a67dcf9f771f93f1f926894de Mon Sep 17 00:00:00 2001 From: Zhang Jinghui Date: Fri, 6 Dec 2019 15:48:17 +0800 Subject: [PATCH 30/34] support queue action by vcctl --- cmd/cli/queue.go | 34 +++++++-- pkg/apis/helpers/helpers.go | 4 ++ pkg/cli/queue/create.go | 10 ++- pkg/cli/queue/delete.go | 58 +++++++++++++++ pkg/cli/queue/delete_test.go | 87 ++++++++++++++++++++++ pkg/cli/queue/get.go | 9 +-- pkg/cli/queue/list.go | 15 ++-- pkg/cli/queue/operate.go | 105 +++++++++++++++++++++++++++ pkg/cli/queue/operate_test.go | 131 ++++++++++++++++++++++++++++++++++ pkg/cli/queue/queue_test.go | 2 +- pkg/cli/queue/util.go | 35 +++++++++ 11 files changed, 471 insertions(+), 19 deletions(-) create mode 100644 pkg/cli/queue/delete.go create mode 100644 pkg/cli/queue/delete_test.go create mode 100644 pkg/cli/queue/operate.go create mode 100644 pkg/cli/queue/operate_test.go diff --git a/cmd/cli/queue.go b/cmd/cli/queue.go index 039c8672b7..46f199b747 100644 --- a/cmd/cli/queue.go +++ b/cmd/cli/queue.go @@ -23,20 +23,40 @@ import ( ) func buildQueueCmd() *cobra.Command { - jobCmd := &cobra.Command{ + queueCmd := &cobra.Command{ Use: "queue", Short: "Queue Operations", } - jobRunCmd := &cobra.Command{ + queueCreateCmd := &cobra.Command{ Use: "create", Short: "creates queue", Run: func(cmd *cobra.Command, args []string) { checkError(cmd, queue.CreateQueue()) }, } - queue.InitRunFlags(jobRunCmd) - jobCmd.AddCommand(jobRunCmd) + queue.InitCreateFlags(queueCreateCmd) + queueCmd.AddCommand(queueCreateCmd) + + queueDeleteCmd := &cobra.Command{ + Use: "delete", + Short: "delete queue", + Run: func(cmd *cobra.Command, args []string) { + checkError(cmd, queue.DeleteQueue()) + }, + } + queue.InitDeleteFlags(queueDeleteCmd) + queueCmd.AddCommand(queueDeleteCmd) + + queueOperateCmd := &cobra.Command{ + Use: "operate queue", + Short: "operate queue", + Run: func(cmd *cobra.Command, args []string) { + checkError(cmd, queue.OperateQueue()) + }, + } + queue.InitOperateFlags(queueOperateCmd) + queueCmd.AddCommand(queueOperateCmd) queueListCmd := &cobra.Command{ Use: "list", @@ -46,7 +66,7 @@ func buildQueueCmd() *cobra.Command { }, } queue.InitListFlags(queueListCmd) - jobCmd.AddCommand(queueListCmd) + queueCmd.AddCommand(queueListCmd) queueGetCmd := &cobra.Command{ Use: "get", @@ -56,7 +76,7 @@ func buildQueueCmd() *cobra.Command { }, } queue.InitGetFlags(queueGetCmd) - jobCmd.AddCommand(queueGetCmd) + queueCmd.AddCommand(queueGetCmd) - return jobCmd + return queueCmd } diff --git a/pkg/apis/helpers/helpers.go b/pkg/apis/helpers/helpers.go index c42601a891..b5df3faf1d 100644 --- a/pkg/apis/helpers/helpers.go +++ b/pkg/apis/helpers/helpers.go @@ -40,6 +40,7 @@ import ( vcbatch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" vcbus "volcano.sh/volcano/pkg/apis/bus/v1alpha1" + schedulerv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" ) // JobKind creates job GroupVersionKind @@ -48,6 +49,9 @@ var JobKind = vcbatch.SchemeGroupVersion.WithKind("Job") // CommandKind creates command GroupVersionKind var CommandKind = vcbus.SchemeGroupVersion.WithKind("Command") +// V1alpha2QueueKind is queue kind with v1alpha2 version +var V1alpha2QueueKind = schedulerv1alpha2.SchemeGroupVersion.WithKind("Queue") + // GetController returns the controller uid func GetController(obj interface{}) types.UID { accessor, err := meta.Accessor(obj) diff --git a/pkg/cli/queue/create.go b/pkg/cli/queue/create.go index 01df533243..68bac301d9 100644 --- a/pkg/cli/queue/create.go +++ b/pkg/cli/queue/create.go @@ -30,20 +30,23 @@ type createFlags struct { Name string Weight int32 + // State is state of Queue + State string } var createQueueFlags = &createFlags{} -// InitRunFlags is used to init all run flags -func InitRunFlags(cmd *cobra.Command) { +// InitCreateFlags is used to init all flags during queue creating +func InitCreateFlags(cmd *cobra.Command) { initFlags(cmd, &createQueueFlags.commonFlags) cmd.Flags().StringVarP(&createQueueFlags.Name, "name", "n", "test", "the name of queue") cmd.Flags().Int32VarP(&createQueueFlags.Weight, "weight", "w", 1, "the weight of the queue") + cmd.Flags().StringVarP(&createQueueFlags.State, "state", "S", "Open", "the state of queue") } -// CreateQueue creates queue +// CreateQueue create queue func CreateQueue() error { config, err := buildConfig(createQueueFlags.Master, createQueueFlags.Kubeconfig) if err != nil { @@ -56,6 +59,7 @@ func CreateQueue() error { }, Spec: schedulingV1alpha2.QueueSpec{ Weight: int32(createQueueFlags.Weight), + State: schedulingV1alpha2.QueueState(createQueueFlags.State), }, } diff --git a/pkg/cli/queue/delete.go b/pkg/cli/queue/delete.go new file mode 100644 index 0000000000..1ec9366325 --- /dev/null +++ b/pkg/cli/queue/delete.go @@ -0,0 +1,58 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "fmt" + + "volcano.sh/volcano/pkg/client/clientset/versioned" + + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type deleteFlags struct { + commonFlags + + // Name is name of queue + Name string +} + +var deleteQueueFlags = &deleteFlags{} + +// InitDeleteFlags is used to init all flags during queue deleting +func InitDeleteFlags(cmd *cobra.Command) { + initFlags(cmd, &deleteQueueFlags.commonFlags) + + cmd.Flags().StringVarP(&deleteQueueFlags.Name, "name", "n", "", "the name of queue") +} + +// DeleteQueue delete queue +func DeleteQueue() error { + config, err := buildConfig(deleteQueueFlags.Master, deleteQueueFlags.Kubeconfig) + if err != nil { + return err + } + + if len(deleteQueueFlags.Name) == 0 { + return fmt.Errorf("Queue name must be specified") + } + + queueClient := versioned.NewForConfigOrDie(config) + return queueClient.SchedulingV1alpha2().Queues().Delete(deleteQueueFlags.Name, &metav1.DeleteOptions{}) +} diff --git a/pkg/cli/queue/delete_test.go b/pkg/cli/queue/delete_test.go new file mode 100644 index 0000000000..5a37394310 --- /dev/null +++ b/pkg/cli/queue/delete_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestDeleteQueue(t *testing.T) { + response := v1alpha2.Queue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-queue", + }, + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + val, err := json.Marshal(response) + if err == nil { + w.Write(val) + } + }) + + server := httptest.NewServer(handler) + defer server.Close() + + deleteQueueFlags.Master = server.URL + testCases := []struct { + Name string + QueueName string + ExpectValue error + }{ + { + Name: "Normal Case Delete Queue Succeed", + QueueName: "normal-case", + ExpectValue: nil, + }, + { + Name: "Abnormal Case Delete Queue Failed For Name Not Specified", + QueueName: "", + ExpectValue: fmt.Errorf("Queue name must be specified"), + }, + } + + for _, testCase := range testCases { + deleteQueueFlags.Name = testCase.QueueName + + err := DeleteQueue() + if false == reflect.DeepEqual(err, testCase.ExpectValue) { + t.Errorf("Case '%s' failed, expected: '%v', got '%v'", testCase.Name, testCase.ExpectValue, err) + } + } +} + +func TestInitDeleteFlags(t *testing.T) { + var cmd cobra.Command + InitDeleteFlags(&cmd) + + if cmd.Flag("name") == nil { + t.Errorf("Could not find the flag name") + } +} diff --git a/pkg/cli/queue/get.go b/pkg/cli/queue/get.go index 919df10c6f..da46d1909d 100644 --- a/pkg/cli/queue/get.go +++ b/pkg/cli/queue/get.go @@ -70,13 +70,14 @@ func GetQueue() error { // PrintQueue prints queue information func PrintQueue(queue *v1alpha2.Queue, writer io.Writer) { - _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s\n", - Name, Weight, Pending, Running, Unknown) + _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s%-8s%-8s\n", + Name, Weight, State, Inqueue, Pending, Running, Unknown) if err != nil { fmt.Printf("Failed to print queue command result: %s.\n", err) } - _, err = fmt.Fprintf(writer, "%-25s%-8d%-8d%-8d%-8d\n", - queue.Name, queue.Spec.Weight, queue.Status.Pending, queue.Status.Running, queue.Status.Unknown) + _, err = fmt.Fprintf(writer, "%-25s%-8d%-8s%-8d%-8d%-8d%-8d\n", + queue.Name, queue.Spec.Weight, queue.Status.State, queue.Status.Inqueue, + queue.Status.Pending, queue.Status.Running, queue.Status.Unknown) if err != nil { fmt.Printf("Failed to print queue command result: %s.\n", err) } diff --git a/pkg/cli/queue/list.go b/pkg/cli/queue/list.go index 33bd1b7d8a..b961320d2b 100644 --- a/pkg/cli/queue/list.go +++ b/pkg/cli/queue/list.go @@ -48,6 +48,12 @@ const ( // Unknown status of the queue Unknown string = "Unknown" + + // Inqueue status of queue + Inqueue string = "Inqueue" + + // State is state of queue + State string = "State" ) var listQueueFlags = &listFlags{} @@ -81,14 +87,15 @@ func ListQueue() error { // PrintQueues prints queue information func PrintQueues(queues *v1alpha2.QueueList, writer io.Writer) { - _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s\n", - Name, Weight, Pending, Running, Unknown) + _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s%-8s%-8s\n", + Name, Weight, State, Inqueue, Pending, Running, Unknown) if err != nil { fmt.Printf("Failed to print queue command result: %s.\n", err) } for _, queue := range queues.Items { - _, err = fmt.Fprintf(writer, "%-25s%-8d%-8d%-8d%-8d\n", - queue.Name, queue.Spec.Weight, queue.Status.Pending, queue.Status.Running, queue.Status.Unknown) + _, err = fmt.Fprintf(writer, "%-25s%-8d%-8s%-8d%-8d%-8d%-8d\n", + queue.Name, queue.Spec.Weight, queue.Status.State, queue.Status.Inqueue, + queue.Status.Pending, queue.Status.Running, queue.Status.Unknown) if err != nil { fmt.Printf("Failed to print queue command result: %s.\n", err) } diff --git a/pkg/cli/queue/operate.go b/pkg/cli/queue/operate.go new file mode 100644 index 0000000000..ac37f979b2 --- /dev/null +++ b/pkg/cli/queue/operate.go @@ -0,0 +1,105 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "fmt" + + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + "volcano.sh/volcano/pkg/client/clientset/versioned" + + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ActionOpen is `open` action + ActionOpen = "open" + // ActionClose is `close` action + ActionClose = "close" + // ActionUpdate is `update` action + ActionUpdate = "update" +) + +type operateFlags struct { + commonFlags + + // Name is name of queue + Name string + // Weight is weight of queue + Weight int32 + // Action is operation action of queue + Action string +} + +var operateQueueFlags = &operateFlags{} + +// InitOperateFlags is used to init all flags during queue operating +func InitOperateFlags(cmd *cobra.Command) { + initFlags(cmd, &operateQueueFlags.commonFlags) + + cmd.Flags().StringVarP(&operateQueueFlags.Name, "name", "n", "", "the name of queue") + cmd.Flags().Int32VarP(&operateQueueFlags.Weight, "weight", "w", 0, "the weight of the queue") + cmd.Flags().StringVarP(&operateQueueFlags.Action, "action", "a", "", + "operate action to queue, valid actions are open, close, update") +} + +// OperateQueue operates queue +func OperateQueue() error { + config, err := buildConfig(operateQueueFlags.Master, operateQueueFlags.Kubeconfig) + if err != nil { + return err + } + + if len(operateQueueFlags.Name) == 0 { + return fmt.Errorf("Queue name must be specified") + } + + var action schedulingv1alpha2.QueueAction + + switch operateQueueFlags.Action { + case ActionOpen: + action = schedulingv1alpha2.OpenQueueAction + case ActionClose: + action = schedulingv1alpha2.CloseQueueAction + case ActionUpdate: + if operateQueueFlags.Weight == 0 { + return fmt.Errorf("When %s queue %s, weight must be specified, "+ + "the value must be greater than 0", ActionUpdate, operateQueueFlags.Name) + } + + queueClient := versioned.NewForConfigOrDie(config) + queue, err := queueClient.SchedulingV1alpha2().Queues().Get(operateQueueFlags.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + queue.Spec.Weight = int32(operateQueueFlags.Weight) + + _, err = queueClient.SchedulingV1alpha2().Queues().Update(queue) + + return err + case "": + return fmt.Errorf("Action can not be null") + default: + return fmt.Errorf("Action %s invalid, valid actions are %s, %s and %s", + operateQueueFlags.Action, ActionOpen, ActionClose, ActionUpdate) + } + + return createQueueCommand(config, action) +} diff --git a/pkg/cli/queue/operate_test.go b/pkg/cli/queue/operate_test.go new file mode 100644 index 0000000000..a3265ce7be --- /dev/null +++ b/pkg/cli/queue/operate_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestOperateQueue(t *testing.T) { + response := v1alpha2.Queue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-queue", + }, + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + val, err := json.Marshal(response) + if err == nil { + w.Write(val) + } + }) + + server := httptest.NewServer(handler) + defer server.Close() + + operateQueueFlags.Master = server.URL + testCases := []struct { + Name string + QueueName string + Weight int32 + Action string + ExpectValue error + }{ + { + Name: "Normal Case Operate Queue Succeed, Action close", + QueueName: "normal-case-action-close", + Action: ActionClose, + ExpectValue: nil, + }, + { + Name: "Normal Case Operate Queue Succeed, Action open", + QueueName: "normal-case-action-open", + Action: ActionOpen, + ExpectValue: nil, + }, + { + Name: "Normal Case Operate Queue Succeed, Update Weight", + QueueName: "normal-case-update-weight", + Action: ActionUpdate, + Weight: 3, + ExpectValue: nil, + }, + { + Name: "Abnormal Case Update Queue Failed For Invalid Weight", + QueueName: "abnormal-case-invalid-weight", + Action: ActionUpdate, + ExpectValue: fmt.Errorf("When %s queue %s, weight must be specified, "+ + "the value must be greater than 0", ActionUpdate, "abnormal-case-invalid-weight"), + }, + { + Name: "Abnormal Case Operate Queue Failed For Name Not Specified", + QueueName: "", + ExpectValue: fmt.Errorf("Queue name must be specified"), + }, + { + Name: "Abnormal Case Operate Queue Failed For Action null", + QueueName: "abnormal-case-null-action", + Action: "", + ExpectValue: fmt.Errorf("Action can not be null"), + }, + { + Name: "Abnormal Case Operate Queue Failed For Action Invalid", + QueueName: "abnormal-case-invalid-action", + Action: "invalid", + ExpectValue: fmt.Errorf("Action %s invalid, valid actions are %s, %s and %s", + "invalid", ActionOpen, ActionClose, ActionUpdate), + }, + } + + for _, testCase := range testCases { + operateQueueFlags.Name = testCase.QueueName + operateQueueFlags.Action = testCase.Action + operateQueueFlags.Weight = testCase.Weight + + err := OperateQueue() + if false == reflect.DeepEqual(err, testCase.ExpectValue) { + t.Errorf("Case '%s' failed, expected: '%v', got '%v'", testCase.Name, testCase.ExpectValue, err) + } + } +} + +func TestInitOperateFlags(t *testing.T) { + var cmd cobra.Command + InitOperateFlags(&cmd) + + if cmd.Flag("name") == nil { + t.Errorf("Could not find the flag name") + } + if cmd.Flag("weight") == nil { + t.Errorf("Could not find the flag weight") + } + if cmd.Flag("action") == nil { + t.Errorf("Could not find the flag action") + } +} diff --git a/pkg/cli/queue/queue_test.go b/pkg/cli/queue/queue_test.go index 126487eca2..c49e3ad48d 100644 --- a/pkg/cli/queue/queue_test.go +++ b/pkg/cli/queue/queue_test.go @@ -80,7 +80,7 @@ func getCommonFlags(master string) commonFlags { } func TestCreateQueue(t *testing.T) { - InitRunFlags(&cobra.Command{}) + InitCreateFlags(&cobra.Command{}) server := getTestQueueHTTPServer(t) defer server.Close() diff --git a/pkg/cli/queue/util.go b/pkg/cli/queue/util.go index f6617f287d..f862ac642d 100644 --- a/pkg/cli/queue/util.go +++ b/pkg/cli/queue/util.go @@ -17,8 +17,16 @@ limitations under the License. package queue import ( + "fmt" "os" + "strings" + busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1" + "volcano.sh/volcano/pkg/apis/helpers" + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + "volcano.sh/volcano/pkg/client/clientset/versioned" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" // Initialize client auth plugin. @@ -35,3 +43,30 @@ func homeDir() string { func buildConfig(master, kubeconfig string) (*rest.Config, error) { return clientcmd.BuildConfigFromFlags(master, kubeconfig) } + +func createQueueCommand(config *rest.Config, action schedulingv1alpha2.QueueAction) error { + queueClient := versioned.NewForConfigOrDie(config) + queue, err := queueClient.SchedulingV1alpha2().Queues().Get(operateQueueFlags.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + ctrlRef := metav1.NewControllerRef(queue, helpers.V1alpha2QueueKind) + cmd := &busv1alpha1.Command{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-%s-", + queue.Name, strings.ToLower(string(action))), + OwnerReferences: []metav1.OwnerReference{ + *ctrlRef, + }, + }, + TargetObject: ctrlRef, + Action: string(action), + } + + if _, err := queueClient.BusV1alpha1().Commands("default").Create(cmd); err != nil { + return err + } + + return nil +} From f8e7ff4918fa615360a0e1e8c8617d5ad87d5228 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Sat, 7 Dec 2019 16:01:36 +0800 Subject: [PATCH 31/34] Adapte to helm v3 --- hack/run-e2e-kind.sh | 15 ++------------- .../chart/volcano/templates/default-queue.yaml | 6 ------ 2 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 installer/helm/chart/volcano/templates/default-queue.yaml diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 6554130967..80c38feac4 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -25,32 +25,21 @@ function kind-up-cluster { } function install-volcano { - echo "Preparing helm tiller service account" - kubectl create serviceaccount --namespace kube-system tiller - kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller - install-helm - helm init --service-account tiller --kubeconfig ${KUBECONFIG} --wait echo "Pulling required docker images" docker pull ${MPI_EXAMPLE_IMAGE} docker pull ${TF_EXAMPLE_IMAGE} - echo "Loading docker images into kind cluster" - kind load docker-image ${MPI_EXAMPLE_IMAGE} ${CLUSTER_CONTEXT} - kind load docker-image ${TF_EXAMPLE_IMAGE} ${CLUSTER_CONTEXT} - echo "Install volcano chart" - helm install volcano installer/helm/chart/volcano --namespace kube-system --name ${CLUSTER_NAME} --kubeconfig ${KUBECONFIG} --set basic.image_tag_version=${TAG} --set basic.scheduler_config_file=config/volcano-scheduler-ci.conf --wait + helm install ${CLUSTER_NAME} installer/helm/chart/volcano --namespace kube-system --kubeconfig ${KUBECONFIG} --set basic.image_tag_version=${TAG} --set basic.scheduler_config_file=config/volcano-scheduler-ci.conf --wait } function uninstall-volcano { - helm delete ${CLUSTER_NAME} --purge --kubeconfig ${KUBECONFIG} + helm uninstall ${CLUSTER_NAME} -n kube-system } function generate-log { - echo "Generating tiller log files" - kubectl logs deployment/tiller-deploy -n kube-system > helm-tiller.log echo "Generating volcano log files" kubectl logs deployment/${CLUSTER_NAME}-admission -n kube-system > volcano-admission.log kubectl logs deployment/${CLUSTER_NAME}-controllers -n kube-system > volcano-controller.log diff --git a/installer/helm/chart/volcano/templates/default-queue.yaml b/installer/helm/chart/volcano/templates/default-queue.yaml deleted file mode 100644 index 3aa233f5db..0000000000 --- a/installer/helm/chart/volcano/templates/default-queue.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: scheduling.incubator.k8s.io/v1alpha1 -kind: Queue -metadata: - name: default -spec: - weight: 1 From 764a158b85ac2e51089ac32928384e8f6d242f98 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Fri, 6 Dec 2019 15:36:00 +0800 Subject: [PATCH 32/34] Cleanup e2e framework to speed up e2e --- docs/development/development.md | 2 +- hack/run-e2e-kind.sh | 2 +- test/e2e/admission.go | 10 +-- test/e2e/cli_util.go | 11 ++- test/e2e/command.go | 8 +- test/e2e/job_controlled_resource.go | 4 +- test/e2e/job_error_handling.go | 41 +++++---- test/e2e/job_lifecycle.go | 16 ++-- test/e2e/job_plugins.go | 6 +- test/e2e/job_scheduling.go | 28 +++--- test/e2e/mpi.go | 2 +- test/e2e/pg_controller.go | 4 +- test/e2e/predicates.go | 12 +-- test/e2e/queue.go | 2 +- test/e2e/tensorflow.go | 2 +- test/e2e/util.go | 130 +++++++++++----------------- test/e2e/vcctl.go | 6 +- 17 files changed, 135 insertions(+), 151 deletions(-) diff --git a/docs/development/development.md b/docs/development/development.md index 490023b33c..8c88df7d24 100644 --- a/docs/development/development.md +++ b/docs/development/development.md @@ -118,7 +118,7 @@ make e2e-test-kind If you want to run e2e test in a existing cluster with volcano deployed, run the following: ```bash -export VK_BIN= need to set vcctl binary path (eg:.../src/volcano.sh/volcano/_output/bin/) +export VC_BIN= need to set vcctl binary path (eg:.../src/volcano.sh/volcano/_output/bin/) KUBECONFIG=${KUBECONFIG} go test ./test/e2e ``` diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 80c38feac4..530b89a2a3 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -1,7 +1,7 @@ #!/bin/bash export VK_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.. -export VK_BIN=${VK_ROOT}/${BIN_DIR}/${BIN_OSARCH} +export VC_BIN=${VK_ROOT}/${BIN_DIR}/${BIN_OSARCH} export LOG_LEVEL=3 export SHOW_VOLCANO_LOGS=${SHOW_VOLCANO_LOGS:-1} export CLEANUP_CLUSTER=${CLEANUP_CLUSTER:-1} diff --git a/test/e2e/admission.go b/test/e2e/admission.go index a6a631935c..02531eecb7 100644 --- a/test/e2e/admission.go +++ b/test/e2e/admission.go @@ -34,7 +34,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { It("Default queue would be added", func() { jobName := "job-default-queue" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) _, err := createJobInner(context, &jobSpec{ @@ -60,7 +60,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { It("Invalid CPU unit", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) namespace := "test" @@ -109,7 +109,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { It("Invalid memory unit", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) namespace := "test" @@ -160,7 +160,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { It("Create default-scheduler pod", func() { podName := "pod-default-scheduler" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) pod := &corev1.Pod{ @@ -188,7 +188,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { podName := "pod-volcano" pgName := "pending-pg" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) pg := &schedulingv1alpha2.PodGroup{ diff --git a/test/e2e/cli_util.go b/test/e2e/cli_util.go index cc69af6eb5..b4d8bd3b6e 100644 --- a/test/e2e/cli_util.go +++ b/test/e2e/cli_util.go @@ -72,7 +72,10 @@ func RunCliCommand(command []string) string { command = append(command, "--master", masterURL()) } command = append(command, "--kubeconfig", kubeconfigPath(homeDir())) - output, err := exec.Command(VolcanoCliBinary(), command...).Output() + vcctl := VolcanoCliBinary() + Expect(fileExist(vcctl)).To(BeTrue(), fmt.Sprintf( + "vcctl binary: %s is required for E2E tests, please update VC_BIN environment", vcctl)) + output, err := exec.Command(vcctl, command...).Output() Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Command %s failed to execute: %s", strings.Join(command, ""), err)) return string(output) @@ -83,8 +86,10 @@ func RunCliCommandWithoutKubeConfig(command []string) string { if masterURL() != "" { command = append(command, "--master", masterURL()) } - - output, err := exec.Command(VolcanoCliBinary(), command...).Output() + vcctl := VolcanoCliBinary() + Expect(fileExist(vcctl)).To(BeTrue(), fmt.Sprintf( + "vcctl binary: %s is required for E2E tests, please update VC_BIN environment", vcctl)) + output, err := exec.Command(vcctl, command...).Output() Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Command %s failed to execute: %s", strings.Join(command, ""), err)) return string(output) diff --git a/test/e2e/command.go b/test/e2e/command.go index aee135e681..8c5755f742 100644 --- a/test/e2e/command.go +++ b/test/e2e/command.go @@ -36,7 +36,7 @@ var _ = Describe("Job E2E Test: Test Job Command", func() { var outBuffer bytes.Buffer jobName := "test-job" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -70,7 +70,7 @@ var _ = Describe("Job E2E Test: Test Job Command", func() { jobName := "test-suspend-running-job" taskName := "long-live-task" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) job := createJob(context, &jobSpec{ @@ -113,7 +113,7 @@ var _ = Describe("Job E2E Test: Test Job Command", func() { }) It("Suspend pending job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -157,7 +157,7 @@ var _ = Describe("Job E2E Test: Test Job Command", func() { jobName := "test-del-job" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) diff --git a/test/e2e/job_controlled_resource.go b/test/e2e/job_controlled_resource.go index c19e01755c..9f1d094eae 100644 --- a/test/e2e/job_controlled_resource.go +++ b/test/e2e/job_controlled_resource.go @@ -32,7 +32,7 @@ var _ = Describe("Job E2E Test: Test Job PVCs", func() { taskName := "pvctask" pvName := "job-pv-name" pvcName := "job-pvc-name-exist" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) var tt v12.HostPathType @@ -136,7 +136,7 @@ var _ = Describe("Job E2E Test: Test Job PVCs", func() { It("Generate PodGroup and valid minResource when creating job", func() { jobName := "job-name-podgroup" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) resource := v12.ResourceList{ diff --git a/test/e2e/job_error_handling.go b/test/e2e/job_error_handling.go index 7bc72c1837..7be56559f4 100644 --- a/test/e2e/job_error_handling.go +++ b/test/e2e/job_error_handling.go @@ -17,9 +17,10 @@ limitations under the License. package e2e import ( + "strconv" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "strconv" "k8s.io/api/core/v1" @@ -31,7 +32,7 @@ import ( var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodFailed; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -68,7 +69,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodFailed; Action: TerminateJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -105,7 +106,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodFailed; Action: AbortJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -142,7 +143,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodEvicted; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -186,7 +187,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodEvicted; Action: TerminateJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -230,7 +231,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodEvicted; Action: AbortJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -274,7 +275,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: Any; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -318,7 +319,7 @@ var _ = Describe("Job Error Handling", func() { It("Job error handling: Restart job when job is unschedulable", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -377,7 +378,7 @@ var _ = Describe("Job Error Handling", func() { It("Job error handling: Abort job when job is unschedulable", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -432,7 +433,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: TaskCompleted; Action: CompletedJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -472,7 +473,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, error code: 3; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -510,7 +511,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event[]: PodEvicted, PodFailed; Action: TerminateJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -556,7 +557,7 @@ var _ = Describe("Job Error Handling", func() { }) It("Task level LifecyclePolicy, Event: PodFailed; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -592,7 +593,7 @@ var _ = Describe("Job Error Handling", func() { }) It("Task level LifecyclePolicy, Event: PodEvicted; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -636,7 +637,7 @@ var _ = Describe("Job Error Handling", func() { }) It("Task level LifecyclePolicy, Event: PodEvicted; Action: TerminateJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -679,7 +680,7 @@ var _ = Describe("Job Error Handling", func() { }) It("Task level LifecyclePolicy, Event: TaskCompleted; Action: CompletedJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -719,7 +720,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodFailed; Action: AbortJob and Task level lifecyclePolicy, Event : PodFailed; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -762,7 +763,9 @@ var _ = Describe("Job Error Handling", func() { It("Task Priority", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{ + priorityClasses: []string{masterPriority, workerPriority}, + }) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) diff --git a/test/e2e/job_lifecycle.go b/test/e2e/job_lifecycle.go index 17b750ae27..c01d1cef79 100644 --- a/test/e2e/job_lifecycle.go +++ b/test/e2e/job_lifecycle.go @@ -30,7 +30,7 @@ import ( var _ = Describe("Job Life Cycle", func() { It("Delete job that is pending state", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -62,7 +62,7 @@ var _ = Describe("Job Life Cycle", func() { It("Delete job that is Running state", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -93,7 +93,7 @@ var _ = Describe("Job Life Cycle", func() { It("Delete job that is Completed state", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -126,7 +126,7 @@ var _ = Describe("Job Life Cycle", func() { It("Delete job that is Failed job", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -165,7 +165,7 @@ var _ = Describe("Job Life Cycle", func() { It("Delete job that is terminated job", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -204,7 +204,7 @@ var _ = Describe("Job Life Cycle", func() { It("Create and Delete job with CPU requirement", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -243,7 +243,7 @@ var _ = Describe("Job Life Cycle", func() { }) It("Checking Event Generation for job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) job := createJob(context, &jobSpec{ @@ -271,7 +271,7 @@ var _ = Describe("Job Life Cycle", func() { }) It("Checking Unschedulable Event Generation for job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) nodeName, rep := computeNode(context, oneCPU) diff --git a/test/e2e/job_plugins.go b/test/e2e/job_plugins.go index 24f688bae3..d8b4204f25 100644 --- a/test/e2e/job_plugins.go +++ b/test/e2e/job_plugins.go @@ -33,7 +33,7 @@ var _ = Describe("Job E2E Test: Test Job Plugins", func() { namespace := "test" taskName := "task" foundVolume := false - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) nodeName, rep := computeNode(context, oneCPU) @@ -105,7 +105,7 @@ var _ = Describe("Job E2E Test: Test Job Plugins", func() { namespace := "test" taskName := "task" foundVolume := false - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) _, rep := computeNode(context, oneCPU) @@ -178,7 +178,7 @@ var _ = Describe("Job E2E Test: Test Job Plugins", func() { taskName := "task" foundVolume := false foundEnv := false - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) _, rep := computeNode(context, oneCPU) diff --git a/test/e2e/job_scheduling.go b/test/e2e/job_scheduling.go index 6e45c27b56..b1379afc80 100644 --- a/test/e2e/job_scheduling.go +++ b/test/e2e/job_scheduling.go @@ -36,7 +36,7 @@ import ( var _ = Describe("Job E2E Test", func() { It("Schedule Job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -57,7 +57,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Schedule Multiple Jobs", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -91,7 +91,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Gang scheduling", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU)/2 + 1 @@ -128,7 +128,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Gang scheduling: Full Occupied", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -159,7 +159,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Preemption", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -191,7 +191,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Multiple Preemption", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -232,7 +232,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Schedule BestEffort Job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -262,7 +262,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Statement", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -299,7 +299,7 @@ var _ = Describe("Job E2E Test", func() { }) It("support binpack policy", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -382,7 +382,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Schedule v1.Job type using Volcano scheduler", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) namespace := "test" parallel := int32(2) @@ -418,7 +418,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Schedule v1.Job type using Volcano scheduler with error case", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) namespace := "test" parallel := int32(2) @@ -479,7 +479,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Namespace Fair Share", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) const fairShareNamespace = "fairshare" @@ -585,7 +585,9 @@ var _ = Describe("Job E2E Test", func() { }) It("Queue Fair Share", func() { - context := initTestContext() + context := initTestContext(options{ + queues: []string{defaultQueue1, defaultQueue2}, + }) defer cleanupTestContext(context) slot := halfCPU diff --git a/test/e2e/mpi.go b/test/e2e/mpi.go index 4ba32c7112..1549210509 100644 --- a/test/e2e/mpi.go +++ b/test/e2e/mpi.go @@ -25,7 +25,7 @@ import ( var _ = Describe("MPI E2E Test", func() { It("will run and complete finally", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU diff --git a/test/e2e/pg_controller.go b/test/e2e/pg_controller.go index 8f3490230a..ac82cfd12b 100644 --- a/test/e2e/pg_controller.go +++ b/test/e2e/pg_controller.go @@ -30,7 +30,7 @@ var _ = Describe("PG E2E Test: Test PG controller", func() { podName := "pod-volcano" namespace := "test" label := map[string]string{"schedulerName": "volcano"} - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rc := &corev1.ReplicationController{ @@ -85,7 +85,7 @@ var _ = Describe("PG E2E Test: Test PG controller", func() { podName := "pod-default-scheduler" namespace := "test" label := map[string]string{"a": "b"} - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rc := &corev1.ReplicationController{ diff --git a/test/e2e/predicates.go b/test/e2e/predicates.go index 1a106587d8..7d0e3d0b98 100644 --- a/test/e2e/predicates.go +++ b/test/e2e/predicates.go @@ -27,7 +27,7 @@ import ( var _ = Describe("Predicates E2E Test", func() { It("Hostport", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) nn := clusterNodeNumber(context) @@ -55,7 +55,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("NodeAffinity", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -104,7 +104,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("Pod Affinity", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -153,7 +153,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("Pod Anti-Affinity", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -203,7 +203,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("Taints", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) taints := []v1.Taint{ @@ -242,7 +242,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("Taints and Tolerations", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) taints := []v1.Taint{ diff --git a/test/e2e/queue.go b/test/e2e/queue.go index cbdc46366a..a58c46b8b4 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -27,7 +27,7 @@ import ( var _ = Describe("Queue E2E Test", func() { It("Reclaim", func() { - context := initTestContext() + context := initTestContext("", []string{defaultQueue1, defaultQueue2}, nil) defer cleanupTestContext(context) slot := oneCPU diff --git a/test/e2e/tensorflow.go b/test/e2e/tensorflow.go index 50851dc90a..947e24c899 100644 --- a/test/e2e/tensorflow.go +++ b/test/e2e/tensorflow.go @@ -28,7 +28,7 @@ import ( var _ = Describe("TensorFlow E2E Test", func() { It("Will Start in pending state and goes through other phases to get complete phase", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) jobName := "tensorflow-dist-mnist" diff --git a/test/e2e/util.go b/test/e2e/util.go index f89d739346..9592378271 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -97,9 +97,9 @@ func kubeconfigPath(home string) string { return filepath.Join(home, ".kube", "config") // default kubeconfig path is $HOME/.kube/config } -//VolcanoCliBinary function gets the volcano cli binary +// VolcanoCliBinary function gets the volcano cli binary func VolcanoCliBinary() string { - if bin := os.Getenv("VK_BIN"); bin != "" { + if bin := os.Getenv("VC_BIN"); bin != "" { return filepath.Join(bin, "vcctl") } return "" @@ -109,62 +109,49 @@ type context struct { kubeclient *kubernetes.Clientset vcclient *vcclient.Clientset - namespace string - queues []string + namespace string + queues []string + priorityClasses []string +} + +type options struct { + namespace string + queues []string + priorityClasses []string } -func initTestContext() *context { - cxt := &context{ - namespace: defaultNamespace, - queues: []string{defaultQueue1, defaultQueue2}, +func initTestContext(o options) *context { + if o.namespace == "" { + o.namespace = defaultNamespace + } + ctx := &context{ + namespace: o.namespace, + queues: o.queues, + priorityClasses: o.priorityClasses, } home := homeDir() Expect(home).NotTo(Equal("")) configPath := kubeconfigPath(home) Expect(configPath).NotTo(Equal("")) - vcctl := VolcanoCliBinary() - Expect(fileExist(vcctl)).To(BeTrue(), fmt.Sprintf( - "vcctl binary: %s is required for E2E tests, please update VK_BIN environment", vcctl)) - config, err := clientcmd.BuildConfigFromFlags(masterURL(), configPath) - Expect(err).NotTo(HaveOccurred()) - cxt.vcclient = vcclient.NewForConfigOrDie(config) - cxt.kubeclient = kubernetes.NewForConfigOrDie(config) - - //Ensure at least one worker is ready - err = waitClusterReady(cxt) - Expect(err).NotTo(HaveOccurred(), - "k8s cluster is required to have one ready worker node at least.") - - _, err = cxt.kubeclient.CoreV1().Namespaces().Create(&v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: cxt.namespace, - }, - }) + config, err := clientcmd.BuildConfigFromFlags(masterURL(), configPath) Expect(err).NotTo(HaveOccurred()) - createQueues(cxt) + ctx.vcclient = vcclient.NewForConfigOrDie(config) + ctx.kubeclient = kubernetes.NewForConfigOrDie(config) - _, err = cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Create(&schedv1.PriorityClass{ + _, err = ctx.kubeclient.CoreV1().Namespaces().Create(&v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: masterPriority, + Name: ctx.namespace, }, - Value: 100, - GlobalDefault: false, }) Expect(err).NotTo(HaveOccurred()) - _, err = cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Create(&schedv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: workerPriority, - }, - Value: 1, - GlobalDefault: false, - }) - Expect(err).NotTo(HaveOccurred()) + createQueues(ctx) + createPriorityClasses(ctx) - return cxt + return ctx } func namespaceNotExist(ctx *context) wait.ConditionFunc { @@ -181,20 +168,6 @@ func namespaceNotExistWithName(ctx *context, name string) wait.ConditionFunc { } } -func queueNotExist(ctx *context) wait.ConditionFunc { - return func() (bool, error) { - for _, q := range ctx.queues { - var err error - _, err = ctx.vcclient.SchedulingV1alpha2().Queues().Get(q, metav1.GetOptions{}) - if !(err != nil && errors.IsNotFound(err)) { - return false, err - } - } - - return true, nil - } -} - func fileExist(name string) bool { if _, err := os.Stat(name); err != nil { if os.IsNotExist(err) { @@ -204,40 +177,21 @@ func fileExist(name string) bool { return true } -func cleanupTestContext(cxt *context) { +func cleanupTestContext(ctx *context) { foreground := metav1.DeletePropagationForeground - - err := cxt.kubeclient.CoreV1().Namespaces().Delete(cxt.namespace, &metav1.DeleteOptions{ - PropagationPolicy: &foreground, - }) - Expect(err).NotTo(HaveOccurred()) - - deleteQueues(cxt) - - err = cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Delete(masterPriority, &metav1.DeleteOptions{ + err := ctx.kubeclient.CoreV1().Namespaces().Delete(ctx.namespace, &metav1.DeleteOptions{ PropagationPolicy: &foreground, }) Expect(err).NotTo(HaveOccurred()) - err = cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Delete(workerPriority, &metav1.DeleteOptions{ - PropagationPolicy: &foreground, - }) - Expect(err).NotTo(HaveOccurred()) + deleteQueues(ctx) - // Wait for namespace deleted. - err = wait.Poll(100*time.Millisecond, twoMinute, namespaceNotExist(cxt)) - Expect(err).NotTo(HaveOccurred()) - - // Wait for queues deleted - err = wait.Poll(100*time.Millisecond, twoMinute, queueNotExist(cxt)) - Expect(err).NotTo(HaveOccurred()) + deletePriorityClasses(ctx) } func createQueues(cxt *context) { - var err error - for _, q := range cxt.queues { - _, err = cxt.vcclient.SchedulingV1alpha2().Queues().Create(&schedulingv1alpha2.Queue{ + _, err := cxt.vcclient.SchedulingV1alpha2().Queues().Create(&schedulingv1alpha2.Queue{ ObjectMeta: metav1.ObjectMeta{ Name: q, }, @@ -262,6 +216,26 @@ func deleteQueues(cxt *context) { } } +func createPriorityClasses(cxt *context) { + for _, pc := range cxt.priorityClasses { + _, err := cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Create(&schedv1.PriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: pc, + }, + Value: 100, + GlobalDefault: false, + }) + Expect(err).NotTo(HaveOccurred()) + } +} + +func deletePriorityClasses(cxt *context) { + for _, pc := range cxt.priorityClasses { + err := cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Delete(pc, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + } +} + type taskSpec struct { name string min, rep int32 diff --git a/test/e2e/vcctl.go b/test/e2e/vcctl.go index 0e83575d0e..fdfa5a1a63 100644 --- a/test/e2e/vcctl.go +++ b/test/e2e/vcctl.go @@ -106,7 +106,7 @@ Global Flags: Expect(exist).Should(Equal(true)) }) - It("Command: vkctl job suspend -n {$JobName} --help", func() { + It("Command: vcctl job suspend -n {$JobName} --help", func() { kubeConfig := os.Getenv("KUBECONFIG") var output = ` abort a job @@ -131,7 +131,7 @@ Global Flags: Expect(exist).Should(Equal(true)) }) - It("vkctl job resume -n {$JobName} --help", func() { + It("vcctl job resume -n {$JobName} --help", func() { kubeConfig := os.Getenv("KUBECONFIG") var output = ` resume a job @@ -156,7 +156,7 @@ Global Flags: Expect(exist).Should(Equal(true)) }) - It("vkctl job run --help", func() { + It("vcctl job run --help", func() { kubeConfig := os.Getenv("KUBECONFIG") var output = ` run job by parameters from the command line From 6fc23bc1c0d34b8909cfd1a519a3443e2ac372c8 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Sat, 7 Dec 2019 16:07:34 +0800 Subject: [PATCH 33/34] fix buid --- test/e2e/queue.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/queue.go b/test/e2e/queue.go index a58c46b8b4..dd7243b6e3 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -27,7 +27,9 @@ import ( var _ = Describe("Queue E2E Test", func() { It("Reclaim", func() { - context := initTestContext("", []string{defaultQueue1, defaultQueue2}, nil) + context := initTestContext(options{ + queues: []string{defaultQueue1, defaultQueue2}, + }) defer cleanupTestContext(context) slot := oneCPU From e0718d62c169fcd9f709dd75ee50d553635a1c8e Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Mon, 9 Dec 2019 14:20:23 +0800 Subject: [PATCH 34/34] Add wait namesapce not exist --- test/e2e/util.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/e2e/util.go b/test/e2e/util.go index 9592378271..ebf3be6693 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -187,6 +187,10 @@ func cleanupTestContext(ctx *context) { deleteQueues(ctx) deletePriorityClasses(ctx) + + // Wait for namespace deleted. + err = wait.Poll(100*time.Millisecond, twoMinute, namespaceNotExist(ctx)) + Expect(err).NotTo(HaveOccurred()) } func createQueues(cxt *context) {