Priority-based exclusive placement

#go.mod#

@@ -0,0 +1,88 @@
+module sigs.k8s.io/jobset
+
+go 1.23.0
+
+require (
+	github.com/google/go-cmp v0.6.0
+	github.com/onsi/ginkgo/v2 v2.20.2
+	github.com/onsi/gomega v1.34.2
+	github.com/open-policy-agent/cert-controller v0.11.0
+	github.com/prometheus/client_golang v1.20.4
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	k8s.io/api v0.31.1
+	k8s.io/apimachinery v0.31.1
+	k8s.io/apiserver v0.31.1
+	k8s.io/client-go v0.31.1
+	k8s.io/code-generator v0.31.1
+	k8s.io/component-base v0.31.1
+	k8s.io/klog v1.0.0
+	k8s.io/klog/v2 v2.130.1
+	k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	sigs.k8s.io/controller-runtime v0.19.0
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/zapr v1.3.0 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/term v0.23.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.31.0 // indirect
+	k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)

api/jobset/v1alpha2/jobset_types.go

@@ -42,6 +42,7 @@ const (
 	// job placement per topology group (defined as the label value).
 	// If set at the ReplicatedJob level, all child jobs from the target ReplicatedJobs will be scheduled
 	// using exclusive job placement per topology group.
+	// Exclusive placement is enforced within a priority level.
 	ExclusiveKey string = "alpha.jobset.sigs.k8s.io/exclusive-topology"
 	// NodeSelectorStrategyKey is an annotation that acts as a flag, the value does not matter.
 	// If set, the JobSet controller will automatically inject nodeSelectors for the JobSetNameKey label to

cloudbuild.yaml

@@ -13,6 +13,7 @@ steps:
     - GIT_TAG=$_GIT_TAG
     - EXTRA_TAG=$_PULL_BASE_REF
     - DOCKER_BUILDX_CMD=/buildx-entrypoint
+    - GOTOOLCHAIN=auto
 substitutions:
   # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
   # can be used as a substitution

go.mod

@@ -1,6 +1,6 @@
 module sigs.k8s.io/jobset
 
-go 1.22.0
+go 1.22.8
 
 require (
 	github.com/google/go-cmp v0.6.0

pkg/constants/constants.go

@@ -28,6 +28,10 @@ const (
 	// the JobSet is currently on.
 	RestartsKey = "jobset.sigs.k8s.io/restart-attempt"
 
+	// PriorityKey is a label key to record the pod priority. This is needed to enfroce exclusive placement
+	// only among jobs within the same priority.
+	PriorityKey = "jobset.sigs.k8s.io/priority"
+
 	// MaxParallelism defines the maximum number of parallel Job creations/deltions that
 	// the JobSet controller can perform.
 	MaxParallelism = 50

pkg/webhooks/pod_mutating_webhook.go

@@ -27,6 +27,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2"
+	"sigs.k8s.io/jobset/pkg/constants"
 )
 
 // +kubebuilder:webhook:path=/mutate--v1-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create,versions=v1,name=mpod.kb.io,sideEffects=None,admissionReviewVersions=v1
@@ -82,6 +83,9 @@ func (p *podWebhook) Default(ctx context.Context, obj runtime.Object) error {
 // scheduled on.
 func (p *podWebhook) patchPod(ctx context.Context, pod *corev1.Pod) error {
 	log := ctrl.LoggerFrom(ctx)
+	if pod.Spec.Priority != nil {
+		pod.Labels[constants.PriorityKey] = fmt.Sprint(*pod.Spec.Priority)
+	}
 	if pod.Annotations[batchv1.JobCompletionIndexAnnotation] == "0" {
 		log.V(3).Info(fmt.Sprintf("pod webhook: setting exclusive affinities for pod: %s", pod.Name))
 		setExclusiveAffinities(pod)
@@ -128,6 +132,11 @@ func setExclusiveAffinities(pod *corev1.Pod) {
 					Operator: metav1.LabelSelectorOpNotIn,
 					Values:   []string{pod.Labels[jobset.JobKey]},
 				},
+				{
+					Key:      constants.PriorityKey,
+					Operator: metav1.LabelSelectorOpIn,
+					Values:   []string{pod.Labels[constants.PriorityKey]},
+				},
 			}},
 			TopologyKey:       pod.Annotations[jobset.ExclusiveKey],
 			NamespaceSelector: &metav1.LabelSelector{},