From 294bfa8be819d206c3408a57d3e3dde3b8401e1c Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Tue, 7 May 2024 16:03:48 +0800 Subject: [PATCH] Support CEL for matchExpressions --- kustomize/crd/bases/kwok.x-k8s.io_stages.yaml | 33 ++++- pkg/apis/internalversion/cel_types.go | 23 +++ pkg/apis/internalversion/stage_types.go | 14 +- .../zz_generated.conversion.go | 90 ++++++++++-- .../internalversion/zz_generated.deepcopy.go | 52 ++++++- pkg/apis/v1alpha1/cel_types.go | 23 +++ pkg/apis/v1alpha1/stage_types.go | 23 ++- pkg/apis/v1alpha1/zz_generated.deepcopy.go | 52 ++++++- pkg/kwok/controllers/controller.go | 24 ++- pkg/kwok/controllers/node_controller.go | 2 +- pkg/kwok/controllers/node_controller_test.go | 2 +- pkg/kwok/controllers/pod_controller.go | 2 +- pkg/kwok/controllers/pod_controller_test.go | 2 +- pkg/kwok/controllers/stage_controller.go | 2 +- pkg/kwok/controllers/stage_controller_test.go | 12 +- pkg/kwok/controllers/stages_manager.go | 6 +- pkg/tools/stage/stage.go | 10 +- pkg/utils/cel/environment.go | 9 ++ pkg/utils/lifecycle/lifecycle.go | 80 +++++++--- site/content/en/docs/generated/apis.md | 137 ++++++++++++++---- 20 files changed, 494 insertions(+), 104 deletions(-) create mode 100644 pkg/apis/internalversion/cel_types.go create mode 100644 pkg/apis/v1alpha1/cel_types.go diff --git a/kustomize/crd/bases/kwok.x-k8s.io_stages.yaml b/kustomize/crd/bases/kwok.x-k8s.io_stages.yaml index c45698a06c..cae79a8d96 100644 --- a/kustomize/crd/bases/kwok.x-k8s.io_stages.yaml +++ b/kustomize/crd/bases/kwok.x-k8s.io_stages.yaml @@ -232,13 +232,16 @@ spec: description: MatchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: |- - SelectorRequirement is a resource selector requirement is a selector that contains values, a key, - and an operator that relates the key and values. + description: SelectorExpression is a resource selector expression + is a set of requirements that must be true for a match. properties: + expression: + description: Expression represents the expression which + will be evaluated by CEL. + type: string key: - description: The name of the scope that the selector applies - to. + description: Key represents the expression which will be + evaluated by JQ. type: string operator: description: Represents a scope's relationship to a set @@ -252,10 +255,24 @@ spec: items: type: string type: array - required: - - key - - operator type: object + x-kubernetes-validations: + - message: expression and key are mutually exclusive + rule: has(self.expression) != has(self.key) + - message: key and operator must be set together + rule: '!has(self.expression) || (has(self.key) && has(self.operator))' + - messageExpression: '''operator '' + self.operator + '' is + not supported''' + rule: '!has(self.expression) || (self.operator in [''In'', + ''NotIn'', ''Exists'', ''DoesNotExist''])' + - messageExpression: '''value must be empty when using the '' + + self.operator + '' operator''' + rule: '!has(self.expression) || !(self.operator in [''In'', + ''NotIn'']) || !has(self.value)' + - messageExpression: '''value must not be empty when using the + '' + self.operator + '' operator''' + rule: '!has(self.expression) || !(self.operator in [''Exists'', + ''DoesNotExist'']) || has(self.value)' type: array matchLabels: additionalProperties: diff --git a/pkg/apis/internalversion/cel_types.go b/pkg/apis/internalversion/cel_types.go new file mode 100644 index 0000000000..046c39753a --- /dev/null +++ b/pkg/apis/internalversion/cel_types.go @@ -0,0 +1,23 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internalversion + +// ExpressionCEL is the expression which will be evaluated by CEL. +type ExpressionCEL struct { + // Expression represents the expression which will be evaluated by CEL. + Expression string +} diff --git a/pkg/apis/internalversion/stage_types.go b/pkg/apis/internalversion/stage_types.go index 1fcfd2b5b8..0031e90265 100644 --- a/pkg/apis/internalversion/stage_types.go +++ b/pkg/apis/internalversion/stage_types.go @@ -163,13 +163,19 @@ type StageSelector struct { // operator is "In", and the values array contains only "value". The requirements are ANDed. MatchAnnotations map[string]string // MatchExpressions is a list of label selector requirements. The requirements are ANDed. - MatchExpressions []SelectorRequirement + MatchExpressions []SelectorExpression } -// SelectorRequirement is a resource selector requirement is a selector that contains values, a key, +// SelectorExpression is a resource selector expression is a set of requirements that must be true for a match. +type SelectorExpression struct { + *ExpressionCEL + *SelectorJQ +} + +// SelectorJQ is a resource selector requirement is a selector that contains values, a key, // and an operator that relates the key and values. -type SelectorRequirement struct { - // The name of the scope that the selector applies to. +type SelectorJQ struct { + // Key represents the expression which will be evaluated by JQ. Key string // Represents a scope's relationship to a set of values. Operator SelectorOperator diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index 65008ceaae..4c444e4377 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -260,6 +260,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ExpressionCEL)(nil), (*v1alpha1.ExpressionCEL)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_internalversion_ExpressionCEL_To_v1alpha1_ExpressionCEL(a.(*ExpressionCEL), b.(*v1alpha1.ExpressionCEL), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1alpha1.ExpressionCEL)(nil), (*ExpressionCEL)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ExpressionCEL_To_internalversion_ExpressionCEL(a.(*v1alpha1.ExpressionCEL), b.(*ExpressionCEL), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*ExpressionFromSource)(nil), (*v1alpha1.ExpressionFromSource)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_internalversion_ExpressionFromSource_To_v1alpha1_ExpressionFromSource(a.(*ExpressionFromSource), b.(*v1alpha1.ExpressionFromSource), scope) }); err != nil { @@ -550,13 +560,23 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*SelectorRequirement)(nil), (*v1alpha1.SelectorRequirement)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_internalversion_SelectorRequirement_To_v1alpha1_SelectorRequirement(a.(*SelectorRequirement), b.(*v1alpha1.SelectorRequirement), scope) + if err := s.AddGeneratedConversionFunc((*SelectorExpression)(nil), (*v1alpha1.SelectorExpression)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_internalversion_SelectorExpression_To_v1alpha1_SelectorExpression(a.(*SelectorExpression), b.(*v1alpha1.SelectorExpression), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1alpha1.SelectorExpression)(nil), (*SelectorExpression)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_SelectorExpression_To_internalversion_SelectorExpression(a.(*v1alpha1.SelectorExpression), b.(*SelectorExpression), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*SelectorJQ)(nil), (*v1alpha1.SelectorJQ)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_internalversion_SelectorJQ_To_v1alpha1_SelectorJQ(a.(*SelectorJQ), b.(*v1alpha1.SelectorJQ), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha1.SelectorRequirement)(nil), (*SelectorRequirement)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_SelectorRequirement_To_internalversion_SelectorRequirement(a.(*v1alpha1.SelectorRequirement), b.(*SelectorRequirement), scope) + if err := s.AddGeneratedConversionFunc((*v1alpha1.SelectorJQ)(nil), (*SelectorJQ)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_SelectorJQ_To_internalversion_SelectorJQ(a.(*v1alpha1.SelectorJQ), b.(*SelectorJQ), scope) }); err != nil { return err } @@ -1327,6 +1347,26 @@ func Convert_v1alpha1_ExecTargetLocal_To_internalversion_ExecTargetLocal(in *v1a return autoConvert_v1alpha1_ExecTargetLocal_To_internalversion_ExecTargetLocal(in, out, s) } +func autoConvert_internalversion_ExpressionCEL_To_v1alpha1_ExpressionCEL(in *ExpressionCEL, out *v1alpha1.ExpressionCEL, s conversion.Scope) error { + out.Expression = in.Expression + return nil +} + +// Convert_internalversion_ExpressionCEL_To_v1alpha1_ExpressionCEL is an autogenerated conversion function. +func Convert_internalversion_ExpressionCEL_To_v1alpha1_ExpressionCEL(in *ExpressionCEL, out *v1alpha1.ExpressionCEL, s conversion.Scope) error { + return autoConvert_internalversion_ExpressionCEL_To_v1alpha1_ExpressionCEL(in, out, s) +} + +func autoConvert_v1alpha1_ExpressionCEL_To_internalversion_ExpressionCEL(in *v1alpha1.ExpressionCEL, out *ExpressionCEL, s conversion.Scope) error { + out.Expression = in.Expression + return nil +} + +// Convert_v1alpha1_ExpressionCEL_To_internalversion_ExpressionCEL is an autogenerated conversion function. +func Convert_v1alpha1_ExpressionCEL_To_internalversion_ExpressionCEL(in *v1alpha1.ExpressionCEL, out *ExpressionCEL, s conversion.Scope) error { + return autoConvert_v1alpha1_ExpressionCEL_To_internalversion_ExpressionCEL(in, out, s) +} + func autoConvert_internalversion_ExpressionFromSource_To_v1alpha1_ExpressionFromSource(in *ExpressionFromSource, out *v1alpha1.ExpressionFromSource, s conversion.Scope) error { out.ExpressionFrom = in.ExpressionFrom return nil @@ -2342,28 +2382,50 @@ func Convert_v1alpha1_SecurityContext_To_internalversion_SecurityContext(in *v1a return autoConvert_v1alpha1_SecurityContext_To_internalversion_SecurityContext(in, out, s) } -func autoConvert_internalversion_SelectorRequirement_To_v1alpha1_SelectorRequirement(in *SelectorRequirement, out *v1alpha1.SelectorRequirement, s conversion.Scope) error { +func autoConvert_internalversion_SelectorExpression_To_v1alpha1_SelectorExpression(in *SelectorExpression, out *v1alpha1.SelectorExpression, s conversion.Scope) error { + out.ExpressionCEL = (*v1alpha1.ExpressionCEL)(unsafe.Pointer(in.ExpressionCEL)) + out.SelectorJQ = (*v1alpha1.SelectorJQ)(unsafe.Pointer(in.SelectorJQ)) + return nil +} + +// Convert_internalversion_SelectorExpression_To_v1alpha1_SelectorExpression is an autogenerated conversion function. +func Convert_internalversion_SelectorExpression_To_v1alpha1_SelectorExpression(in *SelectorExpression, out *v1alpha1.SelectorExpression, s conversion.Scope) error { + return autoConvert_internalversion_SelectorExpression_To_v1alpha1_SelectorExpression(in, out, s) +} + +func autoConvert_v1alpha1_SelectorExpression_To_internalversion_SelectorExpression(in *v1alpha1.SelectorExpression, out *SelectorExpression, s conversion.Scope) error { + out.ExpressionCEL = (*ExpressionCEL)(unsafe.Pointer(in.ExpressionCEL)) + out.SelectorJQ = (*SelectorJQ)(unsafe.Pointer(in.SelectorJQ)) + return nil +} + +// Convert_v1alpha1_SelectorExpression_To_internalversion_SelectorExpression is an autogenerated conversion function. +func Convert_v1alpha1_SelectorExpression_To_internalversion_SelectorExpression(in *v1alpha1.SelectorExpression, out *SelectorExpression, s conversion.Scope) error { + return autoConvert_v1alpha1_SelectorExpression_To_internalversion_SelectorExpression(in, out, s) +} + +func autoConvert_internalversion_SelectorJQ_To_v1alpha1_SelectorJQ(in *SelectorJQ, out *v1alpha1.SelectorJQ, s conversion.Scope) error { out.Key = in.Key out.Operator = v1alpha1.SelectorOperator(in.Operator) out.Values = *(*[]string)(unsafe.Pointer(&in.Values)) return nil } -// Convert_internalversion_SelectorRequirement_To_v1alpha1_SelectorRequirement is an autogenerated conversion function. -func Convert_internalversion_SelectorRequirement_To_v1alpha1_SelectorRequirement(in *SelectorRequirement, out *v1alpha1.SelectorRequirement, s conversion.Scope) error { - return autoConvert_internalversion_SelectorRequirement_To_v1alpha1_SelectorRequirement(in, out, s) +// Convert_internalversion_SelectorJQ_To_v1alpha1_SelectorJQ is an autogenerated conversion function. +func Convert_internalversion_SelectorJQ_To_v1alpha1_SelectorJQ(in *SelectorJQ, out *v1alpha1.SelectorJQ, s conversion.Scope) error { + return autoConvert_internalversion_SelectorJQ_To_v1alpha1_SelectorJQ(in, out, s) } -func autoConvert_v1alpha1_SelectorRequirement_To_internalversion_SelectorRequirement(in *v1alpha1.SelectorRequirement, out *SelectorRequirement, s conversion.Scope) error { +func autoConvert_v1alpha1_SelectorJQ_To_internalversion_SelectorJQ(in *v1alpha1.SelectorJQ, out *SelectorJQ, s conversion.Scope) error { out.Key = in.Key out.Operator = SelectorOperator(in.Operator) out.Values = *(*[]string)(unsafe.Pointer(&in.Values)) return nil } -// Convert_v1alpha1_SelectorRequirement_To_internalversion_SelectorRequirement is an autogenerated conversion function. -func Convert_v1alpha1_SelectorRequirement_To_internalversion_SelectorRequirement(in *v1alpha1.SelectorRequirement, out *SelectorRequirement, s conversion.Scope) error { - return autoConvert_v1alpha1_SelectorRequirement_To_internalversion_SelectorRequirement(in, out, s) +// Convert_v1alpha1_SelectorJQ_To_internalversion_SelectorJQ is an autogenerated conversion function. +func Convert_v1alpha1_SelectorJQ_To_internalversion_SelectorJQ(in *v1alpha1.SelectorJQ, out *SelectorJQ, s conversion.Scope) error { + return autoConvert_v1alpha1_SelectorJQ_To_internalversion_SelectorJQ(in, out, s) } func autoConvert_internalversion_Stage_To_v1alpha1_Stage(in *Stage, out *v1alpha1.Stage, s conversion.Scope) error { @@ -2555,7 +2617,7 @@ func Convert_v1alpha1_StageResourceRef_To_internalversion_StageResourceRef(in *v func autoConvert_internalversion_StageSelector_To_v1alpha1_StageSelector(in *StageSelector, out *v1alpha1.StageSelector, s conversion.Scope) error { out.MatchLabels = *(*map[string]string)(unsafe.Pointer(&in.MatchLabels)) out.MatchAnnotations = *(*map[string]string)(unsafe.Pointer(&in.MatchAnnotations)) - out.MatchExpressions = *(*[]v1alpha1.SelectorRequirement)(unsafe.Pointer(&in.MatchExpressions)) + out.MatchExpressions = *(*[]v1alpha1.SelectorExpression)(unsafe.Pointer(&in.MatchExpressions)) return nil } @@ -2567,7 +2629,7 @@ func Convert_internalversion_StageSelector_To_v1alpha1_StageSelector(in *StageSe func autoConvert_v1alpha1_StageSelector_To_internalversion_StageSelector(in *v1alpha1.StageSelector, out *StageSelector, s conversion.Scope) error { out.MatchLabels = *(*map[string]string)(unsafe.Pointer(&in.MatchLabels)) out.MatchAnnotations = *(*map[string]string)(unsafe.Pointer(&in.MatchAnnotations)) - out.MatchExpressions = *(*[]SelectorRequirement)(unsafe.Pointer(&in.MatchExpressions)) + out.MatchExpressions = *(*[]SelectorExpression)(unsafe.Pointer(&in.MatchExpressions)) return nil } diff --git a/pkg/apis/internalversion/zz_generated.deepcopy.go b/pkg/apis/internalversion/zz_generated.deepcopy.go index 3bbf08b6a7..bea64b87d5 100644 --- a/pkg/apis/internalversion/zz_generated.deepcopy.go +++ b/pkg/apis/internalversion/zz_generated.deepcopy.go @@ -545,6 +545,22 @@ func (in *ExecTargetLocal) DeepCopy() *ExecTargetLocal { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpressionCEL) DeepCopyInto(out *ExpressionCEL) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpressionCEL. +func (in *ExpressionCEL) DeepCopy() *ExpressionCEL { + if in == nil { + return nil + } + out := new(ExpressionCEL) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExpressionFromSource) DeepCopyInto(out *ExpressionFromSource) { *out = *in @@ -1163,7 +1179,33 @@ func (in *SecurityContext) DeepCopy() *SecurityContext { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SelectorRequirement) DeepCopyInto(out *SelectorRequirement) { +func (in *SelectorExpression) DeepCopyInto(out *SelectorExpression) { + *out = *in + if in.ExpressionCEL != nil { + in, out := &in.ExpressionCEL, &out.ExpressionCEL + *out = new(ExpressionCEL) + **out = **in + } + if in.SelectorJQ != nil { + in, out := &in.SelectorJQ, &out.SelectorJQ + *out = new(SelectorJQ) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorExpression. +func (in *SelectorExpression) DeepCopy() *SelectorExpression { + if in == nil { + return nil + } + out := new(SelectorExpression) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SelectorJQ) DeepCopyInto(out *SelectorJQ) { *out = *in if in.Values != nil { in, out := &in.Values, &out.Values @@ -1173,12 +1215,12 @@ func (in *SelectorRequirement) DeepCopyInto(out *SelectorRequirement) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorRequirement. -func (in *SelectorRequirement) DeepCopy() *SelectorRequirement { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorJQ. +func (in *SelectorJQ) DeepCopy() *SelectorJQ { if in == nil { return nil } - out := new(SelectorRequirement) + out := new(SelectorJQ) in.DeepCopyInto(out) return out } @@ -1373,7 +1415,7 @@ func (in *StageSelector) DeepCopyInto(out *StageSelector) { } if in.MatchExpressions != nil { in, out := &in.MatchExpressions, &out.MatchExpressions - *out = make([]SelectorRequirement, len(*in)) + *out = make([]SelectorExpression, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/apis/v1alpha1/cel_types.go b/pkg/apis/v1alpha1/cel_types.go new file mode 100644 index 0000000000..1d230b78ef --- /dev/null +++ b/pkg/apis/v1alpha1/cel_types.go @@ -0,0 +1,23 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// ExpressionCEL is the expression which will be evaluated by CEL. +type ExpressionCEL struct { + // Expression represents the expression which will be evaluated by CEL. + Expression string `json:"expression,omitempty"` +} diff --git a/pkg/apis/v1alpha1/stage_types.go b/pkg/apis/v1alpha1/stage_types.go index a26d04908c..d05ea7b560 100644 --- a/pkg/apis/v1alpha1/stage_types.go +++ b/pkg/apis/v1alpha1/stage_types.go @@ -218,16 +218,27 @@ type StageSelector struct { // operator is "In", and the values array contains only "value". The requirements are ANDed. MatchAnnotations map[string]string `json:"matchAnnotations,omitempty"` // MatchExpressions is a list of label selector requirements. The requirements are ANDed. - MatchExpressions []SelectorRequirement `json:"matchExpressions,omitempty"` + MatchExpressions []SelectorExpression `json:"matchExpressions,omitempty"` } -// SelectorRequirement is a resource selector requirement is a selector that contains values, a key, +// SelectorExpression is a resource selector expression is a set of requirements that must be true for a match. +// +kubebuilder:validation:XValidation:rule="has(self.expression) != has(self.key)",message="expression and key are mutually exclusive" +// +kubebuilder:validation:XValidation:rule="!has(self.expression) || (has(self.key) && has(self.operator))",message="key and operator must be set together" +// +kubebuilder:validation:XValidation:rule="!has(self.expression) || (self.operator in ['In', 'NotIn', 'Exists', 'DoesNotExist'])",messageExpression="'operator ' + self.operator + ' is not supported'" +// +kubebuilder:validation:XValidation:rule="!has(self.expression) || !(self.operator in ['In', 'NotIn']) || !has(self.value)",messageExpression="'value must be empty when using the ' + self.operator + ' operator'" +// +kubebuilder:validation:XValidation:rule="!has(self.expression) || !(self.operator in ['Exists', 'DoesNotExist']) || has(self.value)",messageExpression="'value must not be empty when using the ' + self.operator + ' operator'" +type SelectorExpression struct { + *ExpressionCEL `json:",inline"` + *SelectorJQ `json:",inline"` +} + +// SelectorJQ is a resource selector requirement is a selector that contains values, a key, // and an operator that relates the key and values. -type SelectorRequirement struct { - // The name of the scope that the selector applies to. - Key string `json:"key"` +type SelectorJQ struct { + // Key represents the expression which will be evaluated by JQ. + Key string `json:"key,omitempty"` // Represents a scope's relationship to a set of values. - Operator SelectorOperator `json:"operator"` + Operator SelectorOperator `json:"operator,omitempty"` // An array of string values. // If the operator is In, NotIn, Intersection or NotIntersection, the values array must be non-empty. // If the operator is Exists or DoesNotExist, the values array must be empty. diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 5609c20eb9..f05423743b 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -910,6 +910,22 @@ func (in *ExecTargetLocal) DeepCopy() *ExecTargetLocal { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpressionCEL) DeepCopyInto(out *ExpressionCEL) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpressionCEL. +func (in *ExpressionCEL) DeepCopy() *ExpressionCEL { + if in == nil { + return nil + } + out := new(ExpressionCEL) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExpressionFromSource) DeepCopyInto(out *ExpressionFromSource) { *out = *in @@ -1629,7 +1645,33 @@ func (in *SecurityContext) DeepCopy() *SecurityContext { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SelectorRequirement) DeepCopyInto(out *SelectorRequirement) { +func (in *SelectorExpression) DeepCopyInto(out *SelectorExpression) { + *out = *in + if in.ExpressionCEL != nil { + in, out := &in.ExpressionCEL, &out.ExpressionCEL + *out = new(ExpressionCEL) + **out = **in + } + if in.SelectorJQ != nil { + in, out := &in.SelectorJQ, &out.SelectorJQ + *out = new(SelectorJQ) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorExpression. +func (in *SelectorExpression) DeepCopy() *SelectorExpression { + if in == nil { + return nil + } + out := new(SelectorExpression) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SelectorJQ) DeepCopyInto(out *SelectorJQ) { *out = *in if in.Values != nil { in, out := &in.Values, &out.Values @@ -1639,12 +1681,12 @@ func (in *SelectorRequirement) DeepCopyInto(out *SelectorRequirement) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorRequirement. -func (in *SelectorRequirement) DeepCopy() *SelectorRequirement { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorJQ. +func (in *SelectorJQ) DeepCopy() *SelectorJQ { if in == nil { return nil } - out := new(SelectorRequirement) + out := new(SelectorJQ) in.DeepCopyInto(out) return out } @@ -1892,7 +1934,7 @@ func (in *StageSelector) DeepCopyInto(out *StageSelector) { } if in.MatchExpressions != nil { in, out := &in.MatchExpressions, &out.MatchExpressions - *out = make([]SelectorRequirement, len(*in)) + *out = make([]SelectorExpression, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/kwok/controllers/controller.go b/pkg/kwok/controllers/controller.go index 1b39c798cd..98e0d7a971 100644 --- a/pkg/kwok/controllers/controller.go +++ b/pkg/kwok/controllers/controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "maps" "os" "time" @@ -43,6 +44,7 @@ import ( "sigs.k8s.io/kwok/pkg/client/clientset/versioned" "sigs.k8s.io/kwok/pkg/config/resources" "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/cel" "sigs.k8s.io/kwok/pkg/utils/client" "sigs.k8s.io/kwok/pkg/utils/gotpl" "sigs.k8s.io/kwok/pkg/utils/informer" @@ -59,6 +61,7 @@ var ( // Controller is a fake kubelet implementation that can be used to test type Controller struct { conf Config + env *cel.Environment stagesManager *StagesManager @@ -155,7 +158,26 @@ func NewController(conf Config) (*Controller, error) { return nil, err } + types := slices.Clone(cel.DefaultTypes) + conversions := slices.Clone(cel.DefaultConversions) + funcs := maps.Clone(cel.DefaultFuncs) + methods := maps.Clone(cel.FuncsToMethods(cel.DefaultFuncs)) + vars := map[string]any{ + "self": map[any]any{}, + } + env, err := cel.NewEnvironment(cel.EnvironmentConfig{ + Types: types, + Conversions: conversions, + Methods: methods, + Funcs: funcs, + Vars: vars, + }) + if err != nil { + return nil, err + } + c := &Controller{ + env: env, conf: conf, } @@ -538,7 +560,7 @@ func (c *Controller) Start(ctx context.Context) error { if len(c.conf.LocalStages) != 0 { for ref, stage := range c.conf.LocalStages { - lifecycle, err := lifecycle.NewLifecycle(stage) + lifecycle, err := lifecycle.NewLifecycle(stage, c.env) if err != nil { return err } diff --git a/pkg/kwok/controllers/node_controller.go b/pkg/kwok/controllers/node_controller.go index ce050621cc..fbeee028a0 100644 --- a/pkg/kwok/controllers/node_controller.go +++ b/pkg/kwok/controllers/node_controller.go @@ -284,7 +284,7 @@ func (c *NodeController) preprocess(ctx context.Context, node *corev1.Node) erro } lc := c.lifecycle.Get() - stage, err := lc.Match(ctx, node.Labels, node.Annotations, data) + stage, err := lc.Match(ctx, node.Labels, node.Annotations, node, data) if err != nil { return fmt.Errorf("stage match: %w", err) } diff --git a/pkg/kwok/controllers/node_controller_test.go b/pkg/kwok/controllers/node_controller_test.go index 8871ba883d..9d4dac7d50 100644 --- a/pkg/kwok/controllers/node_controller_test.go +++ b/pkg/kwok/controllers/node_controller_test.go @@ -79,7 +79,7 @@ func TestNodeController(t *testing.T) { nodeInit, _ := config.UnmarshalWithType[*internalversion.Stage](nodefast.DefaultNodeInit) nodeStages := []*internalversion.Stage{nodeInit} - lc, _ := lifecycle.NewLifecycle(nodeStages) + lc, _ := lifecycle.NewLifecycle(nodeStages, nil) nodes, err := NewNodeController(NodeControllerConfig{ TypedClient: clientset, NodeIP: "10.0.0.1", diff --git a/pkg/kwok/controllers/pod_controller.go b/pkg/kwok/controllers/pod_controller.go index 4c3588d7e3..5dd7f1c5d8 100644 --- a/pkg/kwok/controllers/pod_controller.go +++ b/pkg/kwok/controllers/pod_controller.go @@ -219,7 +219,7 @@ func (c *PodController) preprocess(ctx context.Context, pod *corev1.Pod) error { } lc := c.lifecycle.Get() - stage, err := lc.Match(ctx, pod.Labels, pod.Annotations, data) + stage, err := lc.Match(ctx, pod.Labels, pod.Annotations, pod, data) if err != nil { return fmt.Errorf("stage match: %w", err) } diff --git a/pkg/kwok/controllers/pod_controller_test.go b/pkg/kwok/controllers/pod_controller_test.go index 7d8e2f89b7..7219965554 100644 --- a/pkg/kwok/controllers/pod_controller_test.go +++ b/pkg/kwok/controllers/pod_controller_test.go @@ -191,7 +191,7 @@ func TestPodController(t *testing.T) { t.Fatal(fmt.Errorf("failed to watch nodes: %w", err)) } - lc, _ := lifecycle.NewLifecycle(podStages) + lc, _ := lifecycle.NewLifecycle(podStages, nil) annotationSelector, _ := labels.Parse("fake=custom") pods, err := NewPodController(PodControllerConfig{ TypedClient: clientset, diff --git a/pkg/kwok/controllers/stage_controller.go b/pkg/kwok/controllers/stage_controller.go index a74847a2bc..3ee9ad09f7 100644 --- a/pkg/kwok/controllers/stage_controller.go +++ b/pkg/kwok/controllers/stage_controller.go @@ -196,7 +196,7 @@ func (c *StageController) preprocess(ctx context.Context, resource *unstructured } lc := c.lifecycle.Get() - stage, err := lc.Match(ctx, resource.GetLabels(), resource.GetAnnotations(), data) + stage, err := lc.Match(ctx, resource.GetLabels(), resource.GetAnnotations(), resource, data) if err != nil { return fmt.Errorf("stage match: %w", err) } diff --git a/pkg/kwok/controllers/stage_controller_test.go b/pkg/kwok/controllers/stage_controller_test.go index 71ed4fa005..e80154c361 100644 --- a/pkg/kwok/controllers/stage_controller_test.go +++ b/pkg/kwok/controllers/stage_controller_test.go @@ -61,11 +61,13 @@ func TestStageController(t *testing.T) { }, Spec: internalversion.StageSpec{ Selector: &internalversion.StageSelector{ - MatchExpressions: []internalversion.SelectorRequirement{ + MatchExpressions: []internalversion.SelectorExpression{ { - Key: ".status.phase", - Operator: "NotIn", - Values: []string{"Available"}, + SelectorJQ: &internalversion.SelectorJQ{ + Key: ".status.phase", + Operator: "NotIn", + Values: []string{"Available"}, + }, }, }, }, @@ -80,7 +82,7 @@ func TestStageController(t *testing.T) { }, }, }, - }) + }, nil) patchMeta, _ := strategicpatch.NewPatchMetaFromStruct(corev1.PersistentVolume{}) controller, err := NewStageController(StageControllerConfig{ PlayStageParallelism: 1, diff --git a/pkg/kwok/controllers/stages_manager.go b/pkg/kwok/controllers/stages_manager.go index b61978c607..0dc73b57aa 100644 --- a/pkg/kwok/controllers/stages_manager.go +++ b/pkg/kwok/controllers/stages_manager.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/kwok/pkg/apis/internalversion" "sigs.k8s.io/kwok/pkg/config/resources" "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/cel" "sigs.k8s.io/kwok/pkg/utils/lifecycle" "sigs.k8s.io/kwok/pkg/utils/sets" "sigs.k8s.io/kwok/pkg/utils/slices" @@ -29,6 +30,7 @@ import ( // StagesManagerConfig is the configuration for a stages manager type StagesManagerConfig struct { + Env *cel.Environment StageGetter resources.DynamicGetter[[]*internalversion.Stage] StartFunc func(ctx context.Context, ref internalversion.StageResourceRef, lifecycle resources.Getter[lifecycle.Lifecycle]) error } @@ -36,6 +38,7 @@ type StagesManagerConfig struct { // StagesManager is a stages manager // It is a dynamic getter for stages and start a stage controller type StagesManager struct { + env *cel.Environment stageGetter resources.DynamicGetter[[]*internalversion.Stage] startFunc func(ctx context.Context, ref internalversion.StageResourceRef, lifecycle resources.Getter[lifecycle.Lifecycle]) error cache map[internalversion.StageResourceRef]context.CancelCauseFunc @@ -44,6 +47,7 @@ type StagesManager struct { // NewStagesManager creates a stage controller manager func NewStagesManager(conf StagesManagerConfig) *StagesManager { return &StagesManager{ + env: conf.Env, stageGetter: conf.StageGetter, startFunc: conf.StartFunc, cache: map[internalversion.StageResourceRef]context.CancelCauseFunc{}, @@ -90,7 +94,7 @@ func (c *StagesManager) manage(ctx context.Context) { return nil, false } - lifecycleStage, err := lifecycle.NewStage(stage) + lifecycleStage, err := lifecycle.NewStage(stage, c.env) if err != nil { logger.Error("failed to create lifecycle stage", err, "ref", ref) return nil, false diff --git a/pkg/tools/stage/stage.go b/pkg/tools/stage/stage.go index 2c238c8333..4f9e2fd241 100644 --- a/pkg/tools/stage/stage.go +++ b/pkg/tools/stage/stage.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/utils/expression" "sigs.k8s.io/kwok/pkg/utils/gotpl" "sigs.k8s.io/kwok/pkg/utils/lifecycle" "sigs.k8s.io/kwok/pkg/utils/slices" @@ -60,12 +61,17 @@ func TestingStages(ctx context.Context, target any, stages []*internalversion.St return stage.Spec.ResourceRef == want }) - lc, err := lifecycle.NewLifecycle(stages) + lc, err := lifecycle.NewLifecycle(stages, nil) if err != nil { return nil, err } - lcstages, err := lc.ListAllPossible(ctx, testTarget.GetLabels(), testTarget.GetAnnotations(), testTarget) + jsonStandard, err := expression.ToJSONStandard(testTarget) + if err != nil { + return nil, err + } + + lcstages, err := lc.ListAllPossible(ctx, testTarget.GetLabels(), testTarget.GetAnnotations(), testTarget, jsonStandard) if err != nil { return nil, err } diff --git a/pkg/utils/cel/environment.go b/pkg/utils/cel/environment.go index acbf162394..d87ec9d7f7 100644 --- a/pkg/utils/cel/environment.go +++ b/pkg/utils/cel/environment.go @@ -144,3 +144,12 @@ func AsString(refVal ref.Val) (string, error) { } return string(v), nil } + +// AsBool returns the bool value of a ref.Val +func AsBool(refVal ref.Val) (bool, error) { + v, ok := refVal.(types.Bool) + if !ok { + return false, fmt.Errorf("unsupported type: %T", v) + } + return bool(v), nil +} diff --git a/pkg/utils/lifecycle/lifecycle.go b/pkg/utils/lifecycle/lifecycle.go index 15c1fd33b6..b65afa40eb 100644 --- a/pkg/utils/lifecycle/lifecycle.go +++ b/pkg/utils/lifecycle/lifecycle.go @@ -25,15 +25,16 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/utils/cel" "sigs.k8s.io/kwok/pkg/utils/expression" "sigs.k8s.io/kwok/pkg/utils/format" ) // NewLifecycle returns a new Lifecycle. -func NewLifecycle(stages []*internalversion.Stage) (Lifecycle, error) { +func NewLifecycle(stages []*internalversion.Stage, env *cel.Environment) (Lifecycle, error) { lcs := Lifecycle{} for _, stage := range stages { - lc, err := NewStage(stage) + lc, err := NewStage(stage, env) if err != nil { return nil, fmt.Errorf("lifecycle stage: %w", err) } @@ -48,10 +49,10 @@ func NewLifecycle(stages []*internalversion.Stage) (Lifecycle, error) { // Lifecycle is a list of lifecycle stage. type Lifecycle []*Stage -func (s Lifecycle) match(label, annotation labels.Set, data interface{}) ([]*Stage, error) { +func (s Lifecycle) match(ctx context.Context, label, annotation labels.Set, data, jsonStandard any) ([]*Stage, error) { out := []*Stage{} for _, stage := range s { - ok, err := stage.match(label, annotation, data) + ok, err := stage.match(ctx, label, annotation, data, jsonStandard) if err != nil { return nil, err } @@ -63,12 +64,15 @@ func (s Lifecycle) match(label, annotation labels.Set, data interface{}) ([]*Sta } // ListAllPossible returns all possible stages. -func (s Lifecycle) ListAllPossible(ctx context.Context, label, annotation labels.Set, data interface{}) ([]*Stage, error) { - data, err := expression.ToJSONStandard(data) - if err != nil { - return nil, err +func (s Lifecycle) ListAllPossible(ctx context.Context, label, annotation labels.Set, data, jsonStandard any) ([]*Stage, error) { + if jsonStandard == nil { + j, err := expression.ToJSONStandard(data) + if err != nil { + return nil, err + } + jsonStandard = j } - stages, err := s.match(label, annotation, data) + stages, err := s.match(ctx, label, annotation, data, jsonStandard) if err != nil { return nil, err } @@ -122,12 +126,15 @@ func (s Lifecycle) ListAllPossible(ctx context.Context, label, annotation labels } // Match returns matched stage. -func (s Lifecycle) Match(ctx context.Context, label, annotation labels.Set, data interface{}) (*Stage, error) { - data, err := expression.ToJSONStandard(data) - if err != nil { - return nil, err +func (s Lifecycle) Match(ctx context.Context, label, annotation labels.Set, data, jsonStandard any) (*Stage, error) { + if jsonStandard == nil { + j, err := expression.ToJSONStandard(data) + if err != nil { + return nil, err + } + jsonStandard = j } - stages, err := s.match(label, annotation, data) + stages, err := s.match(ctx, label, annotation, data, jsonStandard) if err != nil { return nil, err } @@ -191,7 +198,7 @@ func (s Lifecycle) Match(ctx context.Context, label, annotation labels.Set, data } // NewStage returns a new Stage. -func NewStage(s *internalversion.Stage) (*Stage, error) { +func NewStage(s *internalversion.Stage, env *cel.Environment) (*Stage, error) { stage := &Stage{ name: s.Name, } @@ -208,11 +215,22 @@ func NewStage(s *internalversion.Stage) (*Stage, error) { } if selector.MatchExpressions != nil { for _, express := range selector.MatchExpressions { - requirement, err := expression.NewRequirement(express.Key, express.Operator, express.Values) - if err != nil { - return nil, err + switch { + case express.SelectorJQ != nil: + requirement, err := expression.NewRequirement(express.Key, express.Operator, express.Values) + if err != nil { + return nil, err + } + stage.matchExpressions = append(stage.matchExpressions, requirement) + case express.ExpressionCEL != nil: + program, err := env.Compile(express.Expression) + if err != nil { + return nil, err + } + stage.matchConditions = append(stage.matchConditions, program) + default: + return nil, fmt.Errorf("invalid expression") } - stage.matchExpressions = append(stage.matchExpressions, requirement) } } @@ -272,6 +290,7 @@ type Stage struct { matchLabels labels.Selector matchAnnotations labels.Selector matchExpressions []*expression.Requirement + matchConditions []cel.Program weight expression.IntGetter next *internalversion.StageNext @@ -282,7 +301,7 @@ type Stage struct { immediateNextStage bool } -func (s *Stage) match(label, annotation labels.Set, jsonStandard interface{}) (bool, error) { +func (s *Stage) match(ctx context.Context, label, annotation labels.Set, data, jsonStandard any) (bool, error) { if s.matchLabels != nil { if !s.matchLabels.Matches(label) { return false, nil @@ -296,7 +315,7 @@ func (s *Stage) match(label, annotation labels.Set, jsonStandard interface{}) (b if s.matchExpressions != nil { for _, requirement := range s.matchExpressions { - ok, err := requirement.Matches(context.Background(), jsonStandard) + ok, err := requirement.Matches(ctx, jsonStandard) if err != nil { return false, err } @@ -305,6 +324,25 @@ func (s *Stage) match(label, annotation labels.Set, jsonStandard interface{}) (b } } } + + if s.matchConditions != nil { + for _, program := range s.matchConditions { + val, _, err := program.ContextEval(ctx, map[string]any{ + "self": data, + }) + if err != nil { + return false, err + } + ok, err := cel.AsBool(val) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + } + } + return true, nil } diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index 526dc5a77f..f0e081de31 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -4847,6 +4847,38 @@ SecurityContext +

+ExpressionCEL + # +

+

+Appears on: +SelectorExpression +

+

+

ExpressionCEL is the expression which will be evaluated by CEL.

+

+ + + + + + + + + + + + + +
FieldDescription
+expression + +string + + +

Expression represents the expression which will be evaluated by CEL.

+

ExpressionFromSource # @@ -5816,58 +5848,67 @@ int64 -

-SelectorOperator -(string alias) - # +

+SelectorExpression + #

Appears on: -SelectorRequirement +StageSelector

-

SelectorOperator is a label selector operator is the set of operators that can be used in a selector requirement.

+

SelectorExpression is a resource selector expression is a set of requirements that must be true for a match.

- + - - - - - - - - - - - -
ValueField Description
"DoesNotExist"

SelectorOpDoesNotExist is the negated existence operator.

+		 +ExpressionCEL + + +ExpressionCEL + +
"Exists"

SelectorOpExists is the existence operator.

+		 +

+(Members of ExpressionCEL are embedded into this type.) +
"In"

SelectorOpIn is the set inclusion operator.

+		 +SelectorJQ + + +SelectorJQ + +
"NotIn"

SelectorOpNotIn is the negated set inclusion operator.

+		 +

+(Members of SelectorJQ are embedded into this type.) +
-

-SelectorRequirement - # +

+SelectorJQ + #

Appears on: -StageSelector +SelectorExpression

-

SelectorRequirement is a resource selector requirement is a selector that contains values, a key, +

SelectorJQ is a resource selector requirement is a selector that contains values, a key, and an operator that relates the key and values.

@@ -5886,7 +5927,7 @@ string @@ -5917,6 +5958,48 @@ If the operator is Exists or DoesNotExist, the values array must be empty.

-

The name of the scope that the selector applies to.

+

Key represents the expression which will be evaluated by JQ.
+

+SelectorOperator +(string alias) + # +

+

+Appears on: +SelectorJQ +

+

+

SelectorOperator is a label selector operator is the set of operators that can be used in a selector requirement.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
"DoesNotExist"

SelectorOpDoesNotExist is the negated existence operator.

+
"Exists"

SelectorOpExists is the existence operator.

+
"In"

SelectorOpIn is the set inclusion operator.

+
"NotIn"

SelectorOpNotIn is the negated set inclusion operator.

+

StageDelay # @@ -6431,8 +6514,8 @@ operator is “In”, and the values array contains only “value&rd matchExpressions - -[]SelectorRequirement + +[]SelectorExpression