Skip to content

Commit

Permalink
Introduce a new API named WorkloadRebalancer to support rescheduling.
Browse files Browse the repository at this point in the history
Signed-off-by: chaosi-zju <chaosi@zju.edu.cn>
  • Loading branch information
chaosi-zju committed Apr 22, 2024
1 parent f3b943d commit 760b6cc
Show file tree
Hide file tree
Showing 35 changed files with 2,820 additions and 14 deletions.
1,037 changes: 1,037 additions & 0 deletions api/openapi-spec/swagger.json

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -1128,6 +1128,18 @@ spec:
- name
type: object
type: array
rescheduleTriggeredAt:
description: "RescheduleTriggeredAt is a timestamp representing when
the referenced resource is triggered rescheduling. When this field
is updated, it means a rescheduling is manually triggered by user,
and the expected behavior of this action is to do a complete recalculation
without referring to last scheduling results. It works with the
status.lastScheduledTime field, and only when this timestamp is
later than timestamp in status.lastScheduledTime will the rescheduling
actually execute, otherwise, ignored. \n It is represented in RFC3339
form (like '2006-01-02T15:04:05Z') and is in UTC."
format: date-time
type: string
resource:
description: Resource represents the Kubernetes resource to be propagated.
properties:
Expand Down Expand Up @@ -1279,6 +1291,12 @@ spec:
- type
type: object
type: array
lastScheduledTime:
description: LastScheduledTime representing the latest timestamp when
scheduler successfully finished a scheduling. It is represented
in RFC3339 form (like '2006-01-02T15:04:05Z') and is in UTC.
format: date-time
type: string
schedulerObservedGeneration:
description: SchedulerObservedGeneration is the generation(.metadata.generation)
observed by the scheduler. If SchedulerObservedGeneration is less
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1128,6 +1128,18 @@ spec:
- name
type: object
type: array
rescheduleTriggeredAt:
description: "RescheduleTriggeredAt is a timestamp representing when
the referenced resource is triggered rescheduling. When this field
is updated, it means a rescheduling is manually triggered by user,
and the expected behavior of this action is to do a complete recalculation
without referring to last scheduling results. It works with the
status.lastScheduledTime field, and only when this timestamp is
later than timestamp in status.lastScheduledTime will the rescheduling
actually execute, otherwise, ignored. \n It is represented in RFC3339
form (like '2006-01-02T15:04:05Z') and is in UTC."
format: date-time
type: string
resource:
description: Resource represents the Kubernetes resource to be propagated.
properties:
Expand Down Expand Up @@ -1279,6 +1291,12 @@ spec:
- type
type: object
type: array
lastScheduledTime:
description: LastScheduledTime representing the latest timestamp when
scheduler successfully finished a scheduling. It is represented
in RFC3339 form (like '2006-01-02T15:04:05Z') and is in UTC.
format: date-time
type: string
schedulerObservedGeneration:
description: SchedulerObservedGeneration is the generation(.metadata.generation)
observed by the scheduler. If SchedulerObservedGeneration is less
Expand Down
14 changes: 14 additions & 0 deletions cmd/controller-manager/app/controllermanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ import (
"github.com/karmada-io/karmada/pkg/controllers/remediation"
"github.com/karmada-io/karmada/pkg/controllers/status"
"github.com/karmada-io/karmada/pkg/controllers/unifiedauth"
"github.com/karmada-io/karmada/pkg/controllers/workloadrebalancer"
"github.com/karmada-io/karmada/pkg/dependenciesdistributor"
"github.com/karmada-io/karmada/pkg/detector"
"github.com/karmada-io/karmada/pkg/features"
Expand Down Expand Up @@ -232,6 +233,7 @@ func init() {
controllers["endpointsliceCollect"] = startEndpointSliceCollectController
controllers["endpointsliceDispatch"] = startEndpointSliceDispatchController
controllers["remedy"] = startRemedyController
controllers["workloadRebalancer"] = startWorkloadRebalancerController
}

func startClusterController(ctx controllerscontext.Context) (enabled bool, err error) {
Expand Down Expand Up @@ -707,6 +709,18 @@ func startRemedyController(ctx controllerscontext.Context) (enabled bool, err er
return true, nil
}

func startWorkloadRebalancerController(ctx controllerscontext.Context) (enabled bool, err error) {
workloadRebalancer := workloadrebalancer.RebalancerController{
Client: ctx.Mgr.GetClient(),
}
err = workloadRebalancer.SetupWithManager(ctx.Mgr)
if err != nil {
return false, err
}

return true, nil
}

// setupControllers initialize controllers and setup one by one.
func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stopChan <-chan struct{}) {
restConfig := mgr.GetConfig()
Expand Down
6 changes: 6 additions & 0 deletions hack/tools/swagger/generateswagger.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"k8s.io/kube-openapi/pkg/validation/spec"

"github.com/karmada-io/karmada/hack/tools/swagger/lib"
appsv1alpha1 "github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1"
autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1"
Expand Down Expand Up @@ -112,6 +113,10 @@ func main() {
remedyv1alpha1.SchemeGroupVersion.WithResource(remedyv1alpha1.ResourcePluralRemedy),
remedyv1alpha1.SchemeGroupVersion.WithResource(remedyv1alpha1.ResourceSingularRemedy), meta.RESTScopeRoot)

mapper.AddSpecific(appsv1alpha1.SchemeGroupVersion.WithKind(appsv1alpha1.ResourceKindWorkloadRebalancer),
appsv1alpha1.SchemeGroupVersion.WithResource(appsv1alpha1.ResourcePluralWorkloadRebalancer),
appsv1alpha1.SchemeGroupVersion.WithResource(appsv1alpha1.ResourceSingularWorkloadRebalancer), meta.RESTScopeRoot)

spec, err := lib.RenderOpenAPISpec(lib.Config{
Info: spec.InfoProps{
Title: "Karmada OpenAPI",
Expand Down Expand Up @@ -145,6 +150,7 @@ func main() {
{GVR: autoscalingv1alpha1.SchemeGroupVersion.WithResource(autoscalingv1alpha1.ResourcePluralFederatedHPA), NamespaceScoped: autoscalingv1alpha1.ResourceNamespaceScopedFederatedHPA},
{GVR: autoscalingv1alpha1.SchemeGroupVersion.WithResource(autoscalingv1alpha1.ResourcePluralCronFederatedHPA), NamespaceScoped: autoscalingv1alpha1.ResourceNamespaceScopedCronFederatedHPA},
{GVR: remedyv1alpha1.SchemeGroupVersion.WithResource(remedyv1alpha1.ResourcePluralRemedy), NamespaceScoped: remedyv1alpha1.ResourceNamespaceScopedRemedy},
{GVR: appsv1alpha1.SchemeGroupVersion.WithResource(appsv1alpha1.ResourcePluralWorkloadRebalancer), NamespaceScoped: appsv1alpha1.ResourceNamespaceScopedWorkloadRebalancer},
},
Mapper: mapper,
})
Expand Down
17 changes: 14 additions & 3 deletions hack/update-codegen.sh
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,11 @@ deepcopy-gen \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--output-file-base=zz_generated.deepcopy
deepcopy-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-file-base=zz_generated.deepcopy

echo "Generating with register-gen"
register-gen \
Expand Down Expand Up @@ -167,6 +172,11 @@ register-gen \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--output-file-base=zz_generated.register
register-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-file-base=zz_generated.register

echo "Generating with conversion-gen"
conversion-gen \
Expand All @@ -184,7 +194,7 @@ echo "Generating with client-gen"
client-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-base="" \
--input=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--input=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/generated/clientset \
--clientset-name=versioned
client-gen \
Expand All @@ -197,7 +207,7 @@ client-gen \
echo "Generating with lister-gen"
lister-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/generated/listers
lister-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
Expand All @@ -207,7 +217,7 @@ lister-gen \
echo "Generating with informer-gen"
informer-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--versioned-clientset-package=github.com/karmada-io/karmada/pkg/generated/clientset/versioned \
--listers-package=github.com/karmada-io/karmada/pkg/generated/listers \
--output-package=github.com/karmada-io/karmada/pkg/generated/informers
Expand All @@ -229,6 +239,7 @@ openapi-gen \
--input-dirs "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" \
--input-dirs "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1" \
--input-dirs "github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1" \
--input-dirs "github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1" \
--input-dirs "k8s.io/api/core/v1,k8s.io/apimachinery/pkg/api/resource" \
--input-dirs "k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/apimachinery/pkg/runtime,k8s.io/apimachinery/pkg/version" \
--input-dirs "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1,k8s.io/api/admissionregistration/v1,k8s.io/api/networking/v1" \
Expand Down
21 changes: 21 additions & 0 deletions pkg/apis/apps/v1alpha1/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package v1alpha1 is the v1alpha1 version of the API.
// +k8s:deepcopy-gen=package,register
// +k8s:openapi-gen=true
// +groupName=apps.karmada.io
package v1alpha1
125 changes: 125 additions & 0 deletions pkg/apis/apps/v1alpha1/workloadrebalancer_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

const (
// ResourceKindWorkloadRebalancer is kind name of WorkloadRebalancer.
ResourceKindWorkloadRebalancer = "WorkloadRebalancer"
// ResourceSingularWorkloadRebalancer is singular name of WorkloadRebalancer.
ResourceSingularWorkloadRebalancer = "workloadrebalancer"
// ResourcePluralWorkloadRebalancer is kind plural name of WorkloadRebalancer.
ResourcePluralWorkloadRebalancer = "workloadrebalancers"
// ResourceNamespaceScopedWorkloadRebalancer indicates if WorkloadRebalancer is NamespaceScoped.
ResourceNamespaceScopedWorkloadRebalancer = false
)

// +genclient
// +genclient:nonNamespaced
// +kubebuilder:resource:path=workloadrebalancers,scope="Cluster",shortName=wr,categories={karmada-io}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// WorkloadRebalancer represents the desired behavior and status of a job which can enforces a rescheduling.
//
// Notes: make sure the clocks of controller-manager and scheduler are synchronized when using this API.
type WorkloadRebalancer struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

// Spec represents the specification of the desired behavior of WorkloadRebalancer.
// +required
Spec WorkloadRebalancerSpec `json:"spec"`

// Status represents the status of WorkloadRebalancer.
// +optional
Status WorkloadRebalancerStatus `json:"status,omitempty"`
}

// WorkloadRebalancerSpec represents the specification of the desired behavior of Reschedule.
//
// Notes: this API represents a one-time execution process, once the object is created, the execution process begins,
// and it will not respond to any modification of the spec field.
type WorkloadRebalancerSpec struct {
// Workloads used to specify the list of expected resource.
// Nil or empty list is not allowed.
// +kubebuilder:validation:MinItems=1
// +required
Workloads []Workload `json:"workloads"`
}

// Workload the expected resource.
type Workload struct {
// APIVersion represents the API version of the target resource.
// +required
APIVersion string `json:"apiVersion"`

// Kind represents the Kind of the target resource.
// +required
Kind string `json:"kind"`

// Name of the target resource.
// +required
Name string `json:"name"`

// Namespace of the target resource.
// Default is empty, which means it is a non-namespacescoped resource.
// +optional
Namespace string `json:"namespace,omitempty"`
}

// WorkloadRebalancerStatus contains information about the current status of a WorkloadRebalancer
// updated periodically by schedule trigger controller.
type WorkloadRebalancerStatus struct {
// ObservedWorkloads contains information about the execution states and messages of target resources.
// +optional
ObservedWorkloads []ObservedWorkload `json:"observedWorkloads,omitempty"`
}

// ObservedWorkload the observed resource.
type ObservedWorkload struct {
Workload `json:",inline"`

// State the observed state of resource.
// +optional
State ObservedState `json:"state,omitempty"`

// Reason represents a machine-readable description of why this workload failed.
Reason metav1.StatusReason `json:"reason,omitempty"`
}

// ObservedState the specific extent to which the resource has been executed
type ObservedState string

const (
// Failed the resource has been triggered a rescheduling failed.
Failed ObservedState = "Failed"
// Success the resource has been triggered a scheduling success.
Success ObservedState = "Failed"
)

// +kubebuilder:resource:scope="Cluster"
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// WorkloadRebalancerList contains a list of WorkloadRebalancer
type WorkloadRebalancerList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`

// Items holds a list of WorkloadRebalancer.
Items []WorkloadRebalancer `json:"items"`
}
Loading

0 comments on commit 760b6cc

Please sign in to comment.