Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

KEP-2170: Implement runtime framework #2248

Merged
merged 12 commits into from
Oct 17, 2024
Merged

KEP-2170: Implement runtime framework #2248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
013425d
KEP-2170: Implement runtime framework interfaces
tenzen-y Aug 27, 2024
d64cb34
Remove grep dependency
tenzen-y Sep 10, 2024
a0cae9a
KEP-2170: Implement ValidateObjects interface to the runtime framework
tenzen-y Oct 10, 2024
a527f86
KEP-2170: Expose the TrainingRuntime and ClusterTrainingRuntime Kind
tenzen-y Oct 10, 2024
2aaae2b
KEP-2170: Remove unneeded scheme field from the internal TrainingRuntime
tenzen-y Oct 10, 2024
1c88c85
Rephrase the error message
tenzen-y Oct 16, 2024
1c94ede
Distinguish TrainingRuntime and ClusterTrainingRuntime when creating …
tenzen-y Oct 16, 2024
338cf6e
Propagate the TrainJob labels and annotations to the JobSet
tenzen-y Oct 16, 2024
498ff28
Remove PodAnnotations from the runtime info
tenzen-y Oct 16, 2024
ba81beb
Implement TrainingRuntime ReplicatedJob validation
tenzen-y Oct 16, 2024
2f616e5
Add TODO comments
tenzen-y Oct 16, 2024
0c376d3
Replace queueSuspendedTrainJob with queueSuspendedTrainJobs
tenzen-y Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/unittests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ jobs:

- name: Run Go test for v2
run: |
make testv2
make test-integrationv2 ENVTEST_K8S_VERSION=${{ matrix.kubernetes-version }}

- name: Coveralls report
Expand Down
4 changes: 4 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ test: envtest
test-integrationv2: envtest
KUBEBUILDER_ASSETS="$(shell setup-envtest use $(ENVTEST_K8S_VERSION) -p path)" go test ./test/... -coverprofile cover.out

.PHONY: testv2
testv2:
go test ./pkg/controller.v2/... ./pkg/runtime.v2/... ./pkg/webhook.v2/... ./pkg/util.v2/... -coverprofile cover.out
tenzen-y marked this conversation as resolved.
Show resolved Hide resolved

envtest:
ifndef HAS_SETUP_ENVTEST
go install sigs.k8s.io/controller-runtime/tools/setup-envtest@bf15e44028f908c790721fc8fe67c7bf2d06a611 # v0.17.2
Expand Down
25 changes: 18 additions & 7 deletions cmd/training-operator.v2alpha1/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package main

import (
"context"
"crypto/tls"
"errors"
"flag"
Expand All @@ -25,7 +26,7 @@ import (

zaplog "go.uber.org/zap"
"go.uber.org/zap/zapcore"
"k8s.io/apimachinery/pkg/runtime"
apiruntime "k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -34,22 +35,25 @@ import (
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"
jobsetv1alpha2 "sigs.k8s.io/jobset/api/jobset/v1alpha2"
schedulerpluginsv1alpha1 "sigs.k8s.io/scheduler-plugins/apis/scheduling/v1alpha1"

kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1"
"github.com/kubeflow/training-operator/pkg/cert"
controllerv2 "github.com/kubeflow/training-operator/pkg/controller.v2"
runtimecore "github.com/kubeflow/training-operator/pkg/runtime.v2/core"
webhookv2 "github.com/kubeflow/training-operator/pkg/webhook.v2"
)

var (
scheme = runtime.NewScheme()
scheme = apiruntime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
)

func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(kubeflowv2.AddToScheme(scheme))
utilruntime.Must(jobsetv1alpha2.AddToScheme(scheme))
utilruntime.Must(schedulerpluginsv1alpha1.AddToScheme(scheme))
}

func main() {
Expand Down Expand Up @@ -127,27 +131,34 @@ func main() {
os.Exit(1)
}

ctx := ctrl.SetupSignalHandler()

setupProbeEndpoints(mgr, certsReady)
// Set up controllers using goroutines to start the manager quickly.
go setupControllers(mgr, certsReady)
go setupControllers(ctx, mgr, certsReady)

setupLog.Info("Starting manager")
if err = mgr.Start(ctrl.SetupSignalHandler()); err != nil {
if err = mgr.Start(ctx); err != nil {
setupLog.Error(err, "Could not run manager")
os.Exit(1)
}
}

func setupControllers(mgr ctrl.Manager, certsReady <-chan struct{}) {
func setupControllers(ctx context.Context, mgr ctrl.Manager, certsReady <-chan struct{}) {
setupLog.Info("Waiting for certificate generation to complete")
<-certsReady
setupLog.Info("Certs ready")

if failedCtrlName, err := controllerv2.SetupControllers(mgr); err != nil {
runtimes, err := runtimecore.New(ctx, mgr.GetClient(), mgr.GetFieldIndexer())
if err != nil {
setupLog.Error(err, "Could not initialize runtimes")
os.Exit(1)
}
if failedCtrlName, err := controllerv2.SetupControllers(mgr, runtimes); err != nil {
setupLog.Error(err, "Could not create controller", "controller", failedCtrlName)
os.Exit(1)
}
if failedWebhook, err := webhookv2.Setup(mgr); err != nil {
if failedWebhook, err := webhookv2.Setup(mgr, runtimes); err != nil {
setupLog.Error(err, "Could not create webhook", "webhook", failedWebhook)
os.Exit(1)
}
Expand Down
66 changes: 33 additions & 33 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,49 +3,50 @@ module github.com/kubeflow/training-operator
go 1.22

require (
github.com/go-logr/logr v1.4.1
github.com/go-logr/logr v1.4.2
github.com/google/go-cmp v0.6.0
github.com/onsi/ginkgo/v2 v2.17.1
github.com/onsi/gomega v1.32.0
github.com/onsi/ginkgo/v2 v2.19.0
github.com/onsi/gomega v1.33.1
github.com/open-policy-agent/cert-controller v0.10.1
github.com/prometheus/client_golang v1.18.0
github.com/sirupsen/logrus v1.9.0
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
go.uber.org/zap v1.27.0
k8s.io/api v0.29.3
k8s.io/apimachinery v0.29.3
k8s.io/client-go v0.29.3
k8s.io/code-generator v0.29.3
k8s.io/klog/v2 v2.110.1
k8s.io/api v0.29.5
k8s.io/apimachinery v0.29.5
k8s.io/client-go v0.29.5
k8s.io/code-generator v0.29.5
k8s.io/klog/v2 v2.120.1
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
sigs.k8s.io/controller-runtime v0.17.3
sigs.k8s.io/jobset v0.5.2
sigs.k8s.io/kueue v0.6.3
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need kueue dependency ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This dependency came from

training-operator/pkg/runtime.v2/runtime.go

Line 125 in 22da8af

PodRequests: kueuelr.TotalRequests(&spec.podSpec),
.

This allows us to set the appropriate required resources for PodGroup. If we remove this dependency, we need to just copy Kueue's "TotalRequests" function here. I believe that just coping and pasting is not the ideal way.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be great to reduce the dependency. Maybe copy and paste is okay in this case as long as we provide a reference to the original source

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's not simple implementation. That's so complex, multiple files and lines codes.
So, I would propose keeping it here and then (after kube 1.32) switching to the kube library as I mentioned in #2280.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we discussed with @tenzen-y offline that after k/k separates this utility function, we will remove dependency on Kueue.

andreyvelich marked this conversation as resolved.
Show resolved Hide resolved
sigs.k8s.io/scheduler-plugins v0.28.9
sigs.k8s.io/yaml v1.4.0
volcano.sh/apis v1.9.0
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch v5.9.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.8.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-logr/zapr v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand All @@ -56,30 +57,29 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/mod v0.16.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.19.0 // indirect
golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/oauth2 v0.20.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.33.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.29.2 // indirect
k8s.io/component-base v0.29.2 // indirect
k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
k8s.io/component-base v0.29.5 // indirect
k8s.io/gengo v0.0.0-20240404160639-a0386bf69313 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
)
Loading
Loading