From 12f205ef89f2fe3151588100162227eb085e086a Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Wed, 27 Mar 2024 14:14:20 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9A=A0=EF=B8=8F=20Breaking=20Changes:=20Migr?= =?UTF-8?q?ate=20model=20subcommand=20to=20model=20analyze=20(#1060)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Yi Chen --- charts/modeljob/templates/job.yaml | 10 +- charts/pytorchjob/templates/ingress.yaml | 4 +- .../templates/seldondeployment.yaml | 8 +- charts/tfjob/templates/ingress.yaml | 2 +- charts/trtserving/templates/deployment.yaml | 4 +- docs/model/benchmark/benchmark_torchscript.md | 12 +- docs/model/index.md | 5 +- docs/model/optimize/optimize_torchscript.md | 10 +- docs/model/profile/profile_torchscript.md | 12 +- pkg/apis/arenaclient/analyze_client.go | 102 +++++++++++++++++ pkg/apis/arenaclient/arenaclient.go | 4 +- pkg/apis/arenaclient/model_client.go | 103 ------------------ .../model/{ => analyze}/benchmark_builder.go | 5 +- .../model/{ => analyze}/evaluate_builder.go | 5 +- pkg/apis/model/{ => analyze}/job.go | 2 +- .../model/{ => analyze}/optimize_builder.go | 5 +- .../model/{ => analyze}/profile_builder.go | 5 +- pkg/commands/model/analyze/analyze.go | 22 ++++ pkg/commands/model/{ => analyze}/delete.go | 7 +- pkg/commands/model/{ => analyze}/get.go | 7 +- pkg/commands/model/{ => analyze}/list.go | 7 +- .../model/{ => analyze}/submit_benchmark.go | 9 +- .../model/{ => analyze}/submit_evaluate.go | 9 +- .../model/{ => analyze}/submit_optimize.go | 13 ++- .../model/{ => analyze}/submit_profile.go | 9 +- pkg/commands/model/model.go | 24 +--- pkg/model/{ => analyze}/delete.go | 5 +- pkg/model/{ => analyze}/get.go | 9 +- pkg/model/{ => analyze}/list.go | 13 ++- pkg/model/{ => analyze}/model.go | 9 +- pkg/model/{ => analyze}/model_interface.go | 5 +- pkg/model/{ => analyze}/submit_benchmark.go | 5 +- pkg/model/{ => analyze}/submit_evaluate.go | 5 +- pkg/model/{ => analyze}/submit_optimize.go | 5 +- pkg/model/{ => analyze}/submit_profile.go | 5 +- 35 files changed, 242 insertions(+), 224 deletions(-) create mode 100644 pkg/apis/arenaclient/analyze_client.go delete mode 100644 pkg/apis/arenaclient/model_client.go rename pkg/apis/model/{ => analyze}/benchmark_builder.go (99%) rename pkg/apis/model/{ => analyze}/evaluate_builder.go (99%) rename pkg/apis/model/{ => analyze}/job.go (97%) rename pkg/apis/model/{ => analyze}/optimize_builder.go (99%) rename pkg/apis/model/{ => analyze}/profile_builder.go (99%) create mode 100644 pkg/commands/model/analyze/analyze.go rename pkg/commands/model/{ => analyze}/delete.go (89%) rename pkg/commands/model/{ => analyze}/get.go (89%) rename pkg/commands/model/{ => analyze}/list.go (89%) rename pkg/commands/model/{ => analyze}/submit_benchmark.go (87%) rename pkg/commands/model/{ => analyze}/submit_evaluate.go (87%) rename pkg/commands/model/{ => analyze}/submit_optimize.go (78%) rename pkg/commands/model/{ => analyze}/submit_profile.go (88%) rename pkg/model/{ => analyze}/delete.go (97%) rename pkg/model/{ => analyze}/get.go (99%) rename pkg/model/{ => analyze}/list.go (99%) rename pkg/model/{ => analyze}/model.go (99%) rename pkg/model/{ => analyze}/model_interface.go (98%) rename pkg/model/{ => analyze}/submit_benchmark.go (89%) rename pkg/model/{ => analyze}/submit_evaluate.go (89%) rename pkg/model/{ => analyze}/submit_optimize.go (88%) rename pkg/model/{ => analyze}/submit_profile.go (87%) diff --git a/charts/modeljob/templates/job.yaml b/charts/modeljob/templates/job.yaml index 902540036..1c8e41d06 100644 --- a/charts/modeljob/templates/job.yaml +++ b/charts/modeljob/templates/job.yaml @@ -158,12 +158,12 @@ spec: {{- if gt (int $gpuCount) 0}} nvidia.com/gpu: {{ .Values.gpuCount }} {{- end }} - {{ - if gt (int $gpuMemory) 0 }} - aliyun.com/gpu-mem: { { .Values.gpuMemory }} - {{ - end } } - {{ - if gt (int $gpuCore) 0 }} + {{- if gt (int $gpuMemory) 0 }} + aliyun.com/gpu-mem: {{ .Values.gpuMemory }} + {{- end }} + {{- if gt (int $gpuCore) 0 }} aliyun.com/gpu-core.percentage: {{ .Values.gpuCore }} - {{ - end }} + {{- end }} volumeMounts: {{- if .Values.dataset }} {{- range $pvcName, $mntPath := .Values.dataset}} diff --git a/charts/pytorchjob/templates/ingress.yaml b/charts/pytorchjob/templates/ingress.yaml index d65616792..9b8742a72 100644 --- a/charts/pytorchjob/templates/ingress.yaml +++ b/charts/pytorchjob/templates/ingress.yaml @@ -13,8 +13,8 @@ metadata: createdBy: "PyTorchJob" controller-name: pytorch-operator group-name: kubeflow.org - job-name: { { .Release.Name } } - pytorch-job-name: { { .Release.Name } } + job-name: {{ .Release.Name }} + pytorch-job-name: {{ .Release.Name }} spec: rules: - http: diff --git a/charts/seldon-core/templates/seldondeployment.yaml b/charts/seldon-core/templates/seldondeployment.yaml index 2a42d431d..1a2105ef8 100644 --- a/charts/seldon-core/templates/seldondeployment.yaml +++ b/charts/seldon-core/templates/seldondeployment.yaml @@ -41,12 +41,12 @@ spec: {{- if gt (int $gpuCount) 0 }} nvidia.com/gpu: {{ .Values.gpuCount }} {{- end }} - {{ - if gt (int $gpuMemory) 0 }} + {{- if gt (int $gpuMemory) 0 }} aliyun.com/gpu-mem: {{ .Values.gpuMemory }} - {{ - end } } - {{ - if gt (int $gpuCore) 0 }} + {{- end }} + {{- if gt (int $gpuCore) 0 }} aliyun.com/gpu-core.percentage: {{ .Values.gpuCore }} - {{ - end }} + {{- end }} graph: implementation: {{ .Values.implementation }} modelUri: {{ .Values.modelUri }} diff --git a/charts/tfjob/templates/ingress.yaml b/charts/tfjob/templates/ingress.yaml index 28537f1e9..aa27f0039 100644 --- a/charts/tfjob/templates/ingress.yaml +++ b/charts/tfjob/templates/ingress.yaml @@ -13,7 +13,7 @@ metadata: role: tensorboard createdBy: "TFJob" group-name: kubeflow.org - tf-job-name: { { .Release.Name } } + tf-job-name: {{ .Release.Name }} spec: rules: - http: diff --git a/charts/trtserving/templates/deployment.yaml b/charts/trtserving/templates/deployment.yaml index a5425fdd5..5070b6d43 100755 --- a/charts/trtserving/templates/deployment.yaml +++ b/charts/trtserving/templates/deployment.yaml @@ -104,9 +104,9 @@ spec: {{- if gt (int $gpuMemory) 0}} aliyun.com/gpu-mem: {{ .Values.gpuMemory }} {{- end }} - {{ - if gt (int $gpuCore) 0 }} + {{- if gt (int $gpuCore) 0 }} aliyun.com/gpu-core.percentage: {{ .Values.gpuCore }} - {{ - end }} + {{- end }} volumeMounts: {{- if .Values.shareMemory }} - mountPath: /dev/shm diff --git a/docs/model/benchmark/benchmark_torchscript.md b/docs/model/benchmark/benchmark_torchscript.md index 1d7d492b3..b16c2dc6e 100644 --- a/docs/model/benchmark/benchmark_torchscript.md +++ b/docs/model/benchmark/benchmark_torchscript.md @@ -65,7 +65,7 @@ Then give a profile configuration file named config.json like below. 3\. Submit a model benchmark job. ```shell -$ arena model benchmark \ +$ arena model analyze benchmark \ --name=resnet18-benchmark \ --namespace=default \ --image=registry.cn-beijing.aliyuncs.com/kube-ai/easy-inference:1.0.0 \ @@ -80,13 +80,13 @@ $ arena model benchmark \ job.batch/resnet18-benchmark created INFO[0000] The model benchmark job resnet18-benchmark has been submitted successfully -INFO[0000] You can run `arena model get resnet18-benchmark` to check the job status +INFO[0000] You can run `arena model analyze get resnet18-benchmark` to check the job status ``` 4\. List all the model benchmark jobs. ```shell -$ arena model list +$ arena model analyze list NAMESPACE NAME STATUS TYPE DURATION AGE GPU(Requested) default resnet18-benchmark RUNNING Benchmark 23s 23s 1 @@ -95,7 +95,7 @@ default resnet18-benchmark RUNNING Benchmark 23s 23s 1 5\. Get model benchmark job detail info. ```shell -$ arena model get resnet18-benchmark +$ arena model analyze get resnet18-benchmark Name: resnet18-benchmark Namespace: default Type: Benchmark @@ -125,7 +125,3 @@ Benchmark finished, cost 60.00157570838928 s Benchmark result: {"p90_latency": 3.806, "p95_latency": 3.924, "p99_latency": 4.781, "min_latency": 3.665, "max_latency": 1555.418, "mean_latency": 3.88, "median_latency": 3.731, "throughput": 257, "gpu_mem_used": 1.47, "gpu_utilization": 38.39514839785918} ``` - - - - diff --git a/docs/model/index.md b/docs/model/index.md index 18ab0c01b..6d1672f93 100644 --- a/docs/model/index.md +++ b/docs/model/index.md @@ -1,7 +1,6 @@ # Model Analyze Guide -Welcome to the Arena Model Guide! This guide covers how to use the ``arena cli`` to profile the model to find performance bottleneck, and how to use tensorrt to optimize the inference performance, you can also benchmark the model to get inference metrics like qps, latency, gpu usage and so on. This page outlines the most common situations and questions that bring readers to this section. - +Welcome to the Arena Model Analyze Guide! This guide covers how to use the `arena cli` to profile the model to find performance bottleneck, and how to use tensorrt to optimize the inference performance, you can also benchmark the model to get inference metrics like qps, latency, gpu usage and so on. This page outlines the most common situations and questions that bring readers to this section. ## Who should use this guide? @@ -15,8 +14,6 @@ After training you may get some models. If you want to know the model performanc * I want to [optimize the torchscript module with tensorrt](optimize/optimize_torchscript.md). - ## Benchmark the model inference * I want to [benchmark the torchscript inference performance](benchmark/benchmark_torchscript.md). - diff --git a/docs/model/optimize/optimize_torchscript.md b/docs/model/optimize/optimize_torchscript.md index df888080d..8ba1d1ed7 100644 --- a/docs/model/optimize/optimize_torchscript.md +++ b/docs/model/optimize/optimize_torchscript.md @@ -65,7 +65,7 @@ Then give a profile configuration file named config.json like below. 3\. Submit a model optimize job. ```shell -$ arena model optimize \ +$ arena model analyze optimize \ --name=resnet18-optimize \ --namespace=default \ --image=registry.cn-beijing.aliyuncs.com/kube-ai/easy-inference:1.0.0 \ @@ -78,13 +78,13 @@ $ arena model optimize \ job.batch/resnet18-optimize created INFO[0002] The model optimize job resnet18-optimize has been submitted successfully -INFO[0002] You can run `arena model get resnet18-optimize` to check the job status +INFO[0002] You can run `arena model analyze get resnet18-optimize` to check the job status ``` 4\. List all the model optimize jobs. ```shell -$ arena model list +$ arena model analyze list NAMESPACE NAME STATUS TYPE DURATION AGE GPU(Requested) default-group resnet18-optimize RUNNING Optimize 0s 1m 1 @@ -93,7 +93,7 @@ default-group resnet18-optimize RUNNING Optimize 0s 1m 1 5\. Get model optimize job detail info. ```shell -$ arena model get resnet18-profile +$ arena model analyze get resnet18-profile Name: resnet18-optimize Namespace: default-group Type: Optimize @@ -111,4 +111,4 @@ Instances: resnet18-optimize-xrd6w ContainerCreating 1m 0/1 0 cn-shenzhen.192.168.1.209 ``` -6\. After the optimize job finished, you can see a new torchscript modue named opt_resnet18.pt in --export-path. \ No newline at end of file +6\. After the optimize job finished, you can see a new torchscript modue named opt_resnet18.pt in --export-path. diff --git a/docs/model/profile/profile_torchscript.md b/docs/model/profile/profile_torchscript.md index 624db83e4..9347b9e1a 100644 --- a/docs/model/profile/profile_torchscript.md +++ b/docs/model/profile/profile_torchscript.md @@ -65,7 +65,7 @@ Then give a profile configuration file named config.json like below. 3\. Submit a model profile job. ```shell -$ arena model profile \ +$ arena model analyze profile \ --name=resnet18-profile \ --namespace=default \ --image=registry.cn-beijing.aliyuncs.com/kube-ai/easy-inference:1.0.0 \ @@ -82,13 +82,13 @@ service/resnet18-profile-tensorboard created deployment.apps/resnet18-profile-tensorboard created job.batch/resnet18-profile created INFO[0001] The model profile job resnet18-profile has been submitted successfully -INFO[0001] You can run `arena model get resnet18-profile` to check the job status +INFO[0001] You can run `arena model analyze get resnet18-profile` to check the job status ``` 4\. List all the profile jobs. ```shell -$ arena model list +$ arena model analyze list NAMESPACE NAME STATUS TYPE DURATION AGE GPU(Requested) default resnet18-profile RUNNING Profile 34s 34s 1 @@ -97,7 +97,7 @@ default resnet18-profile RUNNING Profile 34s 34s 1 5\. Get model profile job detail info. ```shell -$ arena model get resnet18-profile +$ arena model analyze get resnet18-profile Name: resnet18-profile Namespace: default Type: Profile @@ -126,6 +126,4 @@ Forwarding from 127.0.0.1:6006 -> 6006 Forwarding from [::1]:6006 -> 6006 ``` - - -![tensorboard](./1-torchscript-profile-result.jpg) \ No newline at end of file +![tensorboard](./1-torchscript-profile-result.jpg) diff --git a/pkg/apis/arenaclient/analyze_client.go b/pkg/apis/arenaclient/analyze_client.go new file mode 100644 index 000000000..cbf4ea902 --- /dev/null +++ b/pkg/apis/arenaclient/analyze_client.go @@ -0,0 +1,102 @@ +package arenaclient + +import ( + "github.com/kubeflow/arena/pkg/apis/config" + apisanalyze "github.com/kubeflow/arena/pkg/apis/model/analyze" + "github.com/kubeflow/arena/pkg/apis/types" + "github.com/kubeflow/arena/pkg/apis/utils" + "github.com/kubeflow/arena/pkg/model/analyze" +) + +type AnalyzeClient struct { + namespace string + configer *config.ArenaConfiger +} + +func NewAnalyzeClient(namespace string, configer *config.ArenaConfiger) *AnalyzeClient { + return &AnalyzeClient{ + namespace: namespace, + configer: configer, + } +} + +// Namespace sets the namespace,this operation does not change the default namespace +func (m *AnalyzeClient) Namespace(namespace string) *AnalyzeClient { + copyModelJobClient := &AnalyzeClient{ + namespace: namespace, + configer: m.configer, + } + return copyModelJobClient +} + +func (m *AnalyzeClient) Submit(job *apisanalyze.Job) error { + switch job.Type() { + case types.ModelProfileJob: + args := job.Args().(*types.ModelProfileArgs) + return analyze.SubmitModelProfileJob(args.Namespace, args) + case types.ModelOptimizeJob: + args := job.Args().(*types.ModelOptimizeArgs) + return analyze.SubmitModelOptimizeJob(args.Namespace, args) + case types.ModelBenchmarkJob: + args := job.Args().(*types.ModelBenchmarkArgs) + return analyze.SubmitModelBenchmarkJob(args.Namespace, args) + case types.ModelEvaluateJob: + args := job.Args().(*types.ModelEvaluateArgs) + return analyze.SubmitModelEvaluateJob(args.Namespace, args) + } + return nil +} + +func (m *AnalyzeClient) Get(jobType types.ModelJobType, name string) (*types.ModelJobInfo, error) { + job, err := analyze.SearchModelJob(m.namespace, name, jobType) + if err != nil { + return nil, err + } + + jobInfo := job.Convert2JobInfo() + return &jobInfo, nil +} + +func (m *AnalyzeClient) GetAndPrint(jobType types.ModelJobType, name string, format string) error { + job, err := analyze.SearchModelJob(m.namespace, name, jobType) + if err != nil { + return err + } + + analyze.PrintModelJob(job, utils.TransferPrintFormat(format)) + return nil +} + +func (m *AnalyzeClient) List(allNamespaces bool, jobType types.ModelJobType) ([]*types.ModelJobInfo, error) { + jobs, err := analyze.ListModelJobs(m.namespace, allNamespaces, jobType) + if err != nil { + return nil, err + } + + var jobInfos []*types.ModelJobInfo + for _, job := range jobs { + jobInfo := job.Convert2JobInfo() + jobInfos = append(jobInfos, &jobInfo) + } + return jobInfos, nil +} + +func (m *AnalyzeClient) ListAndPrint(allNamespaces bool, jobType types.ModelJobType, format string) error { + jobs, err := analyze.ListModelJobs(m.namespace, allNamespaces, jobType) + if err != nil { + return err + } + + analyze.PrintAllModelJobs(jobs, allNamespaces, utils.TransferPrintFormat(format)) + return nil +} + +func (m *AnalyzeClient) Delete(jobType types.ModelJobType, jobNames ...string) error { + for _, jobName := range jobNames { + err := analyze.DeleteModelJob(m.namespace, jobName, jobType) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/apis/arenaclient/arenaclient.go b/pkg/apis/arenaclient/arenaclient.go index 29bdf9f4a..4e9e1ca6c 100644 --- a/pkg/apis/arenaclient/arenaclient.go +++ b/pkg/apis/arenaclient/arenaclient.go @@ -83,6 +83,6 @@ func (a *ArenaClient) Evaluate() *EvaluateClient { return NewEvaluateClient(a.namespace, a.arenaConfiger) } -func (a *ArenaClient) Model() *ModelClient { - return NewModelClient(a.namespace, a.arenaConfiger) +func (a *ArenaClient) Analyze() *AnalyzeClient { + return NewAnalyzeClient(a.namespace, a.arenaConfiger) } diff --git a/pkg/apis/arenaclient/model_client.go b/pkg/apis/arenaclient/model_client.go deleted file mode 100644 index 07716a515..000000000 --- a/pkg/apis/arenaclient/model_client.go +++ /dev/null @@ -1,103 +0,0 @@ -package arenaclient - -import ( - "github.com/kubeflow/arena/pkg/apis/config" - apismodel "github.com/kubeflow/arena/pkg/apis/model" - "github.com/kubeflow/arena/pkg/apis/types" - "github.com/kubeflow/arena/pkg/apis/utils" - "github.com/kubeflow/arena/pkg/model" -) - -type ModelClient struct { - namespace string - configer *config.ArenaConfiger -} - -// NewModelClient creates a ModelClient -func NewModelClient(namespace string, configer *config.ArenaConfiger) *ModelClient { - return &ModelClient{ - namespace: namespace, - configer: configer, - } -} - -// Namespace sets the namespace,this operation does not change the default namespace -func (m *ModelClient) Namespace(namespace string) *ModelClient { - copyModelJobClient := &ModelClient{ - namespace: namespace, - configer: m.configer, - } - return copyModelJobClient -} - -func (m *ModelClient) Submit(job *apismodel.Job) error { - switch job.Type() { - case types.ModelProfileJob: - args := job.Args().(*types.ModelProfileArgs) - return model.SubmitModelProfileJob(args.Namespace, args) - case types.ModelOptimizeJob: - args := job.Args().(*types.ModelOptimizeArgs) - return model.SubmitModelOptimizeJob(args.Namespace, args) - case types.ModelBenchmarkJob: - args := job.Args().(*types.ModelBenchmarkArgs) - return model.SubmitModelBenchmarkJob(args.Namespace, args) - case types.ModelEvaluateJob: - args := job.Args().(*types.ModelEvaluateArgs) - return model.SubmitModelEvaluateJob(args.Namespace, args) - } - return nil -} - -func (m *ModelClient) Get(jobType types.ModelJobType, name string) (*types.ModelJobInfo, error) { - job, err := model.SearchModelJob(m.namespace, name, jobType) - if err != nil { - return nil, err - } - - jobInfo := job.Convert2JobInfo() - return &jobInfo, nil -} - -func (m *ModelClient) GetAndPrint(jobType types.ModelJobType, name string, format string) error { - job, err := model.SearchModelJob(m.namespace, name, jobType) - if err != nil { - return err - } - - model.PrintModelJob(job, utils.TransferPrintFormat(format)) - return nil -} - -func (m *ModelClient) List(allNamespaces bool, jobType types.ModelJobType) ([]*types.ModelJobInfo, error) { - jobs, err := model.ListModelJobs(m.namespace, allNamespaces, jobType) - if err != nil { - return nil, err - } - - var jobInfos []*types.ModelJobInfo - for _, job := range jobs { - jobInfo := job.Convert2JobInfo() - jobInfos = append(jobInfos, &jobInfo) - } - return jobInfos, nil -} - -func (m *ModelClient) ListAndPrint(allNamespaces bool, jobType types.ModelJobType, format string) error { - jobs, err := model.ListModelJobs(m.namespace, allNamespaces, jobType) - if err != nil { - return err - } - - model.PrintAllModelJobs(jobs, allNamespaces, utils.TransferPrintFormat(format)) - return nil -} - -func (m *ModelClient) Delete(jobType types.ModelJobType, jobNames ...string) error { - for _, jobName := range jobNames { - err := model.DeleteModelJob(m.namespace, jobName, jobType) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/apis/model/benchmark_builder.go b/pkg/apis/model/analyze/benchmark_builder.go similarity index 99% rename from pkg/apis/model/benchmark_builder.go rename to pkg/apis/model/analyze/benchmark_builder.go index ad0c1d21f..be6d89020 100644 --- a/pkg/apis/model/benchmark_builder.go +++ b/pkg/apis/model/analyze/benchmark_builder.go @@ -1,10 +1,11 @@ -package model +package analyze import ( "fmt" + "strings" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/argsbuilder" - "strings" ) type ModelBenchmarkArgsBuilder struct { diff --git a/pkg/apis/model/evaluate_builder.go b/pkg/apis/model/analyze/evaluate_builder.go similarity index 99% rename from pkg/apis/model/evaluate_builder.go rename to pkg/apis/model/analyze/evaluate_builder.go index 6a56b332f..eee52818c 100644 --- a/pkg/apis/model/evaluate_builder.go +++ b/pkg/apis/model/analyze/evaluate_builder.go @@ -1,10 +1,11 @@ -package model +package analyze import ( "fmt" + "strings" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/argsbuilder" - "strings" ) type ModelEvaluateJobBuilder struct { diff --git a/pkg/apis/model/job.go b/pkg/apis/model/analyze/job.go similarity index 97% rename from pkg/apis/model/job.go rename to pkg/apis/model/analyze/job.go index ab5f6a0a7..1f5e78c29 100644 --- a/pkg/apis/model/job.go +++ b/pkg/apis/model/analyze/job.go @@ -1,4 +1,4 @@ -package model +package analyze import "github.com/kubeflow/arena/pkg/apis/types" diff --git a/pkg/apis/model/optimize_builder.go b/pkg/apis/model/analyze/optimize_builder.go similarity index 99% rename from pkg/apis/model/optimize_builder.go rename to pkg/apis/model/analyze/optimize_builder.go index 0c726c72a..3527637cd 100644 --- a/pkg/apis/model/optimize_builder.go +++ b/pkg/apis/model/analyze/optimize_builder.go @@ -1,10 +1,11 @@ -package model +package analyze import ( "fmt" + "strings" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/argsbuilder" - "strings" ) type ModelOptimizeJobBuilder struct { diff --git a/pkg/apis/model/profile_builder.go b/pkg/apis/model/analyze/profile_builder.go similarity index 99% rename from pkg/apis/model/profile_builder.go rename to pkg/apis/model/analyze/profile_builder.go index d4a62d6be..2a8044eec 100644 --- a/pkg/apis/model/profile_builder.go +++ b/pkg/apis/model/analyze/profile_builder.go @@ -1,10 +1,11 @@ -package model +package analyze import ( "fmt" + "strings" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/argsbuilder" - "strings" ) type ModelProfileJobBuilder struct { diff --git a/pkg/commands/model/analyze/analyze.go b/pkg/commands/model/analyze/analyze.go new file mode 100644 index 000000000..35df8e77d --- /dev/null +++ b/pkg/commands/model/analyze/analyze.go @@ -0,0 +1,22 @@ +package analyze + +import "github.com/spf13/cobra" + +func NewAnalyzeCommand() *cobra.Command { + var command = &cobra.Command{ + Use: "analyze", + Short: "Submit a model analyze job. (experimental feature)", + Run: func(cmd *cobra.Command, args []string) { + cmd.HelpFunc()(cmd, args) + }, + } + command.AddCommand(NewSubmitModelProfileJobCommand()) + command.AddCommand(NewSubmitModelOptimizeJobCommand()) + command.AddCommand(NewSubmitModelBenchmarkJobCommand()) + command.AddCommand(NewSubmitModelEvaluateJobCommand()) + command.AddCommand(NewGetModelJobCommand()) + command.AddCommand(NewListModelJobsCommand()) + command.AddCommand(NewDeleteModelJobCommand()) + + return command +} diff --git a/pkg/commands/model/delete.go b/pkg/commands/model/analyze/delete.go similarity index 89% rename from pkg/commands/model/delete.go rename to pkg/commands/model/analyze/delete.go index cfc507b71..972c2c569 100644 --- a/pkg/commands/model/delete.go +++ b/pkg/commands/model/analyze/delete.go @@ -1,7 +1,8 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/arenaclient" "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/apis/utils" @@ -13,7 +14,7 @@ func NewDeleteModelJobCommand() *cobra.Command { var jobType string var command = &cobra.Command{ Use: "delete", - Short: "Delete a model job", + Short: "Delete a model analyze job", PreRun: func(cmd *cobra.Command, args []string) { _ = viper.BindPFlags(cmd.Flags()) }, @@ -33,7 +34,7 @@ func NewDeleteModelJobCommand() *cobra.Command { if err != nil { return err } - return client.Model().Delete(utils.TransferModelJobType(jobType), names...) + return client.Analyze().Delete(utils.TransferModelJobType(jobType), names...) }, } diff --git a/pkg/commands/model/get.go b/pkg/commands/model/analyze/get.go similarity index 89% rename from pkg/commands/model/get.go rename to pkg/commands/model/analyze/get.go index b04220138..d5fca3dcf 100644 --- a/pkg/commands/model/get.go +++ b/pkg/commands/model/analyze/get.go @@ -1,7 +1,8 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/arenaclient" "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/apis/utils" @@ -14,7 +15,7 @@ func NewGetModelJobCommand() *cobra.Command { var output string var command = &cobra.Command{ Use: "get", - Short: "Get a model job", + Short: "Get a model analyze job", PreRun: func(cmd *cobra.Command, args []string) { _ = viper.BindPFlags(cmd.Flags()) }, @@ -34,7 +35,7 @@ func NewGetModelJobCommand() *cobra.Command { if err != nil { return err } - return client.Model().GetAndPrint(utils.TransferModelJobType(jobType), name, output) + return client.Analyze().GetAndPrint(utils.TransferModelJobType(jobType), name, output) }, } diff --git a/pkg/commands/model/list.go b/pkg/commands/model/analyze/list.go similarity index 89% rename from pkg/commands/model/list.go rename to pkg/commands/model/analyze/list.go index 8cea32cd9..cc2b6a53f 100644 --- a/pkg/commands/model/list.go +++ b/pkg/commands/model/analyze/list.go @@ -1,7 +1,8 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/arenaclient" "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/apis/utils" @@ -15,7 +16,7 @@ func NewListModelJobsCommand() *cobra.Command { var jobType string var command = &cobra.Command{ Use: "list", - Short: "List all the model jobs", + Short: "List all model analyze jobs", Aliases: []string{"ls"}, PreRun: func(cmd *cobra.Command, args []string) { _ = viper.BindPFlags(cmd.Flags()) @@ -31,7 +32,7 @@ func NewListModelJobsCommand() *cobra.Command { if err != nil { return fmt.Errorf("failed to create arena client: %v", err) } - return client.Model().ListAndPrint(allNamespaces, utils.TransferModelJobType(jobType), format) + return client.Analyze().ListAndPrint(allNamespaces, utils.TransferModelJobType(jobType), format) }, } command.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "show all the namespaces") diff --git a/pkg/commands/model/submit_benchmark.go b/pkg/commands/model/analyze/submit_benchmark.go similarity index 87% rename from pkg/commands/model/submit_benchmark.go rename to pkg/commands/model/analyze/submit_benchmark.go index cc1734613..23f62ccd2 100644 --- a/pkg/commands/model/submit_benchmark.go +++ b/pkg/commands/model/analyze/submit_benchmark.go @@ -1,17 +1,18 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/arenaclient" "github.com/kubeflow/arena/pkg/apis/config" - "github.com/kubeflow/arena/pkg/apis/model" + "github.com/kubeflow/arena/pkg/apis/model/analyze" "github.com/kubeflow/arena/pkg/apis/types" "github.com/spf13/cobra" "github.com/spf13/viper" ) func NewSubmitModelBenchmarkJobCommand() *cobra.Command { - builder := model.NewModelBenchmarkArgsBuilder() + builder := analyze.NewModelBenchmarkArgsBuilder() var command = &cobra.Command{ Use: "benchmark", Short: "Submit a model benchmark job", @@ -33,7 +34,7 @@ func NewSubmitModelBenchmarkJobCommand() *cobra.Command { if err != nil { return fmt.Errorf("failed to validate command args: %v", err) } - return client.Model().Submit(job) + return client.Analyze().Submit(job) }, } builder.AddCommandFlags(command) diff --git a/pkg/commands/model/submit_evaluate.go b/pkg/commands/model/analyze/submit_evaluate.go similarity index 87% rename from pkg/commands/model/submit_evaluate.go rename to pkg/commands/model/analyze/submit_evaluate.go index 6cc0049da..236e45b2d 100644 --- a/pkg/commands/model/submit_evaluate.go +++ b/pkg/commands/model/analyze/submit_evaluate.go @@ -1,17 +1,18 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/arenaclient" "github.com/kubeflow/arena/pkg/apis/config" - "github.com/kubeflow/arena/pkg/apis/model" + "github.com/kubeflow/arena/pkg/apis/model/analyze" "github.com/kubeflow/arena/pkg/apis/types" "github.com/spf13/cobra" "github.com/spf13/viper" ) func NewSubmitModelEvaluateJobCommand() *cobra.Command { - builder := model.NewModelEvaluateJobBuilder() + builder := analyze.NewModelEvaluateJobBuilder() var command = &cobra.Command{ Use: "evaluate", Short: "Submit a model evaluate job", @@ -33,7 +34,7 @@ func NewSubmitModelEvaluateJobCommand() *cobra.Command { if err != nil { return fmt.Errorf("failed to validate command args: %v", err) } - return client.Model().Submit(job) + return client.Analyze().Submit(job) }, } builder.AddCommandFlags(command) diff --git a/pkg/commands/model/submit_optimize.go b/pkg/commands/model/analyze/submit_optimize.go similarity index 78% rename from pkg/commands/model/submit_optimize.go rename to pkg/commands/model/analyze/submit_optimize.go index f447b4ac6..3d677fb77 100644 --- a/pkg/commands/model/submit_optimize.go +++ b/pkg/commands/model/analyze/submit_optimize.go @@ -1,20 +1,21 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/arenaclient" "github.com/kubeflow/arena/pkg/apis/config" - "github.com/kubeflow/arena/pkg/apis/model" + "github.com/kubeflow/arena/pkg/apis/model/analyze" "github.com/kubeflow/arena/pkg/apis/types" "github.com/spf13/cobra" "github.com/spf13/viper" ) func NewSubmitModelOptimizeJobCommand() *cobra.Command { - builder := model.NewModelOptimizeJobBuilder() + builder := analyze.NewModelOptimizeJobBuilder() var command = &cobra.Command{ Use: "optimize", - Short: "Submit a model optimize job, this is a experimental feature", + Short: "Submit a model optimize job", PreRun: func(cmd *cobra.Command, args []string) { _ = viper.BindPFlags(cmd.Flags()) }, @@ -27,13 +28,13 @@ func NewSubmitModelOptimizeJobCommand() *cobra.Command { IsDaemonMode: false, }) if err != nil { - return fmt.Errorf("failed to create arena client: %v\n", err) + return fmt.Errorf("failed to create arena client: %v", err) } job, err := builder.Namespace(config.GetArenaConfiger().GetNamespace()).Command(args).Build() if err != nil { return fmt.Errorf("failed to validate command args: %v", err) } - return client.Model().Submit(job) + return client.Analyze().Submit(job) }, } builder.AddCommandFlags(command) diff --git a/pkg/commands/model/submit_profile.go b/pkg/commands/model/analyze/submit_profile.go similarity index 88% rename from pkg/commands/model/submit_profile.go rename to pkg/commands/model/analyze/submit_profile.go index be8b5aa78..debf9e779 100644 --- a/pkg/commands/model/submit_profile.go +++ b/pkg/commands/model/analyze/submit_profile.go @@ -1,17 +1,18 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/arenaclient" "github.com/kubeflow/arena/pkg/apis/config" - "github.com/kubeflow/arena/pkg/apis/model" + "github.com/kubeflow/arena/pkg/apis/model/analyze" "github.com/kubeflow/arena/pkg/apis/types" "github.com/spf13/cobra" "github.com/spf13/viper" ) func NewSubmitModelProfileJobCommand() *cobra.Command { - builder := model.NewModelProfileJobBuilder() + builder := analyze.NewModelProfileJobBuilder() var command = &cobra.Command{ Use: "profile", Short: "Submit a model profile job", @@ -33,7 +34,7 @@ func NewSubmitModelProfileJobCommand() *cobra.Command { if err != nil { return fmt.Errorf("failed to validate command args: %v", err) } - return client.Model().Submit(job) + return client.Analyze().Submit(job) }, } builder.AddCommandFlags(command) diff --git a/pkg/commands/model/model.go b/pkg/commands/model/model.go index 2c89e6a0c..acdfee93d 100644 --- a/pkg/commands/model/model.go +++ b/pkg/commands/model/model.go @@ -1,33 +1,19 @@ package model -import "github.com/spf13/cobra" +import ( + "github.com/kubeflow/arena/pkg/commands/model/analyze" -var ( - serveLong = `submit a model analyze job. - -Available Commands: - profile Submit a model profile job. - evaluate Submit a model evaluate job. - optimize Submit a model optimize job. - benchmark Submit a model benchmark job` + "github.com/spf13/cobra" ) func NewModelCommand() *cobra.Command { var command = &cobra.Command{ Use: "model", - Short: "Submit a model analyze job. (experimental feature)", - Long: serveLong, + Short: "Model manage and model analyze", Run: func(cmd *cobra.Command, args []string) { cmd.HelpFunc()(cmd, args) }, } - command.AddCommand(NewSubmitModelProfileJobCommand()) - command.AddCommand(NewSubmitModelOptimizeJobCommand()) - command.AddCommand(NewSubmitModelBenchmarkJobCommand()) - command.AddCommand(NewSubmitModelEvaluateJobCommand()) - command.AddCommand(NewGetModelJobCommand()) - command.AddCommand(NewListModelJobsCommand()) - command.AddCommand(NewDeleteModelJobCommand()) - + command.AddCommand(analyze.NewAnalyzeCommand()) return command } diff --git a/pkg/model/delete.go b/pkg/model/analyze/delete.go similarity index 97% rename from pkg/model/delete.go rename to pkg/model/analyze/delete.go index f610bb724..5dfd50347 100644 --- a/pkg/model/delete.go +++ b/pkg/model/analyze/delete.go @@ -1,10 +1,11 @@ -package model +package analyze import ( + "strings" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/workflow" log "github.com/sirupsen/logrus" - "strings" ) func DeleteModelJob(namespace, name string, jobType types.ModelJobType) error { diff --git a/pkg/model/get.go b/pkg/model/analyze/get.go similarity index 99% rename from pkg/model/get.go rename to pkg/model/analyze/get.go index f1f6f514d..388e453ab 100644 --- a/pkg/model/get.go +++ b/pkg/model/analyze/get.go @@ -1,14 +1,15 @@ -package model +package analyze import ( "encoding/json" "fmt" - "github.com/kubeflow/arena/pkg/apis/types" - "github.com/kubeflow/arena/pkg/apis/utils" - "gopkg.in/yaml.v2" "os" "strings" "text/tabwriter" + + "github.com/kubeflow/arena/pkg/apis/types" + "github.com/kubeflow/arena/pkg/apis/utils" + "gopkg.in/yaml.v2" ) var getJobTemplate = ` diff --git a/pkg/model/list.go b/pkg/model/analyze/list.go similarity index 99% rename from pkg/model/list.go rename to pkg/model/analyze/list.go index 3093771e8..5de13d78b 100644 --- a/pkg/model/list.go +++ b/pkg/model/analyze/list.go @@ -1,19 +1,20 @@ -package model +package analyze import ( "encoding/json" "fmt" - "github.com/kubeflow/arena/pkg/apis/types" - "github.com/kubeflow/arena/pkg/apis/utils" - "github.com/kubeflow/arena/pkg/util" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" "io" "os" "strconv" "strings" "text/tabwriter" "time" + + "github.com/kubeflow/arena/pkg/apis/types" + "github.com/kubeflow/arena/pkg/apis/utils" + "github.com/kubeflow/arena/pkg/util" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) func ListModelJobs(namespace string, allNamespaces bool, modelJobType types.ModelJobType) ([]ModelJob, error) { diff --git a/pkg/model/model.go b/pkg/model/analyze/model.go similarity index 99% rename from pkg/model/model.go rename to pkg/model/analyze/model.go index 5fdf0312d..2c40f350d 100644 --- a/pkg/model/model.go +++ b/pkg/model/analyze/model.go @@ -1,7 +1,11 @@ -package model +package analyze import ( "fmt" + "math" + "strings" + "time" + "github.com/kubeflow/arena/pkg/apis/config" "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/apis/utils" @@ -12,9 +16,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "math" - "strings" - "time" ) type modelProcessor struct { diff --git a/pkg/model/model_interface.go b/pkg/model/analyze/model_interface.go similarity index 98% rename from pkg/model/model_interface.go rename to pkg/model/analyze/model_interface.go index f40857a87..70db65020 100644 --- a/pkg/model/model_interface.go +++ b/pkg/model/analyze/model_interface.go @@ -1,11 +1,12 @@ -package model +package analyze import ( + "time" + "github.com/kubeflow/arena/pkg/apis/types" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "time" ) type ModelJob interface { diff --git a/pkg/model/submit_benchmark.go b/pkg/model/analyze/submit_benchmark.go similarity index 89% rename from pkg/model/submit_benchmark.go rename to pkg/model/analyze/submit_benchmark.go index 84d9c1dac..895d7499a 100644 --- a/pkg/model/submit_benchmark.go +++ b/pkg/model/analyze/submit_benchmark.go @@ -1,7 +1,8 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/util" "github.com/kubeflow/arena/pkg/workflow" @@ -23,6 +24,6 @@ func SubmitModelBenchmarkJob(namespace string, args *types.ModelBenchmarkArgs) e return err } log.Infof("The model benchmark job %s has been submitted successfully", args.Name) - log.Infof("You can run `arena model get %s` to check the job status", args.Name) + log.Infof("You can run `arena model analyze get %s` to check the job status", args.Name) return nil } diff --git a/pkg/model/submit_evaluate.go b/pkg/model/analyze/submit_evaluate.go similarity index 89% rename from pkg/model/submit_evaluate.go rename to pkg/model/analyze/submit_evaluate.go index 101ba539c..d7b76e108 100644 --- a/pkg/model/submit_evaluate.go +++ b/pkg/model/analyze/submit_evaluate.go @@ -1,7 +1,8 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/util" "github.com/kubeflow/arena/pkg/workflow" @@ -22,6 +23,6 @@ func SubmitModelEvaluateJob(namespace string, args *types.ModelEvaluateArgs) err return err } log.Infof("The model evaluate job %s has been submitted successfully", args.Name) - log.Infof("You can run `arena model get %s` to check the job status", args.Name) + log.Infof("You can run `arena model analyze get %s` to check the job status", args.Name) return nil } diff --git a/pkg/model/submit_optimize.go b/pkg/model/analyze/submit_optimize.go similarity index 88% rename from pkg/model/submit_optimize.go rename to pkg/model/analyze/submit_optimize.go index b634c301a..70e000358 100644 --- a/pkg/model/submit_optimize.go +++ b/pkg/model/analyze/submit_optimize.go @@ -1,7 +1,8 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/util" "github.com/kubeflow/arena/pkg/workflow" @@ -22,6 +23,6 @@ func SubmitModelOptimizeJob(namespace string, args *types.ModelOptimizeArgs) err return err } log.Infof("The model optimize job %s has been submitted successfully", args.Name) - log.Infof("You can run `arena model get %s` to check the job status", args.Name) + log.Infof("You can run `arena model analyze get %s` to check the job status", args.Name) return nil } diff --git a/pkg/model/submit_profile.go b/pkg/model/analyze/submit_profile.go similarity index 87% rename from pkg/model/submit_profile.go rename to pkg/model/analyze/submit_profile.go index a8b4f08db..4de003b42 100644 --- a/pkg/model/submit_profile.go +++ b/pkg/model/analyze/submit_profile.go @@ -1,7 +1,8 @@ -package model +package analyze import ( "fmt" + "github.com/kubeflow/arena/pkg/apis/types" "github.com/kubeflow/arena/pkg/util" "github.com/kubeflow/arena/pkg/workflow" @@ -22,6 +23,6 @@ func SubmitModelProfileJob(namespace string, args *types.ModelProfileArgs) error return err } log.Infof("The model profile job %s has been submitted successfully", args.Name) - log.Infof("You can run `arena model get %s` to check the job status", args.Name) + log.Infof("You can run `arena model analyze get %s` to check the job status", args.Name) return nil }