From 23310e17d172491c44158a7d07290e2d172e5fdc Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Fri, 26 Jun 2020 22:20:04 -0700 Subject: [PATCH] *: support "CL2UseHostNetworkPods" Signed-off-by: Gyuho Lee --- .../eks/create-cluster-loader.go | 3 +++ eks/cluster-loader/cluster-loader.go | 3 +++ eks/cluster-loader/local/cluster-loader.go | 1 + eks/cluster-loader/remote/cluster-loader.go | 3 ++- eksconfig/README.md | 2 ++ eksconfig/add-on-cluster-loader-local.go | 4 ++++ eksconfig/add-on-cluster-loader-remote.go | 4 ++++ eksconfig/default.yaml | 18 +++++++++--------- 8 files changed, 28 insertions(+), 10 deletions(-) diff --git a/cmd/aws-k8s-tester/eks/create-cluster-loader.go b/cmd/aws-k8s-tester/eks/create-cluster-loader.go index 5aaee24b5..60c8bc51b 100644 --- a/cmd/aws-k8s-tester/eks/create-cluster-loader.go +++ b/cmd/aws-k8s-tester/eks/create-cluster-loader.go @@ -43,6 +43,7 @@ var ( clusterLoaderSmallGroupSize int clusterLoaderSmallStatefulSetsPerNamespace int clusterLoaderMediumStatefulSetsPerNamespace int + clusterLoaderCL2UseHostNetworkPods bool clusterLoaderCL2LoadTestThroughput int clusterLoaderCL2EnablePVS bool clusterLoaderPrometheusScrapeKubeProxy bool @@ -80,6 +81,7 @@ func newCreateClusterLoader() *cobra.Command { cmd.PersistentFlags().IntVar(&clusterLoaderSmallGroupSize, "small-group-size", 5, "small group size") cmd.PersistentFlags().IntVar(&clusterLoaderSmallStatefulSetsPerNamespace, "small-stateful-sets-per-namespace", 0, "small stateful sets per namespace") cmd.PersistentFlags().IntVar(&clusterLoaderMediumStatefulSetsPerNamespace, "medium-stateful-sets-per-namespace", 0, "medium stateful sets per namespace") + cmd.PersistentFlags().BoolVar(&clusterLoaderCL2UseHostNetworkPods, "cl2-use-host-network-pods", false, "clusterloader2 use host network pods to bypass CNI") cmd.PersistentFlags().IntVar(&clusterLoaderCL2LoadTestThroughput, "cl2-load-test-throughput", 20, "clusterloader2 test throughput") cmd.PersistentFlags().BoolVar(&clusterLoaderCL2EnablePVS, "cl2-enable-pvs", false, "'true' to enable CL2 PVS") cmd.PersistentFlags().BoolVar(&clusterLoaderPrometheusScrapeKubeProxy, "prometheus-scrape-kube-proxy", false, "'true' to enable Prometheus scrape kube-proxy") @@ -157,6 +159,7 @@ func createClusterLoaderFunc(cmd *cobra.Command, args []string) { SmallStatefulSetsPerNamespace: clusterLoaderSmallStatefulSetsPerNamespace, MediumStatefulSetsPerNamespace: clusterLoaderMediumStatefulSetsPerNamespace, + CL2UseHostNetworkPods: clusterLoaderCL2UseHostNetworkPods, CL2LoadTestThroughput: clusterLoaderCL2LoadTestThroughput, CL2EnablePVS: clusterLoaderCL2EnablePVS, PrometheusScrapeKubeProxy: clusterLoaderPrometheusScrapeKubeProxy, diff --git a/eks/cluster-loader/cluster-loader.go b/eks/cluster-loader/cluster-loader.go index faefeaffc..332876b1d 100644 --- a/eks/cluster-loader/cluster-loader.go +++ b/eks/cluster-loader/cluster-loader.go @@ -81,6 +81,7 @@ type Config struct { SmallStatefulSetsPerNamespace int MediumStatefulSetsPerNamespace int + CL2UseHostNetworkPods bool CL2LoadTestThroughput int CL2EnablePVS bool PrometheusScrapeKubeProxy bool @@ -515,6 +516,7 @@ func (ld *loader) writeTestOverrides() (err error) { // ref. https://github.com/kubernetes/perf-tests/tree/master/clusterloader2/testing/load // ref. https://github.com/kubernetes/perf-tests/tree/master/clusterloader2/testing/overrides +// ref. https://github.com/kubernetes/perf-tests/pull/1345 const TemplateTestOverrides = `NODES_PER_NAMESPACE: {{ .NodesPerNamespace }} PODS_PER_NODE: {{ .PodsPerNode }} BIG_GROUP_SIZE: {{ .BigGroupSize }} @@ -522,6 +524,7 @@ MEDIUM_GROUP_SIZE: {{ .MediumGroupSize }} SMALL_GROUP_SIZE: {{ .SmallGroupSize }} SMALL_STATEFUL_SETS_PER_NAMESPACE: {{ .SmallStatefulSetsPerNamespace }} MEDIUM_STATEFUL_SETS_PER_NAMESPACE: {{ .MediumStatefulSetsPerNamespace }} +CL2_USE_HOST_NETWORK_PODS: {{ .CL2UseHostNetworkPods }} CL2_LOAD_TEST_THROUGHPUT: {{ .CL2LoadTestThroughput }} CL2_ENABLE_PVS: {{ .CL2EnablePVS }} PROMETHEUS_SCRAPE_KUBE_PROXY: {{ .PrometheusScrapeKubeProxy }} diff --git a/eks/cluster-loader/local/cluster-loader.go b/eks/cluster-loader/local/cluster-loader.go index c5c2f4b34..c482a9b26 100644 --- a/eks/cluster-loader/local/cluster-loader.go +++ b/eks/cluster-loader/local/cluster-loader.go @@ -93,6 +93,7 @@ func (ts *tester) Create() (err error) { SmallStatefulSetsPerNamespace: ts.cfg.EKSConfig.AddOnClusterLoaderLocal.SmallStatefulSetsPerNamespace, MediumStatefulSetsPerNamespace: ts.cfg.EKSConfig.AddOnClusterLoaderLocal.MediumStatefulSetsPerNamespace, + CL2UseHostNetworkPods: ts.cfg.EKSConfig.AddOnClusterLoaderLocal.CL2UseHostNetworkPods, CL2LoadTestThroughput: ts.cfg.EKSConfig.AddOnClusterLoaderLocal.CL2LoadTestThroughput, CL2EnablePVS: ts.cfg.EKSConfig.AddOnClusterLoaderLocal.CL2EnablePVS, PrometheusScrapeKubeProxy: ts.cfg.EKSConfig.AddOnClusterLoaderLocal.PrometheusScrapeKubeProxy, diff --git a/eks/cluster-loader/remote/cluster-loader.go b/eks/cluster-loader/remote/cluster-loader.go index 9cb05362e..6ac052867 100644 --- a/eks/cluster-loader/remote/cluster-loader.go +++ b/eks/cluster-loader/remote/cluster-loader.go @@ -522,7 +522,7 @@ func (ts *tester) createObject() (batchv1.Job, string, error) { // ref. https://github.com/kubernetes/client-go/blob/master/examples/in-cluster-client-configuration/main.go // ref. https://github.com/kubernetes/perf-tests/pull/1295 - testerCmd := fmt.Sprintf("/aws-k8s-tester eks create cluster-loader --partition=%s --region=%s --s3-bucket-name=%s --cluster-loader-path=/clusterloader2 --test-config-path=/clusterloader2-test-config.yaml --report-dir=/var/log/cluster-loader-remote --report-tar-gz-path=/var/log/cluster-loader-remote.tar.gz --report-tar-gz-s3-path=%s --log-path=/var/log/cluster-loader-remote.log --log-s3-path=%s --pod-startup-latency-path=/var/log/cluster-loader-remote.pod-startup-latency-output.json --pod-startup-latency-s3-path=%s --runs=%d --timeout=%v --nodes=%d --nodes-per-namespace=%d --pods-per-node=%d --big-group-size=%d --medium-group-size=%d --small-group-size=%d --small-stateful-sets-per-namespace=%d --medium-stateful-sets-per-namespace=%d --cl2-load-test-throughput=%d --cl2-enable-pvs=%v --prometheus-scrape-kube-proxy=%v --enable-system-pod-metrics=%v", + testerCmd := fmt.Sprintf("/aws-k8s-tester eks create cluster-loader --partition=%s --region=%s --s3-bucket-name=%s --cluster-loader-path=/clusterloader2 --test-config-path=/clusterloader2-test-config.yaml --report-dir=/var/log/cluster-loader-remote --report-tar-gz-path=/var/log/cluster-loader-remote.tar.gz --report-tar-gz-s3-path=%s --log-path=/var/log/cluster-loader-remote.log --log-s3-path=%s --pod-startup-latency-path=/var/log/cluster-loader-remote.pod-startup-latency-output.json --pod-startup-latency-s3-path=%s --runs=%d --timeout=%v --nodes=%d --nodes-per-namespace=%d --pods-per-node=%d --big-group-size=%d --medium-group-size=%d --small-group-size=%d --small-stateful-sets-per-namespace=%d --medium-stateful-sets-per-namespace=%d --cl2-use-host-network-pods=%v --cl2-load-test-throughput=%d --cl2-enable-pvs=%v --prometheus-scrape-kube-proxy=%v --enable-system-pod-metrics=%v", ts.cfg.EKSConfig.Partition, ts.cfg.EKSConfig.Region, ts.cfg.EKSConfig.S3BucketName, @@ -539,6 +539,7 @@ func (ts *tester) createObject() (batchv1.Job, string, error) { ts.cfg.EKSConfig.AddOnClusterLoaderRemote.SmallGroupSize, ts.cfg.EKSConfig.AddOnClusterLoaderRemote.SmallStatefulSetsPerNamespace, ts.cfg.EKSConfig.AddOnClusterLoaderRemote.MediumStatefulSetsPerNamespace, + ts.cfg.EKSConfig.AddOnClusterLoaderRemote.CL2UseHostNetworkPods, ts.cfg.EKSConfig.AddOnClusterLoaderRemote.CL2LoadTestThroughput, ts.cfg.EKSConfig.AddOnClusterLoaderRemote.CL2EnablePVS, ts.cfg.EKSConfig.AddOnClusterLoaderRemote.PrometheusScrapeKubeProxy, diff --git a/eksconfig/README.md b/eksconfig/README.md index dd9ea51e0..db05a8514 100644 --- a/eksconfig/README.md +++ b/eksconfig/README.md @@ -776,6 +776,7 @@ AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_ENABLE=true \ | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_SMALL_GROUP_SIZE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.SmallGroupSize | int | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_SMALL_STATEFUL_SETS_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.SmallStatefulSetsPerNamespace | int | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_MEDIUM_STATEFUL_SETS_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.MediumStatefulSetsPerNamespace | int | +| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_USE_HOST_NETWORK_PODS | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.CL2UseHostNetworkPods | bool | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_LOAD_TEST_THROUGHPUT | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.CL2LoadTestThroughput | int | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_CL2_ENABLE_PVS | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.CL2EnablePVS | bool | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_LOCAL_PROMETHEUS_SCRAPE_KUBE_PROXY | read-only "false" | *eksconfig.AddOnClusterLoaderLocal.PrometheusScrapeKubeProxy | bool | @@ -814,6 +815,7 @@ AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_VERSION_UPGRADE_ENABLE=true \ | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_SMALL_GROUP_SIZE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.SmallGroupSize | int | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_SMALL_STATEFUL_SETS_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.SmallStatefulSetsPerNamespace | int | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_MEDIUM_STATEFUL_SETS_PER_NAMESPACE | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.MediumStatefulSetsPerNamespace | int | +| AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_USE_HOST_NETWORK_PODS | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.CL2UseHostNetworkPods | bool | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_LOAD_TEST_THROUGHPUT | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.CL2LoadTestThroughput | int | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_ENABLE_PVS | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.CL2EnablePVS | bool | | AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_PROMETHEUS_SCRAPE_KUBE_PROXY | read-only "false" | *eksconfig.AddOnClusterLoaderRemote.PrometheusScrapeKubeProxy | bool | diff --git a/eksconfig/add-on-cluster-loader-local.go b/eksconfig/add-on-cluster-loader-local.go index 031e2017e..f47105385 100644 --- a/eksconfig/add-on-cluster-loader-local.go +++ b/eksconfig/add-on-cluster-loader-local.go @@ -94,6 +94,8 @@ type AddOnClusterLoaderLocal struct { SmallStatefulSetsPerNamespace int `json:"small-stateful-sets-per-namespace"` MediumStatefulSetsPerNamespace int `json:"medium-stateful-sets-per-namespace"` + // ref. https://github.com/kubernetes/perf-tests/pull/1345 + CL2UseHostNetworkPods bool `json:"cl2-use-host-network-pods"` CL2LoadTestThroughput int `json:"cl2-load-test-throughput"` CL2EnablePVS bool `json:"cl2-enable-pvs"` PrometheusScrapeKubeProxy bool `json:"prometheus-scrape-kube-proxy"` @@ -140,6 +142,8 @@ func getDefaultAddOnClusterLoaderLocal() *AddOnClusterLoaderLocal { SmallStatefulSetsPerNamespace: 0, MediumStatefulSetsPerNamespace: 0, + CL2UseHostNetworkPods: false, + // ref. https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/testing/load/kubemark/throughput_override.yaml CL2LoadTestThroughput: 20, CL2EnablePVS: false, diff --git a/eksconfig/add-on-cluster-loader-remote.go b/eksconfig/add-on-cluster-loader-remote.go index d7b38eb1b..09985cc64 100644 --- a/eksconfig/add-on-cluster-loader-remote.go +++ b/eksconfig/add-on-cluster-loader-remote.go @@ -101,6 +101,8 @@ type AddOnClusterLoaderRemote struct { SmallStatefulSetsPerNamespace int `json:"small-stateful-sets-per-namespace"` MediumStatefulSetsPerNamespace int `json:"medium-stateful-sets-per-namespace"` + // ref. https://github.com/kubernetes/perf-tests/pull/1345 + CL2UseHostNetworkPods bool `json:"cl2-use-host-network-pods"` CL2LoadTestThroughput int `json:"cl2-load-test-throughput"` CL2EnablePVS bool `json:"cl2-enable-pvs"` PrometheusScrapeKubeProxy bool `json:"prometheus-scrape-kube-proxy"` @@ -147,6 +149,8 @@ func getDefaultAddOnClusterLoaderRemote() *AddOnClusterLoaderRemote { SmallStatefulSetsPerNamespace: 0, MediumStatefulSetsPerNamespace: 0, + CL2UseHostNetworkPods: false, + // ref. https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/testing/load/kubemark/throughput_override.yaml CL2LoadTestThroughput: 20, CL2EnablePVS: false, diff --git a/eksconfig/default.yaml b/eksconfig/default.yaml index aa242b676..1f5f2d8b2 100644 --- a/eksconfig/default.yaml +++ b/eksconfig/default.yaml @@ -24,7 +24,7 @@ log-level: info log-outputs: - stderr - /home/ANT.AMAZON.COM/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.log -name: eks-2020062621-paper9byvcuv +name: eks-2020062622-watergg1p4z7 on-failure-delete: true on-failure-delete-wait-seconds: 120 parameters: @@ -40,10 +40,10 @@ parameters: role-arn: "" role-cfn-stack-id: "" role-cfn-stack-yaml-path: /home/ANT.AMAZON.COM/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.role.cfn.yaml - role-cfn-stack-yaml-s3-key: eks-2020062621-paper9byvcuv/default.role.cfn.yaml + role-cfn-stack-yaml-s3-key: eks-2020062622-watergg1p4z7/default.role.cfn.yaml role-create: true role-managed-policy-arns: null - role-name: eks-2020062621-paper9byvcuv-role + role-name: eks-2020062622-watergg1p4z7-role role-service-principals: null signing-name: eks tags: null @@ -51,19 +51,19 @@ parameters: version-value: 1.16 vpc-cfn-stack-id: "" vpc-cfn-stack-yaml-path: /home/ANT.AMAZON.COM/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.vpc.cfn.yaml - vpc-cfn-stack-yaml-s3-key: eks-2020062621-paper9byvcuv/default.vpc.cfn.yaml + vpc-cfn-stack-yaml-s3-key: eks-2020062622-watergg1p4z7/default.vpc.cfn.yaml vpc-create: true vpc-id: "" partition: aws region: us-west-2 remote-access-commands-output-path: /home/ANT.AMAZON.COM/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.ssh.sh remote-access-key-create: true -remote-access-key-name: eks-2020062621-paper9byvcuv-remote-access-key -remote-access-private-key-path: /tmp/sparkling371p66.insecure.key +remote-access-key-name: eks-2020062622-watergg1p4z7-remote-access-key +remote-access-private-key-path: /tmp/linuxozrs641djd.insecure.key s3-bucket-create: true s3-bucket-create-keep: true s3-bucket-lifecycle-expiration-days: 0 -s3-bucket-name: eks-2020062621-paper9byvcuv-s3-bucket +s3-bucket-name: eks-2020062622-watergg1p4z7-s3-bucket status: aws-account-id: "" aws-credential-path: "" @@ -75,9 +75,9 @@ status: cluster-ca-decoded: "" cluster-cfn-stack-id: "" cluster-cfn-stack-yaml-path: /home/ANT.AMAZON.COM/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/default.cluster.cfn.yaml - cluster-cfn-stack-yaml-s3-key: eks-2020062621-paper9byvcuv/default.cluster.cfn.yaml + cluster-cfn-stack-yaml-s3-key: eks-2020062622-watergg1p4z7/default.cluster.cfn.yaml cluster-control-plane-security-group-id: "" - cluster-metrics-raw-output-dir: /home/ANT.AMAZON.COM/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/eks-2020062621-paper9byvcuv-metrics + cluster-metrics-raw-output-dir: /home/ANT.AMAZON.COM/leegyuho/go/src/github.com/aws/aws-k8s-tester/eksconfig/eks-2020062622-watergg1p4z7-metrics cluster-oidc-issuer-arn: "" cluster-oidc-issuer-ca-thumbprint: "" cluster-oidc-issuer-host-path: ""