Skip to content

Commit

Permalink
*: configure s3 dir
Browse files Browse the repository at this point in the history 
Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
  • Loading branch information
@gyuho
gyuho committed Jun 24, 2020
1 parent d7341db commit 53a0169
Show file tree
Hide file tree
Showing 28 changed files with 1,336 additions and 608 deletions.
40 changes: 25 additions & 15 deletions cmd/aws-k8s-tester/eks/create-configmaps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"

Expand All @@ -26,7 +27,6 @@ var (
configmapsPartition string
configmapsRegion string
configmapsS3BucketName string
configmapsS3DirName string

configmapsClients int
configmapsClientQPS float32
Expand All @@ -37,6 +37,10 @@ var (

configmapsNamespace string

configmapsWritesRawJSONS3Dir string
configmapsWritesSummaryJSONS3Dir string
configmapsWritesSummaryTableS3Dir string

configmapsWritesOutputNamePrefix string

configmapsBlock bool
Expand All @@ -52,14 +56,18 @@ func newCreateConfigMaps() *cobra.Command {
cmd.PersistentFlags().StringVar(&configmapsPartition, "partition", "aws", "partition for AWS API")
cmd.PersistentFlags().StringVar(&configmapsRegion, "region", "us-west-2", "region for AWS API")
cmd.PersistentFlags().StringVar(&configmapsS3BucketName, "s3-bucket-name", "", "S3 bucket name to upload results")
cmd.PersistentFlags().StringVar(&configmapsS3DirName, "s3-dir-name", "", "S3 directory name to upload results")
cmd.PersistentFlags().IntVar(&configmapsClients, "clients", eksconfig.DefaultClients, "Number of clients to create")
cmd.PersistentFlags().Float32Var(&configmapsClientQPS, "client-qps", eksconfig.DefaultClientQPS, "kubelet client setup for QPS")
cmd.PersistentFlags().IntVar(&configmapsClientBurst, "client-burst", eksconfig.DefaultClientBurst, "kubelet client setup for burst")
cmd.PersistentFlags().DurationVar(&configmapsClientTimeout, "client-timeout", eksconfig.DefaultClientTimeout, "kubelet client timeout")
cmd.PersistentFlags().IntVar(&configmapsObjects, "objects", 0, "Size of object per write (0 to disable writes)")
cmd.PersistentFlags().IntVar(&configmapsObjectSize, "object-size", 0, "Size of object per write (0 to disable writes)")
cmd.PersistentFlags().StringVar(&configmapsNamespace, "namespace", "default", "namespace to send writes")

cmd.PersistentFlags().StringVar(&configmapsWritesRawJSONS3Dir, "writes-raw-json-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&configmapsWritesSummaryJSONS3Dir, "writes-summary-json-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&configmapsWritesSummaryTableS3Dir, "writes-summary-table-s3-dir", "", "s3 directory prefix to upload")

cmd.PersistentFlags().StringVar(&configmapsWritesOutputNamePrefix, "writes-output-name-prefix", "", "Write results output name prefix in /var/log/")
cmd.PersistentFlags().BoolVar(&configmapsBlock, "block", false, "true to block process exit after cluster loader complete")
return cmd
Expand Down Expand Up @@ -139,19 +147,21 @@ func createConfigMapsFunc(cmd *cobra.Command, args []string) {
sfx := randutil.String(7)

loader := config_maps.New(config_maps.Config{
Logger: lg,
Stopc: stopc,
S3API: s3.New(awsSession),
S3BucketName: configmapsS3BucketName,
S3DirName: configmapsS3DirName,
Client: cli,
ClientTimeout: configmapsClientTimeout,
Namespace: configmapsNamespace,
Objects: configmapsObjects,
ObjectSize: configmapsObjectSize,
WritesJSONPath: "/var/log/" + configmapsWritesOutputNamePrefix + "-" + sfx + "-writes.json",
WritesSummaryJSONPath: "/var/log/" + configmapsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.json",
WritesSummaryTablePath: "/var/log/" + configmapsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.txt",
Logger: lg,
Stopc: stopc,
S3API: s3.New(awsSession),
S3BucketName: configmapsS3BucketName,
Client: cli,
ClientTimeout: configmapsClientTimeout,
Namespace: configmapsNamespace,
Objects: configmapsObjects,
ObjectSize: configmapsObjectSize,
WritesRawJSONPath: "/var/log/" + configmapsWritesOutputNamePrefix + "-" + sfx + "-writes.json",
WritesRawJSONS3Key: filepath.Join(configmapsWritesRawJSONS3Dir, configmapsWritesOutputNamePrefix+"-"+sfx+"-writes.json"),
WritesSummaryJSONPath: "/var/log/" + configmapsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.json",
WritesSummaryJSONS3Key: filepath.Join(configmapsWritesSummaryJSONS3Dir, configmapsWritesOutputNamePrefix+"-"+sfx+"-writes-summary.json"),
WritesSummaryTablePath: "/var/log/" + configmapsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.txt",
WritesSummaryTableS3Key: filepath.Join(configmapsWritesSummaryTableS3Dir, configmapsWritesOutputNamePrefix+"-"+sfx+"-writes-summary.txt"),
})
loader.Start()
loader.Stop()
Expand Down
18 changes: 14 additions & 4 deletions cmd/aws-k8s-tester/eks/create-csrs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"

Expand All @@ -26,7 +27,6 @@ var (
csrsPartition string
csrsRegion string
csrsS3BucketName string
csrsS3DirName string

csrsClients int
csrsClientQPS float32
Expand All @@ -35,6 +35,10 @@ var (
csrsObjects int
csrsInitialRequestConditionType string

csrsWritesRawJSONS3Dir string
csrsWritesSummaryJSONS3Dir string
csrsWritesSummaryTableS3Dir string

csrsWritesOutputNamePrefix string

csrsBlock bool
Expand All @@ -50,12 +54,16 @@ func newCreateCSRs() *cobra.Command {
cmd.PersistentFlags().StringVar(&csrsPartition, "partition", "aws", "partition for AWS API")
cmd.PersistentFlags().StringVar(&csrsRegion, "region", "us-west-2", "region for AWS API")
cmd.PersistentFlags().StringVar(&csrsS3BucketName, "s3-bucket-name", "", "S3 bucket name to upload results")
cmd.PersistentFlags().StringVar(&csrsS3DirName, "s3-dir-name", "", "S3 directory name to upload results")
cmd.PersistentFlags().IntVar(&csrsClients, "clients", eksconfig.DefaultClients, "Number of clients to create")
cmd.PersistentFlags().Float32Var(&csrsClientQPS, "client-qps", eksconfig.DefaultClientQPS, "kubelet client setup for QPS")
cmd.PersistentFlags().IntVar(&csrsClientBurst, "client-burst", eksconfig.DefaultClientBurst, "kubelet client setup for burst")
cmd.PersistentFlags().DurationVar(&csrsClientTimeout, "client-timeout", eksconfig.DefaultClientTimeout, "kubelet client timeout")
cmd.PersistentFlags().IntVar(&csrsObjects, "objects", 0, "Size of object per write (0 to disable writes)")

cmd.PersistentFlags().StringVar(&csrsWritesRawJSONS3Dir, "writes-raw-json-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&csrsWritesSummaryJSONS3Dir, "writes-summary-json-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&csrsWritesSummaryTableS3Dir, "writes-summary-table-s3-dir", "", "s3 directory prefix to upload")

cmd.PersistentFlags().StringVar(&csrsInitialRequestConditionType, "initial-request-condition-type", "", "Initial CSR condition type")
cmd.PersistentFlags().StringVar(&csrsWritesOutputNamePrefix, "writes-output-name-prefix", "", "Write results output name prefix in /var/log/")
cmd.PersistentFlags().BoolVar(&csrsBlock, "block", false, "true to block process exit after cluster loader complete")
Expand Down Expand Up @@ -140,14 +148,16 @@ func createCSRsFunc(cmd *cobra.Command, args []string) {
Stopc: stopc,
S3API: s3.New(awsSession),
S3BucketName: csrsS3BucketName,
S3DirName: csrsS3DirName,
Client: cli,
ClientTimeout: csrsClientTimeout,
Objects: csrsObjects,
InitialRequestConditionType: "",
WritesJSONPath: "/var/log/" + csrsWritesOutputNamePrefix + "-" + sfx + "-writes.json",
WritesRawJSONPath: "/var/log/" + csrsWritesOutputNamePrefix + "-" + sfx + "-writes.json",
WritesRawJSONS3Key: filepath.Join(csrsWritesRawJSONS3Dir, csrsWritesOutputNamePrefix+"-"+sfx+"-writes.json"),
WritesSummaryJSONPath: "/var/log/" + csrsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.json",
WritesSummaryJSONS3Key: filepath.Join(csrsWritesSummaryJSONS3Dir, csrsWritesOutputNamePrefix+"-"+sfx+"-writes-summary.json"),
WritesSummaryTablePath: "/var/log/" + csrsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.txt",
WritesSummaryTableS3Key: filepath.Join(csrsWritesSummaryTableS3Dir, csrsWritesOutputNamePrefix+"-"+sfx+"-writes-summary.txt"),
})
loader.Start()
loader.Stop()
Expand Down
57 changes: 38 additions & 19 deletions cmd/aws-k8s-tester/eks/create-secrets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"

Expand All @@ -26,7 +27,6 @@ var (
secretsPartition string
secretsRegion string
secretsS3BucketName string
secretsS3DirName string

secretsClients int
secretsClientQPS float32
Expand All @@ -38,6 +38,13 @@ var (
secretsNamespace string
secretsNamePrefix string

secretsWritesRawJSONS3Dir string
secretsWritesSummaryJSONS3Dir string
secretsWritesSummaryTableS3Dir string
secretsReadsRawJSONS3Dir string
secretsReadsSummaryJSONS3Dir string
secretsReadsSummaryTableS3Dir string

secretsWritesOutputNamePrefix string
secretsReadsOutputNamePrefix string

Expand All @@ -54,7 +61,6 @@ func newCreateSecrets() *cobra.Command {
cmd.PersistentFlags().StringVar(&secretsPartition, "partition", "aws", "partition for AWS API")
cmd.PersistentFlags().StringVar(&secretsRegion, "region", "us-west-2", "region for AWS API")
cmd.PersistentFlags().StringVar(&secretsS3BucketName, "s3-bucket-name", "", "S3 bucket name to upload results")
cmd.PersistentFlags().StringVar(&secretsS3DirName, "s3-dir-name", "", "S3 directory name to upload results")
cmd.PersistentFlags().IntVar(&secretsClients, "clients", eksconfig.DefaultClients, "Number of clients to create")
cmd.PersistentFlags().Float32Var(&secretsClientQPS, "client-qps", eksconfig.DefaultClientQPS, "kubelet client setup for QPS")
cmd.PersistentFlags().IntVar(&secretsClientBurst, "client-burst", eksconfig.DefaultClientBurst, "kubelet client setup for burst")
Expand All @@ -63,6 +69,14 @@ func newCreateSecrets() *cobra.Command {
cmd.PersistentFlags().IntVar(&secretsObjectSize, "object-size", 0, "Size of object per write (0 to disable writes)")
cmd.PersistentFlags().StringVar(&secretsNamespace, "namespace", "default", "namespace to send writes")
cmd.PersistentFlags().StringVar(&secretsNamePrefix, "name-prefix", "", "prefix of Secret name")

cmd.PersistentFlags().StringVar(&secretsWritesRawJSONS3Dir, "writes-raw-json-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&secretsWritesSummaryJSONS3Dir, "writes-summary-json-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&secretsWritesSummaryTableS3Dir, "writes-summary-table-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&secretsReadsRawJSONS3Dir, "reads-raw-json-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&secretsReadsSummaryJSONS3Dir, "reads-summary-json-s3-dir", "", "s3 directory prefix to upload")
cmd.PersistentFlags().StringVar(&secretsReadsSummaryTableS3Dir, "reads-summary-table-s3-dir", "", "s3 directory prefix to upload")

cmd.PersistentFlags().StringVar(&secretsWritesOutputNamePrefix, "writes-output-name-prefix", "", "writes results output name prefix in /var/log/")
cmd.PersistentFlags().StringVar(&secretsReadsOutputNamePrefix, "reads-output-name-prefix", "", "reads results output name prefix in /var/log/")
cmd.PersistentFlags().BoolVar(&secretsBlock, "block", false, "true to block process exit after cluster loader complete")
Expand Down Expand Up @@ -143,23 +157,28 @@ func createSecretsFunc(cmd *cobra.Command, args []string) {
sfx := randutil.String(7)

loader := secrets.New(secrets.Config{
Logger: lg,
Stopc: stopc,
S3API: s3.New(awsSession),
S3BucketName: secretsS3BucketName,
S3DirName: secretsS3DirName,
Client: cli,
ClientTimeout: secretsClientTimeout,
Namespace: secretsNamespace,
NamePrefix: secretsNamePrefix,
Objects: secretsObjects,
ObjectSize: secretsObjectSize,
WritesJSONPath: "/var/log/" + secretsWritesOutputNamePrefix + "-" + sfx + "-writes.json",
WritesSummaryJSONPath: "/var/log/" + secretsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.json",
WritesSummaryTablePath: "/var/log/" + secretsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.txt",
ReadsJSONPath: "/var/log/" + secretsReadsOutputNamePrefix + "-" + sfx + "-reads.json",
ReadsSummaryJSONPath: "/var/log/" + secretsReadsOutputNamePrefix + "-" + sfx + "-reads-summary.json",
ReadsSummaryTablePath: "/var/log/" + secretsReadsOutputNamePrefix + "-" + sfx + "-reads-summary.txt",
Logger: lg,
Stopc: stopc,
S3API: s3.New(awsSession),
S3BucketName: secretsS3BucketName,
Client: cli,
ClientTimeout: secretsClientTimeout,
Namespace: secretsNamespace,
NamePrefix: secretsNamePrefix,
Objects: secretsObjects,
ObjectSize: secretsObjectSize,
WritesRawJSONPath: "/var/log/" + secretsWritesOutputNamePrefix + "-" + sfx + "-writes.json",
WritesRawJSONS3Key: filepath.Join(secretsWritesRawJSONS3Dir, secretsWritesOutputNamePrefix+"-"+sfx+"-writes.json"),
WritesSummaryJSONPath: "/var/log/" + secretsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.json",
WritesSummaryJSONS3Key: filepath.Join(secretsWritesSummaryJSONS3Dir, secretsWritesOutputNamePrefix+"-"+sfx+"-writes-summary.json"),
WritesSummaryTablePath: "/var/log/" + secretsWritesOutputNamePrefix + "-" + sfx + "-writes-summary.txt",
WritesSummaryTableS3Key: filepath.Join(secretsWritesSummaryTableS3Dir, secretsWritesOutputNamePrefix+"-"+sfx+"-writes-summary.txt"),
ReadsRawJSONPath: "/var/log/" + secretsReadsOutputNamePrefix + "-" + sfx + "-reads.json",
ReadsRawJSONS3Key: filepath.Join(secretsReadsRawJSONS3Dir, secretsReadsOutputNamePrefix+"-"+sfx+"-reads.json"),
ReadsSummaryJSONPath: "/var/log/" + secretsReadsOutputNamePrefix + "-" + sfx + "-reads-summary.json",
ReadsSummaryJSONS3Key: filepath.Join(secretsReadsSummaryJSONS3Dir, secretsReadsOutputNamePrefix+"-"+sfx+"-reads-summary.json"),
ReadsSummaryTablePath: "/var/log/" + secretsReadsOutputNamePrefix + "-" + sfx + "-reads-summary.txt",
ReadsSummaryTableS3Key: filepath.Join(secretsReadsSummaryTableS3Dir, secretsReadsOutputNamePrefix+"-"+sfx+"-reads-summary.txt"),
})
loader.Start()
loader.Stop()
Expand Down
Loading

0 comments on commit 53a0169

Please sign in to comment.