From 04b91fbf429822ed9057f608cbe06faa64c83f97 Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Mon, 14 Jun 2021 15:35:14 +0200 Subject: [PATCH 1/4] feat: docker-compose for test environment using MinIO --- test/docker-compose.yml | 61 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 test/docker-compose.yml diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100755 index 00000000..a9635a65 --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,61 @@ +--- +version: "3" +services: + terraboard-dev: + build: + context: ../ + dockerfile: ./Dockerfile + environment: + AWS_ACCESS_KEY_ID: root + AWS_SECRET_ACCESS_KEY: mypassword + AWS_BUCKET: test-bucket + AWS_REGION: eu-west-1 + AWS_ENDPOINT: http://minio:9000/ + AWS_OTHER_COMPATIBLE_PROVIDER: "true" + AWS_FORCE_PATH_STYLE: "true" + TERRABOARD_LOG_LEVEL: debug + DB_PASSWORD: mypassword + DB_SSLMODE: disable + GODEBUG: netdns=go + depends_on: + - "db" + - "minio" + volumes: + - ../static:/static:ro + ports: + - "8080:8080" + + minio: + image: minio/minio:latest + environment: + MINIO_ROOT_USER: root + MINIO_ROOT_PASSWORD: mypassword + expose: + - "9000" + ports: + - "9200:9000" + volumes: + - ./data:/data + command: server /data + + db: + image: postgres:9.5 + environment: + POSTGRES_USER: gorm + POSTGRES_PASSWORD: mypassword + POSTGRES_DB: gorm + volumes: + - tb-data:/var/lib/postgresql/data + + pgadmin: + container_name: pgadmin4_container + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: admin@admin.com + PGADMIN_DEFAULT_PASSWORD: root + ports: + - "5050:80" + +volumes: + tb-data: {} \ No newline at end of file From 577adc7b2a1666a284644373cbbf96cf3d0e503c Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Mon, 14 Jun 2021 15:38:26 +0200 Subject: [PATCH 2/4] feat: update AWS provider to be compatible with MinIO (and others S3 compatible ones) --- config/config.go | 13 +++++----- state/aws.go | 43 ++++++++++++++++++++++++---------- test/.gitignore | 2 ++ test/data/test-bucket/.gitkeep | 0 4 files changed, 39 insertions(+), 19 deletions(-) create mode 100755 test/.gitignore create mode 100644 test/data/test-bucket/.gitkeep diff --git a/config/config.go b/config/config.go index 685e24fa..6f2f203d 100644 --- a/config/config.go +++ b/config/config.go @@ -40,12 +40,13 @@ type S3BucketConfig struct { // AWSConfig stores the DynamoDB table and S3 Bucket configuration type AWSConfig struct { - DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."` - S3 S3BucketConfig `group:"S3 Options" yaml:"s3"` - Endpoint string `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."` - Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` - APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` - ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` + DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."` + S3 S3BucketConfig `group:"S3 Options" yaml:"s3"` + Endpoint string `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."` + Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` + APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` + ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` + OtherS3CompatibleProvider bool `long:"aws-other-compatible-provider" env:"AWS_OTHER_COMPATIBLE_PROVIDER" yaml:"other-compatible-provider" description:"Enable compatibility mode to support other providers s3 compatible (MinIO for exemple)"` } // TFEConfig stores the Terraform Enterprise configuration diff --git a/state/aws.go b/state/aws.go index b0f90dd5..ccce2c48 100644 --- a/state/aws.go +++ b/state/aws.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "time" aws_sdk "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" @@ -19,12 +20,13 @@ import ( // AWS is a state provider type, leveraging S3 and DynamoDB type AWS struct { - svc *s3.S3 - dynamoSvc *dynamodb.DynamoDB - bucket string - dynamoTable string - keyPrefix string - fileExtension []string + svc *s3.S3 + dynamoSvc *dynamodb.DynamoDB + bucket string + dynamoTable string + keyPrefix string + fileExtension []string + otherS3CompatibleProvider bool } // NewAWS creates an AWS object @@ -52,17 +54,23 @@ func NewAWS(c *config.Config) AWS { awsConfig.S3ForcePathStyle = &c.AWS.S3.ForcePathStyle return AWS{ - svc: s3.New(sess, awsConfig), - bucket: c.AWS.S3.Bucket, - keyPrefix: c.AWS.S3.KeyPrefix, - fileExtension: c.AWS.S3.FileExtension, - dynamoSvc: dynamodb.New(sess, awsConfig), - dynamoTable: c.AWS.DynamoDBTable, + svc: s3.New(sess, awsConfig), + bucket: c.AWS.S3.Bucket, + keyPrefix: c.AWS.S3.KeyPrefix, + fileExtension: c.AWS.S3.FileExtension, + dynamoSvc: dynamodb.New(sess, awsConfig), + dynamoTable: c.AWS.DynamoDBTable, + otherS3CompatibleProvider: c.AWS.OtherS3CompatibleProvider, } } // GetLocks returns a map of locks by State path func (a *AWS) GetLocks() (locks map[string]LockInfo, err error) { + if a.otherS3CompatibleProvider { + locks = make(map[string]LockInfo) + return + } + if a.dynamoTable == "" { err = fmt.Errorf("no dynamoDB table provided, not getting locks") return @@ -94,6 +102,7 @@ func (a *AWS) GetLocks() (locks map[string]LockInfo, err error) { locks[strings.TrimPrefix(info.Path, infoPrefix)] = info } } + return } @@ -138,7 +147,7 @@ func (a *AWS) GetState(st, versionID string) (sf *statefile.File, err error) { Bucket: aws_sdk.String(a.bucket), Key: aws_sdk.String(st), } - if versionID != "" { + if versionID != "" && !a.otherS3CompatibleProvider { input.VersionId = &versionID } result, err := a.svc.GetObjectWithContext(context.Background(), input) @@ -168,6 +177,14 @@ func (a *AWS) GetState(st, versionID string) (sf *statefile.File, err error) { // GetVersions returns a slice of Version objects func (a *AWS) GetVersions(state string) (versions []Version, err error) { versions = []Version{} + if a.otherS3CompatibleProvider { + versions = append(versions, Version{ + ID: "1", + LastModified: time.Now(), + }) + return + } + result, err := a.svc.ListObjectVersions(&s3.ListObjectVersionsInput{ Bucket: aws_sdk.String(a.bucket), Prefix: aws_sdk.String(state), diff --git a/test/.gitignore b/test/.gitignore new file mode 100755 index 00000000..24b61d13 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,2 @@ +data/* +!data/test-bucket \ No newline at end of file diff --git a/test/data/test-bucket/.gitkeep b/test/data/test-bucket/.gitkeep new file mode 100644 index 00000000..e69de29b From 15636799e115fb981f2a650533c427a1d2378943 Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Mon, 14 Jun 2021 15:54:17 +0200 Subject: [PATCH 3/4] docs: update README with new configuration variable + add some missing ones --- README.md | 4 ++++ config/config.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 291165e1..b0fdf33c 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,10 @@ The precedence of configurations is as described below. |`--aws-external-id` | `AWS_EXTERNAL_ID` | `aws.external-id` | External ID to use when assuming role | - | |`--key-prefix` | `AWS_KEY_PREFIX` | `aws.key-prefix` | AWS Key Prefix | - | |`--file-extension` | `AWS_FILE_EXTENSION` | `aws.file-extension` | File extension(s) of state files. Use multiple CLI flags or a comma separated list ENV variable | .tfstate | +|`--aws-region` | `AWS_REGION` | `aws.region` | AWS region | - | +|`--aws-endpoint` | `AWS_ENDPOINT` | `aws.endpoint` | Custom AWS endpoint | - | +|`--aws-other-compatible-provider` | `AWS_OTHER_COMPATIBLE_PROVIDER` | `aws.other-compatible-provider` | Enable compatibility mode to support other providers s3 compatible (MinIO for example), disable locks support & versionning | false | +|`--force-path-style` | `AWS_FORCE_PATH_STYLE` | `aws.s3.force-path-style` | Force path style S3 bucket calls. | false | |`--base-url` | `TERRABOARD_BASE_URL` | `web.base-url` | Base URL | / | |`--logout-url` | `TERRABOARD_LOGOUT_URL` | `web.logout-url` | Logout URL | - | |`--tfe-address` | `TFE_ADDRESS` | `tfe.tfe-address` | Terraform Enterprise address for states access | - | diff --git a/config/config.go b/config/config.go index 6f2f203d..486e3a1c 100644 --- a/config/config.go +++ b/config/config.go @@ -46,7 +46,7 @@ type AWSConfig struct { Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` - OtherS3CompatibleProvider bool `long:"aws-other-compatible-provider" env:"AWS_OTHER_COMPATIBLE_PROVIDER" yaml:"other-compatible-provider" description:"Enable compatibility mode to support other providers s3 compatible (MinIO for exemple)"` + OtherS3CompatibleProvider bool `long:"aws-other-compatible-provider" env:"AWS_OTHER_COMPATIBLE_PROVIDER" yaml:"other-compatible-provider" description:"Enable compatibility mode to support other providers s3 compatible (MinIO for example), disable locks support & versionning"` } // TFEConfig stores the Terraform Enterprise configuration From 402457d88a2dfb22b9017147c99a498b662fe799 Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Tue, 15 Jun 2021 09:45:31 +0200 Subject: [PATCH 4/4] feat: replace AWS_OTHER_COMPATIBLE_PROVIDER by two distinct generic parameters --- README.md | 3 ++- config/config.go | 21 ++++++++++++++------- example.yml | 4 ++++ state/aws.go | 36 +++++++++++++++++++----------------- test/docker-compose.yml | 3 ++- 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b0fdf33c..2d64a9cf 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,8 @@ The precedence of configurations is as described below. |`--file-extension` | `AWS_FILE_EXTENSION` | `aws.file-extension` | File extension(s) of state files. Use multiple CLI flags or a comma separated list ENV variable | .tfstate | |`--aws-region` | `AWS_REGION` | `aws.region` | AWS region | - | |`--aws-endpoint` | `AWS_ENDPOINT` | `aws.endpoint` | Custom AWS endpoint | - | -|`--aws-other-compatible-provider` | `AWS_OTHER_COMPATIBLE_PROVIDER` | `aws.other-compatible-provider` | Enable compatibility mode to support other providers s3 compatible (MinIO for example), disable locks support & versionning | false | +|`--no-locks` | `TERRABOARD_NO_LOCKS` | `provider.no-locks` | Disable locks support from Terraboard (useful for S3 compatible providers like MinIO) | false | +|`--no-versioning` | `TERRABOARD_NO_VERSIONING` | `provider.no-versioning` | Disable versioning support from Terraboard (useful for S3 compatible providers like MinIO) | false | |`--force-path-style` | `AWS_FORCE_PATH_STYLE` | `aws.s3.force-path-style` | Force path style S3 bucket calls. | false | |`--base-url` | `TERRABOARD_BASE_URL` | `web.base-url` | Base URL | / | |`--logout-url` | `TERRABOARD_LOGOUT_URL` | `web.logout-url` | Logout URL | - | diff --git a/config/config.go b/config/config.go index 486e3a1c..6184b119 100644 --- a/config/config.go +++ b/config/config.go @@ -40,13 +40,12 @@ type S3BucketConfig struct { // AWSConfig stores the DynamoDB table and S3 Bucket configuration type AWSConfig struct { - DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."` - S3 S3BucketConfig `group:"S3 Options" yaml:"s3"` - Endpoint string `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."` - Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` - APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` - ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` - OtherS3CompatibleProvider bool `long:"aws-other-compatible-provider" env:"AWS_OTHER_COMPATIBLE_PROVIDER" yaml:"other-compatible-provider" description:"Enable compatibility mode to support other providers s3 compatible (MinIO for example), disable locks support & versionning"` + DynamoDBTable string `long:"dynamodb-table" env:"AWS_DYNAMODB_TABLE" yaml:"dynamodb-table" description:"AWS DynamoDB table for locks."` + S3 S3BucketConfig `group:"S3 Options" yaml:"s3"` + Endpoint string `long:"aws-endpoint" env:"AWS_ENDPOINT" yaml:"endpoint" description:"AWS endpoint."` + Region string `long:"aws-region" env:"AWS_REGION" yaml:"region" description:"AWS region."` + APPRoleArn string `long:"aws-role-arn" env:"APP_ROLE_ARN" yaml:"app-role-arn" description:"Role ARN to Assume."` + ExternalID string `long:"aws-external-id" env:"AWS_EXTERNAL_ID" yaml:"external-id" description:"External ID to use when assuming role."` } // TFEConfig stores the Terraform Enterprise configuration @@ -75,12 +74,20 @@ type WebConfig struct { LogoutURL string `long:"logout-url" env:"TERRABOARD_LOGOUT_URL" yaml:"logout-url" description:"Logout URL."` } +// ProviderConfig stores genral provider parameters +type ProviderConfig struct { + NoVersioning bool `long:"no-versioning" env:"TERRABOARD_NO_VERSIONING" yaml:"no-versioning" description:"Disable versioning support from Terraboard (useful for S3 compatible providers like MinIO)"` + NoLocks bool `long:"no-locks" env:"TERRABOARD_NO_LOCKS" yaml:"no-locks" description:"Disable locks support from Terraboard (useful for S3 compatible providers like MinIO)"` +} + // Config stores the handler's configuration and UI interface parameters type Config struct { Version bool `short:"V" long:"version" description:"Display version."` ConfigFilePath string `short:"c" long:"config-file" env:"CONFIG_FILE" description:"Config File path"` + Provider ProviderConfig `group:"General Provider Options" yaml:"provider"` + Log LogConfig `group:"Logging Options" yaml:"log"` DB DBConfig `group:"Database Options" yaml:"database"` diff --git a/example.yml b/example.yml index 4935c8b9..75bbca38 100644 --- a/example.yml +++ b/example.yml @@ -11,6 +11,10 @@ database: no-sync: false sync-interval: 5 +provider: + no-locks: "true" + no-versioning: "false" + aws: dynamodb-table: terraboard s3: diff --git a/state/aws.go b/state/aws.go index ccce2c48..f54a202b 100644 --- a/state/aws.go +++ b/state/aws.go @@ -20,13 +20,14 @@ import ( // AWS is a state provider type, leveraging S3 and DynamoDB type AWS struct { - svc *s3.S3 - dynamoSvc *dynamodb.DynamoDB - bucket string - dynamoTable string - keyPrefix string - fileExtension []string - otherS3CompatibleProvider bool + svc *s3.S3 + dynamoSvc *dynamodb.DynamoDB + bucket string + dynamoTable string + keyPrefix string + fileExtension []string + noLocks bool + noVersioning bool } // NewAWS creates an AWS object @@ -54,19 +55,20 @@ func NewAWS(c *config.Config) AWS { awsConfig.S3ForcePathStyle = &c.AWS.S3.ForcePathStyle return AWS{ - svc: s3.New(sess, awsConfig), - bucket: c.AWS.S3.Bucket, - keyPrefix: c.AWS.S3.KeyPrefix, - fileExtension: c.AWS.S3.FileExtension, - dynamoSvc: dynamodb.New(sess, awsConfig), - dynamoTable: c.AWS.DynamoDBTable, - otherS3CompatibleProvider: c.AWS.OtherS3CompatibleProvider, + svc: s3.New(sess, awsConfig), + bucket: c.AWS.S3.Bucket, + keyPrefix: c.AWS.S3.KeyPrefix, + fileExtension: c.AWS.S3.FileExtension, + dynamoSvc: dynamodb.New(sess, awsConfig), + dynamoTable: c.AWS.DynamoDBTable, + noLocks: c.Provider.NoLocks, + noVersioning: c.Provider.NoVersioning, } } // GetLocks returns a map of locks by State path func (a *AWS) GetLocks() (locks map[string]LockInfo, err error) { - if a.otherS3CompatibleProvider { + if a.noLocks { locks = make(map[string]LockInfo) return } @@ -147,7 +149,7 @@ func (a *AWS) GetState(st, versionID string) (sf *statefile.File, err error) { Bucket: aws_sdk.String(a.bucket), Key: aws_sdk.String(st), } - if versionID != "" && !a.otherS3CompatibleProvider { + if versionID != "" && !a.noVersioning { input.VersionId = &versionID } result, err := a.svc.GetObjectWithContext(context.Background(), input) @@ -177,7 +179,7 @@ func (a *AWS) GetState(st, versionID string) (sf *statefile.File, err error) { // GetVersions returns a slice of Version objects func (a *AWS) GetVersions(state string) (versions []Version, err error) { versions = []Version{} - if a.otherS3CompatibleProvider { + if a.noVersioning { versions = append(versions, Version{ ID: "1", LastModified: time.Now(), diff --git a/test/docker-compose.yml b/test/docker-compose.yml index a9635a65..6b8492db 100755 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -11,9 +11,10 @@ services: AWS_BUCKET: test-bucket AWS_REGION: eu-west-1 AWS_ENDPOINT: http://minio:9000/ - AWS_OTHER_COMPATIBLE_PROVIDER: "true" AWS_FORCE_PATH_STYLE: "true" TERRABOARD_LOG_LEVEL: debug + TERRABOARD_NO_LOCKS: "true" + TERRABOARD_NO_VERSIONING: "true" DB_PASSWORD: mypassword DB_SSLMODE: disable GODEBUG: netdns=go