From 548d05040fea541d04ecf47f378b4b99a072668e Mon Sep 17 00:00:00 2001 From: Evgeny Arhipov Date: Sun, 20 Aug 2023 14:56:56 +0200 Subject: [PATCH] Add additional options to control storage endpoints on import --- .../yandex-import/Config-not-required.mdx | 7 ++ .../yandex-import/post-processor.go | 40 ++++++- .../yandex-import/post-processor.hcl2spec.go | 106 +++++++++--------- post-processor/yandex-import/storage.go | 5 +- 4 files changed, 103 insertions(+), 55 deletions(-) diff --git a/docs-partials/post-processor/yandex-import/Config-not-required.mdx b/docs-partials/post-processor/yandex-import/Config-not-required.mdx index d5d7d3f5..5b40a560 100644 --- a/docs-partials/post-processor/yandex-import/Config-not-required.mdx +++ b/docs-partials/post-processor/yandex-import/Config-not-required.mdx @@ -14,4 +14,11 @@ after the import process has completed. Possible values are: `true` to leave it in the bucket, `false` to remove it. Default is `false`. +- `storage_endpoint` (string) - StorageEndpoint custom Yandex Object Storage endpoint to upload image, Default `storage.yandexcloud.net`. + +- `storage_endpoint_autoresolve` (bool) - StorageEndpointAutoresolve auto resolve storage endpoint via YC Public API ListEndpoints call. Option has + precedence over 'storage_endpoint' option. + +- `storage_region` (string) - StorageRegion custom Yandex Object region. Default `ru-central1` + diff --git a/post-processor/yandex-import/post-processor.go b/post-processor/yandex-import/post-processor.go index dbc13acf..eb515a38 100644 --- a/post-processor/yandex-import/post-processor.go +++ b/post-processor/yandex-import/post-processor.go @@ -9,6 +9,7 @@ package yandeximport import ( "context" "fmt" + "log" "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/packer-plugin-sdk/common" @@ -17,9 +18,15 @@ import ( "github.com/hashicorp/packer-plugin-sdk/template/interpolate" "github.com/hashicorp/packer-plugin-yandex/builder/yandex" yandexexport "github.com/hashicorp/packer-plugin-yandex/post-processor/yandex-export" + "github.com/yandex-cloud/go-genproto/yandex/cloud/endpoint" "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility" ) +const ( + defaultStorageEndpoint = "storage.yandexcloud.net" + defaultStorageRegion = "ru-central1" +) + type Config struct { common.PackerConfig `mapstructure:",squash"` yandex.AccessConfig `mapstructure:",squash"` @@ -42,6 +49,14 @@ type Config struct { // leave it in the bucket, `false` to remove it. Default is `false`. SkipClean bool `mapstructure:"skip_clean" required:"false"` + // StorageEndpoint custom Yandex Object Storage endpoint to upload image, Default `storage.yandexcloud.net`. + StorageEndpoint string `mapstructure:"storage_endpoint" required:"false"` + // StorageEndpointAutoresolve auto resolve storage endpoint via YC Public API ListEndpoints call. Option has + // precedence over 'storage_endpoint' option. + StorageEndpointAutoresolve bool `mapstructure:"storage_endpoint_autoresolve" required:"false"` + // StorageRegion custom Yandex Object region. Default `ru-central1` + StorageRegion string `mapstructure:"storage_region" required:"false"` + ctx interpolate.Context } @@ -124,7 +139,30 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packersdk.Ui, artifa return nil, false, false, err } - storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret()) + if p.config.StorageEndpoint == "" { + p.config.StorageEndpoint = defaultStorageEndpoint + } + + if p.config.StorageRegion == "" { + p.config.StorageRegion = defaultStorageRegion + } + + if p.config.StorageEndpointAutoresolve { + ui.Say("Resolving storage endpoint...") + response, err := client.SDK().ApiEndpoint().ApiEndpoint().Get(ctx, &endpoint.GetApiEndpointRequest{ + ApiEndpointId: "storage", + }) + + if err != nil { + return nil, false, false, err + } + + p.config.StorageEndpoint = response.Address + } + + log.Printf("[DEBUG] Using storage endpoint: '%s'", p.config.StorageEndpoint) + + storageClient, err := newYCStorageClient(p.config.StorageEndpoint, respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret()) if err != nil { return nil, false, false, fmt.Errorf("error create object storage client: %s", err) } diff --git a/post-processor/yandex-import/post-processor.hcl2spec.go b/post-processor/yandex-import/post-processor.hcl2spec.go index 08be7133..72165083 100644 --- a/post-processor/yandex-import/post-processor.hcl2spec.go +++ b/post-processor/yandex-import/post-processor.hcl2spec.go @@ -10,31 +10,34 @@ import ( // FlatConfig is an auto-generated flat version of Config. // Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. type FlatConfig struct { - PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"` - PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"` - PackerCoreVersion *string `mapstructure:"packer_core_version" cty:"packer_core_version" hcl:"packer_core_version"` - PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"` - PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"` - PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"` - PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"` - PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"` - Endpoint *string `mapstructure:"endpoint" required:"false" cty:"endpoint" hcl:"endpoint"` - ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"` - Token *string `mapstructure:"token" required:"true" cty:"token" hcl:"token"` - MaxRetries *int `mapstructure:"max_retries" cty:"max_retries" hcl:"max_retries"` - FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id" hcl:"folder_id"` - ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"` - ImageName *string `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"` - ImageDescription *string `mapstructure:"image_description" required:"false" cty:"image_description" hcl:"image_description"` - ImageFamily *string `mapstructure:"image_family" required:"false" cty:"image_family" hcl:"image_family"` - ImageLabels map[string]string `mapstructure:"image_labels" required:"false" cty:"image_labels" hcl:"image_labels"` - ImageMinDiskSizeGb *int `mapstructure:"image_min_disk_size_gb" required:"false" cty:"image_min_disk_size_gb" hcl:"image_min_disk_size_gb"` - ImageProductIDs []string `mapstructure:"image_product_ids" required:"false" cty:"image_product_ids" hcl:"image_product_ids"` - ImagePooled *bool `mapstructure:"image_pooled" required:"false" cty:"image_pooled" hcl:"image_pooled"` - SkipCreateImage *bool `mapstructure:"skip_create_image" required:"false" cty:"skip_create_image" hcl:"skip_create_image"` - Bucket *string `mapstructure:"bucket" required:"false" cty:"bucket" hcl:"bucket"` - ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"` - SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"` + PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"` + PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"` + PackerCoreVersion *string `mapstructure:"packer_core_version" cty:"packer_core_version" hcl:"packer_core_version"` + PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"` + PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"` + PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"` + PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"` + PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"` + Endpoint *string `mapstructure:"endpoint" required:"false" cty:"endpoint" hcl:"endpoint"` + ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"` + Token *string `mapstructure:"token" required:"true" cty:"token" hcl:"token"` + MaxRetries *int `mapstructure:"max_retries" cty:"max_retries" hcl:"max_retries"` + FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id" hcl:"folder_id"` + ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"` + ImageName *string `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"` + ImageDescription *string `mapstructure:"image_description" required:"false" cty:"image_description" hcl:"image_description"` + ImageFamily *string `mapstructure:"image_family" required:"false" cty:"image_family" hcl:"image_family"` + ImageLabels map[string]string `mapstructure:"image_labels" required:"false" cty:"image_labels" hcl:"image_labels"` + ImageMinDiskSizeGb *int `mapstructure:"image_min_disk_size_gb" required:"false" cty:"image_min_disk_size_gb" hcl:"image_min_disk_size_gb"` + ImageProductIDs []string `mapstructure:"image_product_ids" required:"false" cty:"image_product_ids" hcl:"image_product_ids"` + ImagePooled *bool `mapstructure:"image_pooled" required:"false" cty:"image_pooled" hcl:"image_pooled"` + SkipCreateImage *bool `mapstructure:"skip_create_image" required:"false" cty:"skip_create_image" hcl:"skip_create_image"` + Bucket *string `mapstructure:"bucket" required:"false" cty:"bucket" hcl:"bucket"` + ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"` + SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"` + StorageEndpoint *string `mapstructure:"storage_endpoint" required:"false" cty:"storage_endpoint" hcl:"storage_endpoint"` + StorageEndpointAutoresolve *bool `mapstructure:"storage_endpoint_autoresolve" required:"false" cty:"storage_endpoint_autoresolve" hcl:"storage_endpoint_autoresolve"` + StorageRegion *string `mapstructure:"storage_region" required:"false" cty:"storage_region" hcl:"storage_region"` } // FlatMapstructure returns a new FlatConfig. @@ -49,31 +52,34 @@ func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } // The decoded values from this spec will then be applied to a FlatConfig. func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { s := map[string]hcldec.Spec{ - "packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false}, - "packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false}, - "packer_core_version": &hcldec.AttrSpec{Name: "packer_core_version", Type: cty.String, Required: false}, - "packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false}, - "packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false}, - "packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false}, - "packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false}, - "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, - "endpoint": &hcldec.AttrSpec{Name: "endpoint", Type: cty.String, Required: false}, - "service_account_key_file": &hcldec.AttrSpec{Name: "service_account_key_file", Type: cty.String, Required: false}, - "token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false}, - "max_retries": &hcldec.AttrSpec{Name: "max_retries", Type: cty.Number, Required: false}, - "folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false}, - "service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false}, - "image_name": &hcldec.AttrSpec{Name: "image_name", Type: cty.String, Required: false}, - "image_description": &hcldec.AttrSpec{Name: "image_description", Type: cty.String, Required: false}, - "image_family": &hcldec.AttrSpec{Name: "image_family", Type: cty.String, Required: false}, - "image_labels": &hcldec.AttrSpec{Name: "image_labels", Type: cty.Map(cty.String), Required: false}, - "image_min_disk_size_gb": &hcldec.AttrSpec{Name: "image_min_disk_size_gb", Type: cty.Number, Required: false}, - "image_product_ids": &hcldec.AttrSpec{Name: "image_product_ids", Type: cty.List(cty.String), Required: false}, - "image_pooled": &hcldec.AttrSpec{Name: "image_pooled", Type: cty.Bool, Required: false}, - "skip_create_image": &hcldec.AttrSpec{Name: "skip_create_image", Type: cty.Bool, Required: false}, - "bucket": &hcldec.AttrSpec{Name: "bucket", Type: cty.String, Required: false}, - "object_name": &hcldec.AttrSpec{Name: "object_name", Type: cty.String, Required: false}, - "skip_clean": &hcldec.AttrSpec{Name: "skip_clean", Type: cty.Bool, Required: false}, + "packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false}, + "packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false}, + "packer_core_version": &hcldec.AttrSpec{Name: "packer_core_version", Type: cty.String, Required: false}, + "packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false}, + "packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false}, + "packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false}, + "packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false}, + "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, + "endpoint": &hcldec.AttrSpec{Name: "endpoint", Type: cty.String, Required: false}, + "service_account_key_file": &hcldec.AttrSpec{Name: "service_account_key_file", Type: cty.String, Required: false}, + "token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false}, + "max_retries": &hcldec.AttrSpec{Name: "max_retries", Type: cty.Number, Required: false}, + "folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false}, + "service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false}, + "image_name": &hcldec.AttrSpec{Name: "image_name", Type: cty.String, Required: false}, + "image_description": &hcldec.AttrSpec{Name: "image_description", Type: cty.String, Required: false}, + "image_family": &hcldec.AttrSpec{Name: "image_family", Type: cty.String, Required: false}, + "image_labels": &hcldec.AttrSpec{Name: "image_labels", Type: cty.Map(cty.String), Required: false}, + "image_min_disk_size_gb": &hcldec.AttrSpec{Name: "image_min_disk_size_gb", Type: cty.Number, Required: false}, + "image_product_ids": &hcldec.AttrSpec{Name: "image_product_ids", Type: cty.List(cty.String), Required: false}, + "image_pooled": &hcldec.AttrSpec{Name: "image_pooled", Type: cty.Bool, Required: false}, + "skip_create_image": &hcldec.AttrSpec{Name: "skip_create_image", Type: cty.Bool, Required: false}, + "bucket": &hcldec.AttrSpec{Name: "bucket", Type: cty.String, Required: false}, + "object_name": &hcldec.AttrSpec{Name: "object_name", Type: cty.String, Required: false}, + "skip_clean": &hcldec.AttrSpec{Name: "skip_clean", Type: cty.Bool, Required: false}, + "storage_endpoint": &hcldec.AttrSpec{Name: "storage_endpoint", Type: cty.String, Required: false}, + "storage_endpoint_autoresolve": &hcldec.AttrSpec{Name: "storage_endpoint_autoresolve", Type: cty.Bool, Required: false}, + "storage_region": &hcldec.AttrSpec{Name: "storage_region", Type: cty.String, Required: false}, } return s } diff --git a/post-processor/yandex-import/storage.go b/post-processor/yandex-import/storage.go index f53d031a..d6b42e18 100644 --- a/post-processor/yandex-import/storage.go +++ b/post-processor/yandex-import/storage.go @@ -16,9 +16,6 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" ) -const defaultS3Region = "ru-central1" -const defaultStorageEndpoint = "storage.yandexcloud.net" - func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, error) { var creds *credentials.Credentials @@ -28,7 +25,7 @@ func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, e s3Config := &aws.Config{ Endpoint: aws.String(storageEndpoint), - Region: aws.String(defaultS3Region), + Region: aws.String(defaultStorageRegion), } switch {