diff --git a/pkg/common/common.go b/pkg/common/common.go index 3a1a40577b..ad21635371 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -72,6 +72,9 @@ const ( // AnnotationAllowPrivilegedContainers is the annotation describing whether and how created shoots will have allowPrivilegedContainers configured AnnotationAllowPrivilegedContainers = "metadata.testmachinery.gardener.cloud/allow-privileged-containers" + // AnnotationShootAnnotations is the annotation describing which additional shoot annotations were set (as they could impact the shoot behaviour) + AnnotationShootAnnotations = "metadata.testmachinery.gardener.cloud/shoot-annotations" + // AnnotationFlavorDescription is the annotation to describe the test flavor of the current run testrun AnnotationFlavorDescription = "metadata.testmachinery.gardener.cloud/flavor-description" diff --git a/pkg/common/types_shootflavor.go b/pkg/common/types_shootflavor.go index 7391f8312c..8f04e0f029 100644 --- a/pkg/common/types_shootflavor.go +++ b/pkg/common/types_shootflavor.go @@ -50,6 +50,9 @@ type Shoot struct { // AllowPrivilegedContainers defines whether privileged containers will be allowed in the given shoot or not AllowPrivilegedContainers *bool + // AdditionalAnnotations holds annotations to be added to created shoots + AdditionalAnnotations map[string]string + // Worker pools to test Workers []gardencorev1beta1.Worker } @@ -70,6 +73,10 @@ type ShootFlavor struct { // +optional AllowPrivilegedContainers *bool `json:"allowPrivilegedContainers"` + // AdditionalAnnotations allows to optionally define additional annotations for the created shoot resources + // +optional + AdditionalAnnotations map[string]string `json:"annotations"` + // Worker pools to test Workers []ShootWorkerFlavor `json:"workers"` } diff --git a/pkg/shootflavors/extendedflavors.go b/pkg/shootflavors/extendedflavors.go index 2866c462dd..51305ab2ea 100644 --- a/pkg/shootflavors/extendedflavors.go +++ b/pkg/shootflavors/extendedflavors.go @@ -92,6 +92,7 @@ func NewExtended(k8sClient client.Client, rawFlavors []*common.ExtendedShootFlav shoot: &common.ExtendedShoot{ Shoot: common.Shoot{ Description: rawFlavor.Description, + AdditionalAnnotations: rawFlavor.AdditionalAnnotations, Provider: rawFlavor.Provider, KubernetesVersion: k8sVersion, AllowPrivilegedContainers: rawFlavor.AllowPrivilegedContainers, diff --git a/pkg/shootflavors/extendedflavors_test.go b/pkg/shootflavors/extendedflavors_test.go index 171a3c7137..06adb07bf3 100644 --- a/pkg/shootflavors/extendedflavors_test.go +++ b/pkg/shootflavors/extendedflavors_test.go @@ -86,6 +86,7 @@ var _ = Describe("extended flavor test", func() { ExtendedConfiguration: defaultExtendedCfg, ShootFlavor: common.ShootFlavor{ AllowPrivilegedContainers: pointer.BoolPtr(true), + AdditionalAnnotations: map[string]string{"a": "b"}, Provider: common.CloudProviderGCP, KubernetesVersions: common.ShootKubernetesVersionFlavor{ Versions: &[]gardencorev1beta1.ExpirableVersion{ @@ -111,6 +112,7 @@ var _ = Describe("extended flavor test", func() { Expect(shoot.Get().Shoot).To(Equal(common.Shoot{ Provider: common.CloudProviderGCP, AllowPrivilegedContainers: pointer.BoolPtr(true), + AdditionalAnnotations: map[string]string{"a": "b"}, KubernetesVersion: gardencorev1beta1.ExpirableVersion{Version: "1.15"}, Workers: []gardencorev1beta1.Worker{{Name: "wp1"}}, })) diff --git a/pkg/shootflavors/flavors.go b/pkg/shootflavors/flavors.go index dbfcbce982..0b3c87b615 100644 --- a/pkg/shootflavors/flavors.go +++ b/pkg/shootflavors/flavors.go @@ -82,6 +82,7 @@ func New(rawFlavors []*common.ShootFlavor) (*Flavors, error) { } shoots = append(shoots, &common.Shoot{ + AdditionalAnnotations: rawFlavor.AdditionalAnnotations, Provider: rawFlavor.Provider, KubernetesVersion: k8sVersion, AllowPrivilegedContainers: rawFlavor.AllowPrivilegedContainers, @@ -91,6 +92,7 @@ func New(rawFlavors []*common.ShootFlavor) (*Flavors, error) { continue } shoots = append(shoots, &common.Shoot{ + AdditionalAnnotations: rawFlavor.AdditionalAnnotations, Provider: rawFlavor.Provider, KubernetesVersion: k8sVersion, AllowPrivilegedContainers: rawFlavor.AllowPrivilegedContainers, diff --git a/pkg/shootflavors/flavors_test.go b/pkg/shootflavors/flavors_test.go index 960ea26ce0..3f9cd42d0e 100644 --- a/pkg/shootflavors/flavors_test.go +++ b/pkg/shootflavors/flavors_test.go @@ -76,6 +76,32 @@ var _ = Describe("flavor test", func() { )) }) + It("should return one shoot with additional annotations", func() { + rawFlavors := []*common.ShootFlavor{ + { + Provider: common.CloudProviderGCP, + AdditionalAnnotations: map[string]string{"x": "y"}, + KubernetesVersions: common.ShootKubernetesVersionFlavor{ + Versions: &[]gardencorev1beta1.ExpirableVersion{ + { + Version: "1.15", + }, + }, + }, + }, + } + flavors, err := New(rawFlavors) + Expect(err).ToNot(HaveOccurred()) + Expect(flavors.GetShoots()).To(HaveLen(1)) + Expect(flavors.GetShoots()).To(ConsistOf( + &common.Shoot{ + Provider: common.CloudProviderGCP, + AdditionalAnnotations: map[string]string{"x": "y"}, + KubernetesVersion: gardencorev1beta1.ExpirableVersion{Version: "1.15"}, + }, + )) + }) + It("should return one shoot with disabled allowPrivilegeContainers", func() { rawFlavors := []*common.ShootFlavor{ { diff --git a/pkg/testmachinery/metadata/metadata.go b/pkg/testmachinery/metadata/metadata.go index cf90f88297..dd1d05017a 100644 --- a/pkg/testmachinery/metadata/metadata.go +++ b/pkg/testmachinery/metadata/metadata.go @@ -18,6 +18,7 @@ import ( "fmt" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" "github.com/gardener/test-infra/pkg/common" + "github.com/gardener/test-infra/pkg/util" "strconv" ) @@ -34,6 +35,7 @@ func (m *Metadata) CreateAnnotations() map[string]string { common.AnnotationFlavorDescription: m.FlavorDescription, common.AnnotationDimension: m.GetDimensionFromMetadata("/"), common.AnnotationRetries: strconv.Itoa(m.Retries), + common.AnnotationShootAnnotations: util.MarshalMap(m.Annotations), } if m.AllowPrivilegedContainers != nil { annotations[common.AnnotationAllowPrivilegedContainers] = strconv.FormatBool(*m.AllowPrivilegedContainers) @@ -63,6 +65,11 @@ func (m *Metadata) DeepCopy() *Metadata { // FromTestrun reads metadata from a testrun func FromTestrun(tr *tmv1beta1.Testrun) *Metadata { retries, _ := strconv.Atoi(tr.Annotations[common.AnnotationRetries]) + shootAnnotations, err := util.UnmarshalMap(tr.Annotations[common.AnnotationShootAnnotations]) + if err != nil { + shootAnnotations = make(map[string]string) + shootAnnotations["error"] = err.Error() + } metadata := &Metadata{ Landscape: tr.Annotations[common.AnnotationLandscape], KubernetesVersion: tr.Annotations[common.AnnotationK8sVersion], @@ -72,6 +79,7 @@ func FromTestrun(tr *tmv1beta1.Testrun) *Metadata { Region: tr.Annotations[common.AnnotationRegion], Zone: tr.Annotations[common.AnnotationZone], FlavorDescription: tr.Annotations[common.AnnotationFlavorDescription], + ShootAnnotations: shootAnnotations, Retries: retries, Testrun: TestrunMetadata{ ID: tr.Name, diff --git a/pkg/testmachinery/metadata/types.go b/pkg/testmachinery/metadata/types.go index 7ad0eb097b..9a7b3fadc6 100644 --- a/pkg/testmachinery/metadata/types.go +++ b/pkg/testmachinery/metadata/types.go @@ -26,10 +26,11 @@ type Metadata struct { Region string `json:"region,omitempty"` // todo: schrodit - add support to better persist multiple worker pools with multiple oss, versions and zones - OperatingSystem string `json:"operating_system,omitempty"` - OperatingSystemVersion string `json:"operating_system_version,omitempty"` - Zone string `json:"zone,omitempty"` - AllowPrivilegedContainers *bool `json:"allow_privileged_containers,omitempty"` + OperatingSystem string `json:"operating_system,omitempty"` + OperatingSystemVersion string `json:"operating_system_version,omitempty"` + Zone string `json:"zone,omitempty"` + AllowPrivilegedContainers *bool `json:"allow_privileged_containers,omitempty"` + ShootAnnotations map[string]string `json:"shoot_annotations,omitempty"` // ComponentDescriptor describes the current component_descriptor of the direct landscape-setup components. // It is formatted as an array of components: { name: "my_component", version: "0.0.1" } diff --git a/pkg/testrun_renderer/default/default.go b/pkg/testrun_renderer/default/default.go index 3663e9c6c5..1e56f9903e 100644 --- a/pkg/testrun_renderer/default/default.go +++ b/pkg/testrun_renderer/default/default.go @@ -108,6 +108,7 @@ func Render(cfg *Config) (*v1beta1.Testrun, error) { Namespace: cfg.Shoots.Namespace, K8sVersion: flavor.KubernetesVersion.Version, AllowPrivilegedContainers: flavor.AllowPrivilegedContainers, + ShootAnnotations: flavor.AdditionalAnnotations, }, }) for _, test := range cfg.Shoots.Tests { @@ -120,6 +121,7 @@ func Render(cfg *Config) (*v1beta1.Testrun, error) { Namespace: cfg.Shoots.Namespace, K8sVersion: flavor.KubernetesVersion.Version, AllowPrivilegedContainers: flavor.AllowPrivilegedContainers, + ShootAnnotations: flavor.AdditionalAnnotations, }, }) } diff --git a/pkg/testrun_renderer/templates/shoots.go b/pkg/testrun_renderer/templates/shoots.go index 8fb8c21fe5..aa328fcfcd 100644 --- a/pkg/testrun_renderer/templates/shoots.go +++ b/pkg/testrun_renderer/templates/shoots.go @@ -30,6 +30,7 @@ const ( ConfigInfrastructureProviderPathName = "INFRASTRUCTURE_PROVIDER_CONFIG_FILEPATH" ConfigShootName = "SHOOT_NAME" + ConfigShootAnnotations = "SHOOT_ANNOTATIONS" ConfigProjectNamespaceName = "PROJECT_NAMESPACE" ConfigK8sVersionName = "K8S_VERSION" ConfigCloudproviderName = "CLOUDPROVIDER" @@ -49,6 +50,7 @@ var ( // CreateShootConfig describes the configuration for a create-shoot step type CreateShootConfig struct { ShootName string + ShootAnnotations map[string]string Namespace string K8sVersion string AllowPrivilegedContainers *bool diff --git a/pkg/testrun_renderer/templates/shoots_v1beta1.go b/pkg/testrun_renderer/templates/shoots_v1beta1.go index 5260e1b378..07e8cdf95d 100644 --- a/pkg/testrun_renderer/templates/shoots_v1beta1.go +++ b/pkg/testrun_renderer/templates/shoots_v1beta1.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" "github.com/gardener/test-infra/pkg/common" + "github.com/gardener/test-infra/pkg/util" "strconv" ) @@ -98,6 +99,11 @@ func defaultShootConfig(cfg *CreateShootConfig) []v1beta1.ConfigElement { Name: ConfigSeedName, Value: ConfigSeedValue, }, + { + Type: v1beta1.ConfigTypeEnv, + Name: ConfigShootAnnotations, + Value: util.MarshalMap(cfg.ShootAnnotations), + }, } if cfg.AllowPrivilegedContainers != nil { diff --git a/pkg/testrunner/template/shoot_template_test.go b/pkg/testrunner/template/shoot_template_test.go index e43ca88aaf..5101fd8bc1 100644 --- a/pkg/testrunner/template/shoot_template_test.go +++ b/pkg/testrunner/template/shoot_template_test.go @@ -40,6 +40,7 @@ var _ = Describe("shoot templates", func() { KubernetesVersion: gardencorev1beta1.ExpirableVersion{Version: "1.15.2"}, Workers: []gardencorev1beta1.Worker{{Name: "wp1", Machine: gardencorev1beta1.Machine{Image: &gardencorev1beta1.ShootMachineImage{Name: "core-os"}}}}, AllowPrivilegedContainers: pointer.BoolPtr(false), + AdditionalAnnotations: map[string]string{"a": "b"}, }, ExtendedShootConfiguration: common.ExtendedShootConfiguration{ Name: "test-name", @@ -77,6 +78,7 @@ var _ = Describe("shoot templates", func() { Expect(tr.Annotations).To(HaveKeyWithValue("shoot.region", "region-1")) Expect(tr.Annotations).To(HaveKeyWithValue("shoot.zone", "region-1-1")) Expect(tr.Annotations).To(HaveKeyWithValue("shoot.allowPrivilegedContainers", "false")) + Expect(tr.Annotations).To(HaveKeyWithValue("shoot.shootAnnotations", "a=b")) Expect(tr.Annotations).To(HaveKeyWithValue("shoot.k8sVersion", "1.15.2")) Expect(tr.Annotations).To(HaveKeyWithValue("shoot.k8sPrevPrePatchVersion", "1.15.2")) Expect(tr.Annotations).To(HaveKeyWithValue("shoot.k8sPrevPatchVersion", "1.15.2")) @@ -101,6 +103,7 @@ var _ = Describe("shoot templates", func() { Expect(meta.Region).To(Equal("region-1")) Expect(meta.Zone).To(Equal("region-1-1")) Expect(meta.AllowPrivilegedContainers).To(Equal(pointer.BoolPtr(false))) + Expect(meta.Annotations).To(Equal(map[string]string{"a": "b"})) Expect(meta.OperatingSystem).To(Equal("core-os")) }) diff --git a/pkg/testrunner/template/testdata/shoot/basic/templates/testrun.yaml b/pkg/testrunner/template/testdata/shoot/basic/templates/testrun.yaml index 1487f16a53..25f78c2f19 100644 --- a/pkg/testrunner/template/testdata/shoot/basic/templates/testrun.yaml +++ b/pkg/testrunner/template/testdata/shoot/basic/templates/testrun.yaml @@ -15,6 +15,9 @@ metadata: {{- if hasKey .Values.shoot "allowPrivilegedContainers" }} shoot.allowPrivilegedContainers: "{{ .Values.shoot.allowPrivilegedContainers }}" {{- end }} + {{- if hasKey .Values.shoot "shootAnnotations" }} + shoot.shootAnnotations: "{{ .Values.shoot.shootAnnotations }}" + {{- end }} shoot.workers: {{ required "workers is required" .Values.shoot.workers }} shoot.k8sVersion: {{ required "k8sVersion is required" .Values.shoot.k8sVersion }} shoot.k8sPrevPrePatchVersion: {{ required "k8sPrevPrePatchVersion is required" .Values.shoot.k8sPrevPrePatchVersion }} diff --git a/pkg/testrunner/template/values.go b/pkg/testrunner/template/values.go index d7d84fd66f..8d175ce580 100644 --- a/pkg/testrunner/template/values.go +++ b/pkg/testrunner/template/values.go @@ -153,6 +153,9 @@ func (r *shootValueRenderer) GetValues(shoot *common.ExtendedShoot, defaultValue if shoot.AllowPrivilegedContainers != nil { values["shoot"].(map[string]interface{})["allowPrivilegedContainers"] = shoot.AllowPrivilegedContainers } + if shoot.AdditionalAnnotations != nil { + values["shoot"].(map[string]interface{})["shootAnnotations"] = util.MarshalMap(shoot.AdditionalAnnotations) + } return utils.MergeMaps(defaultValues, values), nil } @@ -172,5 +175,6 @@ func (r *shootValueRenderer) GetMetadata(shoot *common.ExtendedShoot) (*metadata AllowPrivilegedContainers: shoot.AllowPrivilegedContainers, OperatingSystem: shoot.Workers[0].Machine.Image.Name, // todo: check if there a possible multiple workerpools with different images OperatingSystemVersion: operatingsystemversion, + Annotations: shoot.AdditionalAnnotations, }, nil } diff --git a/pkg/util/util.go b/pkg/util/util.go index e879eb4f6a..691ab142e2 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -45,6 +45,7 @@ import ( "time" argov1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" + testfwk "github.com/gardener/gardener/test/framework" tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1" "sigs.k8s.io/yaml" ) @@ -486,3 +487,34 @@ func GetClusterDomainURL(tmClient client.Client) (string, error) { } return matches[1], nil } + +// MarshalMap encodes the given map into a string similar to kubectl --selectors: map '{a:b,c:d}' becomes string 'a=b,c=d' +func MarshalMap(annotations map[string]string) string { + var buf []string + for k, v := range annotations { + buf = append(buf, fmt.Sprintf("%s=%s", k, v)) + } + return strings.Join(buf, ",") +} + +// UnmarshalMap does the opposite of MarshalMap. It decodes the given string into a map, return an error if the string is not in the expected format 'key1=value1,key2=value2' +func UnmarshalMap(cfg string) (map[string]string, error) { + if !testfwk.StringSet(cfg) { + return nil, nil + } + result := make(map[string]string) + annotations := strings.Split(cfg, ",") + for _, annotation := range annotations { + annotation = strings.TrimSpace(annotation) + if !testfwk.StringSet(annotation) { + continue + } + keyValue := strings.Split(annotation, "=") + if len(keyValue) != 2 { + return nil, fmt.Errorf("annotation %s could not be parsed into key and value", annotation) + } + result[keyValue[0]] = keyValue[1] + } + + return result, nil +}