add shootAnnotations to flavors (#294)

pkg/common/common.go

@@ -72,6 +72,9 @@ const (
 	// AnnotationAllowPrivilegedContainers is the annotation describing whether and how created shoots will have allowPrivilegedContainers configured
 	AnnotationAllowPrivilegedContainers = "metadata.testmachinery.gardener.cloud/allow-privileged-containers"
 
+	// AnnotationShootAnnotations is the annotation describing which additional shoot annotations were set (as they could impact the shoot behaviour)
+	AnnotationShootAnnotations = "metadata.testmachinery.gardener.cloud/shoot-annotations"
+
 	// AnnotationFlavorDescription is the annotation to describe the test flavor of the current run testrun
 	AnnotationFlavorDescription = "metadata.testmachinery.gardener.cloud/flavor-description"
 

pkg/common/types_shootflavor.go

@@ -50,6 +50,9 @@ type Shoot struct {
 	// AllowPrivilegedContainers defines whether privileged containers will be allowed in the given shoot or not
 	AllowPrivilegedContainers *bool
 
+	// AdditionalAnnotations holds annotations to be added to created shoots
+	AdditionalAnnotations map[string]string
+
 	// Worker pools to test
 	Workers []gardencorev1beta1.Worker
 }
@@ -70,6 +73,10 @@ type ShootFlavor struct {
 	// +optional
 	AllowPrivilegedContainers *bool `json:"allowPrivilegedContainers"`
 
+	// AdditionalAnnotations allows to optionally define additional annotations for the created shoot resources
+	// +optional
+	AdditionalAnnotations map[string]string `json:"annotations"`
+
 	// Worker pools to test
 	Workers []ShootWorkerFlavor `json:"workers"`
 }

pkg/shootflavors/extendedflavors.go

@@ -92,6 +92,7 @@ func NewExtended(k8sClient client.Client, rawFlavors []*common.ExtendedShootFlav
 					shoot: &common.ExtendedShoot{
 						Shoot: common.Shoot{
 							Description:               rawFlavor.Description,
+							AdditionalAnnotations:     rawFlavor.AdditionalAnnotations,
 							Provider:                  rawFlavor.Provider,
 							KubernetesVersion:         k8sVersion,
 							AllowPrivilegedContainers: rawFlavor.AllowPrivilegedContainers,

pkg/shootflavors/extendedflavors_test.go

@@ -86,6 +86,7 @@ var _ = Describe("extended flavor test", func() {
 			ExtendedConfiguration: defaultExtendedCfg,
 			ShootFlavor: common.ShootFlavor{
 				AllowPrivilegedContainers: pointer.BoolPtr(true),
+				AdditionalAnnotations:     map[string]string{"a": "b"},
 				Provider:                  common.CloudProviderGCP,
 				KubernetesVersions: common.ShootKubernetesVersionFlavor{
 					Versions: &[]gardencorev1beta1.ExpirableVersion{
@@ -111,6 +112,7 @@ var _ = Describe("extended flavor test", func() {
 		Expect(shoot.Get().Shoot).To(Equal(common.Shoot{
 			Provider:                  common.CloudProviderGCP,
 			AllowPrivilegedContainers: pointer.BoolPtr(true),
+			AdditionalAnnotations:     map[string]string{"a": "b"},
 			KubernetesVersion:         gardencorev1beta1.ExpirableVersion{Version: "1.15"},
 			Workers:                   []gardencorev1beta1.Worker{{Name: "wp1"}},
 		}))

pkg/shootflavors/flavors.go

@@ -82,6 +82,7 @@ func New(rawFlavors []*common.ShootFlavor) (*Flavors, error) {
 					}
 
 					shoots = append(shoots, &common.Shoot{
+						AdditionalAnnotations:     rawFlavor.AdditionalAnnotations,
 						Provider:                  rawFlavor.Provider,
 						KubernetesVersion:         k8sVersion,
 						AllowPrivilegedContainers: rawFlavor.AllowPrivilegedContainers,
@@ -91,6 +92,7 @@ func New(rawFlavors []*common.ShootFlavor) (*Flavors, error) {
 				continue
 			}
 			shoots = append(shoots, &common.Shoot{
+				AdditionalAnnotations:     rawFlavor.AdditionalAnnotations,
 				Provider:                  rawFlavor.Provider,
 				KubernetesVersion:         k8sVersion,
 				AllowPrivilegedContainers: rawFlavor.AllowPrivilegedContainers,

pkg/shootflavors/flavors_test.go

@@ -76,6 +76,32 @@ var _ = Describe("flavor test", func() {
 		))
 	})
 
+	It("should return one shoot with additional annotations", func() {
+		rawFlavors := []*common.ShootFlavor{
+			{
+				Provider:              common.CloudProviderGCP,
+				AdditionalAnnotations: map[string]string{"x": "y"},
+				KubernetesVersions: common.ShootKubernetesVersionFlavor{
+					Versions: &[]gardencorev1beta1.ExpirableVersion{
+						{
+							Version: "1.15",
+						},
+					},
+				},
+			},
+		}
+		flavors, err := New(rawFlavors)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(flavors.GetShoots()).To(HaveLen(1))
+		Expect(flavors.GetShoots()).To(ConsistOf(
+			&common.Shoot{
+				Provider:              common.CloudProviderGCP,
+				AdditionalAnnotations: map[string]string{"x": "y"},
+				KubernetesVersion:     gardencorev1beta1.ExpirableVersion{Version: "1.15"},
+			},
+		))
+	})
+
 	It("should return one shoot with disabled allowPrivilegeContainers", func() {
 		rawFlavors := []*common.ShootFlavor{
 			{

pkg/testmachinery/metadata/metadata.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
 	"github.com/gardener/test-infra/pkg/common"
+	"github.com/gardener/test-infra/pkg/util"
 	"strconv"
 )
 
@@ -34,6 +35,7 @@ func (m *Metadata) CreateAnnotations() map[string]string {
 		common.AnnotationFlavorDescription:      m.FlavorDescription,
 		common.AnnotationDimension:              m.GetDimensionFromMetadata("/"),
 		common.AnnotationRetries:                strconv.Itoa(m.Retries),
+		common.AnnotationShootAnnotations:       util.MarshalMap(m.Annotations),
 	}
 	if m.AllowPrivilegedContainers != nil {
 		annotations[common.AnnotationAllowPrivilegedContainers] = strconv.FormatBool(*m.AllowPrivilegedContainers)
@@ -63,6 +65,11 @@ func (m *Metadata) DeepCopy() *Metadata {
 // FromTestrun reads metadata from a testrun
 func FromTestrun(tr *tmv1beta1.Testrun) *Metadata {
 	retries, _ := strconv.Atoi(tr.Annotations[common.AnnotationRetries])
+	shootAnnotations, err := util.UnmarshalMap(tr.Annotations[common.AnnotationShootAnnotations])
+	if err != nil {
+		shootAnnotations = make(map[string]string)
+		shootAnnotations["error"] = err.Error()
+	}
 	metadata := &Metadata{
 		Landscape:              tr.Annotations[common.AnnotationLandscape],
 		KubernetesVersion:      tr.Annotations[common.AnnotationK8sVersion],
@@ -72,6 +79,7 @@ func FromTestrun(tr *tmv1beta1.Testrun) *Metadata {
 		Region:                 tr.Annotations[common.AnnotationRegion],
 		Zone:                   tr.Annotations[common.AnnotationZone],
 		FlavorDescription:      tr.Annotations[common.AnnotationFlavorDescription],
+		ShootAnnotations:       shootAnnotations,
 		Retries:                retries,
 		Testrun: TestrunMetadata{
 			ID:             tr.Name,

pkg/testmachinery/metadata/types.go

@@ -26,10 +26,11 @@ type Metadata struct {
 	Region            string `json:"region,omitempty"`
 
 	// todo: schrodit - add support to better persist multiple worker pools with multiple oss, versions and zones
-	OperatingSystem           string `json:"operating_system,omitempty"`
-	OperatingSystemVersion    string `json:"operating_system_version,omitempty"`
-	Zone                      string `json:"zone,omitempty"`
-	AllowPrivilegedContainers *bool  `json:"allow_privileged_containers,omitempty"`
+	OperatingSystem           string            `json:"operating_system,omitempty"`
+	OperatingSystemVersion    string            `json:"operating_system_version,omitempty"`
+	Zone                      string            `json:"zone,omitempty"`
+	AllowPrivilegedContainers *bool             `json:"allow_privileged_containers,omitempty"`
+	ShootAnnotations          map[string]string `json:"shoot_annotations,omitempty"`
 
 	// ComponentDescriptor describes the current component_descriptor of the direct landscape-setup components.
 	// It is formatted as an array of components: { name: "my_component", version: "0.0.1" }

pkg/testrun_renderer/default/default.go

@@ -108,6 +108,7 @@ func Render(cfg *Config) (*v1beta1.Testrun, error) {
 				Namespace:                 cfg.Shoots.Namespace,
 				K8sVersion:                flavor.KubernetesVersion.Version,
 				AllowPrivilegedContainers: flavor.AllowPrivilegedContainers,
+				ShootAnnotations:          flavor.AdditionalAnnotations,
 			},
 		})
 		for _, test := range cfg.Shoots.Tests {
@@ -120,6 +121,7 @@ func Render(cfg *Config) (*v1beta1.Testrun, error) {
 					Namespace:                 cfg.Shoots.Namespace,
 					K8sVersion:                flavor.KubernetesVersion.Version,
 					AllowPrivilegedContainers: flavor.AllowPrivilegedContainers,
+					ShootAnnotations:          flavor.AdditionalAnnotations,
 				},
 			})
 		}

pkg/testrun_renderer/templates/shoots.go

@@ -30,6 +30,7 @@ const (
 	ConfigInfrastructureProviderPathName = "INFRASTRUCTURE_PROVIDER_CONFIG_FILEPATH"
 
 	ConfigShootName                 = "SHOOT_NAME"
+	ConfigShootAnnotations          = "SHOOT_ANNOTATIONS"
 	ConfigProjectNamespaceName      = "PROJECT_NAMESPACE"
 	ConfigK8sVersionName            = "K8S_VERSION"
 	ConfigCloudproviderName         = "CLOUDPROVIDER"
@@ -49,6 +50,7 @@ var (
 // CreateShootConfig describes the configuration for a create-shoot step
 type CreateShootConfig struct {
 	ShootName                 string
+	ShootAnnotations          map[string]string
 	Namespace                 string
 	K8sVersion                string
 	AllowPrivilegedContainers *bool

pkg/testrun_renderer/templates/shoots_v1beta1.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
 	"github.com/gardener/test-infra/pkg/common"
+	"github.com/gardener/test-infra/pkg/util"
 	"strconv"
 )
 
@@ -98,6 +99,11 @@ func defaultShootConfig(cfg *CreateShootConfig) []v1beta1.ConfigElement {
 			Name:  ConfigSeedName,
 			Value: ConfigSeedValue,
 		},
+		{
+			Type:  v1beta1.ConfigTypeEnv,
+			Name:  ConfigShootAnnotations,
+			Value: util.MarshalMap(cfg.ShootAnnotations),
+		},
 	}
 
 	if cfg.AllowPrivilegedContainers != nil {

pkg/testrunner/template/shoot_template_test.go

@@ -40,6 +40,7 @@ var _ = Describe("shoot templates", func() {
 					KubernetesVersion:         gardencorev1beta1.ExpirableVersion{Version: "1.15.2"},
 					Workers:                   []gardencorev1beta1.Worker{{Name: "wp1", Machine: gardencorev1beta1.Machine{Image: &gardencorev1beta1.ShootMachineImage{Name: "core-os"}}}},
 					AllowPrivilegedContainers: pointer.BoolPtr(false),
+					AdditionalAnnotations:     map[string]string{"a": "b"},
 				},
 				ExtendedShootConfiguration: common.ExtendedShootConfiguration{
 					Name:         "test-name",
@@ -77,6 +78,7 @@ var _ = Describe("shoot templates", func() {
 			Expect(tr.Annotations).To(HaveKeyWithValue("shoot.region", "region-1"))
 			Expect(tr.Annotations).To(HaveKeyWithValue("shoot.zone", "region-1-1"))
 			Expect(tr.Annotations).To(HaveKeyWithValue("shoot.allowPrivilegedContainers", "false"))
+			Expect(tr.Annotations).To(HaveKeyWithValue("shoot.shootAnnotations", "a=b"))
 			Expect(tr.Annotations).To(HaveKeyWithValue("shoot.k8sVersion", "1.15.2"))
 			Expect(tr.Annotations).To(HaveKeyWithValue("shoot.k8sPrevPrePatchVersion", "1.15.2"))
 			Expect(tr.Annotations).To(HaveKeyWithValue("shoot.k8sPrevPatchVersion", "1.15.2"))
@@ -101,6 +103,7 @@ var _ = Describe("shoot templates", func() {
 			Expect(meta.Region).To(Equal("region-1"))
 			Expect(meta.Zone).To(Equal("region-1-1"))
 			Expect(meta.AllowPrivilegedContainers).To(Equal(pointer.BoolPtr(false)))
+			Expect(meta.Annotations).To(Equal(map[string]string{"a": "b"}))
 			Expect(meta.OperatingSystem).To(Equal("core-os"))
 		})
 

pkg/testrunner/template/testdata/shoot/basic/templates/testrun.yaml

@@ -15,6 +15,9 @@ metadata:
         {{- if  hasKey .Values.shoot "allowPrivilegedContainers" }}
         shoot.allowPrivilegedContainers: "{{ .Values.shoot.allowPrivilegedContainers }}"
         {{- end }}
+        {{- if  hasKey .Values.shoot "shootAnnotations" }}
+        shoot.shootAnnotations: "{{ .Values.shoot.shootAnnotations }}"
+        {{- end }}
         shoot.workers: {{ required "workers is required" .Values.shoot.workers }}
         shoot.k8sVersion: {{ required "k8sVersion is required" .Values.shoot.k8sVersion }}
         shoot.k8sPrevPrePatchVersion: {{ required "k8sPrevPrePatchVersion is required" .Values.shoot.k8sPrevPrePatchVersion }}

pkg/testrunner/template/values.go

@@ -153,6 +153,9 @@ func (r *shootValueRenderer) GetValues(shoot *common.ExtendedShoot, defaultValue
 	if shoot.AllowPrivilegedContainers != nil {
 		values["shoot"].(map[string]interface{})["allowPrivilegedContainers"] = shoot.AllowPrivilegedContainers
 	}
+	if shoot.AdditionalAnnotations != nil {
+		values["shoot"].(map[string]interface{})["shootAnnotations"] = util.MarshalMap(shoot.AdditionalAnnotations)
+	}
 	return utils.MergeMaps(defaultValues, values), nil
 }
 
@@ -172,5 +175,6 @@ func (r *shootValueRenderer) GetMetadata(shoot *common.ExtendedShoot) (*metadata
 		AllowPrivilegedContainers: shoot.AllowPrivilegedContainers,
 		OperatingSystem:           shoot.Workers[0].Machine.Image.Name, // todo: check if there a possible multiple workerpools with different images
 		OperatingSystemVersion:    operatingsystemversion,
+		Annotations:               shoot.AdditionalAnnotations,
 	}, nil
 }

pkg/util/util.go

@@ -45,6 +45,7 @@ import (
 	"time"
 
 	argov1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
+	testfwk "github.com/gardener/gardener/test/framework"
 	tmv1beta1 "github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
 	"sigs.k8s.io/yaml"
 )
@@ -486,3 +487,34 @@ func GetClusterDomainURL(tmClient client.Client) (string, error) {
 	}
 	return matches[1], nil
 }
+
+// MarshalMap encodes the given map into a string similar to kubectl --selectors: map '{a:b,c:d}' becomes string 'a=b,c=d'
+func MarshalMap(annotations map[string]string) string {
+	var buf []string
+	for k, v := range annotations {
+		buf = append(buf, fmt.Sprintf("%s=%s", k, v))
+	}
+	return strings.Join(buf, ",")
+}
+
+// UnmarshalMap does the opposite of MarshalMap. It decodes the given string into a map, return an error if the string is not in the expected format 'key1=value1,key2=value2'
+func UnmarshalMap(cfg string) (map[string]string, error) {
+	if !testfwk.StringSet(cfg) {
+		return nil, nil
+	}
+	result := make(map[string]string)
+	annotations := strings.Split(cfg, ",")
+	for _, annotation := range annotations {
+		annotation = strings.TrimSpace(annotation)
+		if !testfwk.StringSet(annotation) {
+			continue
+		}
+		keyValue := strings.Split(annotation, "=")
+		if len(keyValue) != 2 {
+			return nil, fmt.Errorf("annotation %s could not be parsed into key and value", annotation)
+		}
+		result[keyValue[0]] = keyValue[1]
+	}
+
+	return result, nil
+}