From 8c6bcd26e2f6bb4b4e662a9cc553e13c8518f37d Mon Sep 17 00:00:00 2001 From: Lukas Piwowarski Date: Fri, 13 Oct 2023 17:47:13 +0300 Subject: [PATCH 1/2] Add python-tempestconf options This patch introduces tempestconfRun section for the Tempest CR. This field allows a user to specify parameters with which discover-tempest-config should be executed inside the Tempest pod. The documentation is updated as well so that it demonstrates how the new field should be utilized. --- api/bases/test.openstack.org_tempests.yaml | 109 +++++++++++- api/v1beta1/tempest_types.go | 155 ++++++++++++++++-- api/v1beta1/zz_generated.deepcopy.go | 25 ++- .../bases/test.openstack.org_tempests.yaml | 109 +++++++++++- config/samples/test_v1beta1_tempest.yaml | 52 ++++++ controllers/tempest_controller.go | 93 +++++++++++ docs/source/samples/tempest-config.yaml | 34 ++++ docs/source/samples/tempest-deployment.yaml | 3 + pkg/tempest/job.go | 29 ++-- 9 files changed, 576 insertions(+), 33 deletions(-) diff --git a/api/bases/test.openstack.org_tempests.yaml b/api/bases/test.openstack.org_tempests.yaml index 516f5582..01f71c16 100644 --- a/api/bases/test.openstack.org_tempests.yaml +++ b/api/bases/test.openstack.org_tempests.yaml @@ -120,7 +120,7 @@ spec: type: string type: array concurrency: - default: 0 + default: -1 description: Concurrency is the Default concurrency format: int64 type: integer @@ -133,6 +133,113 @@ spec: description: WorkerFile is the detailed concurry spec file type: string type: object + tempestconfRun: + description: TempestSpec PythonTempestconf parts + properties: + append: + default: "" + description: Append values to tempest.conf + type: string + collectTiming: + default: false + description: Collect per-API call timing information. + type: boolean + convertToRaw: + default: false + description: Convert images to raw format before uploading. + type: boolean + create: + default: true + description: Create Tempest resources + type: boolean + createAccountsFile: + default: "" + description: Generate Tempest accounts file. + type: string + debug: + default: false + description: Print debugging information. + type: boolean + deployerInput: + default: "" + description: Path to deployer file + type: string + flavorMinDisk: + default: -1 + description: Specify minimum disk size for new flavors + format: int64 + type: integer + flavorMinMem: + default: -1 + description: Specify minimum memory for new flavors + format: int64 + type: integer + generateProfile: + default: "" + description: Generate a sample profile.yaml file. + type: string + image: + default: "" + description: An image name/path/url to be uploaded to glance if + it’s not already there. + type: string + imageDiskFormat: + default: "" + description: A format of an image to be uploaded to glance. + type: string + insecure: + default: false + description: Explicitly allow client to perform “insecure” TLS + (https) requests. + type: boolean + networkID: + default: "" + description: Specify which network with external connectivity + should be used by the test. + type: string + noDefaultDeployer: + default: false + description: Do not check for the default deployer input in + type: boolean + nonAdmin: + default: false + description: Simulate non-admin credentials. + type: boolean + out: + default: "" + description: Output file + type: string + overrides: + default: "" + description: Override options + type: string + profile: + default: "" + description: python-tempestconf’s profile.yaml file + type: string + remove: + default: "" + description: Append values to tempest.conf + type: string + retryImage: + default: false + description: Allow tempestconf to retry download an image, in + case of failure. + type: boolean + testAccounts: + default: "" + description: Tempest accounts.yaml file + type: string + timeout: + default: -1 + description: Set request timeout (in seconds). + format: int64 + type: integer + verbose: + default: false + description: Print more information about the execution. + type: boolean + type: object required: - containerImage - openStackConfigMap diff --git a/api/v1beta1/tempest_types.go b/api/v1beta1/tempest_types.go index 86f5213c..3433e42f 100644 --- a/api/v1beta1/tempest_types.go +++ b/api/v1beta1/tempest_types.go @@ -46,14 +46,137 @@ type TempestRunSpec struct { // SkippedTests SkippedTests []string `json:"skippedTests,omitempty"` - // +kubebuilder:validation:Optional - // +kubebuilder:default:=0 - // Concurrency is the Default concurrency - Concurrency *int64 `json:"concurrency,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default:=-1 + // Concurrency is the Default concurrency + Concurrency int64 `json:"concurrency,omitempty"` + + // +kubebuilder:validation:Optional + // WorkerFile is the detailed concurry spec file + WorkerFile string `json:"workerFile,omitempty"` +} + +// TempestSpec PythonTempestconf parts +type TempestconfRunSpec struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=true + // Create Tempest resources + Create bool `json:"create"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Collect per-API call timing information. + CollectTiming bool `json:"collectTiming"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Explicitly allow client to perform “insecure” TLS (https) requests. + Insecure bool `json:"insecure"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Do not check for the default deployer input in + NoDefaultDeployer bool `json:"noDefaultDeployer"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Print debugging information. + Debug bool `json:"debug"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Print more information about the execution. + Verbose bool `json:"verbose"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Simulate non-admin credentials. + NonAdmin bool `json:"nonAdmin"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Allow tempestconf to retry download an image, in case of failure. + RetryImage bool `json:"retryImage"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Convert images to raw format before uploading. + ConvertToRaw bool `json:"convertToRaw"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Output file + Out string `json:"out"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Path to deployer file + DeployerInput string `json:"deployerInput"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Tempest accounts.yaml file + TestAccounts string `json:"testAccounts"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Generate Tempest accounts file. + CreateAccountsFile string `json:"createAccountsFile"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // python-tempestconf’s profile.yaml file + Profile string `json:"profile"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Generate a sample profile.yaml file. + GenerateProfile string `json:"generateProfile"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // A format of an image to be uploaded to glance. + ImageDiskFormat string `json:"imageDiskFormat"` - // +kubebuilder:validation:Optional - // WorkerFile is the detailed concurry spec file - WorkerFile string `json:"workerFile,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // An image name/path/url to be uploaded to glance if it’s not already there. + Image string `json:"image"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=-1 + // Specify minimum memory for new flavors + FlavorMinMem int64 `json:"flavorMinMem"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=-1 + // Specify minimum disk size for new flavors + FlavorMinDisk int64 `json:"flavorMinDisk"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Specify which network with external connectivity should be used by the test. + NetworkID string `json:"networkID"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Append values to tempest.conf + Append string `json:"append"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Append values to tempest.conf + Remove string `json:"remove"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // Override options + Overrides string `json:"overrides"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=-1 + // Set request timeout (in seconds). + Timeout int64 `json:"timeout"` } // TempestSpec defines the desired state of Tempest @@ -66,15 +189,15 @@ type TempestSpec struct { // NodeSelector to target subset of worker nodes running this service NodeSelector map[string]string `json:"nodeSelector,omitempty"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Required // +kubebuilder:default=openstack-config - // OpenStackConfigMap is the name of the ConfigMap containing the clouds.yaml - OpenStackConfigMap string `json:"openStackConfigMap"` + // OpenStackConfigMap is the name of the ConfigMap containing the clouds.yaml + OpenStackConfigMap string `json:"openStackConfigMap"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Required // +kubebuilder:default=openstack-config-secret - // OpenStackConfigSecret is the name of the Secret containing the secure.yaml - OpenStackConfigSecret string `json:"openStackConfigSecret"` + // OpenStackConfigSecret is the name of the Secret containing the secure.yaml + OpenStackConfigSecret string `json:"openStackConfigSecret"` // +kubebuilder:validation:Optional // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -90,12 +213,14 @@ type TempestSpec struct { BackoffLimit *int32 `json:"backoffLimit,omitempty"` // +kubebuilder:validation:Optional - TempestRun *TempestRunSpec `json:"tempestRun,omitempty"` + TempestRun *TempestRunSpec `json:"tempestRun,omitempty"` + + // +kubebuilder:validation:Optional + TempestconfRun *TempestconfRunSpec `json:"tempestconfRun,omitempty"` // TODO(slaweq): add more tempest run parameters here } - // MetalLBConfig to configure the MetalLB loadbalancer service type MetalLBConfig struct { // +kubebuilder:validation:Required diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 3f631537..18b58f36 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -133,11 +133,6 @@ func (in *TempestRunSpec) DeepCopyInto(out *TempestRunSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.Concurrency != nil { - in, out := &in.Concurrency, &out.Concurrency - *out = new(int64) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TempestRunSpec. @@ -182,6 +177,11 @@ func (in *TempestSpec) DeepCopyInto(out *TempestSpec) { *out = new(TempestRunSpec) (*in).DeepCopyInto(*out) } + if in.TempestconfRun != nil { + in, out := &in.TempestconfRun, &out.TempestconfRun + *out = new(TempestconfRunSpec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TempestSpec. @@ -237,3 +237,18 @@ func (in *TempestStatus) DeepCopy() *TempestStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TempestconfRunSpec) DeepCopyInto(out *TempestconfRunSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TempestconfRunSpec. +func (in *TempestconfRunSpec) DeepCopy() *TempestconfRunSpec { + if in == nil { + return nil + } + out := new(TempestconfRunSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/test.openstack.org_tempests.yaml b/config/crd/bases/test.openstack.org_tempests.yaml index 516f5582..01f71c16 100644 --- a/config/crd/bases/test.openstack.org_tempests.yaml +++ b/config/crd/bases/test.openstack.org_tempests.yaml @@ -120,7 +120,7 @@ spec: type: string type: array concurrency: - default: 0 + default: -1 description: Concurrency is the Default concurrency format: int64 type: integer @@ -133,6 +133,113 @@ spec: description: WorkerFile is the detailed concurry spec file type: string type: object + tempestconfRun: + description: TempestSpec PythonTempestconf parts + properties: + append: + default: "" + description: Append values to tempest.conf + type: string + collectTiming: + default: false + description: Collect per-API call timing information. + type: boolean + convertToRaw: + default: false + description: Convert images to raw format before uploading. + type: boolean + create: + default: true + description: Create Tempest resources + type: boolean + createAccountsFile: + default: "" + description: Generate Tempest accounts file. + type: string + debug: + default: false + description: Print debugging information. + type: boolean + deployerInput: + default: "" + description: Path to deployer file + type: string + flavorMinDisk: + default: -1 + description: Specify minimum disk size for new flavors + format: int64 + type: integer + flavorMinMem: + default: -1 + description: Specify minimum memory for new flavors + format: int64 + type: integer + generateProfile: + default: "" + description: Generate a sample profile.yaml file. + type: string + image: + default: "" + description: An image name/path/url to be uploaded to glance if + it’s not already there. + type: string + imageDiskFormat: + default: "" + description: A format of an image to be uploaded to glance. + type: string + insecure: + default: false + description: Explicitly allow client to perform “insecure” TLS + (https) requests. + type: boolean + networkID: + default: "" + description: Specify which network with external connectivity + should be used by the test. + type: string + noDefaultDeployer: + default: false + description: Do not check for the default deployer input in + type: boolean + nonAdmin: + default: false + description: Simulate non-admin credentials. + type: boolean + out: + default: "" + description: Output file + type: string + overrides: + default: "" + description: Override options + type: string + profile: + default: "" + description: python-tempestconf’s profile.yaml file + type: string + remove: + default: "" + description: Append values to tempest.conf + type: string + retryImage: + default: false + description: Allow tempestconf to retry download an image, in + case of failure. + type: boolean + testAccounts: + default: "" + description: Tempest accounts.yaml file + type: string + timeout: + default: -1 + description: Set request timeout (in seconds). + format: int64 + type: integer + verbose: + default: false + description: Print more information about the execution. + type: boolean + type: object required: - containerImage - openStackConfigMap diff --git a/config/samples/test_v1beta1_tempest.yaml b/config/samples/test_v1beta1_tempest.yaml index d2ee6e52..9e37e42f 100644 --- a/config/samples/test_v1beta1_tempest.yaml +++ b/config/samples/test_v1beta1_tempest.yaml @@ -1,3 +1,4 @@ +--- apiVersion: test.openstack.org/v1beta1 kind: Tempest metadata: @@ -9,3 +10,54 @@ spec: allowedTests: - tempest.api.identity.v3.* concurrency: 8 + pythonTempestconfRun: + # NOTE: All parameters have default values (use only when you want to override + # the default behaviour) + # create: true + # collectTiming: false + # insecure: false + # noDefaultDeployer: false + # debug: false + # verbose: false + # nonAdmin: false + # retryImage: false + # convertToRaw: false + # out: ./etc/tempest.conf + # flavorMinMem: 128 + # flavorMinDisk: 1 + # timeout: 600 + # imageDiskFormat: qcow2 + # image: https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-disk.img + # The following text will be mounted to the tempest pod + # as deployer_input.yaml + # deployerInput: | + # value1: exmaple_value2 + # value2: example_value2 + + # The following text will be mounted to the tempest pod + # as /etc/test_operator/deployer_input.yaml + # testAccounts: | + # - username: 'multi_role_user' + # tenant_name: 'test_tenant_42' + # password: 'test_password' + # roles: + # - 'fun_role' + # - 'not_an_admin' + # - 'an_admin' + + # The following text will be mounted to the tempest pod + # as /etc/test_operator/profile.yaml + # testAccounts: | + + # createAccountsFile: /path/to/accounts.yaml + # generateProfile: /path/to/profile.yaml + # networkID: + # append: | # <-- Use | to preserve \n + # section1.name1 value1 + # section1.name1 value2 + # remove: | # <-- Use | to preserve \n + # section1.name1 value1 + # section1.name1 value2 + # overrides: | # <-- Use | to preserve \n + # overrides_section1.name1 value1 + # overrides_section1.name1 value2 diff --git a/controllers/tempest_controller.go b/controllers/tempest_controller.go index e1b47367..965ea949 100644 --- a/controllers/tempest_controller.go +++ b/controllers/tempest_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "strconv" "time" "k8s.io/apimachinery/pkg/runtime" @@ -374,6 +375,22 @@ func (r *TempestReconciler) reconcileNormal(ctx context.Context, instance *testv return ctrl.Result{}, nil } +func getDefaultBool(variable bool) string { + if variable { + return "true" + } else { + return "false" + } +} + +func getDefaultInt(variable int64) string { + if variable != -1 { + return strconv.FormatInt(variable, 10) + } else { + return "" + } +} + // generateServiceConfigMaps - create create configmaps which hold scripts and service configuration // TODO add DefaultConfigOverwrite func (r *TempestReconciler) generateServiceConfigMaps( @@ -386,7 +403,9 @@ func (r *TempestReconciler) generateServiceConfigMaps( templateParameters := make(map[string]interface{}) customData := make(map[string]string) + envVars := make(map[string]string) + /* Tempest */ if len(instance.Spec.TempestRun.WorkerFile) != 0 { customData["worker_file.yaml"] = instance.Spec.TempestRun.WorkerFile } @@ -394,6 +413,69 @@ func (r *TempestReconciler) generateServiceConfigMaps( templateParameters["AllowedTests"] = instance.Spec.TempestRun.AllowedTests templateParameters["SkippedTests"] = instance.Spec.TempestRun.SkippedTests + /* tempestconf - start */ + // Files + testOperatorDir := "./etc/test_operator/" + if len(instance.Spec.TempestconfRun.DeployerInput) != 0 { + deployerInputFile := "deployer_input.yaml" + customData[deployerInputFile] = instance.Spec.TempestconfRun.DeployerInput + envVars["TEMPESTCONF_DEPLOYER_INPUT"] = testOperatorDir + deployerInputFile + } + + if len(instance.Spec.TempestconfRun.TestAccounts) != 0 { + accountsFile := "accounts.yaml" + customData[accountsFile] = instance.Spec.TempestconfRun.TestAccounts + envVars["TEMPESTCONF_TEST_ACCOUNTS"] = testOperatorDir + accountsFile + } + + if len(instance.Spec.TempestconfRun.Profile) != 0 { + profileFile := "profile.yaml" + customData[profileFile] = instance.Spec.TempestconfRun.Profile + envVars["TEMPESTCONF_PROFILE"] = testOperatorDir + profileFile + } + + // Bool + tempestconfBoolEnvVars := make(map[string]bool) + tempestconfBoolEnvVars = map[string]bool{ + "TEMPESTCONF_CREATE": instance.Spec.TempestconfRun.Create, + "TEMPESTCONF_COLLECT_TIMING": instance.Spec.TempestconfRun.CollectTiming, + "TEMPESTCONF_INSECURE": instance.Spec.TempestconfRun.Insecure, + "TEMPESTCONF_NO_DEFAULT_DEPLOYER": instance.Spec.TempestconfRun.NoDefaultDeployer, + "TEMPESTCONF_DEBUG": instance.Spec.TempestconfRun.Debug, + "TEMPESTCONF_VERBOSE": instance.Spec.TempestconfRun.Verbose, + "TEMPESTCONF_NON_ADMIN": instance.Spec.TempestconfRun.NonAdmin, + "TEMPESTCONF_RETRY_IMAGE": instance.Spec.TempestconfRun.RetryImage, + "TEMPESTCONF_CONVERT_TO_RAW": instance.Spec.TempestconfRun.ConvertToRaw, + } + + for key, value := range tempestconfBoolEnvVars { + envVars[key] = getDefaultBool(value) + } + + // Int + tempestconfIntEnvVars := make(map[string]int64) + tempestconfIntEnvVars = map[string]int64{ + "TEMPESTCONF_TIMEOUT": instance.Spec.TempestconfRun.Timeout, + "TEMPESTCONF_FLAVOR_MIN_MEM": instance.Spec.TempestconfRun.FlavorMinMem, + "TEMPESTCONF_FLAVOR_MIN_DISK": instance.Spec.TempestconfRun.FlavorMinDisk, + } + + for key, value := range tempestconfIntEnvVars { + envVars[key] = getDefaultInt(value) + } + + // String + envVars["TEMPESTCONF_OUT"] = instance.Spec.TempestconfRun.Out + envVars["TEMPESTCONF_CREATE_ACCOUNTS_FILE"] = instance.Spec.TempestconfRun.CreateAccountsFile + envVars["TEMPESTCONF_GENERATE_PROFILE"] = instance.Spec.TempestconfRun.GenerateProfile + envVars["TEMPESTCONF_IMAGE_DISK_FORMAT"] = instance.Spec.TempestconfRun.ImageDiskFormat + envVars["TEMPESTCONF_IMAGE"] = instance.Spec.TempestconfRun.Image + envVars["TEMPESTCONF_NETWORK_ID"] = instance.Spec.TempestconfRun.NetworkID + envVars["TEMPESTCONF_APPEND"] = instance.Spec.TempestconfRun.Append + envVars["TEMPESTCONF_REMOVE"] = instance.Spec.TempestconfRun.Remove + envVars["TEMPESTCONF_OVERRIDES"] = instance.Spec.TempestconfRun.Overrides + + /* Tempestconf - end */ cms := []util.Template{ // ScriptsConfigMap { @@ -413,6 +495,17 @@ func (r *TempestReconciler) generateServiceConfigMaps( ConfigOptions: templateParameters, CustomData: customData, }, + // configMap - EnvVars + { + Name: fmt.Sprintf("%s-env-vars", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + Labels: cmLabels, + ConfigOptions: templateParameters, + CustomData: envVars, + }, } + return configmap.EnsureConfigMaps(ctx, h, instance, cms, nil) } diff --git a/docs/source/samples/tempest-config.yaml b/docs/source/samples/tempest-config.yaml index 027c3e04..810f8854 100644 --- a/docs/source/samples/tempest-config.yaml +++ b/docs/source/samples/tempest-config.yaml @@ -5,3 +5,37 @@ metadata: data: include.txt: | tempest.api.identity.v3 + + # NOTE: All parameters have default values (use only when you want to override + # the default behaviour) + # TEMPESTCONF_CREATE: "true" + # TEMPESTCONF_INSECURE: "false" + # TEMPESTCONF_COLLECT_TIMING: "false" + # TEMPESTCONF_NO_DEFAULT_DEPLOYER: "false" + # TEMPESTCONF_DEBUG: "false" + # TEMPESTCONF_VERBOSE: "false" + # TEMPESTCONF_NO_RNG: "false" + # TEMPESTCONF_NON_ADMIN: "false" + # TEMPESTCONF_RETRY_IMAGE: "false" + # TEMPESTCONF_CONVERT_TO_RAW: "false" + # TEMPESTCONF_TIMEOUT: "600" + # TEMPESTCONF_OUT: "./etc/tempest.conf" + # TEMPESTCONF_DEPLOYER_INPUT: "/etc/test_operator/deployer_input.yaml" + # TEMPESTCONF_TEST_ACCOUNTS: "/etc/test_operator/accounts.yaml" + # TEMPESTCONF_CREATE_ACCOUNTS_FILE: "/etc/test_operator/accounts.yaml" + # TEMPESTCONF_PROFILE: "/etc/test_operator/profile.yaml" + # TEMPESTCONF_GENERATE_PROFILE: "/etc/test_operator/profile.yaml" + # TEMPESTCONF_IMAGE_DISK_FORMAT: "qcow2" + # TEMPESTCONF_IMAGE: "https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-disk.img" + # TEMPESTCONF_FLAVOR_MIN_MEM: "128" + # TEMPESTCONF_FLAVOR_MIN_DISK: "1" + # TEMPESTCONF_NETWORK_ID: "" + # TEMPESTCONF_APPEND: | + # section.value1 value1 + # section.value1 value2 + # TEMPESTCONF_REMOVE: | + # section.value1 value1 + # section.value1 value2 + # TEMPESTCONF_OVERRIDES: | + # section.value1 value1 + # section.value1 value2 diff --git a/docs/source/samples/tempest-deployment.yaml b/docs/source/samples/tempest-deployment.yaml index ac12f530..c51e247f 100644 --- a/docs/source/samples/tempest-deployment.yaml +++ b/docs/source/samples/tempest-deployment.yaml @@ -41,6 +41,9 @@ spec: # configuration and run tempest yourself. # command: ["/usr/bin/dumb-init", "sleep", "infinity"] restartPolicy: Never + envFrom: + - configMapRef: + name: my-tempest-data volumeMounts: - mountPath: "/var/lib/tempest/external_files/" name: tempest-workdir diff --git a/pkg/tempest/job.go b/pkg/tempest/job.go index a5ecf5de..cc3c0a3c 100644 --- a/pkg/tempest/job.go +++ b/pkg/tempest/job.go @@ -7,7 +7,6 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "strconv" ) // Job - prepare job to run Tempest tests @@ -19,11 +18,6 @@ func Job( envVars := map[string]env.Setter{} runAsUser := int64(42480) runAsGroup := int64(42480) - if instance.Spec.TempestRun.Concurrency != nil { - envVars["TEMPEST_CONCURRENCY"] = env.SetValue(strconv.FormatInt(*instance.Spec.TempestRun.Concurrency, 10)) - } else { - envVars["TEMPEST_CONCURRENCY"] = env.SetValue("0") - } job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -44,14 +38,27 @@ func Job( }, Containers: []corev1.Container{ { - Name: instance.Name + "-tests-runner", - Image: instance.Spec.ContainerImage, - Command: []string{ - "/usr/local/bin/container-scripts/invoke_tempest", - }, + Name: instance.Name + "-tests-runner", + Image: instance.Spec.ContainerImage, Args: []string{}, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: GetVolumeMounts(), + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: instance.Name + "-config-data", + }, + }, + }, + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: instance.Name + "-env-vars", + }, + }, + }, + }, }, }, Volumes: GetVolumes(instance), From abfc40cb227cddb6917fb735c37add9c911d57f6 Mon Sep 17 00:00:00 2001 From: Lukas Piwowarski Date: Thu, 26 Oct 2023 17:43:39 +0300 Subject: [PATCH 2/2] Add tempest options This patch introduces a set of parameters for the tempestRun section in the Tempest CR. This field parameters allow a user to specify parameters with which tempest command should be executed inside the Tempest pod. The documentation is updated as well so that it demonstrates how the new field should be utilized. --- api/bases/test.openstack.org_tempests.yaml | 37 ++-- api/v1beta1/tempest_types.go | 31 +++- api/v1beta1/zz_generated.deepcopy.go | 12 +- .../bases/test.openstack.org_tempests.yaml | 37 ++-- config/samples/test_v1beta1_tempest.yaml | 19 ++- controllers/tempest_controller.go | 158 ++++++++++++------ docs/source/samples/tempest-config.yaml | 22 +++ docs/source/samples/tempest-deployment.yaml | 6 +- 8 files changed, 216 insertions(+), 106 deletions(-) diff --git a/api/bases/test.openstack.org_tempests.yaml b/api/bases/test.openstack.org_tempests.yaml index 01f71c16..191dddb5 100644 --- a/api/bases/test.openstack.org_tempests.yaml +++ b/api/bases/test.openstack.org_tempests.yaml @@ -112,25 +112,34 @@ spec: tempestRun: description: TempestSpec TempestRun parts properties: - allowedTests: - default: - - tempest.api.identity.v3 - description: AllowedTests - items: - type: string - type: array concurrency: default: -1 description: Concurrency is the Default concurrency format: int64 type: integer - skippedTests: - description: SkippedTests - items: - type: string - type: array + excludeList: + default: "" + description: ExcludeList + type: string + includeList: + default: tempest.api.identity.v3 + description: IncludeList + type: string + parallel: + default: true + description: Run tests in parallel + type: boolean + serial: + default: false + description: Serial run + type: boolean + smoke: + default: false + description: Smoke tests + type: boolean workerFile: - description: WorkerFile is the detailed concurry spec file + default: "" + description: WorkerFile is the detailed concurrency spec file type: string type: object tempestconfRun: @@ -210,7 +219,7 @@ spec: description: Output file type: string overrides: - default: "" + default: identity.v3_endpoint_type public description: Override options type: string profile: diff --git a/api/v1beta1/tempest_types.go b/api/v1beta1/tempest_types.go index 3433e42f..e8bbcee7 100644 --- a/api/v1beta1/tempest_types.go +++ b/api/v1beta1/tempest_types.go @@ -38,13 +38,14 @@ type Hash struct { // TempestSpec TempestRun parts type TempestRunSpec struct { // +kubebuilder:validation:Optional - // +kubebuilder:default={"tempest.api.identity.v3"} - // AllowedTests - AllowedTests []string `json:"allowedTests,omitempty"` + // +kubebuilder:default="tempest.api.identity.v3" + // IncludeList + IncludeList string `json:"includeList,omitempty"` // +kubebuilder:validation:Optional - // SkippedTests - SkippedTests []string `json:"skippedTests,omitempty"` + // +kubebuilder:default="" + // ExcludeList + ExcludeList string `json:"excludeList,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default:=-1 @@ -52,7 +53,23 @@ type TempestRunSpec struct { Concurrency int64 `json:"concurrency,omitempty"` // +kubebuilder:validation:Optional - // WorkerFile is the detailed concurry spec file + // +kubebuilder:default:=false + // Smoke tests + Smoke bool `json:"smoke,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default:=true + // Run tests in parallel + Parallel bool `json:"parallel,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default:=false + // Serial run + Serial bool `json:"serial,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default:="" + // WorkerFile is the detailed concurrency spec file WorkerFile string `json:"workerFile,omitempty"` } @@ -169,7 +186,7 @@ type TempestconfRunSpec struct { Remove string `json:"remove"` // +kubebuilder:validation:Optional - // +kubebuilder:default="" + // +kubebuilder:default="identity.v3_endpoint_type public" // Override options Overrides string `json:"overrides"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 18b58f36..ad5c474a 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -123,16 +123,6 @@ func (in *TempestList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TempestRunSpec) DeepCopyInto(out *TempestRunSpec) { *out = *in - if in.AllowedTests != nil { - in, out := &in.AllowedTests, &out.AllowedTests - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.SkippedTests != nil { - in, out := &in.SkippedTests, &out.SkippedTests - *out = make([]string, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TempestRunSpec. @@ -175,7 +165,7 @@ func (in *TempestSpec) DeepCopyInto(out *TempestSpec) { if in.TempestRun != nil { in, out := &in.TempestRun, &out.TempestRun *out = new(TempestRunSpec) - (*in).DeepCopyInto(*out) + **out = **in } if in.TempestconfRun != nil { in, out := &in.TempestconfRun, &out.TempestconfRun diff --git a/config/crd/bases/test.openstack.org_tempests.yaml b/config/crd/bases/test.openstack.org_tempests.yaml index 01f71c16..191dddb5 100644 --- a/config/crd/bases/test.openstack.org_tempests.yaml +++ b/config/crd/bases/test.openstack.org_tempests.yaml @@ -112,25 +112,34 @@ spec: tempestRun: description: TempestSpec TempestRun parts properties: - allowedTests: - default: - - tempest.api.identity.v3 - description: AllowedTests - items: - type: string - type: array concurrency: default: -1 description: Concurrency is the Default concurrency format: int64 type: integer - skippedTests: - description: SkippedTests - items: - type: string - type: array + excludeList: + default: "" + description: ExcludeList + type: string + includeList: + default: tempest.api.identity.v3 + description: IncludeList + type: string + parallel: + default: true + description: Run tests in parallel + type: boolean + serial: + default: false + description: Serial run + type: boolean + smoke: + default: false + description: Smoke tests + type: boolean workerFile: - description: WorkerFile is the detailed concurry spec file + default: "" + description: WorkerFile is the detailed concurrency spec file type: string type: object tempestconfRun: @@ -210,7 +219,7 @@ spec: description: Output file type: string overrides: - default: "" + default: identity.v3_endpoint_type public description: Override options type: string profile: diff --git a/config/samples/test_v1beta1_tempest.yaml b/config/samples/test_v1beta1_tempest.yaml index 9e37e42f..de773703 100644 --- a/config/samples/test_v1beta1_tempest.yaml +++ b/config/samples/test_v1beta1_tempest.yaml @@ -7,10 +7,23 @@ metadata: spec: containerImage: quay.io/podified-antelope-centos9/openstack-tempest:current-podified tempestRun: - allowedTests: - - tempest.api.identity.v3.* + # NOTE: All parameters have default values (use only when you want to override + # the default behaviour) + includeList: | # <-- Use | to preserve \n + tempest.api.identity.v3.* concurrency: 8 - pythonTempestconfRun: + # excludeList: + # - tempest.api.identity.v3.* + # workerFile: | # <-- use | to preserver \n + # - worker: + # - tempest.api.* + # - neutron_tempest_tests + # - worker: + # - tempest.scenario.* + # smoke: false + # serial: false + # parallel: true + tempestconfRun: # NOTE: All parameters have default values (use only when you want to override # the default behaviour) # create: true diff --git a/controllers/tempest_controller.go b/controllers/tempest_controller.go index 965ea949..82bff502 100644 --- a/controllers/tempest_controller.go +++ b/controllers/tempest_controller.go @@ -22,12 +22,6 @@ import ( "strconv" "time" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "github.com/go-logr/logr" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" @@ -44,6 +38,11 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) // TempestReconciler reconciles a Tempest object @@ -391,61 +390,96 @@ func getDefaultInt(variable int64) string { } } -// generateServiceConfigMaps - create create configmaps which hold scripts and service configuration -// TODO add DefaultConfigOverwrite -func (r *TempestReconciler) generateServiceConfigMaps( - ctx context.Context, - h *helper.Helper, - instance *testv1beta1.Tempest, -) error { - // Create/update configmaps from templates - cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(tempest.ServiceName), map[string]string{}) +func setTempestConfigVars(envVars map[string]string, + customData map[string]string, + tempestRun *testv1beta1.TempestRunSpec) { + + testOperatorDir := "/etc/test_operator/" + if tempestRun == nil { + includeListFile := "include.txt" + customData[includeListFile] = "tempest.api.identity.v3" + envVars["TEMPEST_INCLUDE_LIST"] = testOperatorDir + includeListFile + envVars["TEMPEST_PARALLEL"] = "true" + return + } - templateParameters := make(map[string]interface{}) - customData := make(map[string]string) - envVars := make(map[string]string) + // Files + if len(tempestRun.WorkerFile) != 0 { + workerFile := "worker_file.yaml" + customData[workerFile] = tempestRun.WorkerFile + envVars["TEMPEST_WORKER_FILE"] = testOperatorDir + workerFile + } - /* Tempest */ - if len(instance.Spec.TempestRun.WorkerFile) != 0 { - customData["worker_file.yaml"] = instance.Spec.TempestRun.WorkerFile + if len(tempestRun.IncludeList) != 0 { + includeListFile := "include.txt" + customData[includeListFile] = tempestRun.IncludeList + envVars["TEMPEST_INCLUDE_LIST"] = testOperatorDir + includeListFile } - templateParameters["AllowedTests"] = instance.Spec.TempestRun.AllowedTests - templateParameters["SkippedTests"] = instance.Spec.TempestRun.SkippedTests + if len(tempestRun.ExcludeList) != 0 { + excludeListFile := "exclude.txt" + customData[excludeListFile] = tempestRun.ExcludeList + envVars["TEMPEST_EXCLUDE_LIST"] = testOperatorDir + excludeListFile + } + + // Bool + tempestBoolEnvVars := make(map[string]bool) + tempestBoolEnvVars = map[string]bool{ + "TEMPEST_SERIAL": tempestRun.Serial, + "TEMPEST_PARALLEL": tempestRun.Parallel, + "TEMPEST_SMOKE": tempestRun.Smoke, + } + + for key, value := range tempestBoolEnvVars { + envVars[key] = getDefaultBool(value) + } + + // Int + envVars["TEMPEST_CONCURRENCY"] = getDefaultInt(tempestRun.Concurrency) +} + +func setTempestconfConfigVars(envVars map[string]string, + customData map[string]string, + tempestconfRun *testv1beta1.TempestconfRunSpec) { + + if tempestconfRun == nil { + envVars["TEMPESTCONF_CREATE"] = "true" + envVars["TEMPESTCONF_OVERRIDES"] = "identity.v3_endpoint_type public" + return + } - /* tempestconf - start */ // Files - testOperatorDir := "./etc/test_operator/" - if len(instance.Spec.TempestconfRun.DeployerInput) != 0 { + testOperatorDir := "/etc/test_operator/" + if len(tempestconfRun.DeployerInput) != 0 { deployerInputFile := "deployer_input.yaml" - customData[deployerInputFile] = instance.Spec.TempestconfRun.DeployerInput + customData[deployerInputFile] = tempestconfRun.DeployerInput envVars["TEMPESTCONF_DEPLOYER_INPUT"] = testOperatorDir + deployerInputFile } - if len(instance.Spec.TempestconfRun.TestAccounts) != 0 { + if len(tempestconfRun.TestAccounts) != 0 { accountsFile := "accounts.yaml" - customData[accountsFile] = instance.Spec.TempestconfRun.TestAccounts + customData[accountsFile] = tempestconfRun.TestAccounts envVars["TEMPESTCONF_TEST_ACCOUNTS"] = testOperatorDir + accountsFile } - if len(instance.Spec.TempestconfRun.Profile) != 0 { + if len(tempestconfRun.Profile) != 0 { profileFile := "profile.yaml" - customData[profileFile] = instance.Spec.TempestconfRun.Profile + customData[profileFile] = tempestconfRun.Profile envVars["TEMPESTCONF_PROFILE"] = testOperatorDir + profileFile } // Bool tempestconfBoolEnvVars := make(map[string]bool) tempestconfBoolEnvVars = map[string]bool{ - "TEMPESTCONF_CREATE": instance.Spec.TempestconfRun.Create, - "TEMPESTCONF_COLLECT_TIMING": instance.Spec.TempestconfRun.CollectTiming, - "TEMPESTCONF_INSECURE": instance.Spec.TempestconfRun.Insecure, - "TEMPESTCONF_NO_DEFAULT_DEPLOYER": instance.Spec.TempestconfRun.NoDefaultDeployer, - "TEMPESTCONF_DEBUG": instance.Spec.TempestconfRun.Debug, - "TEMPESTCONF_VERBOSE": instance.Spec.TempestconfRun.Verbose, - "TEMPESTCONF_NON_ADMIN": instance.Spec.TempestconfRun.NonAdmin, - "TEMPESTCONF_RETRY_IMAGE": instance.Spec.TempestconfRun.RetryImage, - "TEMPESTCONF_CONVERT_TO_RAW": instance.Spec.TempestconfRun.ConvertToRaw, + "TEMPESTCONF_CREATE": tempestconfRun.Create, + "TEMPESTCONF_COLLECT_TIMING": tempestconfRun.CollectTiming, + "TEMPESTCONF_INSECURE": tempestconfRun.Insecure, + "TEMPESTCONF_NO_DEFAULT_DEPLOYER": tempestconfRun.NoDefaultDeployer, + "TEMPESTCONF_DEBUG": tempestconfRun.Debug, + "TEMPESTCONF_VERBOSE": tempestconfRun.Verbose, + "TEMPESTCONF_NON_ADMIN": tempestconfRun.NonAdmin, + "TEMPESTCONF_RETRY_IMAGE": tempestconfRun.RetryImage, + "TEMPESTCONF_CONVERT_TO_RAW": tempestconfRun.ConvertToRaw, } for key, value := range tempestconfBoolEnvVars { @@ -455,9 +489,9 @@ func (r *TempestReconciler) generateServiceConfigMaps( // Int tempestconfIntEnvVars := make(map[string]int64) tempestconfIntEnvVars = map[string]int64{ - "TEMPESTCONF_TIMEOUT": instance.Spec.TempestconfRun.Timeout, - "TEMPESTCONF_FLAVOR_MIN_MEM": instance.Spec.TempestconfRun.FlavorMinMem, - "TEMPESTCONF_FLAVOR_MIN_DISK": instance.Spec.TempestconfRun.FlavorMinDisk, + "TEMPESTCONF_TIMEOUT": tempestconfRun.Timeout, + "TEMPESTCONF_FLAVOR_MIN_MEM": tempestconfRun.FlavorMinMem, + "TEMPESTCONF_FLAVOR_MIN_DISK": tempestconfRun.FlavorMinDisk, } for key, value := range tempestconfIntEnvVars { @@ -465,15 +499,33 @@ func (r *TempestReconciler) generateServiceConfigMaps( } // String - envVars["TEMPESTCONF_OUT"] = instance.Spec.TempestconfRun.Out - envVars["TEMPESTCONF_CREATE_ACCOUNTS_FILE"] = instance.Spec.TempestconfRun.CreateAccountsFile - envVars["TEMPESTCONF_GENERATE_PROFILE"] = instance.Spec.TempestconfRun.GenerateProfile - envVars["TEMPESTCONF_IMAGE_DISK_FORMAT"] = instance.Spec.TempestconfRun.ImageDiskFormat - envVars["TEMPESTCONF_IMAGE"] = instance.Spec.TempestconfRun.Image - envVars["TEMPESTCONF_NETWORK_ID"] = instance.Spec.TempestconfRun.NetworkID - envVars["TEMPESTCONF_APPEND"] = instance.Spec.TempestconfRun.Append - envVars["TEMPESTCONF_REMOVE"] = instance.Spec.TempestconfRun.Remove - envVars["TEMPESTCONF_OVERRIDES"] = instance.Spec.TempestconfRun.Overrides + envVars["TEMPESTCONF_OUT"] = tempestconfRun.Out + envVars["TEMPESTCONF_CREATE_ACCOUNTS_FILE"] = tempestconfRun.CreateAccountsFile + envVars["TEMPESTCONF_GENERATE_PROFILE"] = tempestconfRun.GenerateProfile + envVars["TEMPESTCONF_IMAGE_DISK_FORMAT"] = tempestconfRun.ImageDiskFormat + envVars["TEMPESTCONF_IMAGE"] = tempestconfRun.Image + envVars["TEMPESTCONF_NETWORK_ID"] = tempestconfRun.NetworkID + envVars["TEMPESTCONF_APPEND"] = tempestconfRun.Append + envVars["TEMPESTCONF_REMOVE"] = tempestconfRun.Remove + envVars["TEMPESTCONF_OVERRIDES"] = tempestconfRun.Overrides +} + +// generateServiceConfigMaps - create create configmaps which hold scripts and service configuration +// TODO add DefaultConfigOverwrite +func (r *TempestReconciler) generateServiceConfigMaps( + ctx context.Context, + h *helper.Helper, + instance *testv1beta1.Tempest, +) error { + // Create/update configmaps from template + cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(tempest.ServiceName), map[string]string{}) + + templateParameters := make(map[string]interface{}) + customData := make(map[string]string) + envVars := make(map[string]string) + + setTempestConfigVars(envVars, customData, instance.Spec.TempestRun) + setTempestconfConfigVars(envVars, customData, instance.Spec.TempestconfRun) /* Tempestconf - end */ cms := []util.Template{ @@ -489,7 +541,6 @@ func (r *TempestReconciler) generateServiceConfigMaps( { Name: fmt.Sprintf("%s-config-data", instance.Name), Namespace: instance.Namespace, - Type: util.TemplateTypeConfig, InstanceType: instance.Kind, Labels: cmLabels, ConfigOptions: templateParameters, @@ -499,7 +550,6 @@ func (r *TempestReconciler) generateServiceConfigMaps( { Name: fmt.Sprintf("%s-env-vars", instance.Name), Namespace: instance.Namespace, - Type: util.TemplateTypeConfig, InstanceType: instance.Kind, Labels: cmLabels, ConfigOptions: templateParameters, diff --git a/docs/source/samples/tempest-config.yaml b/docs/source/samples/tempest-config.yaml index 810f8854..6a46d373 100644 --- a/docs/source/samples/tempest-config.yaml +++ b/docs/source/samples/tempest-config.yaml @@ -5,7 +5,29 @@ metadata: data: include.txt: | tempest.api.identity.v3 + # exclude.txt: | + # tempest.api.identity.v3 + # worker_file.yaml: | + # - worker: + # - tempest.api.* + # - neutron_tempest_tests + # - worker: + # - tempest.scenario.* + # TEMPEST env variables: + # ---------------------- + # NOTE: All parameters have default values (use only when you want to override + # the default behaviour) + TEMPEST_INCLUDE_LIST: "/var/lib/tempest/include.txt" + # TEMPEST_EXCLUDE_LIST: "/var/lib/tempest/exclude.txt" + # TEMPEST_WORKER_FILE: "/var/lib/tempest/worker_file.yaml" + # TEMPEST_CONCURRENCY: 8 + # TEMPEST_SMOKE: true + # TEMPEST_PARALLEL: true + # TEMPEST_SERIAL: true + + # TEMPESTCONF env variables: + # -------------------------- # NOTE: All parameters have default values (use only when you want to override # the default behaviour) # TEMPESTCONF_CREATE: "true" diff --git a/docs/source/samples/tempest-deployment.yaml b/docs/source/samples/tempest-deployment.yaml index c51e247f..81cbea45 100644 --- a/docs/source/samples/tempest-deployment.yaml +++ b/docs/source/samples/tempest-deployment.yaml @@ -47,14 +47,14 @@ spec: volumeMounts: - mountPath: "/var/lib/tempest/external_files/" name: tempest-workdir + - mountPath: "/var/lib/tempest/include.txt" + name: tempest-config + subPath: include.txt - mountPath: "/etc/openstack" name: pre-install - mountPath: "/etc/openstack/clouds.yaml" name: clouds-config subPath: clouds.yaml - - mountPath: "/var/lib/tempest/external_files/include.txt" - name: tempest-config - subPath: include.txt - mountPath: "/etc/openstack/secure.yaml" name: cloud-passwd subPath: secure.yaml