diff --git a/src/api/v1alpha1/zz_generated.deepcopy.go b/src/api/v1alpha1/zz_generated.deepcopy.go index e769192edc..6667147b6d 100644 --- a/src/api/v1alpha1/zz_generated.deepcopy.go +++ b/src/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/src/api/v1beta1/feature_flags.go b/src/api/v1beta1/feature_flags.go index 74e3a69ebe..2cbe5fff7d 100644 --- a/src/api/v1beta1/feature_flags.go +++ b/src/api/v1beta1/feature_flags.go @@ -24,13 +24,14 @@ import ( ) const ( - annotationFeaturePrefix = "alpha.operator.dynatrace.com/feature-" - annotationFeatureDisableActiveGateUpdates = annotationFeaturePrefix + "disable-activegate-updates" - annotationFeatureDisableHostsRequests = annotationFeaturePrefix + "disable-hosts-requests" - annotationFeatureOneAgentMaxUnavailable = annotationFeaturePrefix + "oneagent-max-unavailable" - annotationFeatureEnableWebhookReinvocationPolicy = annotationFeaturePrefix + "enable-webhook-reinvocation-policy" - annotationFeatureIgnoreUnknownState = annotationFeaturePrefix + "ignore-unknown-state" - annotationFeatureIgnoredNamespaces = annotationFeaturePrefix + "ignored-namespaces" + annotationFeaturePrefix = "alpha.operator.dynatrace.com/feature-" + annotationFeatureDisableActiveGateUpdates = annotationFeaturePrefix + "disable-activegate-updates" + annotationFeatureDisableHostsRequests = annotationFeaturePrefix + "disable-hosts-requests" + annotationFeatureOneAgentMaxUnavailable = annotationFeaturePrefix + "oneagent-max-unavailable" + annotationFeatureEnableWebhookReinvocationPolicy = annotationFeaturePrefix + "enable-webhook-reinvocation-policy" + annotationFeatureIgnoreUnknownState = annotationFeaturePrefix + "ignore-unknown-state" + annotationFeatureIgnoredNamespaces = annotationFeaturePrefix + "ignored-namespaces" + annotationFeatureAutomaticKubernetesApiMonitoring = annotationFeaturePrefix + "automatic-kubernetes-api-monitoring" ) var ( @@ -100,3 +101,9 @@ func (dk *DynaKube) FeatureIgnoredNamespaces() []string { } return *ignoredNamespaces } + +// FeatureAutomaticKubernetesApiMonitoring is a feature flag to enable automatic kubernetes api monitoring, +// which ensures that settings for this kubernetes cluster exist in Dynatrace +func (dk *DynaKube) FeatureAutomaticKubernetesApiMonitoring() bool { + return dk.Annotations[annotationFeatureAutomaticKubernetesApiMonitoring] == "true" +} diff --git a/src/api/v1beta1/properties.go b/src/api/v1beta1/properties.go index 1e91b37e11..2864bf085e 100644 --- a/src/api/v1beta1/properties.go +++ b/src/api/v1beta1/properties.go @@ -77,6 +77,10 @@ func (dk *DynaKube) IsActiveGateMode(mode string) bool { return false } +func (dk *DynaKube) KubernetesMonitoringMode() bool { + return dk.IsActiveGateMode(string(KubeMonCapability.DisplayName)) || dk.Spec.KubernetesMonitoring.Enabled +} + // ShouldAutoUpdateOneAgent returns true if the Operator should update OneAgent instances automatically. func (dk *DynaKube) ShouldAutoUpdateOneAgent() bool { if dk.CloudNativeFullstackMode() { diff --git a/src/api/v1beta1/zz_generated.deepcopy.go b/src/api/v1beta1/zz_generated.deepcopy.go index f3fcb7068b..c95b7bd89d 100644 --- a/src/api/v1beta1/zz_generated.deepcopy.go +++ b/src/api/v1beta1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/src/controllers/activegate/reconciler/automaticapimonitoring/config.go b/src/controllers/activegate/reconciler/automaticapimonitoring/config.go new file mode 100644 index 0000000000..18f9f1611f --- /dev/null +++ b/src/controllers/activegate/reconciler/automaticapimonitoring/config.go @@ -0,0 +1,9 @@ +package automaticapimonitoring + +import ( + "github.com/Dynatrace/dynatrace-operator/src/logger" +) + +var ( + log = logger.NewDTLogger().WithName("automatic-api-monitoring") +) diff --git a/src/controllers/activegate/reconciler/automaticapimonitoring/reconciler.go b/src/controllers/activegate/reconciler/automaticapimonitoring/reconciler.go new file mode 100644 index 0000000000..656fa5e8f5 --- /dev/null +++ b/src/controllers/activegate/reconciler/automaticapimonitoring/reconciler.go @@ -0,0 +1,86 @@ +package automaticapimonitoring + +import ( + "fmt" + + "github.com/Dynatrace/dynatrace-operator/src/dtclient" + "github.com/pkg/errors" +) + +type AutomaticApiMonitoringReconciler struct { + dtc dtclient.Client + name string + kubeSystemUUID string +} + +func NewReconciler(dtc dtclient.Client, name, kubeSystemUUID string) *AutomaticApiMonitoringReconciler { + return &AutomaticApiMonitoringReconciler{ + dtc, + name, + kubeSystemUUID, + } +} + +func (r *AutomaticApiMonitoringReconciler) Reconcile() error { + objectID, err := r.ensureSettingExists() + + if err != nil { + return err + } + + if objectID != "" { + log.Info("created kubernetes cluster setting", "name", r.name, "cluster", r.kubeSystemUUID, "object id", objectID) + } else { + log.Info("kubernetes cluster setting already exists", "name", r.name, "cluster", r.kubeSystemUUID) + } + + return nil +} + +func (r *AutomaticApiMonitoringReconciler) ensureSettingExists() (string, error) { + if r.kubeSystemUUID == "" { + return "", errors.New("no kube-system namespace UUID given") + } + + // check if ME with UID exists + var monitoredEntities, err = r.dtc.GetMonitoredEntitiesForKubeSystemUUID(r.kubeSystemUUID) + if err != nil { + return "", fmt.Errorf("error while loading MEs: %s", err.Error()) + } + + // check if Setting for ME exists + settings, err := r.dtc.GetSettingsForMonitoredEntities(monitoredEntities) + if err != nil { + return "", fmt.Errorf("error trying to check if setting exists %s", err.Error()) + } + + if settings.TotalCount > 0 { + return "", nil + } + + // determine newest ME (can be empty string), and create or update a settings object accordingly + meID := determineNewestMonitoredEntity(monitoredEntities) + objectID, err := r.dtc.CreateOrUpdateKubernetesSetting(r.name, r.kubeSystemUUID, meID) + + if err != nil { + return "", err + } + + return objectID, nil +} + +// determineNewestMonitoredEntity returns the ID of the newest entities; or empty string if the slice of entities is empty +func determineNewestMonitoredEntity(entities []dtclient.MonitoredEntity) string { + if len(entities) == 0 { + return "" + } + + var newestMe dtclient.MonitoredEntity + for _, entity := range entities { + if entity.LastSeenTms > newestMe.LastSeenTms { + newestMe = entity + } + } + + return newestMe.EntityId +} diff --git a/src/controllers/activegate/reconciler/automaticapimonitoring/reconciler_test.go b/src/controllers/activegate/reconciler/automaticapimonitoring/reconciler_test.go new file mode 100644 index 0000000000..6bbdf5d528 --- /dev/null +++ b/src/controllers/activegate/reconciler/automaticapimonitoring/reconciler_test.go @@ -0,0 +1,186 @@ +package automaticapimonitoring + +import ( + "testing" + + "github.com/Dynatrace/dynatrace-operator/src/dtclient" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +const ( + testUID = "test-uid" + testName = "test-name" + testObjectID = "test-objectid" +) + +func TestNewDefaultReconiler(t *testing.T) { + createDefaultReconciler(t) +} + +func createDefaultReconciler(t *testing.T) *AutomaticApiMonitoringReconciler { + return createReconciler(t, testUID, []dtclient.MonitoredEntity{}, dtclient.GetSettingsResponse{TotalCount: 0}, "") +} + +func createReconciler(t *testing.T, uid string, monitoredEntities []dtclient.MonitoredEntity, getSettingsResponse dtclient.GetSettingsResponse, objectID string) *AutomaticApiMonitoringReconciler { + mockClient := &dtclient.MockDynatraceClient{} + mockClient.On("GetMonitoredEntitiesForKubeSystemUUID", mock.AnythingOfType("string")). + Return(monitoredEntities, nil) + mockClient.On("GetSettingsForMonitoredEntities", monitoredEntities). + Return(getSettingsResponse, nil) + mockClient.On("CreateOrUpdateKubernetesSetting", testName, testUID, mock.AnythingOfType("string")). + Return(objectID, nil) + + r := NewReconciler(mockClient, testName, uid) + require.NotNil(t, r) + require.NotNil(t, r.dtc) + + return r +} + +func createReconcilerWithError(t *testing.T, monitoredEntitiesError error, getSettingsResponseError error, createSettingsResponseError error) *AutomaticApiMonitoringReconciler { + mockClient := &dtclient.MockDynatraceClient{} + mockClient.On("GetMonitoredEntitiesForKubeSystemUUID", mock.AnythingOfType("string")). + Return([]dtclient.MonitoredEntity{}, monitoredEntitiesError) + mockClient.On("GetSettingsForMonitoredEntities", []dtclient.MonitoredEntity{}). + Return(dtclient.GetSettingsResponse{}, getSettingsResponseError) + mockClient.On("CreateOrUpdateKubernetesSetting", testName, testUID, mock.AnythingOfType("string")). + Return("", createSettingsResponseError) + + r := NewReconciler(mockClient, testName, testUID) + require.NotNil(t, r) + require.NotNil(t, r.dtc) + + return r +} + +func createMonitoredEntities() []dtclient.MonitoredEntity { + return []dtclient.MonitoredEntity{ + {EntityId: "KUBERNETES_CLUSTER-0E30FE4BF2007587", DisplayName: "operator test entity 1", LastSeenTms: 1639483869085}, + {EntityId: "KUBERNETES_CLUSTER-119C75CCDA94799F", DisplayName: "operator test entity 2", LastSeenTms: 1639034988126}, + } +} + +func TestReconcile(t *testing.T) { + t.Run(`reconciler does not fail in with defaults`, func(t *testing.T) { + // arrange + r := createDefaultReconciler(t) + + // act + err := r.Reconcile() + + // assert + assert.NoError(t, err) + }) + + t.Run(`create setting when no monitored entities are existing`, func(t *testing.T) { + // arrange + r := createReconciler(t, testUID, []dtclient.MonitoredEntity{}, dtclient.GetSettingsResponse{}, testObjectID) + + // act + actual, err := r.ensureSettingExists() + + // assert + assert.NoError(t, err) + assert.Equal(t, testObjectID, actual) + }) + + t.Run(`create setting when no settings for the found monitored entities are existing`, func(t *testing.T) { + // arrange + entities := createMonitoredEntities() + r := createReconciler(t, testUID, entities, dtclient.GetSettingsResponse{}, testObjectID) + + // act + actual, err := r.ensureSettingExists() + + // assert + assert.NoError(t, err) + assert.Equal(t, testObjectID, actual) + }) + + t.Run(`don't create setting when settings for the found monitored entities are existing`, func(t *testing.T) { + // arrange + entities := createMonitoredEntities() + r := createReconciler(t, testUID, entities, dtclient.GetSettingsResponse{TotalCount: 1}, testObjectID) + + // act + actual, err := r.ensureSettingExists() + + // assert + assert.NoError(t, err) + assert.Equal(t, "", actual) + }) +} + +func TestReconcileErrors(t *testing.T) { + t.Run(`don't create setting when no kube-system uuid is given`, func(t *testing.T) { + // arrange + r := createReconciler(t, "", []dtclient.MonitoredEntity{}, dtclient.GetSettingsResponse{}, testObjectID) + + // act + actual, err := r.ensureSettingExists() + + // assert + assert.Error(t, err) + assert.Equal(t, "", actual) + }) + + t.Run(`don't create setting when get entities api response is error`, func(t *testing.T) { + // arrange + r := createReconcilerWithError(t, errors.New("could not get monitored entities"), nil, nil) + + // act + actual, err := r.ensureSettingExists() + + // assert + assert.Error(t, err) + assert.Equal(t, "", actual) + }) + + t.Run(`don't create setting when get settings api response is error`, func(t *testing.T) { + // arrange + r := createReconcilerWithError(t, nil, errors.New("could not get settings for monitored entities"), nil) + + // act + actual, err := r.ensureSettingExists() + + // assert + assert.Error(t, err) + assert.Equal(t, "", actual) + }) + + t.Run(`don't create setting when create settings api response is error`, func(t *testing.T) { + // arrange + r := createReconcilerWithError(t, nil, nil, errors.New("could not create monitored entity")) + + // act + actual, err := r.ensureSettingExists() + + // assert + assert.Error(t, err) + assert.Equal(t, "", actual) + }) +} + +func TestDetermineNewestMonitoredEntity(t *testing.T) { + t.Run(`newest monitored entity is correctly calculated`, func(t *testing.T) { + // arrange + // explicit create of entities here to visualize that one has the newest LastSeenTimestamp + // here it is the first one + entities := []dtclient.MonitoredEntity{ + {EntityId: "KUBERNETES_CLUSTER-0E30FE4BF2007587", DisplayName: "operator test entity newest", LastSeenTms: 1639483869085}, + {EntityId: "KUBERNETES_CLUSTER-119C75CCDA94799F", DisplayName: "operator test entity 1", LastSeenTms: 1639034988126}, + {EntityId: "KUBERNETES_CLUSTER-119C75CCDA947993", DisplayName: "operator test entity 2", LastSeenTms: 1639134988126}, + {EntityId: "KUBERNETES_CLUSTER-119C75CCDA94799D", DisplayName: "operator test entity 3", LastSeenTms: 1639234988126}, + } + + // act + newestEntity := determineNewestMonitoredEntity(entities) + + // assert + assert.NotNil(t, newestEntity) + assert.Equal(t, entities[0].EntityId, newestEntity) + }) +} diff --git a/src/controllers/dynakube/dynakube_controller.go b/src/controllers/dynakube/dynakube_controller.go index 6678c4841a..18c77012c7 100644 --- a/src/controllers/dynakube/dynakube_controller.go +++ b/src/controllers/dynakube/dynakube_controller.go @@ -9,6 +9,7 @@ import ( dynatracev1beta1 "github.com/Dynatrace/dynatrace-operator/src/api/v1beta1" "github.com/Dynatrace/dynatrace-operator/src/controllers/activegate/capability" + "github.com/Dynatrace/dynatrace-operator/src/controllers/activegate/reconciler/automaticapimonitoring" rcap "github.com/Dynatrace/dynatrace-operator/src/controllers/activegate/reconciler/capability" "github.com/Dynatrace/dynatrace-operator/src/controllers/dynakube/dtpullsecret" "github.com/Dynatrace/dynatrace-operator/src/controllers/dynakube/dtversion" @@ -199,7 +200,7 @@ func (r *ReconcileDynaKube) reconcileDynaKube(ctx context.Context, dkState *stat dkState.Update(upd, defaultUpdateInterval, "Found updates") dkState.Error(err) - if !r.reconcileActiveGateCapabilities(dkState) { + if !r.reconcileActiveGateCapabilities(dkState, dtc) { return } if dkState.Instance.HostMonitoringMode() { @@ -263,6 +264,7 @@ func (r *ReconcileDynaKube) reconcileDynaKube(ctx context.Context, dkState *stat return } } + } func (r *ReconcileDynaKube) ensureDeleted(obj client.Object) error { @@ -272,7 +274,7 @@ func (r *ReconcileDynaKube) ensureDeleted(obj client.Object) error { return nil } -func (r *ReconcileDynaKube) reconcileActiveGateCapabilities(dkState *status.DynakubeState) bool { +func (r *ReconcileDynaKube) reconcileActiveGateCapabilities(dkState *status.DynakubeState, dtc dtclient.Client) bool { var caps = []capability.Capability{ capability.NewKubeMonCapability(dkState.Instance), capability.NewRoutingCapability(dkState.Instance), @@ -311,6 +313,17 @@ func (r *ReconcileDynaKube) reconcileActiveGateCapabilities(dkState *status.Dyna } } + //start automatic config creation + if dkState.Instance.Status.KubeSystemUUID != "" && + dkState.Instance.FeatureAutomaticKubernetesApiMonitoring() && + dkState.Instance.KubernetesMonitoringMode() { + err := automaticapimonitoring.NewReconciler(dtc, dkState.Instance.Name, dkState.Instance.Status.KubeSystemUUID). + Reconcile() + if err != nil { + log.Error(err, "could not create setting") + } + } + return true } diff --git a/src/controllers/dynakube/dynakube_controller_test.go b/src/controllers/dynakube/dynakube_controller_test.go index 596670441c..bd67096983 100644 --- a/src/controllers/dynakube/dynakube_controller_test.go +++ b/src/controllers/dynakube/dynakube_controller_test.go @@ -12,12 +12,14 @@ import ( "github.com/Dynatrace/dynatrace-operator/src/scheme" "github.com/Dynatrace/dynatrace-operator/src/scheme/fake" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -28,7 +30,8 @@ const ( testAPIToken = "test-api-token" testVersion = "1.217-12345-678910" - testUUID = "test-uuid" + testUUID = "test-uuid" + testObjectID = "test-object-id" testHost = "test-host" testPort = uint32(1234) @@ -101,6 +104,36 @@ func TestReconcileActiveGate_Reconcile(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, statefulSet) }) + t.Run(`Reconcile reconciles automatic kubernetes api monitoring`, func(t *testing.T) { + mockClient := createDTMockClient(dtclient.TokenScopes{dtclient.TokenScopeInstallerDownload}, + dtclient.TokenScopes{dtclient.TokenScopeDataExport, + dtclient.TokenScopeReadConfig, + dtclient.TokenScopeWriteConfig, + }) + instance := &dynatracev1beta1.DynaKube{ + ObjectMeta: metav1.ObjectMeta{ + Name: testName, + Namespace: testNamespace, + Annotations: map[string]string{ + "alpha.operator.dynatrace.com/feature-automatic-kubernetes-api-monitoring": "true", + }, + }, + Spec: dynatracev1beta1.DynaKubeSpec{ + ActiveGate: dynatracev1beta1.ActiveGateSpec{ + Capabilities: []dynatracev1beta1.CapabilityDisplayName{ + dynatracev1beta1.KubeMonCapability.DisplayName, + }, + }, + }} + r := createFakeClientAndReconcile(mockClient, instance, testPaasToken, testAPIToken) + + result, err := r.Reconcile(context.TODO(), reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: testName}, + }) + + assert.NoError(t, err) + assert.NotNil(t, result) + }) } func TestReconcileOnlyOneTokenProvided_Reconcile(t *testing.T) { @@ -307,6 +340,12 @@ func createDTMockClient(paasTokenScopes, apiTokenScopes dtclient.TokenScopes) *d mockClient.On("GetConnectionInfo").Return(dtclient.ConnectionInfo{TenantUUID: "abc123456"}, nil) mockClient.On("GetLatestAgentVersion", dtclient.OsUnix, dtclient.InstallerTypeDefault).Return(testVersion, nil) mockClient.On("GetLatestAgentVersion", dtclient.OsUnix, dtclient.InstallerTypePaaS).Return(testVersion, nil) + mockClient.On("GetMonitoredEntitiesForKubeSystemUUID", mock.AnythingOfType("string")). + Return([]dtclient.MonitoredEntity{}, nil) + mockClient.On("GetSettingsForMonitoredEntities", []dtclient.MonitoredEntity{}). + Return(dtclient.GetSettingsResponse{}, nil) + mockClient.On("CreateOrUpdateKubernetesSetting", testName, testUID, mock.AnythingOfType("string")). + Return(testObjectID, nil) return mockClient } @@ -330,7 +369,10 @@ func createFakeClientAndReconcile(mockClient dtclient.Client, instance *dynatrac ObjectMeta: metav1.ObjectMeta{ Name: kubesystem.Namespace, UID: testUID, - }}) + }, + }, + generateStatefulSetForTesting(testName, testNamespace, "activegate", testUID), + ) r := &ReconcileDynaKube{ client: fakeClient, apiReader: fakeClient, @@ -342,3 +384,154 @@ func createFakeClientAndReconcile(mockClient dtclient.Client, instance *dynatrac return r } + +// generateStatefulSetForTesting prepares an ActiveGate StatefulSet after a Reconciliation of the Dynakube with a specific feature enabled +func generateStatefulSetForTesting(name, namespace, feature, kubeSystemUUID string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-" + feature, + Namespace: namespace, + Labels: map[string]string{ + "dynatrace.com/component": feature, + "operator.dynatrace.com/feature": feature, + "operator.dynatrace.com/instance": name, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "dynatrace.com/v1beta1", + Kind: "DynaKube", + Name: name, + }, + }, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "dynatrace.com/component": feature, + "operator.dynatrace.com/feature": feature, + "operator.dynatrace.com/instance": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "dynatrace.com/component": feature, + "operator.dynatrace.com/feature": feature, + "operator.dynatrace.com/instance": name, + }, + Annotations: map[string]string{ + "internal.operator.dynatrace.com/custom-properties-hash": "", + "internal.operator.dynatrace.com/version": "", + }, + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "truststore-volume", + }, + }, + InitContainers: []corev1.Container{ + { + Name: "certificate-loader", + Command: []string{ + "/bin/bash", + }, + Args: []string{ + "-c", + "/opt/dynatrace/gateway/k8scrt2jks.sh", + }, + WorkingDir: "/var/lib/dynatrace/gateway", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "truststore-volume", + MountPath: "/var/lib/dynatrace/gateway/ssl", + MountPropagation: (*corev1.MountPropagationMode)(nil), + }, + }, + ImagePullPolicy: "Always", + }, + }, + Containers: []corev1.Container{ + { + Name: feature, + Env: []corev1.EnvVar{ + { + Name: "DT_CAPABILITIES", + Value: "kubernetes_monitoring", + }, + { + Name: "DT_ID_SEED_NAMESPACE", + Value: namespace, + }, + { + Name: "DT_ID_SEED_K8S_CLUSTER_ID", + Value: kubeSystemUUID, + }, + { + Name: "DT_DEPLOYMENT_METADATA", + Value: "orchestration_tech=Operator-active_gate;script_version=snapshot;orchestrator_id=" + kubeSystemUUID, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "truststore-volume", + ReadOnly: true, + MountPath: "/opt/dynatrace/gateway/jre/lib/security/cacerts", + SubPath: "k8s-local.jks", + }, + }, + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/rest/health", + Port: intstr.IntOrString{ + IntVal: 9999, + }, + Scheme: "HTTPS", + }, + }, + InitialDelaySeconds: 90, + PeriodSeconds: 15, + FailureThreshold: 3, + }, + ImagePullPolicy: "Always", + }, + }, + ServiceAccountName: "dynatrace-kubernetes-monitoring", + ImagePullSecrets: []corev1.LocalObjectReference{ + { + Name: name + "-pull-secret", + }, + }, + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/arch", + Operator: "In", + Values: []string{ + "amd64", + }, + }, + { + Key: "kubernetes.io/os", + Operator: "In", + Values: []string{ + "linux", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + PodManagementPolicy: "Parallel", + }, + } +} diff --git a/src/dtclient/client.go b/src/dtclient/client.go index 2f0364a92f..b7654c3afb 100644 --- a/src/dtclient/client.go +++ b/src/dtclient/client.go @@ -59,6 +59,17 @@ type Client interface { // GetTenantInfo returns TenantInfo that holds UUID, Tenant Token and Endpoints GetTenantInfo() (*TenantInfo, error) + + // CreateOrUpdateKubernetesSetting returns the object id of the created k8s settings if successful, or an api error otherwise + CreateOrUpdateKubernetesSetting(name, kubeSystemUUID, scope string) (string, error) + + // GetMonitoredEntitiesForKubeSystemUUID returns a (possibly empty) list of k8s monitored entities for the given uuid, + // or an api error otherwise + GetMonitoredEntitiesForKubeSystemUUID(kubeSystemUUID string) ([]MonitoredEntity, error) + + // GetSettingsForMonitoredEntities returns the settings response with the number of settings objects, + // or an api error otherwise + GetSettingsForMonitoredEntities(monitoredEntities []MonitoredEntity) (GetSettingsResponse, error) } // Known OS values. diff --git a/src/dtclient/kubernetes_settings.go b/src/dtclient/kubernetes_settings.go new file mode 100644 index 0000000000..36ae68123f --- /dev/null +++ b/src/dtclient/kubernetes_settings.go @@ -0,0 +1,251 @@ +package dtclient + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "strings" + + "github.com/pkg/errors" +) + +type postKubernetesSettings struct { + Label string `json:"label"` + ClusterIdEnabled bool `json:"clusterIdEnabled"` + ClusterId string `json:"clusterId"` + CloudApplicationPipelineEnabled bool `json:"cloudApplicationPipelineEnabled"` + OpenMetricsPipelineEnabled bool `json:"openMetricsPipelineEnabled"` + Enabled bool `json:"enabled"` + EventProcessingActive bool `json:"eventProcessingActive"` + EventProcessingV2Active bool `json:"eventProcessingV2Active"` + FilterEvents bool `json:"filterEvents"` +} + +type postKubernetesSettingsBody struct { + SchemaId string `json:"schemaId"` + SchemaVersion string `json:"schemaVersion"` + Scope string `json:"scope,omitempty"` + Value postKubernetesSettings `json:"value"` +} + +type monitoredEntitiesResponse struct { + TotalCount int `json:"totalCount"` + PageSize int `json:"pageSize"` + Entities []MonitoredEntity `json:"entities"` +} + +type MonitoredEntity struct { + EntityId string `json:"entityId"` + DisplayName string `json:"displayName"` + LastSeenTms int64 `json:"lastSeenTms"` +} + +type GetSettingsResponse struct { + TotalCount int `json:"totalCount"` +} + +type postSettingsResponse struct { + ObjectId string `json:"objectId"` +} + +type getSettingsErrorResponse struct { + ErrorMessage getSettingsError `json:"error"` +} + +type getSettingsError struct { + Code int + Message string + ConstraintViolations constraintViolations +} + +type constraintViolations []struct { + ParameterLocation string + Location string + Message string + Path string +} + +func (dtc *dynatraceClient) CreateOrUpdateKubernetesSetting(name, kubeSystemUUID, scope string) (string, error) { + if kubeSystemUUID == "" { + return "", errors.New("no kube-system namespace UUID given") + } + + body := []postKubernetesSettingsBody{ + { + SchemaId: "builtin:cloud.kubernetes", + SchemaVersion: "1.0.27", + Value: postKubernetesSettings{ + Enabled: true, + Label: name, + ClusterIdEnabled: true, + ClusterId: kubeSystemUUID, + CloudApplicationPipelineEnabled: true, + OpenMetricsPipelineEnabled: false, + EventProcessingActive: false, + FilterEvents: false, + EventProcessingV2Active: false, + }, + }, + } + + if scope != "" { + body[0].Scope = scope + } + + bodyData, err := json.Marshal(body) + if err != nil { + return "", err + } + + req, err := createBaseRequest(fmt.Sprintf("%s/v2/settings/objects?validateOnly=false", dtc.url), http.MethodPost, dtc.apiToken, bytes.NewReader(bodyData)) + if err != nil { + return "", err + } + + res, err := dtc.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("error making post request to dynatrace api: %s", err.Error()) + } + + resData, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", fmt.Errorf("error reading response: %w", err) + } + + if res.StatusCode != http.StatusOK && + res.StatusCode != http.StatusCreated { + return "", handleErrorArrayResponseFromAPI(resData, res.StatusCode) + } + + var resDataJson []postSettingsResponse + err = json.Unmarshal(resData, &resDataJson) + if err != nil { + return "", err + } + + if len(resDataJson) != 1 { + return "", fmt.Errorf("response is not containing exactly one entry %s", resData) + } + + return resDataJson[0].ObjectId, nil +} + +func (dtc *dynatraceClient) GetMonitoredEntitiesForKubeSystemUUID(kubeSystemUUID string) ([]MonitoredEntity, error) { + if kubeSystemUUID == "" { + return nil, errors.New("no kube-system namespace UUID given") + } + + req, err := createBaseRequest(fmt.Sprintf("%s/v2/entities", dtc.url), http.MethodGet, dtc.apiToken, nil) + if err != nil { + return nil, err + } + + q := req.URL.Query() + q.Add("pageSize", "500") + q.Add("entitySelector", fmt.Sprintf("type(KUBERNETES_CLUSTER),kubernetesClusterId(%s)", kubeSystemUUID)) + q.Add("from", "-365d") + q.Add("fields", "+lastSeenTms") + req.URL.RawQuery = q.Encode() + + res, err := dtc.httpClient.Do(req) + if err != nil { + log.Info("check if ME exists failed") + return nil, err + } + + var resDataJson monitoredEntitiesResponse + err = dtc.unmarshalToJson(res, &resDataJson) + if err != nil { + return nil, fmt.Errorf("error parsing response body: %s", err.Error()) + } + + return resDataJson.Entities, nil +} + +func (dtc *dynatraceClient) GetSettingsForMonitoredEntities(monitoredEntities []MonitoredEntity) (GetSettingsResponse, error) { + if len(monitoredEntities) < 1 { + return GetSettingsResponse{TotalCount: 0}, nil + } + + var scopes []string + for _, entity := range monitoredEntities { + scopes = append(scopes, entity.EntityId) + } + + req, err := createBaseRequest(fmt.Sprintf("%s/v2/settings/objects", dtc.url), http.MethodGet, dtc.apiToken, nil) + if err != nil { + return GetSettingsResponse{}, err + } + + q := req.URL.Query() + q.Add("schemaIds", "builtin:cloud.kubernetes") + q.Add("scopes", strings.Join(scopes, ",")) + req.URL.RawQuery = q.Encode() + + res, err := dtc.httpClient.Do(req) + if err != nil { + log.Info("failed to retrieve MEs") + return GetSettingsResponse{}, err + } + + var resDataJson GetSettingsResponse + err = dtc.unmarshalToJson(res, &resDataJson) + if err != nil { + return GetSettingsResponse{}, fmt.Errorf("error parsing response body: %s", err.Error()) + } + + return resDataJson, nil +} + +func (dtc *dynatraceClient) unmarshalToJson(res *http.Response, resDataJson interface{}) error { + resData, err := dtc.getServerResponseData(res) + + if err != nil { + return fmt.Errorf("error reading response body: %s", err.Error()) + } + err = json.Unmarshal(resData, resDataJson) + + if err != nil { + return fmt.Errorf("error parsing response body: %s", err.Error()) + } + + return nil +} + +func createBaseRequest(url, method, apiToken string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequest(method, url, body) + if err != nil { + return nil, fmt.Errorf("error initializing http request: %s", err.Error()) + } + req.Header.Add("Accept", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Api-Token %s", apiToken)) + + if method == http.MethodPost { + req.Header.Add("Content-Type", "application/json") + } + + return req, nil +} + +func handleErrorArrayResponseFromAPI(response []byte, statusCode int) error { + var se []getSettingsErrorResponse + if err := json.Unmarshal(response, &se); err != nil { + return fmt.Errorf("response error: %d, can't unmarshal json response: %w", statusCode, err) + } + + var sb strings.Builder + sb.WriteString("[Settings Creation]: could not create the Kubernetes setting for the following reason:\n") + + for _, errorResponse := range se { + sb.WriteString(fmt.Sprintf("[%s; Code: %d\n", errorResponse.ErrorMessage.Message, errorResponse.ErrorMessage.Code)) + for _, constraintViolation := range errorResponse.ErrorMessage.ConstraintViolations { + sb.WriteString(fmt.Sprintf("\t- %s\n", constraintViolation.Message)) + } + sb.WriteString("]\n") + } + + return fmt.Errorf(sb.String()) +} diff --git a/src/dtclient/kubernetes_settings_test.go b/src/dtclient/kubernetes_settings_test.go new file mode 100644 index 0000000000..b3980eb542 --- /dev/null +++ b/src/dtclient/kubernetes_settings_test.go @@ -0,0 +1,389 @@ +package dtclient + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testUID = "test-uid" + testName = "test-name" + testObjectID = "test-objectid" + testScope = "test-scope" +) + +func TestDynatraceClient_GetMonitoredEntitiesForKubeSystemUUID(t *testing.T) { + t.Run("monitored entities for this uuid exist", func(t *testing.T) { + // arrange + expected := createMonitoredEntitiesForTesting() + + dynatraceServer := httptest.NewServer(mockDynatraceServerEntitiesHandler(expected, false)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).GetMonitoredEntitiesForKubeSystemUUID(testUID) + + // assert + assert.NotNil(t, actual) + assert.NoError(t, err) + assert.Len(t, actual, 2) + assert.EqualValues(t, expected, actual) + }) + + t.Run("no monitored entities for this uuid exist", func(t *testing.T) { + // arrange + expected := []MonitoredEntity{} + + dynatraceServer := httptest.NewServer(mockDynatraceServerEntitiesHandler(expected, false)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).GetMonitoredEntitiesForKubeSystemUUID(testUID) + + // assert + assert.NotNil(t, actual) + assert.NoError(t, err) + assert.Len(t, actual, 0) + assert.EqualValues(t, expected, actual) + }) + + t.Run("no monitored entities found because no kube-system uuid is provided", func(t *testing.T) { + // arrange + expected := createMonitoredEntitiesForTesting() + + dynatraceServer := httptest.NewServer(mockDynatraceServerEntitiesHandler(expected, true)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).GetMonitoredEntitiesForKubeSystemUUID("") + + // assert + assert.Nil(t, actual) + assert.Error(t, err) + assert.Len(t, actual, 0) + }) + + t.Run("no monitored entities found because of an api error", func(t *testing.T) { + // arrange + expected := createMonitoredEntitiesForTesting() + + dynatraceServer := httptest.NewServer(mockDynatraceServerEntitiesHandler(expected, true)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).GetMonitoredEntitiesForKubeSystemUUID(testUID) + + // assert + assert.Nil(t, actual) + assert.Error(t, err) + assert.Len(t, actual, 0) + }) +} + +func TestDynatraceClient_GetSettingsForMonitoredEntities(t *testing.T) { + t.Run(`settings for the given monitored entities exist`, func(t *testing.T) { + // arrange + expected := createMonitoredEntitiesForTesting() + totalCount := 2 + + dynatraceServer := httptest.NewServer(mockDynatraceServerSettingsHandler(totalCount, "", false)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).GetSettingsForMonitoredEntities(expected) + + // assert + assert.NoError(t, err) + assert.NotNil(t, actual) + assert.True(t, actual.TotalCount > 0) + assert.Equal(t, len(expected), actual.TotalCount) + }) + + t.Run(`no settings for the given monitored entities exist`, func(t *testing.T) { + // arrange + expected := createMonitoredEntitiesForTesting() + totalCount := 0 + + dynatraceServer := httptest.NewServer(mockDynatraceServerSettingsHandler(totalCount, "", false)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).GetSettingsForMonitoredEntities(expected) + + // assert + assert.NoError(t, err) + assert.NotNil(t, actual) + assert.True(t, actual.TotalCount < 1) + }) + + t.Run(`no settings for an empty list of monitored entities exist`, func(t *testing.T) { + // arrange + entities := []MonitoredEntity{} + // it is immaterial what we put here since no http call is executed when the list of + // monitored entities is empty, therefore also no settings will be returned + totalCount := 999 + + dynatraceServer := httptest.NewServer(mockDynatraceServerSettingsHandler(totalCount, "", false)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).GetSettingsForMonitoredEntities(entities) + + // assert + assert.NoError(t, err) + assert.NotNil(t, actual) + assert.True(t, actual.TotalCount == 0) + }) + + t.Run(`no settings found for because of an api error`, func(t *testing.T) { + // arrange + entities := createMonitoredEntitiesForTesting() + // it is immaterial what we put here since the http request is producing an error + totalCount := 999 + + dynatraceServer := httptest.NewServer(mockDynatraceServerSettingsHandler(totalCount, "", true)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).GetSettingsForMonitoredEntities(entities) + + // assert + assert.Error(t, err) + assert.True(t, actual.TotalCount == 0) + }) +} + +func TestDynatraceClient_CreateOrUpdateKubernetesSetting(t *testing.T) { + t.Run(`create settings for the given monitored entity id`, func(t *testing.T) { + // arrange + dynatraceServer := httptest.NewServer(mockDynatraceServerSettingsHandler(1, testObjectID, false)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).CreateOrUpdateKubernetesSetting(testName, testUID, testScope) + + // assert + assert.NotNil(t, actual) + assert.NoError(t, err) + assert.Len(t, actual, len(testObjectID)) + assert.EqualValues(t, testObjectID, actual) + }) + + t.Run(`don't create settings for the given monitored entity id because no kube-system uuid is provided`, func(t *testing.T) { + // arrange + dynatraceServer := httptest.NewServer(mockDynatraceServerSettingsHandler(1, testObjectID, false)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).CreateOrUpdateKubernetesSetting(testName, "", testScope) + + // assert + assert.Error(t, err) + assert.Len(t, actual, 0) + }) + + t.Run(`don't create settings for the given monitored entity id because of api error`, func(t *testing.T) { + // arrange + dynatraceServer := httptest.NewServer(mockDynatraceServerSettingsHandler(1, testObjectID, true)) + defer dynatraceServer.Close() + + skipCert := SkipCertificateValidation(true) + dtc, err := NewClient(dynatraceServer.URL, apiToken, paasToken, skipCert) + require.NoError(t, err) + require.NotNil(t, dtc) + + // act + actual, err := dtc.(*dynatraceClient).CreateOrUpdateKubernetesSetting(testName, testUID, testScope) + + // assert + assert.Error(t, err) + assert.Len(t, actual, 0) + }) +} + +func createMonitoredEntitiesForTesting() []MonitoredEntity { + return []MonitoredEntity{ + {EntityId: "KUBERNETES_CLUSTER-0E30FE4BF2007587", DisplayName: "operator test entity 1", LastSeenTms: 1639483869085}, + {EntityId: "KUBERNETES_CLUSTER-119C75CCDA94799F", DisplayName: "operator test entity 2", LastSeenTms: 1639034988126}, + } +} + +func mockHandleEntitiesRequest(request *http.Request, writer http.ResponseWriter, entities []MonitoredEntity) { + if request.Method == http.MethodGet { + if !strings.Contains(request.Form.Get("entitySelector"), "type(KUBERNETES_CLUSTER)") { + writer.WriteHeader(http.StatusBadRequest) + return + } + + meResponse := monitoredEntitiesResponse{ + TotalCount: len(entities), + PageSize: 500, + Entities: entities, + } + + entitiesResponse, err := json.Marshal(meResponse) + + if err != nil { + return + } + + writer.WriteHeader(http.StatusOK) + writer.Write(entitiesResponse) + } else { + writeError(writer, http.StatusMethodNotAllowed) + } +} + +func mockHandleSettingsRequest(request *http.Request, writer http.ResponseWriter, totalCount int, objectId string) { + if request.Method == http.MethodGet { + if request.Form.Get("schemaIds") != "builtin:cloud.kubernetes" || request.Form.Get("scopes") == "" { + writer.WriteHeader(http.StatusBadRequest) + return + } + + settingsGetResponse, err := json.Marshal(GetSettingsResponse{TotalCount: totalCount}) + + if err != nil { + return + } + + writer.WriteHeader(http.StatusOK) + writer.Write(settingsGetResponse) + } else if request.Method == http.MethodPost { + if request.Body == nil { + writer.WriteHeader(http.StatusBadRequest) + return + } + + body, err := ioutil.ReadAll(request.Body) + + if err != nil { + return + } + + var parsedBody []postKubernetesSettingsBody + err = json.Unmarshal(body, &parsedBody) + + if err != nil { + return + } + + var settingsPostResponse []postSettingsResponse + settingsPostResponse = append(settingsPostResponse, postSettingsResponse{ + ObjectId: objectId, + }) + + settingsPostResponseBytes, err := json.Marshal(settingsPostResponse) + + if err != nil { + return + } + + writer.WriteHeader(http.StatusOK) + writer.Write(settingsPostResponseBytes) + } else { + writeError(writer, http.StatusMethodNotAllowed) + } + +} + +func mockDynatraceServerEntitiesHandler(entities []MonitoredEntity, isError bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if isError { + writeError(w, http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "application/json") + + if r.FormValue("Api-Token") == "" && r.Header.Get("Authorization") == "" { + writeError(w, http.StatusUnauthorized) + } else { + switch r.URL.Path { + case "/v2/entities": + mockHandleEntitiesRequest(r, w, entities) + default: + writeError(w, http.StatusBadRequest) + } + } + } +} + +func mockDynatraceServerSettingsHandler(totalCount int, objectId string, isError bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if isError { + writeError(w, http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "application/json") + + if r.FormValue("Api-Token") == "" && r.Header.Get("Authorization") == "" { + writeError(w, http.StatusUnauthorized) + } else { + switch r.URL.Path { + case "/v2/settings/objects": + mockHandleSettingsRequest(r, w, totalCount, objectId) + default: + writeError(w, http.StatusBadRequest) + } + } + } +} diff --git a/src/dtclient/mock_client.go b/src/dtclient/mock_client.go index 2cbedc1d58..053210fab6 100644 --- a/src/dtclient/mock_client.go +++ b/src/dtclient/mock_client.go @@ -65,3 +65,18 @@ func (o *MockDynatraceClient) GetTokenScopes(token string) (TokenScopes, error) args := o.Called(token) return args.Get(0).(TokenScopes), args.Error(1) } + +func (o *MockDynatraceClient) CreateOrUpdateKubernetesSetting(name string, kubeSystemUUID string, scope string) (string, error) { + args := o.Called(name, kubeSystemUUID, scope) + return args.String(0), args.Error(1) +} + +func (o *MockDynatraceClient) GetMonitoredEntitiesForKubeSystemUUID(kubeSystemUUID string) ([]MonitoredEntity, error) { + args := o.Called(kubeSystemUUID) + return args.Get(0).([]MonitoredEntity), args.Error(1) +} + +func (o *MockDynatraceClient) GetSettingsForMonitoredEntities(monitoredEntities []MonitoredEntity) (GetSettingsResponse, error) { + args := o.Called(monitoredEntities) + return args.Get(0).(GetSettingsResponse), args.Error(1) +}