Skip to content

Commit

Permalink
[#124] support scale subresource for scale to zero
Browse files Browse the repository at this point in the history
  • Loading branch information
gtully authored and brusdev committed Mar 2, 2023
1 parent c365b3a commit 3ba7a87
Show file tree
Hide file tree
Showing 10 changed files with 188 additions and 6 deletions.
4 changes: 4 additions & 0 deletions api/v1beta1/activemqartemis_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,9 @@ type ActiveMQArtemisStatus struct {
//we probably use Deployments as operatorHub shows invalid field podStatus
//see 3scale https://github.com/3scale/3scale-operator/blob/8abbabd926616b98db0e7e736e68e5ceba90ed9d/apis/apps/v1alpha1/apimanager_types.go#L87

//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="Deployment Plan Size"
DeploymentPlanSize int32 `json:"deploymentPlanSize,omitempty"`

// Current state of external referenced resources
//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="External Configurations Status"
ExternalConfigs []ExternalConfigStatus `json:"externalConfigs,omitempty"`
Expand Down Expand Up @@ -617,6 +620,7 @@ type ExternalConfigStatus struct {

//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
//+kubebuilder:subresource:scale:specpath=.spec.deploymentPlan.size,statuspath=.status.deploymentPlanSize
//+kubebuilder:storageversion
//+kubebuilder:resource:path=activemqartemises
//+operator-sdk:csv:customresourcedefinitions:resources={{"Service", "v1"}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1297,6 +1297,8 @@ spec:
path: conditions
x-descriptors:
- urn:alm:descriptor:io.kubernetes.conditions
- displayName: Deployment Plan Size
path: deploymentPlanSize
- description: Current state of external referenced resources
displayName: External Configurations Status
path: externalConfigs
Expand Down
6 changes: 6 additions & 0 deletions bundle/manifests/broker.amq.io_activemqartemises.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2314,6 +2314,9 @@ spec:
- type
type: object
type: array
deploymentPlanSize:
format: int32
type: integer
externalConfigs:
description: Current state of external referenced resources
items:
Expand Down Expand Up @@ -2374,6 +2377,9 @@ spec:
served: true
storage: true
subresources:
scale:
specReplicasPath: .spec.deploymentPlan.size
statusReplicasPath: .status.deploymentPlanSize
status: {}
- name: v2alpha1
schema:
Expand Down
6 changes: 6 additions & 0 deletions config/crd/bases/broker.amq.io_activemqartemises.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2316,6 +2316,9 @@ spec:
- type
type: object
type: array
deploymentPlanSize:
format: int32
type: integer
externalConfigs:
description: Current state of external referenced resources
items:
Expand Down Expand Up @@ -2376,6 +2379,9 @@ spec:
served: true
storage: true
subresources:
scale:
specReplicasPath: .spec.deploymentPlan.size
statusReplicasPath: .status.deploymentPlanSize
status: {}
- name: v2alpha1
schema:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1071,6 +1071,8 @@ spec:
path: conditions
x-descriptors:
- urn:alm:descriptor:io.kubernetes.conditions
- displayName: Deployment Plan Size
path: deploymentPlanSize
- description: Current state of external referenced resources
displayName: External Configurations Status
path: externalConfigs
Expand Down
5 changes: 3 additions & 2 deletions controllers/activemqartemis_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -2440,7 +2440,7 @@ func getPodStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, nam
sfsFound := &appsv1.StatefulSet{}
err := client.Get(context.TODO(), ssNamespacedName, sfsFound)
if err == nil {
status = getSingleStatefulSetStatus(sfsFound)
status = getSingleStatefulSetStatus(sfsFound, cr)
}

// TODO: Remove global usage
Expand All @@ -2458,12 +2458,13 @@ func getPodStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, nam
return status
}

func getSingleStatefulSetStatus(ss *appsv1.StatefulSet) olm.DeploymentStatus {
func getSingleStatefulSetStatus(ss *appsv1.StatefulSet, cr *brokerv1beta1.ActiveMQArtemis) olm.DeploymentStatus {
var ready, starting, stopped []string
var requestedCount = int32(0)
if ss.Spec.Replicas != nil {
requestedCount = *ss.Spec.Replicas
}
cr.Status.DeploymentPlanSize = requestedCount

targetCount := ss.Status.Replicas
readyCount := ss.Status.ReadyReplicas
Expand Down
10 changes: 7 additions & 3 deletions controllers/activemqartemis_reconciler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,15 +200,16 @@ func TestGetSingleStatefulSetStatus(t *testing.T) {
ss.Status.Replicas = 1
ss.Status.ReadyReplicas = 1

statusRunning := getSingleStatefulSetStatus(ss)
cr := &brokerv1beta1.ActiveMQArtemis{}
statusRunning := getSingleStatefulSetStatus(ss, cr)
if statusRunning.Ready[0] != "joe-0" {
t.Errorf("not good!, expect correct 0 ordinal" + statusRunning.Ready[0])
}

ss.Status.Replicas = 0
ss.Status.ReadyReplicas = 0

statusRunning = getSingleStatefulSetStatus(ss)
statusRunning = getSingleStatefulSetStatus(ss, cr)
if statusRunning.Stopped[0] != "joe" {
t.Errorf("not good!, expect ss name in stopped" + statusRunning.Stopped[0])
}
Expand All @@ -218,7 +219,7 @@ func TestGetSingleStatefulSetStatus(t *testing.T) {
ss.Status.Replicas = 2
ss.Status.ReadyReplicas = 1

statusRunning = getSingleStatefulSetStatus(ss)
statusRunning = getSingleStatefulSetStatus(ss, cr)
if statusRunning.Ready[0] != "joe-0" {
t.Errorf("not good!, expect correct 0 ordinal ready" + statusRunning.Ready[0])
}
Expand All @@ -227,6 +228,9 @@ func TestGetSingleStatefulSetStatus(t *testing.T) {
t.Errorf("not good!, expect ss name in starting" + statusRunning.Stopped[0])
}

if cr.Status.DeploymentPlanSize != 2 {
t.Errorf("not good!, status not updated")
}
}

func TestGetConfigAppliedConfigMapName(t *testing.T) {
Expand Down
148 changes: 148 additions & 0 deletions controllers/activemqartemis_scale_zero_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// +kubebuilder:docs-gen:collapse=Apache License

/*
As usual, we start with the necessary imports. We also define some utility variables.
*/
package controllers

import (
"context"
"os"

"k8s.io/client-go/dynamic"
"k8s.io/client-go/scale"

"k8s.io/client-go/discovery"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"k8s.io/apimachinery/pkg/api/meta"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"

brokerv1beta1 "github.com/artemiscloud/activemq-artemis-operator/api/v1beta1"
"github.com/artemiscloud/activemq-artemis-operator/pkg/utils/common"
)

var _ = Describe("subresource scale down", func() {

BeforeEach(func() {
BeforeEachSpec()
})

Context("scale to zero via scale subresource", func() {
It("deploy plan 1", func() {
if os.Getenv("USE_EXISTING_CLUSTER") == "true" {

ctx := context.Background()
brokerCrd := generateArtemisSpec(defaultNamespace)

// need to popluate scale json path
brokerCrd.Spec.DeploymentPlan.Size = common.Int32ToPtr(1)

Expect(k8sClient.Create(ctx, &brokerCrd)).Should(Succeed())

createdBrokerCrd := &brokerv1beta1.ActiveMQArtemis{}
createdBrokerCrdKey := types.NamespacedName{
Name: brokerCrd.Name,
Namespace: defaultNamespace,
}

var crdResourceVersion string
By("verifying started")
Eventually(func(g Gomega) {

g.Expect(k8sClient.Get(ctx, createdBrokerCrdKey, createdBrokerCrd)).Should(Succeed())
g.Expect(meta.IsStatusConditionTrue(createdBrokerCrd.Status.Conditions, brokerv1beta1.ReadyConditionType)).Should(BeTrue())
crdResourceVersion = createdBrokerCrd.GetResourceVersion()

}, existingClusterTimeout, existingClusterInterval).Should(Succeed())

By("scale to zero via scale sub resource")

clientset, err := discovery.NewDiscoveryClientForConfig(k8Manager.GetConfig())
Expect(err).To(BeNil())

scaleClient := scale.New(
clientset.RESTClient(), k8Manager.GetRESTMapper(),
dynamic.LegacyAPIPathResolverFunc,
scale.NewDiscoveryScaleKindResolver(clientset),
)

currentScales := scaleClient.Scales(defaultNamespace)

By("by getting current val via scale sub resource")
resource := schema.GroupResource{Group: artemisGvk.Group, Resource: artemisGvk.Kind}
scaleVal, err := currentScales.Get(ctx, resource, createdBrokerCrd.Name, v1.GetOptions{})
Expect(err).To(BeNil())

Expect(scaleVal.Spec.Replicas).To(BeEquivalentTo(1))

Eventually(func(g Gomega) {
scaleVal, err = currentScales.Get(ctx, resource, createdBrokerCrd.Name, v1.GetOptions{})
g.Expect(err).To(BeNil())

By("by updating via via scale sub resource")
scaleVal.Spec.Replicas = 0
_, err = currentScales.Update(ctx, resource, scaleVal, v1.UpdateOptions{})
Expect(err).To(BeNil())

}, existingClusterTimeout, existingClusterInterval).Should(Succeed())

By("verifying scaled down via status")
Eventually(func(g Gomega) {

g.Expect(k8sClient.Get(ctx, createdBrokerCrdKey, createdBrokerCrd)).Should(Succeed())

g.Expect(meta.IsStatusConditionTrue(createdBrokerCrd.Status.Conditions, brokerv1beta1.ReadyConditionType)).Should(BeFalse())
g.Expect(meta.IsStatusConditionTrue(createdBrokerCrd.Status.Conditions, brokerv1beta1.DeployedConditionType)).Should(BeFalse())
g.Expect(meta.IsStatusConditionTrue(createdBrokerCrd.Status.Conditions, brokerv1beta1.ValidConditionType)).Should(BeTrue())

g.Expect(createdBrokerCrd.Status.DeploymentPlanSize).Should(BeEquivalentTo(0))

By("verify crd updated")
g.Expect(crdResourceVersion).ShouldNot(Equal(createdBrokerCrd.GetResourceVersion()))

}, existingClusterTimeout, existingClusterInterval).Should(Succeed())

By("scaling back up")
Eventually(func(g Gomega) {
scaleVal, err = currentScales.Get(ctx, resource, createdBrokerCrd.Name, v1.GetOptions{})
g.Expect(err).To(BeNil())

By("by updating via via scale sub resource")
scaleVal.Spec.Replicas = 1
_, err = currentScales.Update(ctx, resource, scaleVal, v1.UpdateOptions{})
Expect(err).To(BeNil())

}, existingClusterTimeout, existingClusterInterval).Should(Succeed())

By("verifying restarted")
Eventually(func(g Gomega) {

g.Expect(k8sClient.Get(ctx, createdBrokerCrdKey, createdBrokerCrd)).Should(Succeed())
g.Expect(meta.IsStatusConditionTrue(createdBrokerCrd.Status.Conditions, brokerv1beta1.ReadyConditionType)).Should(BeTrue())

}, existingClusterTimeout, existingClusterInterval).Should(Succeed())

Expect(k8sClient.Delete(ctx, createdBrokerCrd)).Should(Succeed())
}

})
})
})
5 changes: 4 additions & 1 deletion controllers/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/envtest"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/manager"

brokerv1alpha1 "github.com/artemiscloud/activemq-artemis-operator/api/v1alpha1"
brokerv1beta1 "github.com/artemiscloud/activemq-artemis-operator/api/v1beta1"
Expand Down Expand Up @@ -105,6 +106,7 @@ var (
// the manager may be stopped/restarted via tests
managerCtx context.Context
managerCancel context.CancelFunc
k8Manager manager.Manager

stateManager *common.StateManager

Expand Down Expand Up @@ -331,7 +333,8 @@ func createControllerManager(disableMetrics bool, watchNamespace string) {
mgrOptions.LeaderElectionReleaseOnCancel = true

// start our controler
k8Manager, err := ctrl.NewManager(restConfig, mgrOptions)
var err error
k8Manager, err = ctrl.NewManager(restConfig, mgrOptions)
Expect(err).ToNot(HaveOccurred())

// Create and start a new auto detect process for this operator
Expand Down
6 changes: 6 additions & 0 deletions deploy/crds/broker_activemqartemis_crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1508,6 +1508,9 @@ spec:
- type
type: object
type: array
deploymentPlanSize:
format: int32
type: integer
externalConfigs:
description: Current state of external referenced resources
items:
Expand Down Expand Up @@ -1567,6 +1570,9 @@ spec:
served: true
storage: true
subresources:
scale:
specReplicasPath: .spec.deploymentPlan.size
statusReplicasPath: .status.deploymentPlanSize
status: {}
- name: v2alpha1
schema:
Expand Down

0 comments on commit 3ba7a87

Please sign in to comment.