Skip to content

Commit

Permalink
enable addon management feature gate by default (#222)
Browse files Browse the repository at this point in the history
Signed-off-by: zhujian <jiazhu@redhat.com>
  • Loading branch information
zhujian7 authored Jul 14, 2023
1 parent 817aa62 commit 9ff0948
Show file tree
Hide file tree
Showing 14 changed files with 74 additions and 128 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ require (
k8s.io/kube-aggregator v0.27.2
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749
open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63
open-cluster-management.io/api v0.11.1-0.20230703133341-6d7212c2e941
open-cluster-management.io/api v0.11.1-0.20230714020829-ef97df044b15
sigs.k8s.io/controller-runtime v0.15.0
sigs.k8s.io/kube-storage-version-migrator v0.0.5
)
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1158,8 +1158,8 @@ k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1E
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63 h1:GCsAD1jb6wqhXTHdUM/HcWzv5b2NbZ6FxpLZcxa/jhI=
open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63/go.mod h1:V+WUFC7GD89Lc68eXSN/FJebnCH4NjrfF44VsO0YAC8=
open-cluster-management.io/api v0.11.1-0.20230703133341-6d7212c2e941 h1:k10Sx7Th1UDyJ+GYFqWddFq+m6U7x9MHk1g8KwrYy8Y=
open-cluster-management.io/api v0.11.1-0.20230703133341-6d7212c2e941/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
open-cluster-management.io/api v0.11.1-0.20230714020829-ef97df044b15 h1:S2a+NsIlaPNQAFruowBunN2wHnK2JfyrpnRU783WwMc=
open-cluster-management.io/api v0.11.1-0.20230714020829-ef97df044b15/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
Expand Down
10 changes: 5 additions & 5 deletions pkg/operator/helpers/helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1489,15 +1489,15 @@ func TestConvertToFeatureGateFlags(t *testing.T) {
{Feature: "ClusterClaim", Mode: operatorapiv1.FeatureGateModeTypeEnable},
{Feature: "AddonManagement", Mode: operatorapiv1.FeatureGateModeTypeEnable},
},
desiredFlags: []string{"--feature-gates=AddonManagement=true"},
desiredFlags: []string{},
},
{
name: "disable feature",
features: []operatorapiv1.FeatureGate{
{Feature: "ClusterClaim", Mode: operatorapiv1.FeatureGateModeTypeDisable},
{Feature: "AddonManagement", Mode: operatorapiv1.FeatureGateModeTypeDisable},
},
desiredFlags: []string{"--feature-gates=ClusterClaim=false"},
desiredFlags: []string{"--feature-gates=ClusterClaim=false", "--feature-gates=AddonManagement=false"},
},
{
name: "invalid feature",
Expand All @@ -1514,10 +1514,10 @@ func TestConvertToFeatureGateFlags(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
flags, msg := ConvertToFeatureGateFlags("test", tc.features, ocmfeature.DefaultSpokeRegistrationFeatureGates)
if msg != tc.desiredMsg {
t.Errorf("unexpected message, got: %s, desired %s", msg, tc.desiredMsg)
t.Errorf("Name: %s, unexpected message, got: %s, desired %s", tc.name, msg, tc.desiredMsg)
}
if !equality.Semantic.DeepEqual(flags, tc.desiredFlags) {
t.Errorf("Unexpected flags, got %v, desired %v", flags, tc.desiredFlags)
t.Errorf("Name: %s, unexpected flags, got %v, desired %v", tc.name, flags, tc.desiredFlags)
}
})
}
Expand Down Expand Up @@ -1559,7 +1559,7 @@ func TestFeatureGateEnabled(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
enabled := FeatureGateEnabled(tc.features, ocmfeature.DefaultSpokeRegistrationFeatureGates, tc.featureName)
if enabled != tc.desiredResult {
t.Errorf("Expect feature enabled is %v, but got %v", tc.desiredResult, enabled)
t.Errorf("Name: %s, expect feature enabled is %v, but got %v", tc.name, tc.desiredResult, enabled)
}
})
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,6 @@ func assertRegistrationDeployment(t *testing.T, actions []clienttesting.Action,
"agent",
fmt.Sprintf("--spoke-cluster-name=%s", clusterName),
"--bootstrap-kubeconfig=/spoke/bootstrap/kubeconfig",
"--feature-gates=AddonManagement=true",
}

if serverURL != "" {
Expand Down
34 changes: 0 additions & 34 deletions test/e2e/addonmanagement_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (

addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
clusterv1apha1 "open-cluster-management.io/api/cluster/v1alpha1"
operatorapiv1 "open-cluster-management.io/api/operator/v1"

"open-cluster-management.io/ocm/pkg/addon/templateagent"
"open-cluster-management.io/ocm/test/e2e/manifests"
Expand Down Expand Up @@ -62,39 +61,6 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
"addon/signca_secret_rolebinding.yaml",
}

ginkgo.BeforeAll(func() {
// enable addon management feature gate
gomega.Eventually(func() error {
clusterManager, err := t.OperatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
if err != nil {
return err
}
clusterManager.Spec.AddOnManagerConfiguration = &operatorapiv1.AddOnManagerConfiguration{
FeatureGates: []operatorapiv1.FeatureGate{
{
Feature: "AddonManagement",
Mode: operatorapiv1.FeatureGateModeTypeEnable,
},
},
}
_, err = t.OperatorClient.OperatorV1().ClusterManagers().Update(context.TODO(), clusterManager, metav1.UpdateOptions{})
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.Succeed())
})

ginkgo.AfterAll(func() {
// disable addon management feature gate
gomega.Eventually(func() error {
clusterManager, err := t.OperatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
if err != nil {
return err
}
clusterManager.Spec.AddOnManagerConfiguration = &operatorapiv1.AddOnManagerConfiguration{}
_, err = t.OperatorClient.OperatorV1().ClusterManagers().Update(context.TODO(), clusterManager, metav1.UpdateOptions{})
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.Succeed())
})

ginkgo.BeforeEach(func() {
addonInstallNamespace = fmt.Sprintf("%s-addon", agentNamespace)
ginkgo.By("create addon custom sign secret")
Expand Down
5 changes: 4 additions & 1 deletion test/e2e/work_webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,10 @@ var _ = ginkgo.Describe("ManifestWork admission webhook", ginkgo.Label("validati
_, err = impersonatedHubWorkClient.WorkV1().ManifestWorks(clusterName).Create(
context.Background(), work, metav1.CreateOptions{})
gomega.Expect(err).To(gomega.HaveOccurred())
gomega.Expect(errors.IsBadRequest(err)).Should(gomega.BeTrue())
if !errors.IsBadRequest(err) {
// not bad request, assert true=false to show the error message
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
})
})
})
Expand Down
59 changes: 28 additions & 31 deletions test/integration/operator/clustermanager_hosted_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
var hubWorkWebhookDeployment = fmt.Sprintf("%s-work-webhook", clusterManagerName)
var hubAddOnManagerDeployment = fmt.Sprintf("%s-addon-manager-controller", clusterManagerName)
var hubWorkControllerDeployment = fmt.Sprintf("%s-work-controller", clusterManagerName)
var hubAddonManagerDeployment = fmt.Sprintf("%s-addon-manager-controller", clusterManagerName)
var hubRegistrationClusterRole = fmt.Sprintf("open-cluster-management:%s-registration:controller", clusterManagerName)
var hubRegistrationWebhookClusterRole = fmt.Sprintf("open-cluster-management:%s-registration:webhook", clusterManagerName)
var hubWorkWebhookClusterRole = fmt.Sprintf("open-cluster-management:%s-registration:webhook", clusterManagerName)
Expand Down Expand Up @@ -200,6 +201,13 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
return nil
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

gomega.Eventually(func() error {
if _, err := hostedKubeClient.AppsV1().Deployments(hubNamespaceHosted).Get(hostedCtx, hubAddonManagerDeployment, metav1.GetOptions{}); err != nil {
return err
}
return nil
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

// Check service
gomega.Eventually(func() error {
if _, err := hostedKubeClient.CoreV1().Services(hubNamespaceHosted).Get(hostedCtx, "cluster-manager-registration-webhook", metav1.GetOptions{}); err != nil {
Expand Down Expand Up @@ -252,55 +260,37 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
registrationValidtingWebhook := "managedclustervalidators.admission.cluster.open-cluster-management.io"

// Should not apply the webhook config if the replica and observed is not set
_, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, registrationValidtingWebhook, metav1.GetOptions{})
_, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(
hostedCtx, registrationValidtingWebhook, metav1.GetOptions{})
gomega.Expect(err).To(gomega.HaveOccurred())

workValidtingWebhook := "manifestworkvalidators.admission.work.open-cluster-management.io"
// Should not apply the webhook config if the replica and observed is not set
_, err = hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, workValidtingWebhook, metav1.GetOptions{})
_, err = hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(
hostedCtx, workValidtingWebhook, metav1.GetOptions{})
gomega.Expect(err).To(gomega.HaveOccurred())

updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubRegistrationWebhookDeployment)
updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubWorkWebhookDeployment)
updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubWorkControllerDeployment)
updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubAddonManagerDeployment)

gomega.Eventually(func() error {
if _, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, registrationValidtingWebhook, metav1.GetOptions{}); err != nil {
return err
}
return nil
_, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(
hostedCtx, registrationValidtingWebhook, metav1.GetOptions{})
return err
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

gomega.Expect(err).To(gomega.HaveOccurred())

gomega.Eventually(func() error {
if _, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, workValidtingWebhook, metav1.GetOptions{}); err != nil {
return err
}
return nil
_, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(
hostedCtx, workValidtingWebhook, metav1.GetOptions{})
return err
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

util.AssertClusterManagerCondition(clusterManagerName, hostedOperatorClient, "Applied", "ClusterManagerApplied", metav1.ConditionTrue)
})

ginkgo.It("should have expected resource created/deleted successfully when feature gates AddOnManager enabled/disabled", func() {
// Check addon manager default mode
gomega.Eventually(func() error {
clusterManager, err := hostedOperatorClient.OperatorV1().ClusterManagers().Get(context.Background(), clusterManagerName, metav1.GetOptions{})
if err != nil {
return err
}

// Check addon manager enabled mode
clusterManager.Spec.AddOnManagerConfiguration = &operatorapiv1.AddOnManagerConfiguration{
FeatureGates: []operatorapiv1.FeatureGate{
{Feature: "AddonManagement", Mode: operatorapiv1.FeatureGateModeTypeEnable},
},
}
_, err = hostedOperatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
return err
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

// Check clusterrole/clusterrolebinding
gomega.Eventually(func() error {
if _, err := hostedKubeClient.RbacV1().ClusterRoles().Get(context.Background(), hubAddOnManagerClusterRole, metav1.GetOptions{}); err != nil {
Expand Down Expand Up @@ -350,8 +340,15 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
return err
}

clusterManager.Spec.AddOnManagerConfiguration.FeatureGates = []operatorapiv1.FeatureGate{}
clusterManager, err = hostedOperatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
clusterManager.Spec.AddOnManagerConfiguration = &operatorapiv1.AddOnManagerConfiguration{
FeatureGates: []operatorapiv1.FeatureGate{
{
Feature: "AddonManagement",
Mode: operatorapiv1.FeatureGateModeTypeDisable,
},
},
}
_, err = hostedOperatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
return err
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

Expand Down
59 changes: 29 additions & 30 deletions test/integration/operator/clustermanager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
var hubWorkWebhookDeployment = fmt.Sprintf("%s-work-webhook", clusterManagerName)
var hubAddOnManagerDeployment = fmt.Sprintf("%s-addon-manager-controller", clusterManagerName)
var hubWorkControllerDeployment = fmt.Sprintf("%s-work-controller", clusterManagerName)
var hubAddonManagerDeployment = fmt.Sprintf("%s-addon-manager-controller", clusterManagerName)
var hubRegistrationClusterRole = fmt.Sprintf("open-cluster-management:%s-registration:controller", clusterManagerName)
var hubRegistrationWebhookClusterRole = fmt.Sprintf("open-cluster-management:%s-registration:webhook", clusterManagerName)
var hubWorkWebhookClusterRole = fmt.Sprintf("open-cluster-management:%s-work:webhook", clusterManagerName)
Expand Down Expand Up @@ -197,6 +198,14 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
}
return nil
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

gomega.Eventually(func() error {
if _, err := kubeClient.AppsV1().Deployments(hubNamespace).Get(context.Background(), hubAddonManagerDeployment, metav1.GetOptions{}); err != nil {
return err
}
return nil
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

// Check service
gomega.Eventually(func() error {
if _, err := kubeClient.CoreV1().Services(hubNamespace).Get(context.Background(), "cluster-manager-registration-webhook", metav1.GetOptions{}); err != nil {
Expand Down Expand Up @@ -249,53 +258,36 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
registrationValidtingWebhook := "managedclustervalidators.admission.cluster.open-cluster-management.io"

// Should not apply the webhook config if the replica and observed is not set
_, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), registrationValidtingWebhook, metav1.GetOptions{})
_, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(
context.Background(), registrationValidtingWebhook, metav1.GetOptions{})
gomega.Expect(err).To(gomega.HaveOccurred())
workValidtingWebhook := "manifestworkvalidators.admission.work.open-cluster-management.io"
// Should not apply the webhook config if the replica and observed is not set
_, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), workValidtingWebhook, metav1.GetOptions{})
_, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(
context.Background(), workValidtingWebhook, metav1.GetOptions{})
gomega.Expect(err).To(gomega.HaveOccurred())

// Update readyreplica of deployment

updateDeploymentStatus(kubeClient, hubNamespace, hubRegistrationWebhookDeployment)
updateDeploymentStatus(kubeClient, hubNamespace, hubWorkWebhookDeployment)
updateDeploymentStatus(kubeClient, hubNamespace, hubWorkControllerDeployment)
updateDeploymentStatus(kubeClient, hubNamespace, hubAddonManagerDeployment)

gomega.Eventually(func() error {
if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), registrationValidtingWebhook, metav1.GetOptions{}); err != nil {
return err
}
return nil
_, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(
context.Background(), registrationValidtingWebhook, metav1.GetOptions{})
return err
}, eventuallyTimeout*10, eventuallyInterval).Should(gomega.BeNil())

gomega.Eventually(func() error {
if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), workValidtingWebhook, metav1.GetOptions{}); err != nil {
return err
}
return nil
_, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(
context.Background(), workValidtingWebhook, metav1.GetOptions{})
return err
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
util.AssertClusterManagerCondition(clusterManagerName, operatorClient, "Applied", "ClusterManagerApplied", metav1.ConditionTrue)
})

ginkgo.It("should have expected resource created/deleted successfully when feature gates AddOnManager enabled/disabled", func() {
// Check addon manager default mode
gomega.Eventually(func() error {
clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get(context.Background(), clusterManagerName, metav1.GetOptions{})
if err != nil {
return err
}

// Check addon manager enabled mode
clusterManager.Spec.AddOnManagerConfiguration = &operatorapiv1.AddOnManagerConfiguration{
FeatureGates: []operatorapiv1.FeatureGate{
{Feature: "AddonManagement", Mode: operatorapiv1.FeatureGateModeTypeEnable},
},
}
_, err = operatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
return err
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

// Check clusterrole/clusterrolebinding
gomega.Eventually(func() error {
if _, err := kubeClient.RbacV1().ClusterRoles().Get(context.Background(), hubAddOnManagerClusterRole, metav1.GetOptions{}); err != nil {
Expand Down Expand Up @@ -345,8 +337,15 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
return err
}

clusterManager.Spec.AddOnManagerConfiguration.FeatureGates = []operatorapiv1.FeatureGate{}
clusterManager, err = operatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
clusterManager.Spec.AddOnManagerConfiguration = &operatorapiv1.AddOnManagerConfiguration{
FeatureGates: []operatorapiv1.FeatureGate{
{
Feature: "AddonManagement",
Mode: operatorapiv1.FeatureGateModeTypeDisable,
},
},
}
_, err = operatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
return err
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())

Expand Down
10 changes: 2 additions & 8 deletions test/integration/operator/klusterlet_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,6 @@ var _ = ginkgo.Describe("Klusterlet", func() {
},
ClusterName: "testcluster",
Namespace: klusterletNamespace,
RegistrationConfiguration: &operatorapiv1.RegistrationConfiguration{FeatureGates: []operatorapiv1.FeatureGate{
{
Feature: "AddonManagement",
Mode: "Enable",
},
}},
},
}

Expand Down Expand Up @@ -506,7 +500,7 @@ var _ = ginkgo.Describe("Klusterlet", func() {
return false
}
gomega.Expect(len(actual.Spec.Template.Spec.Containers)).Should(gomega.Equal(1))
gomega.Expect(len(actual.Spec.Template.Spec.Containers[0].Args)).Should(gomega.Equal(8))
gomega.Expect(len(actual.Spec.Template.Spec.Containers[0].Args)).Should(gomega.Equal(7))
return actual.Spec.Template.Spec.Containers[0].Args[2] == "--spoke-cluster-name=cluster2"
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

Expand Down Expand Up @@ -1003,7 +997,7 @@ var _ = ginkgo.Describe("Klusterlet", func() {
registrationDeployment, err := kubeClient.AppsV1().Deployments(klusterletNamespace).Get(
context.Background(), registrationDeploymentName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
gomega.Expect(registrationDeployment.Spec.Template.Spec.Containers[0].Args).Should(
gomega.Expect(registrationDeployment.Spec.Template.Spec.Containers[0].Args).ShouldNot(
gomega.ContainElement("--feature-gates=AddonManagement=true"))

ginkgo.By("Check the work-agent has the expected feature gates")
Expand Down
Loading

0 comments on commit 9ff0948

Please sign in to comment.