From 39b3c2e5df9e2f8ca378c8c9dd5ded9f33c2c536 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Wed, 11 Jan 2023 09:37:03 -0500 Subject: [PATCH 1/7] Add Conditions to status * Adds an initial Condition: "Ready" with a "False" status. * Fixed #73 * Include unit-tests Signed-off-by: Todd Short --- api/v1alpha1/operator_types.go | 8 ++- api/v1alpha1/zz_generated.deepcopy.go | 10 ++- ...rators.operatorframework.io_operators.yaml | 69 +++++++++++++++++++ controllers/operator_controller.go | 27 +++++++- controllers/suite_test.go | 64 ++++++++++++++++- 5 files changed, 174 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/operator_types.go b/api/v1alpha1/operator_types.go index 485806d29..f157924fb 100644 --- a/api/v1alpha1/operator_types.go +++ b/api/v1alpha1/operator_types.go @@ -27,8 +27,14 @@ type OperatorSpec struct { PackageName string `json:"packageName"` } +const ( + TypeReady = "Ready" +) + // OperatorStatus defines the observed state of Operator -type OperatorStatus struct{} +type OperatorStatus struct { + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"` +} //+kubebuilder:object:root=true //+kubebuilder:resource:scope=Cluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index cd276d734..f54900db6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,6 +22,7 @@ limitations under the License. package v1alpha1 import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -31,7 +32,7 @@ func (in *Operator) DeepCopyInto(out *Operator) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Operator. @@ -102,6 +103,13 @@ func (in *OperatorSpec) DeepCopy() *OperatorSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorStatus) DeepCopyInto(out *OperatorStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorStatus. diff --git a/config/crd/bases/operators.operatorframework.io_operators.yaml b/config/crd/bases/operators.operatorframework.io_operators.yaml index e6584b095..611fc6f9c 100644 --- a/config/crd/bases/operators.operatorframework.io_operators.yaml +++ b/config/crd/bases/operators.operatorframework.io_operators.yaml @@ -44,6 +44,75 @@ spec: type: object status: description: OperatorStatus defines the observed state of Operator + properties: + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array type: object type: object served: true diff --git a/controllers/operator_controller.go b/controllers/operator_controller.go index f96366935..84ea0c4c7 100644 --- a/controllers/operator_controller.go +++ b/controllers/operator_controller.go @@ -25,6 +25,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" operatorsv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // OperatorReconciler reconciles a Operator object @@ -47,10 +49,33 @@ type OperatorReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.1/pkg/reconcile func (r *OperatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + log := log.FromContext(ctx).WithName("reconcile") + + var operator operatorsv1alpha1.Operator + if err := r.Get(ctx, req.NamespacedName, &operator); err != nil { + log.Error(err, "unable to fetch Operator") + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + readyCondition := metav1.Condition{ + Type: operatorsv1alpha1.TypeReady, + Status: metav1.ConditionFalse, + Reason: "startUp", + Message: "Operator startup", + ObservedGeneration: operator.GetGeneration(), + } + apimeta.SetStatusCondition(&operator.Status.Conditions, readyCondition) + log.Info("set not ready Condition") // TODO(user): your logic here + // This is something that needs to happen if any Status is updated, + // even if an error occurs + if err := r.Status().Update(ctx, &operator); err != nil { + log.Error(err, "unable to update Operator status") + return ctrl.Result{}, err + } + return ctrl.Result{}, nil } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 298c2c7b5..f4331c470 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package controllers_test import ( + "context" "path/filepath" "testing" @@ -31,6 +32,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" operatorsv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" + "github.com/operator-framework/operator-controller/controllers" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" //+kubebuilder:scaffold:imports ) @@ -78,3 +83,60 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) + +var _ = Describe("Reconcile Test", func() { + When("an Operator is created", func() { + var ( + operator *operatorsv1alpha1.Operator + ctx context.Context + opName string + pkgName string + err error + ) + BeforeEach(func() { + ctx = context.Background() + + opName = "operator-test" + pkgName = "package-test" + + operator = &operatorsv1alpha1.Operator{ + ObjectMeta: metav1.ObjectMeta{ + Name: opName, + }, + Spec: operatorsv1alpha1.OperatorSpec{ + PackageName: pkgName, + }, + } + err = k8sClient.Create(ctx, operator) + Expect(err).To(Not(HaveOccurred())) + + or := controllers.OperatorReconciler{ + k8sClient, + scheme.Scheme, + } + _, err = or.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: opName, + }, + }) + Expect(err).To(Not(HaveOccurred())) + }) + AfterEach(func() { + err = k8sClient.Delete(ctx, operator) + Expect(err).To(Not(HaveOccurred())) + }) + It("has a Condition created", func() { + getOperator := &operatorsv1alpha1.Operator{} + + err = k8sClient.Get(ctx, client.ObjectKey{ + Name: opName, + }, getOperator) + Expect(err).To(Not(HaveOccurred())) + + // There should always be a "Ready" condition, regardless of Status. + conds := getOperator.Status.Conditions + Expect(conds).To(Not(BeEmpty())) + Expect(conds).To(ContainElement(HaveField("Type", operatorsv1alpha1.TypeReady))) + }) + }) +}) From 29631dc7439614e30d24d3fbefe595890b5253ea Mon Sep 17 00:00:00 2001 From: Todd Short Date: Wed, 11 Jan 2023 11:26:30 -0500 Subject: [PATCH 2/7] fixup! Add Conditions to status --- api/v1alpha1/operator_types.go | 2 ++ controllers/operator_controller.go | 56 ++++++++++++++++++++---------- controllers/suite_test.go | 12 ++++--- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/api/v1alpha1/operator_types.go b/api/v1alpha1/operator_types.go index f157924fb..5a8d761dd 100644 --- a/api/v1alpha1/operator_types.go +++ b/api/v1alpha1/operator_types.go @@ -29,6 +29,8 @@ type OperatorSpec struct { const ( TypeReady = "Ready" + + StatusNotImplemented = "NotImplemented" ) // OperatorStatus defines the observed state of Operator diff --git a/controllers/operator_controller.go b/controllers/operator_controller.go index 84ea0c4c7..6004165a5 100644 --- a/controllers/operator_controller.go +++ b/controllers/operator_controller.go @@ -19,14 +19,16 @@ package controllers import ( "context" + "k8s.io/apimachinery/pkg/api/equality" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + utilerrors "k8s.io/apimachinery/pkg/util/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" operatorsv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" - apimeta "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // OperatorReconciler reconciles a Operator object @@ -49,33 +51,51 @@ type OperatorReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.1/pkg/reconcile func (r *OperatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := log.FromContext(ctx).WithName("reconcile") + l := log.FromContext(ctx).WithName("reconcile") + l.V(1).Info("starting") + defer l.V(1).Info("ending") - var operator operatorsv1alpha1.Operator - if err := r.Get(ctx, req.NamespacedName, &operator); err != nil { - log.Error(err, "unable to fetch Operator") + var existingOp = &operatorsv1alpha1.Operator{} + if err := r.Get(ctx, req.NamespacedName, existingOp); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + reconciledOp := existingOp.DeepCopy() + res, reconcileErr := r.reconcile(ctx, reconciledOp) + + // Compare status and update if required + if !equality.Semantic.DeepEqual(existingOp.Status, reconciledOp.Status) { + if updateErr := r.Status().Update(ctx, reconciledOp); updateErr != nil { + return res, utilerrors.NewAggregate([]error{reconcileErr, updateErr}) + } + } + + // Compare resources - ignoring status - and update if required + existingOp.Status, reconciledOp.Status = operatorsv1alpha1.OperatorStatus{}, operatorsv1alpha1.OperatorStatus{} + if !equality.Semantic.DeepEqual(existingOp, reconciledOp) { + if updateErr := r.Update(ctx, reconciledOp); updateErr != nil { + return res, utilerrors.NewAggregate([]error{reconcileErr, updateErr}) + } + } + + return res, reconcileErr +} + +// Helper function to do the actual reconcile +func (r *OperatorReconciler) reconcile(ctx context.Context, op *operatorsv1alpha1.Operator) (ctrl.Result, error) { + + // TODO(user): change ReasonNotImplemented when functionality added readyCondition := metav1.Condition{ Type: operatorsv1alpha1.TypeReady, Status: metav1.ConditionFalse, - Reason: "startUp", - Message: "Operator startup", - ObservedGeneration: operator.GetGeneration(), + Reason: operatorsv1alpha1.StatusNotImplemented, + Message: "The Reconcile operation is not implemented", + ObservedGeneration: op.GetGeneration(), } - apimeta.SetStatusCondition(&operator.Status.Conditions, readyCondition) - log.Info("set not ready Condition") + apimeta.SetStatusCondition(&op.Status.Conditions, readyCondition) // TODO(user): your logic here - // This is something that needs to happen if any Status is updated, - // even if an error occurs - if err := r.Status().Update(ctx, &operator); err != nil { - log.Error(err, "unable to update Operator status") - return ctrl.Result{}, err - } - return ctrl.Result{}, nil } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index f4331c470..58a26e1dc 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -18,24 +18,26 @@ package controllers_test import ( "context" + "fmt" "path/filepath" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" operatorsv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" "github.com/operator-framework/operator-controller/controllers" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/reconcile" //+kubebuilder:scaffold:imports ) @@ -96,8 +98,8 @@ var _ = Describe("Reconcile Test", func() { BeforeEach(func() { ctx = context.Background() - opName = "operator-test" - pkgName = "package-test" + opName = fmt.Sprintf("perator-test-%s", rand.String(8)) + pkgName = fmt.Sprintf("package-test-%s", rand.String(8)) operator = &operatorsv1alpha1.Operator{ ObjectMeta: metav1.ObjectMeta{ From f3d098724987e876058d3838ddbc123fe5d76bea Mon Sep 17 00:00:00 2001 From: Todd Short Date: Wed, 11 Jan 2023 11:41:26 -0500 Subject: [PATCH 3/7] fixup! Add Conditions to status --- .github/workflows/unit-test.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/unit-test.yaml diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml new file mode 100644 index 000000000..211df5ba0 --- /dev/null +++ b/.github/workflows/unit-test.yaml @@ -0,0 +1,20 @@ +name: unit-test + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + +jobs: + e2e-kind: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-go@v3 + with: + go-version-file: "go.mod" + - name: Run unit tests + run: | + make test From 1a3d231aaea4846236b0cd425cf52e5a4f309044 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Wed, 11 Jan 2023 11:43:09 -0500 Subject: [PATCH 4/7] fixup! Add Conditions to status --- .github/workflows/unit-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 211df5ba0..02e7154c2 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -8,13 +8,13 @@ on: - main jobs: - e2e-kind: + unit-test-basic: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - uses: actions/setup-go@v3 with: go-version-file: "go.mod" - - name: Run unit tests + - name: Run basic unit tests run: | make test From 407cd2f0348989330624bd2628e58be2b28b6c34 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Wed, 11 Jan 2023 16:21:35 -0500 Subject: [PATCH 5/7] fixup! Add Conditions to status --- api/v1alpha1/operator_types.go | 4 ++++ .../crd/bases/operators.operatorframework.io_operators.yaml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/api/v1alpha1/operator_types.go b/api/v1alpha1/operator_types.go index 5a8d761dd..2207860a0 100644 --- a/api/v1alpha1/operator_types.go +++ b/api/v1alpha1/operator_types.go @@ -35,6 +35,10 @@ const ( // OperatorStatus defines the observed state of Operator type OperatorStatus struct { + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"` } diff --git a/config/crd/bases/operators.operatorframework.io_operators.yaml b/config/crd/bases/operators.operatorframework.io_operators.yaml index 611fc6f9c..894b7699c 100644 --- a/config/crd/bases/operators.operatorframework.io_operators.yaml +++ b/config/crd/bases/operators.operatorframework.io_operators.yaml @@ -113,6 +113,9 @@ spec: - type type: object type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map type: object type: object served: true From 4e05d478bebbfcb496ff01759b7faab5c4db1d88 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Fri, 13 Jan 2023 09:56:14 -0500 Subject: [PATCH 6/7] fixup! Add Conditions to status --- api/v1alpha1/operator_types.go | 4 +++- controllers/operator_controller.go | 2 +- controllers/suite_test.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/v1alpha1/operator_types.go b/api/v1alpha1/operator_types.go index 2207860a0..f39276975 100644 --- a/api/v1alpha1/operator_types.go +++ b/api/v1alpha1/operator_types.go @@ -28,9 +28,11 @@ type OperatorSpec struct { } const ( + // TODO(user): add more Types TypeReady = "Ready" - StatusNotImplemented = "NotImplemented" + // TODO(user): add more Reasons + ReasonNotImplemented = "NotImplemented" ) // OperatorStatus defines the observed state of Operator diff --git a/controllers/operator_controller.go b/controllers/operator_controller.go index 6004165a5..7ee659275 100644 --- a/controllers/operator_controller.go +++ b/controllers/operator_controller.go @@ -88,7 +88,7 @@ func (r *OperatorReconciler) reconcile(ctx context.Context, op *operatorsv1alpha readyCondition := metav1.Condition{ Type: operatorsv1alpha1.TypeReady, Status: metav1.ConditionFalse, - Reason: operatorsv1alpha1.StatusNotImplemented, + Reason: operatorsv1alpha1.ReasonNotImplemented, Message: "The Reconcile operation is not implemented", ObservedGeneration: op.GetGeneration(), } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 58a26e1dc..c67f6f236 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -98,7 +98,7 @@ var _ = Describe("Reconcile Test", func() { BeforeEach(func() { ctx = context.Background() - opName = fmt.Sprintf("perator-test-%s", rand.String(8)) + opName = fmt.Sprintf("operator-test-%s", rand.String(8)) pkgName = fmt.Sprintf("package-test-%s", rand.String(8)) operator = &operatorsv1alpha1.Operator{ From 9f797cc200b439e13adeedd1e7e3ad28734899e8 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Tue, 17 Jan 2023 10:48:43 -0500 Subject: [PATCH 7/7] fixup! Add Conditions to status --- controllers/operator_controller.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/controllers/operator_controller.go b/controllers/operator_controller.go index 7ee659275..8f05fe648 100644 --- a/controllers/operator_controller.go +++ b/controllers/operator_controller.go @@ -63,16 +63,27 @@ func (r *OperatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c reconciledOp := existingOp.DeepCopy() res, reconcileErr := r.reconcile(ctx, reconciledOp) - // Compare status and update if required - if !equality.Semantic.DeepEqual(existingOp.Status, reconciledOp.Status) { + // Do checks before any Update()s, as Update() may modify the resource structure! + updateStatus := !equality.Semantic.DeepEqual(existingOp.Status, reconciledOp.Status) + updateFinalizers := !equality.Semantic.DeepEqual(existingOp.Finalizers, reconciledOp.Finalizers) + + // Compare resources - ignoring status & metadata.finalizers + compareOp := reconciledOp.DeepCopy() + existingOp.Status, compareOp.Status = operatorsv1alpha1.OperatorStatus{}, operatorsv1alpha1.OperatorStatus{} + existingOp.Finalizers, compareOp.Finalizers = []string{}, []string{} + specDiffers := !equality.Semantic.DeepEqual(existingOp, compareOp) + + if updateStatus { if updateErr := r.Status().Update(ctx, reconciledOp); updateErr != nil { return res, utilerrors.NewAggregate([]error{reconcileErr, updateErr}) } } - // Compare resources - ignoring status - and update if required - existingOp.Status, reconciledOp.Status = operatorsv1alpha1.OperatorStatus{}, operatorsv1alpha1.OperatorStatus{} - if !equality.Semantic.DeepEqual(existingOp, reconciledOp) { + if specDiffers { + panic("spec or metadata changed by reconciler") + } + + if updateFinalizers { if updateErr := r.Update(ctx, reconciledOp); updateErr != nil { return res, utilerrors.NewAggregate([]error{reconcileErr, updateErr}) }