diff --git a/api/v1beta1/ocicluster_webhook.go b/api/v1beta1/ocicluster_webhook.go new file mode 100644 index 000000000..fed41f953 --- /dev/null +++ b/api/v1beta1/ocicluster_webhook.go @@ -0,0 +1,110 @@ +/* + * + * Copyright (c) 2022, Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + * + */ + +package v1beta1 + +import ( + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var clusterlogger = ctrl.Log.WithName("ocicluster-resource") + +var ( + _ webhook.Validator = &OCICluster{} +) + +// +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-ocicluster,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=ociclusters,versions=v1beta1,name=validation.ocicluster.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1 + +func (c *OCICluster) SetupWebhookWithManager(mgr ctrl.Manager) error { + clusterlogger.Info("validate create cluster", "name", c.Name) + + return ctrl.NewWebhookManagedBy(mgr). + For(c). + Complete() +} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (c *OCICluster) ValidateCreate() error { + clusterlogger.Info("validate update cluster", "name", c.Name) + + var allErrs field.ErrorList + + allErrs = append(allErrs, c.validate()...) + + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid(c.GroupVersionKind().GroupKind(), c.Name, allErrs) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (c *OCICluster) ValidateDelete() error { + clusterlogger.Info("validate delete cluster", "name", c.Name) + + return nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (c *OCICluster) ValidateUpdate(old runtime.Object) error { + clusterlogger.Info("validate update cluster", "name", c.Name) + + var allErrs field.ErrorList + + oldCluster, ok := old.(*OCICluster) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected an OCICluster but got a %T", old)) + } + + if c.Spec.Region != oldCluster.Spec.Region { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "region"), c.Spec.Region, "field is immutable")) + } + + allErrs = append(allErrs, c.validate()...) + + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid(c.GroupVersionKind().GroupKind(), c.Name, allErrs) +} + +func (c *OCICluster) validate() field.ErrorList { + var allErrs field.ErrorList + + // simple validity test for compartment + if !validOcid(c.Spec.CompartmentId) { + allErrs = append( + allErrs, + field.Invalid(field.NewPath("spec", "compartmentId"), c.Spec.CompartmentId, "field is invalid")) + } + + if len(allErrs) == 0 { + return nil + } + + return allErrs +} diff --git a/api/v1beta1/ocicluster_webhook_test.go b/api/v1beta1/ocicluster_webhook_test.go new file mode 100644 index 000000000..a955b6c02 --- /dev/null +++ b/api/v1beta1/ocicluster_webhook_test.go @@ -0,0 +1,124 @@ +/* + * + * Copyright (c) 2022, Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + * + */ + +package v1beta1 + +import ( + "testing" + + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestOCICluster_ValidateCreate(t *testing.T) { + + tests := []struct { + name string + c *OCICluster + expectErr bool + }{ + { + name: "shouldn't allow bad ImageId", + c: &OCICluster{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIClusterSpec{ + CompartmentId: "badocid", + }, + }, + expectErr: true, + }, + { + name: "should succeed", + c: &OCICluster{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIClusterSpec{ + CompartmentId: "ocid", + }, + }, + expectErr: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + if test.expectErr { + g.Expect(test.c.ValidateCreate()).NotTo(gomega.Succeed()) + } else { + g.Expect(test.c.ValidateCreate()).To(gomega.Succeed()) + } + }) + } +} + +func TestOCICluster_ValidateUpdate(t *testing.T) { + tests := []struct { + name string + c *OCICluster + old *OCICluster + expectErr bool + }{ + { + name: "shouldn't region change", + c: &OCICluster{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIClusterSpec{ + Region: "new-region", + }, + }, + old: &OCICluster{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIClusterSpec{ + Region: "old-region", + }, + }, + expectErr: true, + }, + { + name: "should succeed", + c: &OCICluster{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIClusterSpec{ + CompartmentId: "ocid", + Region: "old-region", + }, + }, + old: &OCICluster{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIClusterSpec{ + Region: "old-region", + }, + }, + expectErr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + if test.expectErr { + g.Expect(test.c.ValidateUpdate(test.old)).NotTo(gomega.Succeed()) + } else { + g.Expect(test.c.ValidateUpdate(test.old)).To(gomega.Succeed()) + } + }) + } + +} diff --git a/api/v1beta1/ocimachinetemplate_webhook.go b/api/v1beta1/ocimachinetemplate_webhook.go new file mode 100644 index 000000000..3b60fd6d1 --- /dev/null +++ b/api/v1beta1/ocimachinetemplate_webhook.go @@ -0,0 +1,108 @@ +/* + * + * Copyright (c) 2022, Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + * + */ + +package v1beta1 + +import ( + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + _ webhook.Validator = &OCIMachineTemplate{} +) + +// +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-ocimachinetemplate,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=ocimachinetemplates,versions=v1beta1,name=validation.ocimachinetemplate.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1 + +func (m *OCIMachineTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(m). + Complete() +} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (m *OCIMachineTemplate) ValidateCreate() error { + clusterlogger.Info("validate create machinetemplate", "name", m.Name) + + var allErrs field.ErrorList + + allErrs = append(allErrs, m.validate()...) + + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid(m.GroupVersionKind().GroupKind(), m.Name, allErrs) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (m *OCIMachineTemplate) ValidateDelete() error { + clusterlogger.Info("validate delete machinetemplate", "name", m.Name) + + return nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (m *OCIMachineTemplate) ValidateUpdate(old runtime.Object) error { + clusterlogger.Info("validate update machinetemplate", "name", m.Name) + + var allErrs field.ErrorList + + allErrs = append(allErrs, m.validate()...) + + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid(m.GroupVersionKind().GroupKind(), m.Name, allErrs) +} + +func (m *OCIMachineTemplate) validate() field.ErrorList { + var allErrs field.ErrorList + + if len(m.Spec.Template.Spec.ImageId) > 0 && !validOcid(m.Spec.Template.Spec.ImageId) { + allErrs = append( + allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "imageId"), + m.Spec.Template.Spec.ImageId, + "field is invalid"), + ) + } + + // simple validity test for compartment + if len(m.Spec.Template.Spec.CompartmentId) > 0 && !validOcid(m.Spec.Template.Spec.CompartmentId) { + allErrs = append( + allErrs, + field.Invalid( + field.NewPath("Spec", "Template", "Spec", "compartmentId"), + m.Spec.Template.Spec.CompartmentId, + "field is invalid"), + ) + } + + if len(allErrs) == 0 { + return nil + } + + return allErrs +} diff --git a/api/v1beta1/ocimachinetemplate_webhook_test.go b/api/v1beta1/ocimachinetemplate_webhook_test.go new file mode 100644 index 000000000..23da2913b --- /dev/null +++ b/api/v1beta1/ocimachinetemplate_webhook_test.go @@ -0,0 +1,105 @@ +/* + * + * Copyright (c) 2022, Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + * + */ + +package v1beta1 + +import ( + "testing" + + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var tests = []struct { + name string + inputTemplate *OCIMachineTemplate + expectErr bool +}{ + { + name: "shouldn't allow bad ImageId", + inputTemplate: &OCIMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIMachineTemplateSpec{ + Template: OCIMachineTemplateResource{ + Spec: OCIMachineSpec{ + ImageId: "badocid", + }, + }, + }, + }, + expectErr: true, + }, + { + name: "shouldn't allow bad CompartmentId", + inputTemplate: &OCIMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIMachineTemplateSpec{ + Template: OCIMachineTemplateResource{ + Spec: OCIMachineSpec{ + CompartmentId: "badocid", + }, + }, + }, + }, + expectErr: true, + }, + { + name: "should succeed", + inputTemplate: &OCIMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: OCIMachineTemplateSpec{ + Template: OCIMachineTemplateResource{ + Spec: OCIMachineSpec{ + ImageId: "ocid", + CompartmentId: "ocid", + }, + }, + }, + }, + expectErr: false, + }, +} + +func TestOCIMachineTemplate_ValidateCreate(t *testing.T) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + if test.expectErr { + g.Expect(test.inputTemplate.ValidateCreate()).NotTo(gomega.Succeed()) + } else { + g.Expect(test.inputTemplate.ValidateCreate()).To(gomega.Succeed()) + } + }) + } +} + +func TestOCIMachineTemplate_ValidateUpdate(t *testing.T) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + if test.expectErr { + g.Expect(test.inputTemplate.ValidateUpdate(nil)).NotTo(gomega.Succeed()) + } else { + g.Expect(test.inputTemplate.ValidateUpdate(nil)).To(gomega.Succeed()) + } + }) + } +} diff --git a/api/v1beta1/webhooks.go b/api/v1beta1/webhooks.go new file mode 100644 index 000000000..afc0702c8 --- /dev/null +++ b/api/v1beta1/webhooks.go @@ -0,0 +1,29 @@ +/* + * + * Copyright (c) 2022, Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + * + */ + +package v1beta1 + +func validOcid(ocid string) bool { + + if len(ocid) >= 4 && ocid[:4] == "ocid" { + return true + } + + return false +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index f9cdf9ea6..5afe11ab2 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright (c) 2022 Oracle and/or its affiliates. +Copyright (c) 2022, Oracle and/or its affiliates. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ limitations under the License. package v1beta1 import ( - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" apiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/errors" ) diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 000000000..476be88c1 --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,24 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize \ No newline at end of file diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 000000000..48e61b9c9 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - certificate.yaml + +configurations: + - kustomizeconfig.yaml \ No newline at end of file diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 000000000..cf148912c --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: + - kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: + - kind: Certificate + group: cert-manager.io + path: spec/commonName + - kind: Certificate + group: cert-manager.io + path: spec/dnsNames + - kind: Certificate + group: cert-manager.io + path: spec/secretName \ No newline at end of file diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 5131ef34e..698803c3c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -15,15 +15,15 @@ resources: patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD -#- patches/webhook_in_ociclusters.yaml -#- patches/webhook_in_ocimachines.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch +- patches/webhook_in_ociclusters.yaml +- patches/webhook_in_ocimachines.yaml +# +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD -#- patches/cainjection_in_ociclusters.yaml -#- patches/cainjection_in_ocimachines.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch +- patches/cainjection_in_ociclusters.yaml +- patches/cainjection_in_ocimachines.yaml +# +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. configurations: diff --git a/config/crd/patches/webhook_in_ociclusters.yaml b/config/crd/patches/webhook_in_ociclusters.yaml index 713171771..c52d5b29b 100644 --- a/config/crd/patches/webhook_in_ociclusters.yaml +++ b/config/crd/patches/webhook_in_ociclusters.yaml @@ -14,3 +14,4 @@ spec: path: /convert conversionReviewVersions: - v1 + - v1beta1 diff --git a/config/crd/patches/webhook_in_ocimachines.yaml b/config/crd/patches/webhook_in_ocimachines.yaml index 0767de44a..479b8cde3 100644 --- a/config/crd/patches/webhook_in_ocimachines.yaml +++ b/config/crd/patches/webhook_in_ocimachines.yaml @@ -14,3 +14,4 @@ spec: path: /convert conversionReviewVersions: - v1 + - v1beta1 diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 3e3be1a73..979bb0a4a 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -21,9 +21,9 @@ bases: - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook +- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -41,39 +41,42 @@ patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- manager_webhook_patch.yaml +- manager_webhook_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml +- webhookcainjection_patch.yaml # the following config is for teaching kustomize how to do var substitution vars: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -#- name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -#- name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -#- name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service +- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace +- name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml +- name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace +- name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service + +configurations: + - kustomizeconfig.yaml \ No newline at end of file diff --git a/config/default/kustomizeconfig.yaml b/config/default/kustomizeconfig.yaml new file mode 100644 index 000000000..2e04a4f01 --- /dev/null +++ b/config/default/kustomizeconfig.yaml @@ -0,0 +1,4 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +varReference: + - kind: Deployment + path: spec/template/spec/volumes/secret/secretName \ No newline at end of file diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 000000000..e116610fa --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: $(SERVICE_NAME)-cert \ No newline at end of file diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 000000000..a4f361168 --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,17 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. + +# TODO: uncomment as part of https://github.com/oracle/cluster-api-provider-oci/issues/56 +#apiVersion: admissionregistration.k8s.io/v1 +#kind: MutatingWebhookConfiguration +#metadata: +# name: mutating-webhook-configuration +# annotations: +# cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +#--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 000000000..fbd74f349 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: + - manifests.yaml + - service.yaml + +configurations: + - kustomizeconfig.yaml \ No newline at end of file diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 000000000..46fb4aaf1 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,27 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: + - kind: Service + version: v1 + fieldSpecs: +# TODO: uncomment as part of https://github.com/oracle/cluster-api-provider-oci/issues/56 +# - kind: MutatingWebhookConfiguration +# group: admissionregistration.k8s.io +# path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +# TODO: uncomment as part of https://github.com/oracle/cluster-api-provider-oci/issues/56 +# - kind: MutatingWebhookConfiguration +# group: admissionregistration.k8s.io +# path: webhooks/clientConfig/service/namespace +# create: true + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: + - path: metadata/annotations \ No newline at end of file diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 000000000..39fae6b8b --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,50 @@ + +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-ocicluster + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.ocicluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ociclusters + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-ocimachinetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.ocimachinetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ocimachinetemplates + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 000000000..9290b06f9 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: webhook-server \ No newline at end of file diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index d094e7aca..8e805e6d7 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /* -Copyright (c) 2022 Oracle and/or its affiliates. +Copyright (c) 2022, Oracle and/or its affiliates. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/main.go b/main.go index a6f347dbb..b7272d96f 100644 --- a/main.go +++ b/main.go @@ -39,8 +39,10 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + webhookPort int + webhookCertDir string ) const ( @@ -58,7 +60,7 @@ func main() { var metricsAddr string var enableLeaderElection bool var probeAddr string - var webhookPort int + //var webhookPort int flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -70,6 +72,8 @@ func main() { 9443, "Webhook Server port.", ) + flag.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs/", + "Webhook cert dir, only used when webhook-port is specified.") opts := zap.Options{ Development: true, @@ -86,6 +90,7 @@ func main() { HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "controller-leader-elect-capoci", + CertDir: webhookCertDir, }) if err != nil { setupLog.Error(err, "unable to start manager") @@ -151,6 +156,17 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", scope.OCIMachineKind) os.Exit(1) } + + if err = (&infrastructurev1beta1.OCICluster{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "OCICluster") + os.Exit(1) + } + + if err = (&infrastructurev1beta1.OCIMachineTemplate{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "OCIMachineTemplate") + os.Exit(1) + } + //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-cluster-class.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-cluster-class.yaml new file mode 100644 index 000000000..24c18505a --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-cluster-class.yaml @@ -0,0 +1,28 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-0 + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: test-cluster-class + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: ssh_authorized_keys + value: ${OCI_SSH_KEY} + - name: compartmentId + value: ${OCI_COMPARTMENT_ID} + - name: imageId + value: ${OCI_IMAGE_ID} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-local-vcn-peering.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-local-vcn-peering.yaml new file mode 100644 index 000000000..2c1dbb939 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-local-vcn-peering.yaml @@ -0,0 +1,1074 @@ +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + stringData: + cloud-provider.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + loadBalancer: + disabled: true + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: oci-cloud-controller-manager + spec: + selector: + matchLabels: + component: oci-cloud-controller-manager + tier: control-plane + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: oci-cloud-controller-manager + tier: control-plane + spec: + serviceAccountName: cloud-controller-manager + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: cfg + secret: + secretName: oci-cloud-controller-manager + - name: kubernetes + hostPath: + path: /etc/kubernetes + containers: + - name: oci-cloud-controller-manager + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + command: ["/usr/local/bin/oci-cloud-controller-manager"] + args: + - --cloud-config=/etc/oci/cloud-provider.yaml + - --cloud-provider=oci + - --leader-elect-resource-lock=configmaps + - -v=2 + volumeMounts: + - name: cfg + mountPath: /etc/oci + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + labels: + kubernetes.io/cluster-service: "true" + rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + + - apiGroups: + - "" + resources: + - services + verbs: + - list + - watch + - patch + + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - patch + - update + + # For leader election + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + + - apiGroups: + - "" + resources: + - endpoints + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - list + - watch + - update + + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + + # For the PVL + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - watch + - patch + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: oci-cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + namespace: default +--- +apiVersion: v1 +data: + csi.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-volume-provisioner + namespace: kube-system + stringData: + config.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-controller + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-controller + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-controller + role: csi-oci + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + containers: + - name: csi-volume-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --volume-name-prefix=csi + - --feature-gates=Topology=true + - --timeout=120s + - --enable-leader-election=true + - --leader-election-type=leases + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v2.2.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --timeout=120s + - --leader-election=true + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: oci-csi-controller-driver + args: + - --v=2 + - --endpoint=unix://var/run/shared-tmpfs/csi.sock + command: + - /usr/local/bin/oci-csi-controller-driver + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + volumes: + - name: config + secret: + secretName: oci-volume-provisioner + - name: kubernetes + hostPath: + path: /etc/kubernetes + - name: shared-tmpfs + emptyDir: {} + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + --- + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: oci-csi-iscsiadm + namespace: kube-system + data: + iscsiadm: | + #!/bin/sh + if [ -x /host/sbin/iscsiadm ]; then + chroot /host /sbin/iscsiadm "$@" + elif [ -x /host/usr/local/sbin/iscsiadm ]; then + chroot /host /usr/local/sbin/iscsiadm "$@" + elif [ -x /host/bin/iscsiadm ]; then + chroot /host /bin/iscsiadm "$@" + elif [ -x /host/usr/local/bin/iscsiadm ]; then + chroot /host /usr/local/bin/iscsiadm "$@" + else + chroot /host iscsiadm "$@" + fi + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-node + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-node + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-node + role: csi-oci + spec: + containers: + - name: oci-csi-node-driver + args: + - --v=2 + - --endpoint=unix:///csi/csi.sock + - --nodeid=$(KUBE_NODE_NAME) + - --loglevel=debug + command: + - /usr/local/bin/oci-csi-node-driver + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/bin:/host/sbin + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + - mountPath: /host + name: host-root + - mountPath: /sbin/iscsiadm + name: chroot-iscsiadm + subPath: iscsiadm + - name: csi-node-registrar + args: + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + securityContext: + privileged: true + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/blockvolume.csi.oraclecloud.com /registration/blockvolume.csi.oraclecloud.com-reg.sock + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + type: "" + name: device-dir + - hostPath: + path: / + type: Directory + name: host-root + - configMap: + name: oci-csi-iscsiadm + defaultMode: 0755 + name: chroot-iscsiadm + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-csi + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-ccm-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-csi-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-csi + strategy: ApplyOnce +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCICluster + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + dns: {} + etcd: {} + kubernetesVersion: ${KUBERNETES_VERSION} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} +spec: + compartmentId: ${OCI_COMPARTMENT_ID} + networkSpec: + vcn: + cidr: 10.1.0.0/16 + networkSecurityGroups: + - egressRules: + - egressRule: + description: Kubernetes API traffic to Control Plane + destination: 10.1.0.0/29 + destinationType: CIDR_BLOCK + isStateless: false + protocol: "6" + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + ingressRules: + - ingressRule: + description: External access to Kubernetes API endpoint + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + name: control-plane-endpoint + role: control-plane-endpoint + - egressRules: + - egressRule: + description: Control Plane access to Internet + destination: 0.0.0.0/0 + destinationType: CIDR_BLOCK + isStateless: false + protocol: all + ingressRules: + - ingressRule: + description: Kubernetes API endpoint to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.0.8/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Control plane node to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Worker Node to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: etcd client communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 2379 + min: 2379 + - ingressRule: + description: etcd peer + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 2380 + min: 2380 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Inbound SSH traffic to Control Plane + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 22 + min: 22 + name: control-plane + role: control-plane + - egressRules: + - egressRule: + description: Worker node access to Internet + destination: 0.0.0.0/0 + destinationType: CIDR_BLOCK + isStateless: false + protocol: all + id: C2F829 + ingressRules: + - ingressRule: + description: Inbound SSH traffic to worker node + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 22 + min: 22 + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Control Plane to worker node Kubelet Communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 10250 + min: 10250 + - ingressRule: + description: Worker node to worker node Kubelet Communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 10250 + min: 10250 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 11.0.64.0/20 + sourceType: CIDR_BLOCK + - ingressRule: + description: Worker node to default NodePort ingress communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 32767 + min: 30000 + name: worker + role: worker + - egressRules: + - egressRule: + description: Service LoadBalancer to default NodePort egress communication + destination: 10.1.64.0/20 + destinationType: CIDR_BLOCK + isStateless: false + protocol: "6" + tcpOptions: + destinationPortRange: + max: 32767 + min: 30000 + ingressRules: + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Accept http traffic on port 80 + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 80 + min: 80 + - ingressRule: + description: Accept https traffic on port 443 + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 443 + min: 443 + name: service-lb + role: service-lb + subnets: + - cidr: 10.1.0.8/29 + name: control-plane-endpoint + role: control-plane-endpoint + type: private + - cidr: 10.1.0.0/29 + name: control-plane + role: control-plane + type: private + - cidr: 10.1.0.32/27 + name: service-lb + role: service-lb + type: public + - cidr: 10.1.64.0/20 + name: worker + role: worker + type: private + vcnPeering: + drg: + id: ${LOCAL_DRG_ID} + peerRouteRules: + - vcnCIDRRange: 10.0.0.0/16 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_CONTROL_PLANE_MACHINE_TYPE} + shapeConfig: + ocpus: "1" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_NODE_MACHINE_TYPE} + shapeConfig: + ocpus: "1" diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml new file mode 100644 index 000000000..25fc5aaa8 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml @@ -0,0 +1,1079 @@ +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + stringData: + cloud-provider.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + loadBalancer: + disabled: true + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: oci-cloud-controller-manager + spec: + selector: + matchLabels: + component: oci-cloud-controller-manager + tier: control-plane + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: oci-cloud-controller-manager + tier: control-plane + spec: + serviceAccountName: cloud-controller-manager + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: cfg + secret: + secretName: oci-cloud-controller-manager + - name: kubernetes + hostPath: + path: /etc/kubernetes + containers: + - name: oci-cloud-controller-manager + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + command: ["/usr/local/bin/oci-cloud-controller-manager"] + args: + - --cloud-config=/etc/oci/cloud-provider.yaml + - --cloud-provider=oci + - --leader-elect-resource-lock=configmaps + - -v=2 + volumeMounts: + - name: cfg + mountPath: /etc/oci + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + labels: + kubernetes.io/cluster-service: "true" + rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + + - apiGroups: + - "" + resources: + - services + verbs: + - list + - watch + - patch + + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - patch + - update + + # For leader election + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + + - apiGroups: + - "" + resources: + - endpoints + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - list + - watch + - update + + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + + # For the PVL + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - watch + - patch + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: oci-cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + namespace: default +--- +apiVersion: v1 +data: + csi.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-volume-provisioner + namespace: kube-system + stringData: + config.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-controller + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-controller + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-controller + role: csi-oci + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + containers: + - name: csi-volume-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --volume-name-prefix=csi + - --feature-gates=Topology=true + - --timeout=120s + - --enable-leader-election=true + - --leader-election-type=leases + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v2.2.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --timeout=120s + - --leader-election=true + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: oci-csi-controller-driver + args: + - --v=2 + - --endpoint=unix://var/run/shared-tmpfs/csi.sock + command: + - /usr/local/bin/oci-csi-controller-driver + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + volumes: + - name: config + secret: + secretName: oci-volume-provisioner + - name: kubernetes + hostPath: + path: /etc/kubernetes + - name: shared-tmpfs + emptyDir: {} + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + --- + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: oci-csi-iscsiadm + namespace: kube-system + data: + iscsiadm: | + #!/bin/sh + if [ -x /host/sbin/iscsiadm ]; then + chroot /host /sbin/iscsiadm "$@" + elif [ -x /host/usr/local/sbin/iscsiadm ]; then + chroot /host /usr/local/sbin/iscsiadm "$@" + elif [ -x /host/bin/iscsiadm ]; then + chroot /host /bin/iscsiadm "$@" + elif [ -x /host/usr/local/bin/iscsiadm ]; then + chroot /host /usr/local/bin/iscsiadm "$@" + else + chroot /host iscsiadm "$@" + fi + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-node + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-node + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-node + role: csi-oci + spec: + containers: + - name: oci-csi-node-driver + args: + - --v=2 + - --endpoint=unix:///csi/csi.sock + - --nodeid=$(KUBE_NODE_NAME) + - --loglevel=debug + command: + - /usr/local/bin/oci-csi-node-driver + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/bin:/host/sbin + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + - mountPath: /host + name: host-root + - mountPath: /sbin/iscsiadm + name: chroot-iscsiadm + subPath: iscsiadm + - name: csi-node-registrar + args: + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + securityContext: + privileged: true + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/blockvolume.csi.oraclecloud.com /registration/blockvolume.csi.oraclecloud.com-reg.sock + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + type: "" + name: device-dir + - hostPath: + path: / + type: Directory + name: host-root + - configMap: + name: oci-csi-iscsiadm + defaultMode: 0755 + name: chroot-iscsiadm + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-csi + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-ccm-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-csi-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-csi + strategy: ApplyOnce +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCICluster + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + dns: {} + etcd: {} + kubernetesVersion: ${KUBERNETES_VERSION} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} +spec: + compartmentId: ${OCI_COMPARTMENT_ID} + networkSpec: + vcn: + cidr: 10.1.0.0/16 + networkSecurityGroups: + - egressRules: + - egressRule: + description: Kubernetes API traffic to Control Plane + destination: 10.1.0.0/29 + destinationType: CIDR_BLOCK + isStateless: false + protocol: "6" + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + ingressRules: + - ingressRule: + description: External access to Kubernetes API endpoint + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + name: control-plane-endpoint + role: control-plane-endpoint + - egressRules: + - egressRule: + description: Control Plane access to Internet + destination: 0.0.0.0/0 + destinationType: CIDR_BLOCK + isStateless: false + protocol: all + ingressRules: + - ingressRule: + description: Kubernetes API endpoint to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.0.8/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Control plane node to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Worker Node to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: etcd client communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 2379 + min: 2379 + - ingressRule: + description: etcd peer + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 2380 + min: 2380 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Inbound SSH traffic to Control Plane + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 22 + min: 22 + name: control-plane + role: control-plane + - egressRules: + - egressRule: + description: Worker node access to Internet + destination: 0.0.0.0/0 + destinationType: CIDR_BLOCK + isStateless: false + protocol: all + id: C2F829 + ingressRules: + - ingressRule: + description: Inbound SSH traffic to worker node + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 22 + min: 22 + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Control Plane to worker node Kubelet Communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 10250 + min: 10250 + - ingressRule: + description: Worker node to worker node Kubelet Communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 10250 + min: 10250 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 11.0.64.0/20 + sourceType: CIDR_BLOCK + - ingressRule: + description: Worker node to default NodePort ingress communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 32767 + min: 30000 + name: worker + role: worker + - egressRules: + - egressRule: + description: Service LoadBalancer to default NodePort egress communication + destination: 10.1.64.0/20 + destinationType: CIDR_BLOCK + isStateless: false + protocol: "6" + tcpOptions: + destinationPortRange: + max: 32767 + min: 30000 + ingressRules: + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Accept http traffic on port 80 + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 80 + min: 80 + - ingressRule: + description: Accept https traffic on port 443 + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 443 + min: 443 + name: service-lb + role: service-lb + subnets: + - cidr: 10.1.0.8/29 + name: control-plane-endpoint + role: control-plane-endpoint + type: private + - cidr: 10.1.0.0/29 + name: control-plane + role: control-plane + type: private + - cidr: 10.1.0.32/27 + name: service-lb + role: service-lb + type: public + - cidr: 10.1.64.0/20 + name: worker + role: worker + type: private + vcnPeering: + drg: + manage: true + peerRouteRules: + - vcnCIDRRange: 10.0.0.0/16 + remotePeeringConnections: + - managePeerRPC: true + peerDRGId: ${PEER_DRG_ID} + peerRegionName: ${PEER_REGION_NAME} + region: ${OCI_ALTERNATIVE_REGION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_ALTERNATIVE_REGION_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_CONTROL_PLANE_MACHINE_TYPE} + shapeConfig: + ocpus: "1" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_ALTERNATIVE_REGION_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_NODE_MACHINE_TYPE} + shapeConfig: + ocpus: "1"