From c2314673829019a2c4e85d76ab573c55f5b41486 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 19 Apr 2022 08:43:54 -0400 Subject: [PATCH] Quick fixes for e2e testing --- api/v1beta1/ocicluster_webhook.go | 5 - api/v1beta1/ocimachinetemplate_webhook.go | 8 +- config/default/webhookcainjection_patch.yaml | 16 +- config/webhook/kustomizeconfig.yaml | 16 +- config/webhook/manifests.yaml | 50 - .../cluster-template-cluster-class.yaml | 28 + .../cluster-template-local-vcn-peering.yaml | 1074 ++++++++++++++++ .../cluster-template-remote-vcn-peering.yaml | 1079 +++++++++++++++++ 8 files changed, 2201 insertions(+), 75 deletions(-) create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-cluster-class.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-local-vcn-peering.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml diff --git a/api/v1beta1/ocicluster_webhook.go b/api/v1beta1/ocicluster_webhook.go index 45c20a19d..fed41f953 100644 --- a/api/v1beta1/ocicluster_webhook.go +++ b/api/v1beta1/ocicluster_webhook.go @@ -34,14 +34,9 @@ var clusterlogger = ctrl.Log.WithName("ocicluster-resource") var ( _ webhook.Validator = &OCICluster{} - _ webhook.Defaulter = &OCICluster{} ) // +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-ocicluster,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=ociclusters,versions=v1beta1,name=validation.ocicluster.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1 -// +kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-ocicluster,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=ociclusters,versions=v1beta1,name=default.ocicluster.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1 - -// Default implements webhook.Defaulter so a webhook will be registered for the type. -func (c *OCICluster) Default() {} func (c *OCICluster) SetupWebhookWithManager(mgr ctrl.Manager) error { clusterlogger.Info("validate create cluster", "name", c.Name) diff --git a/api/v1beta1/ocimachinetemplate_webhook.go b/api/v1beta1/ocimachinetemplate_webhook.go index 9a400c040..c010b44b5 100644 --- a/api/v1beta1/ocimachinetemplate_webhook.go +++ b/api/v1beta1/ocimachinetemplate_webhook.go @@ -29,14 +29,10 @@ import ( var ( _ webhook.Validator = &OCIMachineTemplate{} - _ webhook.Defaulter = &OCIMachineTemplate{} + //_ webhook.Defaulter = &OCIMachineTemplate{} ) // +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-ocimachinetemplate,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=ocimachinetemplates,versions=v1beta1,name=validation.ocimachinetemplate.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1 -// +kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-ocimachinetemplate,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=ocimachinetemplates,versions=v1beta1,name=default.ocimachinetemplate.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1 - -// Default implements webhook.Defaulter so a webhook will be registered for the type. -func (m *OCIMachineTemplate) Default() {} func (m *OCIMachineTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). @@ -84,7 +80,7 @@ func (m *OCIMachineTemplate) ValidateUpdate(old runtime.Object) error { func (m *OCIMachineTemplate) validate() field.ErrorList { var allErrs field.ErrorList - if !validOcid(m.Spec.Template.Spec.ImageId) { + if len(m.Spec.Template.Spec.ImageId) > 0 && !validOcid(m.Spec.Template.Spec.ImageId) { allErrs = append( allErrs, field.Invalid( diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml index 02ab515d4..a4f361168 100644 --- a/config/default/webhookcainjection_patch.yaml +++ b/config/default/webhookcainjection_patch.yaml @@ -1,12 +1,14 @@ # This patch add annotation to admission webhook config and # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) ---- + +# TODO: uncomment as part of https://github.com/oracle/cluster-api-provider-oci/issues/56 +#apiVersion: admissionregistration.k8s.io/v1 +#kind: MutatingWebhookConfiguration +#metadata: +# name: mutating-webhook-configuration +# annotations: +# cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +#--- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml index 15185025d..46fb4aaf1 100644 --- a/config/webhook/kustomizeconfig.yaml +++ b/config/webhook/kustomizeconfig.yaml @@ -4,18 +4,20 @@ nameReference: - kind: Service version: v1 fieldSpecs: - - kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name +# TODO: uncomment as part of https://github.com/oracle/cluster-api-provider-oci/issues/56 +# - kind: MutatingWebhookConfiguration +# group: admissionregistration.k8s.io +# path: webhooks/clientConfig/service/name - kind: ValidatingWebhookConfiguration group: admissionregistration.k8s.io path: webhooks/clientConfig/service/name namespace: - - kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true +# TODO: uncomment as part of https://github.com/oracle/cluster-api-provider-oci/issues/56 +# - kind: MutatingWebhookConfiguration +# group: admissionregistration.k8s.io +# path: webhooks/clientConfig/service/namespace +# create: true - kind: ValidatingWebhookConfiguration group: admissionregistration.k8s.io path: webhooks/clientConfig/service/namespace diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 95b140c6f..39fae6b8b 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -1,54 +1,4 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null - name: mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-ocicluster - failurePolicy: Fail - matchPolicy: Equivalent - name: default.ocicluster.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - ociclusters - sideEffects: None -- admissionReviewVersions: - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-ocimachinetemplate - failurePolicy: Fail - matchPolicy: Equivalent - name: default.ocimachinetemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - ocimachinetemplates - sideEffects: None - --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-cluster-class.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-cluster-class.yaml new file mode 100644 index 000000000..24c18505a --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-cluster-class.yaml @@ -0,0 +1,28 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-0 + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: test-cluster-class + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: ssh_authorized_keys + value: ${OCI_SSH_KEY} + - name: compartmentId + value: ${OCI_COMPARTMENT_ID} + - name: imageId + value: ${OCI_IMAGE_ID} + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-local-vcn-peering.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-local-vcn-peering.yaml new file mode 100644 index 000000000..2c1dbb939 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-local-vcn-peering.yaml @@ -0,0 +1,1074 @@ +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + stringData: + cloud-provider.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + loadBalancer: + disabled: true + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: oci-cloud-controller-manager + spec: + selector: + matchLabels: + component: oci-cloud-controller-manager + tier: control-plane + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: oci-cloud-controller-manager + tier: control-plane + spec: + serviceAccountName: cloud-controller-manager + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: cfg + secret: + secretName: oci-cloud-controller-manager + - name: kubernetes + hostPath: + path: /etc/kubernetes + containers: + - name: oci-cloud-controller-manager + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + command: ["/usr/local/bin/oci-cloud-controller-manager"] + args: + - --cloud-config=/etc/oci/cloud-provider.yaml + - --cloud-provider=oci + - --leader-elect-resource-lock=configmaps + - -v=2 + volumeMounts: + - name: cfg + mountPath: /etc/oci + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + labels: + kubernetes.io/cluster-service: "true" + rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + + - apiGroups: + - "" + resources: + - services + verbs: + - list + - watch + - patch + + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - patch + - update + + # For leader election + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + + - apiGroups: + - "" + resources: + - endpoints + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - list + - watch + - update + + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + + # For the PVL + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - watch + - patch + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: oci-cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + namespace: default +--- +apiVersion: v1 +data: + csi.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-volume-provisioner + namespace: kube-system + stringData: + config.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-controller + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-controller + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-controller + role: csi-oci + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + containers: + - name: csi-volume-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --volume-name-prefix=csi + - --feature-gates=Topology=true + - --timeout=120s + - --enable-leader-election=true + - --leader-election-type=leases + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v2.2.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --timeout=120s + - --leader-election=true + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: oci-csi-controller-driver + args: + - --v=2 + - --endpoint=unix://var/run/shared-tmpfs/csi.sock + command: + - /usr/local/bin/oci-csi-controller-driver + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + volumes: + - name: config + secret: + secretName: oci-volume-provisioner + - name: kubernetes + hostPath: + path: /etc/kubernetes + - name: shared-tmpfs + emptyDir: {} + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + --- + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: oci-csi-iscsiadm + namespace: kube-system + data: + iscsiadm: | + #!/bin/sh + if [ -x /host/sbin/iscsiadm ]; then + chroot /host /sbin/iscsiadm "$@" + elif [ -x /host/usr/local/sbin/iscsiadm ]; then + chroot /host /usr/local/sbin/iscsiadm "$@" + elif [ -x /host/bin/iscsiadm ]; then + chroot /host /bin/iscsiadm "$@" + elif [ -x /host/usr/local/bin/iscsiadm ]; then + chroot /host /usr/local/bin/iscsiadm "$@" + else + chroot /host iscsiadm "$@" + fi + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-node + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-node + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-node + role: csi-oci + spec: + containers: + - name: oci-csi-node-driver + args: + - --v=2 + - --endpoint=unix:///csi/csi.sock + - --nodeid=$(KUBE_NODE_NAME) + - --loglevel=debug + command: + - /usr/local/bin/oci-csi-node-driver + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/bin:/host/sbin + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + - mountPath: /host + name: host-root + - mountPath: /sbin/iscsiadm + name: chroot-iscsiadm + subPath: iscsiadm + - name: csi-node-registrar + args: + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + securityContext: + privileged: true + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/blockvolume.csi.oraclecloud.com /registration/blockvolume.csi.oraclecloud.com-reg.sock + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + type: "" + name: device-dir + - hostPath: + path: / + type: Directory + name: host-root + - configMap: + name: oci-csi-iscsiadm + defaultMode: 0755 + name: chroot-iscsiadm + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-csi + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-ccm-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-csi-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-csi + strategy: ApplyOnce +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCICluster + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + dns: {} + etcd: {} + kubernetesVersion: ${KUBERNETES_VERSION} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} +spec: + compartmentId: ${OCI_COMPARTMENT_ID} + networkSpec: + vcn: + cidr: 10.1.0.0/16 + networkSecurityGroups: + - egressRules: + - egressRule: + description: Kubernetes API traffic to Control Plane + destination: 10.1.0.0/29 + destinationType: CIDR_BLOCK + isStateless: false + protocol: "6" + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + ingressRules: + - ingressRule: + description: External access to Kubernetes API endpoint + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + name: control-plane-endpoint + role: control-plane-endpoint + - egressRules: + - egressRule: + description: Control Plane access to Internet + destination: 0.0.0.0/0 + destinationType: CIDR_BLOCK + isStateless: false + protocol: all + ingressRules: + - ingressRule: + description: Kubernetes API endpoint to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.0.8/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Control plane node to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Worker Node to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: etcd client communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 2379 + min: 2379 + - ingressRule: + description: etcd peer + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 2380 + min: 2380 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Inbound SSH traffic to Control Plane + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 22 + min: 22 + name: control-plane + role: control-plane + - egressRules: + - egressRule: + description: Worker node access to Internet + destination: 0.0.0.0/0 + destinationType: CIDR_BLOCK + isStateless: false + protocol: all + id: C2F829 + ingressRules: + - ingressRule: + description: Inbound SSH traffic to worker node + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 22 + min: 22 + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Control Plane to worker node Kubelet Communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 10250 + min: 10250 + - ingressRule: + description: Worker node to worker node Kubelet Communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 10250 + min: 10250 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 11.0.64.0/20 + sourceType: CIDR_BLOCK + - ingressRule: + description: Worker node to default NodePort ingress communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 32767 + min: 30000 + name: worker + role: worker + - egressRules: + - egressRule: + description: Service LoadBalancer to default NodePort egress communication + destination: 10.1.64.0/20 + destinationType: CIDR_BLOCK + isStateless: false + protocol: "6" + tcpOptions: + destinationPortRange: + max: 32767 + min: 30000 + ingressRules: + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Accept http traffic on port 80 + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 80 + min: 80 + - ingressRule: + description: Accept https traffic on port 443 + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 443 + min: 443 + name: service-lb + role: service-lb + subnets: + - cidr: 10.1.0.8/29 + name: control-plane-endpoint + role: control-plane-endpoint + type: private + - cidr: 10.1.0.0/29 + name: control-plane + role: control-plane + type: private + - cidr: 10.1.0.32/27 + name: service-lb + role: service-lb + type: public + - cidr: 10.1.64.0/20 + name: worker + role: worker + type: private + vcnPeering: + drg: + id: ${LOCAL_DRG_ID} + peerRouteRules: + - vcnCIDRRange: 10.0.0.0/16 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_CONTROL_PLANE_MACHINE_TYPE} + shapeConfig: + ocpus: "1" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_NODE_MACHINE_TYPE} + shapeConfig: + ocpus: "1" diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml new file mode 100644 index 000000000..25fc5aaa8 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml @@ -0,0 +1,1079 @@ +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + stringData: + cloud-provider.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + loadBalancer: + disabled: true + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: oci-cloud-controller-manager + spec: + selector: + matchLabels: + component: oci-cloud-controller-manager + tier: control-plane + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: oci-cloud-controller-manager + tier: control-plane + spec: + serviceAccountName: cloud-controller-manager + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: cfg + secret: + secretName: oci-cloud-controller-manager + - name: kubernetes + hostPath: + path: /etc/kubernetes + containers: + - name: oci-cloud-controller-manager + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + command: ["/usr/local/bin/oci-cloud-controller-manager"] + args: + - --cloud-config=/etc/oci/cloud-provider.yaml + - --cloud-provider=oci + - --leader-elect-resource-lock=configmaps + - -v=2 + volumeMounts: + - name: cfg + mountPath: /etc/oci + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + labels: + kubernetes.io/cluster-service: "true" + rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + + - apiGroups: + - "" + resources: + - services + verbs: + - list + - watch + - patch + + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - patch + - update + + # For leader election + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + + - apiGroups: + - "" + resources: + - endpoints + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - list + - watch + - update + + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + + # For the PVL + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - watch + - patch + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: oci-cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + namespace: default +--- +apiVersion: v1 +data: + csi.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-volume-provisioner + namespace: kube-system + stringData: + config.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-controller + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-controller + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-controller + role: csi-oci + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + containers: + - name: csi-volume-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --volume-name-prefix=csi + - --feature-gates=Topology=true + - --timeout=120s + - --enable-leader-election=true + - --leader-election-type=leases + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v2.2.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --timeout=120s + - --leader-election=true + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: oci-csi-controller-driver + args: + - --v=2 + - --endpoint=unix://var/run/shared-tmpfs/csi.sock + command: + - /usr/local/bin/oci-csi-controller-driver + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + volumes: + - name: config + secret: + secretName: oci-volume-provisioner + - name: kubernetes + hostPath: + path: /etc/kubernetes + - name: shared-tmpfs + emptyDir: {} + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + --- + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: oci-csi-iscsiadm + namespace: kube-system + data: + iscsiadm: | + #!/bin/sh + if [ -x /host/sbin/iscsiadm ]; then + chroot /host /sbin/iscsiadm "$@" + elif [ -x /host/usr/local/sbin/iscsiadm ]; then + chroot /host /usr/local/sbin/iscsiadm "$@" + elif [ -x /host/bin/iscsiadm ]; then + chroot /host /bin/iscsiadm "$@" + elif [ -x /host/usr/local/bin/iscsiadm ]; then + chroot /host /usr/local/bin/iscsiadm "$@" + else + chroot /host iscsiadm "$@" + fi + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-node + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-node + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-node + role: csi-oci + spec: + containers: + - name: oci-csi-node-driver + args: + - --v=2 + - --endpoint=unix:///csi/csi.sock + - --nodeid=$(KUBE_NODE_NAME) + - --loglevel=debug + command: + - /usr/local/bin/oci-csi-node-driver + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/bin:/host/sbin + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + - mountPath: /host + name: host-root + - mountPath: /sbin/iscsiadm + name: chroot-iscsiadm + subPath: iscsiadm + - name: csi-node-registrar + args: + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + securityContext: + privileged: true + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/blockvolume.csi.oraclecloud.com /registration/blockvolume.csi.oraclecloud.com-reg.sock + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + type: "" + name: device-dir + - hostPath: + path: / + type: Directory + name: host-root + - configMap: + name: oci-csi-iscsiadm + defaultMode: 0755 + name: chroot-iscsiadm + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-csi + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-ccm-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-csi-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-csi + strategy: ApplyOnce +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCICluster + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + dns: {} + etcd: {} + kubernetesVersion: ${KUBERNETES_VERSION} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} +spec: + compartmentId: ${OCI_COMPARTMENT_ID} + networkSpec: + vcn: + cidr: 10.1.0.0/16 + networkSecurityGroups: + - egressRules: + - egressRule: + description: Kubernetes API traffic to Control Plane + destination: 10.1.0.0/29 + destinationType: CIDR_BLOCK + isStateless: false + protocol: "6" + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + ingressRules: + - ingressRule: + description: External access to Kubernetes API endpoint + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + name: control-plane-endpoint + role: control-plane-endpoint + - egressRules: + - egressRule: + description: Control Plane access to Internet + destination: 0.0.0.0/0 + destinationType: CIDR_BLOCK + isStateless: false + protocol: all + ingressRules: + - ingressRule: + description: Kubernetes API endpoint to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.0.8/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Control plane node to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: Worker Node to Control Plane(apiserver port) communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 6443 + min: 6443 + - ingressRule: + description: etcd client communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 2379 + min: 2379 + - ingressRule: + description: etcd peer + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 2380 + min: 2380 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Inbound SSH traffic to Control Plane + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 22 + min: 22 + name: control-plane + role: control-plane + - egressRules: + - egressRule: + description: Worker node access to Internet + destination: 0.0.0.0/0 + destinationType: CIDR_BLOCK + isStateless: false + protocol: all + id: C2F829 + ingressRules: + - ingressRule: + description: Inbound SSH traffic to worker node + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 22 + min: 22 + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Control Plane to worker node Kubelet Communication + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 10250 + min: 10250 + - ingressRule: + description: Worker node to worker node Kubelet Communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 10250 + min: 10250 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking (BGP) + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 179 + min: 179 + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 10.1.0.0/29 + sourceType: CIDR_BLOCK + - ingressRule: + description: Calico networking with IP-in-IP enabled + isStateless: false + protocol: "4" + source: 11.0.64.0/20 + sourceType: CIDR_BLOCK + - ingressRule: + description: Worker node to default NodePort ingress communication + isStateless: false + protocol: "6" + source: 10.1.64.0/20 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 32767 + min: 30000 + name: worker + role: worker + - egressRules: + - egressRule: + description: Service LoadBalancer to default NodePort egress communication + destination: 10.1.64.0/20 + destinationType: CIDR_BLOCK + isStateless: false + protocol: "6" + tcpOptions: + destinationPortRange: + max: 32767 + min: 30000 + ingressRules: + - ingressRule: + description: Path discovery + icmpOptions: + code: 3 + type: 3 + isStateless: false + protocol: "1" + source: 10.1.0.0/16 + sourceType: CIDR_BLOCK + - ingressRule: + description: Accept http traffic on port 80 + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 80 + min: 80 + - ingressRule: + description: Accept https traffic on port 443 + isStateless: false + protocol: "6" + source: 0.0.0.0/0 + sourceType: CIDR_BLOCK + tcpOptions: + destinationPortRange: + max: 443 + min: 443 + name: service-lb + role: service-lb + subnets: + - cidr: 10.1.0.8/29 + name: control-plane-endpoint + role: control-plane-endpoint + type: private + - cidr: 10.1.0.0/29 + name: control-plane + role: control-plane + type: private + - cidr: 10.1.0.32/27 + name: service-lb + role: service-lb + type: public + - cidr: 10.1.64.0/20 + name: worker + role: worker + type: private + vcnPeering: + drg: + manage: true + peerRouteRules: + - vcnCIDRRange: 10.0.0.0/16 + remotePeeringConnections: + - managePeerRPC: true + peerDRGId: ${PEER_DRG_ID} + peerRegionName: ${PEER_REGION_NAME} + region: ${OCI_ALTERNATIVE_REGION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_ALTERNATIVE_REGION_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_CONTROL_PLANE_MACHINE_TYPE} + shapeConfig: + ocpus: "1" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_ALTERNATIVE_REGION_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_NODE_MACHINE_TYPE} + shapeConfig: + ocpus: "1"