From 694b3309c8b5b8163814d221c23b4138b8267503 Mon Sep 17 00:00:00 2001 From: prateekpandey14 Date: Wed, 12 May 2021 20:29:28 +0530 Subject: [PATCH] fix(cvc): set the bind address using a go flag in cvc server config - Remove the entrypoint.sh script in Dockerfile , used to set the bind address for cvc api server (backup/restore endpoints) reset/overwrite the other flags passed as args to the deployment container. - Updated the cvc operator deployment spec args to include the --bind flag to set the server IP using downward API status.podIP - Updated the cstor-operator service account name changes as per the latest release changes. - Sync with latest changes in node-disk-manager operator yaml Signed-off-by: prateekpandey14 --- build/cvc-operator/Dockerfile | 4 +- build/cvc-operator/Dockerfile.arm64 | 2 +- build/cvc-operator/cvc-operator.Dockerfile | 4 +- build/cvc-operator/entrypoint.sh | 21 - deploy/cstor-operator.yaml | 14 +- deploy/ndm-operator.yaml | 801 ++++++++++++++++++--- deploy/rbac.yaml | 6 +- pkg/controllers/cstorvolumeconfig/start.go | 6 +- 8 files changed, 706 insertions(+), 152 deletions(-) delete mode 100644 build/cvc-operator/entrypoint.sh diff --git a/build/cvc-operator/Dockerfile b/build/cvc-operator/Dockerfile index 46965f1f..40c87c98 100644 --- a/build/cvc-operator/Dockerfile +++ b/build/cvc-operator/Dockerfile @@ -23,8 +23,6 @@ RUN apk add --no-cache \ ca-certificates COPY cvc-operator /usr/local/bin/cvc-operator -COPY entrypoint.sh /usr/local/bin/ -RUN chmod +x /usr/local/bin/entrypoint.sh ARG ARCH ARG DBUILD_DATE @@ -38,5 +36,5 @@ LABEL org.label-schema.build-date=$DBUILD_DATE LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL LABEL org.label-schema.url=$DBUILD_SITE_URL -ENTRYPOINT entrypoint.sh +ENTRYPOINT ["/usr/local/bin/cvc-operator"] EXPOSE 5757 diff --git a/build/cvc-operator/Dockerfile.arm64 b/build/cvc-operator/Dockerfile.arm64 index 7a6046c2..c8b7bb0a 100644 --- a/build/cvc-operator/Dockerfile.arm64 +++ b/build/cvc-operator/Dockerfile.arm64 @@ -41,5 +41,5 @@ LABEL org.label-schema.build-date=$DBUILD_DATE LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL LABEL org.label-schema.url=$DBUILD_SITE_URL -ENTRYPOINT entrypoint.sh +ENTRYPOINT ["/usr/local/bin/cvc-operator"] EXPOSE 5757 diff --git a/build/cvc-operator/cvc-operator.Dockerfile b/build/cvc-operator/cvc-operator.Dockerfile index d6cd4ba4..7a23ec12 100644 --- a/build/cvc-operator/cvc-operator.Dockerfile +++ b/build/cvc-operator/cvc-operator.Dockerfile @@ -64,8 +64,6 @@ LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL LABEL org.label-schema.url=$DBUILD_SITE_URL COPY --from=build /go/src/github.com/openebs/cstor-operator/bin/cvc-operator/cvc-operator /usr/local/bin/cvc-operator -COPY --from=build /go/src/github.com/openebs/cstor-operator/build/cvc-operator/entrypoint.sh /usr/local/bin/ -RUN chmod +x /usr/local/bin/entrypoint.sh -ENTRYPOINT entrypoint.sh +ENTRYPOINT ["/usr/local/bin/cvc-operator"] EXPOSE 5757 diff --git a/build/cvc-operator/entrypoint.sh b/build/cvc-operator/entrypoint.sh deleted file mode 100644 index b95a1caa..00000000 --- a/build/cvc-operator/entrypoint.sh +++ /dev/null @@ -1,21 +0,0 @@ -#/* -#Copyright 2020 The OpenEBS Authors -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -#*/ -#!/bin/sh - -set -ex - -CVC_API_SERVER_NETWORK="eth0" - -CONTAINER_IP_ADDR=$(ip -4 addr show scope global dev "${CVC_API_SERVER_NETWORK}" | grep inet | awk '{print $2}' | cut -d / -f 1) - -exec /usr/local/bin/cvc-operator --bind="${CONTAINER_IP_ADDR}" 1>&2 diff --git a/deploy/cstor-operator.yaml b/deploy/cstor-operator.yaml index 3bca97b9..727b9eb9 100644 --- a/deploy/cstor-operator.yaml +++ b/deploy/cstor-operator.yaml @@ -23,7 +23,7 @@ spec: openebs.io/component-name: cspc-operator openebs.io/version: dev spec: - serviceAccountName: openebs-maya-operator + serviceAccountName: openebs-cstor-operator containers: - name: cspc-operator imagePullPolicy: IfNotPresent @@ -85,11 +85,15 @@ spec: openebs.io/component-name: cvc-operator openebs.io/version: dev spec: - serviceAccountName: openebs-maya-operator + serviceAccountName: openebs-cstor-operator containers: - name: cvc-operator imagePullPolicy: IfNotPresent image: openebs/cvc-operator:ci + args: + - "--v=5" + - "--leader-election=false" + - "--bind=$(OPENEBS_CVC_POD_IP)" env: # OPENEBS_IO_BASE_DIR is used to configure base directory for openebs on host path. # Where OpenEBS can store required files. Default base path will be /var/openebs @@ -104,6 +108,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPENEBS_CVC_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: OPENEBS_SERVICEACCOUNT_NAME valueFrom: fieldRef: @@ -156,7 +164,7 @@ spec: openebs.io/component-name: cstor-admission-webhook openebs.io/version: dev spec: - serviceAccountName: openebs-maya-operator + serviceAccountName: openebs-cstor-operator containers: - name: admission-webhook image: openebs/cstor-webhook:ci diff --git a/deploy/ndm-operator.yaml b/deploy/ndm-operator.yaml index 1177c3de..fa8c273c 100644 --- a/deploy/ndm-operator.yaml +++ b/deploy/ndm-operator.yaml @@ -1,16 +1,487 @@ -# This is the node-disk-manager related config. -# It can be used to customize the disks probes and filters +# This manifest is autogenerated via 'make manifests' command. +# Do the modification to the yamls in deploy/yamls/ directory +# and then run 'make manifests' command + +# This manifest deploys the OpenEBS NDM components with associated RBAC rules and crds. + + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + name: blockdevices.openebs.io +spec: + group: openebs.io + names: + kind: BlockDevice + listKind: BlockDeviceList + plural: blockdevices + shortNames: + - bd + singular: blockdevice + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.nodeAttributes.nodeName + name: NodeName + type: string + - jsonPath: .spec.path + name: Path + priority: 1 + type: string + - jsonPath: .spec.filesystem.fsType + name: FSType + priority: 1 + type: string + - jsonPath: .spec.capacity.storage + name: Size + type: string + - jsonPath: .status.claimState + name: ClaimState + type: string + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: BlockDevice is the Schema used to represent a BlockDevice CR + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeviceSpec defines the properties and runtime status of a + BlockDevice + properties: + aggregateDevice: + description: AggregateDevice was intended to store the hierachical + information in cases of LVM. However this is currently not implemented + and may need to be re-looked into for better design. To be deprecated + type: string + capacity: + description: Capacity + properties: + logicalSectorSize: + description: LogicalSectorSize is blockdevice logical-sector size + in bytes + format: int32 + type: integer + physicalSectorSize: + description: PhysicalSectorSize is blockdevice physical-Sector + size in bytes + format: int32 + type: integer + storage: + description: Storage is the blockdevice capacity in bytes + format: int64 + type: integer + required: + - storage + type: object + claimRef: + description: ClaimRef is the reference to the BDC which has claimed + this BD + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + details: + description: Details contain static attributes of BD like model,serial, + and so forth + properties: + compliance: + description: Compliance is standards/specifications version implemented + by device firmware such as SPC-1, SPC-2, etc + type: string + deviceType: + description: DeviceType represents the type of device like sparse, + disk, partition, lvm, crypt + enum: + - disk + - partition + - sparse + - loop + - lvm + - crypt + - dm + - mpath + type: string + driveType: + description: DriveType is the type of backing drive, HDD/SSD + enum: + - HDD + - SSD + - Unknown + - "" + type: string + firmwareRevision: + description: FirmwareRevision is the disk firmware revision + type: string + hardwareSectorSize: + description: HardwareSectorSize is the hardware sector size in + bytes + format: int32 + type: integer + logicalBlockSize: + description: LogicalBlockSize is the logical block size in bytes + reported by /sys/class/block/sda/queue/logical_block_size + format: int32 + type: integer + model: + description: Model is model of disk + type: string + physicalBlockSize: + description: PhysicalBlockSize is the physical block size in bytes + reported by /sys/class/block/sda/queue/physical_block_size + format: int32 + type: integer + serial: + description: Serial is serial number of disk + type: string + vendor: + description: Vendor is vendor of disk + type: string + type: object + devlinks: + description: DevLinks contains soft links of a block device like /dev/by-id/... + /dev/by-uuid/... + items: + description: DeviceDevLink holds the mapping between type and links + like by-id type or by-path type link + properties: + kind: + description: Kind is the type of link like by-id or by-path. + enum: + - by-id + - by-path + type: string + links: + description: Links are the soft links + items: + type: string + type: array + type: object + type: array + filesystem: + description: FileSystem contains mountpoint and filesystem type + properties: + fsType: + description: Type represents the FileSystem type of the block + device + type: string + mountPoint: + description: MountPoint represents the mountpoint of the block + device. + type: string + type: object + nodeAttributes: + description: NodeAttributes has the details of the node on which BD + is attached + properties: + nodeName: + description: NodeName is the name of the Kubernetes node resource + on which the device is attached + type: string + type: object + parentDevice: + description: "ParentDevice was intended to store the UUID of the parent + Block Device as is the case for partitioned block devices. \n For + example: /dev/sda is the parent for /dev/sda1 To be deprecated" + type: string + partitioned: + description: Partitioned represents if BlockDevice has partitions + or not (Yes/No) Currently always default to No. To be deprecated + enum: + - "Yes" + - "No" + type: string + path: + description: Path contain devpath (e.g. /dev/sdb) + type: string + required: + - capacity + - devlinks + - nodeAttributes + - path + type: object + status: + description: DeviceStatus defines the observed state of BlockDevice + properties: + claimState: + description: ClaimState represents the claim state of the block device + enum: + - Claimed + - Unclaimed + - Released + type: string + state: + description: State is the current state of the blockdevice (Active/Inactive/Unknown) + enum: + - Active + - Inactive + - Unknown + type: string + required: + - claimState + - state + type: object + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + name: blockdeviceclaims.openebs.io +spec: + group: openebs.io + names: + kind: BlockDeviceClaim + listKind: BlockDeviceClaimList + plural: blockdeviceclaims + shortNames: + - bdc + singular: blockdeviceclaim + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.blockDeviceName + name: BlockDeviceName + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: BlockDeviceClaim is the Schema for the BlockDeviceClaim CR + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeviceClaimSpec defines the request details for a BlockDevice + properties: + blockDeviceName: + description: BlockDeviceName is the reference to the block-device + backing this claim + type: string + blockDeviceNodeAttributes: + description: BlockDeviceNodeAttributes is the attributes on the node + from which a BD should be selected for this claim. It can include + nodename, failure domain etc. + properties: + hostName: + description: HostName represents the hostname of the Kubernetes + node resource where the BD should be present + type: string + nodeName: + description: NodeName represents the name of the Kubernetes node + resource where the BD should be present + type: string + type: object + deviceClaimDetails: + description: Details of the device to be claimed + properties: + allowPartition: + description: AllowPartition represents whether to claim a full + block device or a device that is a partition + type: boolean + blockVolumeMode: + description: 'BlockVolumeMode represents whether to claim a device + in Block mode or Filesystem mode. These are use cases of BlockVolumeMode: + 1) Not specified: VolumeMode check will not be effective 2) + VolumeModeBlock: BD should not have any filesystem or mountpoint + 3) VolumeModeFileSystem: BD should have a filesystem and mountpoint. + If DeviceFormat is specified then the format should match + with the FSType in BD' + type: string + formatType: + description: Format of the device required, eg:ext4, xfs + type: string + type: object + deviceType: + description: DeviceType represents the type of drive like SSD, HDD + etc., + nullable: true + type: string + hostName: + description: Node name from where blockdevice has to be claimed. To + be deprecated. Use NodeAttributes.HostName instead + type: string + resources: + description: Resources will help with placing claims on Capacity, + IOPS + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum resources required. + eg: if storage resource of 10G is requested minimum capacity + of 10G should be available TODO for validating' + type: object + required: + - requests + type: object + selector: + description: Selector is used to find block devices to be considered + for claiming + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + type: object + status: + description: DeviceClaimStatus defines the observed state of BlockDeviceClaim + properties: + phase: + description: Phase represents the current phase of the claim + type: string + required: + - phase + type: object + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Create the OpenEBS namespace +apiVersion: v1 +kind: Namespace +metadata: + name: openebs +--- +# Create NDM configmap apiVersion: v1 kind: ConfigMap metadata: - name: openebs-ndm-config + name: node-disk-manager-config namespace: openebs - labels: - openebs.io/component-name: ndm-config data: - # udev-probe is default or primary probe which should be enabled to run ndm - # filterconfigs contails configs of filters - in their form fo include - # and exclude comma separated strings + # node-disk-manager-config contains config of available probes and filters. + # Probes and Filters will initialize with default values if config for that + # filter or probe are not present in configmap + + # udev-probe is default or primary probe it should be enabled to run ndm + # filterconfigs contains configs of filters. To provide a group of include + # and exclude values add it as , separated string node-disk-manager.config: | probeconfigs: - key: udev-probe @@ -18,7 +489,7 @@ data: state: true - key: seachest-probe name: seachest probe - state: false + state: true - key: smart-probe name: smart probe state: true @@ -36,58 +507,93 @@ data: name: path filter state: true include: "" - exclude: "/dev/loop,/dev/fd0,/dev/sr0,/dev/ram,/dev/dm-,/dev/md,/dev/rbd,/dev/zd" + exclude: "/dev/loop,/dev/fd0,/dev/sr0,/dev/ram,/dev/md,/dev/dm-,/dev/rbd,/dev/zd" +--- +# Create NDM Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openebs-maya-operator + namespace: openebs +--- +# Define Role that allows operations on K8s pods/deployments +# in "openebs" namespace +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: openebs + name: openebs-ndm-operator +rules: +- apiGroups: ["*"] + resources: ["nodes", "pods", "services", "endpoints", "events", "configmaps", "secrets", "jobs"] + verbs: + - '*' +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: + - '*' +- apiGroups: + - openebs.io + resources: + - blockdevices + - blockdeviceclaims + verbs: + - '*' --- +# Bind the Service Account with the Role Privileges. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-ndm-operator + namespace: openebs +subjects: +- kind: ServiceAccount + name: openebs-maya-operator + namespace: openebs +roleRef: + kind: ClusterRole + name: openebs-ndm-operator + apiGroup: rbac.authorization.k8s.io +--- +# Create NDM daemonset apiVersion: apps/v1 kind: DaemonSet metadata: - name: openebs-ndm + name: node-disk-manager namespace: openebs - labels: - name: openebs-ndm - openebs.io/component-name: ndm - openebs.io/version: 2.6.0 spec: - selector: - matchLabels: - name: openebs-ndm - openebs.io/component-name: ndm updateStrategy: type: RollingUpdate + selector: + matchLabels: + name: node-disk-manager template: metadata: labels: - name: openebs-ndm - openebs.io/component-name: ndm - openebs.io/version: 2.6.0 + name: node-disk-manager spec: # By default the node-disk-manager will be run on all kubernetes nodes # If you would like to limit this to only some nodes, say the nodes - # that have storage attached, you could label those node and use - # nodeSelector. - # - # e.g. label the storage nodes with - "openebs.io/nodegroup"="storage-node" + # that have storage attached, you could label those node and use nodeSelector. + # Example: Label the storage nodes with - "openebs.io/nodegroup"="storage-node" # kubectl label node "openebs.io/nodegroup"="storage-node" - #nodeSelector: - # "openebs.io/nodegroup": "storage-node" - serviceAccountName: openebs-maya-operator + # nodeSelector: + # "openebs.io/nodegroup": "storage-node" + # Use host network as container network to monitor udev source using netlink + # to detect disk attach and detach events using fd. hostNetwork: true - # host PID is used to check status of iSCSI Service when the NDM - # API service is enabled - #hostPID: true + hostPID: true + serviceAccountName: openebs-maya-operator containers: - name: node-disk-manager - image: openebs/node-disk-manager:1.2.0 + image: openebs/node-disk-manager:ci args: - - -v=4 - # The feature-gate is used to enable the new UUID algorithm. - - --feature-gates="GPTBasedUUID" - # The feature gate is used to start the gRPC API service. The gRPC server - # starts at 9115 port by default. This feature is currently in Alpha state - # - --feature-gates="APIService" - # The feature gate is used to enable NDM, to create blockdevice resources - # for unused partitions on the OS disk - # - --feature-gates="UseOSDisk" + - -v=4 + - --feature-gates="GPTBasedUUID" + - --feature-gates="APIService" + # Default address is 0.0.0.0:9115, do not use quotes around the address + # - --api-service-address=0.0.0.0:9115 + - --feature-gates="UseOSDisk" imagePullPolicy: IfNotPresent securityContext: privileged: true @@ -96,6 +602,7 @@ spec: mountPath: /host/node-disk-manager.config subPath: node-disk-manager.config readOnly: true + # make udev database available inside container - name: udev mountPath: /run/udev - name: procmount @@ -123,36 +630,32 @@ spec: # if not specified, then sparse files will not be created. - name: SPARSE_FILE_DIR value: "/var/openebs/sparse" - # Size(bytes) of the sparse file to be created. + # Size of the sparse file to be created. - name: SPARSE_FILE_SIZE - value: "10737418240" + value: "1073741824" # Specify the number of sparse files to be created - name: SPARSE_FILE_COUNT - value: "1" - livenessProbe: - exec: - command: - - pgrep - - "ndm" - initialDelaySeconds: 30 - periodSeconds: 60 + value: "5" + # Set the core dump env to enable core dump for NDM daemon + #- name: ENABLE_COREDUMP + # value: "1" volumes: - name: config configMap: - name: openebs-ndm-config + name: node-disk-manager-config - name: udev hostPath: path: /run/udev type: Directory - # mount /proc (to access mount file of process 1 of host) inside container - # to read mount-point of disks and partitions - name: procmount + # mount /proc/1/mounts (mount file of process 1 of host) inside container + # to read which partition is mounted on / path hostPath: path: /proc type: Directory - name: devmount - # the /dev directory is mounted so that we have access to the devices that - # are connected at runtime of the pod. + # the /dev directory is mounted so that we have access to the devices that + # are connected at runtime of the pod. hostPath: path: /dev type: Directory @@ -164,77 +667,147 @@ spec: hostPath: path: /var/openebs/sparse --- +# Create NDM operator deployment apiVersion: apps/v1 kind: Deployment metadata: - name: openebs-ndm-operator + name: node-disk-operator namespace: openebs - labels: - name: openebs-ndm-operator - openebs.io/component-name: ndm-operator - openebs.io/version: 2.6.0 spec: + replicas: 1 + strategy: + type: Recreate selector: matchLabels: - name: openebs-ndm-operator - openebs.io/component-name: ndm-operator + name: node-disk-operator + template: + metadata: + labels: + name: node-disk-operator + spec: + serviceAccountName: openebs-maya-operator + containers: + - name: node-disk-operator + image: openebs/node-disk-operator:ci + ports: + - containerPort: 8080 + name: liveness + imagePullPolicy: IfNotPresent + readinessProbe: + exec: + command: + - stat + - /tmp/operator-sdk-ready + initialDelaySeconds: 4 + periodSeconds: 10 + failureThreshold: 1 + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # the service account of this pod + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_NAME + value: "node-disk-operator" + - name: CLEANUP_JOB_IMAGE + value: "openebs/linux-utils:ci" + - name: OPENEBS_IO_INSTALL_CRD + value: "false" + # OPENEBS_IO_IMAGE_PULL_SECRETS environment variable is used to pass the image pull secrets + # to the cleanup pod launched by NDM operator + #- name: OPENEBS_IO_IMAGE_PULL_SECRETS + # value: "" +--- +# Create NDM cluster exporter deployment. +# This is an optional component and is not required for the basic +# functioning of NDM +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ndm-cluster-exporter + namespace: openebs +spec: replicas: 1 strategy: type: Recreate + selector: + matchLabels: + name: ndm-cluster-exporter template: metadata: labels: - name: openebs-ndm-operator - openebs.io/component-name: ndm-operator - openebs.io/version: 2.6.0 + name: ndm-cluster-exporter spec: serviceAccountName: openebs-maya-operator containers: - - name: node-disk-operator - image: openebs/node-disk-operator:1.2.0 - imagePullPolicy: IfNotPresent - readinessProbe: - exec: - command: - - stat - - /tmp/operator-sdk-ready - initialDelaySeconds: 4 - periodSeconds: 10 - failureThreshold: 1 - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # the service account of the ndm-operator pod - - name: SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: OPERATOR_NAME - value: "node-disk-operator" - - name: CLEANUP_JOB_IMAGE - value: "openebs/linux-utils:2.6.0" - # OPENEBS_IO_INSTALL_CRD environment variable is used to enable/disable CRD installation - # from NDM operator. By default the CRDs will be installed - #- name: OPENEBS_IO_INSTALL_CRD - # value: "true" - # OPENEBS_IO_IMAGE_PULL_SECRETS environment variable is used to pass the image pull secrets - # to the cleanup pod launched by NDM operator - #- name: OPENEBS_IO_IMAGE_PULL_SECRETS - # value: "" - # Process name used for matching is limited to the 15 characters - # present in the pgrep output. - # So fullname can be used here with pgrep (cmd is < 15 chars). - livenessProbe: - exec: - command: - - pgrep - - "ndo" - initialDelaySeconds: 30 - periodSeconds: 60 + - name: ndm-cluster-exporter + image: openebs/node-disk-exporter:ci + command: + - /usr/local/bin/exporter + args: + - "start" + - "--mode=cluster" + - "--port=:9100" + - "--metrics=/metrics" + ports: + - containerPort: 9100 + protocol: TCP + name: metrics + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +--- +# Create NDM node exporter daemonset. +# This is an optional component used for getting disk level +# metrics from each of the storage nodes +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: ndm-node-exporter + namespace: openebs +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + name: ndm-node-exporter + template: + metadata: + labels: + name: ndm-node-exporter + spec: + serviceAccountName: openebs-maya-operator + containers: + - name: node-disk-exporter + image: openebs/node-disk-exporter:ci + command: + - /usr/local/bin/exporter + args: + - "start" + - "--mode=node" + - "--port=:9101" + - "--metrics=/metrics" + ports: + - containerPort: 9101 + protocol: TCP + name: metrics + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace --- diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index eeaaadc5..3e0e7f6b 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -8,7 +8,7 @@ metadata: apiVersion: v1 kind: ServiceAccount metadata: - name: openebs-maya-operator + name: openebs-cstor-operator namespace: openebs --- # Define Role that allows operations on K8s pods/deployments @@ -66,7 +66,7 @@ metadata: name: openebs-cstor-operator subjects: - kind: ServiceAccount - name: openebs-maya-operator + name: openebs-cstor-operator namespace: openebs roleRef: kind: ClusterRole @@ -95,7 +95,7 @@ metadata: name: openebs-cstor-migration subjects: - kind: ServiceAccount - name: openebs-maya-operator + name: openebs-cstor-operator namespace: openebs roleRef: kind: ClusterRole diff --git a/pkg/controllers/cstorvolumeconfig/start.go b/pkg/controllers/cstorvolumeconfig/start.go index 4ba0b175..25878a43 100644 --- a/pkg/controllers/cstorvolumeconfig/start.go +++ b/pkg/controllers/cstorvolumeconfig/start.go @@ -21,10 +21,6 @@ import ( "flag" "os" "os/signal" - - "github.com/pkg/errors" - "k8s.io/klog" - "time" clientset "github.com/openebs/api/v2/pkg/client/clientset/versioned" @@ -33,10 +29,12 @@ import ( server "github.com/openebs/cstor-operators/pkg/server" cvcserver "github.com/openebs/cstor-operators/pkg/server/cstorvolumeconfig" "github.com/openebs/cstor-operators/pkg/snapshot" + "github.com/pkg/errors" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" ) var (