[ceph] prepare upgrade to rook v1.16

system/cc-ceph/Chart.lock

@@ -4,9 +4,9 @@ dependencies:
   version: 1.0.0
 - name: rook-ceph
   repository: https://charts.rook.io/release
-  version: v1.15.0
+  version: v1.16.0
 - name: rook-crds
   repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm
-  version: 0.0.2-rook.1.15.0
-digest: sha256:d8d894ba706ae4a8216ecf28bb9561fafc3b88908f183ba0b6d4b77f296e92f8
-generated: "2024-09-12T10:38:14.973622-04:00"
+  version: 0.0.2-rook.1.16.0
+digest: sha256:d9a8ec1509dcec3a634aada46cbf9434897bfa7bf85bad9adb8c6af157aa08d4
+generated: "2024-12-18T11:34:58.26072907+01:00"

system/cc-ceph/Chart.yaml

@@ -2,16 +2,16 @@ apiVersion: v2
 name: cc-ceph
 description: A Helm chart for the Rook / Ceph Objects inside the Storage Clusters
 type: application
-version: 1.1.1
-appVersion: "1.15.0"
+version: 1.1.2
+appVersion: "1.16.0"
 dependencies:
   - name: owner-info
     repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm
     version: '>= 0.0.0'
   - name: rook-ceph
     # version update should be done in the rook-crds chart as well
     repository: https://charts.rook.io/release
-    version: 1.15.0
+    version: 1.16.0
   - name: rook-crds
     repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm
-    version: '0.0.2-rook.1.15.0'
+    version: '0.0.2-rook.1.16.0'

system/cc-ceph/templates/cephobjectstore-extra.yaml

@@ -0,0 +1,110 @@
+{{- if .Values.objectstore.multiInstance.enabled }}
+apiVersion: ceph.rook.io/v1
+kind: CephObjectRealm
+metadata:
+  name: {{ .Values.objectstore.name }}
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZoneGroup
+metadata:
+  name: {{ .Values.objectstore.name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  realm: {{ .Values.objectstore.name }}
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZone
+metadata:
+  name: {{ .Values.objectstore.name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  zoneGroup: {{ .Values.objectstore.name }}
+{{- if and .Values.rgwTargetPlacements.useRookCRD .Values.rgwTargetPlacements.placements }}
+  sharedPools:
+    poolPlacements:
+{{- range $target := .Values.rgwTargetPlacements.placements }}
+      - name: {{ $target.name }} 
+        metadataPoolName: {{ $target.name }}.rgw.buckets.index
+        dataPoolName: {{ $target.name }}.rgw.buckets.data
+        dataNonECPoolName: {{ $target.name }}.rgw.buckets.non-ec
+        default: {{ $target.default | default false }}
+{{- end }}
+{{- else }}
+  metadataPool: {{ toYaml .Values.objectstore.metadataPool | nindent 4 }}
+  dataPool: {{ toYaml .Values.objectstore.dataPool | nindent 4 }}
+{{- end }}
+{{- range $instance := .Values.objectstore.multiInstance.extraInstances }}
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: {{ $instance.name }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  zone:
+    name: {{ $.Values.objectstore.name }} 
+  hosting:
+{{- if gt (len $instance.gateway.dnsNames) 0 }}
+    advertiseEndpoint:
+      dnsName: {{ $instance.gateway.dnsNames | first }}
+      port: 443
+      useTls: true
+    dnsNames: {{ toYaml $instance.gateway.dnsNames | nindent 8 }}
+{{- end }}
+  gateway:
+    instances: {{ $instance.gateway.instances | default $.Values.objectstore.gateway.instances }}
+    {{- if or $instance.gateway.port $.Values.objectstore.gateway.port }}
+    port: {{ $instance.gateway.port | default $.Values.objectstore.gateway.port }}
+    {{- end }}
+    {{- if or $instance.gateway.securePort $.Values.objectstore.gateway.securePort }}
+    securePort: {{ $instance.gateway.securePort | default $.Values.objectstore.gateway.securePort }}
+    {{- end }}
+    placement:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: kubernetes.metal.cloud.sap/role
+                  operator: In
+                  values:
+                    - {{ $.Values.osd.nodeRole }}
+      # since the CephCluster's network provider is "host", we need to isolate 80/443 port listeners from each other
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchExpressions:
+            - key: app
+              operator: In
+              values:
+              - rook-ceph-rgw
+          topologyKey: kubernetes.io/hostname
+    priorityClassName: system-cluster-critical
+    sslCertificateRef: {{ $instance.gateway.sslCertificateRef | default $.Values.objectstore.gateway.sslCertificateRef }}
+    resources: {{ toYaml ( $instance.gateway.resources | default $.Values.objectstore.gateway.resources) | nindent 6 }}
+  preservePoolsOnDelete: true
+{{- if and $.Values.objectstore.keystone.enabled }}
+{{- with $.Values.objectstore.keystone }}
+  auth:
+    keystone:
+      acceptedRoles:
+{{- range $_, $role := .accepted_roles }}
+        - {{ $role }}
+{{- end }}
+      implicitTenants: {{ .implicit_tenants | quote }}
+      serviceUserSecretName: ceph-keystone-secret
+      tokenCacheSize: {{ .token_cache_size }}
+      url: {{ .url }}
+  protocols:
+{{- if $instance.enabledAPIs }}
+    enableAPIs: {{ toYaml $instance.enabledAPIs | nindent 6 }}
+{{- end }}
+    s3:
+      authUseKeystone: true
+    swift:
+      accountInUrl: {{ .swift_account_in_url }}
+      versioningEnabled: {{ .swift_versioning_enabled }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

system/cc-ceph/templates/cephobjectstore-placement-pools.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.rgwTargetPlacements.enabled }}
 {{- range $target := .Values.rgwTargetPlacements.placements }}
 ---
 apiVersion: ceph.rook.io/v1
@@ -80,7 +79,6 @@ spec:
     nodelete: {{ $.Values.pool.nodelete | quote }}
     nosizechange: {{ $.Values.pool.nosizechange | quote }}
 {{- end }}
-{{- end }}
 {{- if .Values.rgwTargetPlacements.premiumPlacements }}
 {{- range $target := .Values.rgwTargetPlacements.premiumPlacements }}
 ---

system/cc-ceph/templates/cephobjectstore.yaml

@@ -4,8 +4,23 @@ metadata:
   name: {{ .Values.objectstore.name }}
   namespace: {{ .Release.Namespace }}
 spec:
+{{- if .Values.objectstore.multiInstance.enabled }}
+  zone:
+    name: {{ .Values.objectstore.name }} 
+{{- else if and .Values.rgwTargetPlacements.useRookCRD .Values.rgwTargetPlacements.placements }}
+  sharedPools:
+    poolPlacements:
+{{- range $target := .Values.rgwTargetPlacements.placements }}
+      - name: {{ $target.name }} 
+        metadataPoolName: {{ $target.name }}.rgw.buckets.index
+        dataPoolName: {{ $target.name }}.rgw.buckets.data
+        dataNonECPoolName: {{ $target.name }}.rgw.buckets.non-ec
+        default: {{ $target.default | default false }}
+{{- end }}
+{{- else }}
   metadataPool: {{ toYaml .Values.objectstore.metadataPool | nindent 4 }}
   dataPool: {{ toYaml .Values.objectstore.dataPool | nindent 4 }}
+{{- end }}
   hosting:
 {{- if gt (len .Values.objectstore.gateway.dnsNames) 0 }}
     advertiseEndpoint:
@@ -60,6 +75,9 @@ spec:
       tokenCacheSize: {{ .token_cache_size }}
       url: {{ .url }}
   protocols:
+{{- if $.Values.objectstore.enabledAPIs }}
+    enableAPIs: {{ toYaml $.Values.objectstore.enabledAPIs | nindent 6 }}
+{{- end }}
     s3:
       authUseKeystone: true
     swift:

system/cc-ceph/templates/certificate-extra.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.objectstore.multiInstance.enabled }}
+{{- range $instance := .Values.objectstore.multiInstance.extraInstances }}
+{{- range $key, $record := $instance.gateway.dnsNames }}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ $record }}
+spec:
+  dnsNames:
+    - "*.{{ $record }}"
+    - "{{ $record }}"
+  uris:
+    - rook-ceph-rgw-{{ $instance.name }}.rook-ceph.svc
+  issuerRef:
+    group: certmanager.cloud.sap
+    kind: ClusterIssuer
+    name: digicert-issuer
+  secretName: {{ $instance.gateway.sslCertificateRef }}
+  usages:
+    - digital signature
+    - key encipherment
+{{- end }}
+{{- end }}
+{{- end }}

system/cc-ceph/templates/record-extra.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.objectstore.multiInstance.enabled }}
+{{- range $instance := .Values.objectstore.multiInstance.extraInstances }}
+{{- range $key, $record := $instance.gateway.dnsNames }}
+---
+apiVersion: disco.stable.sap.cc/v1
+kind: Record
+metadata:
+  name: "{{ $record }}"
+spec:
+  type: A
+  record: {{ $instance.service.externalIP }}
+  hosts:
+    - "{{ $record }}."
+---
+apiVersion: disco.stable.sap.cc/v1
+kind: Record
+metadata:
+  name: "{{ $record }}-wildcard"
+spec:
+  type: CNAME
+  record: "{{ $record }}."
+  hosts:
+    - "*.{{ $record }}."
+{{- end }}
+{{- end }}
+{{- end }}

system/cc-ceph/templates/service-extra.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.objectstore.multiInstance.enabled }}
+{{- range $instance := .Values.objectstore.multiInstance.extraInstances }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $instance.service.name }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  externalIPs:
+  - {{ $instance.service.externalIP }}
+  type: NodePort
+  sessionAffinity: None
+  externalTrafficPolicy: Local
+  ports:
+    - port: {{ $instance.service.port }}
+      targetPort: {{ $instance.service.port }}
+      protocol: TCP
+      name: rgw-ssl
+  selector:
+    app: {{ $instance.service.selector.app }}
+    rook_cluster: {{ $instance.service.selector.rook_cluster }}
+    rook_object_store: {{ $instance.name }}
+{{- end }}
+{{- end }}

system/cc-ceph/values.yaml

@@ -90,6 +90,7 @@ dashboard:
 objectstore:
   enabled: true
   name: objectstore
+  enabledAPIs: [] # empty - all enabled. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis
   gateway:
     instances: 6
     port: 80
@@ -149,6 +150,30 @@ objectstore:
       password: XXX
       domain: XXX
       project: XXX
+  multiInstance:
+    enabled: false
+    extraInstances:
+      # can inherit/override all config options from objectstore:
+    # - name: objectstore-admin
+    #   gateway:
+    #     instances: 2
+    #     sslCertificateRef: ""
+    #     dnsNames: 
+    #     - dns1-adm
+    #     - dns2-adm
+    #     resources:
+    #       requests:
+    #         cpu: 1
+    #         memory: 2Gi
+    #   service:
+    #     name: ceph-objectstore-admin-external
+    #     port: 443
+    #     externalIP: "10.0.0.1"
+    #     selector:
+    #       app: rgw
+    # define other RGW instances here:
+    # - name: other-instance-name
+
   prysm:
     enabled: true
     repository:
@@ -158,7 +183,8 @@ objectstore:
     interval: "10"
 
 rgwTargetPlacements:
-  enabled: false
+  #  enabled: false deprecate rgwTargetPlacements.enabled because it is true on all envs
+  useRookCRD: false # !!!WARNING set 'true' only for new clusters. Upgrade will not work now.
 
 defaultRgwPools:
   enabled: false # create default rgw pools, see: https://github.com/sapcc/helm-charts/issues/6670