allocate metrics port for multi runtime

charts/juicefs/templates/fuse/daemonset.yaml

@@ -125,9 +125,11 @@ spec:
             successThreshold: 1
             timeoutSeconds: 1
           ports:
-          - containerPort: 9567
+          {{- if .Values.fuse.metricsPort }}
+          - containerPort: {{ .Values.fuse.metricsPort }}
             name: metrics
             protocol: TCP
+          {{- end }}
           securityContext:
             runAsUser: 0
           {{- if .Values.fuse.privileged }}

charts/juicefs/templates/worker/statefuleset.yaml

@@ -82,9 +82,11 @@ spec:
           {{- if .Values.worker.privileged }}
             privileged: true
           {{- end }}
-          {{- if .Values.worker.ports }}
           ports:
-{{ toYaml .Values.worker.ports | trim | indent 10  }}
+          {{- if .Values.worker.metricsPort }}
+            - containerPort: {{ .Values.worker.metricsPort }}
+              name: metrics
+              protocol: TCP
           {{- end }}
           env:
           - name: JFS_FOREGROUND

charts/juicefs/values.yaml

@@ -37,13 +37,13 @@ worker:
   imagePullPolicy: ""
   nodeSelector: ""
   cacheDir: ""
-  ports: []
   envs: []
   command: ""
   mountPath: /mnt/jfs
   statCmd: "stat -c %i /mnt/jfs"
   podManagementPolicy: Parallel
   hostNetwork: false
+  metricsPort: 9567
   resources:
     requests:
 #      cpu: "0.5"
@@ -57,6 +57,7 @@ worker:
   volumes: []
   volumeMounts: []
 
+
 configs:
   name: ""
   accesskeySecret: ""
@@ -75,6 +76,7 @@ configs:
 
 fuse:
   hostNetwork: false
+  metricsPort: 9567
   subPath: ""
   criticalPod: false
   enabled: true

cmd/juicefs/app/juicefs.go

@@ -19,16 +19,19 @@ package app
 import (
 	"os"
 
-	"github.com/fluid-cloudnative/fluid/pkg/utils"
 	"github.com/spf13/cobra"
 	zapOpt "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/net"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
+	"github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator"
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
+
 	"github.com/fluid-cloudnative/fluid"
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	juicefsctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/juicefs"
@@ -47,8 +50,10 @@ var (
 	enableLeaderElection    bool
 	leaderElectionNamespace string
 	development             bool
+	portRange               string
 	maxConcurrentReconciles int
 	pprofAddr               string
+	portAllocatePolicy      string
 )
 
 var startCmd = &cobra.Command{
@@ -69,6 +74,8 @@ func init() {
 	startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results")
 	startCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.")
 	startCmd.Flags().BoolVar(&eventDriven, "event-driven", true, "The reconciler's loop strategy. if it's false, it indicates period driven.")
+	startCmd.Flags().StringVar(&portRange, "runtime-node-port-range", "14000-15999", "Set available port range for JuiceFS")
+	startCmd.Flags().StringVar(&portAllocatePolicy, "port-allocate-policy", "random", "Set port allocating policy, available choice is bitmap or random(default random).")
 }
 
 func handle() {
@@ -116,6 +123,20 @@ func handle() {
 		os.Exit(1)
 	}
 
+	pr, err := net.ParsePortRange(portRange)
+	if err != nil {
+		setupLog.Error(err, "can't parse port range. Port range must be like <min>-<max>")
+		os.Exit(1)
+	}
+	setupLog.Info("port range parsed", "port range", pr.String())
+
+	err = portallocator.SetupRuntimePortAllocator(mgr.GetClient(), pr, portAllocatePolicy, juicefs.GetReservedPorts)
+	if err != nil {
+		setupLog.Error(err, "failed to setup runtime port allocator")
+		os.Exit(1)
+	}
+	setupLog.Info("Set up runtime port allocator", "policy", portAllocatePolicy)
+
 	setupLog.Info("starting juicefsruntime-controller")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem juicefsruntime-controller")

pkg/ddc/jindo/types.go

@@ -17,8 +17,9 @@ limitations under the License.
 package jindo
 
 import (
-	"github.com/fluid-cloudnative/fluid/pkg/common"
 	v1 "k8s.io/api/core/v1"
+
+	"github.com/fluid-cloudnative/fluid/pkg/common"
 )
 
 type Jindo struct {

pkg/ddc/juicefs/const.go

@@ -31,9 +31,10 @@ const (
 
 	PodRoleType = "role"
 
-	WorkerPodRole     = "juicefs-worker"
-	EnterpriseEdition = "enterprise"
-	CommunityEdition  = "community"
+	WorkerPodRole      = "juicefs-worker"
+	EnterpriseEdition  = "enterprise"
+	CommunityEdition   = "community"
+	DefaultMetricsPort = 9567
 
 	MetadataSyncNotDoneMsg               = "[Calculating]"
 	CheckMetadataSyncDoneTimeoutMillisec = 500

pkg/ddc/juicefs/data_load_test.go

@@ -27,7 +27,7 @@ import (
 var valuesConfigMapData = `
 cacheDirs:
   "1":
-    path: /var/foo
+    path: /jfs/cache3:/jfs/cache4
     type: hostPath
 configs:
   accesskeySecret: jfs-secret
@@ -46,8 +46,9 @@ edition: community
 fsGroup: 0
 fullnameOverride: jfsdemo
 fuse:
+  metricsPort: 14001
   command: /bin/mount.juicefs ${METAURL} /runtime-mnt/juicefs/default/jfsdemo/juicefs-fuse
-    -o attr-cache=7200,entry-cache=7200,metrics=0.0.0.0:9567,cache-size=1024,free-space-ratio=0.1,cache-dir=/var/foo,ro
+    -o metrics=0.0.0.0:9567,cache-size=1024,free-space-ratio=0.1,cache-dir=/jfs/cache3:/jfs/cache4
   criticalPod: true
   enabled: true
   hostMountPath: /runtime-mnt/juicefs/default/jfsdemo
@@ -62,11 +63,11 @@ fuse:
   resources: {}
   statCmd: stat -c %i /runtime-mnt/juicefs/default/jfsdemo/juicefs-fuse
   volumeMounts:
-  - mountPath: /var/foo
+  - mountPath: /jfs/cache3:/jfs/cache4
     name: cache-dir-1
   volumes:
   - hostPath:
-      path: /var/foo
+      path: /jfs/cache3:/jfs/cache4
       type: DirectoryOrCreate
     name: cache-dir-1
 group: 0
@@ -81,27 +82,28 @@ owner:
   enabled: true
   kind: JuiceFSRuntime
   name: jfsdemo
-  uid: 7bf75683-c4cd-4f18-9344-3adde1799250
+  uid: 9ae3312b-d5b6-4a3d-895c-7712bfa7d74e
 placement: Exclusive
 runtimeIdentity:
   name: jfsdemo
   namespace: default
 source: ${METAURL}
 user: 0
 worker:
+  metricsPort: 14000
   command: /bin/mount.juicefs ${METAURL} /runtime-mnt/juicefs/default/jfsdemo/juicefs-fuse
-    -o cache-size=1024,free-space-ratio=0.1,cache-dir=/var/foo,ro,metrics=0.0.0.0:9567
+    -o cache-size=1024,free-space-ratio=0.1,cache-dir=/jfs/cache1:/jfs/cache2,metrics=0.0.0.0:9567
   hostNetwork: true
   mountPath: /runtime-mnt/juicefs/default/jfsdemo/juicefs-fuse
   privileged: true
   resources: {}
   statCmd: stat -c %i /runtime-mnt/juicefs/default/jfsdemo/juicefs-fuse
   volumeMounts:
-  - mountPath: /var/foo
+  - mountPath: /jfs/cache1:/jfs/cache2
     name: cache-dir-1
   volumes:
   - hostPath:
-      path: /var/foo
+      path: /jfs/cache1:/jfs/cache2
       type: DirectoryOrCreate
     name: cache-dir-1
 `

pkg/ddc/juicefs/master_internal_test.go

@@ -21,16 +21,20 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/fluid-cloudnative/fluid/pkg/utils/kubectl"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/net"
+
+	"github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/kubectl"
 
 	"github.com/brahma-adshonor/gohook"
-	"github.com/fluid-cloudnative/fluid/pkg/utils/fake"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 
+	"github.com/fluid-cloudnative/fluid/pkg/utils/fake"
+
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	"github.com/fluid-cloudnative/fluid/pkg/utils/helm"
 )
@@ -65,6 +69,9 @@ func TestSetupMasterInternal(t *testing.T) {
 	mockExecInstallReleaseErr := func(name string, namespace string, valueFile string, chartName string) error {
 		return errors.New("fail to install dataload chart")
 	}
+	mockExecCreateConfigMapFromFileErr := func(name string, key, fileName string, namespace string) (err error) {
+		return errors.New("fail to exec command")
+	}
 
 	wrappedUnhookCheckRelease := func() {
 		err := gohook.UnHook(helm.CheckRelease)
@@ -134,10 +141,17 @@ func TestSetupMasterInternal(t *testing.T) {
 			},
 		},
 	}
-	////portallocator.SetupRuntimePortAllocator(client, &net.PortRange{Base: 10, Size: 100}, GetReservedPorts)
+	err := portallocator.SetupRuntimePortAllocator(client, &net.PortRange{Base: 10, Size: 100}, "bitmap", GetReservedPorts)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	err = gohook.Hook(kubectl.CreateConfigMapFromFile, mockExecCreateConfigMapFromFileErr, nil)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
 
 	// check release found
-	err := gohook.Hook(helm.CheckRelease, mockExecCheckReleaseCommonFound, nil)
+	err = gohook.Hook(helm.CheckRelease, mockExecCheckReleaseCommonFound, nil)
 	if err != nil {
 		t.Fatal(err.Error())
 	}

pkg/ddc/juicefs/port_parser.go

@@ -0,0 +1,84 @@
+/*
+  Copyright 2023 The Fluid Authors.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+package juicefs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+
+	"github.com/fluid-cloudnative/fluid/api/v1alpha1"
+	"github.com/fluid-cloudnative/fluid/pkg/common"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/kubeclient"
+)
+
+// GetReservedPorts defines restoration logic for JuiceFSRuntime
+func GetReservedPorts(client client.Client) (ports []int, err error) {
+	var datasets v1alpha1.DatasetList
+	err = client.List(context.TODO(), &datasets)
+	if err != nil {
+		return nil, errors.Wrap(err, "can't list datasets when GetReservedPorts")
+	}
+
+	for _, dataset := range datasets.Items {
+		if len(dataset.Status.Runtimes) != 0 {
+			// Assume there is only one runtime with category "Accelerate"
+			accelerateRuntime := dataset.Status.Runtimes[0]
+			if accelerateRuntime.Type != common.JuiceFSRuntime {
+				continue
+			}
+			configMapName := fmt.Sprintf("%s-%s-values", accelerateRuntime.Name, accelerateRuntime.Type)
+			configMap, err := kubeclient.GetConfigmapByName(client, configMapName, accelerateRuntime.Namespace)
+			if err != nil {
+				return nil, errors.Wrap(err, "GetConfigMapByName when GetReservedPorts")
+			}
+
+			if configMap == nil {
+				continue
+			}
+
+			reservedPorts, err := parsePortsFromConfigMap(configMap)
+			if err != nil {
+				return nil, errors.Wrap(err, "parsePortsFromConfigMap when GetReservedPorts")
+			}
+			ports = append(ports, reservedPorts...)
+		}
+	}
+	return ports, nil
+}
+
+// parsePortsFromConfigMap extracts port usage information given a configMap
+func parsePortsFromConfigMap(configMap *corev1.ConfigMap) (ports []int, err error) {
+	var value JuiceFS
+	if v, ok := configMap.Data["data"]; ok {
+		if err := yaml.Unmarshal([]byte(v), &value); err != nil {
+			return nil, err
+		}
+
+		if value.Worker.HostNetwork && value.Worker.MetricsPort != nil {
+			ports = append(ports, *value.Worker.MetricsPort)
+		}
+		if value.Fuse.HostNetwork && value.Fuse.MetricsPort != nil {
+			ports = append(ports, *value.Fuse.MetricsPort)
+		}
+	}
+	return ports, nil
+}