test: Add FVT tests for PVC storage

Related: #230, #267
Signed-off-by: Christian Kadner <ckadner@us.ibm.com>

.github/workflows/run-fvt.yml

@@ -79,7 +79,7 @@ jobs:
         run: |
           docker images
           kubectl get pods
-          kubectl get servingruntimes
+          kubectl get clusterservingruntimes
       - name: Run FVTs
         run: |
           go install github.com/onsi/ginkgo/v2/ginkgo

Makefile

@@ -46,8 +46,10 @@ test:
 	go test -coverprofile cover.out `go list ./... | grep -v fvt`
 
 # Run fvt tests. This requires an etcd, kubernetes connection, and model serving installation. Ginkgo CLI is used to run them in parallel
+# TODO: reinstate all FVT suites
 fvt:
-	ginkgo -v -p -progress --fail-fast fvt/predictor fvt/scaleToZero --timeout=40m
+	@#ginkgo -v -p --fail-fast fvt/predictor fvt/scaleToZero fvt/storage --timeout=40m
+	ginkgo --fail-fast fvt/storage --timeout=40m --succinct
 
 # Command to regenerate the grpc go files from the proto files
 fvt-protoc:

config/dependencies/fvt.yaml

@@ -123,3 +123,130 @@ stringData:
       "default_bucket": "modelmesh-example-models",
       "region": "us-south"
     }
+  pvc1: |
+    {
+      "type": "pvc",
+      "name": "models-pvc-1"
+    }
+  pvc2: |
+    {
+      "type": "pvc",
+      "name": "models-pvc-2"
+    }
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "models-pvc-1"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "models-pvc-2"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "models-pvc-3"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "pvc-init"
+spec:
+  template:
+    metadata:
+      name: "pvc-init-pod"
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: "copy-pod"
+          image: kserve/modelmesh-minio-examples:latest
+          securityContext:
+            runAsUser: 0
+            allowPrivilegeEscalation: false
+          command: ["/bin/sh", "-ex", "-c"]
+          args:
+            - echo copy model files ...;
+              whoami;
+              ls -al "${SRC_FOLDER}";
+              cp -r "${SRC_FOLDER}"/* "${DST_FOLDER_1}" &&
+              cp -r "${SRC_FOLDER}"/* "${DST_FOLDER_2}" &&
+              cp -r "${SRC_FOLDER}"/* "${DST_FOLDER_3}" &&
+              ls -al "${DST_FOLDER_1}" &&
+              ls -al "${DST_FOLDER_2}" &&
+              ls -al "${DST_FOLDER_3}" &&
+              echo done &&
+              exit 0;
+          env:
+            - name: SRC_FOLDER
+              value: "/data1/modelmesh-example-models"
+            - name: DST_FOLDER_1
+              value: "/mnt/pvc1"
+            - name: DST_FOLDER_2
+              value: "/mnt/pvc2"
+            - name: DST_FOLDER_3
+              value: "/mnt/pvc3"
+          volumeMounts:
+            - name: "pvc1"
+              mountPath: "/mnt/pvc1"
+            - name: "pvc2"
+              mountPath: "/mnt/pvc2"
+            - name: "pvc3"
+              mountPath: "/mnt/pvc3"
+      volumes:
+        - name: "pvc1"
+          persistentVolumeClaim:
+            claimName: "models-pvc-1"
+        - name: "pvc2"
+          persistentVolumeClaim:
+            claimName: "models-pvc-2"
+        - name: "pvc3"
+          persistentVolumeClaim:
+            claimName: "models-pvc-3"
+  backoffLimit: 4
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "pvc-reader"
+spec:
+  containers:
+    - name: main
+      image: ubuntu
+      command: ["/bin/sh", "-ec", "sleep 10000"]
+      volumeMounts:
+        - name: "pvc1"
+          mountPath: "/mnt/pvc1"
+        - name: "pvc2"
+          mountPath: "/mnt/pvc2"
+        - name: "pvc3"
+          mountPath: "/mnt/pvc3"
+  volumes:
+    - name: "pvc1"
+      persistentVolumeClaim:
+        claimName: "models-pvc-1"
+    - name: "pvc2"
+      persistentVolumeClaim:
+        claimName: "models-pvc-2"
+    - name: "pvc3"
+      persistentVolumeClaim:
+        claimName: "models-pvc-3"

config/dependencies/minio-storage-secret.yaml

@@ -1,5 +1,17 @@
+# Copyright 2021 IBM Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 apiVersion: v1
-
 kind: Secret
 metadata:
   name: storage-config

config/manager/kustomization.yaml

@@ -17,5 +17,4 @@ resources:
 images:
   - name: modelmesh-controller
     newName: kserve/modelmesh-controller
-    ## NOTE THIS SHOULD BE REPLACED WITH LATEST CONTROLLER IMAGE TAG
     newTag: latest

config/runtimes/ovms-1.x.yaml

@@ -57,3 +57,6 @@ spec:
     runtimeManagementPort: 8888
     memBufferBytes: 134217728
     modelLoadingTimeoutMillis: 90000
+
+  # TODO: re-enable OpenVino runtime
+  disabled: true

config/runtimes/triton-2.x.yaml

@@ -93,3 +93,6 @@ spec:
     runtimeManagementPort: 8001
     memBufferBytes: 134217728
     modelLoadingTimeoutMillis: 90000
+
+  # TODO: re-enable Triton runtime
+  disabled: true

docs/quickstart.md

@@ -270,7 +270,8 @@ To see more detailed instructions and information, click [here](./predictors/run
 
 ## 4. (Optional) Deleting your ModelMesh Serving installation
 
-To delete all ModelMesh Serving resources that were installed, run the following from the root of the project:
+To delete all ModelMesh Serving resources that were installed, run the following
+command from the root of the project:
 
 ```shell
 ./scripts/delete.sh --namespace modelmesh-serving

fvt/README.md

@@ -33,7 +33,7 @@ If starting with a fresh namespace, install ModelMesh Serving configured for the
 ./scripts/install.sh --namespace modelmesh-serving --fvt --dev-mode-logging
 ```
 
-To re-configure an existing quick-start instance for FVTs, run:
+To re-configure an existing "quickstart" deployment for FVTs, run:
 
 ```Shell
 kubectl apply -f config/dependencies/fvt.yaml

fvt/fvtclient.go

@@ -57,8 +57,8 @@ import (
 	torchserveapi "github.com/kserve/modelmesh-serving/fvt/generated/torchserve/apis"
 )
 
-const predictorTimeout = time.Second * 120
-const timeForStatusToStabilize = time.Second * 5
+const PredictorTimeout = time.Second * 180
+const timeForStatusToStabilize = time.Second * 60
 
 type ModelServingConnectionType int
 
@@ -374,6 +374,13 @@ func (fvt *FVTClient) PrintIsvcs() {
 	}
 }
 
+func (fvt *FVTClient) PrintDescribeIsvc(name string) {
+	err := fvt.RunKubectl("describe", "isvc", name)
+	if err != nil {
+		fvt.log.Error(err, fmt.Sprintf("Error running describe isvc '%s' command", name))
+	}
+}
+
 func (fvt *FVTClient) PrintPods() {
 	err := fvt.RunKubectl("get", "pods")
 	if err != nil {
@@ -412,12 +419,12 @@ func (fvt *FVTClient) TailPodLogs(sinceTime string) {
 
 func (fvt *FVTClient) RunKubectl(args ...string) error {
 	args = append(args, "-n", fvt.namespace)
-	getPredictorCommand := exec.Command("kubectl", args...)
-	getPredictorCommand.Stdout = ginkgo.GinkgoWriter
-	getPredictorCommand.Stderr = ginkgo.GinkgoWriter
-	fvt.log.Info("Running command", "args", strings.Join(getPredictorCommand.Args, " "))
+	kubectlCmd := exec.Command("kubectl", args...)
+	kubectlCmd.Stdout = ginkgo.GinkgoWriter
+	kubectlCmd.Stderr = ginkgo.GinkgoWriter
+	fvt.log.Info("Running command", "args", strings.Join(kubectlCmd.Args, " "))
 	fmt.Fprintf(ginkgo.GinkgoWriter, "=====================================================================================================================================\n")
-	err := getPredictorCommand.Run()
+	err := kubectlCmd.Run()
 	fmt.Fprintf(ginkgo.GinkgoWriter, "=====================================================================================================================================\n")
 	return err
 }
@@ -504,11 +511,11 @@ func (fvt *FVTClient) ConnectToModelServing(connectionType ModelServingConnectio
 	}
 
 	if err := fvt.grpcPortForward.EnsureStarted(); err != nil {
-		return fmt.Errorf("Error with grpc port-forward, could not connect to model serving")
+		return fmt.Errorf("Error with gRPC port-forward, could not connect to model serving")
 	}
 
 	if err := fvt.restPortForward.EnsureStarted(); err != nil {
-		return fmt.Errorf("Error with rest port-forward, could not connect to model serving")
+		return fmt.Errorf("Error with REST port-forward, could not connect to model serving")
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)