Merge pull request #1337 from kubernetes-sigs/fix-GetVolumeStats-win

fix: incorrect volume usage on Windows node with host process deployment
kubernetes-sigs · Aug 16, 2023 · 6ad4199 · 6ad4199
2 parents 4046d86 + 45f8576
commit 6ad4199
Show file tree

Hide file tree

Showing 6 changed files with 135 additions and 49 deletions.
diff --git a/go.mod b/go.mod
@@ -41,6 +41,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
 	github.com/jongio/azidext/go/azidext v0.5.0
 	github.com/onsi/ginkgo/v2 v2.11.0
+	golang.org/x/sys v0.11.0
 	k8s.io/pod-security-admission v0.27.4
 )
 
@@ -132,7 +133,6 @@ require (
 	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/oauth2 v0.7.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/term v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect

diff --git a/pkg/azurefile/azure_common_darwin.go b/pkg/azurefile/azure_common_darwin.go
@@ -20,6 +20,10 @@ limitations under the License.
 package azurefile
 
 import (
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
 	mount "k8s.io/mount-utils"
 )
 
@@ -38,3 +42,8 @@ func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
 func prepareStagePath(path string, m *mount.SafeFormatAndMount) error {
 	return nil
 }
+
+// GetVolumeStats returns volume stats based on the given path.
+func GetVolumeStats(path string, enableWindowsHostProcess bool) (*csi.NodeGetVolumeStatsResponse, error) {
+	return nil, status.Errorf(codes.Internal, "GetVolumeStats is not supported on darwin")
+}
diff --git a/pkg/azurefile/azure_common_linux.go b/pkg/azurefile/azure_common_linux.go
@@ -20,7 +20,13 @@ limitations under the License.
 package azurefile
 
 import (
+	"github.com/container-storage-interface/spec/lib/go/csi"
+
+	"k8s.io/kubernetes/pkg/volume"
 	mount "k8s.io/mount-utils"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func SMBMount(m *mount.SafeFormatAndMount, source, target, fsType string, options, sensitiveMountOptions []string) error {
@@ -38,3 +44,54 @@ func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
 func prepareStagePath(path string, m *mount.SafeFormatAndMount) error {
 	return nil
 }
+
+// GetVolumeStats returns volume stats based on the given path.
+func GetVolumeStats(path string, enableWindowsHostProcess bool) (*csi.NodeGetVolumeStatsResponse, error) {
+	volumeMetrics, err := volume.NewMetricsStatFS(path).GetMetrics()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err)
+	}
+
+	available, ok := volumeMetrics.Available.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume available size(%v)", volumeMetrics.Available)
+	}
+	capacity, ok := volumeMetrics.Capacity.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume capacity size(%v)", volumeMetrics.Capacity)
+	}
+	used, ok := volumeMetrics.Used.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume used size(%v)", volumeMetrics.Used)
+	}
+
+	inodesFree, ok := volumeMetrics.InodesFree.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes free(%v)", volumeMetrics.InodesFree)
+	}
+	inodes, ok := volumeMetrics.Inodes.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes(%v)", volumeMetrics.Inodes)
+	}
+	inodesUsed, ok := volumeMetrics.InodesUsed.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes used(%v)", volumeMetrics.InodesUsed)
+	}
+
+	return &csi.NodeGetVolumeStatsResponse{
+		Usage: []*csi.VolumeUsage{
+			{
+				Unit:      csi.VolumeUsage_BYTES,
+				Available: available,
+				Total:     capacity,
+				Used:      used,
+			},
+			{
+				Unit:      csi.VolumeUsage_INODES,
+				Available: inodesFree,
+				Total:     inodes,
+				Used:      inodesUsed,
+			},
+		},
+	}, nil
+}
diff --git a/pkg/azurefile/azure_common_windows.go b/pkg/azurefile/azure_common_windows.go
@@ -22,7 +22,12 @@ package azurefile
 import (
 	"fmt"
 
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"golang.org/x/sys/windows"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/volume"
 	mount "k8s.io/mount-utils"
 	"sigs.k8s.io/azurefile-csi-driver/pkg/mounter"
 )
@@ -69,3 +74,62 @@ func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
 func prepareStagePath(path string, m *mount.SafeFormatAndMount) error {
 	return removeDir(path, m)
 }
+
+// GetFreeSpace returns the free space of the volume in bytes, total size of the volume in bytes and the used space of the volume in bytes
+func GetFreeSpace(path string) (int64, int64, int64, error) {
+	var totalNumberOfBytes, totalNumberOfFreeBytes uint64
+	dirName := windows.StringToUTF16Ptr(path)
+	if err := windows.GetDiskFreeSpaceEx(dirName, nil, &totalNumberOfBytes, &totalNumberOfFreeBytes); err != nil {
+		return 0, 0, 0, err
+	}
+	return int64(totalNumberOfFreeBytes), int64(totalNumberOfBytes), int64(totalNumberOfBytes - totalNumberOfFreeBytes), nil
+}
+
+// GetVolumeStats returns volume stats based on the given path.
+func GetVolumeStats(path string, enableWindowsHostProcess bool) (*csi.NodeGetVolumeStatsResponse, error) {
+	if enableWindowsHostProcess {
+		// only in host process mode, we can get the free space of the volume, otherwise, we will get permission denied error
+		freeBytesAvailable, totalBytes, totalBytesUsed, err := GetFreeSpace(path)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "failed to get free space on path %s: %v", path, err)
+		}
+		return &csi.NodeGetVolumeStatsResponse{
+			Usage: []*csi.VolumeUsage{
+				{
+					Unit:      csi.VolumeUsage_BYTES,
+					Available: freeBytesAvailable,
+					Total:     totalBytes,
+					Used:      totalBytesUsed,
+				},
+			},
+		}, nil
+	}
+
+	volumeMetrics, err := volume.NewMetricsStatFS(path).GetMetrics()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err)
+	}
+
+	available, ok := volumeMetrics.Available.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume available size(%v)", volumeMetrics.Available)
+	}
+	capacity, ok := volumeMetrics.Capacity.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume capacity size(%v)", volumeMetrics.Capacity)
+	}
+	used, ok := volumeMetrics.Used.AsInt64()
+	if !ok {
+		return nil, status.Errorf(codes.Internal, "failed to transform volume used size(%v)", volumeMetrics.Used)
+	}
+	return &csi.NodeGetVolumeStatsResponse{
+		Usage: []*csi.VolumeUsage{
+			{
+				Unit:      csi.VolumeUsage_BYTES,
+				Available: available,
+				Total:     capacity,
+				Used:      used,
+			},
+		},
+	}, nil
+}
diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go
@@ -29,7 +29,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
-	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 
 	"google.golang.org/grpc/codes"
@@ -432,53 +431,7 @@ func (d *Driver) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeS
 		return nil, status.Errorf(codes.Internal, "failed to stat file %s: %v", req.VolumePath, err)
 	}
 
-	volumeMetrics, err := volume.NewMetricsStatFS(req.VolumePath).GetMetrics()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err)
-	}
-
-	available, ok := volumeMetrics.Available.AsInt64()
-	if !ok {
-		return nil, status.Errorf(codes.Internal, "failed to transform volume available size(%v)", volumeMetrics.Available)
-	}
-	capacity, ok := volumeMetrics.Capacity.AsInt64()
-	if !ok {
-		return nil, status.Errorf(codes.Internal, "failed to transform volume capacity size(%v)", volumeMetrics.Capacity)
-	}
-	used, ok := volumeMetrics.Used.AsInt64()
-	if !ok {
-		return nil, status.Errorf(codes.Internal, "failed to transform volume used size(%v)", volumeMetrics.Used)
-	}
-
-	inodesFree, ok := volumeMetrics.InodesFree.AsInt64()
-	if !ok {
-		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes free(%v)", volumeMetrics.InodesFree)
-	}
-	inodes, ok := volumeMetrics.Inodes.AsInt64()
-	if !ok {
-		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes(%v)", volumeMetrics.Inodes)
-	}
-	inodesUsed, ok := volumeMetrics.InodesUsed.AsInt64()
-	if !ok {
-		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes used(%v)", volumeMetrics.InodesUsed)
-	}
-
-	return &csi.NodeGetVolumeStatsResponse{
-		Usage: []*csi.VolumeUsage{
-			{
-				Unit:      csi.VolumeUsage_BYTES,
-				Available: available,
-				Total:     capacity,
-				Used:      used,
-			},
-			{
-				Unit:      csi.VolumeUsage_INODES,
-				Available: inodesFree,
-				Total:     inodes,
-				Used:      inodesUsed,
-			},
-		},
-	}, nil
+	return GetVolumeStats(req.VolumePath, d.enableWindowsHostProcess)
 }
 
 // NodeExpandVolume node expand volume

diff --git a/pkg/azurefile/nodeserver_test.go b/pkg/azurefile/nodeserver_test.go
@@ -877,6 +877,9 @@ func TestNodeGetVolumeStats(t *testing.T) {
 	d := NewFakeDriver()
 
 	for _, test := range tests {
+		if runtime.GOOS == "darwin" {
+			continue
+		}
 		_, err := d.NodeGetVolumeStats(context.Background(), &test.req)
 		//t.Errorf("[debug] error: %v\n metrics: %v", err, metrics)
 		if !reflect.DeepEqual(err, test.expectedErr) {