From 2bbf3f6579e8befe36e83f269ac46f90df8ed5c7 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Wed, 16 Aug 2023 06:19:29 +0000 Subject: [PATCH] fix: incorrect volume usage on Windows node with host process deployment --- go.mod | 2 +- pkg/azurefile/azure_common_darwin.go | 9 ++++ pkg/azurefile/azure_common_linux.go | 57 ++++++++++++++++++++++++ pkg/azurefile/azure_common_windows.go | 64 +++++++++++++++++++++++++++ pkg/azurefile/nodeserver.go | 49 +------------------- pkg/azurefile/nodeserver_test.go | 3 ++ 6 files changed, 135 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index 00e3fceebd..a3e0892ece 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/jongio/azidext/go/azidext v0.4.0 github.com/onsi/ginkgo/v2 v2.11.0 + golang.org/x/sys v0.11.0 k8s.io/pod-security-admission v0.27.3 ) @@ -132,7 +133,6 @@ require ( golang.org/x/crypto v0.12.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.11.0 // indirect golang.org/x/term v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect diff --git a/pkg/azurefile/azure_common_darwin.go b/pkg/azurefile/azure_common_darwin.go index 5f3f39c04d..127c06e734 100644 --- a/pkg/azurefile/azure_common_darwin.go +++ b/pkg/azurefile/azure_common_darwin.go @@ -20,6 +20,10 @@ limitations under the License. package azurefile import ( + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + mount "k8s.io/mount-utils" ) @@ -38,3 +42,8 @@ func preparePublishPath(path string, m *mount.SafeFormatAndMount) error { func prepareStagePath(path string, m *mount.SafeFormatAndMount) error { return nil } + +// GetVolumeStats returns volume stats based on the given path. +func GetVolumeStats(path string, enableWindowsHostProcess bool) (*csi.NodeGetVolumeStatsResponse, error) { + return nil, status.Errorf(codes.Internal, "GetVolumeStats is not supported on darwin") +} diff --git a/pkg/azurefile/azure_common_linux.go b/pkg/azurefile/azure_common_linux.go index 3e3a5a9e9c..cfbdf5704c 100644 --- a/pkg/azurefile/azure_common_linux.go +++ b/pkg/azurefile/azure_common_linux.go @@ -20,7 +20,13 @@ limitations under the License. package azurefile import ( + "github.com/container-storage-interface/spec/lib/go/csi" + + "k8s.io/kubernetes/pkg/volume" mount "k8s.io/mount-utils" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func SMBMount(m *mount.SafeFormatAndMount, source, target, fsType string, options, sensitiveMountOptions []string) error { @@ -38,3 +44,54 @@ func preparePublishPath(path string, m *mount.SafeFormatAndMount) error { func prepareStagePath(path string, m *mount.SafeFormatAndMount) error { return nil } + +// GetVolumeStats returns volume stats based on the given path. +func GetVolumeStats(path string, enableWindowsHostProcess bool) (*csi.NodeGetVolumeStatsResponse, error) { + volumeMetrics, err := volume.NewMetricsStatFS(path).GetMetrics() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err) + } + + available, ok := volumeMetrics.Available.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume available size(%v)", volumeMetrics.Available) + } + capacity, ok := volumeMetrics.Capacity.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume capacity size(%v)", volumeMetrics.Capacity) + } + used, ok := volumeMetrics.Used.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume used size(%v)", volumeMetrics.Used) + } + + inodesFree, ok := volumeMetrics.InodesFree.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform disk inodes free(%v)", volumeMetrics.InodesFree) + } + inodes, ok := volumeMetrics.Inodes.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform disk inodes(%v)", volumeMetrics.Inodes) + } + inodesUsed, ok := volumeMetrics.InodesUsed.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform disk inodes used(%v)", volumeMetrics.InodesUsed) + } + + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Unit: csi.VolumeUsage_BYTES, + Available: available, + Total: capacity, + Used: used, + }, + { + Unit: csi.VolumeUsage_INODES, + Available: inodesFree, + Total: inodes, + Used: inodesUsed, + }, + }, + }, nil +} diff --git a/pkg/azurefile/azure_common_windows.go b/pkg/azurefile/azure_common_windows.go index c499f8e4d0..9a90225b6f 100644 --- a/pkg/azurefile/azure_common_windows.go +++ b/pkg/azurefile/azure_common_windows.go @@ -22,7 +22,12 @@ package azurefile import ( "fmt" + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/sys/windows" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/volume" mount "k8s.io/mount-utils" "sigs.k8s.io/azurefile-csi-driver/pkg/mounter" ) @@ -69,3 +74,62 @@ func preparePublishPath(path string, m *mount.SafeFormatAndMount) error { func prepareStagePath(path string, m *mount.SafeFormatAndMount) error { return removeDir(path, m) } + +// GetFreeSpace returns the free space of the volume in bytes, total size of the volume in bytes and the used space of the volume in bytes +func GetFreeSpace(path string) (int64, int64, int64, error) { + var totalNumberOfBytes, totalNumberOfFreeBytes uint64 + dirName := windows.StringToUTF16Ptr(path) + if err := windows.GetDiskFreeSpaceEx(dirName, nil, &totalNumberOfBytes, &totalNumberOfFreeBytes); err != nil { + return 0, 0, 0, err + } + return int64(totalNumberOfFreeBytes), int64(totalNumberOfBytes), int64(totalNumberOfBytes - totalNumberOfFreeBytes), nil +} + +// GetVolumeStats returns volume stats based on the given path. +func GetVolumeStats(path string, enableWindowsHostProcess bool) (*csi.NodeGetVolumeStatsResponse, error) { + if enableWindowsHostProcess { + // only in host process mode, we can get the free space of the volume, otherwise, we will get permission denied error + freeBytesAvailable, totalBytes, totalBytesUsed, err := GetFreeSpace(path) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get free space on path %s: %v", path, err) + } + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Unit: csi.VolumeUsage_BYTES, + Available: freeBytesAvailable, + Total: totalBytes, + Used: totalBytesUsed, + }, + }, + }, nil + } + + volumeMetrics, err := volume.NewMetricsStatFS(path).GetMetrics() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err) + } + + available, ok := volumeMetrics.Available.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume available size(%v)", volumeMetrics.Available) + } + capacity, ok := volumeMetrics.Capacity.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume capacity size(%v)", volumeMetrics.Capacity) + } + used, ok := volumeMetrics.Used.AsInt64() + if !ok { + return nil, status.Errorf(codes.Internal, "failed to transform volume used size(%v)", volumeMetrics.Used) + } + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Unit: csi.VolumeUsage_BYTES, + Available: available, + Total: capacity, + Used: used, + }, + }, + }, nil +} diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index 5588be4b3d..e858894214 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "google.golang.org/grpc/codes" @@ -432,53 +431,7 @@ func (d *Driver) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeS return nil, status.Errorf(codes.Internal, "failed to stat file %s: %v", req.VolumePath, err) } - volumeMetrics, err := volume.NewMetricsStatFS(req.VolumePath).GetMetrics() - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err) - } - - available, ok := volumeMetrics.Available.AsInt64() - if !ok { - return nil, status.Errorf(codes.Internal, "failed to transform volume available size(%v)", volumeMetrics.Available) - } - capacity, ok := volumeMetrics.Capacity.AsInt64() - if !ok { - return nil, status.Errorf(codes.Internal, "failed to transform volume capacity size(%v)", volumeMetrics.Capacity) - } - used, ok := volumeMetrics.Used.AsInt64() - if !ok { - return nil, status.Errorf(codes.Internal, "failed to transform volume used size(%v)", volumeMetrics.Used) - } - - inodesFree, ok := volumeMetrics.InodesFree.AsInt64() - if !ok { - return nil, status.Errorf(codes.Internal, "failed to transform disk inodes free(%v)", volumeMetrics.InodesFree) - } - inodes, ok := volumeMetrics.Inodes.AsInt64() - if !ok { - return nil, status.Errorf(codes.Internal, "failed to transform disk inodes(%v)", volumeMetrics.Inodes) - } - inodesUsed, ok := volumeMetrics.InodesUsed.AsInt64() - if !ok { - return nil, status.Errorf(codes.Internal, "failed to transform disk inodes used(%v)", volumeMetrics.InodesUsed) - } - - return &csi.NodeGetVolumeStatsResponse{ - Usage: []*csi.VolumeUsage{ - { - Unit: csi.VolumeUsage_BYTES, - Available: available, - Total: capacity, - Used: used, - }, - { - Unit: csi.VolumeUsage_INODES, - Available: inodesFree, - Total: inodes, - Used: inodesUsed, - }, - }, - }, nil + return GetVolumeStats(req.VolumePath, d.enableWindowsHostProcess) } // NodeExpandVolume node expand volume diff --git a/pkg/azurefile/nodeserver_test.go b/pkg/azurefile/nodeserver_test.go index 8bc018b02a..94147a4531 100644 --- a/pkg/azurefile/nodeserver_test.go +++ b/pkg/azurefile/nodeserver_test.go @@ -877,6 +877,9 @@ func TestNodeGetVolumeStats(t *testing.T) { d := NewFakeDriver() for _, test := range tests { + if runtime.GOOS == "darwin" { + continue + } _, err := d.NodeGetVolumeStats(context.Background(), &test.req) //t.Errorf("[debug] error: %v\n metrics: %v", err, metrics) if !reflect.DeepEqual(err, test.expectedErr) {