From 24c1e3053b6767add29bf257c7943dc6aa5fa91d Mon Sep 17 00:00:00 2001 From: Alex Mazalov Date: Tue, 19 Jan 2021 08:26:42 +0000 Subject: [PATCH] cmd/geth: graceful shutdown if disk is full (#22103) Adding warnings of free disk space left and graceful shutdown when there is not enough space left. This also adds a flag datadir.minfreedisk which can be used to set the trigger for low disk space, and setting it to zero disables the check. Co-authored-by: Martin Holst Swende Co-authored-by: Felix Lange --- cmd/geth/main.go | 3 ++- cmd/geth/usage.go | 1 + cmd/utils/cmd.go | 34 +++++++++++++++++++++++++++++- cmd/utils/diskusage.go | 35 +++++++++++++++++++++++++++++++ cmd/utils/diskusage_windows.go | 38 ++++++++++++++++++++++++++++++++++ cmd/utils/flags.go | 4 ++++ 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 cmd/utils/diskusage.go create mode 100644 cmd/utils/diskusage_windows.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 7af24e65233f..e577ab370dc6 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -65,6 +65,7 @@ var ( utils.LegacyBootnodesV5Flag, utils.DataDirFlag, utils.AncientFlag, + utils.MinFreeDiskSpaceFlag, utils.KeyStoreDirFlag, utils.ExternalSignerFlag, utils.NoUSBFlag, @@ -368,7 +369,7 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend) { debug.Memsize.Add("node", stack) // Start up the node itself - utils.StartNode(stack) + utils.StartNode(ctx, stack) // Unlock any account specifically requested unlockAccounts(ctx, stack) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 78ebb807e1f5..25accc9b7989 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -36,6 +36,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ configFileFlag, utils.DataDirFlag, utils.AncientFlag, + utils.MinFreeDiskSpaceFlag, utils.KeyStoreDirFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 869cf90ea57b..43062168927f 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -26,17 +26,20 @@ import ( "runtime" "strings" "syscall" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" + "gopkg.in/urfave/cli.v1" ) const ( @@ -63,7 +66,7 @@ func Fatalf(format string, args ...interface{}) { os.Exit(1) } -func StartNode(stack *node.Node) { +func StartNode(ctx *cli.Context, stack *node.Node) { if err := stack.Start(); err != nil { Fatalf("Error starting protocol stack: %v", err) } @@ -71,6 +74,17 @@ func StartNode(stack *node.Node) { sigc := make(chan os.Signal, 1) signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(sigc) + + minFreeDiskSpace := eth.DefaultConfig.TrieDirtyCache + if ctx.GlobalIsSet(MinFreeDiskSpaceFlag.Name) { + minFreeDiskSpace = ctx.GlobalInt(MinFreeDiskSpaceFlag.Name) + } else if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { + minFreeDiskSpace = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + } + if minFreeDiskSpace > 0 { + go monitorFreeDiskSpace(sigc, stack.InstanceDir(), uint64(minFreeDiskSpace)*1024*1024) + } + <-sigc log.Info("Got interrupt, shutting down...") go stack.Close() @@ -85,6 +99,24 @@ func StartNode(stack *node.Node) { }() } +func monitorFreeDiskSpace(sigc chan os.Signal, path string, freeDiskSpaceCritical uint64) { + for { + freeSpace, err := getFreeDiskSpace(path) + if err != nil { + log.Warn("Failed to get free disk space", "path", path, "err", err) + break + } + if freeSpace < freeDiskSpaceCritical { + log.Error("Low disk space. Gracefully shutting down Geth to prevent database corruption.", "available", common.StorageSize(freeSpace)) + sigc <- syscall.SIGTERM + break + } else if freeSpace < 2*freeDiskSpaceCritical { + log.Warn("Disk space is running low. Geth will shutdown if disk space runs below critical level.", "available", common.StorageSize(freeSpace), "critical_level", common.StorageSize(freeDiskSpaceCritical)) + } + time.Sleep(60 * time.Second) + } +} + func ImportChain(chain *core.BlockChain, fn string) error { // Watch for Ctrl-C while the import is running. // If a signal is received, the import will stop at the next batch. diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go new file mode 100644 index 000000000000..a822118a39a1 --- /dev/null +++ b/cmd/utils/diskusage.go @@ -0,0 +1,35 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build !windows + +package utils + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +func getFreeDiskSpace(path string) (uint64, error) { + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return 0, fmt.Errorf("failed to call Statfs: %v", err) + } + + // Available blocks * size per block = available space in bytes + return stat.Bavail * uint64(stat.Bsize), nil +} diff --git a/cmd/utils/diskusage_windows.go b/cmd/utils/diskusage_windows.go new file mode 100644 index 000000000000..9bf7740b9941 --- /dev/null +++ b/cmd/utils/diskusage_windows.go @@ -0,0 +1,38 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package utils + +import ( + "fmt" + + "golang.org/x/sys/windows" +) + +func getFreeDiskSpace(path string) (uint64, error) { + + cwd, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, fmt.Errorf("failed to call UTF16PtrFromString: %v", err) + } + + var freeBytesAvailableToCaller, totalNumberOfBytes, totalNumberOfFreeBytes uint64 + if err := windows.GetDiskFreeSpaceEx(cwd, &freeBytesAvailableToCaller, &totalNumberOfBytes, &totalNumberOfFreeBytes); err != nil { + return 0, fmt.Errorf("failed to call GetDiskFreeSpaceEx: %v", err) + } + + return freeBytesAvailableToCaller, nil +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 688e25618dab..df036cbbf07e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -113,6 +113,10 @@ var ( Name: "datadir.ancient", Usage: "Data directory for ancient chain segments (default = inside chaindata)", } + MinFreeDiskSpaceFlag = DirectoryFlag{ + Name: "datadir.minfreedisk", + Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", + } KeyStoreDirFlag = DirectoryFlag{ Name: "keystore", Usage: "Directory for the keystore (default = inside the datadir)",