diff --git a/pkg/disk/monitor.go b/pkg/disk/monitor.go new file mode 100644 index 0000000000..70bbc675fc --- /dev/null +++ b/pkg/disk/monitor.go @@ -0,0 +1,58 @@ +// Copyright (c) 2022 IoTeX Foundation +// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no +// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent +// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache +// License 2.0 that can be found in the LICENSE file. + +package disk + +import ( + "context" + "time" + + "github.com/pkg/errors" + "github.com/shirou/gopsutil/v3/disk" + "go.uber.org/zap" + + "github.com/iotexproject/iotex-core/pkg/log" + "github.com/iotexproject/iotex-core/pkg/routine" + "github.com/iotexproject/iotex-core/pkg/util/fileutil" +) + +// Monitor represents a timer task of checking disk +type Monitor struct { + task *routine.RecurringTask + path string +} + +// NewMonitor creates an instance of timer task +func NewMonitor(path string, internal time.Duration) (*Monitor, error) { + if !fileutil.FileExists(path) { + return nil, errors.Errorf("invalid file path %s", path) + } + m := &Monitor{path: path} + m.task = routine.NewRecurringTask(m.checkDiskSpace, internal) + return m, nil +} + +// Start starts timer task +func (m *Monitor) Start(ctx context.Context) error { + return m.task.Start(ctx) +} + +// Stop stops timer task +func (m *Monitor) Stop(ctx context.Context) error { + return m.task.Stop(ctx) +} + +func (m *Monitor) checkDiskSpace() { + usage, err := disk.Usage(m.path) + if err != nil { + log.L().Error("Failed to get disk usage.", zap.Error(err)) + return + } + // panic if left less than 2% + if usage.UsedPercent > 98.0 || usage.InodesUsedPercent > 98.0 { + log.L().Fatal("No space in device.", zap.Float64("UsedPercent", usage.UsedPercent), zap.Float64("InodesUsedPercent", usage.InodesUsedPercent)) + } +} diff --git a/pkg/disk/monitor_test.go b/pkg/disk/monitor_test.go new file mode 100644 index 0000000000..bcc33f23e0 --- /dev/null +++ b/pkg/disk/monitor_test.go @@ -0,0 +1,38 @@ +// Copyright (c) 2022 IoTeX Foundation +// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no +// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent +// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache +// License 2.0 that can be found in the LICENSE file. + +package disk + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/iotexproject/iotex-core/testutil" +) + +func TestMonitor(t *testing.T) { + require := require.New(t) + m, err := NewMonitor(t.TempDir(), 30*time.Millisecond) + require.NoError(err) + ctx := context.Background() + err = m.Start(ctx) + require.NoError(err) + require.NoError(testutil.WaitUntil(100*time.Millisecond, 1*time.Second, func() (b bool, e error) { + err = m.Stop(ctx) + return err == nil, nil + })) +} + +func BenchmarkCheckSpace(b *testing.B) { + m := &Monitor{path: b.TempDir()} + // BenchmarkCheckSpace-4 522752 2069 ns/op + for i := 0; i < b.N; i++ { + m.checkDiskSpace() + } +} diff --git a/server/itx/server.go b/server/itx/server.go index 69633b6ad3..a44978a707 100644 --- a/server/itx/server.go +++ b/server/itx/server.go @@ -13,6 +13,7 @@ import ( "net/http/pprof" "runtime" "sync" + "time" "github.com/pkg/errors" "go.uber.org/zap" @@ -22,6 +23,7 @@ import ( "github.com/iotexproject/iotex-core/config" "github.com/iotexproject/iotex-core/dispatcher" "github.com/iotexproject/iotex-core/p2p" + "github.com/iotexproject/iotex-core/pkg/disk" "github.com/iotexproject/iotex-core/pkg/ha" "github.com/iotexproject/iotex-core/pkg/log" "github.com/iotexproject/iotex-core/pkg/probe" @@ -230,6 +232,20 @@ func StartServer(ctx context.Context, svr *Server, probeSvr *probe.Server, cfg c }() } + // check disk space + m, err := disk.NewMonitor(cfg.Chain.ChainDBPath, 3*time.Minute) + if err != nil { + log.L().Panic("Failed to create monitor disk space.", zap.Error(err)) + } + if err = m.Start(ctx); err != nil { + log.L().Panic("Failed to start monitor disk space.", zap.Error(err)) + } + defer func() { + if err := m.Stop(ctx); err != nil { + log.L().Panic("Failed to stop monitor disk space.", zap.Error(err)) + } + }() + var adminserv http.Server if cfg.System.HTTPAdminPort > 0 { mux := http.NewServeMux()