[pkg] panic if no space is left (#3636)

* add fsnotify

* modify as 5-minutes timer loop

* add comment

* add comment

* move to maintainment of StartServer

* add BenchmarkCheckSpace

* new struct

* modify comments

* check db partion

pkg/disk/monitor.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2022 IoTeX Foundation
+// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
+// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
+// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
+// License 2.0 that can be found in the LICENSE file.
+
+package disk
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/shirou/gopsutil/v3/disk"
+	"go.uber.org/zap"
+
+	"github.com/iotexproject/iotex-core/pkg/log"
+	"github.com/iotexproject/iotex-core/pkg/routine"
+	"github.com/iotexproject/iotex-core/pkg/util/fileutil"
+)
+
+// Monitor represents a timer task of checking disk
+type Monitor struct {
+	task *routine.RecurringTask
+	path string
+}
+
+// NewMonitor creates an instance of timer task
+func NewMonitor(path string, internal time.Duration) (*Monitor, error) {
+	if !fileutil.FileExists(path) {
+		return nil, errors.Errorf("invalid file path %s", path)
+	}
+	m := &Monitor{path: path}
+	m.task = routine.NewRecurringTask(m.checkDiskSpace, internal)
+	return m, nil
+}
+
+// Start starts timer task
+func (m *Monitor) Start(ctx context.Context) error {
+	return m.task.Start(ctx)
+}
+
+// Stop stops timer task
+func (m *Monitor) Stop(ctx context.Context) error {
+	return m.task.Stop(ctx)
+}
+
+func (m *Monitor) checkDiskSpace() {
+	usage, err := disk.Usage(m.path)
+	if err != nil {
+		log.L().Error("Failed to get disk usage.", zap.Error(err))
+		return
+	}
+	// panic if left less than 2%
+	if usage.UsedPercent > 98.0 || usage.InodesUsedPercent > 98.0 {
+		log.L().Fatal("No space in device.", zap.Float64("UsedPercent", usage.UsedPercent), zap.Float64("InodesUsedPercent", usage.InodesUsedPercent))
+	}
+}

pkg/disk/monitor_test.go

@@ -0,0 +1,38 @@
+// Copyright (c) 2022 IoTeX Foundation
+// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
+// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
+// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
+// License 2.0 that can be found in the LICENSE file.
+
+package disk
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/iotexproject/iotex-core/testutil"
+)
+
+func TestMonitor(t *testing.T) {
+	require := require.New(t)
+	m, err := NewMonitor(t.TempDir(), 30*time.Millisecond)
+	require.NoError(err)
+	ctx := context.Background()
+	err = m.Start(ctx)
+	require.NoError(err)
+	require.NoError(testutil.WaitUntil(100*time.Millisecond, 1*time.Second, func() (b bool, e error) {
+		err = m.Stop(ctx)
+		return err == nil, nil
+	}))
+}
+
+func BenchmarkCheckSpace(b *testing.B) {
+	m := &Monitor{path: b.TempDir()}
+	// BenchmarkCheckSpace-4   	  522752	      2069 ns/op
+	for i := 0; i < b.N; i++ {
+		m.checkDiskSpace()
+	}
+}

server/itx/server.go

@@ -13,6 +13,7 @@ import (
 	"net/http/pprof"
 	"runtime"
 	"sync"
+	"time"
 
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
@@ -22,6 +23,7 @@ import (
 	"github.com/iotexproject/iotex-core/config"
 	"github.com/iotexproject/iotex-core/dispatcher"
 	"github.com/iotexproject/iotex-core/p2p"
+	"github.com/iotexproject/iotex-core/pkg/disk"
 	"github.com/iotexproject/iotex-core/pkg/ha"
 	"github.com/iotexproject/iotex-core/pkg/log"
 	"github.com/iotexproject/iotex-core/pkg/probe"
@@ -230,6 +232,20 @@ func StartServer(ctx context.Context, svr *Server, probeSvr *probe.Server, cfg c
 		}()
 	}
 
+	// check disk space
+	m, err := disk.NewMonitor(cfg.Chain.ChainDBPath, 3*time.Minute)
+	if err != nil {
+		log.L().Panic("Failed to create monitor disk space.", zap.Error(err))
+	}
+	if err = m.Start(ctx); err != nil {
+		log.L().Panic("Failed to start monitor disk space.", zap.Error(err))
+	}
+	defer func() {
+		if err := m.Stop(ctx); err != nil {
+			log.L().Panic("Failed to stop monitor disk space.", zap.Error(err))
+		}
+	}()
+
 	var adminserv http.Server
 	if cfg.System.HTTPAdminPort > 0 {
 		mux := http.NewServeMux()