[pkg] panic if no space is left

pkg/disk/monitor.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2022 IoTeX Foundation
+// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
+// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
+// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
+// License 2.0 that can be found in the LICENSE file.
+
+package disk
+
+import (
+	"context"
+	"time"
+
+	"github.com/shirou/gopsutil/v3/disk"
+	"go.uber.org/zap"
+
+	"github.com/iotexproject/iotex-core/pkg/log"
+	"github.com/iotexproject/iotex-core/pkg/routine"
+)
+
+// Monitor represents a timer task of checking disk
+type Monitor struct {
+	task *routine.RecurringTask
+}
+
+// NewMonitor creates an instance of timer task
+func NewMonitor(internal time.Duration) *Monitor {
+	return &Monitor{
+		task: routine.NewRecurringTask(checkDiskSpace, internal),
+	}
+}
+
+// Start starts timer task
+func (m *Monitor) Start(ctx context.Context) error {
+	return m.task.Start(ctx)
+}
+
+// Stop stops timer task
+func (m *Monitor) Stop(ctx context.Context) error {
+	return m.task.Stop(ctx)
+}
+
+func checkDiskSpace() {
+	usage, err := disk.Usage("/")
+	if err != nil {
+		log.L().Error("Failed to get disk usage.", zap.Error(err))
+		return
+	}
+	// panic if left less than 2%
+	if usage.UsedPercent > 98.0 || usage.InodesUsedPercent > 98.0 {
+		log.L().Fatal("No space in device.", zap.Float64("UsedPercent", usage.UsedPercent), zap.Float64("InodesUsedPercent", usage.InodesUsedPercent))
+	}
+}

pkg/disk/monitor_test.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2022 IoTeX Foundation
+// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
+// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
+// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
+// License 2.0 that can be found in the LICENSE file.
+
+package disk
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/iotexproject/iotex-core/testutil"
+)
+
+func TestMonitor(t *testing.T) {
+	require := require.New(t)
+	m := NewMonitor(30 * time.Millisecond)
+	ctx := context.Background()
+	err := m.Start(ctx)
+	require.NoError(err)
+	require.NoError(testutil.WaitUntil(100*time.Millisecond, 1*time.Second, func() (b bool, e error) {
+		err = m.Stop(ctx)
+		return err == nil, nil
+	}))
+}
+
+func BenchmarkCheckSpace(b *testing.B) {
+	// BenchmarkCheckSpace-4             560012              1849 ns/op             108 B/op          4 allocs/op
+	for i := 0; i < b.N; i++ {
+		checkDiskSpace()
+	}
+}

server/itx/server.go

@@ -13,6 +13,7 @@ import (
 	"net/http/pprof"
 	"runtime"
 	"sync"
+	"time"
 
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
@@ -22,6 +23,7 @@ import (
 	"github.com/iotexproject/iotex-core/config"
 	"github.com/iotexproject/iotex-core/dispatcher"
 	"github.com/iotexproject/iotex-core/p2p"
+	"github.com/iotexproject/iotex-core/pkg/disk"
 	"github.com/iotexproject/iotex-core/pkg/ha"
 	"github.com/iotexproject/iotex-core/pkg/log"
 	"github.com/iotexproject/iotex-core/pkg/probe"
@@ -230,6 +232,17 @@ func StartServer(ctx context.Context, svr *Server, probeSvr *probe.Server, cfg c
 		}()
 	}
 
+	// check disk space
+	m := disk.NewMonitor(3 * time.Minute)
+	if err := m.Start(ctx); err != nil {
+		log.L().Panic("Failed to start monitor disk space", zap.Error(err))
+	}
+	defer func() {
+		if err := m.Stop(ctx); err != nil {
+			log.L().Panic("Failed to stop monitor disk space.", zap.Error(err))
+		}
+	}()
+
 	var adminserv http.Server
 	if cfg.System.HTTPAdminPort > 0 {
 		mux := http.NewServeMux()