[pkg] panic if no space is left

pkg/watch/watch.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2022 IoTeX Foundation
+// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
+// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
+// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
+// License 2.0 that can be found in the LICENSE file.
+
+package watch
+
+import (
+	"context"
+	"time"
+
+	"github.com/shirou/gopsutil/v3/disk"
+	"go.uber.org/zap"
+
+	"github.com/iotexproject/iotex-core/pkg/log"
+	"github.com/iotexproject/iotex-core/pkg/routine"
+)
+
+// Start creates a timer task to check device per watchInternal
+func Start(ctx context.Context, watchInternal time.Duration) func() {
+	task := routine.NewRecurringTask(checkDiskSpace, watchInternal)
+	if err := task.Start(ctx); err != nil {
+		log.L().Panic("Failed to start watch disk space", zap.Error(err))
+	}
+	return func() {
+		if err := task.Stop(ctx); err != nil {
+			log.L().Panic("Failed to stop watch disk space.", zap.Error(err))
+		}
+	}
+}
+
+func checkDiskSpace() {
+	usage, err := disk.Usage("/")
+	if err != nil {
+		log.L().Error("Failed to get disk usage.", zap.Error(err))
+		return
+	}
+	// panic if left less than 2%
+	if usage.UsedPercent > 98.0 || usage.InodesUsedPercent > 98.0 {
+		log.L().Fatal("No space in device, pls clean up.", zap.Float64("UsedPercent", usage.UsedPercent), zap.Float64("InodesUsedPercent", usage.InodesUsedPercent))
+	}
+}

pkg/watch/watch_test.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2022 IoTeX Foundation
+// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
+// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
+// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
+// License 2.0 that can be found in the LICENSE file.
+
+package watch
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/iotexproject/iotex-core/testutil"
+)
+
+func TestStart(t *testing.T) {
+	require := require.New(t)
+	h := Start(context.Background(), 30*time.Millisecond)
+	require.NoError(testutil.WaitUntil(100*time.Millisecond, 1*time.Second, func() (b bool, e error) {
+		h()
+		return true, nil
+	}))
+}

server/main.go

@@ -20,6 +20,7 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/iotexproject/go-pkgs/hash"
 	_ "go.uber.org/automaxprocs"
@@ -31,6 +32,7 @@ import (
 	"github.com/iotexproject/iotex-core/pkg/log"
 	"github.com/iotexproject/iotex-core/pkg/probe"
 	"github.com/iotexproject/iotex-core/pkg/recovery"
+	"github.com/iotexproject/iotex-core/pkg/watch"
 	"github.com/iotexproject/iotex-core/server/itx"
 )
 
@@ -74,8 +76,7 @@ func init() {
 
 func main() {
 	stop := make(chan os.Signal, 1)
-	signal.Notify(stop, os.Interrupt)
-	signal.Notify(stop, syscall.SIGTERM)
+	signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
 	ctx, cancel := context.WithCancel(context.Background())
 	stopped := make(chan struct{})
 	livenessCtx, livenessCancel := context.WithCancel(context.Background())
@@ -103,6 +104,10 @@ func main() {
 		glog.Fatalln("Cannot config global logger, use default one: ", zap.Error(err))
 	}
 
+	// check device
+	stopWatch := watch.Start(ctx, 5*time.Minute)
+	defer stopWatch()
+
 	if err = recovery.SetCrashlogDir(cfg.System.SystemLogDBPath); err != nil {
 		glog.Fatalln("Failed to set directory of crashlog: ", zap.Error(err))
 	}