Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[pkg] panic if no space is left #3636

Merged
merged 15 commits into from
Oct 13, 2022
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ require (
github.com/dustinxie/gmsm v1.4.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/fsnotify/fsnotify v1.5.5-0.20220908164714-85acde25252d // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
Expand Down Expand Up @@ -192,7 +193,7 @@ require (
go.opentelemetry.io/otel/metric v0.31.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
google.golang.org/appengine v1.6.5 // indirect
Expand Down
8 changes: 8 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,10 @@ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/fsnotify/fsnotify v1.5.5-0.20220908164714-85acde25252d h1:Gy0zdYdQplJcIRkhpu9T5g6Get+mZ5ojAW37QSxTQSE=
github.com/fsnotify/fsnotify v1.5.5-0.20220908164714-85acde25252d/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
Expand Down Expand Up @@ -1539,8 +1543,12 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho=
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
Expand Down
73 changes: 73 additions & 0 deletions pkg/fsnotify/fsnotify.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright (c) 2022 IoTeX Foundation
// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
// License 2.0 that can be found in the LICENSE file.

package fsnotify

import (
"context"

"github.com/fsnotify/fsnotify"
"github.com/shirou/gopsutil/v3/disk"
"go.uber.org/zap"

"github.com/iotexproject/iotex-core/pkg/log"
)

// Watch creates dir watcher to hanlde events from os in loop
func Watch(ctx context.Context, dirs ...string) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.L().Fatal("Failed to create fsnotify watcher.", zap.Error(err))
}
defer watcher.Close()

for _, dir := range dirs {
if err = watcher.Add(dir); err != nil {
log.L().Fatal("Failed to watch dir: "+dir, zap.Error(err))
}
}
log.S().Infof("Watch dirs: %+v", dirs)

watchLoop(ctx, watcher)
}

// watchLoop receives events and errors from os
func watchLoop(ctx context.Context, watcher *fsnotify.Watcher) {
for {
select {
case <-ctx.Done():
return
case ev, ok := <-watcher.Events:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when does an event come

if !ok {
log.L().Error("Failed to get watcher events.")
continue
}
// check disk space per write operation
if ev.Has(fsnotify.Write) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fsnotify.Create needed too

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's still left enough space to run program even if Avail is 0 after run df -h .

checkDiskSpace(ctx)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

all filesystem write ops trig a check, may inefficient

}
case err, ok := <-watcher.Errors:
if !ok {
log.L().Error("Failed to get watcher errors.")
continue
}
log.L().Error("watch file error", zap.Error(err))
}
}
}

// checkDiskSpace returns the disk usage info
func checkDiskSpace(ctx context.Context) {
usage, err := disk.Usage("/")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if os disk partition like this

/          /dev/sda
/run       /dev/sdb

and iotex-core run within /run, the check may not work

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"/" checks all disks

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@huangzhiran +1

/    /dev/sda   500G
/run /dev/sdb 100MB

if run on /run, it will failed

if err != nil {
log.L().Error("Failed to get disk usage.", zap.Error(err))
return
}
// panic if no space or no inodes
if usage.Free == 0 || usage.InodesFree == 0 {
log.L().Fatal("No space in device.", zap.Float64("UsedPercent", usage.UsedPercent), zap.Float64("InodesUsedPercent", usage.InodesUsedPercent))
}
}
41 changes: 41 additions & 0 deletions pkg/fsnotify/fsnotify_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Copyright (c) 2022 IoTeX Foundation
// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
// License 2.0 that can be found in the LICENSE file.

package fsnotify

import (
"context"
"os"
"testing"
"time"

"github.com/stretchr/testify/require"

"github.com/iotexproject/iotex-core/testutil"
)

func TestWatch(t *testing.T) {
var (
require = require.New(t)
dir = t.TempDir()
times int
)

ctx, cancel := context.WithCancel(context.Background())
go Watch(ctx, dir)

f, err := os.CreateTemp(dir, "log")
require.NoError(err)
defer f.Close()

require.NoError(testutil.WaitUntil(100*time.Millisecond, 1*time.Second, func() (b bool, e error) {
err = os.WriteFile(f.Name(), []byte("test"), 0666)
require.NoError(err)
times++
return times >= 5, nil
}))
cancel()
}
13 changes: 8 additions & 5 deletions pkg/log/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"log"
"net/http"
"os"
"path/filepath"
"sync"

"go.elastic.co/ecszap"
Expand Down Expand Up @@ -67,14 +68,15 @@ func Logger(name string) *zap.Logger {
}

// InitLoggers initializes the global logger and other sub loggers.
func InitLoggers(globalCfg GlobalConfig, subCfgs map[string]GlobalConfig, opts ...zap.Option) error {
func InitLoggers(globalCfg GlobalConfig, subCfgs map[string]GlobalConfig, opts ...zap.Option) ([]string, error) {
if _, exists := subCfgs[_globalLoggerName]; exists {
return errors.New("'" + _globalLoggerName + "' is a reserved name for global logger")
return nil, errors.New("'" + _globalLoggerName + "' is a reserved name for global logger")
}
subCfgs[_globalLoggerName] = globalCfg
watchDirs := make([]string, 0, len(subCfgs))
for name, cfg := range subCfgs {
if _, exists := _subLoggers[name]; exists {
return errors.Errorf("duplicate sub logger name: %s", name)
return nil, errors.Errorf("duplicate sub logger name: %s", name)
}
if cfg.Zap == nil {
zapCfg := zap.NewProductionConfig()
Expand All @@ -90,8 +92,9 @@ func InitLoggers(globalCfg GlobalConfig, subCfgs map[string]GlobalConfig, opts .
if cfg.StderrRedirectFile != nil {
stderrF, err := os.OpenFile(*cfg.StderrRedirectFile, os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_APPEND, 0600)
if err != nil {
return err
return nil, err
}
watchDirs = append(watchDirs, filepath.Dir(*cfg.StderrRedirectFile))

cores = append(cores, zapcore.NewCore(
zapcore.NewJSONEncoder(cfg.Zap.EncoderConfig),
Expand Down Expand Up @@ -123,7 +126,7 @@ func InitLoggers(globalCfg GlobalConfig, subCfgs map[string]GlobalConfig, opts .
_logMu.Unlock()
}

return nil
return watchDirs, nil
}

// RegisterLevelConfigMux registers log's level config http mux.
Expand Down
10 changes: 6 additions & 4 deletions server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/iotexproject/iotex-core/blockchain/block"
"github.com/iotexproject/iotex-core/blockchain/genesis"
"github.com/iotexproject/iotex-core/config"
"github.com/iotexproject/iotex-core/pkg/fsnotify"
"github.com/iotexproject/iotex-core/pkg/log"
"github.com/iotexproject/iotex-core/pkg/probe"
"github.com/iotexproject/iotex-core/pkg/recovery"
Expand Down Expand Up @@ -74,8 +75,7 @@ func init() {

func main() {
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt)
signal.Notify(stop, syscall.SIGTERM)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
ctx, cancel := context.WithCancel(context.Background())
stopped := make(chan struct{})
livenessCtx, livenessCancel := context.WithCancel(context.Background())
Expand All @@ -99,9 +99,11 @@ func main() {
if err != nil {
glog.Fatalln("Failed to new config.", zap.Error(err))
}
if err = initLogger(cfg); err != nil {
watchDirs, err := initLogger(cfg)
if err != nil {
glog.Fatalln("Cannot config global logger, use default one: ", zap.Error(err))
}
go fsnotify.Watch(ctx, watchDirs...)

if err = recovery.SetCrashlogDir(cfg.System.SystemLogDBPath); err != nil {
glog.Fatalln("Failed to set directory of crashlog: ", zap.Error(err))
Expand Down Expand Up @@ -167,7 +169,7 @@ func main() {
<-livenessCtx.Done()
}

func initLogger(cfg config.Config) error {
func initLogger(cfg config.Config) ([]string, error) {
addr := cfg.Chain.ProducerAddress()
return log.InitLoggers(cfg.Log, cfg.SubLogs, zap.AddCaller(), zap.Fields(
zap.String("ioAddr", addr.String()),
Expand Down
12 changes: 5 additions & 7 deletions tools/bot/server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
_ "go.uber.org/automaxprocs"
"go.uber.org/zap"

"github.com/iotexproject/iotex-core/pkg/fsnotify"
"github.com/iotexproject/iotex-core/pkg/log"
"github.com/iotexproject/iotex-core/tools/bot/config"
"github.com/iotexproject/iotex-core/tools/bot/server/bot"
Expand All @@ -36,10 +37,11 @@ func main() {
fmt.Println("Failed to new config.", zap.Error(err))
return
}
err = initLogger(cfg)
watchDirs, err := initLogger(cfg)
if err != nil {
return
}
go fsnotify.Watch(context.Background(), watchDirs...)
b, err := bot.NewServer(cfg)
if err != nil {
log.L().Fatal("new server:", zap.Error(err))
Expand Down Expand Up @@ -83,10 +85,6 @@ func main() {
select {}
}

func initLogger(cfg config.Config) error {
if err := log.InitLoggers(cfg.Log, cfg.SubLogs); err != nil {
fmt.Println("Cannot config global logger, use default one: ", err)
return err
}
return nil
func initLogger(cfg config.Config) ([]string, error) {
return log.InitLoggers(cfg.Log, cfg.SubLogs)
}