diff --git a/go.mod b/go.mod index 5dec159a32..350f4a58e4 100644 --- a/go.mod +++ b/go.mod @@ -92,7 +92,7 @@ require ( github.com/jeromer/syslogparser v1.1.0 github.com/jsimonetti/rtnetlink v1.4.2 github.com/jxskiss/base62 v1.1.0 - github.com/klauspost/compress v1.17.7 + github.com/klauspost/compress v1.17.8 github.com/klauspost/cpuid/v2 v2.2.7 github.com/linode/go-metadata v0.2.0 github.com/martinlindhe/base36 v1.1.1 @@ -127,7 +127,7 @@ require ( github.com/siderolabs/go-api-signature v0.3.2 github.com/siderolabs/go-blockdevice v0.4.7 github.com/siderolabs/go-blockdevice/v2 v2.0.0-20240405165836-3265299b0192 - github.com/siderolabs/go-circular v0.1.0 + github.com/siderolabs/go-circular v0.2.0 github.com/siderolabs/go-cmd v0.1.1 github.com/siderolabs/go-copy v0.1.0 github.com/siderolabs/go-debug v0.3.0 diff --git a/go.sum b/go.sum index 964dc4cecc..0000c1386a 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,8 @@ github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -656,8 +656,8 @@ github.com/siderolabs/go-blockdevice v0.4.7 h1:2bk4WpEEflGxjrNwp57ye24Pr+cYgAiAe github.com/siderolabs/go-blockdevice v0.4.7/go.mod h1:4PeOuk71pReJj1JQEXDE7kIIQJPVe8a+HZQa+qjxSEA= github.com/siderolabs/go-blockdevice/v2 v2.0.0-20240405165836-3265299b0192 h1:16/cHDGhTUDBtfIftOkuHWhJcQdpa/FwwWPcTq4aOxc= github.com/siderolabs/go-blockdevice/v2 v2.0.0-20240405165836-3265299b0192/go.mod h1:UBbbc+L7hU0UggOQeKCA+Qp3ImGkSeaLfVOiCbxRxEI= -github.com/siderolabs/go-circular v0.1.0 h1:zpBJNUbCZSh0odZxA4Dcj0d3ShLLR2WxKW6hTdAtoiE= -github.com/siderolabs/go-circular v0.1.0/go.mod h1:14XnLf/I3J0VjzTgmwWNGjp58/bdIi4zXppAEx8plfw= +github.com/siderolabs/go-circular v0.2.0 h1:Xca8zrjF/YsujLbwDSojkKzJe7ngetnpuIJn8N78DJI= +github.com/siderolabs/go-circular v0.2.0/go.mod h1:rrYCwHLYWmxqrmZP+LjYtwB2a55lxzQi0Ztu1VpWZSc= github.com/siderolabs/go-cmd v0.1.1 h1:nTouZUSxLeiiEe7hFexSVvaTsY/3O8k1s08BxPRrsps= github.com/siderolabs/go-cmd v0.1.1/go.mod h1:6hY0JG34LxEEwYE8aH2iIHkHX/ir12VRLqfwAf2yJIY= github.com/siderolabs/go-copy v0.1.0 h1:OIWCtSg+rhOtnIZTpT31Gfpn17rv5kwJqQHG+QUEgC8= diff --git a/internal/app/machined/pkg/runtime/logging/circular.go b/internal/app/machined/pkg/runtime/logging/circular.go index 1ee64e34b2..c63ac8cc21 100644 --- a/internal/app/machined/pkg/runtime/logging/circular.go +++ b/internal/app/machined/pkg/runtime/logging/circular.go @@ -15,7 +15,9 @@ import ( "sync" "time" + corezstd "github.com/klauspost/compress/zstd" "github.com/siderolabs/go-circular" + "github.com/siderolabs/go-circular/zstd" "github.com/siderolabs/go-debug" "github.com/siderolabs/go-tail" @@ -24,19 +26,25 @@ import ( // These constants should some day move to config. const ( + // Overall capacity of the log buffer (in raw bytes, memory size will be smaller due to compression). + DesiredCapacity = 1048576 // Some logs are tiny, no need to reserve too much memory. InitialCapacity = 16384 - // Cap each log at 1M. - MaxCapacity = 1048576 - // Safety gap to avoid buffer overruns. - SafetyGap = 2048 + // Chunk capacity is the length of each chunk, it should be + // big enough for the compression to be efficient. + ChunkCapacity = 65536 + // Number of zstd-compressed chunks to keep. + NumCompressedChunks = (DesiredCapacity / ChunkCapacity) - 1 + // Safety gap to avoid buffer overruns, can be lowered as with compression we don't need much. + SafetyGap = 1 ) // CircularBufferLoggingManager implements logging to circular fixed size buffer. type CircularBufferLoggingManager struct { fallbackLogger *log.Logger - buffers sync.Map + buffers sync.Map + compressor circular.Compressor sendersRW sync.RWMutex senders []runtime.LogSender @@ -45,9 +53,19 @@ type CircularBufferLoggingManager struct { // NewCircularBufferLoggingManager initializes new CircularBufferLoggingManager. func NewCircularBufferLoggingManager(fallbackLogger *log.Logger) *CircularBufferLoggingManager { + compressor, err := zstd.NewCompressor( + corezstd.WithEncoderConcurrency(1), + corezstd.WithWindowSize(2*corezstd.MinWindowSize), + ) + if err != nil { + // should not happen + panic(fmt.Sprintf("failed to create zstd compressor: %s", err)) + } + return &CircularBufferLoggingManager{ fallbackLogger: fallbackLogger, sendersChanged: make(chan struct{}), + compressor: compressor, } } @@ -106,7 +124,8 @@ func (manager *CircularBufferLoggingManager) getBuffer(id string, create bool) ( b, err := circular.NewBuffer( circular.WithInitialCapacity(InitialCapacity), - circular.WithMaxCapacity(MaxCapacity), + circular.WithMaxCapacity(ChunkCapacity), + circular.WithNumCompressedChunks(NumCompressedChunks, manager.compressor), circular.WithSafetyGap(SafetyGap)) if err != nil { return nil, err // only configuration issue might raise error