From f6de6eafd0ae5753b72b9431295486a60f5e6d11 Mon Sep 17 00:00:00 2001 From: Milan Lenco Date: Sat, 1 Jul 2023 11:32:14 +0200 Subject: [PATCH] Ensure we get at least one netdump in each topic for tested EVE upgrade If upgrade fails, the recorded netdumps will remain persisted and available for retrieval from the older EVE version. Signed-off-by: Milan Lenco --- pkg/pillar/cmd/zedagent/handleconfig.go | 25 ++++++++++++++++++++----- pkg/pillar/cmd/zedagent/reportinfo.go | 6 +----- pkg/pillar/cmd/zedagent/zedagent.go | 2 ++ pkg/pillar/dpcmanager/dpcmanager.go | 2 ++ pkg/pillar/dpcmanager/netdump.go | 13 ++++++++++++- 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/pkg/pillar/cmd/zedagent/handleconfig.go b/pkg/pillar/cmd/zedagent/handleconfig.go index 60ad521a93..508fa5e8ed 100644 --- a/pkg/pillar/cmd/zedagent/handleconfig.go +++ b/pkg/pillar/cmd/zedagent/handleconfig.go @@ -26,6 +26,7 @@ import ( "github.com/lf-edge/eve/pkg/pillar/types" "github.com/lf-edge/eve/pkg/pillar/utils" fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file" + "github.com/lf-edge/eve/pkg/pillar/zboot" "github.com/lf-edge/eve/pkg/pillar/zedcloud" uuid "github.com/satori/go.uuid" "google.golang.org/protobuf/proto" @@ -1081,14 +1082,28 @@ func isNettraceEnabled(ctx *zedagentContext) bool { return true } -// Function decides if the next call to SendOnAllIntf for /config request should be traced -// and netdump published at the end (see libs/nettrace and pkg/pillar/netdump). -func traceNextConfigReq(ctx *zedagentContext) bool { +// Function decides if the next HTTP request should be traced and netdump published. +func traceNextReq(ctx *zedagentContext, lastNetdump time.Time) bool { if !isNettraceEnabled(ctx) { return false } - return ctx.lastConfigNetdumpPub.IsZero() || - time.Since(ctx.lastConfigNetdumpPub) >= ctx.netdumpInterval + if lastNetdump.IsZero() { + // No netdump published yet. + return true + } + uptime := time.Since(ctx.startTime) + lastNetdumpAge := time.Since(lastNetdump) + // Ensure we get at least one netdump for the currently tested EVE upgrade. + if zboot.IsCurrentPartitionStateInProgress() && lastNetdumpAge > uptime { + return true + } + return lastNetdumpAge >= ctx.netdumpInterval +} + +// Function decides if the next call to SendOnAllIntf for /config request should be traced +// and netdump published at the end (see libs/nettrace and pkg/pillar/netdump). +func traceNextConfigReq(ctx *zedagentContext) bool { + return traceNextReq(ctx, ctx.lastConfigNetdumpPub) } // Publish netdump containing traces of executed config requests. diff --git a/pkg/pillar/cmd/zedagent/reportinfo.go b/pkg/pillar/cmd/zedagent/reportinfo.go index 1739096964..11b1717392 100644 --- a/pkg/pillar/cmd/zedagent/reportinfo.go +++ b/pkg/pillar/cmd/zedagent/reportinfo.go @@ -1147,11 +1147,7 @@ func isUpdating(ctx *zedagentContext) bool { // Function decides if the next call to SendOnAllIntf for /info request should be traced // and netdump published at the end (see libs/nettrace and pkg/pillar/netdump). func traceNextInfoReq(ctx *zedagentContext) bool { - if !isNettraceEnabled(ctx) { - return false - } - return ctx.lastInfoNetdumpPub.IsZero() || - time.Since(ctx.lastInfoNetdumpPub) >= ctx.netdumpInterval + return traceNextReq(ctx, ctx.lastInfoNetdumpPub) } // Publish netdump containing traces of executed /info requests. diff --git a/pkg/pillar/cmd/zedagent/zedagent.go b/pkg/pillar/cmd/zedagent/zedagent.go index 7da84c320d..6b6f5053d8 100644 --- a/pkg/pillar/cmd/zedagent/zedagent.go +++ b/pkg/pillar/cmd/zedagent/zedagent.go @@ -219,6 +219,7 @@ type zedagentContext struct { netdumpInterval time.Duration lastConfigNetdumpPub time.Time // last call to publishConfigNetdump lastInfoNetdumpPub time.Time // last call to publishInfoNetdump + startTime time.Time } // AddAgentSpecificCLIFlags adds CLI options @@ -315,6 +316,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar zedagentCtx.ps = ps zedagentCtx.hangFlag = *zedagentCtx.hangPtr zedagentCtx.fatalFlag = *zedagentCtx.fatalPtr + zedagentCtx.startTime = time.Now() flowlogQueue := make(chan *flowlog.FlowMessage, flowlogQueueCap) triggerDeviceInfo := make(chan destinationBitset, 1) diff --git a/pkg/pillar/dpcmanager/dpcmanager.go b/pkg/pillar/dpcmanager/dpcmanager.go index 9621a42bf1..6520ee7bee 100644 --- a/pkg/pillar/dpcmanager/dpcmanager.go +++ b/pkg/pillar/dpcmanager/dpcmanager.go @@ -129,6 +129,7 @@ type DpcManager struct { netDumper *netdump.NetDumper // nil if netdump is disabled netdumpInterval time.Duration lastNetdumpPub time.Time // last call to publishNetdump + startTime time.Time } // Watchdog : methods used by DpcManager to interact with Watchdog. @@ -238,6 +239,7 @@ func (m *DpcManager) Init(ctx context.Context) error { // Run DpcManager as a separate task with its own loop and a watchdog file. func (m *DpcManager) Run(ctx context.Context) (err error) { + m.startTime = time.Now() m.networkEvents = m.NetworkMonitor.WatchEvents(ctx, "dpc-reconciler") m.wwanEvents, err = m.WwanWatcher.Watch(ctx) if err != nil { diff --git a/pkg/pillar/dpcmanager/netdump.go b/pkg/pillar/dpcmanager/netdump.go index d2cc5f93be..f5411e44b4 100644 --- a/pkg/pillar/dpcmanager/netdump.go +++ b/pkg/pillar/dpcmanager/netdump.go @@ -7,6 +7,7 @@ import ( "time" "github.com/lf-edge/eve/pkg/pillar/netdump" + "github.com/lf-edge/eve/pkg/pillar/zboot" ) // Topic for netdumps of successful connectivity tests. @@ -31,7 +32,17 @@ func (m *DpcManager) traceNextConnTest() bool { if len(m.dpcList.PortConfigList) == 0 || m.dpcList.CurrentIndex != 0 { return false } - return m.lastNetdumpPub.IsZero() || time.Since(m.lastNetdumpPub) >= m.netdumpInterval + if m.lastNetdumpPub.IsZero() { + // No netdump published yet for DPC testing. + return true + } + uptime := time.Since(m.startTime) + lastNetdumpAge := time.Since(m.lastNetdumpPub) + // Ensure we get at least one netdump for the currently tested EVE upgrade. + if zboot.IsCurrentPartitionStateInProgress() && lastNetdumpAge > uptime { + return true + } + return lastNetdumpAge >= m.netdumpInterval } // Publish netdump containing traces of executed connectivity probes.