Log custom plugin stderr only if the status is not ok.

Otherwise with plugins that run frequently and report ok status, the
logs are filled with unnecessary noise and significantly increases log
size.

pkg/custompluginmonitor/plugin/plugin.go

@@ -89,7 +89,7 @@ func (p *Plugin) Run() {
 
 // run each rule in parallel and wait for them to complete
 func (p *Plugin) runRules() {
-	glog.Info("Start to run custom plugins")
+	glog.V(3).Info("Start to run custom plugins")
 
 	for _, rule := range p.config.Rules {
 		p.syncChan <- struct{}{}
@@ -120,7 +120,7 @@ func (p *Plugin) runRules() {
 	}
 
 	p.Wait()
-	glog.Info("Finish running custom plugins")
+	glog.V(3).Info("Finish running custom plugins")
 }
 
 // readFromReader reads the maxBytes from the reader and drains the rest.
@@ -203,12 +203,6 @@ func (p *Plugin) run(rule cpmtypes.CustomRule) (exitStatus cpmtypes.Status, outp
 		}
 	}
 
-	// log the stderr from the plugin
-	if len(stderr) != 0 {
-		glog.Infof("Start logs from plugin %q \n %s", rule.Path, string(stderr))
-		glog.Infof("End logs from plugin %q", rule.Path)
-	}
-
 	// trim suffix useless bytes
 	output = string(stdout)
 	output = strings.TrimSpace(output)
@@ -227,8 +221,10 @@ func (p *Plugin) run(rule cpmtypes.CustomRule) (exitStatus cpmtypes.Status, outp
 	case 0:
 		return cpmtypes.OK, output
 	case 1:
+		logPluginStderr(rule.Path, string(stderr))
 		return cpmtypes.NonOK, output
 	default:
+		logPluginStderr(rule.Path, string(stderr))
 		return cpmtypes.Unknown, output
 	}
 }
@@ -237,3 +233,10 @@ func (p *Plugin) Stop() {
 	p.tomb.Stop()
 	glog.Info("Stop plugin execution")
 }
+
+func logPluginStderr(path, logs string) {
+	if len(logs) != 0 {
+		glog.Infof("Start logs from plugin %q \n %s", path, string(logs))
+		glog.Infof("End logs from plugin %q", path)
+	}
+}

pkg/healthchecker/health_checker.go

@@ -31,6 +31,7 @@ import (
 )
 
 type healthChecker struct {
+	component       string
 	enableRepair    bool
 	healthCheckFunc func() bool
 	// The repair is "best-effort" and ignores the error from the underlying actions.
@@ -45,6 +46,7 @@ type healthChecker struct {
 // NewHealthChecker returns a new health checker configured with the given options.
 func NewHealthChecker(hco *options.HealthCheckerOptions) (types.HealthChecker, error) {
 	hc := &healthChecker{
+		component:          hco.Component,
 		enableRepair:       hco.EnableRepair,
 		crictlPath:         hco.CriCtlPath,
 		healthCheckTimeout: hco.HealthCheckTimeout,
@@ -139,14 +141,14 @@ func (hc *healthChecker) CheckHealth() bool {
 	// The service is unhealthy.
 	// Attempt repair based on flag.
 	if hc.enableRepair {
-		glog.Infof("health-checker: component is unhealthy, proceeding to repair")
 		// repair if the service has been up for the cool down period.
 		uptime, err := hc.uptimeFunc()
 		if err != nil {
-			glog.Infof("health-checker: %v\n", err.Error())
+			glog.Infof("error in getting uptime for %v: %v\n", hc.component, err.Error())
 		}
-		glog.Infof("health-checker: component uptime: %v\n", uptime)
+		glog.Infof("%v is unhealthy, component uptime: %v\n", hc.component, uptime)
 		if uptime > hc.coolDownTime {
+			glog.Infof("%v cooldown period of %v exceeded, repairing", hc.component, hc.coolDownTime)
 			hc.repairFunc()
 		}
 	}
@@ -159,10 +161,9 @@ func execCommand(timeout time.Duration, command string, args ...string) (string,
 	defer cancel()
 
 	cmd := exec.CommandContext(ctx, command, args...)
-	glog.Infof("health-checker: executing command : %v\n", cmd)
 	out, err := cmd.Output()
 	if err != nil {
-		glog.Infof("health-checker: command failed : %v, %v\n", err.Error(), out)
+		glog.Infof("command %v failed: %v, %v\n", cmd, err.Error(), out)
 		return "", err
 	}
 	return strings.TrimSuffix(string(out), "\n"), nil