Skip to content
This repository has been archived by the owner on Mar 28, 2020. It is now read-only.

Commit

Permalink
debug: log operator actions to hostPath volume
Browse files Browse the repository at this point in the history
  • Loading branch information
hasbro17 committed Jul 12, 2017
1 parent 9fabd46 commit da8966f
Show file tree
Hide file tree
Showing 5 changed files with 145 additions and 7 deletions.
2 changes: 2 additions & 0 deletions cmd/operator/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/coreos/etcd-operator/pkg/backup/s3/s3config"
"github.com/coreos/etcd-operator/pkg/chaos"
"github.com/coreos/etcd-operator/pkg/controller"
"github.com/coreos/etcd-operator/pkg/debug"
"github.com/coreos/etcd-operator/pkg/garbagecollection"
"github.com/coreos/etcd-operator/pkg/util/constants"
"github.com/coreos/etcd-operator/pkg/util/k8sutil"
Expand Down Expand Up @@ -72,6 +73,7 @@ var (

func init() {
flag.BoolVar(&analyticsEnabled, "analytics", true, "Send analytical event (Cluster Created/Deleted etc.) to Google Analytics")
flag.StringVar(&debug.DebugFilePath, "debug-logfile-path", "", "path where the debug logfile will be written, recommended to be under: /var/tmp/etcd-operator/debug/ to avoid any issue with lack of write permissions")

flag.StringVar(&pvProvisioner, "pv-provisioner", constants.PVProvisionerGCEPD, "persistent volume provisioner type")
flag.StringVar(&awsSecret, "backup-aws-secret", "",
Expand Down
33 changes: 26 additions & 7 deletions pkg/cluster/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"time"

"github.com/coreos/etcd-operator/pkg/backup/s3/s3config"
"github.com/coreos/etcd-operator/pkg/debug"
"github.com/coreos/etcd-operator/pkg/garbagecollection"
"github.com/coreos/etcd-operator/pkg/spec"
"github.com/coreos/etcd-operator/pkg/util/etcdutil"
Expand Down Expand Up @@ -66,6 +67,8 @@ type Config struct {

type Cluster struct {
logger *logrus.Entry
// debug logger for self hosted cluster
debugLogger *debug.DebugLogger

config Config

Expand Down Expand Up @@ -93,14 +96,20 @@ type Cluster struct {

func New(config Config, cl *spec.Cluster, stopC <-chan struct{}, wg *sync.WaitGroup) *Cluster {
lg := logrus.WithField("pkg", "cluster").WithField("cluster-name", cl.Metadata.Name)
var debugLogger *debug.DebugLogger
if cl.Spec.SelfHosted != nil {
debugLogger = debug.New(cl.Metadata.Name)
}

c := &Cluster{
logger: lg,
config: config,
cluster: cl,
eventCh: make(chan *clusterEvent, 100),
stopCh: make(chan struct{}),
status: cl.Status.Copy(),
gc: garbagecollection.New(config.KubeCli, cl.Metadata.Namespace),
logger: lg,
debugLogger: debugLogger,
config: config,
cluster: cl,
eventCh: make(chan *clusterEvent, 100),
stopCh: make(chan struct{}),
status: cl.Status.Copy(),
gc: garbagecollection.New(config.KubeCli, cl.Metadata.Namespace),
}

wg.Add(1)
Expand Down Expand Up @@ -247,6 +256,9 @@ func (c *Cluster) run(stopC <-chan struct{}) {
c.reportFailedStatus()

c.logger.Infof("deleting the failed cluster")
if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil {
c.debugLogger.LogMessage("deleting the failed cluster")
}
c.delete()
}

Expand Down Expand Up @@ -477,6 +489,9 @@ func (c *Cluster) removePod(name string) error {
ns := c.cluster.Metadata.Namespace
opts := metav1.NewDeleteOptions(podTerminationGracePeriod)
err := c.config.KubeCli.Core().Pods(ns).Delete(name, opts)
if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil {
c.debugLogger.LogPodDeletion(name, err)
}
if err != nil {
if !k8sutil.IsKubernetesResourceNotFoundError(err) {
return err
Expand Down Expand Up @@ -632,4 +647,8 @@ func (c *Cluster) logSpecUpdate(newSpec spec.ClusterSpec) {
for _, m := range strings.Split(string(newSpecBytes), "\n") {
c.logger.Info(m)
}

if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil {
c.debugLogger.LogClusterSpecUpdate(string(oldSpecBytes), string(newSpecBytes))
}
}
11 changes: 11 additions & 0 deletions pkg/cluster/self_hosted.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,9 @@ func (c *Cluster) addOneSelfHostedMember() error {
pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, c.members.ClientURLs(), c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())

_, err = c.config.KubeCli.CoreV1().Pods(ns).Create(pod)
if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil {
c.debugLogger.LogPodCreation(pod, err)
}
if err != nil {
return err
}
Expand All @@ -154,7 +157,11 @@ func (c *Cluster) newSelfHostedSeedMember() error {
initialCluster := []string{newMember.Name + "=" + newMember.PeerURL()}

pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())

_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 3*60*time.Second)
if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil {
c.debugLogger.LogPodCreation(pod, err)
}
if err != nil {
return err
}
Expand Down Expand Up @@ -191,7 +198,11 @@ func (c *Cluster) migrateBootMember() error {

pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
ns := c.cluster.Metadata.Namespace

_, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 3*60*time.Second)
if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil {
c.debugLogger.LogPodCreation(pod, err)
}
if err != nil {
return err
}
Expand Down
97 changes: 97 additions & 0 deletions pkg/debug/debug_logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package debug

import (
"os"
"path"

"github.com/coreos/etcd-operator/pkg/util/k8sutil"

"github.com/Sirupsen/logrus"
"k8s.io/client-go/pkg/api/v1"
)

var (
// This flag should be set to enable debug logging
DebugFilePath string
)

type DebugLogger struct {
// regular log to stdout
logger *logrus.Entry
// log to file for debugging self hosted clusters
fileLogger *logrus.Logger
}

func New(clusterName string) *DebugLogger {
if len(DebugFilePath) == 0 {
return nil
}

logger := logrus.WithField("pkg", "debug")
err := os.MkdirAll(path.Dir(DebugFilePath), 0755)
if err != nil {
logger.Errorf("Could not create debug log directory (%v), debug logging will not be performed: %v", path.Dir(DebugFilePath), err)
return nil
}

logFile, err := os.OpenFile(DebugFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
logger.Errorf("failed to open debug log file(%v): %v", DebugFilePath, err)
return nil
}

l := logrus.New()
l.Out = logFile
l.Infof("Starting debug logs for self-hosted etcd cluster: %v", clusterName)
return &DebugLogger{
logger: logrus.WithField("pkg", "debug"),
fileLogger: l,
}
}

func (dl *DebugLogger) LogPodCreation(pod *v1.Pod, podCreationErr error) {
if podCreationErr != nil {
dl.fileLogger.Infof("failed to create pod (%s): %v ", pod.Name, podCreationErr)
return
}

podSpec, err := k8sutil.PodSpecToPrettyJSON(pod)
if err != nil {
dl.fileLogger.Infof("failed to get readable spec for pod(%v): %v ", pod.Name, err)
}
dl.fileLogger.Infof("created pod (%s) with spec: %s\n", pod.Name, podSpec)
}

func (dl *DebugLogger) LogPodDeletion(podName string, podDeletionErr error) {
if podDeletionErr != nil {
if !k8sutil.IsKubernetesResourceNotFoundError(podDeletionErr) {
dl.fileLogger.Infof("failed to delete pod (%s): %v ", podName, podDeletionErr)
return
}
dl.fileLogger.Infof("pod (%s) not found while trying to delete: %v ", podName, podDeletionErr)
return
}
dl.fileLogger.Infof("deleted pod (%s)", podName)
}

func (dl *DebugLogger) LogClusterSpecUpdate(oldSpec, newSpec string) {
dl.fileLogger.Infof("spec update: \nOld:\n%v \nNew:\n%v\n", oldSpec, newSpec)
}

func (dl *DebugLogger) LogMessage(msg string) {
dl.fileLogger.Infof(msg)
}
9 changes: 9 additions & 0 deletions pkg/util/k8sutil/pod_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package k8sutil

import (
"encoding/json"
"fmt"

"github.com/coreos/etcd-operator/pkg/spec"
Expand Down Expand Up @@ -168,3 +169,11 @@ func getPodReadyCondition(status *v1.PodStatus) *v1.PodCondition {
}
return nil
}

func PodSpecToPrettyJSON(pod *v1.Pod) (string, error) {
bytes, err := json.MarshalIndent(pod.Spec, "", " ")
if err != nil {
return "", err
}
return string(bytes), nil
}

0 comments on commit da8966f

Please sign in to comment.