diff --git a/cmd/operator/main.go b/cmd/operator/main.go index a671de6fe..b4120ed61 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -28,6 +28,7 @@ import ( "github.com/coreos/etcd-operator/pkg/backup/s3/s3config" "github.com/coreos/etcd-operator/pkg/chaos" "github.com/coreos/etcd-operator/pkg/controller" + "github.com/coreos/etcd-operator/pkg/debug" "github.com/coreos/etcd-operator/pkg/garbagecollection" "github.com/coreos/etcd-operator/pkg/util/constants" "github.com/coreos/etcd-operator/pkg/util/k8sutil" @@ -72,6 +73,7 @@ var ( func init() { flag.BoolVar(&analyticsEnabled, "analytics", true, "Send analytical event (Cluster Created/Deleted etc.) to Google Analytics") + flag.StringVar(&debug.DebugFilePath, "debug-logfile-path", "", "path where the debug logfile will be written, recommended to be under: /var/tmp/etcd-operator/debug/ to avoid any issue with lack of write permissions") flag.StringVar(&pvProvisioner, "pv-provisioner", constants.PVProvisionerGCEPD, "persistent volume provisioner type") flag.StringVar(&awsSecret, "backup-aws-secret", "", diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 3ec1f2044..fa9db7eea 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -25,6 +25,7 @@ import ( "time" "github.com/coreos/etcd-operator/pkg/backup/s3/s3config" + "github.com/coreos/etcd-operator/pkg/debug" "github.com/coreos/etcd-operator/pkg/garbagecollection" "github.com/coreos/etcd-operator/pkg/spec" "github.com/coreos/etcd-operator/pkg/util/etcdutil" @@ -66,6 +67,8 @@ type Config struct { type Cluster struct { logger *logrus.Entry + // debug logger for self hosted cluster + debugLogger *debug.DebugLogger config Config @@ -93,14 +96,20 @@ type Cluster struct { func New(config Config, cl *spec.Cluster, stopC <-chan struct{}, wg *sync.WaitGroup) *Cluster { lg := logrus.WithField("pkg", "cluster").WithField("cluster-name", cl.Metadata.Name) + var debugLogger *debug.DebugLogger + if cl.Spec.SelfHosted != nil { + debugLogger = debug.New(cl.Metadata.Name) + } + c := &Cluster{ - logger: lg, - config: config, - cluster: cl, - eventCh: make(chan *clusterEvent, 100), - stopCh: make(chan struct{}), - status: cl.Status.Copy(), - gc: garbagecollection.New(config.KubeCli, cl.Metadata.Namespace), + logger: lg, + debugLogger: debugLogger, + config: config, + cluster: cl, + eventCh: make(chan *clusterEvent, 100), + stopCh: make(chan struct{}), + status: cl.Status.Copy(), + gc: garbagecollection.New(config.KubeCli, cl.Metadata.Namespace), } wg.Add(1) @@ -247,6 +256,9 @@ func (c *Cluster) run(stopC <-chan struct{}) { c.reportFailedStatus() c.logger.Infof("deleting the failed cluster") + if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil { + c.debugLogger.LogMessage("deleting the failed cluster") + } c.delete() } @@ -477,6 +489,9 @@ func (c *Cluster) removePod(name string) error { ns := c.cluster.Metadata.Namespace opts := metav1.NewDeleteOptions(podTerminationGracePeriod) err := c.config.KubeCli.Core().Pods(ns).Delete(name, opts) + if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil { + c.debugLogger.LogPodDeletion(name, err) + } if err != nil { if !k8sutil.IsKubernetesResourceNotFoundError(err) { return err @@ -632,4 +647,8 @@ func (c *Cluster) logSpecUpdate(newSpec spec.ClusterSpec) { for _, m := range strings.Split(string(newSpecBytes), "\n") { c.logger.Info(m) } + + if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil { + c.debugLogger.LogClusterSpecUpdate(string(oldSpecBytes), string(newSpecBytes)) + } } diff --git a/pkg/cluster/self_hosted.go b/pkg/cluster/self_hosted.go index 045b19d85..9c776122a 100644 --- a/pkg/cluster/self_hosted.go +++ b/pkg/cluster/self_hosted.go @@ -132,6 +132,9 @@ func (c *Cluster) addOneSelfHostedMember() error { pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, c.members.ClientURLs(), c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner()) _, err = c.config.KubeCli.CoreV1().Pods(ns).Create(pod) + if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil { + c.debugLogger.LogPodCreation(pod, err) + } if err != nil { return err } @@ -154,7 +157,11 @@ func (c *Cluster) newSelfHostedSeedMember() error { initialCluster := []string{newMember.Name + "=" + newMember.PeerURL()} pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner()) + _, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 3*60*time.Second) + if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil { + c.debugLogger.LogPodCreation(pod, err) + } if err != nil { return err } @@ -191,7 +198,11 @@ func (c *Cluster) migrateBootMember() error { pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner()) ns := c.cluster.Metadata.Namespace + _, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 3*60*time.Second) + if c.cluster.Spec.SelfHosted != nil && c.debugLogger != nil { + c.debugLogger.LogPodCreation(pod, err) + } if err != nil { return err } diff --git a/pkg/debug/debug_logger.go b/pkg/debug/debug_logger.go new file mode 100644 index 000000000..ab4b5e5e4 --- /dev/null +++ b/pkg/debug/debug_logger.go @@ -0,0 +1,97 @@ +// Copyright 2017 The etcd-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package debug + +import ( + "os" + "path" + + "github.com/coreos/etcd-operator/pkg/util/k8sutil" + + "github.com/Sirupsen/logrus" + "k8s.io/client-go/pkg/api/v1" +) + +var ( + // This flag should be set to enable debug logging + DebugFilePath string +) + +type DebugLogger struct { + // regular log to stdout + logger *logrus.Entry + // log to file for debugging self hosted clusters + fileLogger *logrus.Logger +} + +func New(clusterName string) *DebugLogger { + if len(DebugFilePath) == 0 { + return nil + } + + logger := logrus.WithField("pkg", "debug") + err := os.MkdirAll(path.Dir(DebugFilePath), 0755) + if err != nil { + logger.Errorf("Could not create debug log directory (%v), debug logging will not be performed: %v", path.Dir(DebugFilePath), err) + return nil + } + + logFile, err := os.OpenFile(DebugFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + logger.Errorf("failed to open debug log file(%v): %v", DebugFilePath, err) + return nil + } + + l := logrus.New() + l.Out = logFile + l.Infof("Starting debug logs for self-hosted etcd cluster: %v", clusterName) + return &DebugLogger{ + logger: logrus.WithField("pkg", "debug"), + fileLogger: l, + } +} + +func (dl *DebugLogger) LogPodCreation(pod *v1.Pod, podCreationErr error) { + if podCreationErr != nil { + dl.fileLogger.Infof("failed to create pod (%s): %v ", pod.Name, podCreationErr) + return + } + + podSpec, err := k8sutil.PodSpecToPrettyJSON(pod) + if err != nil { + dl.fileLogger.Infof("failed to get readable spec for pod(%v): %v ", pod.Name, err) + } + dl.fileLogger.Infof("created pod (%s) with spec: %s\n", pod.Name, podSpec) +} + +func (dl *DebugLogger) LogPodDeletion(podName string, podDeletionErr error) { + if podDeletionErr != nil { + if !k8sutil.IsKubernetesResourceNotFoundError(podDeletionErr) { + dl.fileLogger.Infof("failed to delete pod (%s): %v ", podName, podDeletionErr) + return + } + dl.fileLogger.Infof("pod (%s) not found while trying to delete: %v ", podName, podDeletionErr) + return + } + dl.fileLogger.Infof("deleted pod (%s)", podName) +} + +func (dl *DebugLogger) LogClusterSpecUpdate(oldSpec, newSpec string) { + dl.fileLogger.Infof("spec update: \nOld:\n%v \nNew:\n%v\n", oldSpec, newSpec) +} + +func (dl *DebugLogger) LogMessage(msg string) { + dl.fileLogger.Infof(msg) +} diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 86098cfff..d05d38a65 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -15,6 +15,7 @@ package k8sutil import ( + "encoding/json" "fmt" "github.com/coreos/etcd-operator/pkg/spec" @@ -168,3 +169,11 @@ func getPodReadyCondition(status *v1.PodStatus) *v1.PodCondition { } return nil } + +func PodSpecToPrettyJSON(pod *v1.Pod) (string, error) { + bytes, err := json.MarshalIndent(pod.Spec, "", " ") + if err != nil { + return "", err + } + return string(bytes), nil +}