Skip to content

Commit

Permalink
Add LogCollector to E2E framework
Browse files Browse the repository at this point in the history
  • Loading branch information
fabriziopandini committed Sep 3, 2020
1 parent edbf631 commit 5037a94
Show file tree
Hide file tree
Showing 10 changed files with 195 additions and 13 deletions.
6 changes: 6 additions & 0 deletions test/e2e/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,13 @@ func setupSpecNamespace(ctx context.Context, specName string, clusterProxy frame
}

func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string, namespace *corev1.Namespace, cancelWatches context.CancelFunc, cluster *clusterv1.Cluster, intervalsGetter func(spec, key string) []interface{}, skipCleanup bool) {
Byf("Dumping logs from the %q workload cluster", cluster.Name)

// Dump all the logs from the workload cluster before deleting them.
clusterProxy.CollectWorkloadClusterLogs(ctx, cluster.Namespace, cluster.Name, filepath.Join(artifactFolder, "clusters", cluster.Name, "machines"))

Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name)

// Dump all Cluster API related resources to artifacts before deleting them.
framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{
Lister: clusterProxy.GetClient(),
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
kubeconfigPath := parts[3]

e2eConfig = loadE2EConfig(configPath)
bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme())
bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), framework.WithMachineLogCollector(framework.DockerLogCollector{}))
})

// Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads).
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/kcp_adoption.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu

By("Creating a workload cluster")

clusterName := fmt.Sprintf("cluster-%s", util.RandomString(6))
clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))
client := input.BootstrapClusterProxy.GetClient()
WaitForClusterIntervals := input.E2EConfig.GetIntervals(specName, "wait-cluster")
WaitForControlPlaneIntervals := input.E2EConfig.GetIntervals(specName, "wait-control-plane")
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/kcp_upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput)
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: fmt.Sprintf("cluster-%s", util.RandomString(6)),
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
Expand Down Expand Up @@ -121,7 +121,7 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput)
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: fmt.Sprintf("cluster-%s", util.RandomString(6)),
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
ControlPlaneMachineCount: pointer.Int64Ptr(3),
WorkerMachineCount: pointer.Int64Ptr(1),
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/md_upgrades.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func MachineDeploymentUpgradesSpec(ctx context.Context, inputGetter func() Machi
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: fmt.Sprintf("cluster-%s", util.RandomString(6)),
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/mhc_remediations.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: fmt.Sprintf("cluster-%s", util.RandomString(6)),
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/quick_start.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: fmt.Sprintf("cluster-%s", util.RandomString(6)),
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/self_hosted.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: fmt.Sprintf("cluster-%s", util.RandomString(6)),
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
Expand Down Expand Up @@ -124,7 +124,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
ClusterProxy: selfHostedClusterProxy,
ClusterctlConfigPath: input.ClusterctlConfigPath,
InfrastructureProviders: input.E2EConfig.InfrastructureProviders(),
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", "self-hosted"),
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name),
}, input.E2EConfig.GetIntervals(specName, "wait-controllers")...)

//TODO: refactor in to an helper func e.g. "MoveToSelfHostedAndWait"
Expand Down Expand Up @@ -161,13 +161,13 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{
Lister: selfHostedClusterProxy.GetClient(),
Namespace: namespace.Name,
LogPath: filepath.Join(input.ArtifactFolder, "clusters", "self-hosted", "resources"),
LogPath: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name, "resources"),
})
}
if selfHostedCluster != nil {
By("Moving the cluster back to bootstrap")
clusterctl.Move(ctx, clusterctl.MoveInput{
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", "self-hosted"),
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name),
ClusterctlConfigPath: input.ClusterctlConfigPath,
FromKubeconfigPath: selfHostedClusterProxy.GetKubeconfigPath(),
ToKubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
Expand Down
67 changes: 65 additions & 2 deletions test/framework/cluster_proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"io/ioutil"
"net/url"
"os"
"path"
goruntime "runtime"
"strings"

Expand Down Expand Up @@ -65,33 +66,61 @@ type ClusterProxy interface {
// GetWorkloadCluster returns a proxy to a workload cluster defined in the Kubernetes cluster.
GetWorkloadCluster(ctx context.Context, namespace, name string) ClusterProxy

// CollectWorkloadClusterLogs collects machines logs from the workload cluster.
CollectWorkloadClusterLogs(ctx context.Context, namespace, name, outputPath string)

// Dispose proxy's internal resources (the operation does not affects the Kubernetes cluster).
// This should be implemented as a synchronous function.
Dispose(context.Context)
}

// ClusterLogCollector defines an object that can collect logs from a machine.
type ClusterLogCollector interface {
// CollectMachineLog collects log from a machine.
// TODO: describe output folder struct
CollectMachineLog(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error
}

// Option is a configuration option supplied to NewClusterProxy
type Option func(*clusterProxy)

// WithMachineLogCollector allows to define the machine log collector to be used with this Cluster.
func WithMachineLogCollector(logCollector ClusterLogCollector) Option {
return func(c *clusterProxy) {
c.logCollector = logCollector
}
}

// clusterProxy provides a base implementation of the ClusterProxy interface.
type clusterProxy struct {
name string
kubeconfigPath string
scheme *runtime.Scheme
shouldCleanupKubeconfig bool
logCollector ClusterLogCollector
}

// NewClusterProxy returns a clusterProxy given a KubeconfigPath and the scheme defining the types hosted in the cluster.
// If a kubeconfig file isn't provided, standard kubeconfig locations will be used (kubectl loading rules apply).
func NewClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme) ClusterProxy {
func NewClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme, options ...Option) ClusterProxy {
Expect(scheme).NotTo(BeNil(), "scheme is required for NewClusterProxy")

if kubeconfigPath == "" {
kubeconfigPath = clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename()
}
return &clusterProxy{

proxy := &clusterProxy{
name: name,
kubeconfigPath: kubeconfigPath,
scheme: scheme,
shouldCleanupKubeconfig: false,
}

for _, o := range options {
o(proxy)
}

return proxy
}

// newFromAPIConfig returns a clusterProxy given a api.Config and the scheme defining the types hosted in the cluster.
Expand Down Expand Up @@ -192,6 +221,40 @@ func (p *clusterProxy) GetWorkloadCluster(ctx context.Context, namespace, name s
return newFromAPIConfig(name, config, p.scheme)
}

// CollectWorkloadClusterLogs collects machines logs from the workload cluster.
func (p *clusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace, name, outputPath string) {
if p.logCollector == nil {
return
}

machines, err := getMachinesInCluster(ctx, p.GetClient(), namespace, name)
Expect(err).ToNot(HaveOccurred(), "Failed to get machines for the %s/%s cluster", namespace, name)

for i := range machines.Items {
m := &machines.Items[i]
err := p.logCollector.CollectMachineLog(ctx, p.GetClient(), m, path.Join(outputPath, m.GetName()))
if err != nil {
// NB. we are treating failures in collecting logs as a non blocking operation (best effort)
fmt.Printf("Failed to get logs for machine %s, cluster %s/%s: %v\n", m.GetName(), namespace, name, err)
}
}
}

func getMachinesInCluster(ctx context.Context, c client.Client, namespace, name string) (*clusterv1.MachineList, error) {
if name == "" {
return nil, nil
}

machineList := &clusterv1.MachineList{}
labels := map[string]string{clusterv1.ClusterLabelName: name}

if err := c.List(ctx, machineList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil {
return nil, err
}

return machineList, nil
}

func (p *clusterProxy) getKubeconfig(ctx context.Context, namespace string, name string) *api.Config {
cl := p.GetClient()

Expand Down
113 changes: 113 additions & 0 deletions test/framework/docker_logcollector.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
"context"
"fmt"
"os"
"path/filepath"

clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/kind/pkg/errors"
"sigs.k8s.io/kind/pkg/exec"
)

// DockerLogCollector collect logs from a CAPD workload cluster.
type DockerLogCollector struct{}

func (k DockerLogCollector) CollectMachineLog(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error {
containerName := fmt.Sprintf("%s-%s", m.Spec.ClusterName, m.Name)
execToPathFn := func(outputFileName, command string, args ...string) func() error {
return func() error {
f, err := fileOnHost(filepath.Join(outputPath, outputFileName))
if err != nil {
return err
}
defer f.Close()
return execOnContainer(containerName, f, command, args...)
}
}
return errors.AggregateConcurrent([]func() error{
execToPathFn(
"journal.log",
"journalctl", "--no-pager", "--output=short-precise",
),
execToPathFn(
"kern.log",
"journalctl", "--no-pager", "--output=short-precise", "-k",
),
execToPathFn(
"kubelet-version.txt",
"kubelet", "--version",
),
execToPathFn(
"kubelet.log",
"journalctl", "--no-pager", "--output=short-precise", "-u", "kubelet.service",
),
execToPathFn(
"containerd-info.txt",
"crictl", "info",
),
execToPathFn(
"containerd.log",
"journalctl", "--no-pager", "--output=short-precise", "-u", "containerd.service",
),
})
}

// fileOnHost is a helper to create a file at path
// even if the parent directory doesn't exist
// in which case it will be created with ModePerm
func fileOnHost(path string) (*os.File, error) {
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
return nil, err
}
return os.Create(path)
}

// execOnContainer is an helper that runs a command on a CAPD node/container
func execOnContainer(containerName string, fileOnHost *os.File, command string, args ...string) error {
dockerArgs := []string{
"exec",
// run with privileges so we can remount etc..
// this might not make sense in the most general sense, but it is
// important to many kind commands
"--privileged",
}
// specify the container and command, after this everything will be
// args the the command in the container rather than to docker
dockerArgs = append(
dockerArgs,
containerName, // ... against the container
command, // with the command specified
)
dockerArgs = append(
dockerArgs,
// finally, with the caller args
args...,
)

cmd := exec.Command("docker", dockerArgs...)
cmd.SetEnv("PATH", os.Getenv("PATH"))

cmd.SetStderr(fileOnHost)
cmd.SetStdout(fileOnHost)

return errors.WithStack(cmd.Run())
}

0 comments on commit 5037a94

Please sign in to comment.