-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #4758 from hashicorp/f-driver-plugin-docker
docklog: add go-plugin for forwarding of docker logs
- Loading branch information
Showing
7 changed files
with
751 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
package docklog | ||
|
||
import ( | ||
"context" | ||
|
||
"github.com/hashicorp/nomad/drivers/docker/docklog/proto" | ||
) | ||
|
||
// dockerLoggerClient implements the dockerLogger interface for client side requests | ||
type dockerLoggerClient struct { | ||
client proto.DockerLoggerClient | ||
} | ||
|
||
// Start proxies the Start client side func to the protobuf interface | ||
func (c *dockerLoggerClient) Start(opts *StartOpts) error { | ||
req := &proto.StartRequest{ | ||
Endpoint: opts.Endpoint, | ||
ContainerId: opts.ContainerID, | ||
StdoutFifo: opts.Stdout, | ||
StderrFifo: opts.Stderr, | ||
|
||
TlsCert: opts.TLSCert, | ||
TlsKey: opts.TLSKey, | ||
TlsCa: opts.TLSCA, | ||
} | ||
_, err := c.client.Start(context.Background(), req) | ||
return err | ||
} | ||
|
||
// Stop proxies the Stop client side func to the protobuf interface | ||
func (c *dockerLoggerClient) Stop() error { | ||
req := &proto.StopRequest{} | ||
_, err := c.client.Stop(context.Background(), req) | ||
return err | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,141 @@ | ||
package docklog | ||
|
||
import ( | ||
"fmt" | ||
"io" | ||
|
||
docker "github.com/fsouza/go-dockerclient" | ||
hclog "github.com/hashicorp/go-hclog" | ||
multierror "github.com/hashicorp/go-multierror" | ||
"github.com/hashicorp/nomad/client/lib/fifo" | ||
"golang.org/x/net/context" | ||
) | ||
|
||
// DockerLogger is a small utility to forward logs from a docker container to a target | ||
// destination | ||
type DockerLogger interface { | ||
Start(*StartOpts) error | ||
Stop() error | ||
} | ||
|
||
// StartOpts are the options needed to start docker log monitoring | ||
type StartOpts struct { | ||
// Endpoint sets the docker client endpoint, defaults to environment if not set | ||
Endpoint string | ||
|
||
// ContainerID of the container to monitor logs for | ||
ContainerID string | ||
|
||
// Stdout path to fifo | ||
Stdout string | ||
//Stderr path to fifo | ||
Stderr string | ||
|
||
// TLS settings for docker client | ||
TLSCert string | ||
TLSKey string | ||
TLSCA string | ||
} | ||
|
||
// NewDockerLogger returns an implementation of the DockerLogger interface | ||
func NewDockerLogger(logger hclog.Logger) DockerLogger { | ||
return &dockerLogger{logger: logger} | ||
} | ||
|
||
// dockerLogger implements the DockerLogger interface | ||
type dockerLogger struct { | ||
logger hclog.Logger | ||
|
||
stdout io.WriteCloser | ||
stderr io.WriteCloser | ||
cancelCtx context.CancelFunc | ||
} | ||
|
||
// Start log monitoring | ||
func (d *dockerLogger) Start(opts *StartOpts) error { | ||
client, err := d.getDockerClient(opts) | ||
if err != nil { | ||
return fmt.Errorf("failed to open docker client: %v", err) | ||
} | ||
|
||
if d.stdout == nil { | ||
stdout, err := fifo.Open(opts.Stdout) | ||
if err != nil { | ||
return fmt.Errorf("failed to open fifo for path %s: %v", opts.Stdout, err) | ||
} | ||
d.stdout = stdout | ||
} | ||
if d.stderr == nil { | ||
stderr, err := fifo.Open(opts.Stderr) | ||
if err != nil { | ||
return fmt.Errorf("failed to open fifo for path %s: %v", opts.Stdout, err) | ||
} | ||
d.stderr = stderr | ||
} | ||
ctx, cancel := context.WithCancel(context.Background()) | ||
d.cancelCtx = cancel | ||
|
||
logOpts := docker.LogsOptions{ | ||
Context: ctx, | ||
Container: opts.ContainerID, | ||
OutputStream: d.stdout, | ||
ErrorStream: d.stderr, | ||
Since: 0, | ||
Follow: true, | ||
Stdout: true, | ||
Stderr: true, | ||
} | ||
|
||
go func() { client.Logs(logOpts) }() | ||
return nil | ||
|
||
} | ||
|
||
// Stop log monitoring | ||
func (d *dockerLogger) Stop() error { | ||
if d.cancelCtx != nil { | ||
d.cancelCtx() | ||
} | ||
if d.stdout != nil { | ||
d.stdout.Close() | ||
} | ||
if d.stderr != nil { | ||
d.stderr.Close() | ||
} | ||
return nil | ||
} | ||
|
||
func (d *dockerLogger) getDockerClient(opts *StartOpts) (*docker.Client, error) { | ||
var err error | ||
var merr multierror.Error | ||
var newClient *docker.Client | ||
|
||
// Default to using whatever is configured in docker.endpoint. If this is | ||
// not specified we'll fall back on NewClientFromEnv which reads config from | ||
// the DOCKER_* environment variables DOCKER_HOST, DOCKER_TLS_VERIFY, and | ||
// DOCKER_CERT_PATH. This allows us to lock down the config in production | ||
// but also accept the standard ENV configs for dev and test. | ||
if opts.Endpoint != "" { | ||
if opts.TLSCert+opts.TLSKey+opts.TLSCA != "" { | ||
d.logger.Debug("using TLS client connection to docker", "endpoint", opts.Endpoint) | ||
newClient, err = docker.NewTLSClient(opts.Endpoint, opts.TLSCert, opts.TLSKey, opts.TLSCA) | ||
if err != nil { | ||
merr.Errors = append(merr.Errors, err) | ||
} | ||
} else { | ||
d.logger.Debug("using plaintext client connection to docker", "endpoint", opts.Endpoint) | ||
newClient, err = docker.NewClient(opts.Endpoint) | ||
if err != nil { | ||
merr.Errors = append(merr.Errors, err) | ||
} | ||
} | ||
} else { | ||
d.logger.Debug("using client connection initialized from environment") | ||
newClient, err = docker.NewClientFromEnv() | ||
if err != nil { | ||
merr.Errors = append(merr.Errors, err) | ||
} | ||
} | ||
|
||
return newClient, merr.ErrorOrNil() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
package docklog | ||
|
||
import ( | ||
"bytes" | ||
"fmt" | ||
"testing" | ||
|
||
docker "github.com/fsouza/go-dockerclient" | ||
"github.com/hashicorp/nomad/helper/testlog" | ||
"github.com/hashicorp/nomad/testutil" | ||
"github.com/stretchr/testify/require" | ||
"golang.org/x/net/context" | ||
) | ||
|
||
func TestDockerLogger(t *testing.T) { | ||
t.Parallel() | ||
require := require.New(t) | ||
|
||
client, err := docker.NewClientFromEnv() | ||
if err != nil { | ||
t.Skip("docker unavailable:", err) | ||
} | ||
|
||
containerConf := docker.CreateContainerOptions{ | ||
Config: &docker.Config{ | ||
Cmd: []string{ | ||
"/bin/ash", "-c", "touch /tmp/docklog; tail -f /tmp/docklog", | ||
}, | ||
Image: "alpine", | ||
}, | ||
Context: context.Background(), | ||
} | ||
|
||
container, err := client.CreateContainer(containerConf) | ||
require.NoError(err) | ||
|
||
defer client.RemoveContainer(docker.RemoveContainerOptions{ | ||
ID: container.ID, | ||
Force: true, | ||
}) | ||
|
||
err = client.StartContainer(container.ID, nil) | ||
require.NoError(err) | ||
|
||
testutil.WaitForResult(func() (bool, error) { | ||
container, err = client.InspectContainer(container.ID) | ||
if err != nil { | ||
return false, err | ||
} | ||
if !container.State.Running { | ||
return false, fmt.Errorf("container not running") | ||
} | ||
return true, nil | ||
}, func(err error) { | ||
require.NoError(err) | ||
}) | ||
|
||
stdout := &noopCloser{bytes.NewBuffer(nil)} | ||
stderr := &noopCloser{bytes.NewBuffer(nil)} | ||
|
||
dl := NewDockerLogger(testlog.HCLogger(t)).(*dockerLogger) | ||
dl.stdout = stdout | ||
dl.stderr = stderr | ||
require.NoError(dl.Start(&StartOpts{ | ||
ContainerID: container.ID, | ||
})) | ||
|
||
echoToContainer(t, client, container.ID, "abc") | ||
echoToContainer(t, client, container.ID, "123") | ||
|
||
testutil.WaitForResult(func() (bool, error) { | ||
act := stdout.String() | ||
if "abc\n123\n" != act { | ||
return false, fmt.Errorf("expected abc\\n123\\n for stdout but got %s", act) | ||
} | ||
|
||
return true, nil | ||
}, func(err error) { | ||
require.NoError(err) | ||
}) | ||
} | ||
|
||
func echoToContainer(t *testing.T, client *docker.Client, id string, line string) { | ||
op := docker.CreateExecOptions{ | ||
Container: id, | ||
Cmd: []string{ | ||
"/bin/ash", "-c", | ||
fmt.Sprintf("echo %s >>/tmp/docklog", line), | ||
}, | ||
} | ||
|
||
exec, err := client.CreateExec(op) | ||
require.NoError(t, err) | ||
require.NoError(t, client.StartExec(exec.ID, docker.StartExecOptions{Detach: true})) | ||
} | ||
|
||
type noopCloser struct { | ||
*bytes.Buffer | ||
} | ||
|
||
func (*noopCloser) Close() error { | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
package docklog | ||
|
||
import ( | ||
"context" | ||
"os/exec" | ||
|
||
hclog "github.com/hashicorp/go-hclog" | ||
plugin "github.com/hashicorp/go-plugin" | ||
"github.com/hashicorp/nomad/drivers/docker/docklog/proto" | ||
"github.com/hashicorp/nomad/helper/discover" | ||
"github.com/hashicorp/nomad/plugins/base" | ||
"google.golang.org/grpc" | ||
) | ||
|
||
const pluginName = "docker_logger" | ||
|
||
// LaunchDockerLogger launches an instance of DockerLogger | ||
// TODO: Integrate with base plugin loader | ||
func LaunchDockerLogger(logger hclog.Logger) (DockerLogger, *plugin.Client, error) { | ||
logger = logger.Named(pluginName) | ||
bin, err := discover.NomadExecutable() | ||
if err != nil { | ||
return nil, nil, err | ||
} | ||
|
||
client := plugin.NewClient(&plugin.ClientConfig{ | ||
HandshakeConfig: base.Handshake, | ||
Plugins: map[string]plugin.Plugin{ | ||
pluginName: &Plugin{impl: NewDockerLogger(hclog.L().Named(pluginName))}, | ||
}, | ||
Cmd: exec.Command(bin, pluginName), | ||
AllowedProtocols: []plugin.Protocol{ | ||
plugin.ProtocolGRPC, | ||
}, | ||
}) | ||
|
||
rpcClient, err := client.Client() | ||
if err != nil { | ||
return nil, nil, err | ||
} | ||
|
||
raw, err := rpcClient.Dispense(pluginName) | ||
if err != nil { | ||
return nil, nil, err | ||
} | ||
|
||
l := raw.(DockerLogger) | ||
return l, client, nil | ||
|
||
} | ||
|
||
// Plugin is the go-plugin implementation | ||
type Plugin struct { | ||
plugin.NetRPCUnsupportedPlugin | ||
impl DockerLogger | ||
} | ||
|
||
// GRPCServer registered the server side implementation with the grpc server | ||
func (p *Plugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { | ||
proto.RegisterDockerLoggerServer(s, &dockerLoggerServer{ | ||
impl: p.impl, | ||
broker: broker, | ||
}) | ||
return nil | ||
} | ||
|
||
// GRPCClient returns a client side implementation of the plugin | ||
func (p *Plugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { | ||
return &dockerLoggerClient{client: proto.NewDockerLoggerClient(c)}, nil | ||
} |
Oops, something went wrong.