-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Implemented Port Labeling and Driver Configurations #415
Merged
Merged
Changes from 23 commits
Commits
Show all changes
27 commits
Select commit
Hold shift + click to select a range
fbb082c
Adding the port hcl object to example
diptanu 7f6e940
Implemented port labeling and driver configs
diptanu 4d16daf
DRYed the code
diptanu 4186e70
Making the config for drivers public
diptanu e8236c0
Fixed the parsing and encoding logic
diptanu a52bff8
Fixed the api tests
diptanu 5c95e7f
Fixed the client tests
diptanu 51e3c99
Fixing the scheduler tests
diptanu 87f49c4
Fixed the tests for jobspec
diptanu 0d69778
Fixed compilation issues with driver tests
diptanu 05d21c5
Fixed the allocdir tests
diptanu 72f82a7
Fixed the test related to setting env variables in tasks
diptanu 76a005b
Fixed the exec driver config
diptanu b6e3b8d
Fixed the structs test
diptanu 7b2e3ee
Making sure that there is only one port_map block in the docker drive…
diptanu 7ebe235
Fixed the java driver config
diptanu 042532f
RawToSting true for structs codec. Needed for encoding strings in nil…
dadgar 363fced
mapstructure close quotes
dadgar 174ac10
Adding comments to fields in the docker driver config
diptanu 041f48d
Fixed the docker auth config
diptanu c01f2a3
Made the code more DRY
diptanu bc2efb7
jobspec: add test for types of nested configs
mitchellh 3d4fdbb
Avoid map[interface{}]interface{} conversions in the msgpack codec
dadgar 1d1e4bd
Merge branch 'master' into f-port-labels
cbednarski 28757ca
Change logging port as %d to logging port.Value
cbednarski 6403375
go fmt
cbednarski b8d468f
Addressed some review comments
diptanu File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,13 +16,45 @@ import ( | |
"github.com/hashicorp/nomad/client/driver/args" | ||
"github.com/hashicorp/nomad/client/fingerprint" | ||
"github.com/hashicorp/nomad/nomad/structs" | ||
"github.com/mitchellh/mapstructure" | ||
) | ||
|
||
type DockerDriver struct { | ||
DriverContext | ||
fingerprint.StaticFingerprinter | ||
} | ||
|
||
type DockerAuthConfig struct { | ||
UserName string `mapstructure:"auth.username"` // user name of the registry | ||
Password string `mapstructure:"auth.password"` // password to access the registry | ||
Email string `mapstructure:"auth.email"` // email address of the user who is allowed to access the registry | ||
ServerAddress string `mapstructure:"auth.server_address"` // server address of the registry | ||
|
||
} | ||
|
||
type DockerDriverConfig struct { | ||
DockerAuthConfig | ||
ImageName string `mapstructure:"image"` // Container's Image Name | ||
Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up | ||
Args string `mapstructure:"args"` // The arguments to the Command/Entrypoint | ||
NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none | ||
PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and the ports exposed on the container | ||
Privileged bool `mapstructure:"privileged"` // Flag to run the container in priviledged mode | ||
DNS string `mapstructure:"dns_server"` // DNS Server for containers | ||
SearchDomains string `mapstructure:"search_domains"` | ||
} | ||
|
||
func (c *DockerDriverConfig) Validate() error { | ||
if c.ImageName == "" { | ||
return fmt.Errorf("Docker Driver needs an image name") | ||
} | ||
|
||
if len(c.PortMap) > 1 { | ||
return fmt.Errorf("Only one port_map block is allowed in the docker driver config") | ||
} | ||
return nil | ||
} | ||
|
||
type dockerPID struct { | ||
ImageID string | ||
ContainerID string | ||
|
@@ -116,7 +148,7 @@ func (d *DockerDriver) containerBinds(alloc *allocdir.AllocDir, task *structs.Ta | |
} | ||
|
||
// createContainer initializes a struct needed to call docker.client.CreateContainer() | ||
func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (docker.CreateContainerOptions, error) { | ||
func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, driverConfig *DockerDriverConfig) (docker.CreateContainerOptions, error) { | ||
var c docker.CreateContainerOptions | ||
if task.Resources == nil { | ||
d.logger.Printf("[ERR] driver.docker: task.Resources is empty") | ||
|
@@ -134,8 +166,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (do | |
env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) | ||
|
||
config := &docker.Config{ | ||
Env: env.List(), | ||
Image: task.Config["image"], | ||
Image: driverConfig.ImageName, | ||
} | ||
|
||
hostConfig := &docker.HostConfig{ | ||
|
@@ -184,22 +215,18 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (do | |
return c, fmt.Errorf("Unable to parse docker.privileged.enabled: %s", err) | ||
} | ||
|
||
if v, ok := task.Config["privileged"]; ok { | ||
taskPrivileged, err := strconv.ParseBool(v) | ||
if err != nil { | ||
return c, fmt.Errorf("Unable to parse boolean value from task config option 'privileged': %v", err) | ||
} | ||
if taskPrivileged && !hostPrivileged { | ||
if driverConfig.Privileged { | ||
if !hostPrivileged { | ||
return c, fmt.Errorf(`Unable to set privileged flag since "docker.privileged.enabled" is false`) | ||
} | ||
|
||
hostConfig.Privileged = taskPrivileged | ||
hostConfig.Privileged = driverConfig.Privileged | ||
} | ||
|
||
// set DNS servers | ||
dns, ok := task.Config["dns-servers"] | ||
dns := driverConfig.DNS | ||
|
||
if ok && dns != "" { | ||
if dns != "" { | ||
for _, v := range strings.Split(dns, ",") { | ||
ip := strings.TrimSpace(v) | ||
if net.ParseIP(ip) != nil { | ||
|
@@ -211,16 +238,16 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (do | |
} | ||
|
||
// set DNS search domains | ||
dnsSearch, ok := task.Config["search-domains"] | ||
dnsSearch := driverConfig.SearchDomains | ||
|
||
if ok && dnsSearch != "" { | ||
if dnsSearch != "" { | ||
for _, v := range strings.Split(dnsSearch, ",") { | ||
hostConfig.DNSSearch = append(hostConfig.DNSSearch, strings.TrimSpace(v)) | ||
} | ||
} | ||
|
||
mode, ok := task.Config["network_mode"] | ||
if !ok || mode == "" { | ||
mode := driverConfig.NetworkMode | ||
if mode == "" { | ||
// docker default | ||
d.logger.Printf("[WARN] driver.docker: no mode specified for networking, defaulting to bridge") | ||
mode = "bridge" | ||
|
@@ -245,73 +272,70 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (do | |
publishedPorts := map[docker.Port][]docker.PortBinding{} | ||
exposedPorts := map[docker.Port]struct{}{} | ||
|
||
for _, port := range network.ListStaticPorts() { | ||
publishedPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} | ||
publishedPorts[docker.Port(strconv.Itoa(port)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} | ||
for _, port := range network.ReservedPorts { | ||
publishedPorts[docker.Port(strconv.Itoa(port.Value)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port.Value)}} | ||
publishedPorts[docker.Port(strconv.Itoa(port.Value)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port.Value)}} | ||
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)\n", network.IP, port, port) | ||
exposedPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = struct{}{} | ||
exposedPorts[docker.Port(strconv.Itoa(port)+"/udp")] = struct{}{} | ||
exposedPorts[docker.Port(strconv.Itoa(port.Value)+"/tcp")] = struct{}{} | ||
exposedPorts[docker.Port(strconv.Itoa(port.Value)+"/udp")] = struct{}{} | ||
d.logger.Printf("[DEBUG] driver.docker: exposed port %d\n", port) | ||
} | ||
|
||
for label, port := range network.MapDynamicPorts() { | ||
// If the label is numeric we expect that there is a service | ||
// listening on that port inside the container. In this case we'll | ||
// setup a mapping from our random host port to the label port. | ||
// | ||
// Otherwise we'll setup a direct 1:1 mapping from the host port to | ||
// the container, and assume that the process inside will read the | ||
// environment variable and bind to the correct port. | ||
if _, err := strconv.Atoi(label); err == nil { | ||
publishedPorts[docker.Port(label+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} | ||
publishedPorts[docker.Port(label+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} | ||
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %s (mapped)", network.IP, port, label) | ||
exposedPorts[docker.Port(label+"/tcp")] = struct{}{} | ||
exposedPorts[docker.Port(label+"/udp")] = struct{}{} | ||
d.logger.Printf("[DEBUG] driver.docker: exposed port %d\n", port) | ||
} else { | ||
publishedPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} | ||
publishedPorts[docker.Port(strconv.Itoa(port)+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: strconv.Itoa(port)}} | ||
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d for label %s\n", network.IP, port, port, label) | ||
exposedPorts[docker.Port(strconv.Itoa(port)+"/tcp")] = struct{}{} | ||
exposedPorts[docker.Port(strconv.Itoa(port)+"/udp")] = struct{}{} | ||
d.logger.Printf("[DEBUG] driver.docker: exposed port %d\n", port) | ||
containerToHostPortMap := make(map[string]int) | ||
for _, port := range network.DynamicPorts { | ||
containerPort, ok := driverConfig.PortMap[0][port.Label] | ||
if !ok { | ||
containerPort = port.Value | ||
} | ||
cp := strconv.Itoa(containerPort) | ||
hostPort := strconv.Itoa(port.Value) | ||
publishedPorts[docker.Port(cp+"/tcp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPort}} | ||
publishedPorts[docker.Port(cp+"/udp")] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPort}} | ||
d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPort) | ||
exposedPorts[docker.Port(cp+"/tcp")] = struct{}{} | ||
exposedPorts[docker.Port(cp+"/udp")] = struct{}{} | ||
d.logger.Printf("[DEBUG] driver.docker: exposed port %d\n", hostPort) | ||
containerToHostPortMap[cp] = port.Value | ||
} | ||
|
||
env.SetPorts(containerToHostPortMap) | ||
hostConfig.PortBindings = publishedPorts | ||
config.ExposedPorts = exposedPorts | ||
} | ||
|
||
rawArgs, hasArgs := task.Config["args"] | ||
parsedArgs, err := args.ParseAndReplace(rawArgs, env.Map()) | ||
parsedArgs, err := args.ParseAndReplace(driverConfig.Args, env.Map()) | ||
if err != nil { | ||
return c, err | ||
} | ||
|
||
// If the user specified a custom command to run as their entrypoint, we'll | ||
// inject it here. | ||
if command, ok := task.Config["command"]; ok { | ||
if command := driverConfig.Command; command != "" { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
cmd := []string{command} | ||
if hasArgs { | ||
if driverConfig.Args != "" { | ||
cmd = append(cmd, parsedArgs...) | ||
} | ||
config.Cmd = cmd | ||
} else if hasArgs { | ||
} else if driverConfig.Args != "" { | ||
d.logger.Println("[DEBUG] driver.docker: ignoring args because command not specified") | ||
} | ||
|
||
config.Env = env.List() | ||
return docker.CreateContainerOptions{ | ||
Config: config, | ||
HostConfig: hostConfig, | ||
}, nil | ||
} | ||
|
||
func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { | ||
// Get the image from config | ||
image, ok := task.Config["image"] | ||
if !ok || image == "" { | ||
return nil, fmt.Errorf("Image not specified") | ||
var driverConfig DockerDriverConfig | ||
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil { | ||
return nil, err | ||
} | ||
image := driverConfig.ImageName | ||
|
||
if err := driverConfig.Validate(); err != nil { | ||
return nil, err | ||
} | ||
if task.Resources == nil { | ||
return nil, fmt.Errorf("Resources are not specified") | ||
|
@@ -361,10 +385,10 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle | |
} | ||
|
||
authOptions := docker.AuthConfiguration{ | ||
Username: task.Config["auth.username"], | ||
Password: task.Config["auth.password"], | ||
Email: task.Config["auth.email"], | ||
ServerAddress: task.Config["auth.server-address"], | ||
Username: driverConfig.UserName, | ||
Password: driverConfig.Password, | ||
Email: driverConfig.Email, | ||
ServerAddress: driverConfig.ServerAddress, | ||
} | ||
|
||
err = client.PullImage(pullOptions, authOptions) | ||
|
@@ -384,7 +408,7 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle | |
d.logger.Printf("[DEBUG] driver.docker: using image %s", dockerImage.ID) | ||
d.logger.Printf("[INFO] driver.docker: identified image %s as %s", image, dockerImage.ID) | ||
|
||
config, err := d.createContainer(ctx, task) | ||
config, err := d.createContainer(ctx, task, &driverConfig) | ||
if err != nil { | ||
d.logger.Printf("[ERR] driver.docker: %s", err) | ||
return nil, fmt.Errorf("Failed to create container config for image %s", image) | ||
|
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Missed one? :)