Merge branch 'master' into f-win-service

.circleci/config.yml

@@ -298,7 +298,7 @@ commands:
     parameters:
       version:
         type: string
-        default: 1.0.0
+        default: 1.2.3
     steps:
       - run:
           name: Install Vault << parameters.version >>

README.md

@@ -123,8 +123,6 @@ Who Uses Nomad
   * [Tech at N26 - The Bank in the Cloud](https://medium.com/insiden26/tech-at-n26-the-bank-in-the-cloud-e5ff818b528b)
 * Elsevier
   * [Eslevier’s Container Framework with Nomad, Terraform, and Consul](https://www.hashicorp.com/resources/elsevier-nomad-container-framework-demo)
-* Palantir
-  * [Enterprise Security at Palantir with the HashiCorp stack](https://www.hashicorp.com/resources/enterprise-security-hashicorp-stack)
 * Graymeta
   * [Backend Batch Processing At Scale with Nomad](https://www.hashicorp.com/resources/backend-batch-processing-nomad)
 * NIH NCBI

client/vaultclient/vaultclient.go

@@ -45,14 +45,6 @@ type VaultClient interface {
 	// StopRenewToken removes the token from the min-heap, stopping its
 	// renewal.
 	StopRenewToken(string) error
-
-	// RenewLease renews a vault secret's lease and adds the lease
-	// identifier to the min-heap for periodic renewal.
-	RenewLease(string, int) (<-chan error, error)
-
-	// StopRenewLease removes a secret's lease ID from the min-heap,
-	// stopping its renewal.
-	StopRenewLease(string) error
 }
 
 // Implementation of VaultClient interface to interact with vault and perform
@@ -325,44 +317,6 @@ func (c *vaultClient) RenewToken(token string, increment int) (<-chan error, err
 	return errCh, nil
 }
 
-// RenewLease renews the supplied lease identifier for a supplied duration (in
-// seconds) and adds it to the min-heap so that it gets renewed periodically by
-// the renewal loop. Any error returned during renewal will be written to a
-// buffered channel and the channel is returned instead of an actual error.
-// This helps the caller be notified of a renewal failure asynchronously for
-// appropriate actions to be taken. The caller of this function need not have
-// to close the error channel.
-func (c *vaultClient) RenewLease(leaseId string, increment int) (<-chan error, error) {
-	if leaseId == "" {
-		err := fmt.Errorf("missing lease ID")
-		return nil, err
-	}
-
-	if increment < 1 {
-		err := fmt.Errorf("increment cannot be less than 1")
-		return nil, err
-	}
-
-	// Create a buffered error channel
-	errCh := make(chan error, 1)
-
-	// Create a renewal request using the supplied lease and duration
-	renewalReq := &vaultClientRenewalRequest{
-		errCh:     errCh,
-		id:        leaseId,
-		increment: increment,
-	}
-
-	// Renew the secret and send any error to the dedicated error channel
-	if err := c.renew(renewalReq); err != nil {
-		c.logger.Error("error during renewal of lease", "error", err)
-		metrics.IncrCounter([]string{"client", "vault", "renew_lease_error"}, 1)
-		return nil, err
-	}
-
-	return errCh, nil
-}
-
 // renew is a common method to handle renewal of both tokens and secret leases.
 // It invokes a token renewal or a secret's lease renewal. If renewal is
 // successful, min-heap is updated based on the duration after which it needs
@@ -558,12 +512,6 @@ func (c *vaultClient) StopRenewToken(token string) error {
 	return c.stopRenew(token)
 }
 
-// StopRenewLease removes the item from the heap which represents the given
-// lease identifier.
-func (c *vaultClient) StopRenewLease(leaseId string) error {
-	return c.stopRenew(leaseId)
-}
-
 // stopRenew removes the given identifier from the heap and signals the renewal
 // loop to compute the next best candidate for renewal.
 func (c *vaultClient) stopRenew(id string) error {

client/vaultclient/vaultclient_testing.go

@@ -111,10 +111,6 @@ func (vc *MockVaultClient) StopRenewToken(token string) error {
 	return nil
 }
 
-func (vc *MockVaultClient) RenewLease(leaseId string, interval int) (<-chan error, error) {
-	return nil, nil
-}
-func (vc *MockVaultClient) StopRenewLease(leaseId string) error                   { return nil }
 func (vc *MockVaultClient) Start()                                                {}
 func (vc *MockVaultClient) Stop()                                                 {}
 func (vc *MockVaultClient) GetConsulACL(string, string) (*vaultapi.Secret, error) { return nil, nil }

command/agent/command.go

@@ -279,14 +279,14 @@ func (c *Command) readConfig() *Config {
 		config.PluginDir = filepath.Join(config.DataDir, "plugins")
 	}
 
-	if !c.isValidConfig(config) {
+	if !c.isValidConfig(config, cmdConfig) {
 		return nil
 	}
 
 	return config
 }
 
-func (c *Command) isValidConfig(config *Config) bool {
+func (c *Command) isValidConfig(config, cmdConfig *Config) bool {
 
 	// Check that the server is running in at least one mode.
 	if !(config.Server.Enabled || config.Client.Enabled) {
@@ -362,11 +362,12 @@ func (c *Command) isValidConfig(config *Config) bool {
 	}
 
 	// Check the bootstrap flags
-	if config.Server.BootstrapExpect > 0 && !config.Server.Enabled {
+	if !config.Server.Enabled && cmdConfig.Server.BootstrapExpect > 0 {
+		// report an error if BootstrapExpect is set in CLI but server is disabled
 		c.Ui.Error("Bootstrap requires server mode to be enabled")
 		return false
 	}
-	if config.Server.BootstrapExpect == 1 {
+	if config.Server.Enabled && config.Server.BootstrapExpect == 1 {
 		c.Ui.Error("WARNING: Bootstrap mode enabled! Potentially unsafe operation.")
 	}
 

command/quota_apply.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/hcl/hcl/ast"
 	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/jobspec"
 	"github.com/mitchellh/mapstructure"
 	"github.com/posener/complete"
 )
@@ -261,6 +262,7 @@ func parseQuotaResource(result *api.Resources, list *ast.ObjectList) error {
 	valid := []string{
 		"cpu",
 		"memory",
+		"network",
 	}
 	if err := helper.CheckHCLKeys(listVal, valid); err != nil {
 		return multierror.Prefix(err, "resources ->")
@@ -275,5 +277,20 @@ func parseQuotaResource(result *api.Resources, list *ast.ObjectList) error {
 		return err
 	}
 
+	// Find the network ObjectList, parse it
+	nw := listVal.Filter("network")
+	if len(nw.Items) > 0 {
+		rl, err := jobspec.ParseNetwork(nw)
+		if err != nil {
+			return multierror.Prefix(err, "resources ->")
+		}
+		if rl != nil {
+			if rl.Mode != "" || rl.HasPorts() {
+				return fmt.Errorf("resources -> network only allows mbits")
+			}
+			result.Networks = []*api.NetworkResource{rl}
+		}
+	}
+
 	return nil
 }

command/quota_apply_test.go

@@ -8,8 +8,10 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/hashicorp/nomad/api"
 	"github.com/mitchellh/cli"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestQuotaApplyCommand_Implements(t *testing.T) {
@@ -97,3 +99,42 @@ func TestQuotaApplyCommand_Good_JSON(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Len(t, quotas, 1)
 }
+
+func TestQuotaApplyNetwork(t *testing.T) {
+	t.Parallel()
+
+	mbits := 20
+
+	cases := []struct {
+		hcl string
+		q   *api.QuotaSpec
+		err string
+	}{{
+		hcl: `limit {region = "global", region_limit {network {mbits = 20}}}`,
+		q: &api.QuotaSpec{
+			Limits: []*api.QuotaLimit{{
+				Region: "global",
+				RegionLimit: &api.Resources{
+					Networks: []*api.NetworkResource{{
+						MBits: &mbits,
+					}},
+				},
+			}},
+		},
+		err: "",
+	}, {
+		hcl: `limit {region = "global", region_limit {network { mbits = 20, device = "eth0"}}}`,
+		q:   nil,
+		err: "1 error(s) occurred:\n\n* limit -> region_limit -> resources -> network -> invalid key: device",
+	}}
+
+	for _, c := range cases {
+		t.Run(c.hcl, func(t *testing.T) {
+			q, err := parseQuotaSpec([]byte(c.hcl))
+			require.Equal(t, c.q, q)
+			if c.err != "" {
+				require.EqualError(t, err, c.err)
+			}
+		})
+	}
+}

drivers/docker/driver.go

@@ -912,7 +912,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 
 	// Setup port mapping and exposed ports
 	if len(task.Resources.NomadResources.Networks) == 0 {
-		logger.Debug("no network interfaces are available")
 		if len(driverConfig.PortMap) > 0 {
 			return c, fmt.Errorf("Trying to map ports but no network interface is available")
 		}

drivers/docker/driver_test.go

@@ -1009,6 +1009,23 @@ func TestDockerDriver_CreateContainerConfig(t *testing.T) {
 	require.Equal(t, containerName, c.Name)
 }
 
+func TestDockerDriver_CreateContainerConfig_User(t *testing.T) {
+	t.Parallel()
+
+	task, cfg, _ := dockerTask(t)
+	task.User = "random-user-1"
+
+	require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
+
+	dh := dockerDriverHarness(t, nil)
+	driver := dh.Impl().(*Driver)
+
+	c, err := driver.createContainerConfig(task, cfg, "org/repo:0.1")
+	require.NoError(t, err)
+
+	require.Equal(t, task.User, c.Config.User)
+}
+
 func TestDockerDriver_CreateContainerConfig_Labels(t *testing.T) {
 	t.Parallel()
 

e2e/terraform/compute.tf

@@ -1,31 +1,3 @@
-data "template_file" "user_data_server" {
-  template = file("${path.root}/user-data-server.sh")
-
-  vars = {
-    server_count = var.server_count
-    region       = var.region
-    retry_join   = var.retry_join
-  }
-}
-
-data "template_file" "user_data_client" {
-  template = file("${path.root}/user-data-client.sh")
-  count    = var.client_count
-
-  vars = {
-    region     = var.region
-    retry_join = var.retry_join
-  }
-}
-
-data "template_file" "nomad_client_config" {
-  template = file("${path.root}/configs/client.hcl")
-}
-
-data "template_file" "nomad_server_config" {
-  template = "}"
-}
-
 resource "aws_instance" "server" {
   ami                    = data.aws_ami.main.image_id
   instance_type          = var.instance_type
@@ -41,14 +13,12 @@ resource "aws_instance" "server" {
     User           = data.aws_caller_identity.current.arn
   }
 
-  user_data            = data.template_file.user_data_server.rendered
   iam_instance_profile = aws_iam_instance_profile.instance_profile.name
 
+  # copy up all provisioning scripts and configs
   provisioner "file" {
-    content = file(
-      "${path.root}/configs/${var.indexed == false ? "server.hcl" : "indexed/server-${count.index}.hcl"}",
-    )
-    destination = "/tmp/server.hcl"
+    source      = "shared/"
+    destination = "/ops/shared"
 
     connection {
       host        = coalesce(self.public_ip, self.private_ip)
@@ -57,9 +27,11 @@ resource "aws_instance" "server" {
       private_key = module.keys.private_key_pem
     }
   }
+
   provisioner "remote-exec" {
     inline = [
-      "/ops/shared/config/provision-server.sh ${var.nomad_sha}",
+      "chmod +x /ops/shared/config/provision-server.sh",
+      "/ops/shared/config/provision-server.sh aws ${var.server_count} '${var.nomad_sha}' '${var.indexed == false ? "server.hcl" : "indexed/server-${count.index}.hcl"}'",
     ]
 
     connection {
@@ -94,14 +66,12 @@ resource "aws_instance" "client" {
     delete_on_termination = "true"
   }
 
-  user_data            = element(data.template_file.user_data_client.*.rendered, count.index)
   iam_instance_profile = aws_iam_instance_profile.instance_profile.name
 
+  # copy up all provisioning scripts and configs
   provisioner "file" {
-    content = file(
-      "${path.root}/configs/${var.indexed == false ? "client.hcl" : "indexed/client-${count.index}.hcl"}",
-    )
-    destination = "/tmp/client.hcl"
+    source      = "shared/"
+    destination = "/ops/shared"
 
     connection {
       host        = coalesce(self.public_ip, self.private_ip)
@@ -113,7 +83,8 @@ resource "aws_instance" "client" {
 
   provisioner "remote-exec" {
     inline = [
-      "/ops/shared/config/provision-client.sh ${var.nomad_sha}",
+      "chmod +x /ops/shared/config/provision-client.sh",
+      "/ops/shared/config/provision-client.sh aws '${var.nomad_sha}' '${var.indexed == false ? "client.hcl" : "indexed/client-${count.index}.hcl"}'",
     ]
 
     connection {
@@ -124,4 +95,3 @@ resource "aws_instance" "client" {
     }
   }
 }
-