e2e: add a test for using private registry with podman driver

This PR adds an e2e test case that stands up a private docker registry
and has a podman task run a container from an image in that private
registry. Currently the podman driver only supports basic user:pass
authentication embeded in the task. We can use this registry to iterate
on future support for credentials helpers, etc.

e2e/podman/input/auth_basic.hcl

@@ -0,0 +1,87 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+# This job runs a podman task using a container stored in a private registry
+# configured with basic authentication. The registry.hcl job should be running
+# and healthy before running this job. The registry_address and registry_port
+# HCL variables must be provided.
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+  default     = "localhost"
+}
+
+variable "registry_port" {
+  type        = number
+  description = "The HTTP port of the local registry"
+}
+
+variable "registry_username" {
+  type        = string
+  description = "The Basic Auth username of the local registry"
+  default     = "e2euser"
+}
+
+variable "registry_password" {
+  type        = string
+  description = "The Basic Auth password of the local registry"
+  default     = "e2epassword"
+}
+
+locals {
+  registry_auth = base64encode("${var.registry_username}:${var.registry_password}")
+}
+
+job "auth_basic" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "basic" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    network {
+      mode = "host"
+    }
+
+    task "echo" {
+      driver = "podman"
+
+      #       template {
+      #         data        = <<EOH
+      # {
+      #   "auths": {
+      #     "${var.registry_address}:${var.registry_port}": {
+      #       "auth": "${local.registry_auth}"
+      #     }
+      #   }
+      # }
+      #         EOH
+      #         destination = "local/auth.json"
+      #       }
+
+      config {
+        image = "${var.registry_address}:${var.registry_port}/docker.io/library/bash:private"
+        args  = ["echo", "The auth basic test is OK!"]
+
+        auth {
+          username   = "e2euser"
+          password   = "e2epassword"
+          tls_verify = false
+        }
+      }
+
+      resources {
+        cpu    = 50
+        memory = 32
+      }
+    }
+  }
+}

e2e/podman/input/podman_basic.hcl → e2e/podman/input/redis.hcl

@@ -1,7 +1,9 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: MPL-2.0
 
-job "podman_basic" {
+# This is a simple redis job using the podman task driver.
+
+job "redis" {
 
   constraint {
     attribute = "${attr.kernel.name}"
@@ -19,7 +21,7 @@ job "podman_basic" {
       driver = "podman"
 
       config {
-        image = "redis:7"
+        image = "docker.io/library/redis:7"
         ports = ["db"]
       }
 

e2e/podman/input/registry.hcl

@@ -0,0 +1,122 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+# This job stands up a private container registry for use in e2e tests.
+# In a post start task we then upload some default images for convenience.
+#
+# <address>:<port>/docker.io/library/bash:private
+# <address>:<port>/docker.io/library/python:private
+#
+# Note that the <address>:<port> is dynamic and can be found using NSD.
+# Note that credentials are required (e.g. podman login), and are
+# user: e2euser
+# password: e2epassword
+
+job "registry" {
+  type = "service"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "registry-server" {
+
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    network {
+      mode = "host"
+      port "registryhttp" {}
+    }
+
+    service {
+      provider = "nomad"
+      name     = "registry"
+      port     = "registryhttp"
+      check {
+        name     = "registry-http"
+        type     = "http"
+        path     = "/"
+        interval = "10s"
+        timeout  = "3s"
+      }
+    }
+
+    task "registry" {
+      driver = "podman"
+
+      template {
+        data        = <<EOH
+e2euser:$2y$05$QpRvGkM/CMG.AG/G7Uh6guULMIlv1ZvjwfPa6dNjdkH.fhTzcpLDC
+        EOH
+        destination = "local/auth.txt"
+      }
+
+      config {
+        image        = "docker.io/library/registry:2"
+        ports        = ["registryhttp"]
+        network_mode = "host"
+      }
+
+      env {
+        REGISTRY_HTTP_ADDR           = "${NOMAD_ADDR_registryhttp}"
+        REGISTRY_AUTH                = "htpasswd"
+        REGISTRY_AUTH_HTPASSWD_REALM = "Registry Realm"
+        REGISTRY_AUTH_HTPASSWD_PATH  = "local/auth.txt"
+      }
+
+      resources {
+        cpu    = 50
+        memory = 128
+      }
+    }
+
+
+    task "registry-preload" {
+      user   = "root"
+      driver = "raw_exec"
+
+      lifecycle {
+        hook    = "poststart"
+        sidecar = false
+      }
+
+      template {
+        data        = <<EOH
+{
+  "auths": {
+    "localhost:{{- env "NOMAD_PORT_registryhttp" -}}": {
+      "auth": "ZTJldXNlcjplMmVwYXNzd29yZA=="
+    }
+  }
+}
+        EOH
+        destination = "local/auth.json"
+      }
+
+      template {
+        data        = <<EOH
+set -euo pipefail
+podman pull docker.io/library/bash:5
+podman pull docker.io/library/python:3
+podman push --tls-verify=false --authfile=local/auth.json docker.io/library/bash:5 localhost:{{- env "NOMAD_PORT_registryhttp" -}}/docker.io/library/bash:private
+podman push --tls-verify=false --authfile=local/auth.json docker.io/library/python:3 localhost:{{- env "NOMAD_PORT_registryhttp" -}}/docker.io/library/python:private
+        EOH
+        destination = "local/script.sh"
+      }
+
+      config {
+        command = "bash"
+        args    = ["-c", "chmod +x local/script.sh && local/script.sh"]
+      }
+
+      resources {
+        cpu    = 50
+        memory = 32
+      }
+    }
+  }
+}

e2e/podman/podman_test.go

@@ -4,36 +4,54 @@
 package podman
 
 import (
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/nomad/e2e/e2eutil"
-	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/e2e/v3/cluster3"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
 	"github.com/shoenig/test/must"
 )
 
 func TestPodman(t *testing.T) {
-	nomad := e2eutil.NomadClient(t)
+	cluster3.Establish(t,
+		cluster3.Leader(),
+		cluster3.LinuxClients(1),
+	)
 
-	e2eutil.WaitForLeader(t, nomad)
-	e2eutil.WaitForNodesReady(t, nomad, 1)
-
-	t.Run("testBasic", testBasic)
+	t.Run("testRedis", testRedis)
+	t.Run("testAuthBasic", testAuthBasic)
 }
 
-func testBasic(t *testing.T) {
-	nomad := e2eutil.NomadClient(t)
-	jobID := "podman-basic-" + uuid.Short()
-	jobIDs := []string{jobID}
-	t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs))
-
-	// start job
-	e2eutil.RegisterAndWaitForAllocs(t, nomad, "./input/podman_basic.hcl", jobID, "")
+func testRedis(t *testing.T) {
+	job, cleanup := jobs3.Submit(t, "./input/redis.hcl")
+	t.Cleanup(cleanup)
 
-	// get alloc id
-	allocID := e2eutil.SingleAllocID(t, jobID, "", 0)
+	logs := job.TaskLogs("cache", "redis")
+	must.StrContains(t, logs.Stdout, "oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo")
+}
 
-	// check logs for redis startup
-	logs, err := e2eutil.AllocTaskLogs(allocID, "redis", e2eutil.LogsStdOut)
-	must.NoError(t, err)
-	must.StrContains(t, logs, "oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo")
+func testAuthBasic(t *testing.T) {
+	// start the private registry
+	_, regCleanup := jobs3.Submit(t, "./input/registry.hcl",
+		jobs3.Timeout(90*time.Second), // may need to pull 3 images
+	)
+	t.Cleanup(regCleanup)
+
+	// find the private registry service
+	services, _, err := e2eutil.NomadClient(t).Services().Get("registry", nil)
+	must.NoError(t, err, must.Sprint("failed to find registry service"))
+	must.Len(t, 1, services, must.Sprint("expected 1 registry service"))
+
+	// run the private bash image
+	regService := services[0]
+	bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_basic.hcl",
+		jobs3.Var("registry_address", regService.Address),
+		jobs3.Var("registry_port", strconv.Itoa(regService.Port)),
+		jobs3.WaitComplete("basic"),
+	)
+	t.Cleanup(bashCleanup)
+	logs := bashJob.TaskLogs("basic", "echo")
+	must.StrContains(t, logs.Stdout, "The auth basic test is OK!")
 }

e2e/v3/jobs3/jobs3.go

@@ -96,6 +96,7 @@ func (sub *Submission) getTaskLogs(allocID, task string) Logs {
 	read := func(path string) string {
 		rc, err := fsAPI.ReadAt(alloc, path, 0, 0, queryOpts)
 		must.NoError(sub.t, err, must.Sprint("failed to read logs"))
+		must.NoError(sub.t, err, must.Sprintf("failed to read logs"))
 		b, err := io.ReadAll(rc)
 		must.NoError(sub.t, rc.Close(), must.Sprint("failed to close log stream"))
 		return string(b)