-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
e2e: lifecycle hook tests for batch and service jobs
- Loading branch information
Showing
4 changed files
with
388 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,118 @@ | ||
# lifecycle hook test job for batch jobs. touches, removes, and tests | ||
# for the existence of files to assert the order of running tasks. | ||
# all tasks should exit 0 and the alloc dir should contain the following | ||
# files: ./init-ran, ./main-ran, ./cleanup-run | ||
|
||
job "batch-lifecycle" { | ||
|
||
datacenters = ["dc1"] | ||
|
||
type = "batch" | ||
|
||
group "test" { | ||
|
||
task "init" { | ||
|
||
lifecycle { | ||
hook = "prestart" | ||
} | ||
|
||
driver = "docker" | ||
|
||
config { | ||
image = "busybox:1" | ||
command = "/bin/sh" | ||
args = ["local/prestart.sh"] | ||
} | ||
|
||
template { | ||
data = <<EOT | ||
#!/bin/sh | ||
sleep 1 | ||
touch ${NOMAD_ALLOC_DIR}/init-ran | ||
touch ${NOMAD_ALLOC_DIR}/init-running | ||
if [ -f ${NOMAD_ALLOC_DIR}/main ]; then exit 7; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/cleanup ]; then exit 8; fi | ||
rm ${NOMAD_ALLOC_DIR}/init-running | ||
EOT | ||
|
||
destination = "local/prestart.sh" | ||
|
||
} | ||
|
||
resources { | ||
cpu = 64 | ||
memory = 64 | ||
} | ||
} | ||
|
||
task "main" { | ||
|
||
driver = "docker" | ||
|
||
config { | ||
image = "busybox:1" | ||
command = "/bin/sh" | ||
args = ["local/main.sh"] | ||
} | ||
|
||
template { | ||
data = <<EOT | ||
#!/bin/sh | ||
sleep 1 | ||
touch ${NOMAD_ALLOC_DIR}/main-ran | ||
touch ${NOMAD_ALLOC_DIR}/main-running | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 9; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 10; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/cleanup ]; then exit 11; fi | ||
rm ${NOMAD_ALLOC_DIR}/main-running | ||
EOT | ||
|
||
destination = "local/main.sh" | ||
} | ||
|
||
resources { | ||
cpu = 64 | ||
memory = 64 | ||
} | ||
} | ||
|
||
|
||
task "cleanup" { | ||
|
||
lifecycle { | ||
hook = "poststop" | ||
} | ||
|
||
driver = "docker" | ||
|
||
config { | ||
image = "busybox:1" | ||
command = "/bin/sh" | ||
args = ["local/cleanup.sh"] | ||
} | ||
|
||
template { | ||
data = <<EOT | ||
#!/bin/sh | ||
sleep 1 | ||
touch ${NOMAD_ALLOC_DIR}/cleanup-ran | ||
touch ${NOMAD_ALLOC_DIR}/cleanup-running | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 12; fi | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/main-ran ]; then exit 13; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 14; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/main-running ]; then exit 15; fi | ||
rm ${NOMAD_ALLOC_DIR}/cleanup-running | ||
EOT | ||
|
||
destination = "local/cleanup.sh" | ||
} | ||
|
||
resources { | ||
cpu = 64 | ||
memory = 64 | ||
} | ||
} | ||
|
||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,160 @@ | ||
# lifecycle hook test job for service jobs. touches, removes, and tests | ||
# for the existence of files to assert the order of running tasks. | ||
# after stopping, the alloc dir should contain the following files: | ||
# files: ./init-ran, ./sidecar-ran, ./main-ran, ./cleanup-run but not | ||
# the ./main-running, ./sidecar-running, or ./cleanup-running files | ||
|
||
job "service-lifecycle" { | ||
|
||
datacenters = ["dc1"] | ||
|
||
type = "batch" | ||
|
||
group "test" { | ||
|
||
task "init" { | ||
|
||
lifecycle { | ||
hook = "prestart" | ||
} | ||
|
||
driver = "docker" | ||
|
||
config { | ||
image = "busybox:1" | ||
command = "/bin/sh" | ||
args = ["local/prestart.sh"] | ||
} | ||
|
||
template { | ||
data = <<EOT | ||
#!/bin/sh | ||
sleep 1 | ||
touch ${NOMAD_ALLOC_DIR}/init-ran | ||
touch ${NOMAD_ALLOC_DIR}/init-running | ||
if [ -f ${NOMAD_ALLOC_DIR}/main ]; then exit 7; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/cleanup ]; then exit 8; fi | ||
rm ${NOMAD_ALLOC_DIR}/init-running | ||
EOT | ||
|
||
destination = "local/prestart.sh" | ||
|
||
} | ||
|
||
resources { | ||
cpu = 64 | ||
memory = 64 | ||
} | ||
} | ||
|
||
task "sidecar" { | ||
|
||
lifecycle { | ||
hook = "prestart" | ||
sidecar = true | ||
} | ||
|
||
driver = "docker" | ||
|
||
config { | ||
image = "busybox:1" | ||
command = "/bin/sh" | ||
args = ["local/sidecar.sh"] | ||
} | ||
|
||
template { | ||
data = <<EOT | ||
#!/bin/sh | ||
touch ${NOMAD_ALLOC_DIR}/sidecar-ran | ||
touch ${NOMAD_ALLOC_DIR}/sidecar-running | ||
sleep 5 | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/main ]; then exit 9; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/cleanup ]; then exit 10; fi | ||
sleep 300 | ||
EOT | ||
|
||
destination = "local/sidecar.sh" | ||
|
||
} | ||
|
||
resources { | ||
cpu = 64 | ||
memory = 64 | ||
} | ||
} | ||
|
||
task "main" { | ||
|
||
driver = "docker" | ||
|
||
config { | ||
image = "busybox:1" | ||
command = "/bin/sh" | ||
args = ["local/main.sh"] | ||
} | ||
|
||
template { | ||
data = <<EOT | ||
#!/bin/sh | ||
touch ${NOMAD_ALLOC_DIR}/main-ran | ||
touch ${NOMAD_ALLOC_DIR}/main-running | ||
sleep 5 | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 11; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 12; fi | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-ran ]; then exit 13; fi | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-running ]; then exit 14; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/cleanup ]; then exit 15; fi | ||
sleep 300 | ||
EOT | ||
|
||
destination = "local/main.sh" | ||
} | ||
|
||
resources { | ||
cpu = 64 | ||
memory = 64 | ||
} | ||
} | ||
|
||
|
||
task "cleanup" { | ||
|
||
lifecycle { | ||
hook = "poststop" | ||
} | ||
|
||
driver = "docker" | ||
|
||
config { | ||
image = "busybox:1" | ||
command = "/bin/sh" | ||
args = ["local/cleanup.sh"] | ||
} | ||
|
||
template { | ||
data = <<EOT | ||
#!/bin/sh | ||
touch ${NOMAD_ALLOC_DIR}/cleanup-ran | ||
touch ${NOMAD_ALLOC_DIR}/cleanup-running | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/init-ran ]; then exit 16; fi | ||
if [ -f ${NOMAD_ALLOC_DIR}/init-running ]; then exit 17; fi | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-ran ]; then exit 18; fi | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/sidecar-running ]; then exit 19; fi | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/main-ran ]; then exit 20; fi | ||
if [ ! -f ${NOMAD_ALLOC_DIR}/main-running ]; then exit 20; fi | ||
rm ${NOMAD_ALLOC_DIR}/sidecar-running | ||
rm ${NOMAD_ALLOC_DIR}/main-running | ||
rm ${NOMAD_ALLOC_DIR}/cleanup-running | ||
EOT | ||
|
||
destination = "local/cleanup.sh" | ||
} | ||
|
||
resources { | ||
cpu = 64 | ||
memory = 64 | ||
} | ||
} | ||
|
||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
package lifecycle | ||
|
||
import ( | ||
"github.com/hashicorp/nomad/api" | ||
"github.com/hashicorp/nomad/e2e/e2eutil" | ||
"github.com/hashicorp/nomad/e2e/framework" | ||
"github.com/hashicorp/nomad/helper/uuid" | ||
"github.com/hashicorp/nomad/nomad/structs" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
type LifecycleE2ETest struct { | ||
framework.TC | ||
jobIDs []string | ||
} | ||
|
||
func init() { | ||
framework.AddSuites(&framework.TestSuite{ | ||
Component: "Lifecycle", | ||
CanRunLocal: true, | ||
Cases: []framework.TestCase{new(LifecycleE2ETest)}, | ||
}) | ||
} | ||
|
||
// Ensure cluster has leader and at least 1 client node | ||
// in a ready state before running tests | ||
func (tc *LifecycleE2ETest) BeforeAll(f *framework.F) { | ||
e2eutil.WaitForLeader(f.T(), tc.Nomad()) | ||
e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) | ||
} | ||
|
||
// TestBatchJob runs a batch job with prestart and poststop hooks | ||
func (tc *LifecycleE2ETest) TestBatchJob(f *framework.F) { | ||
t := f.T() | ||
require := require.New(t) | ||
nomadClient := tc.Nomad() | ||
uuid := uuid.Generate() | ||
jobID := "lifecycle-" + uuid[0:8] | ||
tc.jobIDs = append(tc.jobIDs, jobID) | ||
|
||
allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, "lifecycle/inputs/batch.nomad", jobID, "") | ||
require.Equal(1, len(allocs)) | ||
allocID := allocs[0].ID | ||
|
||
// wait for the job to stop and assert we stopped successfully, not failed | ||
e2eutil.WaitForAllocStopped(t, nomadClient, allocID) | ||
alloc, _, err := nomadClient.Allocations().Info(allocID, nil) | ||
require.NoError(err) | ||
require.Equal(structs.AllocClientStatusComplete, alloc.ClientStatus) | ||
|
||
// assert the files were written as expected | ||
afi, _, err := nomadClient.AllocFS().List(alloc, "alloc", nil) | ||
require.NoError(err) | ||
expected := map[string]bool{ | ||
"init-ran": true, "main-ran": true, "cleanup-ran": true, | ||
"init-running": false, "main-running": false, "cleanup-running": false} | ||
got := checkFiles(expected, afi) | ||
require.Equal(expected, got) | ||
} | ||
|
||
// TestServiceJob runs a service job with prestart and poststop hooks | ||
func (tc *LifecycleE2ETest) TestServiceJob(f *framework.F) { | ||
t := f.T() | ||
require := require.New(t) | ||
nomadClient := tc.Nomad() | ||
uuid := uuid.Generate() | ||
jobID := "lifecycle-" + uuid[0:8] | ||
tc.jobIDs = append(tc.jobIDs, jobID) | ||
|
||
allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, "lifecycle/inputs/service.nomad", jobID, "") | ||
require.Equal(1, len(allocs)) | ||
allocID := allocs[0].ID | ||
|
||
e2eutil.WaitForAllocRunning(t, nomadClient, allocID) | ||
|
||
alloc, _, err := nomadClient.Allocations().Info(allocID, nil) | ||
require.NoError(err) | ||
|
||
// stop the job | ||
_, _, err = nomadClient.Jobs().Deregister(jobID, false, nil) | ||
require.NoError(err) | ||
e2eutil.WaitForAllocStopped(t, nomadClient, allocID) | ||
|
||
// assert the files were written as expected | ||
afi, _, err := nomadClient.AllocFS().List(alloc, "alloc", nil) | ||
require.NoError(err) | ||
expected := map[string]bool{ | ||
"init-ran": true, "sidecar-ran": true, "main-ran": true, "cleanup-ran": true, | ||
"init-running": false, "sidecar-running": false, | ||
"main-running": false, "cleanup-running": false} | ||
got := checkFiles(expected, afi) | ||
require.Equal(expected, got) | ||
} | ||
|
||
// checkFiles returns a map of whether the expected files were found | ||
// in the file info response | ||
func checkFiles(expected map[string]bool, got []*api.AllocFileInfo) map[string]bool { | ||
results := map[string]bool{} | ||
for expect := range expected { | ||
results[expect] = false | ||
} | ||
for _, file := range got { | ||
// there will be files unrelated to the test, so ignore those | ||
if _, ok := results[file.Name]; ok { | ||
results[file.Name] = true | ||
} | ||
} | ||
return results | ||
} |