e2e: fix and modernize rescheduling test (#19105) (#19106)

The E2E test suite for rescheduling had a few bugs:
* Using the command line to stop a job with a failing deployment returns a non-zero exit
  code, which would cause an otherwise passing test to fail.
* Two of the input jobs were actually invalid but were only correctly detected
  as such because of #17342

This changeset also updates the whole test suite to move it off the v1
"framework". A few test assertions are also de-flaked.

Fixes: #19076

Co-authored-by: Tim Gross <tgross@hashicorp.com>

e2e/e2e_test.go

@@ -28,7 +28,6 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/podman"
 	_ "github.com/hashicorp/nomad/e2e/quotas"
 	_ "github.com/hashicorp/nomad/e2e/remotetasks"
-	_ "github.com/hashicorp/nomad/e2e/rescheduling"
 	_ "github.com/hashicorp/nomad/e2e/scaling"
 	_ "github.com/hashicorp/nomad/e2e/scalingpolicies"
 	_ "github.com/hashicorp/nomad/e2e/scheduler_sysbatch"
@@ -42,6 +41,7 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/disconnectedclients"
 	_ "github.com/hashicorp/nomad/e2e/namespaces"
 	_ "github.com/hashicorp/nomad/e2e/nodedrain"
+	_ "github.com/hashicorp/nomad/e2e/rescheduling"
 	_ "github.com/hashicorp/nomad/e2e/volumes"
 )
 

e2e/rescheduling/doc.go

@@ -0,0 +1,7 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package rescheduling
+
+// This package contains only tests, so this is a placeholder file to
+// make sure builds don't fail with "no non-test Go files in" errors

e2e/rescheduling/input/rescheduling_canary_autorevert.nomad

@@ -27,7 +27,7 @@ job "test" {
       min_healthy_time  = "1s"
       auto_revert       = true
       healthy_deadline  = "2s"
-      progress_deadline = "3s"
+      progress_deadline = "5s"
     }
 
     restart {

e2e/rescheduling/input/rescheduling_maxp_autorevert.nomad

@@ -26,7 +26,7 @@ job "demo3" {
       min_healthy_time  = "1s"
       auto_revert       = true
       healthy_deadline  = "2s"
-      progress_deadline = "3s"
+      progress_deadline = "5s"
     }
 
     restart {

e2e/rescheduling/input/rescheduling_progressdeadline_fail.nomad

@@ -24,7 +24,7 @@ job "demo2" {
     update {
       # we want the first allocation to take a while before we give up on it,
       # so that we can check the deployment's progress deadline before and
-      # after it becomes healthy
+      # after we determine it will never become healthy
       min_healthy_time  = "10s"
       healthy_deadline  = "15s"
       progress_deadline = "20s"
@@ -39,7 +39,7 @@ job "demo2" {
     }
 
     reschedule {
-      unlimited      = "true"
+      unlimited      = true
       delay_function = "constant"
       delay          = "5s"
     }

e2e/rescheduling/input/rescheduling_success.nomad

@@ -28,9 +28,10 @@ job "test3" {
     }
 
     reschedule {
-      attempts  = 2
-      interval  = "5m"
-      unlimited = false
+      delay          = "5s"
+      delay_function = "constant"
+      unlimited      = true
     }
+
   }
 }