Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle Jobs with ttl_seconds_after_finished = 0 correctly #2596

Merged
merged 10 commits into from
Oct 30, 2024
3 changes: 3 additions & 0 deletions .changelog/2596.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
Properly handle Kubernetes Jobs with ttl_seconds_after_finished = 0 to prevent unnecessary recreation.
```
114 changes: 111 additions & 3 deletions kubernetes/resource_kubernetes_job_v1.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"fmt"
"log"
"strconv"
"time"

"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
Expand All @@ -16,6 +17,7 @@ import (
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
pkgApi "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
Expand All @@ -28,6 +30,7 @@ func resourceKubernetesJobV1() *schema.Resource {
ReadContext: resourceKubernetesJobV1Read,
UpdateContext: resourceKubernetesJobV1Update,
DeleteContext: resourceKubernetesJobV1Delete,
CustomizeDiff: resourceKubernetesJobV1CustomizeDiff,
Importer: &schema.ResourceImporter{
StateContext: schema.ImportStatePassthroughContext,
},
Expand All @@ -48,6 +51,78 @@ func resourceKubernetesJobV1() *schema.Resource {
}
}

func resourceKubernetesJobV1CustomizeDiff(ctx context.Context, d *schema.ResourceDiff, meta interface{}) error {
if d.Id() == "" {
log.Printf("[DEBUG] Resource ID is empty, resource not created yet.")
return nil
}

// Retrieve old and new TTL values as strings
oldTTLRaw, newTTLRaw := d.GetChange("spec.0.ttl_seconds_after_finished")

var oldTTLStr, newTTLStr string

if oldTTLRaw != nil {
oldTTLStr, _ = oldTTLRaw.(string)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can just omit the , _ here if you're not going to check the type assertion.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

,_ is not omitted, thanks for the feedback!

}
if newTTLRaw != nil {
newTTLStr, _ = newTTLRaw.(string)
}

oldTTLInt, err := strconv.Atoi(oldTTLStr)
if err != nil {
oldTTLInt = 0
JaylonmcShan03 marked this conversation as resolved.
Show resolved Hide resolved
}
newTTLInt, err := strconv.Atoi(newTTLStr)
if err != nil {
newTTLInt = 0
}

conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
return err
}

namespace, name, err := idParts(d.Id())
if err != nil {
return err
}

// Check if the Job exists
_, err = conn.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
// Job is missing
if oldTTLInt >= 0 {
if oldTTLInt != newTTLInt {
// TTL value changed; force recreation
JaylonmcShan03 marked this conversation as resolved.
Show resolved Hide resolved
log.Printf("[DEBUG] Job %s not found and ttl_seconds_after_finished changed from %d to %d; forcing recreation", d.Id(), oldTTLInt, newTTLInt)
d.ForceNew("spec.0.ttl_seconds_after_finished")
return nil
} else {
// TTL remains the same; suppress diff
log.Printf("[DEBUG] Job %s not found and ttl_seconds_after_finished remains %d; suppressing diff", d.Id(), oldTTLInt)
d.Clear("spec")
d.Clear("metadata")
return nil
}
}
} else {
return err
}
} else {
// Job exists, check if TTL changed
if oldTTLInt != newTTLInt {
// TTL changed; force recreation
log.Printf("[DEBUG] Job %s exists and ttl_seconds_after_finished changed from %d to %d; forcing recreation", d.Id(), oldTTLInt, newTTLInt)
d.ForceNew("spec.0.ttl_seconds_after_finished")
return nil
}
}

return nil
}

func resourceKubernetesJobV1Schema() map[string]*schema.Schema {
return map[string]*schema.Schema{
"metadata": jobMetadataSchema(),
Expand Down Expand Up @@ -118,8 +193,17 @@ func resourceKubernetesJobV1Read(ctx context.Context, d *schema.ResourceData, me
return diag.FromErr(err)
}
if !exists {
d.SetId("")
return diag.Diagnostics{}
// Check if ttl_seconds_after_finished is set
if ttl, ok := d.GetOk("spec.0.ttl_seconds_after_finished"); ok {
// ttl_seconds_after_finished is set, Job is deleted due to TTL
// We don't need to remove the resource from the state
log.Printf("[INFO] Job %s has been deleted by Kubernetes due to TTL (ttl_seconds_after_finished = %v), keeping resource in state", d.Id(), ttl)
return diag.Diagnostics{}
} else {
// ttl_seconds_after_finished is not set, remove the resource from the state
d.SetId("")
return diag.Diagnostics{}
}
}
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
Expand Down Expand Up @@ -173,6 +257,31 @@ func resourceKubernetesJobV1Update(ctx context.Context, d *schema.ResourceData,
return diag.FromErr(err)
}

// Attempt to get the Job
_, err = conn.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
// Job is missing; check TTL
ttlAttr := d.Get("spec.0.ttl_seconds_after_finished")
ttlStr, _ := ttlAttr.(string)
ttlInt, err := strconv.Atoi(ttlStr)
if err != nil {
ttlInt = 0
}

if ttlInt >= 0 {
// Job was deleted due to TTL nothing to update
log.Printf("[INFO] Job %s not found but ttl_seconds_after_finished = %v; nothing to update", d.Id(), ttlInt)
return nil
}

// Job was deleted unexpectedly; return an error
return diag.Errorf("Job %s not found; cannot update because it has been deleted", d.Id())
}
return diag.Errorf("Error retrieving Job: %s", err)
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think you need this logic either. In the read function it's going to check if the job exists and then set the ID to "" if it doesn't which will force recreation and so the update function wont get called.

I tried commenting this code out and all your tests still pass too.


// Proceed with the update as usual
ops := patchMetadata("metadata.0.", "/metadata/", d)

if d.HasChange("spec") {
Expand Down Expand Up @@ -204,7 +313,6 @@ func resourceKubernetesJobV1Update(ctx context.Context, d *schema.ResourceData,
}
return resourceKubernetesJobV1Read(ctx, d, meta)
}

func resourceKubernetesJobV1Delete(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
conn, err := meta.(KubeClientsets).MainClientset()
if err != nil {
Expand Down
126 changes: 126 additions & 0 deletions kubernetes/resource_kubernetes_job_v1_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,82 @@ func TestAccKubernetesJobV1_ttl_seconds_after_finished(t *testing.T) {
})
}

func TestAccKubernetesJobV1_customizeDiff_ttlZero(t *testing.T) {
var conf batchv1.Job
name := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(10))
imageName := busyboxImage
resourceName := "kubernetes_job_v1.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() {
testAccPreCheck(t)
skipIfClusterVersionLessThan(t, "1.21.0")
},
ProviderFactories: testAccProviderFactories,
Steps: []resource.TestStep{
// Step 1: Create the Job
{
Config: testAccKubernetesJobV1Config_customizeDiff_ttlZero(name, imageName),
Check: resource.ComposeAggregateTestCheckFunc(
testAccCheckKubernetesJobV1Exists(resourceName, &conf),
resource.TestCheckResourceAttr(resourceName, "spec.0.ttl_seconds_after_finished", "0"),
),
},
// Step 2: Wait for the Job to complete and be deleted
{
PreConfig: func() {
time.Sleep(70 * time.Second)
},
Config: testAccKubernetesJobV1Config_customizeDiff_ttlZero(name, imageName),
PlanOnly: true,
ExpectNonEmptyPlan: false,
},
},
})
}

func TestAccKubernetesJobV1_updateTTLFromZero(t *testing.T) {
var conf batchv1.Job
name := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(10))
imageName := busyboxImage
resourceName := "kubernetes_job_v1.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() {
testAccPreCheck(t)
skipIfClusterVersionLessThan(t, "1.21.0")
},
ProviderFactories: testAccProviderFactories,
Steps: []resource.TestStep{
// Step 1: Create the Job with ttl_seconds_after_finished = 0
{
Config: testAccKubernetesJobV1Config_customizeDiff_ttlZero(name, imageName),
Check: resource.ComposeAggregateTestCheckFunc(
testAccCheckKubernetesJobV1Exists(resourceName, &conf),
resource.TestCheckResourceAttr(resourceName, "spec.0.ttl_seconds_after_finished", "0"),
),
},
// Step 2: Wait for the Job to complete and be deleted
{
PreConfig: func() {
time.Sleep(70 * time.Second)
},
Config: testAccKubernetesJobV1Config_customizeDiff_ttlZero(name, imageName),
PlanOnly: true,
ExpectNonEmptyPlan: false,
},
// Step 3: Update the Job to ttl_seconds_after_finished = 5
{
Config: testAccKubernetesJobV1Config_customizeDiff_ttlFive(name, imageName),
Check: resource.ComposeAggregateTestCheckFunc(
testAccCheckKubernetesJobV1Exists(resourceName, &conf),
resource.TestCheckResourceAttr(resourceName, "spec.0.ttl_seconds_after_finished", "5"),
),
},
},
})
}

func testAccCheckJobV1Waited(minDuration time.Duration) func(*terraform.State) error {
// NOTE this works because this function is called when setting up the test
// and the function it returns is called after the resource has been created
Expand Down Expand Up @@ -516,3 +592,53 @@ func testAccKubernetesJobV1Config_modified(name, imageName string) string {
wait_for_completion = false
}`, name, imageName)
}

func testAccKubernetesJobV1Config_customizeDiff_ttlZero(name, imageName string) string {
JaylonmcShan03 marked this conversation as resolved.
Show resolved Hide resolved
return fmt.Sprintf(`
resource "kubernetes_job_v1" "test" {
metadata {
name = "%s"
}
spec {
ttl_seconds_after_finished = 0
template {
metadata {}
spec {
container {
name = "wait-test"
image = "%s"
command = ["sleep", "60"]
}
restart_policy = "Never"
}
}
}
wait_for_completion = false
}
`, name, imageName)
}

func testAccKubernetesJobV1Config_customizeDiff_ttlFive(name, imageName string) string {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of having 2 configs, you could just make the ttl value of a parameter to the function and template it in.

return fmt.Sprintf(`
resource "kubernetes_job_v1" "test" {
metadata {
name = "%s"
}
spec {
ttl_seconds_after_finished = 5
template {
metadata {}
spec {
container {
name = "wait-test"
image = "%s"
command = ["sleep", "60"]
JaylonmcShan03 marked this conversation as resolved.
Show resolved Hide resolved
}
restart_policy = "Never"
}
}
}
wait_for_completion = false
}
`, name, imageName)
}
Loading