From fbc0f57f3f8edab98d709194947ccdb044064cb4 Mon Sep 17 00:00:00 2001 From: Krutika Dhananjay Date: Fri, 18 Oct 2024 13:39:43 +0530 Subject: [PATCH 1/2] Add support for AllVersionsExpiration ilm rule This is a minio-only rule for removing all versions of an object upon expiration. The rule takes two parameters: Days and DeleteMarker. "Days" indicates the number of days after which the object and all its versions must be expired. The optional "DeleteMarker" flag indicates that the expiration be applied if the latest version of this object is a delete marker. These specific changes in minio-go are for consumption by mc in a different PR. --- pkg/lifecycle/lifecycle.go | 29 ++++++++++ pkg/lifecycle/lifecycle_test.go | 94 +++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index e706b57de..624337f31 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -434,12 +434,37 @@ func (de DelMarkerExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElemen return enc.EncodeElement(delMarkerExp(de), start) } +// AllVersionsExpiration represents AllVersionsExpiration actions element in an ILM policy +type AllVersionsExpiration struct { + XMLName xml.Name `xml:"AllVersionsExpiration" json:"-"` + Days int `xml:"Days,omitempty" json:"Days,omitempty"` + DeleteMarker ExpireDeleteMarker `xml:"DeleteMarker,omitempty" json:"DeleteMarker,omitempty"` +} + +// IsDaysNull returns true if days field is null +func (e AllVersionsExpiration) IsDaysNull() bool { + return e.Days == 0 +} + +func (e AllVersionsExpiration) IsNull() bool { + return e.IsDaysNull() +} + +func (ave AllVersionsExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if ave.IsNull() { + return nil + } + type allVersionsExp AllVersionsExpiration + return enc.EncodeElement(allVersionsExp(ave), start) +} + // MarshalJSON customizes json encoding by omitting empty values func (r Rule) MarshalJSON() ([]byte, error) { type rule struct { AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `json:"AbortIncompleteMultipartUpload,omitempty"` Expiration *Expiration `json:"Expiration,omitempty"` DelMarkerExpiration *DelMarkerExpiration `json:"DelMarkerExpiration,omitempty"` + AllVersionsExpiration *AllVersionsExpiration `json:"AllVersionsExpiration,omitempty"` ID string `json:"ID"` RuleFilter *Filter `json:"Filter,omitempty"` NoncurrentVersionExpiration *NoncurrentVersionExpiration `json:"NoncurrentVersionExpiration,omitempty"` @@ -475,6 +500,9 @@ func (r Rule) MarshalJSON() ([]byte, error) { if !r.NoncurrentVersionTransition.isNull() { newr.NoncurrentVersionTransition = &r.NoncurrentVersionTransition } + if !r.AllVersionsExpiration.IsNull() { + newr.AllVersionsExpiration = &r.AllVersionsExpiration + } return json.Marshal(newr) } @@ -485,6 +513,7 @@ type Rule struct { AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty" json:"AbortIncompleteMultipartUpload,omitempty"` Expiration Expiration `xml:"Expiration,omitempty" json:"Expiration,omitempty"` DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty" json:"DelMarkerExpiration,omitempty"` + AllVersionsExpiration AllVersionsExpiration `xml:"AllVersionsExpiration,omitempty" json:"AllVersionsExpiration,omitempty"` ID string `xml:"ID" json:"ID"` RuleFilter Filter `xml:"Filter,omitempty" json:"Filter,omitempty"` NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty" json:"NoncurrentVersionExpiration,omitempty"` diff --git a/pkg/lifecycle/lifecycle_test.go b/pkg/lifecycle/lifecycle_test.go index d730ed3b3..9fa380e16 100644 --- a/pkg/lifecycle/lifecycle_test.go +++ b/pkg/lifecycle/lifecycle_test.go @@ -273,6 +273,28 @@ func TestLifecycleJSONRoundtrip(t *testing.T) { ID: "rule-7", Status: "Enabled", }, + { + AllVersionsExpiration: AllVersionsExpiration{ + Days: 10, + }, + ID: "rule-8", + Status: "Enabled", + }, + { + AllVersionsExpiration: AllVersionsExpiration{ + Days: 0, + }, + ID: "rule-9", + Status: "Enabled", + }, + { + AllVersionsExpiration: AllVersionsExpiration{ + Days: 7, + DeleteMarker: ExpireDeleteMarker(true), + }, + ID: "rule-10", + Status: "Enabled", + }, }, } @@ -291,6 +313,10 @@ func TestLifecycleJSONRoundtrip(t *testing.T) { t.Fatalf("expected %#v got %#v", lc.Rules[i].NoncurrentVersionTransition, got.Rules[i].NoncurrentVersionTransition) } + if !lc.Rules[i].NoncurrentVersionExpiration.equals(got.Rules[i].NoncurrentVersionExpiration) { + t.Fatalf("expected %#v got %#v", lc.Rules[i].NoncurrentVersionExpiration, got.Rules[i].NoncurrentVersionExpiration) + } + if !lc.Rules[i].Transition.equals(got.Rules[i].Transition) { t.Fatalf("expected %#v got %#v", lc.Rules[i].Transition, got.Rules[i].Transition) } @@ -300,6 +326,9 @@ func TestLifecycleJSONRoundtrip(t *testing.T) { if !lc.Rules[i].DelMarkerExpiration.equals(got.Rules[i].DelMarkerExpiration) { t.Fatalf("expected %#v got %#v", lc.Rules[i].DelMarkerExpiration, got.Rules[i].DelMarkerExpiration) } + if !lc.Rules[i].AllVersionsExpiration.equals(got.Rules[i].AllVersionsExpiration) { + t.Fatalf("expected %#v got %#v", lc.Rules[i].AllVersionsExpiration, got.Rules[i].AllVersionsExpiration) + } } } @@ -352,6 +381,27 @@ func TestLifecycleXMLRoundtrip(t *testing.T) { Days: 5, }, }, + { + ID: "all-versions-expiration-1", + Status: "Enabled", + AllVersionsExpiration: AllVersionsExpiration{ + Days: 5, + }, + }, + { + ID: "all-versions-expiration-2", + Status: "Enabled", + AllVersionsExpiration: AllVersionsExpiration{ + Days: 10, + DeleteMarker: ExpireDeleteMarker(true), + }, + RuleFilter: Filter{ + Tag: Tag{ + Key: "key-1", + Value: "value-1", + }, + }, + }, }, } @@ -374,6 +424,18 @@ func TestLifecycleXMLRoundtrip(t *testing.T) { if !lc.Rules[i].Transition.equals(got.Rules[i].Transition) { t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].Transition, got.Rules[i].Transition) } + + if !lc.Rules[i].NoncurrentVersionExpiration.equals(got.Rules[i].NoncurrentVersionExpiration) { + t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].NoncurrentVersionExpiration, got.Rules[i].NoncurrentVersionExpiration) + } + + if !lc.Rules[i].DelMarkerExpiration.equals(got.Rules[i].DelMarkerExpiration) { + t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].DelMarkerExpiration, got.Rules[i].DelMarkerExpiration) + } + + if !lc.Rules[i].AllVersionsExpiration.equals(got.Rules[i].AllVersionsExpiration) { + t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].AllVersionsExpiration, got.Rules[i].AllVersionsExpiration) + } } } @@ -381,6 +443,10 @@ func (n NoncurrentVersionTransition) equals(m NoncurrentVersionTransition) bool return n.NoncurrentDays == m.NoncurrentDays && n.StorageClass == m.StorageClass } +func (n NoncurrentVersionExpiration) equals(m NoncurrentVersionExpiration) bool { + return n.NoncurrentDays == m.NoncurrentDays && n.NewerNoncurrentVersions == m.NewerNoncurrentVersions +} + func (t Transition) equals(u Transition) bool { return t.Days == u.Days && t.Date.Equal(u.Date.Time) && t.StorageClass == u.StorageClass } @@ -389,6 +455,10 @@ func (a DelMarkerExpiration) equals(b DelMarkerExpiration) bool { return a.Days == b.Days } +func (a AllVersionsExpiration) equals(b AllVersionsExpiration) bool { + return a.Days == b.Days && a.DeleteMarker == b.DeleteMarker +} + func TestExpiredObjectDeleteMarker(t *testing.T) { expected := []byte(`{"Rules":[{"Expiration":{"ExpiredObjectDeleteMarker":true},"ID":"expired-object-delete-marker","Status":"Enabled"}]}`) lc := Configuration{ @@ -411,3 +481,27 @@ func TestExpiredObjectDeleteMarker(t *testing.T) { t.Fatalf("Expected %s but got %s", expected, got) } } + +func TestAllVersionsExpiration(t *testing.T) { + expected := []byte(`{"Rules":[{"AllVersionsExpiration":{"Days":2,"DeleteMarker":true},"ID":"all-versions-expiration","Status":"Enabled"}]}`) + lc := Configuration{ + Rules: []Rule{ + { + AllVersionsExpiration: AllVersionsExpiration{ + Days: 2, + DeleteMarker: ExpireDeleteMarker(true), + }, + ID: "all-versions-expiration", + Status: "Enabled", + }, + }, + } + + got, err := json.Marshal(lc) + if err != nil { + t.Fatalf("Failed to marshal due to %v", err) + } + if !bytes.Equal(expected, got) { + t.Fatalf("Expected %s but got %s", expected, got) + } +} From 55f722b5fb35ebd67dfe75619dad9e1e4302308b Mon Sep 17 00:00:00 2001 From: Krutika Dhananjay Date: Mon, 28 Oct 2024 16:03:31 +0530 Subject: [PATCH 2/2] Address review comments --- pkg/lifecycle/lifecycle.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index 624337f31..344af2b78 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -441,21 +441,18 @@ type AllVersionsExpiration struct { DeleteMarker ExpireDeleteMarker `xml:"DeleteMarker,omitempty" json:"DeleteMarker,omitempty"` } -// IsDaysNull returns true if days field is null -func (e AllVersionsExpiration) IsDaysNull() bool { - return e.Days == 0 -} - +// IsNull returns true if days field is 0 func (e AllVersionsExpiration) IsNull() bool { - return e.IsDaysNull() + return e.Days == 0 } -func (ave AllVersionsExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { - if ave.IsNull() { +// MarshalXML satisfies xml.Marshaler to provide custom encoding +func (e AllVersionsExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if e.IsNull() { return nil } type allVersionsExp AllVersionsExpiration - return enc.EncodeElement(allVersionsExp(ave), start) + return enc.EncodeElement(allVersionsExp(e), start) } // MarshalJSON customizes json encoding by omitting empty values