Skip to content

Commit

Permalink
fetching minobject attributes while listing
Browse files Browse the repository at this point in the history
  • Loading branch information
@anushka567
anushka567 committed Oct 25, 2024
1 parent db860f8 commit 98e6bb3
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 114 deletions.
2 changes: 1 addition & 1 deletion internal/storage/bucket_handle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ func (bh *bucketHandle) ListObjects(ctx context.Context, req *gcs.ListObjectsReq
IncludeFoldersAsPrefixes: req.IncludeFoldersAsPrefixes,
//MaxResults: , (Field not present in storage.Query of Go Storage Library but present in ListObjectsQuery in Jacobsa code.)
}
err = query.SetAttrSelection([]string{"Name", "Size", "Generation", "Metageneration", "Updated", "Metadata"})
err = query.SetAttrSelection([]string{"Name", "Size", "Generation", "Metageneration", "Updated", "Metadata", "ContentEncoding", "CRC32C"})
if err != nil {
err = fmt.Errorf("error while setting attribute selection for List Object query :%w", err)
return
Expand Down
169 changes: 56 additions & 113 deletions internal/storage/fake/testing/bucket_tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,31 @@ func (t *bucketTest) matchesStartTime(start time.Time) Matcher {
return timeutil.TimeNear(start, slop)
}

// Given gcs.minObject and gcs.extendedObjectAttributes , assert on attributes of gcs.Object
func (t *bucketTest) assertOnObjectAttributes(minObj *gcs.MinObject, extendedAttr *gcs.ExtendedObjectAttributes, o *gcs.Object) {
ExpectThat(minObj.Name, Equals(o.Name))
ExpectThat(minObj.Size, Equals(o.Size))
ExpectThat(minObj.Generation, Equals(o.Generation))
ExpectThat(minObj.MetaGeneration, Equals(o.MetaGeneration))
ExpectThat(minObj.Updated, DeepEquals(o.Updated))
ExpectThat(minObj.Metadata, DeepEquals(o.Metadata))
ExpectThat(minObj.ContentEncoding, Equals(o.ContentEncoding))
ExpectThat(minObj.CRC32C, Equals(o.CRC32C))
ExpectThat(extendedAttr.ContentType, Equals(o.ContentType))
ExpectThat(extendedAttr.ContentLanguage, Equals(o.ContentLanguage))
ExpectThat(extendedAttr.CacheControl, Equals(o.CacheControl))
ExpectThat(extendedAttr.Owner, Equals(o.Owner))
ExpectThat(extendedAttr.MD5, Equals(o.MD5))
ExpectThat(extendedAttr.MediaLink, Equals(o.MediaLink))
ExpectThat(extendedAttr.StorageClass, Equals(o.StorageClass))
ExpectThat(extendedAttr.Deleted, DeepEquals(o.Deleted))
ExpectThat(extendedAttr.ComponentCount, Equals(o.ComponentCount))
ExpectThat(extendedAttr.ContentDisposition, Equals(o.ContentDisposition))
ExpectThat(extendedAttr.CustomTime, Equals(o.CustomTime))
ExpectThat(extendedAttr.EventBasedHold, Equals(o.EventBasedHold))
ExpectThat(extendedAttr.Acl, DeepEquals(o.Acl))
}

////////////////////////////////////////////////////////////////////////
// Create
////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -551,27 +576,7 @@ func (t *createTest) ObjectAttributes_Default() {
ReturnExtendedObjectAttributes: true})
AssertEq(nil, err)

ExpectThat(minObj.Name, Equals(o.Name))
ExpectThat(minObj.Size, Equals(o.Size))
ExpectThat(minObj.Generation, Equals(o.Generation))
ExpectThat(minObj.MetaGeneration, Equals(o.MetaGeneration))
ExpectThat(minObj.Updated, DeepEquals(o.Updated))
ExpectThat(minObj.Metadata, DeepEquals(o.Metadata))
ExpectThat(minObj.ContentEncoding, Equals(o.ContentEncoding))
ExpectThat(minObj.CRC32C, Equals(o.CRC32C))
ExpectThat(extendedAttr.ContentType, Equals(o.ContentType))
ExpectThat(extendedAttr.ContentLanguage, Equals(o.ContentLanguage))
ExpectThat(extendedAttr.CacheControl, Equals(o.CacheControl))
ExpectThat(extendedAttr.Owner, Equals(o.Owner))
ExpectThat(extendedAttr.MD5, Equals(o.MD5))
ExpectThat(extendedAttr.MediaLink, Equals(o.MediaLink))
ExpectThat(extendedAttr.StorageClass, Equals(o.StorageClass))
ExpectThat(extendedAttr.Deleted, DeepEquals(o.Deleted))
ExpectThat(extendedAttr.ComponentCount, Equals(o.ComponentCount))
ExpectThat(extendedAttr.ContentDisposition, Equals(o.ContentDisposition))
ExpectThat(extendedAttr.CustomTime, Equals(o.CustomTime))
ExpectThat(extendedAttr.EventBasedHold, Equals(o.EventBasedHold))
ExpectThat(extendedAttr.Acl, DeepEquals(o.Acl))
t.assertOnObjectAttributes(minObj, extendedAttr, o)
}

func (t *createTest) ObjectAttributes_Explicit() {
Expand Down Expand Up @@ -625,28 +630,7 @@ func (t *createTest) ObjectAttributes_Explicit() {
ReturnExtendedObjectAttributes: true})
AssertEq(nil, err)

ExpectThat(minObj.Name, Equals(o.Name))
ExpectThat(minObj.Size, Equals(o.Size))
ExpectThat(minObj.Generation, Equals(o.Generation))
ExpectThat(minObj.MetaGeneration, Equals(o.MetaGeneration))
ExpectThat(minObj.Updated, DeepEquals(o.Updated))
ExpectThat(minObj.Metadata, DeepEquals(o.Metadata))
ExpectThat(minObj.ContentEncoding, Equals(o.ContentEncoding))
ExpectThat(minObj.CRC32C, Equals(o.CRC32C))
ExpectThat(extendedAttr.ContentType, Equals(o.ContentType))
ExpectThat(extendedAttr.ContentLanguage, Equals(o.ContentLanguage))
ExpectThat(extendedAttr.CacheControl, Equals(o.CacheControl))
ExpectThat(extendedAttr.Owner, Equals(o.Owner))
ExpectThat(extendedAttr.MD5, Equals(o.MD5))
ExpectThat(extendedAttr.MediaLink, Equals(o.MediaLink))
ExpectThat(extendedAttr.StorageClass, Equals(o.StorageClass))
ExpectThat(extendedAttr.Deleted, DeepEquals(o.Deleted))
ExpectThat(extendedAttr.ComponentCount, Equals(o.ComponentCount))
ExpectThat(extendedAttr.ContentDisposition, Equals(o.ContentDisposition))
ExpectThat(extendedAttr.CustomTime, Equals(o.CustomTime))
ExpectThat(extendedAttr.EventBasedHold, Equals(o.EventBasedHold))
ExpectThat(extendedAttr.Acl, DeepEquals(o.Acl))

t.assertOnObjectAttributes(minObj, extendedAttr, o)
}

func (t *createTest) ErrorAfterPartialContents() {
Expand Down Expand Up @@ -3251,28 +3235,7 @@ func (t *updateTest) RemoveAllFields() {
ReturnExtendedObjectAttributes: true})
AssertEq(nil, err)

ExpectThat(minObj.Name, Equals(o.Name))
ExpectThat(minObj.Size, Equals(o.Size))
ExpectThat(minObj.Generation, Equals(o.Generation))
ExpectThat(minObj.MetaGeneration, Equals(o.MetaGeneration))
ExpectThat(minObj.Updated, DeepEquals(o.Updated))
ExpectThat(minObj.Metadata, DeepEquals(o.Metadata))
ExpectThat(minObj.ContentEncoding, Equals(o.ContentEncoding))
ExpectThat(minObj.CRC32C, Equals(o.CRC32C))
ExpectThat(extendedAttr.ContentType, Equals(o.ContentType))
ExpectThat(extendedAttr.ContentLanguage, Equals(o.ContentLanguage))
ExpectThat(extendedAttr.CacheControl, Equals(o.CacheControl))
ExpectThat(extendedAttr.Owner, Equals(o.Owner))
ExpectThat(extendedAttr.MD5, Equals(o.MD5))
ExpectThat(extendedAttr.MediaLink, Equals(o.MediaLink))
ExpectThat(extendedAttr.StorageClass, Equals(o.StorageClass))
ExpectThat(extendedAttr.Deleted, DeepEquals(o.Deleted))
ExpectThat(extendedAttr.ComponentCount, Equals(o.ComponentCount))
ExpectThat(extendedAttr.ContentDisposition, Equals(o.ContentDisposition))
ExpectThat(extendedAttr.CustomTime, Equals(o.CustomTime))
ExpectThat(extendedAttr.EventBasedHold, Equals(o.EventBasedHold))
ExpectThat(extendedAttr.Acl, DeepEquals(o.Acl))

t.assertOnObjectAttributes(minObj, extendedAttr, o)
}

func (t *updateTest) ModifyAllFields() {
Expand Down Expand Up @@ -3317,20 +3280,15 @@ func (t *updateTest) ModifyAllFields() {

ExpectThat(o.Metadata, DeepEquals(createReq.Metadata))

// Check that a listing agrees.
listing, err := t.bucket.ListObjects(t.ctx, &gcs.ListObjectsRequest{})
// Make sure it matches when we stat object.
minObj, extendedAttr, err := t.bucket.StatObject(
t.ctx,
&gcs.StatObjectRequest{Name: o.Name,
ForceFetchFromGcs: true,
ReturnExtendedObjectAttributes: true})
AssertEq(nil, err)

AssertThat(listing.CollapsedRuns, ElementsAre())
AssertEq("", listing.ContinuationToken)

AssertEq(1, len(listing.Objects))
ExpectThat(listing.Objects[0].Name, Equals(o.Name))
ExpectThat(listing.Objects[0].Size, Equals(o.Size))
ExpectThat(listing.Objects[0].Generation, Equals(o.Generation))
ExpectThat(listing.Objects[0].MetaGeneration, Equals(o.MetaGeneration))
ExpectThat(listing.Objects[0].Updated, DeepEquals(o.Updated))
ExpectThat(listing.Objects[0].Metadata, DeepEquals(o.Metadata))
t.assertOnObjectAttributes(minObj, extendedAttr, o)
}

func (t *updateTest) MixedModificationsToFields() {
Expand Down Expand Up @@ -3375,20 +3333,15 @@ func (t *updateTest) MixedModificationsToFields() {

ExpectThat(o.Metadata, DeepEquals(createReq.Metadata))

// Check that a listing agrees.
listing, err := t.bucket.ListObjects(t.ctx, &gcs.ListObjectsRequest{})
// Make sure it matches when we stat object.
minObj, extendedAttr, err := t.bucket.StatObject(
t.ctx,
&gcs.StatObjectRequest{Name: o.Name,
ForceFetchFromGcs: true,
ReturnExtendedObjectAttributes: true})
AssertEq(nil, err)

AssertThat(listing.CollapsedRuns, ElementsAre())
AssertEq("", listing.ContinuationToken)

AssertEq(1, len(listing.Objects))
ExpectThat(listing.Objects[0].Name, Equals(o.Name))
ExpectThat(listing.Objects[0].Size, Equals(o.Size))
ExpectThat(listing.Objects[0].Generation, Equals(o.Generation))
ExpectThat(listing.Objects[0].MetaGeneration, Equals(o.MetaGeneration))
ExpectThat(listing.Objects[0].Updated, DeepEquals(o.Updated))
ExpectThat(listing.Objects[0].Metadata, DeepEquals(o.Metadata))
t.assertOnObjectAttributes(minObj, extendedAttr, o)
}

func (t *updateTest) AddUserMetadata() {
Expand Down Expand Up @@ -3423,20 +3376,15 @@ func (t *updateTest) AddUserMetadata() {
"1": "burrito",
}))

// Check that a listing agrees.
listing, err := t.bucket.ListObjects(t.ctx, &gcs.ListObjectsRequest{})
// Make sure it matches when we stat object.
minObj, extendedAttr, err := t.bucket.StatObject(
t.ctx,
&gcs.StatObjectRequest{Name: o.Name,
ForceFetchFromGcs: true,
ReturnExtendedObjectAttributes: true})
AssertEq(nil, err)

AssertThat(listing.CollapsedRuns, ElementsAre())
AssertEq("", listing.ContinuationToken)

AssertEq(1, len(listing.Objects))
ExpectThat(listing.Objects[0].Name, Equals(o.Name))
ExpectThat(listing.Objects[0].Size, Equals(o.Size))
ExpectThat(listing.Objects[0].Generation, Equals(o.Generation))
ExpectThat(listing.Objects[0].MetaGeneration, Equals(o.MetaGeneration))
ExpectThat(listing.Objects[0].Updated, DeepEquals(o.Updated))
ExpectThat(listing.Objects[0].Metadata, DeepEquals(o.Metadata))
t.assertOnObjectAttributes(minObj, extendedAttr, o)
}

func (t *updateTest) MixedModificationsToUserMetadata() {
Expand Down Expand Up @@ -3485,20 +3433,15 @@ func (t *updateTest) MixedModificationsToUserMetadata() {
"3": "updated",
}))

// Check that a listing agrees.
listing, err := t.bucket.ListObjects(t.ctx, &gcs.ListObjectsRequest{})
// Make sure it matches when we stat object.
minObj, extendedAttr, err := t.bucket.StatObject(
t.ctx,
&gcs.StatObjectRequest{Name: o.Name,
ForceFetchFromGcs: true,
ReturnExtendedObjectAttributes: true})
AssertEq(nil, err)

AssertThat(listing.CollapsedRuns, ElementsAre())
AssertEq("", listing.ContinuationToken)

AssertEq(1, len(listing.Objects))
ExpectThat(listing.Objects[0].Name, Equals(o.Name))
ExpectThat(listing.Objects[0].Size, Equals(o.Size))
ExpectThat(listing.Objects[0].Generation, Equals(o.Generation))
ExpectThat(listing.Objects[0].MetaGeneration, Equals(o.MetaGeneration))
ExpectThat(listing.Objects[0].Updated, DeepEquals(o.Updated))
ExpectThat(listing.Objects[0].Metadata, DeepEquals(o.Metadata))
t.assertOnObjectAttributes(minObj, extendedAttr, o)
}

func (t *updateTest) UpdateTime() {
Expand Down

0 comments on commit 98e6bb3

Please sign in to comment.