From cf1a6d91cd954a62b791bd7c6b28aebc8a44c0a9 Mon Sep 17 00:00:00 2001 From: Alexxxxxx <118710506+alexgao001@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:29:15 +0800 Subject: [PATCH] feat: provide APIs for delegate upload and update (#228) * delegate uplaod * delegate uplaod * delegate upload * delegate upload * delegate upload * delegate upload * add comments --- client/api_bucket.go | 14 +++++-- client/api_client.go | 1 - client/api_object.go | 82 +++++++++++++++++++++++++++++++++----- client/api_payment.go | 1 - e2e/basesuite/suite.go | 5 ++- e2e/e2e_storage_test.go | 25 ++++++++++-- examples/crosschain_gov.go | 3 +- examples/storage.go | 6 ++- go.mod | 4 +- go.sum | 8 ++-- types/option.go | 3 ++ 11 files changed, 122 insertions(+), 30 deletions(-) diff --git a/client/api_bucket.go b/client/api_bucket.go index b39e2b87..9f344012 100644 --- a/client/api_bucket.go +++ b/client/api_bucket.go @@ -38,6 +38,7 @@ type IBucketClient interface { UpdateBucketVisibility(ctx context.Context, bucketName string, visibility storageTypes.VisibilityType, opt types.UpdateVisibilityOption) (string, error) UpdateBucketInfo(ctx context.Context, bucketName string, opts types.UpdateBucketOptions) (string, error) UpdateBucketPaymentAddr(ctx context.Context, bucketName string, paymentAddr sdk.AccAddress, opt types.UpdatePaymentOption) (string, error) + ToggleSPAsDelegatedAgent(ctx context.Context, bucketName string, opt types.UpdateBucketOptions) (string, error) HeadBucket(ctx context.Context, bucketName string) (*storageTypes.BucketInfo, error) HeadBucketByID(ctx context.Context, bucketID string) (*storageTypes.BucketInfo, error) PutBucketPolicy(ctx context.Context, bucketName string, principal types.Principal, statements []*permTypes.Statement, opt types.PutPolicyOption) (string, error) @@ -333,6 +334,16 @@ func (c *Client) UpdateBucketInfo(ctx context.Context, bucketName string, opts t return c.sendTxn(ctx, updateBucketMsg, opts.TxOpts) } +func (c *Client) ToggleSPAsDelegatedAgent(ctx context.Context, bucketName string, opt types.UpdateBucketOptions, +) (string, error) { + _, err := c.HeadBucket(ctx, bucketName) + if err != nil { + return "", err + } + msg := storageTypes.NewMsgToggleSPAsDelegatedAgent(c.MustGetDefaultAccount().GetAddress(), bucketName) + return c.sendTxn(ctx, msg, opt.TxOpts) +} + // HeadBucket - query the bucketInfo on chain by bucket name, return the bucket info if exists. // // - ctx: Context variables for the current API call. @@ -584,7 +595,6 @@ func (c *Client) ListBuckets(ctx context.Context, opts types.ListBucketsOptions) bufStr := buf.String() err = xml.Unmarshal([]byte(bufStr), &listBucketsResult) - // TODO(annie) remove tolerance for unmarshal err after structs got stabilized if err != nil { return types.ListBucketsResult{}, err @@ -981,7 +991,6 @@ func (c *Client) MigrateBucket(ctx context.Context, bucketName string, dstPrimar // // - ret2: Return error when the request of cancel migration failed, otherwise return nil. func (c *Client) CancelMigrateBucket(ctx context.Context, bucketName string, opts types.CancelMigrateBucketOptions) (string, error) { - cancelMigrateBucketMsg := storageTypes.NewMsgCancelMigrateBucket(c.MustGetDefaultAccount().GetAddress(), bucketName) err := cancelMigrateBucketMsg.ValidateBasic() @@ -1033,7 +1042,6 @@ func (c *Client) CancelMigrateBucket(ctx context.Context, bucketName string, opt // // - ret2: Return error when the request failed, otherwise return nil. func (c *Client) ListBucketsByPaymentAccount(ctx context.Context, paymentAccount string, opts types.ListBucketsByPaymentAccountOptions) (types.ListBucketsByPaymentAccountResult, error) { - _, err := sdk.AccAddressFromHexUnsafe(paymentAccount) if err != nil { return types.ListBucketsByPaymentAccountResult{}, err diff --git a/client/api_client.go b/client/api_client.go index 1d833925..283510bb 100644 --- a/client/api_client.go +++ b/client/api_client.go @@ -206,7 +206,6 @@ func New(chainID string, endpoint string, option Option) (IClient, error) { } else { // fetch sp endpoints info from chain err = c.refreshStorageProviders(context.Background()) - if err != nil { return nil, err } diff --git a/client/api_object.go b/client/api_object.go index c1ee309c..9a354fbd 100644 --- a/client/api_object.go +++ b/client/api_object.go @@ -40,6 +40,8 @@ type IObjectClient interface { CancelUpdateObjectContent(ctx context.Context, bucketName, objectName string, opts types.CancelUpdateObjectOption) (string, error) PutObject(ctx context.Context, bucketName, objectName string, objectSize int64, reader io.Reader, opts types.PutObjectOptions) error putObjectResumable(ctx context.Context, bucketName, objectName string, objectSize int64, reader io.Reader, opts types.PutObjectOptions) error + DelegatePutObject(ctx context.Context, bucketName, objectName string, objectSize int64, reader io.Reader, opts types.PutObjectOptions) error + DelegateUpdateObjectContent(ctx context.Context, bucketName, objectName string, objectSize int64, reader io.Reader, opts types.PutObjectOptions) error FPutObject(ctx context.Context, bucketName, objectName, filePath string, opts types.PutObjectOptions) (err error) CancelCreateObject(ctx context.Context, bucketName, objectName string, opt types.CancelCreateOption) (string, error) DeleteObject(ctx context.Context, bucketName, objectName string, opt types.DeleteObjectOption) (string, error) @@ -316,9 +318,11 @@ func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, o func (c *Client) putObject(ctx context.Context, bucketName, objectName string, objectSize int64, reader io.Reader, opts types.PutObjectOptions, ) (err error) { - if err := c.headSPObjectInfo(ctx, bucketName, objectName); err != nil { - log.Error().Msg(fmt.Sprintf("fail to head object %s , err %v ", objectName, err)) - return err + if !opts.Delegated { + if err := c.headSPObjectInfo(ctx, bucketName, objectName); err != nil { + log.Error().Msg(fmt.Sprintf("fail to head object %s , err %v ", objectName, err)) + return err + } } var contentType string @@ -327,6 +331,16 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, o } else { contentType = types.ContentDefault } + urlValues := make(url.Values) + + if opts.Delegated { + urlValues.Set("delegate", "") + urlValues.Set("is_update", strconv.FormatBool(opts.IsUpdate)) + urlValues.Set("payload_size", strconv.FormatInt(objectSize, 10)) + if !opts.IsUpdate { + urlValues.Set("visibility", strconv.FormatInt(int64(opts.Visibility), 10)) + } + } reqMeta := requestMeta{ bucketName: bucketName, @@ -334,6 +348,7 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, o contentSHA256: types.EmptyStringSHA256, contentLength: objectSize, contentType: contentType, + urlValues: urlValues, } var sendOpt sendOptions @@ -376,13 +391,18 @@ func DefaultUploadSegment(id int) error { func (c *Client) putObjectResumable(ctx context.Context, bucketName, objectName string, objectSize int64, reader io.Reader, opts types.PutObjectOptions, ) (err error) { - if err := c.headSPObjectInfo(ctx, bucketName, objectName); err != nil { - return err - } + var offset uint64 - offset, err := c.getObjectResumableUploadOffset(ctx, bucketName, objectName) - if err != nil { - return err + if !opts.Delegated { + if err = c.headSPObjectInfo(ctx, bucketName, objectName); err != nil { + return err + } + offset, err = c.getObjectResumableUploadOffset(ctx, bucketName, objectName) + if err != nil { + return err + } + } else { + offset = 0 } // Total data read and written to server. should be equal to @@ -450,6 +470,14 @@ func (c *Client) putObjectResumable(ctx context.Context, bucketName, objectName urlValues.Set("offset", strconv.FormatInt(totalUploadedSize, 10)) urlValues.Set("complete", strconv.FormatBool(complete)) + if opts.Delegated { + urlValues.Set("delegate", "") + urlValues.Set("is_update", strconv.FormatBool(opts.IsUpdate)) + urlValues.Set("payload_size", strconv.FormatInt(objectSize, 10)) + if !opts.IsUpdate { + urlValues.Set("visibility", strconv.FormatInt(int64(opts.Visibility), 10)) + } + } reqMeta := requestMeta{ bucketName: bucketName, objectName: objectName, @@ -575,7 +603,6 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, endpoint = c.forceToUseSpecifiedSpEndpointForDownloadOnly } else { endpoint, err = c.getSPUrlByBucket(bucketName) - if err != nil { log.Error().Msg(fmt.Sprintf("route endpoint by bucket: %s failed, err: %s", bucketName, err.Error())) return nil, types.ObjectStat{}, err @@ -1445,3 +1472,38 @@ func (c *Client) ListObjectPolicies(ctx context.Context, objectName, bucketName return policies, nil } + +func (c *Client) DelegatePutObject(ctx context.Context, bucketName, objectName string, objectSize int64, + reader io.Reader, opts types.PutObjectOptions, +) (err error) { + if objectSize <= 0 { + return errors.New("object size should be more than 0") + } + params, err := c.GetParams() + if err != nil { + return err + } + opts.Delegated = true + // minPartSize: 16MB + if opts.PartSize == 0 { + opts.PartSize = types.MinPartSize + } + if opts.PartSize%params.GetMaxSegmentSize() != 0 { + return errors.New("part size should be an integer multiple of the segment size") + } + + // upload an entire object to the storage provider in a single request + if objectSize <= int64(opts.PartSize) || opts.DisableResumable { + return c.putObject(ctx, bucketName, objectName, objectSize, reader, opts) + } + + // resumableupload + return c.putObjectResumable(ctx, bucketName, objectName, objectSize, reader, opts) +} + +func (c *Client) DelegateUpdateObjectContent(ctx context.Context, bucketName, objectName string, objectSize int64, + reader io.Reader, opts types.PutObjectOptions, +) (err error) { + opts.IsUpdate = true + return c.DelegatePutObject(ctx, bucketName, objectName, objectSize, reader, opts) +} diff --git a/client/api_payment.go b/client/api_payment.go index 410fafc9..8180d06f 100644 --- a/client/api_payment.go +++ b/client/api_payment.go @@ -203,7 +203,6 @@ func (c *Client) ListUserPaymentAccounts(ctx context.Context, opts types.ListUse bufStr := buf.String() err = xml.Unmarshal([]byte(bufStr), &paymentAccounts) - if err != nil { return types.ListUserPaymentAccountsResult{}, err } diff --git a/e2e/basesuite/suite.go b/e2e/basesuite/suite.go index f3ee1aeb..fc3f7b9a 100644 --- a/e2e/basesuite/suite.go +++ b/e2e/basesuite/suite.go @@ -4,11 +4,12 @@ import ( "bufio" "context" "fmt" - storageTypes "github.com/bnb-chain/greenfield/x/storage/types" "os" "path/filepath" "time" + storageTypes "github.com/bnb-chain/greenfield/x/storage/types" + "github.com/bnb-chain/greenfield-go-sdk/client" "github.com/bnb-chain/greenfield-go-sdk/types" "github.com/stretchr/testify/suite" @@ -90,7 +91,7 @@ func (s *BaseSuite) WaitSealObject(bucketName string, objectName string) { for i := 0; i < 100; i++ { objectDetail, err = s.Client.HeadObject(s.ClientContext, bucketName, objectName) s.Require().NoError(err) - if objectDetail.ObjectInfo.GetObjectStatus() == storageTypes.OBJECT_STATUS_SEALED { + if objectDetail.ObjectInfo.GetObjectStatus() == storageTypes.OBJECT_STATUS_SEALED && !objectDetail.ObjectInfo.GetIsUpdating() { break } time.Sleep(3 * time.Second) diff --git a/e2e/e2e_storage_test.go b/e2e/e2e_storage_test.go index d92d0bad..6777570f 100644 --- a/e2e/e2e_storage_test.go +++ b/e2e/e2e_storage_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + "github.com/bnb-chain/greenfield/types/resource" + "cosmossdk.io/math" "github.com/stretchr/testify/suite" @@ -21,7 +23,6 @@ import ( types2 "github.com/bnb-chain/greenfield/sdk/types" storageTestUtil "github.com/bnb-chain/greenfield/testutil/storage" greenfield_types "github.com/bnb-chain/greenfield/types" - "github.com/bnb-chain/greenfield/types/resource" permTypes "github.com/bnb-chain/greenfield/x/permission/types" spTypes "github.com/bnb-chain/greenfield/x/sp/types" storageTypes "github.com/bnb-chain/greenfield/x/storage/types" @@ -38,7 +39,7 @@ func (s *StorageTestSuite) SetupSuite() { spList, err := s.Client.ListStorageProviders(s.ClientContext, false) s.Require().NoError(err) for _, sp := range spList { - if sp.Endpoint != "https://sp0.greenfield.io" { + if sp.Endpoint != "https://sp0.greenfield.io" && sp.Id == 1 { s.PrimarySP = sp break } @@ -287,6 +288,22 @@ func (s *StorageTestSuite) Test_Object() { s.Require().NoError(err) _, err = s.Client.HeadObject(s.ClientContext, bucketName, objectName) s.Require().Error(err) + + objectName2 := storageTestUtil.GenRandomObjectName() + err = s.Client.DelegatePutObject(s.ClientContext, bucketName, objectName2, objectSize, bytes.NewReader(buffer.Bytes()), types.PutObjectOptions{}) + s.Require().NoError(err) + s.WaitSealObject(bucketName, objectName2) + + var newBuffer bytes.Buffer + for i := 0; i < 1024*300*40; i++ { + newBuffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + newObjectSize := int64(newBuffer.Len()) + s.T().Logf("newObjectSize: %d", newObjectSize) + + err = s.Client.DelegateUpdateObjectContent(s.ClientContext, bucketName, objectName2, newObjectSize, bytes.NewReader(newBuffer.Bytes()), types.PutObjectOptions{}) + s.Require().NoError(err) + s.WaitSealObject(bucketName, objectName2) } func (s *StorageTestSuite) Test_Group() { @@ -652,7 +669,7 @@ func (s *StorageTestSuite) Test_Upload_Object_With_Tampering_Content() { } func (s *StorageTestSuite) Test_Group_with_Tag() { - //create group with tag + // create group with tag groupName := storageTestUtil.GenRandomGroupName() groupOwner := s.DefaultAccount.GetAddress() @@ -674,7 +691,7 @@ func (s *StorageTestSuite) Test_Group_with_Tag() { } func (s *StorageTestSuite) Test_CreateGroup_And_Set_Tag() { - //create group with tag + // create group with tag groupName := storageTestUtil.GenRandomGroupName() groupOwner := s.DefaultAccount.GetAddress() diff --git a/examples/crosschain_gov.go b/examples/crosschain_gov.go index d753578f..cd0e37a5 100644 --- a/examples/crosschain_gov.go +++ b/examples/crosschain_gov.go @@ -2,6 +2,8 @@ package main import ( "context" + "log" + "cosmossdk.io/math" "github.com/bnb-chain/greenfield-go-sdk/client" "github.com/bnb-chain/greenfield-go-sdk/types" @@ -10,7 +12,6 @@ import ( authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" govtypes "github.com/cosmos/cosmos-sdk/x/gov/types" govv1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1" - "log" ) func main() { diff --git a/examples/storage.go b/examples/storage.go index 6bbdcd46..2997917a 100644 --- a/examples/storage.go +++ b/examples/storage.go @@ -133,8 +133,10 @@ func main() { log.Printf("object: %s has been deleted\n", objectName) // list buckets - paymentBuckets, err := cli.ListBucketsByPaymentAccount(ctx, paymentAddr, types.ListBucketsByPaymentAccountOptions{Endpoint: httpsAddr, - SPAddress: ""}) + paymentBuckets, err := cli.ListBucketsByPaymentAccount(ctx, paymentAddr, types.ListBucketsByPaymentAccountOptions{ + Endpoint: httpsAddr, + SPAddress: "", + }) log.Println("list buckets by payment account result:") for _, bucket := range paymentBuckets.Buckets { i := bucket.BucketInfo diff --git a/go.mod b/go.mod index a6e7af88..e0f05b76 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/math v1.0.1 - github.com/bnb-chain/greenfield v1.4.1-0.20240221082420-50b8ec14277e + github.com/bnb-chain/greenfield v1.5.2-0.20240315090203-f28b240981f6 github.com/bnb-chain/greenfield-common/go v0.0.0-20240228080631-2683b0ee669a github.com/cometbft/cometbft v0.37.2 github.com/consensys/gnark-crypto v0.7.0 @@ -149,7 +149,7 @@ replace ( github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v1.2.0 github.com/cometbft/cometbft-db => github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 - github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.4.1-0.20240221065455-ef1f7f0d2659 + github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.5.1-0.20240318023627-97f1fb45ef8c github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1 github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 ) diff --git a/go.sum b/go.sum index ef142aab..85dcd28d 100644 --- a/go.sum +++ b/go.sum @@ -152,16 +152,16 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 h1:41iFGWnSlI2gVpmOtVTJZNodLdLQLn/KsJqFvXwnd/s= github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= -github.com/bnb-chain/greenfield v1.4.1-0.20240221082420-50b8ec14277e h1:7zxM7j3KjRQojNt40Jb+JiweL8Mxm0m4PU7YsaU9CZ4= -github.com/bnb-chain/greenfield v1.4.1-0.20240221082420-50b8ec14277e/go.mod h1:sIm6fjiv6WFNKNl1Lcg4WBEvyb8yy79eLs9xzbkmf4A= +github.com/bnb-chain/greenfield v1.5.2-0.20240315090203-f28b240981f6 h1:XZg4CbWG5szMzqQwcApWUEXBYNFCJbmRlPTdlfcUm0Y= +github.com/bnb-chain/greenfield v1.5.2-0.20240315090203-f28b240981f6/go.mod h1:z970om1k0EPmDFCUvxZpufQz3a1bOP7QriaZbaywaVY= github.com/bnb-chain/greenfield-cometbft v1.2.0 h1:LTStppZS9WkVj0TfEYKkk5OAQDGfYlUefWByr7Zr018= github.com/bnb-chain/greenfield-cometbft v1.2.0/go.mod h1:WVOEZ59UYM2XePQH47/IQfcInspDn8wbRXhFSJrbU1c= github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 h1:XcWulGacHVRiSCx90Q8Y//ajOrLNBQWR/KDB89dy3cU= github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1/go.mod h1:ey1CiK4bYo1RBNJLRiVbYr5CMdSxci9S/AZRINLtppI= github.com/bnb-chain/greenfield-common/go v0.0.0-20240228080631-2683b0ee669a h1:VjUknQkIcqkjYCt1hmfpinM7kToOBuUU+KykrrqFsEM= github.com/bnb-chain/greenfield-common/go v0.0.0-20240228080631-2683b0ee669a/go.mod h1:K9jK80fbahciC+FAvrch8Qsbw9ZkvVgjfKsqrzPTAVA= -github.com/bnb-chain/greenfield-cosmos-sdk v1.4.1-0.20240221065455-ef1f7f0d2659 h1:ytOD5CuSsmV9pe9HXXJEsxiDKxHzOYShSG8s21Yw5Xw= -github.com/bnb-chain/greenfield-cosmos-sdk v1.4.1-0.20240221065455-ef1f7f0d2659/go.mod h1:XF8U3VN1euzLkIR5xiSNyQSnBabvnD86oz6fgdrpteQ= +github.com/bnb-chain/greenfield-cosmos-sdk v1.5.1-0.20240318023627-97f1fb45ef8c h1:Eszn0PMtdYr/i6PYTBmGcToittIN1MZMzmwdmMYeBnA= +github.com/bnb-chain/greenfield-cosmos-sdk v1.5.1-0.20240318023627-97f1fb45ef8c/go.mod h1:XF8U3VN1euzLkIR5xiSNyQSnBabvnD86oz6fgdrpteQ= github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20231129013257-1e407f209b02 h1:cwuShQ+MlvwkfbOz79BRF4aYjgKAuSyugoCtXE8tWgM= github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20231129013257-1e407f209b02/go.mod h1:vhsZxXE9tYJeYB5JR4hPhd6Pc/uPf7j1T8IJ7p9FdeM= github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231129013257-1e407f209b02 h1:RMrZep7e2dcMk4E5YysMdCn6PekI3vA2WHfnMQ/kg18= diff --git a/types/option.go b/types/option.go index 482578e4..ec684173 100644 --- a/types/option.go +++ b/types/option.go @@ -233,6 +233,9 @@ type PutObjectOptions struct { TxnHash string // TxnHash indicates the transaction hash creating the object meta on chain. DisableResumable bool // DisableResumable indicates whether upload the object to Storage Provider via resumable upload. PartSize uint64 + Delegated bool // Delegated indicates that the request to SP will require SP to create/update objet behalf of the uploader. + IsUpdate bool // IsUpdate indicates that the request to SP is a delegated update object request. + Visibility storageTypes.VisibilityType } // GetObjectOptions contains the options for `GetObject` API.