From c635815285bb7c083c246d96168af97d5288f659 Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Wed, 9 Oct 2024 00:52:54 +0300 Subject: [PATCH 1/3] neofs: add pool for searching With ObjectSearchInitter ObjectSearch can be done both NeoFS SDK client or pool. Signed-off-by: Ekaterina Pavlova --- pkg/services/oracle/neofs/neofs.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/services/oracle/neofs/neofs.go b/pkg/services/oracle/neofs/neofs.go index af245463e3..7f2ddd5645 100644 --- a/pkg/services/oracle/neofs/neofs.go +++ b/pkg/services/oracle/neofs/neofs.go @@ -236,8 +236,13 @@ func parseRange(s string) (*object.Range, error) { return r, nil } +// ObjectSearchInitter defines the interface for initializing object search. +type ObjectSearchInitter interface { + ObjectSearchInit(ctx context.Context, containerID cid.ID, s user.Signer, prm client.PrmObjectSearch) (*client.ObjectListReader, error) +} + // ObjectSearch returns a list of object IDs from the provided container. -func ObjectSearch(ctx context.Context, c *client.Client, priv *keys.PrivateKey, containerIDStr string, prm client.PrmObjectSearch) ([]oid.ID, error) { +func ObjectSearch(ctx context.Context, initter ObjectSearchInitter, priv *keys.PrivateKey, containerIDStr string, prm client.PrmObjectSearch) ([]oid.ID, error) { var ( s = user.NewAutoIDSignerRFC6979(priv.PrivateKey) objectIDs []oid.ID @@ -247,7 +252,7 @@ func ObjectSearch(ctx context.Context, c *client.Client, priv *keys.PrivateKey, if err != nil { return nil, fmt.Errorf("%w: %w", ErrInvalidContainer, err) } - reader, err := c.ObjectSearchInit(ctx, containerID, s, prm) + reader, err := initter.ObjectSearchInit(ctx, containerID, s, prm) if err != nil { return nil, fmt.Errorf("failed to initiate object search: %w", err) } From 3c0605be5c3a5267a3440307e688d4b0111e778e Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Wed, 9 Oct 2024 00:53:33 +0300 Subject: [PATCH 2/3] docs: adjust default IndexFileAttribute Signed-off-by: Ekaterina Pavlova --- config/protocol.testnet.yml | 2 +- docs/neofs-blockstorage.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/protocol.testnet.yml b/config/protocol.testnet.yml index 9a31251a2b..ecdfd6eed6 100644 --- a/config/protocol.testnet.yml +++ b/config/protocol.testnet.yml @@ -112,4 +112,4 @@ ApplicationConfiguration: IndexFileSize: 128000 ContainerID: "EPGuD26wYgQJbmDdVBoYoNZiMKHwFMJT3A5WqPjdUHxH" BlockAttribute: "block" - IndexFileAttribute: "oid" + IndexFileAttribute: "index" diff --git a/docs/neofs-blockstorage.md b/docs/neofs-blockstorage.md index 9755713805..4ecd1b8de0 100644 --- a/docs/neofs-blockstorage.md +++ b/docs/neofs-blockstorage.md @@ -16,12 +16,12 @@ attributes: - primary node index (`primary:0`) - block hash in the LE form (`hash:5412a781caf278c0736556c0e544c7cfdbb6e3c62ae221ef53646be89364566b`) - previous block hash in the LE form (`prevHash:3654a054d82a8178c7dfacecc2c57282e23468a42ee407f14506368afe22d929`) - - millisecond-precision block timestamp (`time:1627894840919`) + - millisecond-precision block timestamp (`timestamp:1627894840919`) Each index file is an object containing a constant-sized batch of raw block object IDs in binary form ordered by block index. Each index file is marked with the following attributes: - - index file identifier with consecutive file index value (`oid:0`) + - index file identifier with consecutive file index value (`index:0`) - the number of OIDs included into index file (`size:128000`) ### NeoFS BlockFetcher From 0a4e1d1ef96cecb53fdc66450429bdce661a5ef9 Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Fri, 27 Sep 2024 13:07:18 +0300 Subject: [PATCH 3/3] cli: add upload-bin This command is used for keeping container with blocks for blockfetcher updated. Close #3578 Signed-off-by: Ekaterina Pavlova README: add archival notice (#520) People are likely to find this repo and we better have some directions for them. Signed-off-by: Roman Khimov Signed-off-by: Ekaterina Pavlova --- cli/util/convert.go | 53 +++++ cli/util/uploader.go | 432 +++++++++++++++++++++++++++++++++++++++++ go.sum | 63 ++++++ pkg/vm/testdata/neo-vm | 2 +- 4 files changed, 549 insertions(+), 1 deletion(-) create mode 100644 cli/util/uploader.go diff --git a/cli/util/convert.go b/cli/util/convert.go index 26c520877a..23959a533b 100644 --- a/cli/util/convert.go +++ b/cli/util/convert.go @@ -6,6 +6,7 @@ import ( "fmt" "os" + "github.com/nspcc-dev/neo-go/cli/cmdargs" "github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/cli/options" "github.com/nspcc-dev/neo-go/cli/txctx" @@ -32,6 +33,51 @@ func NewCommands() []*cli.Command { txctx.AwaitFlag, }, options.RPC...) txCancelFlags = append(txCancelFlags, options.Wallet...) + uploadBinFlags := append([]cli.Flag{ + &cli.StringSliceFlag{ + Name: "fs-rpc-endpoint", + Aliases: []string{"fsr"}, + Usage: "List of NeoFS storage node RPC addresses (comma-separated or multiple --fs-rpc-endpoint flags)", + Required: true, + Action: func(ctx *cli.Context, fsRpcEndpoints []string) error { + for _, endpoint := range fsRpcEndpoints { + if endpoint == "" { + return cli.Exit("NeoFS RPC endpoint cannot contain empty values", 1) + } + } + return nil + }, + }, + &cli.StringFlag{ + Name: "container", + Aliases: []string{"cid"}, + Usage: "NeoFS container ID to upload blocks to", + Required: true, + Action: cmdargs.EnsureNotEmpty("container"), + }, + &cli.StringFlag{ + Name: "block-attribute", + Usage: "Attribute key of the block object", + Required: true, + Action: cmdargs.EnsureNotEmpty("block-attribute"), + }, + &cli.StringFlag{ + Name: "index-attribute", + Usage: "Attribute key of the index file object", + Required: true, + Action: cmdargs.EnsureNotEmpty("index-attribute"), + }, + &flags.AddressFlag{ + Name: "address", + Usage: "Address to use for signing the uploading and searching transactions in NeoFS", + }, + &cli.UintFlag{ + Name: "index-file-size", + Usage: "Size of index file", + Value: 128000, + }, + }, options.RPC...) + uploadBinFlags = append(uploadBinFlags, options.Wallet...) return []*cli.Command{ { Name: "util", @@ -109,6 +155,13 @@ func NewCommands() []*cli.Command { }, }, }, + { + Name: "upload-bin", + Usage: "Fetch blocks from RPC node and upload them to the NeoFS container", + UsageText: "neo-go util upload-bin --fs-rpc-endpoint [,[...]] --container --block-attribute block --index-attribute index --rpc-endpoint [--timeout