From e1705071f4a66cfc90bc536e1cbabc4e4db5f9f4 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Thu, 1 Jul 2021 16:04:09 -0300 Subject: [PATCH 1/5] pow: prepare with aggregation Signed-off-by: Ignacio Hagopian --- cmd/pow/cmd/prepare/prepare.go | 147 ++++++++++++++++++++++++++------- go.mod | 6 +- go.sum | 16 ++-- 3 files changed, 128 insertions(+), 41 deletions(-) diff --git a/cmd/pow/cmd/prepare/prepare.go b/cmd/pow/cmd/prepare/prepare.go index 33ca5ab9..9384ff6c 100644 --- a/cmd/pow/cmd/prepare/prepare.go +++ b/cmd/pow/cmd/prepare/prepare.go @@ -12,8 +12,12 @@ import ( "sync" "time" + files "github.com/ipfs/go-ipfs-files" + unixfile "github.com/ipfs/go-unixfs/file" + "github.com/cheggaaa/pb/v3" "github.com/dustin/go-humanize" + aggregator "github.com/filecoin-project/go-dagaggregator-unixfs" bsrv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-car" "github.com/ipfs/go-cid" @@ -36,6 +40,7 @@ func init() { prepare.Flags().String("tmpdir", os.TempDir(), "path of folder where a temporal blockstore is created for processing data") prepare.Flags().String("ipfs-api", "", "IPFS HTTP API multiaddress that stores the cid (only for Cid processing instead of file/folder path)") prepare.Flags().Bool("json", false, "avoid pretty output and use json formatting") + prepare.Flags().Bool("aggregate", false, "aggregates a folder of files") commp.Flags().Bool("json", false, "avoid pretty output and use json formatting") commp.Flags().Bool("skip-car-validation", false, "skips CAR validation when processing a path") @@ -86,7 +91,7 @@ If a Cid is provided, an extra --ipfs-api flag should be provided to connect to if err != nil { c.Fatal(fmt.Errorf("parsing json flag: %s", err)) } - dataCid, dagService, cls, err := prepareDAGService(cmd, args, quiet) + dataCid, dagService, _, cls, err := prepareDAGService(cmd, args, quiet) if err != nil { c.Fatal(fmt.Errorf("creating dag-service: %s", err)) } @@ -147,7 +152,7 @@ You can use the --skip-car-validation, but usually shouldn't be done unless you c.Fatal(fmt.Errorf("parsing json flag: %s", err)) } if jsonFlag { - printJSONResult(pieceSize, pieceCID) + printJSONResult(pieceSize, cid.Undef, pieceCID, nil) return } c.Message("Piece-size: %d (%s)", pieceSize, humanize.IBytes(pieceSize)) @@ -183,7 +188,7 @@ The piece-size and piece-cid are printed to stderr. For scripting usage, its rec if err != nil { c.Fatal(fmt.Errorf("parsing json flag: %s", err)) } - dataCid, dagService, cls, err := prepareDAGService(cmd, args, json) + payloadCid, dagService, aggrFiles, cls, err := prepareDAGService(cmd, args, json) if err != nil { c.Fatal(fmt.Errorf("creating temporal dag-service: %s", err)) } @@ -216,7 +221,7 @@ The piece-size and piece-cid are printed to stderr. For scripting usage, its rec c.Fatal(fmt.Errorf("closing car writer: %s", err)) } }() - if err := car.WriteCar(ctx, dagService, []cid.Cid{dataCid}, pwCAR); err != nil { + if err := car.WriteCar(ctx, dagService, []cid.Cid{payloadCid}, pwCAR); err != nil { writeCarErr = err return } @@ -249,63 +254,101 @@ The piece-size and piece-cid are printed to stderr. For scripting usage, its rec c.Fatal(fmt.Errorf("calculating piece-size and PieceCID: %s", err)) } if json { - printJSONResult(pieceSize, pieceCid) + printJSONResult(pieceSize, payloadCid, pieceCid, aggrFiles) + + aggregate, err := cmd.Flags().GetBool("aggregate") + if err != nil { + c.Fatal(fmt.Errorf("aggregate flag: %s", err)) + } + if aggregate { + rootNd, err := dagService.Get(ctx, payloadCid) + if err != nil { + c.Fatal(fmt.Errorf("get root node: %s", err)) + } + manifestLnk := rootNd.Links()[0] + manifestNd, err := dagService.Get(ctx, manifestLnk.Cid) + if err != nil { + c.Fatal(fmt.Errorf("get manfiest node: %s", err)) + } + manifestF, err := unixfile.NewUnixfsFile(context.Background(), dagService, manifestNd) + if err != nil { + c.Fatal(fmt.Errorf("get unifxfs manifest file: %s", err)) + } + manifest := manifestF.(files.File) + fManifest, err := os.Create(args[1] + ".manifest") + if err != nil { + c.Fatal(fmt.Errorf("creating manifest file: %s", err)) + + } + defer func() { + if err := fManifest.Close(); err != nil { + c.Fatal(fmt.Errorf("closing manifest file: %s", err)) + } + }() + if _, err := io.Copy(fManifest, manifest); err != nil { + c.Fatal(fmt.Errorf("writing manifest file: %s", err)) + } + } return } c.Message("Created CAR file, and piece digest in %.02fs.", time.Since(start).Seconds()) + c.Message("Payload cid: %s", payloadCid) c.Message("Piece size: %d (%s)", pieceSize, humanize.IBytes(pieceSize)) c.Message("Piece CID: %s\n", pieceCid) c.Message("Lotus offline-deal command:") - c.Message("lotus client deal --manual-piece-cid=%s --manual-piece-size=%d %s ", pieceCid, pieceSize, dataCid) + c.Message("lotus client deal --manual-piece-cid=%s --manual-piece-size=%d %s ", pieceCid, pieceSize, payloadCid) }, } type closeFunc func() error -func prepareDAGService(cmd *cobra.Command, args []string, quiet bool) (cid.Cid, ipld.DAGService, closeFunc, error) { +func prepareDAGService(cmd *cobra.Command, args []string, quiet bool) (cid.Cid, ipld.DAGService, []aggregatedFile, closeFunc, error) { ipfsAPI, err := cmd.Flags().GetString("ipfs-api") if err != nil { - return cid.Undef, nil, nil, fmt.Errorf("getting ipfs api flag: %s", err) + return cid.Undef, nil, nil, nil, fmt.Errorf("getting ipfs api flag: %s", err) + } + aggregate, err := cmd.Flags().GetBool("aggregate") + if err != nil { + return cid.Undef, nil, nil, nil, fmt.Errorf("aggregate flag: %s", err) } - if ipfsAPI == "" { path := args[0] tmpDir, err := cmd.Flags().GetString("tmpdir") if err != nil { - return cid.Undef, nil, nil, fmt.Errorf("getting tmpdir directory: %s", err) + return cid.Undef, nil, nil, nil, fmt.Errorf("getting tmpdir directory: %s", err) } dagService, cls, err := createTmpDAGService(tmpDir) if err != nil { - return cid.Undef, nil, nil, fmt.Errorf("creating temporary dag-service: %s", err) + return cid.Undef, nil, nil, nil, fmt.Errorf("creating temporary dag-service: %s", err) } - dataCid, err := dagify(context.Background(), dagService, path, quiet) + dataCid, aggregatedFiles, err := dagify(context.Background(), dagService, path, quiet, aggregate) if err != nil { - return cid.Undef, nil, nil, fmt.Errorf("creating dag for data: %s", err) + return cid.Undef, nil, nil, nil, fmt.Errorf("creating dag for data: %s", err) } - return dataCid, dagService, cls, nil + return dataCid, dagService, aggregatedFiles, cls, nil } if len(args) == 0 { - return cid.Undef, nil, nil, fmt.Errorf("cid argument is empty") + return cid.Undef, nil, nil, nil, fmt.Errorf("cid argument is empty") } dataCid, err := cid.Decode(args[0]) if err != nil { - return cid.Undef, nil, nil, fmt.Errorf("parsing cid: %s", err) + return cid.Undef, nil, nil, nil, fmt.Errorf("parsing cid: %s", err) } ipfsAPIMA, err := multiaddr.NewMultiaddr(ipfsAPI) if err != nil { - return cid.Undef, nil, nil, fmt.Errorf("parsing ipfs-api multiaddress: %s", err) + return cid.Undef, nil, nil, nil, fmt.Errorf("parsing ipfs-api multiaddress: %s", err) } ipfs, err := httpapi.NewApi(ipfsAPIMA) if err != nil { - return cid.Undef, nil, nil, fmt.Errorf("creating ipfs client: %s", err) + return cid.Undef, nil, nil, nil, fmt.Errorf("creating ipfs client: %s", err) } - return dataCid, ipfs.Dag(), closeFunc(func() error { return nil }), nil + return dataCid, ipfs.Dag(), nil, closeFunc(func() error { return nil }), nil } func createTmpDAGService(tmpDir string) (ipld.DAGService, closeFunc, error) { @@ -333,30 +376,36 @@ func createTmpDAGService(tmpDir string) (ipld.DAGService, closeFunc, error) { var jsonOutput = io.Writer(os.Stderr) -func printJSONResult(pieceSize uint64, pieceCID cid.Cid) { +func printJSONResult(pieceSize uint64, payloadCid, pieceCID cid.Cid, aggrFiles []aggregatedFile) { outData := struct { - PieceSize uint64 `json:"piece_size"` - PieceCid string `json:"piece_cid"` + PayloadCid string `json:"payload_cid,omitempty"` + PieceSize uint64 `json:"piece_size"` + PieceCid string `json:"piece_cid"` + AggregatedFiles []aggregatedFile `json:"files,omitempty"` }{ - PieceSize: pieceSize, - PieceCid: pieceCID.String(), + PieceSize: pieceSize, + PieceCid: pieceCID.String(), + AggregatedFiles: aggrFiles, + } + if payloadCid.Defined() { + outData.PayloadCid = payloadCid.String() } out, err := json.Marshal(outData) c.CheckErr(err) fmt.Fprint(jsonOutput, string(out)) } -func dagify(ctx context.Context, dagService ipld.DAGService, path string, quiet bool) (cid.Cid, error) { +func dagify(ctx context.Context, dagService ipld.DAGService, path string, quiet bool, aggregate bool) (cid.Cid, []aggregatedFile, error) { var progressChan chan int64 if !quiet { f, err := os.Open(path) if err != nil { - return cid.Undef, fmt.Errorf("opening path: %s", err) + return cid.Undef, nil, fmt.Errorf("opening path: %s", err) } stat, err := f.Stat() if err != nil { _ = f.Close() - return cid.Undef, fmt.Errorf("getting stat of data: %s", err) + return cid.Undef, nil, fmt.Errorf("getting stat of data: %s", err) } _ = f.Close() @@ -375,7 +424,7 @@ func dagify(ctx context.Context, dagService ipld.DAGService, path string, quiet return err }) if err != nil { - return cid.Undef, fmt.Errorf("walking path: %s", err) + return cid.Undef, nil, fmt.Errorf("walking path: %s", err) } } bar := pb.StartNew(dataSize) @@ -395,10 +444,44 @@ func dagify(ctx context.Context, dagService ipld.DAGService, path string, quiet c.Message("DAG created in %.02fs.", time.Since(start).Seconds()) }() } - dataCid, err := dataprep.Dagify(ctx, dagService, path, progressChan) - if err != nil { - return cid.Undef, fmt.Errorf("creating dag for data: %s", err) + + var aggregatedFiles []aggregatedFile + var dataCid cid.Cid + var err error + if !aggregate { + dataCid, err = dataprep.Dagify(ctx, dagService, path, progressChan) + if err != nil { + return cid.Undef, nil, fmt.Errorf("creating dag for data: %s", err) + } + } else { + var lst []aggregator.AggregateDagEntry + err = filepath.Walk(path, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + dcid, err := dataprep.Dagify(ctx, dagService, p, progressChan) + if err != nil { + return err + } + lst = append(lst, aggregator.AggregateDagEntry{ + RootCid: dcid, + }) + aggregatedFiles = append(aggregatedFiles, aggregatedFile{Name: p, Cid: dcid.String()}) + return err + }) + dataCid, err = aggregator.Aggregate(ctx, dagService, lst) + if err != nil { + return cid.Undef, nil, fmt.Errorf("aggregating: %s", err) + } } - return dataCid, nil + return dataCid, aggregatedFiles, nil +} + +type aggregatedFile struct { + Name string `json:"name"` + Cid string `jsong:"cid"` } diff --git a/go.mod b/go.mod index 60454075..9d6fd383 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dustin/go-humanize v1.0.0 github.com/filecoin-project/go-address v0.0.5 + github.com/filecoin-project/go-dagaggregator-unixfs v0.1.0 github.com/filecoin-project/go-data-transfer v1.4.3 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a github.com/filecoin-project/go-fil-commp-hashhash v0.0.0-20210329070555-d36b89a2504c @@ -36,7 +37,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e github.com/ipfs/go-graphsync v0.7.0 // indirect github.com/ipfs/go-ipfs v0.8.0 - github.com/ipfs/go-ipfs-blockstore v1.0.3 + github.com/ipfs/go-ipfs-blockstore v1.0.4 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.8 github.com/ipfs/go-ipfs-http-client v0.1.0 @@ -44,6 +45,7 @@ require ( github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-log/v2 v2.1.3 github.com/ipfs/go-merkledag v0.3.2 + github.com/ipfs/go-unixfs v0.2.6 github.com/ipfs/interface-go-ipfs-core v0.4.0 github.com/ipld/go-car v0.2.1-0.20210322190947-cffd36d39d90 // indirect github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15 @@ -81,4 +83,6 @@ require ( nhooyr.io/websocket v1.8.6 // indirect ) +replace github.com/ipfs/go-unixfs => github.com/ipfs/go-unixfs v0.2.2 + replace github.com/dgraph-io/badger/v2 => github.com/dgraph-io/badger/v2 v2.2007.2 diff --git a/go.sum b/go.sum index 41064183..a0011321 100644 --- a/go.sum +++ b/go.sum @@ -62,7 +62,6 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= @@ -302,6 +301,8 @@ github.com/filecoin-project/go-commp-utils v0.1.0 h1:PaDxoXYh1TXnnz5kA/xSObpAQwc github.com/filecoin-project/go-commp-utils v0.1.0/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-dagaggregator-unixfs v0.1.0 h1:ZwAAj4n6uTNcVRmzibEBZUxJLs61cdpaxChBgEhURsg= +github.com/filecoin-project/go-dagaggregator-unixfs v0.1.0/go.mod h1:WTuJWgBQY0omnQqa8kRPT9O0Uj5wQOgslVMUuTeHdJ8= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= github.com/filecoin-project/go-data-transfer v1.4.3 h1:ECEw69NOfmEZ7XN1NSBvj3KTbbH2mIczQs+Z2w4bD7c= github.com/filecoin-project/go-data-transfer v1.4.3/go.mod h1:n8kbDQXWrY1c4UgfMa9KERxNCWbOTDwdNhf2MpN9dpo= @@ -715,8 +716,9 @@ github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2Is github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v1.0.0/go.mod h1:knLVdhVU9L7CC4T+T4nvGdeUIPAXlnd9zmXfp+9MIjU= github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE= -github.com/ipfs/go-ipfs-blockstore v1.0.3 h1:RDhK6fdg5YsonkpMuMpdvk/pRtOQlrIRIybuQfkvB2M= github.com/ipfs/go-ipfs-blockstore v1.0.3/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE= +github.com/ipfs/go-ipfs-blockstore v1.0.4 h1:DZdeya9Vu4ttvlGheQPGrj6kWehXnYZRFCp9EsZQ1hI= +github.com/ipfs/go-ipfs-blockstore v1.0.4/go.mod h1:uL7/gTJ8QIZ3MtA3dWf+s1a0U3fJy2fcEZAsovpRp+w= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= @@ -821,12 +823,8 @@ github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= -github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= -github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= -github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= -github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= -github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= -github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= +github.com/ipfs/go-unixfs v0.2.2 h1:eTkDT9F0dn4qHmBMVRMZbziwyqLRcogjtPYqMgZYmQs= +github.com/ipfs/go-unixfs v0.2.2/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/interface-go-ipfs-core v0.2.3/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= @@ -1430,6 +1428,8 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g github.com/multiformats/go-multibase v0.0.2/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multicodec v0.2.0 h1:MUzKZWxOFagwLLtlx96pub9zwDQAbMAf1k9fXOdc3so= +github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= From 3270a40c6b5ade0658bebc9e428c1b98b338baa1 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 2 Jul 2021 09:25:56 -0300 Subject: [PATCH 2/5] pow: check flag combinations and move manifest generation Signed-off-by: Ignacio Hagopian --- cmd/pow/cmd/prepare/prepare.go | 70 +++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/cmd/pow/cmd/prepare/prepare.go b/cmd/pow/cmd/prepare/prepare.go index 9384ff6c..6487d300 100644 --- a/cmd/pow/cmd/prepare/prepare.go +++ b/cmd/pow/cmd/prepare/prepare.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -188,6 +189,18 @@ The piece-size and piece-cid are printed to stderr. For scripting usage, its rec if err != nil { c.Fatal(fmt.Errorf("parsing json flag: %s", err)) } + aggregate, err := cmd.Flags().GetBool("aggregate") + if err != nil { + c.Fatal(fmt.Errorf("aggregate flag: %s", err)) + } + ipfsAPI, err := cmd.Flags().GetString("ipfs-api") + if err != nil { + c.Fatal(fmt.Errorf("get api addr flag: %s", err)) + } + if ipfsAPI != "" && aggregate { + c.Fatal(errors.New("the --aggregate flag can't be used with a remote go-ipfs node")) + } + payloadCid, dagService, aggrFiles, cls, err := prepareDAGService(cmd, args, json) if err != nil { c.Fatal(fmt.Errorf("creating temporal dag-service: %s", err)) @@ -253,42 +266,39 @@ The piece-size and piece-cid are printed to stderr. For scripting usage, its rec if errCommP != nil { c.Fatal(fmt.Errorf("calculating piece-size and PieceCID: %s", err)) } - if json { - printJSONResult(pieceSize, payloadCid, pieceCid, aggrFiles) - aggregate, err := cmd.Flags().GetBool("aggregate") + if aggregate { + rootNd, err := dagService.Get(ctx, payloadCid) if err != nil { - c.Fatal(fmt.Errorf("aggregate flag: %s", err)) + c.Fatal(fmt.Errorf("get root node: %s", err)) } - if aggregate { - rootNd, err := dagService.Get(ctx, payloadCid) - if err != nil { - c.Fatal(fmt.Errorf("get root node: %s", err)) - } - manifestLnk := rootNd.Links()[0] - manifestNd, err := dagService.Get(ctx, manifestLnk.Cid) - if err != nil { - c.Fatal(fmt.Errorf("get manfiest node: %s", err)) - } - manifestF, err := unixfile.NewUnixfsFile(context.Background(), dagService, manifestNd) - if err != nil { - c.Fatal(fmt.Errorf("get unifxfs manifest file: %s", err)) - } - manifest := manifestF.(files.File) - fManifest, err := os.Create(args[1] + ".manifest") - if err != nil { - c.Fatal(fmt.Errorf("creating manifest file: %s", err)) + manifestLnk := rootNd.Links()[0] + manifestNd, err := dagService.Get(ctx, manifestLnk.Cid) + if err != nil { + c.Fatal(fmt.Errorf("get manfiest node: %s", err)) + } + manifestF, err := unixfile.NewUnixfsFile(context.Background(), dagService, manifestNd) + if err != nil { + c.Fatal(fmt.Errorf("get unifxfs manifest file: %s", err)) + } + manifest := manifestF.(files.File) + fManifest, err := os.Create(args[1] + ".manifest") + if err != nil { + c.Fatal(fmt.Errorf("creating manifest file: %s", err)) + } + defer func() { + if err := fManifest.Close(); err != nil { + c.Fatal(fmt.Errorf("closing manifest file: %s", err)) } - defer func() { - if err := fManifest.Close(); err != nil { - c.Fatal(fmt.Errorf("closing manifest file: %s", err)) - } - }() - if _, err := io.Copy(fManifest, manifest); err != nil { - c.Fatal(fmt.Errorf("writing manifest file: %s", err)) - } + }() + if _, err := io.Copy(fManifest, manifest); err != nil { + c.Fatal(fmt.Errorf("writing manifest file: %s", err)) } + } + + if json { + printJSONResult(pieceSize, payloadCid, pieceCid, aggrFiles) return } c.Message("Created CAR file, and piece digest in %.02fs.", time.Since(start).Seconds()) From a131b6020b80950501d2e183e50f418186527000 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 2 Jul 2021 09:46:35 -0300 Subject: [PATCH 3/5] typo Signed-off-by: Ignacio Hagopian --- cmd/pow/cmd/prepare/prepare.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pow/cmd/prepare/prepare.go b/cmd/pow/cmd/prepare/prepare.go index 6487d300..ff2c2d8d 100644 --- a/cmd/pow/cmd/prepare/prepare.go +++ b/cmd/pow/cmd/prepare/prepare.go @@ -275,7 +275,7 @@ The piece-size and piece-cid are printed to stderr. For scripting usage, its rec manifestLnk := rootNd.Links()[0] manifestNd, err := dagService.Get(ctx, manifestLnk.Cid) if err != nil { - c.Fatal(fmt.Errorf("get manfiest node: %s", err)) + c.Fatal(fmt.Errorf("get manifest node: %s", err)) } manifestF, err := unixfile.NewUnixfsFile(context.Background(), dagService, manifestNd) if err != nil { From 729fc333577c678737f92e59a7aa300355a3673e Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 2 Jul 2021 09:46:57 -0300 Subject: [PATCH 4/5] make docs Signed-off-by: Ignacio Hagopian --- cli-docs/pow/pow_offline_prepare.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cli-docs/pow/pow_offline_prepare.md b/cli-docs/pow/pow_offline_prepare.md index 667da094..0ecc5f60 100644 --- a/cli-docs/pow/pow_offline_prepare.md +++ b/cli-docs/pow/pow_offline_prepare.md @@ -24,6 +24,7 @@ pow offline prepare [cid | path] [output CAR file path] [flags] ### Options ``` + --aggregate aggregates a folder of files -h, --help help for prepare --ipfs-api string IPFS HTTP API multiaddress that stores the cid (only for Cid processing instead of file/folder path) --json avoid pretty output and use json formatting From 71c599f46a68f267bd4a53cd50f7c6910e05e9c6 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 2 Jul 2021 09:54:33 -0300 Subject: [PATCH 5/5] lints and tests Signed-off-by: Ignacio Hagopian --- cmd/pow/cmd/prepare/prepare.go | 4 ++++ cmd/pow/cmd/prepare/prepare_test.go | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/pow/cmd/prepare/prepare.go b/cmd/pow/cmd/prepare/prepare.go index ff2c2d8d..30ee57c4 100644 --- a/cmd/pow/cmd/prepare/prepare.go +++ b/cmd/pow/cmd/prepare/prepare.go @@ -482,6 +482,10 @@ func dagify(ctx context.Context, dagService ipld.DAGService, path string, quiet aggregatedFiles = append(aggregatedFiles, aggregatedFile{Name: p, Cid: dcid.String()}) return err }) + if err != nil { + return cid.Undef, nil, fmt.Errorf("walking the target folder: %s", err) + } + dataCid, err = aggregator.Aggregate(ctx, dagService, lst) if err != nil { return cid.Undef, nil, fmt.Errorf("aggregating: %s", err) diff --git a/cmd/pow/cmd/prepare/prepare_test.go b/cmd/pow/cmd/prepare/prepare_test.go index c58d0c38..ec94a918 100644 --- a/cmd/pow/cmd/prepare/prepare_test.go +++ b/cmd/pow/cmd/prepare/prepare_test.go @@ -22,9 +22,9 @@ func TestOfflinePreparation(t *testing.T) { size int json string }{ - {size: 10000, json: `{"piece_size":16384,"piece_cid":"baga6ea4seaqjuk4uh5g7cu5znbvrr7wvfsn2l3xj47rbymvi63uiiroya44lkiy"}`}, - {size: 1000, json: `{"piece_size":2048,"piece_cid":"baga6ea4seaqadahcx4ct54tlbvgkqlhmif7kxxkvxz3yf3vr2e4puhvsxdbrgka"}`}, - {size: 100, json: `{"piece_size":256,"piece_cid":"baga6ea4seaqd4hgfl6texpf377k7igx2ga2mfwn3lb4c4kdpaq3g3oao2yftuki"}`}, + {size: 10000, json: `{"payload_cid":"QmRP8TCp9bthhzLACAao6mh8cfLypqXncdNbzuPtuqFYP7","piece_size":16384,"piece_cid":"baga6ea4seaqjuk4uh5g7cu5znbvrr7wvfsn2l3xj47rbymvi63uiiroya44lkiy"}`}, + {size: 1000, json: `{"payload_cid":"QmQRAjpSLWziADGz8p5ezxTguFNn18yYbSnduKqMrbJ93c","piece_size":2048,"piece_cid":"baga6ea4seaqadahcx4ct54tlbvgkqlhmif7kxxkvxz3yf3vr2e4puhvsxdbrgka"}`}, + {size: 100, json: `{"payload_cid":"QmY6zHswPvyZkAyxwM9uup1DDPb67hZqChv8hnyu4MrFWk","piece_size":256,"piece_cid":"baga6ea4seaqd4hgfl6texpf377k7igx2ga2mfwn3lb4c4kdpaq3g3oao2yftuki"}`}, } for _, test := range testCases {