diff --git a/go.mod b/go.mod index 2bc86eba..4e4d1277 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,13 @@ require ( github.com/filecoin-project/dagstore v0.2.2-0.20210722035743-7ddd92f518b4 github.com/filecoin-project/go-address v0.0.5 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 - github.com/filecoin-project/go-commp-utils v0.1.0 + github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 github.com/filecoin-project/go-data-transfer v1.7.0 github.com/filecoin-project/go-ds-versioning v0.1.0 + github.com/filecoin-project/go-fil-commcid v0.1.0 + github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 github.com/filecoin-project/go-multistore v0.0.3 - github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 + github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.1 diff --git a/go.sum b/go.sum index ed52276b..c8944830 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7 github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= -github.com/filecoin-project/go-commp-utils v0.1.0 h1:PaDxoXYh1TXnnz5kA/xSObpAQwcJSUs4Szb72nuaNdk= -github.com/filecoin-project/go-commp-utils v0.1.0/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= +github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 h1:U9Z+76pHCKBmtdxFV7JFZJj7OVm12I6dEKwtMVbq5p0= +github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= @@ -134,8 +134,11 @@ github.com/filecoin-project/go-data-transfer v1.7.0/go.mod h1:GLRr5BmLEqsLwXfiRD github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88OqLYEo6roi+GiIeOh8= +github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -143,8 +146,9 @@ github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxl github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= -github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 h1:+/4aUeUoKr6AKfPE3mBhXA5spIV6UcKdTYDPNU2Tdmg= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak= +github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 h1:0BogtftbcgyBx4lP2JWM00ZK7/pXmgnrDqKp9aLTgVs= +github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= @@ -844,8 +848,8 @@ github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/c-for-go v0.0.0-20201112171043-ea6dce5809cb h1:/7/dQyiKnxAOj9L69FhST7uMe17U015XPzX7cy+5ykM= -github.com/xlab/c-for-go v0.0.0-20201112171043-ea6dce5809cb/go.mod h1:pbNsDSxn1ICiNn9Ct4ZGNrwzfkkwYbx/lw8VuyutFIg= +github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 h1:wb7xrDzfkLgPHsSEBm+VSx6aDdi64VtV0xvP0E6j8bk= +github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I= github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 h1:Sw125DKxZhPUI4JLlWugkzsrlB50jR9v2khiD9FxuSo= github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk= github.com/xorcare/golden v0.6.0 h1:E8emU8bhyMIEpYmgekkTUaw4vtcrRE+Wa0c5wYIcgXc= @@ -914,7 +918,6 @@ golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1099,7 +1102,6 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201112185108-eeaa07dd7696/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -1198,12 +1200,8 @@ honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/golex v1.0.1 h1:EYKY1a3wStt0RzHaH8mdSRNg78Ub0OHxYfCRWw35YtM= -modernc.org/golex v1.0.1/go.mod h1:QCA53QtsT1NdGkaZZkF5ezFwk4IXh4BGNafAARTC254= -modernc.org/lex v1.0.0/go.mod h1:G6rxMTy3cH2iA0iXL/HRRv4Znu8MK4higxph/lE7ypk= -modernc.org/lexer v1.0.0/go.mod h1:F/Dld0YKYdZCLQ7bD0USbWL4YKCyTDRDHiDTOs0q0vk= modernc.org/mathutil v1.1.1 h1:FeylZSVX8S+58VsyJlkEj2bcpdytmp9MmDKZkKx8OIE= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc= diff --git a/retrievalmarket/impl/integration_test.go b/retrievalmarket/impl/integration_test.go index dbfffe13..3e275582 100644 --- a/retrievalmarket/impl/integration_test.go +++ b/retrievalmarket/impl/integration_test.go @@ -440,7 +440,7 @@ func TestClientCanMakeDealWithProvider(t *testing.T) { } // ------- SET UP CLIENT - ctx, cancel := context.WithTimeout(bgCtx, 10*time.Second) + ctx, cancel := context.WithTimeout(bgCtx, 60*time.Second) defer cancel() provider := setupProvider(bgCtx, t, testData, payloadCID, pieceInfo, carFile.Name(), expectedQR, @@ -561,11 +561,12 @@ CurrentInterval: %d } assert.Equal(t, retrievalmarket.DealStatusCompleted, clientDealState.Status) } - ctx, cancel = context.WithTimeout(bgCtx, 5*time.Second) - defer cancel() + + ctxProv, cancelProv := context.WithTimeout(bgCtx, 10*time.Second) + defer cancelProv() var providerDealState retrievalmarket.ProviderDealState select { - case <-ctx.Done(): + case <-ctxProv.Done(): t.Error("provider never saw completed deal") t.FailNow() case providerDealState = <-providerDealStateChan: diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index 7793a631..732341d2 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -17,6 +17,8 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer" versioning "github.com/filecoin-project/go-ds-versioning/pkg" versionedfsm "github.com/filecoin-project/go-ds-versioning/pkg/fsm" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" @@ -313,7 +315,7 @@ func (p *Provider) ImportDataForDeal(ctx context.Context, propCid cid.Cid, data _ = n // TODO: verify n? - pieceSize := uint64(tempfi.Size()) + carSize := uint64(tempfi.Size()) _, err = tempfi.Seek(0, io.SeekStart) if err != nil { @@ -327,16 +329,30 @@ func (p *Provider) ImportDataForDeal(ctx context.Context, propCid cid.Cid, data return xerrors.Errorf("failed to determine proof type: %w", err) } - pieceCid, err := generatePieceCommitment(proofType, tempfi, pieceSize) + pieceCid, err := generatePieceCommitment(proofType, tempfi, carSize) if err != nil { cleanup() return xerrors.Errorf("failed to generate commP: %w", err) } + if carSizePadded := padreader.PaddedSize(carSize).Padded(); carSizePadded < d.Proposal.PieceSize { + // need to pad up! + rawPaddedCommp, err := commp.PadCommP( + // we know how long a pieceCid "hash" is, just blindly extract the trailing 32 bytes + pieceCid.Hash()[len(pieceCid.Hash())-32:], + uint64(carSizePadded), + uint64(d.Proposal.PieceSize), + ) + if err != nil { + return err + } + pieceCid, _ = commcid.DataCommitmentV1ToCID(rawPaddedCommp) + } + // Verify CommP matches if !pieceCid.Equals(d.Proposal.PieceCID) { cleanup() - return xerrors.Errorf("given data does not match expected commP (got: %x, expected %x)", pieceCid, d.Proposal.PieceCID) + return xerrors.Errorf("given data does not match expected commP (got: %s, expected %s)", pieceCid, d.Proposal.PieceCID) } return p.deals.Send(propCid, storagemarket.ProviderEventVerifiedData, tempfi.Path(), filestore.Path("")) diff --git a/storagemarket/impl/provider_environments.go b/storagemarket/impl/provider_environments.go index af46f216..b69c14bf 100644 --- a/storagemarket/impl/provider_environments.go +++ b/storagemarket/impl/provider_environments.go @@ -13,6 +13,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-commp-utils/writer" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/piecestore" @@ -79,7 +82,7 @@ func (p *providerDealEnvironment) CleanReadWriteBlockstore(proposalCid cid.Cid, } // GeneratePieceCommitment generates the pieceCid for the CARv1 deal payload in the CARv2 file that already exists at the given path. -func (p *providerDealEnvironment) GeneratePieceCommitment(proposalCid cid.Cid, carV2FilePath string) (c cid.Cid, path filestore.Path, finalErr error) { +func (p *providerDealEnvironment) GeneratePieceCommitment(proposalCid cid.Cid, carV2FilePath string, dealSize abi.PaddedPieceSize) (c cid.Cid, path filestore.Path, finalErr error) { rd, err := carv2.OpenReader(carV2FilePath) if err != nil { return cid.Undef, "", xerrors.Errorf("failed to get CARv2 reader, proposalCid=%s, carV2FilePath=%s: %w", proposalCid, carV2FilePath, err) @@ -119,6 +122,20 @@ func (p *providerDealEnvironment) GeneratePieceCommitment(proposalCid cid.Cid, c return cid.Undef, "", xerrors.Errorf("failed to get CommP: %w", err) } + if cidAndSize.PieceSize < dealSize { + // need to pad up! + rawPaddedCommp, err := commp.PadCommP( + // we know how long a pieceCid "hash" is, just blindly extract the trailing 32 bytes + cidAndSize.PieceCID.Hash()[len(cidAndSize.PieceCID.Hash())-32:], + uint64(cidAndSize.PieceSize), + uint64(dealSize), + ) + if err != nil { + return cid.Undef, "", err + } + cidAndSize.PieceCID, _ = commcid.DataCommitmentV1ToCID(rawPaddedCommp) + } + return cidAndSize.PieceCID, filestore.Path(""), err } diff --git a/storagemarket/impl/provider_environments_test.go b/storagemarket/impl/provider_environments_test.go index b83a5137..7f107fbc 100644 --- a/storagemarket/impl/provider_environments_test.go +++ b/storagemarket/impl/provider_environments_test.go @@ -9,19 +9,22 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/go-state-types/abi" ) func TestGeneratePieceCommitment(t *testing.T) { + // both payload.txt and payload2.txt are about 18kb long + pieceSize := abi.PaddedPieceSize(32768) _, carV2File1 := shared_testutil.GenFullCARv2FromNormalFile(t, filepath.Join("storagemarket", "fixtures", "payload.txt")) defer os.Remove(carV2File1) _, carV2File2 := shared_testutil.GenFullCARv2FromNormalFile(t, filepath.Join("storagemarket", "fixtures", "payload2.txt")) defer os.Remove(carV2File2) - commP1 := genProviderCommP(t, carV2File1) - commP2 := genProviderCommP(t, carV2File2) + commP1 := genProviderCommP(t, carV2File1, pieceSize) + commP2 := genProviderCommP(t, carV2File2, pieceSize) - commP3 := genProviderCommP(t, carV2File1) - commP4 := genProviderCommP(t, carV2File2) + commP3 := genProviderCommP(t, carV2File1, pieceSize) + commP4 := genProviderCommP(t, carV2File2, pieceSize) require.Equal(t, commP1, commP3) require.Equal(t, commP2, commP4) @@ -31,15 +34,15 @@ func TestGeneratePieceCommitment(t *testing.T) { // fails when CARv2 file path isn't a valid one. env := &providerDealEnvironment{} - pieceCid, _, err := env.GeneratePieceCommitment(cid.Cid{}, "randpath") + pieceCid, _, err := env.GeneratePieceCommitment(cid.Cid{}, "randpath", pieceSize) require.Error(t, err) require.Contains(t, err.Error(), "no such file or directory") require.Equal(t, cid.Undef, pieceCid) } -func genProviderCommP(t *testing.T, carv2 string) cid.Cid { +func genProviderCommP(t *testing.T, carv2 string, pieceSize abi.PaddedPieceSize) cid.Cid { env := &providerDealEnvironment{} - pieceCid, _, err := env.GeneratePieceCommitment(cid.Cid{}, carv2) + pieceCid, _, err := env.GeneratePieceCommitment(cid.Cid{}, carv2, pieceSize) require.NoError(t, err) require.NotEqual(t, pieceCid, cid.Undef) return pieceCid diff --git a/storagemarket/impl/providerstates/provider_states.go b/storagemarket/impl/providerstates/provider_states.go index 389fefbe..647e8626 100644 --- a/storagemarket/impl/providerstates/provider_states.go +++ b/storagemarket/impl/providerstates/provider_states.go @@ -42,7 +42,7 @@ type ProviderDealEnvironment interface { CleanReadWriteBlockstore(proposalCid cid.Cid, carFilePath string) error - GeneratePieceCommitment(proposalCid cid.Cid, carV2FilePath string) (cid.Cid, filestore.Path, error) + GeneratePieceCommitment(proposalCid cid.Cid, carV2FilePath string, dealSize abi.PaddedPieceSize) (cid.Cid, filestore.Path, error) Address() address.Address Node() storagemarket.StorageProviderNode @@ -207,7 +207,7 @@ func VerifyData(ctx fsm.Context, environment ProviderDealEnvironment, deal stora return ctx.Trigger(storagemarket.ProviderEventDataVerificationFailed, xerrors.Errorf("failed to finalize read-write blockstore: %w", err), filestore.Path(""), filestore.Path("")) } - pieceCid, metadataPath, err := environment.GeneratePieceCommitment(deal.ProposalCid, deal.CARv2FilePath) + pieceCid, metadataPath, err := environment.GeneratePieceCommitment(deal.ProposalCid, deal.CARv2FilePath, deal.Proposal.PieceSize) if err != nil { return ctx.Trigger(storagemarket.ProviderEventDataVerificationFailed, xerrors.Errorf("error generating CommP: %w", err), filestore.Path(""), filestore.Path("")) } @@ -355,14 +355,16 @@ func HandoffDeal(ctx fsm.Context, environment ProviderDealEnvironment, deal stor return ctx.Trigger(storagemarket.ProviderEventDealHandedOff) } -func handoffDeal(ctx context.Context, environment ProviderDealEnvironment, deal storagemarket.MinerDeal, reader io.Reader, size uint64) (*storagemarket.PackingResult, error) { - // Note that even though padreader.New returns an UnpaddedPieceSize, it is - // *actually* returning the padded piece size cast to UnpaddedPieceSize. - paddedReader, paddedSize := padreader.New(reader, size) +func handoffDeal(ctx context.Context, environment ProviderDealEnvironment, deal storagemarket.MinerDeal, reader io.Reader, payloadSize uint64) (*storagemarket.PackingResult, error) { + // because we use the PadReader directly during AP we need to produce the + // correct amount of zeroes + // (alternative would be to keep precise track of sector offsets for each + // piece which is just too much work for a seldom used feature) + paddedReader, err := padreader.NewInflator(reader, payloadSize, deal.Proposal.PieceSize.Unpadded()) + if err != nil { + return nil, err + } - // Note that even though OnDealComplete takes an UnpaddedPieceSize as a - // parameter, the sealing code *actually* requires a padded piece size, - // which is what we pass here. return environment.Node().OnDealComplete( ctx, storagemarket.MinerDeal{ @@ -375,7 +377,7 @@ func handoffDeal(ctx context.Context, environment ProviderDealEnvironment, deal DealID: deal.DealID, FastRetrieval: deal.FastRetrieval, }, - paddedSize, + deal.Proposal.PieceSize.Unpadded(), paddedReader, ) } diff --git a/storagemarket/impl/providerstates/provider_states_test.go b/storagemarket/impl/providerstates/provider_states_test.go index 9388b3ab..a4a53c10 100644 --- a/storagemarket/impl/providerstates/provider_states_test.go +++ b/storagemarket/impl/providerstates/provider_states_test.go @@ -1438,7 +1438,7 @@ func makeExecutor(ctx context.Context, environment.ask = defaultAsk } if environment.pieceSize == 0 { - environment.pieceSize = uint64(defaultPieceSize) + environment.pieceSize = uint64(defaultPieceSize.Unpadded()) } if environment.pieceReader == nil { environment.pieceReader = newStubbedReadCloser(nil) @@ -1551,7 +1551,7 @@ func (fe *fakeEnvironment) CleanReadWriteBlockstore(proposalCid cid.Cid, carFile return nil } -func (fe *fakeEnvironment) GeneratePieceCommitment(proposalCid cid.Cid, carV2FilePath string) (cid.Cid, filestore.Path, error) { +func (fe *fakeEnvironment) GeneratePieceCommitment(proposalCid cid.Cid, carV2FilePath string, dealSize abi.PaddedPieceSize) (cid.Cid, filestore.Path, error) { return fe.pieceCid, fe.metadataPath, fe.generateCommPError } diff --git a/storagemarket/impl/providerutils/providerutils.go b/storagemarket/impl/providerutils/providerutils.go index 433666e3..902f419b 100644 --- a/storagemarket/impl/providerutils/providerutils.go +++ b/storagemarket/impl/providerutils/providerutils.go @@ -85,19 +85,19 @@ func GeneratePieceCommitmentWithMetadata( proofType abi.RegisteredSealProof, payloadCid cid.Cid, selector ipld.Node, - storeID *multistore.StoreID) (cid.Cid, filestore.Path, error) { + storeID *multistore.StoreID) (cid.Cid, abi.UnpaddedPieceSize, filestore.Path, error) { metadataFile, err := fileStore.CreateTemp() if err != nil { - return cid.Cid{}, "", err + return cid.Cid{}, 0, "", err } blockRecorder := blockrecorder.RecordEachBlockTo(metadataFile) - pieceCid, _, err := commPGenerator(proofType, payloadCid, selector, storeID, blockRecorder) + pieceCid, psize, err := commPGenerator(proofType, payloadCid, selector, storeID, blockRecorder) _ = metadataFile.Close() if err != nil { _ = fileStore.Delete(metadataFile.Path()) - return cid.Cid{}, "", err + return cid.Cid{}, 0, "", err } - return pieceCid, metadataFile.Path(), err + return pieceCid, psize, metadataFile.Path(), err } // LoadBlockLocations loads a metadata file then converts it to a map of cid -> blockLocation diff --git a/storagemarket/impl/providerutils/providerutils_test.go b/storagemarket/impl/providerutils/providerutils_test.go index bcfadf4d..1f30516f 100644 --- a/storagemarket/impl/providerutils/providerutils_test.go +++ b/storagemarket/impl/providerutils/providerutils_test.go @@ -161,7 +161,7 @@ func TestCommPGenerationWithMetadata(t *testing.T) { t.Run(testName, func(t *testing.T) { fcp := &fakeCommPGenerator{pieceCid, pieceSize, testCase.commPErr} fs := shared_testutil.NewTestFileStore(testCase.fileStoreParams) - resultPieceCid, resultMetadataPath, resultErr := providerutils.GeneratePieceCommitmentWithMetadata( + resultPieceCid, _, resultMetadataPath, resultErr := providerutils.GeneratePieceCommitmentWithMetadata( fs, fcp.GenerateCommPToFile, proofType, payloadCid, selector, &storeID) require.Equal(t, resultPieceCid, testCase.expectedPieceCid) require.Equal(t, resultMetadataPath, testCase.expectedMetadataPath) diff --git a/storagemarket/impl/requestvalidation/common.go b/storagemarket/impl/requestvalidation/common.go index 41a41aa5..cc5f010c 100644 --- a/storagemarket/impl/requestvalidation/common.go +++ b/storagemarket/impl/requestvalidation/common.go @@ -15,7 +15,6 @@ import ( // Will succeed only if: // - voucher has correct type // - voucher references an active deal -// - referenced deal matches the client // - referenced deal matches the given base CID // - referenced deal is in an acceptable state func ValidatePush( @@ -34,9 +33,6 @@ func ValidatePush( if err != nil { return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) } - if deal.Client != sender { - return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer) - } if !deal.Ref.Root.Equals(baseCid) { return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", deal.Proposal.PieceCID.String(), baseCid.String(), ErrWrongPiece) @@ -53,7 +49,6 @@ func ValidatePush( // Will succeed only if: // - voucher has correct type // - voucher references an active deal -// - referenced deal matches the receiver (miner) // - referenced deal matches the given base CID // - referenced deal is in an acceptable state func ValidatePull( @@ -71,9 +66,6 @@ func ValidatePull( return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) } - if deal.Miner != receiver { - return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer) - } if !deal.DataRef.Root.Equals(baseCid) { return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", deal.Proposal.PieceCID.String(), baseCid.String(), ErrWrongPiece) } diff --git a/storagemarket/impl/requestvalidation/request_validation_test.go b/storagemarket/impl/requestvalidation/request_validation_test.go index 85428f89..3de91720 100644 --- a/storagemarket/impl/requestvalidation/request_validation_test.go +++ b/storagemarket/impl/requestvalidation/request_validation_test.go @@ -176,21 +176,6 @@ func AssertPushValidator(t *testing.T, validator datatransfer.RequestValidator, t.Fatal("Push should fail if there is no deal stored") } }) - t.Run("ValidatePush fails wrong miner", func(t *testing.T) { - otherClient := peer.ID("otherclient") - minerDeal, err := newMinerDeal(otherClient, storagemarket.StorageDealProposalAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - ref := minerDeal.Ref - _, err = validator.ValidatePush(false, datatransfer.ChannelID{}, sender, &rv.StorageDataTransferVoucher{minerDeal.ProposalCid}, ref.Root, nil) - if !xerrors.Is(err, rv.ErrWrongPeer) { - t.Fatal("Push should fail if miner address is incorrect") - } - }) t.Run("ValidatePush fails wrong piece ref", func(t *testing.T) { minerDeal, err := newMinerDeal(sender, storagemarket.StorageDealProposalAccepted) if err != nil { @@ -249,21 +234,6 @@ func AssertValidatesPulls(t *testing.T, validator datatransfer.RequestValidator, t.Fatal("Pull should fail if there is no deal stored") } }) - t.Run("ValidatePull fails wrong client", func(t *testing.T) { - otherMiner := peer.ID("otherminer") - clientDeal, err := newClientDeal(otherMiner, storagemarket.StorageDealProposalAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - payloadCid := clientDeal.DataRef.Root - _, err = validator.ValidatePull(false, datatransfer.ChannelID{}, receiver, &rv.StorageDataTransferVoucher{clientDeal.ProposalCid}, payloadCid, nil) - if !xerrors.Is(err, rv.ErrWrongPeer) { - t.Fatal("Pull should fail if miner address is incorrect") - } - }) t.Run("ValidatePull fails wrong piece ref", func(t *testing.T) { clientDeal, err := newClientDeal(receiver, storagemarket.StorageDealProposalAccepted) if err != nil {