Merge pull request #6280 from filecoin-project/feat/pieceread-outside…

…-scheduler Bypass task scheduler for reading unsealed pieces
filecoin-project · Jun 7, 2021 · fadc79a · fadc79a
2 parents 0117584 + 8625da3
commit fadc79a
Show file tree

Hide file tree

Showing 44 changed files with 3,139 additions and 1,135 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -331,7 +331,7 @@ jobs:
       - run: cd extern/filecoin-ffi && make
       - run:
           name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../..
+          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy
       - run:
           name: "build lotus-soup testplan"
           command: pushd testplans/lotus-soup && go build -tags=testground .

diff --git a/api/api_worker.go b/api/api_worker.go
@@ -2,7 +2,6 @@ package api
 
 import (
 	"context"
-	"io"
 
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
@@ -43,7 +42,6 @@ type Worker interface {
 	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)                                                                                 //perm:admin
 	MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 //perm:admin
 	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
-	ReadPiece(context.Context, io.Writer, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (storiface.CallID, error)                                                               //perm:admin
 	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin
 
 	TaskDisable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin

diff --git a/api/mocks/mock_full.go b/api/mocks/mock_full.go
diff --git a/api/proxy_gen.go b/api/proxy_gen.go
diff --git a/api/test/deals.go b/api/test/deals.go
@@ -441,7 +441,7 @@ func TestOfflineDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, st
 	require.Eventually(t, func() bool {
 		cd, _ := s.client.ClientGetDealInfo(s.ctx, *proposalCid)
 		return cd.State == storagemarket.StorageDealCheckForAcceptance
-	}, 1*time.Second, 100*time.Millisecond, "actual deal status is %s", storagemarket.DealStates[cd.State])
+	}, 30*time.Second, 1*time.Second, "actual deal status is %s", storagemarket.DealStates[cd.State])
 
 	// Create a CAR file from the raw file
 	carFileDir, err := ioutil.TempDir(os.TempDir(), "test-make-deal-car")

diff --git a/api/v0api/v0mocks/mock_full.go b/api/v0api/v0mocks/mock_full.go
diff --git a/api/version.go b/api/version.go
@@ -57,8 +57,8 @@ var (
 	FullAPIVersion0 = newVer(1, 3, 0)
 	FullAPIVersion1 = newVer(2, 1, 0)
 
-	MinerAPIVersion0  = newVer(1, 0, 1)
-	WorkerAPIVersion0 = newVer(1, 0, 0)
+	MinerAPIVersion0  = newVer(1, 1, 0)
+	WorkerAPIVersion0 = newVer(1, 1, 0)
 )
 
 //nolint:varcheck,deadcode

diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz
diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz
diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz
diff --git a/cli/servicesmock_test.go b/cli/servicesmock_test.go
diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go
@@ -362,9 +362,10 @@ var runCmd = &cli.Command{
 			return xerrors.Errorf("could not get api info: %w", err)
 		}
 
-		remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"))
+		remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"),
+			&stores.DefaultPartialFileHandler{})
 
-		fh := &stores.FetchHandler{Local: localStore}
+		fh := &stores.FetchHandler{Local: localStore, PfHandler: &stores.DefaultPartialFileHandler{}}
 		remoteHandler := func(w http.ResponseWriter, r *http.Request) {
 			if !auth.HasPerm(r.Context(), nil, api.PermAdmin) {
 				w.WriteHeader(401)

diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go
@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -453,14 +454,23 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 			wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix))
 			smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix))
 
-			smgr, err := sectorstorage.New(ctx, lr, stores.NewIndex(), sectorstorage.SealerConfig{
+			si := stores.NewIndex()
+
+			lstor, err := stores.NewLocal(ctx, lr, si, nil)
+			if err != nil {
+				return err
+			}
+			stor := stores.NewRemote(lstor, si, http.Header(sa), 10, &stores.DefaultPartialFileHandler{})
+
+			smgr, err := sectorstorage.New(ctx, lstor, stor, lr, si, sectorstorage.SealerConfig{
 				ParallelFetchLimit: 10,
 				AllowAddPiece:      true,
 				AllowPreCommit1:    true,
 				AllowPreCommit2:    true,
 				AllowCommit:        true,
 				AllowUnseal:        true,
-			}, nil, sa, wsts, smsts)
+			}, wsts, smsts)
+
 			if err != nil {
 				return err
 			}

diff --git a/documentation/en/api-v0-methods-worker.md b/documentation/en/api-v0-methods-worker.md
@@ -15,8 +15,6 @@
   * [MoveStorage](#MoveStorage)
 * [Process](#Process)
   * [ProcessSession](#ProcessSession)
-* [Read](#Read)
-  * [ReadPiece](#ReadPiece)
 * [Release](#Release)
   * [ReleaseUnsealed](#ReleaseUnsealed)
 * [Seal](#Seal)
@@ -263,41 +261,6 @@ Inputs: `null`
 
 Response: `"07070707-0707-0707-0707-070707070707"`
 
-## Read
-
-
-### ReadPiece
-
-
-Perms: admin
-
-Inputs:
-```json
-[
-  {},
-  {
-    "ID": {
-      "Miner": 1000,
-      "Number": 9
-    },
-    "ProofType": 8
-  },
-  1040384,
-  1024
-]
-```
-
-Response:
-```json
-{
-  "Sector": {
-    "Miner": 1000,
-    "Number": 9
-  },
-  "ID": "07070707-0707-0707-0707-070707070707"
-}
-```
-
 ## Release