GC & multitenancy (#712)

* rebase & squash Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * trackstore progress Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * more progress Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * extensive trackstore tests Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * fixes Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * revert ci change Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * lints Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * fix docs Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * start migrations Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * progress Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * migration progress Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * storageinfo migr Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * progress Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * work Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * migration tests and changes Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * pinstore migration tests Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * lints Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * punctuation Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * change Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * rename Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * rename again Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * forward only Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * lint Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * fix admin ctx Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * allow for non-transactional migrations & real migration test & fixes Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * resume deal fixes & migration Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * improve comment Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com> * cover the stage folder usecase Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>
textileio · Dec 2, 2020 · 458e9c0 · 458e9c0
1 parent 2e43f9b
commit 458e9c0
Show file tree

Hide file tree

Showing 86 changed files with 5,633 additions and 1,677 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,7 @@
 
 # Test binary, built with `go test -c`
 *.test
+cover.out
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out

diff --git a/Dockerfile b/Dockerfile
@@ -4,7 +4,7 @@ RUN mkdir /app
 WORKDIR /app 
 
 COPY go.mod go.sum ./
-RUN go mod download
+RUN go mod download -x
 COPY . .
 RUN POW_BUILD_FLAGS="CGO_ENABLED=0 GOOS=linux" make build-powd
 RUN POW_BUILD_FLAGS="CGO_ENABLED=0 GOOS=linux" make build-pow

diff --git a/api/client/admin/admin.go b/api/client/admin/admin.go
@@ -9,6 +9,7 @@ type Admin struct {
 	StorageJobs *StorageJobs
 	Users       *Users
 	Wallet      *Wallet
+	Data        *Data
 }
 
 // NewAdmin creates a new admin API.
@@ -17,5 +18,6 @@ func NewAdmin(client adminPb.AdminServiceClient) *Admin {
 		StorageJobs: &StorageJobs{client: client},
 		Users:       &Users{client: client},
 		Wallet:      &Wallet{client: client},
+		Data:        &Data{client: client},
 	}
 }
diff --git a/api/client/admin/hs.go b/api/client/admin/hs.go
@@ -0,0 +1,22 @@
+package admin
+
+import (
+	"context"
+
+	proto "github.com/textileio/powergate/api/gen/powergate/admin/v1"
+)
+
+// Data provides access to Powergate data admin APIs.
+type Data struct {
+	client proto.AdminServiceClient
+}
+
+// GCStaged unpins staged data not related to queued or executing jobs.
+func (w *Data) GCStaged(ctx context.Context) (*proto.GCStagedResponse, error) {
+	return w.client.GCStaged(ctx, &proto.GCStagedRequest{})
+}
+
+// PinnedCids returns pinned cids information of hot-storage.
+func (w *Data) PinnedCids(ctx context.Context) (*proto.PinnedCidsResponse, error) {
+	return w.client.PinnedCids(ctx, &proto.PinnedCidsRequest{})
+}
diff --git a/api/client/data.go b/api/client/data.go
@@ -97,13 +97,18 @@ func (d *Data) StageFolder(ctx context.Context, ipfsRevProxyAddr string, folderP
 	defer func() { _ = ff.Close() }()
 	opts := []options.UnixfsAddOption{
 		options.Unixfs.CidVersion(1),
-		options.Unixfs.Pin(false),
+		options.Unixfs.Pin(true),
 	}
 	pth, err := ipfs.Unixfs().Add(context.Background(), files.ToDir(ff), opts...)
 	if err != nil {
 		return "", err
 	}
 
+	_, err = d.client.StageCid(ctx, &userPb.StageCidRequest{Cid: pth.Cid().String()})
+	if err != nil {
+		return "", fmt.Errorf("stage pinning cid: %s", err)
+	}
+
 	return pth.Cid().String(), nil
 }
 

diff --git a/api/client/utils_test.go b/api/client/utils_test.go
@@ -2,7 +2,6 @@ package client
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"testing"
 	"time"
@@ -25,9 +24,7 @@ func defaultServerConfig(t *testing.T) server.Config {
 	gatewayHostAddr := fmt.Sprintf("0.0.0.0:%d", freePort(t))
 	indexRawJSONHostAddr := fmt.Sprintf("0.0.0.0:%d", freePort(t))
 
-	repoPath, err := ioutil.TempDir("/tmp/powergate", ".powergate-*")
-	require.NoError(t, err)
-
+	repoPath := t.TempDir()
 	dipfs, cls := tests.LaunchIPFSDocker(t)
 	t.Cleanup(func() { cls() })
 
@@ -56,6 +53,7 @@ func defaultServerConfig(t *testing.T) server.Config {
 		MinerSelector:               "reputation",
 		FFSDealFinalityTimeout:      time.Minute * 30,
 		FFSMaxParallelDealPreparing: 1,
+		FFSGCAutomaticGCInterval:    0,
 		DealWatchPollDuration:       time.Second * 15,
 		SchedMaxParallel:            10,
 		AskIndexQueryAskTimeout:     time.Second * 3,