swarm/storage: Filehasher prototypes

cmd/swarm/list.go

@@ -37,6 +37,7 @@ var listCommand = cli.Command{
 }
 
 func list(ctx *cli.Context) {
+	fmt.Println("foo\n", ctx.GlobalString("password"))
 	args := ctx.Args()
 
 	if len(args) < 1 {

swarm/bmt/bmt_test.go

@@ -23,14 +23,23 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
+	"os"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/log"
 )
 
+func init() {
+	hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true))
+	hf := log.LvlFilterHandler(1, hs)
+	h := log.CallerFileHandler(hf)
+	log.Root().SetHandler(h)
+}
+
 // the actual data length generated (could be longer than max datalength of the BMT)
 const BufferSize = 4128
 

swarm/storage/chunker.go

@@ -298,7 +298,6 @@ func (tc *TreeChunker) split(ctx context.Context, depth int, treeSize int64, add
 	// dept > 0
 	// intermediate chunk containing child nodes hashes
 	branchCnt := (size + treeSize - 1) / treeSize
-
 	var chunk = make([]byte, branchCnt*tc.hashSize+8)
 	var pos, i int64
 
@@ -336,6 +335,8 @@ func (tc *TreeChunker) split(ctx context.Context, depth int, treeSize int64, add
 	case tc.jobC <- &hashJob{addr, chunk, size, parentWg}:
 	case <-tc.quitC:
 	}
+
+	log.Trace("chunkdata", "d", chunk)
 }
 
 func (tc *TreeChunker) runWorker(ctx context.Context) {

swarm/storage/chunker_test.go

@@ -70,6 +70,41 @@ func testRandomBrokenData(n int, tester *chunkerTester) {
 	tester.t.Logf(" Address = %v\n", key)
 }
 
+func testSerialData(usePyramid bool, hash string, n int, tester *chunkerTester) Address {
+	if tester.inputs == nil {
+		tester.inputs = make(map[uint64][]byte)
+	}
+	input, found := tester.inputs[uint64(n)]
+	var data io.Reader
+	if !found {
+		data, input = generateSerialData(n, 255, 0)
+		tester.inputs[uint64(n)] = input
+	} else {
+		data = io.LimitReader(bytes.NewReader(input), int64(n))
+	}
+
+	putGetter := newTestHasherStore(NewMapChunkStore(), hash)
+
+	var addr Address
+	var wait func(context.Context) error
+	var err error
+	ctx := context.TODO()
+	if usePyramid {
+		addr, wait, err = PyramidSplit(ctx, data, putGetter, putGetter)
+	} else {
+		addr, wait, err = TreeSplit(ctx, data, int64(n), putGetter)
+	}
+	if err != nil {
+		tester.t.Fatalf(err.Error())
+	}
+	tester.t.Logf(" Key = %v\n", addr)
+	err = wait(ctx)
+	if err != nil {
+		tester.t.Fatalf(err.Error())
+	}
+	return addr
+}
+
 func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) Address {
 	if tester.inputs == nil {
 		tester.inputs = make(map[uint64][]byte)
@@ -229,6 +264,21 @@ func TestDataAppend(t *testing.T) {
 	}
 }
 
+func TestSerialData(t *testing.T) {
+	sizes := []int{4096 * 129}
+	tester := &chunkerTester{t: t}
+
+	for _, s := range sizes {
+		treeChunkerKey := testSerialData(false, BMTHash, s, tester)
+		//		pyramidChunkerKey := testRandomData(true, SHA3Hash, s, tester)
+		//		if treeChunkerKey.String() != pyramidChunkerKey.String() {
+		//			tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String())
+		//		}
+		t.Logf("chunker result: %s", treeChunkerKey)
+	}
+
+}
+
 func TestRandomData(t *testing.T) {
 	// This test can validate files up to a relatively short length, as tree chunker slows down drastically.
 	// Validation of longer files is done by TestLocalStoreAndRetrieve in swarm package.

swarm/storage/common_test.go

@@ -162,6 +162,24 @@ func mget(store ChunkStore, hs []Address, f func(h Address, chunk Chunk) error)
 	return err
 }
 
+func generateRandomData(l int) (r io.Reader, slice []byte) {
+	slice = make([]byte, l)
+	if _, err := rand.Read(slice); err != nil {
+		panic("rand error")
+	}
+	r = io.LimitReader(bytes.NewReader(slice), int64(l))
+	return
+}
+
+func generateSerialData(l int, mod int, offset int) (r io.Reader, slice []byte) {
+	slice = make([]byte, l)
+	for i := 0; i < len(slice); i++ {
+		slice[i] = byte((i + offset) % mod)
+	}
+	r = io.LimitReader(bytes.NewReader(slice), int64(l))
+	return
+}
+
 func testDataReader(l int) (r io.Reader) {
 	return io.LimitReader(rand.Reader, int64(l))
 }