diff --git a/README.md b/README.md index 7b1b5b2..4daffe8 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,27 @@ # go-ipfs-chunker +> go-ipfs-chunker implements data Splitters for go-ipfs. + [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-chunker?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-chunker) [![Build Status](https://travis-ci.org/ipfs/go-ipfs-chunker.svg?branch=master)](https://travis-ci.org/ipfs/go-ipfs-chunker) -> go-ipfs-chunker implements data Splitters for go-ipfs. +## ❗ This repo is no longer maintained. +👉 We highly recommend switching to the maintained version at https://github.com/ipfs/boxo/tree/main/chunker. +🏎️ Good news! There is [tooling and documentation](https://github.com/ipfs/boxo#migrating-to-boxo) to expedite a switch in your repo. -`go-ipfs-chunker` provides the `Splitter` interface. IPFS splitters read data from a reader an create "chunks". These chunks are used to build the ipfs DAGs (Merkle Tree) and are the base unit to obtain the sums that ipfs uses to address content. +⚠️ If you continue using this repo, please note that security fixes will not be provided (unless someone steps in to maintain it). -The package provides a `SizeSplitter` which creates chunks of equal size and it is used by default in most cases, and a `rabin` fingerprint chunker. This chunker will attempt to split data in a way that the resulting blocks are the same when the data has repetitive patterns, thus optimizing the resulting DAGs. +📚 Learn more, including how to take the maintainership mantle or ask questions, [here](https://github.com/ipfs/boxo/wiki/Copied-or-Migrated-Repos-FAQ). -## Lead Maintainer -[Steven Allen](https://github.com/Stebalien) +## Summary + +`go-ipfs-chunker` provides the `Splitter` interface. IPFS splitters read data from a reader an create "chunks". These chunks are used to build the ipfs DAGs (Merkle Tree) and are the base unit to obtain the sums that ipfs uses to address content. + +The package provides a `SizeSplitter` which creates chunks of equal size and it is used by default in most cases, and a `rabin` fingerprint chunker. This chunker will attempt to split data in a way that the resulting blocks are the same when the data has repetitive patterns, thus optimizing the resulting DAGs. ## Table of Contents diff --git a/buzhash.go b/buzhash.go index 83ab019..3e80164 100644 --- a/buzhash.go +++ b/buzhash.go @@ -13,6 +13,7 @@ const ( buzMask = 1<<17 - 1 ) +// Deprecated: use github.com/ipfs/boxo/chunker.Buzhash type Buzhash struct { r io.Reader buf []byte @@ -21,6 +22,7 @@ type Buzhash struct { err error } +// Deprecated: use github.com/ipfs/boxo/chunker.NewBuzhash func NewBuzhash(r io.Reader) *Buzhash { return &Buzhash{ r: r, diff --git a/parse.go b/parse.go index dee8304..8599795 100644 --- a/parse.go +++ b/parse.go @@ -10,23 +10,32 @@ import ( const ( // DefaultBlockSize is the chunk size that splitters produce (or aim to). + // + // Deprecated: use github.com/ipfs/boxo/chunker.DefaultBlockSize DefaultBlockSize int64 = 1024 * 256 // No leaf block should contain more than 1MiB of payload data ( wrapping overhead aside ) // This effectively mandates the maximum chunk size // See discussion at https://github.com/ipfs/go-ipfs-chunker/pull/21#discussion_r369124879 for background + // + // Deprecated: use github.com/ipfs/boxo/chunker.ChunkSizeLimit ChunkSizeLimit int = 1048576 ) var ( + // Deprecated: use github.com/ipfs/boxo/chunker.ErrRabinMin ErrRabinMin = errors.New("rabin min must be greater than 16") - ErrSize = errors.New("chunker size must be greater than 0") - ErrSizeMax = fmt.Errorf("chunker parameters may not exceed the maximum chunk size of %d", ChunkSizeLimit) + // Deprecated: use github.com/ipfs/boxo/chunker.ErrSize + ErrSize = errors.New("chunker size must be greater than 0") + // Deprecated: use github.com/ipfs/boxo/chunker.ErrSizeMax + ErrSizeMax = fmt.Errorf("chunker parameters may not exceed the maximum chunk size of %d", ChunkSizeLimit) ) // FromString returns a Splitter depending on the given string: // it supports "default" (""), "size-{size}", "rabin", "rabin-{blocksize}", // "rabin-{min}-{avg}-{max}" and "buzhash". +// +// Deprecated: use github.com/ipfs/boxo/chunker.FromString func FromString(r io.Reader, chunker string) (Splitter, error) { switch { case chunker == "" || chunker == "default": diff --git a/rabin.go b/rabin.go index 4247057..3ee7a69 100644 --- a/rabin.go +++ b/rabin.go @@ -8,10 +8,14 @@ import ( ) // IpfsRabinPoly is the irreducible polynomial of degree 53 used by for Rabin. +// +// Deprecated: use github.com/ipfs/boxo/chunker.IpfsRabinPoly var IpfsRabinPoly = chunker.Pol(17437180132763653) // Rabin implements the Splitter interface and splits content with Rabin // fingerprints. +// +// Deprecated: use github.com/ipfs/boxo/chunker.Rabin type Rabin struct { r *chunker.Chunker reader io.Reader @@ -19,6 +23,8 @@ type Rabin struct { // NewRabin creates a new Rabin splitter with the given // average block size. +// +// Deprecated: use github.com/ipfs/boxo/chunker.NewRabin func NewRabin(r io.Reader, avgBlkSize uint64) *Rabin { min := avgBlkSize / 3 max := avgBlkSize + (avgBlkSize / 2) @@ -28,6 +34,8 @@ func NewRabin(r io.Reader, avgBlkSize uint64) *Rabin { // NewRabinMinMax returns a new Rabin splitter which uses // the given min, average and max block sizes. +// +// Deprecated: use github.com/ipfs/boxo/chunker.NewRabinMinMax func NewRabinMinMax(r io.Reader, min, avg, max uint64) *Rabin { h := fnv.New32a() ch := chunker.New(r, IpfsRabinPoly, h, avg, min, max) diff --git a/splitting.go b/splitting.go index a137820..5b637da 100644 --- a/splitting.go +++ b/splitting.go @@ -15,21 +15,29 @@ var log = logging.Logger("chunk") // A Splitter reads bytes from a Reader and creates "chunks" (byte slices) // that can be used to build DAG nodes. +// +// Deprecated: use github.com/ipfs/boxo/chunker.Splitter type Splitter interface { Reader() io.Reader NextBytes() ([]byte, error) } // SplitterGen is a splitter generator, given a reader. +// +// Deprecated: use github.com/ipfs/boxo/chunker.SplitterGen type SplitterGen func(r io.Reader) Splitter // DefaultSplitter returns a SizeSplitter with the DefaultBlockSize. +// +// Deprecated: use github.com/ipfs/boxo/chunker.DefaultSplitter func DefaultSplitter(r io.Reader) Splitter { return NewSizeSplitter(r, DefaultBlockSize) } // SizeSplitterGen returns a SplitterGen function which will create // a splitter with the given size when called. +// +// Deprecated: use github.com/ipfs/boxo/chunker.SizeSplitterGen func SizeSplitterGen(size int64) SplitterGen { return func(r io.Reader) Splitter { return NewSizeSplitter(r, size) @@ -38,6 +46,8 @@ func SizeSplitterGen(size int64) SplitterGen { // Chan returns a channel that receives each of the chunks produced // by a splitter, along with another one for errors. +// +// Deprecated: use github.com/ipfs/boxo/chunker.Chan func Chan(s Splitter) (<-chan []byte, <-chan error) { out := make(chan []byte) errs := make(chan error, 1) @@ -66,6 +76,8 @@ type sizeSplitterv2 struct { } // NewSizeSplitter returns a new size-based Splitter with the given block size. +// +// Deprecated: use github.com/ipfs/boxo/chunker.NewSizeSplitter func NewSizeSplitter(r io.Reader, size int64) Splitter { return &sizeSplitterv2{ r: r,