diff --git a/cache/contenthash/checksum_test.go b/cache/contenthash/checksum_test.go index a7ee85cde4f6..846cd9d2e823 100644 --- a/cache/contenthash/checksum_test.go +++ b/cache/contenthash/checksum_test.go @@ -401,7 +401,7 @@ func TestPersistence(t *testing.T) { time.Sleep(100 * time.Millisecond) // saving happens on the background - // we can't close snapshotter and open it twice (especially, its internal boltdb store) + // we can't close snapshotter and open it twice (especially, its internal bbolt store) cm.Close() getDefaultManager().lru.Purge() cm = setupCacheManager(t, tmpdir, snapshotter) diff --git a/cache/manager_test.go b/cache/manager_test.go index cb3ddb661952..3468bce1111c 100644 --- a/cache/manager_test.go +++ b/cache/manager_test.go @@ -331,7 +331,7 @@ func TestLazyCommit(t *testing.T) { err = cm.Close() require.NoError(t, err) - // we can't close snapshotter and open it twice (especially, its internal boltdb store) + // we can't close snapshotter and open it twice (especially, its internal bbolt store) cm = getCacheManager(t, tmpdir, snapshotter) snap2, err = cm.Get(ctx, snap.ID()) diff --git a/cache/metadata.go b/cache/metadata.go index ae26699dcd26..9929868844c9 100644 --- a/cache/metadata.go +++ b/cache/metadata.go @@ -3,10 +3,10 @@ package cache import ( "time" - "github.com/boltdb/bolt" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/client" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) const sizeUnknown int64 = -1 diff --git a/cache/metadata/metadata.go b/cache/metadata/metadata.go index 461ff6cfd7fe..9da270b4e619 100644 --- a/cache/metadata/metadata.go +++ b/cache/metadata/metadata.go @@ -6,9 +6,9 @@ import ( "strings" "sync" - "github.com/boltdb/bolt" "github.com/pkg/errors" "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" ) const ( diff --git a/cache/metadata/metadata_test.go b/cache/metadata/metadata_test.go index c79caf1744e1..7e3d5b055df6 100644 --- a/cache/metadata/metadata_test.go +++ b/cache/metadata/metadata_test.go @@ -6,8 +6,8 @@ import ( "path/filepath" "testing" - "github.com/boltdb/bolt" "github.com/stretchr/testify/require" + bolt "go.etcd.io/bbolt" ) func TestGetSetSearch(t *testing.T) { diff --git a/cmd/buildctl/debug/dumpmetadata.go b/cmd/buildctl/debug/dumpmetadata.go index 10afd4ac3f7c..67270c6c879c 100644 --- a/cmd/buildctl/debug/dumpmetadata.go +++ b/cmd/buildctl/debug/dumpmetadata.go @@ -8,9 +8,9 @@ import ( "path/filepath" "time" - "github.com/boltdb/bolt" "github.com/moby/buildkit/util/appdefaults" "github.com/urfave/cli" + bolt "go.etcd.io/bbolt" ) var DumpMetadataCommand = cli.Command{ diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 8c3d6fb25284..2f74ad94f9e2 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -30,7 +30,7 @@ import ( "github.com/moby/buildkit/frontend/gateway" "github.com/moby/buildkit/frontend/gateway/forwarder" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/solver/boltdbcachestorage" + "github.com/moby/buildkit/solver/bboltcachestorage" "github.com/moby/buildkit/util/apicaps" "github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/appdefaults" @@ -500,7 +500,7 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err frontends["dockerfile.v0"] = forwarder.NewGatewayForwarder(wc, dockerfile.Build) frontends["gateway.v0"] = gateway.NewGatewayFrontend(wc) - cacheStorage, err := boltdbcachestorage.NewStore(filepath.Join(cfg.Root, "cache.db")) + cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(cfg.Root, "cache.db")) if err != nil { return nil, err } diff --git a/docs/solver.md b/docs/solver.md index 27beb4d0f936..45b81c5cb078 100644 --- a/docs/solver.md +++ b/docs/solver.md @@ -133,7 +133,7 @@ Edge state is cleaned up when a final job that loaded the vertexes that they are Cache providers determine if there is a result that matches the cache keys generated during the build that could be reused instead of fully reevaluating the vertex and its inputs. There can be multiple cache providers, and specific providers can be defined per vertex using the vertex options. -There are multiple backend implementations for cache providers, in-memory one used in unit tests, the default local one using boltdb and one based on cache manifests in a remote registry. +There are multiple backend implementations for cache providers, in-memory one used in unit tests, the default local one using bbolt and one based on cache manifests in a remote registry. Simplified cache provider has following methods: diff --git a/snapshot/blobmapping/snapshotter.go b/snapshot/blobmapping/snapshotter.go index e91241b7b648..d6861ae5d605 100644 --- a/snapshot/blobmapping/snapshotter.go +++ b/snapshot/blobmapping/snapshotter.go @@ -3,13 +3,13 @@ package blobmapping import ( "context" - "github.com/boltdb/bolt" "github.com/containerd/containerd/content" "github.com/containerd/containerd/snapshots" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/snapshot" digest "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" ) const blobKey = "blobmapping.blob" diff --git a/solver/boltdbcachestorage/storage.go b/solver/bboltcachestorage/storage.go similarity index 99% rename from solver/boltdbcachestorage/storage.go rename to solver/bboltcachestorage/storage.go index 0b38fe8610c7..19755816cd5a 100644 --- a/solver/boltdbcachestorage/storage.go +++ b/solver/bboltcachestorage/storage.go @@ -1,14 +1,14 @@ -package boltdbcachestorage +package bboltcachestorage import ( "bytes" "encoding/json" "fmt" - "github.com/boltdb/bolt" "github.com/moby/buildkit/solver" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) const ( diff --git a/solver/boltdbcachestorage/storage_test.go b/solver/bboltcachestorage/storage_test.go similarity index 95% rename from solver/boltdbcachestorage/storage_test.go rename to solver/bboltcachestorage/storage_test.go index 69a349990971..5e72c889b47d 100644 --- a/solver/boltdbcachestorage/storage_test.go +++ b/solver/bboltcachestorage/storage_test.go @@ -1,4 +1,4 @@ -package boltdbcachestorage +package bboltcachestorage import ( "io/ioutil" diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 1804c85e5521..ee9c3af7bfb3 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -16,7 +16,6 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/mount" "github.com/docker/docker/pkg/locker" "github.com/moby/buildkit/cache" @@ -37,6 +36,7 @@ import ( "github.com/opencontainers/runc/libcontainer/system" "github.com/pkg/errors" "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) diff --git a/source/git/gitsource.go b/source/git/gitsource.go index c0afc3b8048f..47d304b79235 100644 --- a/source/git/gitsource.go +++ b/source/git/gitsource.go @@ -11,7 +11,6 @@ import ( "regexp" "strings" - "github.com/boltdb/bolt" "github.com/docker/docker/pkg/locker" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" @@ -22,6 +21,7 @@ import ( "github.com/moby/buildkit/util/progress/logs" "github.com/pkg/errors" "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" ) var validHex = regexp.MustCompile(`^[a-f0-9]{40}$`) diff --git a/source/http/httpsource.go b/source/http/httpsource.go index 58c70400989c..39939128cd9e 100644 --- a/source/http/httpsource.go +++ b/source/http/httpsource.go @@ -15,7 +15,6 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/docker/docker/pkg/locker" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" @@ -24,6 +23,7 @@ import ( "github.com/moby/buildkit/util/tracing" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type Opt struct { diff --git a/source/local/local.go b/source/local/local.go index b2deb5c15b63..8ffe0c09ed36 100644 --- a/source/local/local.go +++ b/source/local/local.go @@ -6,7 +6,6 @@ import ( "fmt" "time" - "github.com/boltdb/bolt" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/contenthash" "github.com/moby/buildkit/cache/metadata" @@ -20,6 +19,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/tonistiigi/fsutil" + bolt "go.etcd.io/bbolt" "golang.org/x/time/rate" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/vendor.conf b/vendor.conf index 2811f25717a9..00f8c83244d6 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,12 +1,12 @@ -github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd github.com/pkg/errors v0.8.0 +go.etcd.io/bbolt v1.3.1-etcd.8 github.com/stretchr/testify v1.1.4 github.com/davecgh/go-spew v1.1.0 github.com/pmezard/go-difflib v1.0.0 golang.org/x/sys 1b2967e3c290b7c545b3db0deeda16e9be4f98a2 -github.com/containerd/containerd 1597270d0468ccebd29b78164c2e902514f426fd +github.com/containerd/containerd d97a907f7f781c0ab8340877d8e6b53cc7f1c2f6 github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/sirupsen/logrus v1.0.0 @@ -19,16 +19,16 @@ github.com/golang/protobuf v1.1.0 github.com/containerd/continuity f44b615e492bdfb371aae2f76ec694d9da1db537 github.com/opencontainers/image-spec v1.0.1 github.com/opencontainers/runc 20aff4f0488c6d4b8df4d85b4f63f1f704c11abd -github.com/Microsoft/go-winio v0.4.10 +github.com/Microsoft/go-winio v0.4.11 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c -github.com/opencontainers/runtime-spec d810dbc60d8c5aeeb3d054bd1132fab2121968ce # v1.0.1-43-gd810dbc -github.com/containerd/go-runc acb7c88cac264acca9b5eae187a117f4d77a1292 +github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d +github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3 github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 -github.com/Microsoft/hcsshim 44c060121b68e8bdc40b411beba551f3b4ee9e55 +github.com/Microsoft/hcsshim v0.7.3 golang.org/x/crypto 0709b304e793a5edb4a2c0145f281ecdc20838a4 github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c @@ -50,8 +50,9 @@ github.com/docker/distribution 30578ca32960a4d368bf6db67b0a33c2a1f3dc6f github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 github.com/docker/cli 99576756eb3303b7af8102c502f21a912e3c1af6 https://github.com/tonistiigi/docker-cli.git github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 -github.com/docker/libnetwork 822e5b59d346b7ad0735df2c8e445e9787320e67 +github.com/docker/libnetwork 36d3bed0e9f4b3c8c66df9bd45278bb90b33e911 github.com/BurntSushi/toml 3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005 +github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 diff --git a/vendor/github.com/Microsoft/go-winio/ea.go b/vendor/github.com/Microsoft/go-winio/ea.go index b37e930d6af6..4051c1b33bfe 100644 --- a/vendor/github.com/Microsoft/go-winio/ea.go +++ b/vendor/github.com/Microsoft/go-winio/ea.go @@ -1,137 +1,137 @@ -package winio - -import ( - "bytes" - "encoding/binary" - "errors" -) - -type fileFullEaInformation struct { - NextEntryOffset uint32 - Flags uint8 - NameLength uint8 - ValueLength uint16 -} - -var ( - fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) - - errInvalidEaBuffer = errors.New("invalid extended attribute buffer") - errEaNameTooLarge = errors.New("extended attribute name too large") - errEaValueTooLarge = errors.New("extended attribute value too large") -) - -// ExtendedAttribute represents a single Windows EA. -type ExtendedAttribute struct { - Name string - Value []byte - Flags uint8 -} - -func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { - var info fileFullEaInformation - err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) - if err != nil { - err = errInvalidEaBuffer - return - } - - nameOffset := fileFullEaInformationSize - nameLen := int(info.NameLength) - valueOffset := nameOffset + int(info.NameLength) + 1 - valueLen := int(info.ValueLength) - nextOffset := int(info.NextEntryOffset) - if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { - err = errInvalidEaBuffer - return - } - - ea.Name = string(b[nameOffset : nameOffset+nameLen]) - ea.Value = b[valueOffset : valueOffset+valueLen] - ea.Flags = info.Flags - if info.NextEntryOffset != 0 { - nb = b[info.NextEntryOffset:] - } - return -} - -// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION -// buffer retrieved from BackupRead, ZwQueryEaFile, etc. -func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { - for len(b) != 0 { - ea, nb, err := parseEa(b) - if err != nil { - return nil, err - } - - eas = append(eas, ea) - b = nb - } - return -} - -func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { - if int(uint8(len(ea.Name))) != len(ea.Name) { - return errEaNameTooLarge - } - if int(uint16(len(ea.Value))) != len(ea.Value) { - return errEaValueTooLarge - } - entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) - withPadding := (entrySize + 3) &^ 3 - nextOffset := uint32(0) - if !last { - nextOffset = withPadding - } - info := fileFullEaInformation{ - NextEntryOffset: nextOffset, - Flags: ea.Flags, - NameLength: uint8(len(ea.Name)), - ValueLength: uint16(len(ea.Value)), - } - - err := binary.Write(buf, binary.LittleEndian, &info) - if err != nil { - return err - } - - _, err = buf.Write([]byte(ea.Name)) - if err != nil { - return err - } - - err = buf.WriteByte(0) - if err != nil { - return err - } - - _, err = buf.Write(ea.Value) - if err != nil { - return err - } - - _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) - if err != nil { - return err - } - - return nil -} - -// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION -// buffer for use with BackupWrite, ZwSetEaFile, etc. -func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { - var buf bytes.Buffer - for i := range eas { - last := false - if i == len(eas)-1 { - last = true - } - - err := writeEa(&buf, &eas[i], last) - if err != nil { - return nil, err - } - } - return buf.Bytes(), nil -} +package winio + +import ( + "bytes" + "encoding/binary" + "errors" +) + +type fileFullEaInformation struct { + NextEntryOffset uint32 + Flags uint8 + NameLength uint8 + ValueLength uint16 +} + +var ( + fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) + + errInvalidEaBuffer = errors.New("invalid extended attribute buffer") + errEaNameTooLarge = errors.New("extended attribute name too large") + errEaValueTooLarge = errors.New("extended attribute value too large") +) + +// ExtendedAttribute represents a single Windows EA. +type ExtendedAttribute struct { + Name string + Value []byte + Flags uint8 +} + +func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { + var info fileFullEaInformation + err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) + if err != nil { + err = errInvalidEaBuffer + return + } + + nameOffset := fileFullEaInformationSize + nameLen := int(info.NameLength) + valueOffset := nameOffset + int(info.NameLength) + 1 + valueLen := int(info.ValueLength) + nextOffset := int(info.NextEntryOffset) + if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { + err = errInvalidEaBuffer + return + } + + ea.Name = string(b[nameOffset : nameOffset+nameLen]) + ea.Value = b[valueOffset : valueOffset+valueLen] + ea.Flags = info.Flags + if info.NextEntryOffset != 0 { + nb = b[info.NextEntryOffset:] + } + return +} + +// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION +// buffer retrieved from BackupRead, ZwQueryEaFile, etc. +func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { + for len(b) != 0 { + ea, nb, err := parseEa(b) + if err != nil { + return nil, err + } + + eas = append(eas, ea) + b = nb + } + return +} + +func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { + if int(uint8(len(ea.Name))) != len(ea.Name) { + return errEaNameTooLarge + } + if int(uint16(len(ea.Value))) != len(ea.Value) { + return errEaValueTooLarge + } + entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) + withPadding := (entrySize + 3) &^ 3 + nextOffset := uint32(0) + if !last { + nextOffset = withPadding + } + info := fileFullEaInformation{ + NextEntryOffset: nextOffset, + Flags: ea.Flags, + NameLength: uint8(len(ea.Name)), + ValueLength: uint16(len(ea.Value)), + } + + err := binary.Write(buf, binary.LittleEndian, &info) + if err != nil { + return err + } + + _, err = buf.Write([]byte(ea.Name)) + if err != nil { + return err + } + + err = buf.WriteByte(0) + if err != nil { + return err + } + + _, err = buf.Write(ea.Value) + if err != nil { + return err + } + + _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) + if err != nil { + return err + } + + return nil +} + +// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION +// buffer for use with BackupWrite, ZwSetEaFile, etc. +func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { + var buf bytes.Buffer + for i := range eas { + last := false + if i == len(eas)-1 { + last = true + } + + err := writeEa(&buf, &eas[i], last) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go b/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go index c37dec8c7b97..e9e45c0306dc 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go @@ -2,10 +2,16 @@ package guid import ( "crypto/rand" + "encoding/json" "fmt" "io" + "strconv" + "strings" ) +var _ = (json.Marshaler)(&GUID{}) +var _ = (json.Unmarshaler)(&GUID{}) + type GUID [16]byte func New() GUID { @@ -20,3 +26,44 @@ func New() GUID { func (g GUID) String() string { return fmt.Sprintf("%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x-%02x", g[3], g[2], g[1], g[0], g[5], g[4], g[7], g[6], g[8:10], g[10:]) } + +func FromString(s string) GUID { + if len(s) != 36 { + panic(fmt.Sprintf("invalid GUID length: %d", len(s))) + } + if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + panic("invalid GUID format") + } + indexOrder := [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34, + } + byteOrder := [16]int{ + 3, 2, 1, 0, + 5, 4, + 7, 6, + 8, 9, + 10, 11, 12, 13, 14, 15, + } + var g GUID + for i, x := range indexOrder { + b, err := strconv.ParseInt(s[x:x+2], 16, 16) + if err != nil { + panic(err) + } + g[byteOrder[i]] = byte(b) + } + return g +} + +func (g GUID) MarshalJSON() ([]byte, error) { + return json.Marshal(g.String()) +} + +func (g *GUID) UnmarshalJSON(data []byte) error { + *g = FromString(strings.Trim(string(data), "\"")) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go index 0de4a706aa18..8294d66d7b48 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go @@ -2,6 +2,7 @@ package hcs import ( "encoding/json" + "fmt" "io" "sync" "syscall" @@ -83,7 +84,10 @@ func (process *Process) Kill() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("TerminateProcess %s: %d", process.SystemID(), process.Pid()), &completed) err := hcsTerminateProcess(process.handle, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return makeProcessError(process, operation, err, events) @@ -177,7 +181,10 @@ func (process *Process) Properties() (*ProcessStatus, error) { resultp *uint16 propertiesp *uint16 ) + completed := false + go syscallWatcher(fmt.Sprintf("GetProcessProperties %s: %d", process.SystemID(), process.Pid()), &completed) err := hcsGetProcessProperties(process.handle, &propertiesp, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, makeProcessError(process, operation, err, events) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go index 41ff2877b5bc..57afd5ec6ba3 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go @@ -2,6 +2,7 @@ package hcs import ( "encoding/json" + "fmt" "os" "strconv" "sync" @@ -63,7 +64,10 @@ func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System, resultp *uint16 identity syscall.Handle ) + completed := false + go syscallWatcher(fmt.Sprintf("CreateCompleteSystem %s: %s", id, hcsDocument), &completed) createError := hcsCreateComputeSystem(id, hcsDocument, identity, &computeSystem.handle, &resultp) + completed = true if createError == nil || IsPending(createError) { if err := computeSystem.registerCallback(); err != nil { @@ -74,7 +78,7 @@ func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System, } } - events, err := processAsyncHcsResult(createError, resultp, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.Duration) + events, err := processAsyncHcsResult(createError, resultp, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.SystemCreate) if err != nil { if err == ErrTimeout { // Terminate the compute system if it still exists. We're okay to @@ -135,7 +139,10 @@ func GetComputeSystems(q schema1.ComputeSystemQuery) ([]schema1.ContainerPropert resultp *uint16 computeSystemsp *uint16 ) + completed := false + go syscallWatcher(fmt.Sprintf("GetComputeSystems %s:", query), &completed) err = hcsEnumerateComputeSystems(query, &computeSystemsp, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, &HcsError{Op: operation, Err: err, Events: events} @@ -192,8 +199,11 @@ func (computeSystem *System) Start() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("StartComputeSystem %s:", computeSystem.ID()), &completed) err := hcsStartComputeSystem(computeSystem.handle, "", &resultp) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.Duration) + completed = true + events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart) if err != nil { return makeSystemError(computeSystem, "Start", "", err, events) } @@ -219,7 +229,10 @@ func (computeSystem *System) Shutdown() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("ShutdownComputeSystem %s:", computeSystem.ID()), &completed) err := hcsShutdownComputeSystem(computeSystem.handle, "", &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return makeSystemError(computeSystem, "Shutdown", "", err, events) @@ -242,7 +255,10 @@ func (computeSystem *System) Terminate() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("TerminateComputeSystem %s:", computeSystem.ID()), &completed) err := hcsTerminateComputeSystem(computeSystem.handle, "", &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return makeSystemError(computeSystem, "Terminate", "", err, events) @@ -291,7 +307,10 @@ func (computeSystem *System) Properties(types ...schema1.PropertyType) (*schema1 } var resultp, propertiesp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("GetComputeSystemProperties %s:", computeSystem.ID()), &completed) err = hcsGetComputeSystemProperties(computeSystem.handle, string(queryj), &propertiesp, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, makeSystemError(computeSystem, "Properties", "", err, events) @@ -320,8 +339,11 @@ func (computeSystem *System) Pause() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("PauseComputeSystem %s:", computeSystem.ID()), &completed) err := hcsPauseComputeSystem(computeSystem.handle, "", &resultp) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.Duration) + completed = true + events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause) if err != nil { return makeSystemError(computeSystem, "Pause", "", err, events) } @@ -342,8 +364,11 @@ func (computeSystem *System) Resume() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("ResumeComputeSystem %s:", computeSystem.ID()), &completed) err := hcsResumeComputeSystem(computeSystem.handle, "", &resultp) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.Duration) + completed = true + events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume) if err != nil { return makeSystemError(computeSystem, "Resume", "", err, events) } @@ -375,7 +400,10 @@ func (computeSystem *System) CreateProcess(c interface{}) (*Process, error) { configuration := string(configurationb) logrus.Debugf(title+" config=%s", configuration) + completed := false + go syscallWatcher(fmt.Sprintf("CreateProcess %s: %s", computeSystem.ID(), configuration), &completed) err = hcsCreateProcess(computeSystem.handle, configuration, &processInfo, &processHandle, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, makeSystemError(computeSystem, "CreateProcess", configuration, err, events) @@ -415,7 +443,10 @@ func (computeSystem *System) OpenProcess(pid int) (*Process, error) { return nil, makeSystemError(computeSystem, "OpenProcess", "", ErrAlreadyClosed, nil) } + completed := false + go syscallWatcher(fmt.Sprintf("OpenProcess %s: %d", computeSystem.ID(), pid), &completed) err := hcsOpenProcess(computeSystem.handle, uint32(pid), &processHandle, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, makeSystemError(computeSystem, "OpenProcess", "", err, events) @@ -451,7 +482,11 @@ func (computeSystem *System) Close() error { return makeSystemError(computeSystem, "Close", "", err, nil) } - if err := hcsCloseComputeSystem(computeSystem.handle); err != nil { + completed := false + go syscallWatcher(fmt.Sprintf("CloseComputeSystem %s:", computeSystem.ID()), &completed) + err := hcsCloseComputeSystem(computeSystem.handle) + completed = true + if err != nil { return makeSystemError(computeSystem, "Close", "", err, nil) } @@ -537,7 +572,10 @@ func (computeSystem *System) Modify(config interface{}) error { logrus.Debugf(title + " " + requestString) var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("ModifyComputeSystem %s: %s", computeSystem.ID(), requestString), &completed) err = hcsModifyComputeSystem(computeSystem.handle, requestString, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return makeSystemError(computeSystem, "Modify", requestString, err, events) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go new file mode 100644 index 000000000000..6b94bc9ff817 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go @@ -0,0 +1,30 @@ +package hcs + +import ( + "time" + + "github.com/Microsoft/hcsshim/internal/timeout" + "github.com/sirupsen/logrus" +) + +// syscallWatcher is used as a very simple goroutine around calls into +// the platform. In some cases, we have seen HCS APIs not returning due to +// various bugs, and the goroutine making the syscall ends up not returning, +// prior to its async callback. By spinning up a syscallWatcher, it allows +// us to at least log a warning if a syscall doesn't complete in a reasonable +// amount of time. +// +// Usage is: +// +// completed := false +// go syscallWatcher("some description", &completed) +// +// completed = true +// +func syscallWatcher(description string, syscallCompleted *bool) { + time.Sleep(timeout.SyscallWatcher) + if *syscallCompleted { + return + } + logrus.Warnf("%s: Did not complete within %s. This may indicate a platform issue. If it appears to be making no forward progress, obtain the stacks and see is there is a syscall stuck in the platform API for a significant length of time.", description, timeout.SyscallWatcher) +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go index 32f4e070c0ec..2f5bf8f555bb 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go @@ -1,4 +1,4 @@ -// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT +// Code generated by 'go generate'; DO NOT EDIT. package interop diff --git a/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go index 776adbe7a0a7..709b9d3475d1 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go @@ -1,4 +1,4 @@ -// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT +// Code generated by 'go generate'; DO NOT EDIT. package safefile diff --git a/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go b/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go index e4253f400051..ff3b6572e659 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go @@ -6,21 +6,65 @@ import ( "time" ) -// Duration is the default time to wait for various operations. -// - Waiting for async notifications from HCS -// - Waiting for processes to launch through -// - Waiting to copy data to/from a launched processes stdio pipes. -// -// This can be overridden through environment variable `HCS_TIMEOUT_SECONDS` +var ( + // defaultTimeout is the timeout for most operations that is not overridden. + defaultTimeout = 4 * time.Minute -var Duration = 4 * time.Minute + // defaultTimeoutTestdRetry is the retry loop timeout for testd to respond + // for a disk to come online in LCOW. + defaultTimeoutTestdRetry = 5 * time.Second +) + +// External variables for HCSShim consumers to use. +var ( + // SystemCreate is the timeout for creating a compute system + SystemCreate time.Duration = defaultTimeout + + // SystemStart is the timeout for starting a compute system + SystemStart time.Duration = defaultTimeout + + // SystemPause is the timeout for pausing a compute system + SystemPause time.Duration = defaultTimeout + + // SystemResume is the timeout for resuming a compute system + SystemResume time.Duration = defaultTimeout + + // SyscallWatcher is the timeout before warning of a potential stuck platform syscall. + SyscallWatcher time.Duration = defaultTimeout + + // Tar2VHD is the timeout for the tar2vhd operation to complete + Tar2VHD time.Duration = defaultTimeout + + // ExternalCommandToStart is the timeout for external commands to start + ExternalCommandToStart = defaultTimeout + + // ExternalCommandToComplete is the timeout for external commands to complete. + // Generally this means copying data from their stdio pipes. + ExternalCommandToComplete = defaultTimeout + + // TestDRetryLoop is the timeout for testd retry loop when onlining a SCSI disk in LCOW + TestDRetryLoop = defaultTimeoutTestdRetry +) func init() { - envTimeout := os.Getenv("HCSSHIM_TIMEOUT_SECONDS") + SystemCreate = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMCREATE", SystemCreate) + SystemStart = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMSTART", SystemStart) + SystemPause = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMPAUSE", SystemPause) + SystemResume = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMRESUME", SystemResume) + SyscallWatcher = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSCALLWATCHER", SyscallWatcher) + Tar2VHD = durationFromEnvironment("HCSSHIM_TIMEOUT_TAR2VHD", Tar2VHD) + ExternalCommandToStart = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDSTART", ExternalCommandToStart) + ExternalCommandToComplete = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDCOMPLETE", ExternalCommandToComplete) + TestDRetryLoop = durationFromEnvironment("HCSSHIM_TIMEOUT_TESTDRETRYLOOP", TestDRetryLoop) +} + +func durationFromEnvironment(env string, defaultValue time.Duration) time.Duration { + envTimeout := os.Getenv(env) if len(envTimeout) > 0 { e, err := strconv.Atoi(envTimeout) if err == nil && e > 0 { - Duration = time.Second * time.Duration(e) + return time.Second * time.Duration(e) } } + return defaultValue } diff --git a/vendor/github.com/containerd/containerd/README.md b/vendor/github.com/containerd/containerd/README.md index d3e4aa4357ef..c4d9dffdbdc5 100644 --- a/vendor/github.com/containerd/containerd/README.md +++ b/vendor/github.com/containerd/containerd/README.md @@ -224,7 +224,7 @@ This will be the best place to discuss design and implementation. For sync communication we have a community slack with a #containerd channel that everyone is welcome to join and chat about development. -**Slack:** https://dockr.ly/community +**Slack:** https://join.slack.com/t/dockercommunity/shared_invite/enQtNDM4NjAwNDMyOTUwLWZlMDZmYWRjZjk4Zjc5ZGQ5NWZkOWI1Yjk2NGE3ZWVlYjYxM2VhYjczOWIyZDFhZTE3NTUwZWQzMjhmNGYyZTg ### Reporting security issues diff --git a/vendor/github.com/containerd/containerd/content/local/writer.go b/vendor/github.com/containerd/containerd/content/local/writer.go index a6579a9d2102..10df4a4c5408 100644 --- a/vendor/github.com/containerd/containerd/content/local/writer.go +++ b/vendor/github.com/containerd/containerd/content/local/writer.go @@ -132,11 +132,11 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, // clean up!! defer os.RemoveAll(w.path) + if _, err := os.Stat(target); err == nil { + // collision with the target file! + return errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", dgst) + } if err := os.Rename(ingest, target); err != nil { - if os.IsExist(err) { - // collision with the target file! - return errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", dgst) - } return err } commitTime := time.Now() diff --git a/vendor/github.com/containerd/containerd/diff/walking/differ.go b/vendor/github.com/containerd/containerd/diff/walking/differ.go index 462d70856844..f06aa016f519 100644 --- a/vendor/github.com/containerd/containerd/diff/walking/differ.go +++ b/vendor/github.com/containerd/containerd/diff/walking/differ.go @@ -139,7 +139,9 @@ func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, o dgst := cw.Digest() if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil { - return errors.Wrap(err, "failed to commit") + if !errdefs.IsAlreadyExists(err) { + return errors.Wrap(err, "failed to commit") + } } info, err := s.store.Info(ctx, dgst) diff --git a/vendor/github.com/containerd/containerd/install.go b/vendor/github.com/containerd/containerd/install.go index 2aa8b039469b..5e4c6a2c8d40 100644 --- a/vendor/github.com/containerd/containerd/install.go +++ b/vendor/github.com/containerd/containerd/install.go @@ -33,25 +33,14 @@ import ( // Install a binary image into the opt service func (c *Client) Install(ctx context.Context, image Image, opts ...InstallOpts) error { - resp, err := c.IntrospectionService().Plugins(ctx, &introspectionapi.PluginsRequest{ - Filters: []string{ - "id==opt", - }, - }) - if err != nil { - return err - } - if len(resp.Plugins) != 1 { - return errors.New("opt service not enabled") - } - path := resp.Plugins[0].Exports["path"] - if path == "" { - return errors.New("opt path not exported") - } var config InstallConfig for _, o := range opts { o(&config) } + path, err := c.getInstallPath(ctx, config) + if err != nil { + return err + } var ( cs = image.ContentStore() platform = platforms.Default() @@ -89,3 +78,25 @@ func (c *Client) Install(ctx context.Context, image Image, opts ...InstallOpts) } return nil } + +func (c *Client) getInstallPath(ctx context.Context, config InstallConfig) (string, error) { + if config.Path != "" { + return config.Path, nil + } + resp, err := c.IntrospectionService().Plugins(ctx, &introspectionapi.PluginsRequest{ + Filters: []string{ + "id==opt", + }, + }) + if err != nil { + return "", err + } + if len(resp.Plugins) != 1 { + return "", errors.New("opt service not enabled") + } + path := resp.Plugins[0].Exports["path"] + if path == "" { + return "", errors.New("opt path not exported") + } + return path, nil +} diff --git a/vendor/github.com/containerd/containerd/install_opts.go b/vendor/github.com/containerd/containerd/install_opts.go index b11e7f3d6d0f..b0c9213cb277 100644 --- a/vendor/github.com/containerd/containerd/install_opts.go +++ b/vendor/github.com/containerd/containerd/install_opts.go @@ -25,6 +25,8 @@ type InstallConfig struct { Libs bool // Replace will overwrite existing binaries or libs in the opt directory Replace bool + // Path to install libs and binaries to + Path string } // WithInstallLibs installs libs from the image @@ -36,3 +38,10 @@ func WithInstallLibs(c *InstallConfig) { func WithInstallReplace(c *InstallConfig) { c.Replace = true } + +// WithInstallPath sets the optional install path +func WithInstallPath(path string) InstallOpts { + return func(c *InstallConfig) { + c.Path = path + } +} diff --git a/vendor/github.com/containerd/containerd/metadata/bolt.go b/vendor/github.com/containerd/containerd/metadata/bolt.go index 156a33593f43..6ea460866361 100644 --- a/vendor/github.com/containerd/containerd/metadata/bolt.go +++ b/vendor/github.com/containerd/containerd/metadata/bolt.go @@ -19,8 +19,8 @@ package metadata import ( "context" - "github.com/boltdb/bolt" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type transactionKey struct{} diff --git a/vendor/github.com/containerd/containerd/metadata/boltutil/helpers.go b/vendor/github.com/containerd/containerd/metadata/boltutil/helpers.go index 4b2ede876360..1240188577be 100644 --- a/vendor/github.com/containerd/containerd/metadata/boltutil/helpers.go +++ b/vendor/github.com/containerd/containerd/metadata/boltutil/helpers.go @@ -19,8 +19,8 @@ package boltutil import ( "time" - "github.com/boltdb/bolt" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) var ( diff --git a/vendor/github.com/containerd/containerd/metadata/buckets.go b/vendor/github.com/containerd/containerd/metadata/buckets.go index fcf4c29597f3..51d40a7f3601 100644 --- a/vendor/github.com/containerd/containerd/metadata/buckets.go +++ b/vendor/github.com/containerd/containerd/metadata/buckets.go @@ -17,11 +17,11 @@ package metadata import ( - "github.com/boltdb/bolt" digest "github.com/opencontainers/go-digest" + bolt "go.etcd.io/bbolt" ) -// The layout where a "/" delineates a bucket is desribed in the following +// The layout where a "/" delineates a bucket is described in the following // section. Please try to follow this as closely as possible when adding // functionality. We can bolster this with helpers and more structure if that // becomes an issue. @@ -164,11 +164,11 @@ func getSnapshotterBucket(tx *bolt.Tx, namespace, snapshotter string) *bolt.Buck } func createBlobBucket(tx *bolt.Tx, namespace string, dgst digest.Digest) (*bolt.Bucket, error) { - bkt, err := createBucketIfNotExists(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectContent, bucketKeyObjectBlob, []byte(dgst.String())) + bkt, err := createBucketIfNotExists(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectContent, bucketKeyObjectBlob) if err != nil { return nil, err } - return bkt, nil + return bkt.CreateBucket([]byte(dgst.String())) } func getBlobsBucket(tx *bolt.Tx, namespace string) *bolt.Bucket { diff --git a/vendor/github.com/containerd/containerd/metadata/containers.go b/vendor/github.com/containerd/containerd/metadata/containers.go index 2c9726fc2734..6e5622c36f57 100644 --- a/vendor/github.com/containerd/containerd/metadata/containers.go +++ b/vendor/github.com/containerd/containerd/metadata/containers.go @@ -21,7 +21,6 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/filters" @@ -32,6 +31,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type containerStore struct { diff --git a/vendor/github.com/containerd/containerd/metadata/content.go b/vendor/github.com/containerd/containerd/metadata/content.go index 7293f1bfae29..ac4440060b82 100644 --- a/vendor/github.com/containerd/containerd/metadata/content.go +++ b/vendor/github.com/containerd/containerd/metadata/content.go @@ -23,7 +23,6 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/filters" @@ -34,6 +33,7 @@ import ( digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type contentStore struct { @@ -592,9 +592,6 @@ func (nw *namespacedWriter) commit(ctx context.Context, tx *bolt.Tx, size int64, } size = nw.desc.Size actual = nw.desc.Digest - if getBlobBucket(tx, nw.namespace, actual) != nil { - return "", errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", actual) - } } else { status, err := nw.w.Status() if err != nil { @@ -606,18 +603,16 @@ func (nw *namespacedWriter) commit(ctx context.Context, tx *bolt.Tx, size int64, size = status.Offset actual = nw.w.Digest() - if err := nw.w.Commit(ctx, size, expected); err != nil { - if !errdefs.IsAlreadyExists(err) { - return "", err - } - if getBlobBucket(tx, nw.namespace, actual) != nil { - return "", errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", actual) - } + if err := nw.w.Commit(ctx, size, expected); err != nil && !errdefs.IsAlreadyExists(err) { + return "", err } } bkt, err := createBlobBucket(tx, nw.namespace, actual) if err != nil { + if err == bolt.ErrBucketExists { + return "", errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", actual) + } return "", err } diff --git a/vendor/github.com/containerd/containerd/metadata/db.go b/vendor/github.com/containerd/containerd/metadata/db.go index 0ea8ad78ff73..507d6d22d011 100644 --- a/vendor/github.com/containerd/containerd/metadata/db.go +++ b/vendor/github.com/containerd/containerd/metadata/db.go @@ -23,12 +23,12 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/content" "github.com/containerd/containerd/gc" "github.com/containerd/containerd/log" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) const ( @@ -43,7 +43,7 @@ const ( // dbVersion represents updates to the schema // version which are additions and compatible with // prior version of the same schema. - dbVersion = 2 + dbVersion = 3 ) // DB represents a metadata database backed by a bolt diff --git a/vendor/github.com/containerd/containerd/metadata/gc.go b/vendor/github.com/containerd/containerd/metadata/gc.go index eb4d0c7c339b..a670ce1a3e09 100644 --- a/vendor/github.com/containerd/containerd/metadata/gc.go +++ b/vendor/github.com/containerd/containerd/metadata/gc.go @@ -23,10 +23,10 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/gc" "github.com/containerd/containerd/log" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) const ( diff --git a/vendor/github.com/containerd/containerd/metadata/images.go b/vendor/github.com/containerd/containerd/metadata/images.go index 62b1179cd33b..6ae9dc85541c 100644 --- a/vendor/github.com/containerd/containerd/metadata/images.go +++ b/vendor/github.com/containerd/containerd/metadata/images.go @@ -23,7 +23,6 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/filters" "github.com/containerd/containerd/images" @@ -33,6 +32,7 @@ import ( digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type imageStore struct { diff --git a/vendor/github.com/containerd/containerd/metadata/leases.go b/vendor/github.com/containerd/containerd/metadata/leases.go index 8835065d1ea4..635c3b3c37e4 100644 --- a/vendor/github.com/containerd/containerd/metadata/leases.go +++ b/vendor/github.com/containerd/containerd/metadata/leases.go @@ -20,7 +20,6 @@ import ( "context" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/filters" "github.com/containerd/containerd/leases" @@ -28,6 +27,7 @@ import ( "github.com/containerd/containerd/namespaces" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) // LeaseManager manages the create/delete lifecyle of leases diff --git a/vendor/github.com/containerd/containerd/metadata/migrations.go b/vendor/github.com/containerd/containerd/metadata/migrations.go index 3ffda6b3bfd8..34febdd15965 100644 --- a/vendor/github.com/containerd/containerd/metadata/migrations.go +++ b/vendor/github.com/containerd/containerd/metadata/migrations.go @@ -16,7 +16,7 @@ package metadata -import "github.com/boltdb/bolt" +import bolt "go.etcd.io/bbolt" type migration struct { schema string @@ -45,6 +45,11 @@ var migrations = []migration{ version: 2, migrate: migrateIngests, }, + { + schema: "v1", + version: 3, + migrate: noOpMigration, + }, } // addChildLinks Adds children key to the snapshotters to enforce snapshot @@ -154,3 +159,10 @@ func migrateIngests(tx *bolt.Tx) error { return nil } + +// noOpMigration was for a database change from boltdb/bolt which is no +// longer being supported, to go.etcd.io/bbolt which is the currently +// maintained repo for boltdb. +func noOpMigration(tx *bolt.Tx) error { + return nil +} diff --git a/vendor/github.com/containerd/containerd/metadata/namespaces.go b/vendor/github.com/containerd/containerd/metadata/namespaces.go index 000bef6c9cf5..74951eb5c5b6 100644 --- a/vendor/github.com/containerd/containerd/metadata/namespaces.go +++ b/vendor/github.com/containerd/containerd/metadata/namespaces.go @@ -19,11 +19,11 @@ package metadata import ( "context" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" l "github.com/containerd/containerd/labels" "github.com/containerd/containerd/namespaces" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type namespaceStore struct { diff --git a/vendor/github.com/containerd/containerd/metadata/snapshot.go b/vendor/github.com/containerd/containerd/metadata/snapshot.go index fda380a6abc7..75b80b0bc59c 100644 --- a/vendor/github.com/containerd/containerd/metadata/snapshot.go +++ b/vendor/github.com/containerd/containerd/metadata/snapshot.go @@ -23,7 +23,6 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/labels" "github.com/containerd/containerd/log" @@ -32,6 +31,7 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type snapshotter struct { diff --git a/vendor/github.com/containerd/containerd/oci/spec_opts.go b/vendor/github.com/containerd/containerd/oci/spec_opts.go index 47ffd8c9f369..50c77396d203 100644 --- a/vendor/github.com/containerd/containerd/oci/spec_opts.go +++ b/vendor/github.com/containerd/containerd/oci/spec_opts.go @@ -315,8 +315,14 @@ func WithImageConfigArgs(image Image, args []string) SpecOpts { } s.Process.Cwd = cwd if config.User != "" { - return WithUser(config.User)(ctx, client, c, s) + if err := WithUser(config.User)(ctx, client, c, s); err != nil { + return err + } + return WithAdditionalGIDs(fmt.Sprintf("%d", s.Process.User.UID))(ctx, client, c, s) } + // we should query the image's /etc/group for additional GIDs + // even if there is no specified user in the image config + return WithAdditionalGIDs("root")(ctx, client, c, s) } else if s.Windows != nil { s.Process.Env = config.Env s.Process.Args = append(config.Entrypoint, config.Cmd...) @@ -478,12 +484,13 @@ func WithUser(userstr string) SpecOpts { } f := func(root string) error { if username != "" { - uid, _, err = getUIDGIDFromPath(root, func(u user.User) bool { + user, err := getUserFromPath(root, func(u user.User) bool { return u.Name == username }) if err != nil { return err } + uid = uint32(user.Uid) } if groupname != "" { gid, err = getGIDFromPath(root, func(g user.Group) bool { @@ -541,7 +548,7 @@ func WithUserID(uid uint32) SpecOpts { if !isRootfsAbs(s.Root.Path) { return errors.Errorf("rootfs absolute path is required") } - uuid, ugid, err := getUIDGIDFromPath(s.Root.Path, func(u user.User) bool { + user, err := getUserFromPath(s.Root.Path, func(u user.User) bool { return u.Uid == int(uid) }) if err != nil { @@ -551,7 +558,7 @@ func WithUserID(uid uint32) SpecOpts { } return err } - s.Process.User.UID, s.Process.User.GID = uuid, ugid + s.Process.User.UID, s.Process.User.GID = uint32(user.Uid), uint32(user.Gid) return nil } @@ -567,7 +574,7 @@ func WithUserID(uid uint32) SpecOpts { return err } return mount.WithTempMount(ctx, mounts, func(root string) error { - uuid, ugid, err := getUIDGIDFromPath(root, func(u user.User) bool { + user, err := getUserFromPath(root, func(u user.User) bool { return u.Uid == int(uid) }) if err != nil { @@ -577,7 +584,7 @@ func WithUserID(uid uint32) SpecOpts { } return err } - s.Process.User.UID, s.Process.User.GID = uuid, ugid + s.Process.User.UID, s.Process.User.GID = uint32(user.Uid), uint32(user.Gid) return nil }) } @@ -595,13 +602,13 @@ func WithUsername(username string) SpecOpts { if !isRootfsAbs(s.Root.Path) { return errors.Errorf("rootfs absolute path is required") } - uid, gid, err := getUIDGIDFromPath(s.Root.Path, func(u user.User) bool { + user, err := getUserFromPath(s.Root.Path, func(u user.User) bool { return u.Name == username }) if err != nil { return err } - s.Process.User.UID, s.Process.User.GID = uid, gid + s.Process.User.UID, s.Process.User.GID = uint32(user.Uid), uint32(user.Gid) return nil } if c.Snapshotter == "" { @@ -616,13 +623,13 @@ func WithUsername(username string) SpecOpts { return err } return mount.WithTempMount(ctx, mounts, func(root string) error { - uid, gid, err := getUIDGIDFromPath(root, func(u user.User) bool { + user, err := getUserFromPath(root, func(u user.User) bool { return u.Name == username }) if err != nil { return err } - s.Process.User.UID, s.Process.User.GID = uid, gid + s.Process.User.UID, s.Process.User.GID = uint32(user.Uid), uint32(user.Gid) return nil }) } else if s.Windows != nil { @@ -634,6 +641,71 @@ func WithUsername(username string) SpecOpts { } } +// WithAdditionalGIDs sets the OCI spec's additionalGids array to any additional groups listed +// for a particular user in the /etc/groups file of the image's root filesystem +// The passed in user can be either a uid or a username. +func WithAdditionalGIDs(userstr string) SpecOpts { + return func(ctx context.Context, client Client, c *containers.Container, s *Spec) (err error) { + setProcess(s) + setAdditionalGids := func(root string) error { + var username string + uid, err := strconv.Atoi(userstr) + if err == nil { + user, err := getUserFromPath(root, func(u user.User) bool { + return u.Uid == uid + }) + if err != nil { + if os.IsNotExist(err) || err == errNoUsersFound { + return nil + } + return err + } + username = user.Name + } else { + username = userstr + } + gids, err := getSupplementalGroupsFromPath(root, func(g user.Group) bool { + // we only want supplemental groups + if g.Name == username { + return false + } + for _, entry := range g.List { + if entry == username { + return true + } + } + return false + }) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + s.Process.User.AdditionalGids = gids + return nil + } + if c.Snapshotter == "" && c.SnapshotKey == "" { + if !isRootfsAbs(s.Root.Path) { + return errors.Errorf("rootfs absolute path is required") + } + return setAdditionalGids(s.Root.Path) + } + if c.Snapshotter == "" { + return errors.Errorf("no snapshotter set for container") + } + if c.SnapshotKey == "" { + return errors.Errorf("rootfs snapshot not created for container") + } + snapshotter := client.SnapshotService(c.Snapshotter) + mounts, err := snapshotter.Mounts(ctx, c.SnapshotKey) + if err != nil { + return err + } + return mount.WithTempMount(ctx, mounts, setAdditionalGids) + } +} + // WithCapabilities sets Linux capabilities on the process func WithCapabilities(caps []string) SpecOpts { return func(_ context.Context, _ Client, _ *containers.Container, s *Spec) error { @@ -681,20 +753,19 @@ func WithAmbientCapabilities(caps []string) SpecOpts { var errNoUsersFound = errors.New("no users found") -func getUIDGIDFromPath(root string, filter func(user.User) bool) (uid, gid uint32, err error) { +func getUserFromPath(root string, filter func(user.User) bool) (user.User, error) { ppath, err := fs.RootPath(root, "/etc/passwd") if err != nil { - return 0, 0, err + return user.User{}, err } users, err := user.ParsePasswdFileFilter(ppath, filter) if err != nil { - return 0, 0, err + return user.User{}, err } if len(users) == 0 { - return 0, 0, errNoUsersFound + return user.User{}, errNoUsersFound } - u := users[0] - return uint32(u.Uid), uint32(u.Gid), nil + return users[0], nil } var errNoGroupsFound = errors.New("no groups found") @@ -715,6 +786,26 @@ func getGIDFromPath(root string, filter func(user.Group) bool) (gid uint32, err return uint32(g.Gid), nil } +func getSupplementalGroupsFromPath(root string, filter func(user.Group) bool) ([]uint32, error) { + gpath, err := fs.RootPath(root, "/etc/group") + if err != nil { + return []uint32{}, err + } + groups, err := user.ParseGroupFileFilter(gpath, filter) + if err != nil { + return []uint32{}, err + } + if len(groups) == 0 { + // if there are no additional groups; just return an empty set + return []uint32{}, nil + } + addlGids := []uint32{} + for _, grp := range groups { + addlGids = append(addlGids, uint32(grp.Gid)) + } + return addlGids, nil +} + func isRootfsAbs(root string) bool { return filepath.IsAbs(root) } diff --git a/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go b/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go index 9c589575ef9a..7716a591e00d 100644 --- a/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go +++ b/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go @@ -23,11 +23,11 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/metadata/boltutil" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) var ( diff --git a/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go b/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go index fa5e6c78be19..69ba3ea96bbc 100644 --- a/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go +++ b/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go @@ -25,9 +25,9 @@ import ( "context" "sync" - "github.com/boltdb/bolt" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) // Transactor is used to finalize an active transaction. diff --git a/vendor/github.com/containerd/containerd/task.go b/vendor/github.com/containerd/containerd/task.go index 750075f8dbdf..6806e1162081 100644 --- a/vendor/github.com/containerd/containerd/task.go +++ b/vendor/github.com/containerd/containerd/task.go @@ -607,8 +607,11 @@ func writeContent(ctx context.Context, store content.Ingester, mediaType, ref st if err != nil { return d, err } + if err := writer.Commit(ctx, size, "", opts...); err != nil { - return d, err + if !errdefs.IsAlreadyExists(err) { + return d, err + } } return v1.Descriptor{ MediaType: mediaType, diff --git a/vendor/github.com/containerd/containerd/vendor.conf b/vendor/github.com/containerd/containerd/vendor.conf index 216122309040..1bcd620ea3c3 100644 --- a/vendor/github.com/containerd/containerd/vendor.conf +++ b/vendor/github.com/containerd/containerd/vendor.conf @@ -19,7 +19,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.0 github.com/gogo/protobuf v1.0.0 github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef github.com/golang/protobuf v1.1.0 -github.com/opencontainers/runtime-spec d810dbc60d8c5aeeb3d054bd1132fab2121968ce # v1.0.1-43-gd810dbc +github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d github.com/opencontainers/runc 20aff4f0488c6d4b8df4d85b4f63f1f704c11abd github.com/sirupsen/logrus v1.0.0 github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c @@ -34,17 +34,17 @@ github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 github.com/Microsoft/go-winio v0.4.10 github.com/Microsoft/hcsshim 44c060121b68e8bdc40b411beba551f3b4ee9e55 -github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 github.com/containerd/ttrpc 94dde388801693c54f88a6596f713b51a8b30b2d github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 gotest.tools v2.1.0 github.com/google/go-cmp v0.1.0 +go.etcd.io/bbolt v1.3.1-etcd.8 # cri dependencies -github.com/containerd/cri v1.11.1 -github.com/containerd/go-cni 5882530828ecf62032409b298a3e8b19e08b6534 +github.com/containerd/cri 9f39e3289533fc228c5e5fcac0a6dbdd60c6047b # release/1.2 branch +github.com/containerd/go-cni 6d7b509a054a3cb1c35ed1865d4fde2f0cb547cd github.com/blang/semver v3.1.0 github.com/containernetworking/cni v0.6.0 github.com/containernetworking/plugins v0.7.0 @@ -52,32 +52,33 @@ github.com/davecgh/go-spew v1.1.0 github.com/docker/distribution b38e5838b7b2f2ad48e06ec4b500011976080621 github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528 -github.com/emicklei/go-restful ff4f55a206334ef123e4f79bbf348980da81ca46 -github.com/ghodss/yaml 73d445a93680fa1a78ae23a5839bad48f32ba1ee +github.com/emicklei/go-restful v2.2.1 +github.com/ghodss/yaml v1.0.0 github.com/golang/glog 44145f04b68cf362d9c4df2182967c2275eaefed github.com/google/gofuzz 44d81051d367757e1c7c6a5a86423ece9afcf63c github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55 github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f -github.com/json-iterator/go f2b4162afba35581b6d4a50d3b8f34e33c144682 -github.com/modern-go/reflect2 05fbef0ca5da472bbf96c9322b84a53edc03c9fd +github.com/json-iterator/go 1.1.5 +github.com/modern-go/reflect2 1.0.1 github.com/modern-go/concurrent 1.0.3 github.com/opencontainers/runtime-tools v0.6.0 -github.com/opencontainers/selinux 4a2974bf1ee960774ffd517717f1f45325af0206 +github.com/opencontainers/selinux b6fa367ed7f534f9ba25391cc2d467085dbb445a github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 -github.com/tchap/go-patricia 5ad6cdb7538b0097d5598c7e57f0a24072adf7dc +github.com/tchap/go-patricia v2.2.6 github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6 github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874 golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067 +golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4 golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 -gopkg.in/yaml.v2 53feefa2559fb8dfa8d81baad31be332c97d6c77 -k8s.io/api 9e5ffd1f1320950b238cfce291b926411f0af722 -k8s.io/apimachinery ed135c5b96450fd24e5e981c708114fbbd950697 -k8s.io/apiserver a90e3a95c2e91b944bfca8225c4e0d12e42a9eb5 -k8s.io/client-go 03bfb9bdcfe5482795b999f39ca3ed9ad42ce5bb -k8s.io/kubernetes v1.11.0 -k8s.io/utils 733eca437aa39379e4bcc25e726439dfca40fcff +gopkg.in/yaml.v2 v2.2.1 +k8s.io/api 012f271b5d41baad56190c5f1ae19bff16df0fd8 +k8s.io/apimachinery 6429050ef506887d121f3e7306e894f8900d8a63 +k8s.io/apiserver e9312c15296b6c2c923ebd5031ff5d1d5fd022d7 +k8s.io/client-go 37c3c02ec96533daec0dbda1f39a6b1d68505c79 +k8s.io/kubernetes v1.12.0-beta.1 +k8s.io/utils 982821ea41da7e7c15f3d3738921eb2e7e241ccd # zfs dependencies github.com/containerd/zfs 9a0b8b8b5982014b729cd34eb7cd7a11062aa6ec diff --git a/vendor/github.com/containerd/go-runc/io.go b/vendor/github.com/containerd/go-runc/io.go index 24a419d18b73..6cf0410c9d33 100644 --- a/vendor/github.com/containerd/go-runc/io.go +++ b/vendor/github.com/containerd/go-runc/io.go @@ -34,6 +34,24 @@ type StartCloser interface { CloseAfterStart() error } +// IOOpt sets I/O creation options +type IOOpt func(*IOOption) + +// IOOption holds I/O creation options +type IOOption struct { + OpenStdin bool + OpenStdout bool + OpenStderr bool +} + +func defaultIOOption() *IOOption { + return &IOOption{ + OpenStdin: true, + OpenStdout: true, + OpenStderr: true, + } +} + func newPipe() (*pipe, error) { r, w, err := os.Pipe() if err != nil { @@ -65,14 +83,23 @@ type pipeIO struct { } func (i *pipeIO) Stdin() io.WriteCloser { + if i.in == nil { + return nil + } return i.in.w } func (i *pipeIO) Stdout() io.ReadCloser { + if i.out == nil { + return nil + } return i.out.r } func (i *pipeIO) Stderr() io.ReadCloser { + if i.err == nil { + return nil + } return i.err.r } @@ -83,28 +110,38 @@ func (i *pipeIO) Close() error { i.out, i.err, } { - if cerr := v.Close(); err == nil { - err = cerr + if v != nil { + if cerr := v.Close(); err == nil { + err = cerr + } } } return err } func (i *pipeIO) CloseAfterStart() error { - for _, f := range []*os.File{ - i.out.w, - i.err.w, + for _, f := range []*pipe{ + i.out, + i.err, } { - f.Close() + if f != nil { + f.w.Close() + } } return nil } // Set sets the io to the exec.Cmd func (i *pipeIO) Set(cmd *exec.Cmd) { - cmd.Stdin = i.in.r - cmd.Stdout = i.out.w - cmd.Stderr = i.err.w + if i.in != nil { + cmd.Stdin = i.in.r + } + if i.out != nil { + cmd.Stdout = i.out.w + } + if i.err != nil { + cmd.Stderr = i.err.w + } } func NewSTDIO() (IO, error) { diff --git a/vendor/github.com/containerd/go-runc/io_unix.go b/vendor/github.com/containerd/go-runc/io_unix.go index e0420ead61ae..567cd072e5f8 100644 --- a/vendor/github.com/containerd/go-runc/io_unix.go +++ b/vendor/github.com/containerd/go-runc/io_unix.go @@ -24,8 +24,15 @@ import ( ) // NewPipeIO creates pipe pairs to be used with runc -func NewPipeIO(uid, gid int) (i IO, err error) { - var pipes []*pipe +func NewPipeIO(uid, gid int, opts ...IOOpt) (i IO, err error) { + option := defaultIOOption() + for _, o := range opts { + o(option) + } + var ( + pipes []*pipe + stdin, stdout, stderr *pipe + ) // cleanup in case of an error defer func() { if err != nil { @@ -34,33 +41,33 @@ func NewPipeIO(uid, gid int) (i IO, err error) { } } }() - stdin, err := newPipe() - if err != nil { - return nil, err - } - pipes = append(pipes, stdin) - if err = unix.Fchown(int(stdin.r.Fd()), uid, gid); err != nil { - return nil, errors.Wrap(err, "failed to chown stdin") - } - - stdout, err := newPipe() - if err != nil { - return nil, err - } - pipes = append(pipes, stdout) - if err = unix.Fchown(int(stdout.w.Fd()), uid, gid); err != nil { - return nil, errors.Wrap(err, "failed to chown stdout") + if option.OpenStdin { + if stdin, err = newPipe(); err != nil { + return nil, err + } + pipes = append(pipes, stdin) + if err = unix.Fchown(int(stdin.r.Fd()), uid, gid); err != nil { + return nil, errors.Wrap(err, "failed to chown stdin") + } } - - stderr, err := newPipe() - if err != nil { - return nil, err + if option.OpenStdout { + if stdout, err = newPipe(); err != nil { + return nil, err + } + pipes = append(pipes, stdout) + if err = unix.Fchown(int(stdout.w.Fd()), uid, gid); err != nil { + return nil, errors.Wrap(err, "failed to chown stdout") + } } - pipes = append(pipes, stderr) - if err = unix.Fchown(int(stderr.w.Fd()), uid, gid); err != nil { - return nil, errors.Wrap(err, "failed to chown stderr") + if option.OpenStderr { + if stderr, err = newPipe(); err != nil { + return nil, err + } + pipes = append(pipes, stderr) + if err = unix.Fchown(int(stderr.w.Fd()), uid, gid); err != nil { + return nil, errors.Wrap(err, "failed to chown stderr") + } } - return &pipeIO{ in: stdin, out: stdout, diff --git a/vendor/github.com/containerd/go-runc/io_windows.go b/vendor/github.com/containerd/go-runc/io_windows.go index 4ac557ef8c6c..fc56ac4f301a 100644 --- a/vendor/github.com/containerd/go-runc/io_windows.go +++ b/vendor/github.com/containerd/go-runc/io_windows.go @@ -19,8 +19,15 @@ package runc // NewPipeIO creates pipe pairs to be used with runc -func NewPipeIO() (i IO, err error) { - var pipes []*pipe +func NewPipeIO(opts ...IOOpt) (i IO, err error) { + option := defaultIOOption() + for _, o := range opts { + o(option) + } + var ( + pipes []*pipe + stdin, stdout, stderr *pipe + ) // cleanup in case of an error defer func() { if err != nil { @@ -29,24 +36,24 @@ func NewPipeIO() (i IO, err error) { } } }() - stdin, err := newPipe() - if err != nil { - return nil, err + if option.OpenStdin { + if stdin, err = newPipe(); err != nil { + return nil, err + } + pipes = append(pipes, stdin) } - pipes = append(pipes, stdin) - - stdout, err := newPipe() - if err != nil { - return nil, err + if option.OpenStdout { + if stdout, err = newPipe(); err != nil { + return nil, err + } + pipes = append(pipes, stdout) } - pipes = append(pipes, stdout) - - stderr, err := newPipe() - if err != nil { - return nil, err + if option.OpenStderr { + if stderr, err = newPipe(); err != nil { + return nil, err + } + pipes = append(pipes, stderr) } - pipes = append(pipes, stderr) - return &pipeIO{ in: stdin, out: stdout, diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go index ac9c89d8054f..96262afab396 100644 --- a/vendor/github.com/containerd/go-runc/runc.go +++ b/vendor/github.com/containerd/go-runc/runc.go @@ -608,9 +608,8 @@ func parseVersion(data []byte) (Version, error) { var v Version parts := strings.Split(strings.TrimSpace(string(data)), "\n") if len(parts) != 3 { - return v, ErrParseRuncVersion + return v, nil } - for i, p := range []struct { dest *string split string diff --git a/vendor/github.com/docker/libnetwork/README.md b/vendor/github.com/docker/libnetwork/README.md index 536f8aa2b3eb..a9020381abe8 100644 --- a/vendor/github.com/docker/libnetwork/README.md +++ b/vendor/github.com/docker/libnetwork/README.md @@ -1,6 +1,6 @@ # libnetwork - networking for containers -[![Circle CI](https://circleci.com/gh/docker/libnetwork/tree/master.svg?style=svg)](https://circleci.com/gh/docker/libnetwork/tree/master) [![Coverage Status](https://coveralls.io/repos/docker/libnetwork/badge.svg)](https://coveralls.io/r/docker/libnetwork) [![GoDoc](https://godoc.org/github.com/docker/libnetwork?status.svg)](https://godoc.org/github.com/docker/libnetwork) +[![Circle CI](https://circleci.com/gh/docker/libnetwork/tree/master.svg?style=svg)](https://circleci.com/gh/docker/libnetwork/tree/master) [![Coverage Status](https://coveralls.io/repos/docker/libnetwork/badge.svg)](https://coveralls.io/r/docker/libnetwork) [![GoDoc](https://godoc.org/github.com/docker/libnetwork?status.svg)](https://godoc.org/github.com/docker/libnetwork) [![Go Report Card](https://goreportcard.com/badge/github.com/docker/libnetwork)](https://goreportcard.com/report/github.com/docker/libnetwork) Libnetwork provides a native Go implementation for connecting containers @@ -15,6 +15,17 @@ There are many networking solutions available to suit a broad range of use-cases ```go +import ( + "fmt" + "log" + + "github.com/docker/docker/pkg/reexec" + "github.com/docker/libnetwork" + "github.com/docker/libnetwork/config" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/options" +) + func main() { if reexec.Init() { return diff --git a/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go b/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go index 5cb251b13103..23caf7f12083 100644 --- a/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go +++ b/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go @@ -14,6 +14,11 @@ import ( "github.com/sirupsen/logrus" ) +const ( + // DefaultResolvConf points to the default file used for dns configuration on a linux machine + DefaultResolvConf = "/etc/resolv.conf" +) + var ( // Note: the default IPv4 & IPv6 resolvers are set to Google's Public DNS defaultIPv4Dns = []string{"nameserver 8.8.8.8", "nameserver 8.8.4.4"} @@ -50,15 +55,7 @@ type File struct { // Get returns the contents of /etc/resolv.conf and its hash func Get() (*File, error) { - resolv, err := ioutil.ReadFile("/etc/resolv.conf") - if err != nil { - return nil, err - } - hash, err := ioutils.HashData(bytes.NewReader(resolv)) - if err != nil { - return nil, err - } - return &File{Content: resolv, Hash: hash}, nil + return GetSpecific(DefaultResolvConf) } // GetSpecific returns the contents of the user specified resolv.conf file and its hash diff --git a/vendor/github.com/docker/libnetwork/types/types.go b/vendor/github.com/docker/libnetwork/types/types.go index 164b18096ced..b102ba4c390b 100644 --- a/vendor/github.com/docker/libnetwork/types/types.go +++ b/vendor/github.com/docker/libnetwork/types/types.go @@ -7,6 +7,8 @@ import ( "net" "strconv" "strings" + + "github.com/ishidawataru/sctp" ) // constants for the IP address type @@ -96,6 +98,8 @@ func (p PortBinding) HostAddr() (net.Addr, error) { return &net.UDPAddr{IP: p.HostIP, Port: int(p.HostPort)}, nil case TCP: return &net.TCPAddr{IP: p.HostIP, Port: int(p.HostPort)}, nil + case SCTP: + return &sctp.SCTPAddr{IP: []net.IP{p.HostIP}, Port: int(p.HostPort)}, nil default: return nil, ErrInvalidProtocolBinding(p.Proto.String()) } @@ -108,6 +112,8 @@ func (p PortBinding) ContainerAddr() (net.Addr, error) { return &net.UDPAddr{IP: p.IP, Port: int(p.Port)}, nil case TCP: return &net.TCPAddr{IP: p.IP, Port: int(p.Port)}, nil + case SCTP: + return &sctp.SCTPAddr{IP: []net.IP{p.IP}, Port: int(p.Port)}, nil default: return nil, ErrInvalidProtocolBinding(p.Proto.String()) } @@ -139,7 +145,12 @@ func (p *PortBinding) String() string { return ret } -// FromString reads the TransportPort structure from string +// FromString reads the PortBinding structure from string s. +// String s is a triple of "protocol/containerIP:port/hostIP:port" +// containerIP and hostIP can be in dotted decimal ("192.0.2.1") or IPv6 ("2001:db8::68") form. +// Zoned addresses ("169.254.0.23%eth0" or "fe80::1ff:fe23:4567:890a%eth0") are not supported. +// If string s is incorrectly formatted or the IP addresses or ports cannot be parsed, FromString +// returns an error. func (p *PortBinding) FromString(s string) error { ps := strings.Split(s, "/") if len(ps) != 3 { @@ -161,21 +172,19 @@ func (p *PortBinding) FromString(s string) error { } func parseIPPort(s string) (net.IP, uint16, error) { - pp := strings.Split(s, ":") - if len(pp) != 2 { - return nil, 0, BadRequestErrorf("invalid format: %s", s) + hoststr, portstr, err := net.SplitHostPort(s) + if err != nil { + return nil, 0, err } - var ip net.IP - if pp[0] != "" { - if ip = net.ParseIP(pp[0]); ip == nil { - return nil, 0, BadRequestErrorf("invalid ip: %s", pp[0]) - } + ip := net.ParseIP(hoststr) + if ip == nil { + return nil, 0, BadRequestErrorf("invalid ip: %s", hoststr) } - port, err := strconv.ParseUint(pp[1], 10, 16) + port, err := strconv.ParseUint(portstr, 10, 16) if err != nil { - return nil, 0, BadRequestErrorf("invalid port: %s", pp[1]) + return nil, 0, BadRequestErrorf("invalid port: %s", portstr) } return ip, uint16(port), nil @@ -233,6 +242,8 @@ const ( TCP = 6 // UDP is for the UDP ip protocol UDP = 17 + // SCTP is for the SCTP ip protocol + SCTP = 132 ) // Protocol represents an IP protocol number @@ -246,6 +257,8 @@ func (p Protocol) String() string { return "tcp" case UDP: return "udp" + case SCTP: + return "sctp" default: return fmt.Sprintf("%d", p) } @@ -260,6 +273,8 @@ func ParseProtocol(s string) Protocol { return UDP case "tcp": return TCP + case "sctp": + return SCTP default: return 0 } @@ -317,6 +332,8 @@ func CompareIPNet(a, b *net.IPNet) bool { } // GetMinimalIP returns the address in its shortest form +// If ip contains an IPv4-mapped IPv6 address, the 4-octet form of the IPv4 address will be returned. +// Otherwise ip is returned unchanged. func GetMinimalIP(ip net.IP) net.IP { if ip != nil && ip.To4() != nil { return ip.To4() diff --git a/vendor/github.com/docker/libnetwork/vendor.conf b/vendor/github.com/docker/libnetwork/vendor.conf index bdcb5a1a5453..f2bb55223e3f 100644 --- a/vendor/github.com/docker/libnetwork/vendor.conf +++ b/vendor/github.com/docker/libnetwork/vendor.conf @@ -1,53 +1,50 @@ github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 -github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060 -github.com/Microsoft/go-winio v0.4.5 -github.com/Microsoft/hcsshim v0.6.5 +github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 +github.com/Microsoft/go-winio v0.4.11 +github.com/Microsoft/hcsshim v0.7.3 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 -github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904 github.com/codegangsta/cli a65b733b303f0055f8d324d805f393cd3e7a7904 -github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e -github.com/containerd/continuity 22694c680ee48fb8f50015b44618517e2bde77e8 +github.com/containerd/continuity d3c23511c1bf5851696cba83143d9cbcd666869b github.com/coreos/etcd v3.2.1 github.com/coreos/go-semver v0.2.0 -github.com/coreos/go-systemd v4 github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d +go.etcd.io/bbolt v1.3.1-etcd.8 -github.com/docker/docker a3efe9722f34af5cf4443fe3a5c4e4e3e0457b54 -github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d +github.com/docker/docker 162ba6016def672690ee4a1f3978368853a1e149 +github.com/docker/go-connections 7beb39f0b969b075d1325fecb092faf27fd357b6 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/docker/go-units 9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1 -github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef +github.com/docker/libkv 458977154600b9f23984d9f4b82e79570b5ae12b github.com/godbus/dbus v4.0.0 -github.com/gogo/protobuf v0.4 -github.com/golang/protobuf 7a211bcf3bce0e3f1d74f9894916e6f116ae83b4 +github.com/gogo/protobuf v1.0.0 github.com/gorilla/context v1.1 github.com/gorilla/mux v1.1 github.com/hashicorp/consul v0.5.2 github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b github.com/hashicorp/go-multierror fcdddc395df1ddf4247c69bd436e84cfa0733f7e -github.com/hashicorp/memberlist v0.1.0 +github.com/hashicorp/memberlist 3d8438da9589e7b608a83ffac1ef8211486bcb7c github.com/sean-/seed e2103e2c35297fb7e17febb81e49b312087a2372 -github.com/hashicorp/go-sockaddr acd314c5781ea706c710d9ea70069fd2e110d61d +github.com/hashicorp/go-sockaddr 6d291a969b86c4b633730bfc6b8b9d64c3aafed9 github.com/hashicorp/serf 598c54895cc5a7b1a24a398d635e8c0ea0959870 github.com/mattn/go-shellwords v1.0.3 -github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 -github.com/mrunalp/fileutils ed869b029674c0e9ce4c0dfa781405c2d9946d08 -github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb -github.com/opencontainers/image-spec 372ad780f63454fbbbbcc7cf80e5b90245c13e13 -github.com/opencontainers/runc 0351df1c5a66838d0c392b4ac4cf9450de844e2d -github.com/opencontainers/runtime-spec v1.0.0 -github.com/opencontainers/selinux v1.0.0-rc1 +github.com/miekg/dns v1.0.7 +github.com/opencontainers/go-digest v1.0.0-rc1 +github.com/opencontainers/image-spec v1.0.1 +github.com/opencontainers/runc 69663f0bd4b60df09991c08812a60108003fa340 +github.com/opencontainers/runtime-spec v1.0.1 github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 -github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 github.com/sirupsen/logrus v1.0.3 -github.com/stretchr/testify dab07ac62d4905d3e48d17dc549c684ac3b7c15a -github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 -golang.org/x/crypto 558b6879de74bc843225cde5686419267ff707ca -golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 -golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5 +golang.org/x/crypto 1a580b3eff7814fc9b40602fd35256c63b50f491 +golang.org/x/net 0ed95abb35c445290478a5348a7b38bb154135fd +golang.org/x/sys 37707fdb30a5b38865cfb95e5aab41707daec7fd +golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5 github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9 +github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb + +gotest.tools v2.1.0 +github.com/google/go-cmp v0.2.0 diff --git a/vendor/github.com/ishidawataru/sctp/LICENSE b/vendor/github.com/ishidawataru/sctp/LICENSE new file mode 100644 index 000000000000..8dada3edaf50 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/ishidawataru/sctp/README.md b/vendor/github.com/ishidawataru/sctp/README.md new file mode 100644 index 000000000000..574ececa8626 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/README.md @@ -0,0 +1,18 @@ +Stream Control Transmission Protocol (SCTP) +---- + +[![Build Status](https://travis-ci.org/ishidawataru/sctp.svg?branch=master)](https://travis-ci.org/ishidawataru/sctp/builds) + +Examples +---- + +See `example/sctp.go` + +```go +$ cd example +$ go build +$ # run example SCTP server +$ ./example -server -port 1000 -ip 10.10.0.1,10.20.0.1 +$ # run example SCTP client +$ ./example -port 1000 -ip 10.10.0.1,10.20.0.1 +``` diff --git a/vendor/github.com/ishidawataru/sctp/sctp.go b/vendor/github.com/ishidawataru/sctp/sctp.go new file mode 100644 index 000000000000..cac1a889ca30 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/sctp.go @@ -0,0 +1,656 @@ +package sctp + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + "unsafe" +) + +const ( + SOL_SCTP = 132 + + SCTP_BINDX_ADD_ADDR = 0x01 + SCTP_BINDX_REM_ADDR = 0x02 + + MSG_NOTIFICATION = 0x8000 +) + +const ( + SCTP_RTOINFO = iota + SCTP_ASSOCINFO + SCTP_INITMSG + SCTP_NODELAY + SCTP_AUTOCLOSE + SCTP_SET_PEER_PRIMARY_ADDR + SCTP_PRIMARY_ADDR + SCTP_ADAPTATION_LAYER + SCTP_DISABLE_FRAGMENTS + SCTP_PEER_ADDR_PARAMS + SCTP_DEFAULT_SENT_PARAM + SCTP_EVENTS + SCTP_I_WANT_MAPPED_V4_ADDR + SCTP_MAXSEG + SCTP_STATUS + SCTP_GET_PEER_ADDR_INFO + SCTP_DELAYED_ACK_TIME + SCTP_DELAYED_ACK = SCTP_DELAYED_ACK_TIME + SCTP_DELAYED_SACK = SCTP_DELAYED_ACK_TIME + + SCTP_SOCKOPT_BINDX_ADD = 100 + SCTP_SOCKOPT_BINDX_REM = 101 + SCTP_SOCKOPT_PEELOFF = 102 + SCTP_GET_PEER_ADDRS = 108 + SCTP_GET_LOCAL_ADDRS = 109 + SCTP_SOCKOPT_CONNECTX = 110 + SCTP_SOCKOPT_CONNECTX3 = 111 +) + +const ( + SCTP_EVENT_DATA_IO = 1 << iota + SCTP_EVENT_ASSOCIATION + SCTP_EVENT_ADDRESS + SCTP_EVENT_SEND_FAILURE + SCTP_EVENT_PEER_ERROR + SCTP_EVENT_SHUTDOWN + SCTP_EVENT_PARTIAL_DELIVERY + SCTP_EVENT_ADAPTATION_LAYER + SCTP_EVENT_AUTHENTICATION + SCTP_EVENT_SENDER_DRY + + SCTP_EVENT_ALL = SCTP_EVENT_DATA_IO | SCTP_EVENT_ASSOCIATION | SCTP_EVENT_ADDRESS | SCTP_EVENT_SEND_FAILURE | SCTP_EVENT_PEER_ERROR | SCTP_EVENT_SHUTDOWN | SCTP_EVENT_PARTIAL_DELIVERY | SCTP_EVENT_ADAPTATION_LAYER | SCTP_EVENT_AUTHENTICATION | SCTP_EVENT_SENDER_DRY +) + +type SCTPNotificationType int + +const ( + SCTP_SN_TYPE_BASE = SCTPNotificationType(iota + (1 << 15)) + SCTP_ASSOC_CHANGE + SCTP_PEER_ADDR_CHANGE + SCTP_SEND_FAILED + SCTP_REMOTE_ERROR + SCTP_SHUTDOWN_EVENT + SCTP_PARTIAL_DELIVERY_EVENT + SCTP_ADAPTATION_INDICATION + SCTP_AUTHENTICATION_INDICATION + SCTP_SENDER_DRY_EVENT +) + +type NotificationHandler func([]byte) error + +type EventSubscribe struct { + DataIO uint8 + Association uint8 + Address uint8 + SendFailure uint8 + PeerError uint8 + Shutdown uint8 + PartialDelivery uint8 + AdaptationLayer uint8 + Authentication uint8 + SenderDry uint8 +} + +const ( + SCTP_CMSG_INIT = iota + SCTP_CMSG_SNDRCV + SCTP_CMSG_SNDINFO + SCTP_CMSG_RCVINFO + SCTP_CMSG_NXTINFO +) + +const ( + SCTP_UNORDERED = 1 << iota + SCTP_ADDR_OVER + SCTP_ABORT + SCTP_SACK_IMMEDIATELY + SCTP_EOF +) + +const ( + SCTP_MAX_STREAM = 0xffff +) + +type InitMsg struct { + NumOstreams uint16 + MaxInstreams uint16 + MaxAttempts uint16 + MaxInitTimeout uint16 +} + +type SndRcvInfo struct { + Stream uint16 + SSN uint16 + Flags uint16 + _ uint16 + PPID uint32 + Context uint32 + TTL uint32 + TSN uint32 + CumTSN uint32 + AssocID int32 +} + +type SndInfo struct { + SID uint16 + Flags uint16 + PPID uint32 + Context uint32 + AssocID int32 +} + +type GetAddrsOld struct { + AssocID int32 + AddrNum int32 + Addrs uintptr +} + +type NotificationHeader struct { + Type uint16 + Flags uint16 + Length uint32 +} + +type SCTPState uint16 + +const ( + SCTP_COMM_UP = SCTPState(iota) + SCTP_COMM_LOST + SCTP_RESTART + SCTP_SHUTDOWN_COMP + SCTP_CANT_STR_ASSOC +) + +var nativeEndian binary.ByteOrder +var sndRcvInfoSize uintptr + +func init() { + i := uint16(1) + if *(*byte)(unsafe.Pointer(&i)) == 0 { + nativeEndian = binary.BigEndian + } else { + nativeEndian = binary.LittleEndian + } + info := SndRcvInfo{} + sndRcvInfoSize = unsafe.Sizeof(info) +} + +func toBuf(v interface{}) []byte { + var buf bytes.Buffer + binary.Write(&buf, nativeEndian, v) + return buf.Bytes() +} + +func htons(h uint16) uint16 { + if nativeEndian == binary.LittleEndian { + return (h << 8 & 0xff00) | (h >> 8 & 0xff) + } + return h +} + +var ntohs = htons + +func setNumOstreams(fd, num int) error { + param := InitMsg{ + NumOstreams: uint16(num), + } + optlen := unsafe.Sizeof(param) + _, _, err := setsockopt(fd, SCTP_INITMSG, uintptr(unsafe.Pointer(¶m)), uintptr(optlen)) + return err +} + +type SCTPAddr struct { + IP []net.IP + Port int +} + +func (a *SCTPAddr) ToRawSockAddrBuf() []byte { + buf := []byte{} + p := htons(uint16(a.Port)) + for _, ip := range a.IP { + if ip.To4() != nil { + s := syscall.RawSockaddrInet4{ + Family: syscall.AF_INET, + Port: p, + } + copy(s.Addr[:], ip.To4()) + buf = append(buf, toBuf(s)...) + } else { + s := syscall.RawSockaddrInet6{ + Family: syscall.AF_INET6, + Port: p, + } + copy(s.Addr[:], ip) + buf = append(buf, toBuf(s)...) + } + } + return buf +} + +func (a *SCTPAddr) String() string { + var b bytes.Buffer + + for n, i := range a.IP { + if a.IP[n].To4() != nil { + b.WriteString(i.String()) + } else if a.IP[n].To16() != nil { + b.WriteRune('[') + b.WriteString(i.String()) + b.WriteRune(']') + } + if n < len(a.IP)-1 { + b.WriteRune('/') + } + } + b.WriteRune(':') + b.WriteString(strconv.Itoa(a.Port)) + return b.String() +} + +func (a *SCTPAddr) Network() string { return "sctp" } + +func ResolveSCTPAddr(network, addrs string) (*SCTPAddr, error) { + tcpnet := "" + switch network { + case "", "sctp": + case "sctp4": + tcpnet = "tcp4" + case "sctp6": + tcpnet = "tcp6" + default: + return nil, fmt.Errorf("invalid net: %s", network) + } + elems := strings.Split(addrs, "/") + if len(elems) == 0 { + return nil, fmt.Errorf("invalid input: %s", addrs) + } + ipaddrs := make([]net.IP, 0, len(elems)) + for _, e := range elems[:len(elems)-1] { + tcpa, err := net.ResolveTCPAddr(tcpnet, e+":") + if err != nil { + return nil, err + } + ipaddrs = append(ipaddrs, tcpa.IP) + } + tcpa, err := net.ResolveTCPAddr(tcpnet, elems[len(elems)-1]) + if err != nil { + return nil, err + } + if tcpa.IP != nil { + ipaddrs = append(ipaddrs, tcpa.IP) + } else { + ipaddrs = nil + } + return &SCTPAddr{ + IP: ipaddrs, + Port: tcpa.Port, + }, nil +} + +func SCTPConnect(fd int, addr *SCTPAddr) (int, error) { + buf := addr.ToRawSockAddrBuf() + param := GetAddrsOld{ + AddrNum: int32(len(buf)), + Addrs: uintptr(uintptr(unsafe.Pointer(&buf[0]))), + } + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(fd, SCTP_SOCKOPT_CONNECTX3, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err == nil { + return int(param.AssocID), nil + } else if err != syscall.ENOPROTOOPT { + return 0, err + } + r0, _, err := setsockopt(fd, SCTP_SOCKOPT_CONNECTX, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) + return int(r0), err +} + +func SCTPBind(fd int, addr *SCTPAddr, flags int) error { + var option uintptr + switch flags { + case SCTP_BINDX_ADD_ADDR: + option = SCTP_SOCKOPT_BINDX_ADD + case SCTP_BINDX_REM_ADDR: + option = SCTP_SOCKOPT_BINDX_REM + default: + return syscall.EINVAL + } + + buf := addr.ToRawSockAddrBuf() + _, _, err := setsockopt(fd, option, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) + return err +} + +type SCTPConn struct { + _fd int32 + notificationHandler NotificationHandler +} + +func (c *SCTPConn) fd() int { + return int(atomic.LoadInt32(&c._fd)) +} + +func NewSCTPConn(fd int, handler NotificationHandler) *SCTPConn { + conn := &SCTPConn{ + _fd: int32(fd), + notificationHandler: handler, + } + return conn +} + +func (c *SCTPConn) Write(b []byte) (int, error) { + return c.SCTPWrite(b, nil) +} + +func (c *SCTPConn) Read(b []byte) (int, error) { + n, _, err := c.SCTPRead(b) + if n < 0 { + n = 0 + } + return n, err +} + +func (c *SCTPConn) SetInitMsg(numOstreams, maxInstreams, maxAttempts, maxInitTimeout int) error { + param := InitMsg{ + NumOstreams: uint16(numOstreams), + MaxInstreams: uint16(maxInstreams), + MaxAttempts: uint16(maxAttempts), + MaxInitTimeout: uint16(maxInitTimeout), + } + optlen := unsafe.Sizeof(param) + _, _, err := setsockopt(c.fd(), SCTP_INITMSG, uintptr(unsafe.Pointer(¶m)), uintptr(optlen)) + return err +} + +func (c *SCTPConn) SubscribeEvents(flags int) error { + var d, a, ad, sf, p, sh, pa, ada, au, se uint8 + if flags&SCTP_EVENT_DATA_IO > 0 { + d = 1 + } + if flags&SCTP_EVENT_ASSOCIATION > 0 { + a = 1 + } + if flags&SCTP_EVENT_ADDRESS > 0 { + ad = 1 + } + if flags&SCTP_EVENT_SEND_FAILURE > 0 { + sf = 1 + } + if flags&SCTP_EVENT_PEER_ERROR > 0 { + p = 1 + } + if flags&SCTP_EVENT_SHUTDOWN > 0 { + sh = 1 + } + if flags&SCTP_EVENT_PARTIAL_DELIVERY > 0 { + pa = 1 + } + if flags&SCTP_EVENT_ADAPTATION_LAYER > 0 { + ada = 1 + } + if flags&SCTP_EVENT_AUTHENTICATION > 0 { + au = 1 + } + if flags&SCTP_EVENT_SENDER_DRY > 0 { + se = 1 + } + param := EventSubscribe{ + DataIO: d, + Association: a, + Address: ad, + SendFailure: sf, + PeerError: p, + Shutdown: sh, + PartialDelivery: pa, + AdaptationLayer: ada, + Authentication: au, + SenderDry: se, + } + optlen := unsafe.Sizeof(param) + _, _, err := setsockopt(c.fd(), SCTP_EVENTS, uintptr(unsafe.Pointer(¶m)), uintptr(optlen)) + return err +} + +func (c *SCTPConn) SubscribedEvents() (int, error) { + param := EventSubscribe{} + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(c.fd(), SCTP_EVENTS, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err != nil { + return 0, err + } + var flags int + if param.DataIO > 0 { + flags |= SCTP_EVENT_DATA_IO + } + if param.Association > 0 { + flags |= SCTP_EVENT_ASSOCIATION + } + if param.Address > 0 { + flags |= SCTP_EVENT_ADDRESS + } + if param.SendFailure > 0 { + flags |= SCTP_EVENT_SEND_FAILURE + } + if param.PeerError > 0 { + flags |= SCTP_EVENT_PEER_ERROR + } + if param.Shutdown > 0 { + flags |= SCTP_EVENT_SHUTDOWN + } + if param.PartialDelivery > 0 { + flags |= SCTP_EVENT_PARTIAL_DELIVERY + } + if param.AdaptationLayer > 0 { + flags |= SCTP_EVENT_ADAPTATION_LAYER + } + if param.Authentication > 0 { + flags |= SCTP_EVENT_AUTHENTICATION + } + if param.SenderDry > 0 { + flags |= SCTP_EVENT_SENDER_DRY + } + return flags, nil +} + +func (c *SCTPConn) SetDefaultSentParam(info *SndRcvInfo) error { + optlen := unsafe.Sizeof(*info) + _, _, err := setsockopt(c.fd(), SCTP_DEFAULT_SENT_PARAM, uintptr(unsafe.Pointer(info)), uintptr(optlen)) + return err +} + +func (c *SCTPConn) GetDefaultSentParam() (*SndRcvInfo, error) { + info := &SndRcvInfo{} + optlen := unsafe.Sizeof(*info) + _, _, err := getsockopt(c.fd(), SCTP_DEFAULT_SENT_PARAM, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(&optlen))) + return info, err +} + +func resolveFromRawAddr(ptr unsafe.Pointer, n int) (*SCTPAddr, error) { + addr := &SCTPAddr{ + IP: make([]net.IP, n), + } + + switch family := (*(*syscall.RawSockaddrAny)(ptr)).Addr.Family; family { + case syscall.AF_INET: + addr.Port = int(ntohs(uint16((*(*syscall.RawSockaddrInet4)(ptr)).Port))) + tmp := syscall.RawSockaddrInet4{} + size := unsafe.Sizeof(tmp) + for i := 0; i < n; i++ { + a := *(*syscall.RawSockaddrInet4)(unsafe.Pointer( + uintptr(ptr) + size*uintptr(i))) + addr.IP[i] = a.Addr[:] + } + case syscall.AF_INET6: + addr.Port = int(ntohs(uint16((*(*syscall.RawSockaddrInet4)(ptr)).Port))) + tmp := syscall.RawSockaddrInet6{} + size := unsafe.Sizeof(tmp) + for i := 0; i < n; i++ { + a := *(*syscall.RawSockaddrInet6)(unsafe.Pointer( + uintptr(ptr) + size*uintptr(i))) + addr.IP[i] = a.Addr[:] + } + default: + return nil, fmt.Errorf("unknown address family: %d", family) + } + return addr, nil +} + +func sctpGetAddrs(fd, id, optname int) (*SCTPAddr, error) { + + type getaddrs struct { + assocId int32 + addrNum uint32 + addrs [4096]byte + } + param := getaddrs{ + assocId: int32(id), + } + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(fd, uintptr(optname), uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err != nil { + return nil, err + } + return resolveFromRawAddr(unsafe.Pointer(¶m.addrs), int(param.addrNum)) +} + +func (c *SCTPConn) SCTPGetPrimaryPeerAddr() (*SCTPAddr, error) { + + type sctpGetSetPrim struct { + assocId int32 + addrs [128]byte + } + param := sctpGetSetPrim{ + assocId: int32(0), + } + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(c.fd(), SCTP_PRIMARY_ADDR, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err != nil { + return nil, err + } + return resolveFromRawAddr(unsafe.Pointer(¶m.addrs), 1) +} + +func (c *SCTPConn) SCTPLocalAddr(id int) (*SCTPAddr, error) { + return sctpGetAddrs(c.fd(), id, SCTP_GET_LOCAL_ADDRS) +} + +func (c *SCTPConn) SCTPRemoteAddr(id int) (*SCTPAddr, error) { + return sctpGetAddrs(c.fd(), id, SCTP_GET_PEER_ADDRS) +} + +func (c *SCTPConn) LocalAddr() net.Addr { + addr, err := sctpGetAddrs(c.fd(), 0, SCTP_GET_LOCAL_ADDRS) + if err != nil { + return nil + } + return addr +} + +func (c *SCTPConn) RemoteAddr() net.Addr { + addr, err := sctpGetAddrs(c.fd(), 0, SCTP_GET_PEER_ADDRS) + if err != nil { + return nil + } + return addr +} + +func (c *SCTPConn) PeelOff(id int) (*SCTPConn, error) { + type peeloffArg struct { + assocId int32 + sd int + } + param := peeloffArg{ + assocId: int32(id), + } + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(c.fd(), SCTP_SOCKOPT_PEELOFF, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err != nil { + return nil, err + } + return &SCTPConn{_fd: int32(param.sd)}, nil +} + +func (c *SCTPConn) SetDeadline(t time.Time) error { + return syscall.EOPNOTSUPP +} + +func (c *SCTPConn) SetReadDeadline(t time.Time) error { + return syscall.EOPNOTSUPP +} + +func (c *SCTPConn) SetWriteDeadline(t time.Time) error { + return syscall.EOPNOTSUPP +} + +type SCTPListener struct { + fd int + m sync.Mutex +} + +func (ln *SCTPListener) Addr() net.Addr { + laddr, err := sctpGetAddrs(ln.fd, 0, SCTP_GET_LOCAL_ADDRS) + if err != nil { + return nil + } + return laddr +} + +type SCTPSndRcvInfoWrappedConn struct { + conn *SCTPConn +} + +func NewSCTPSndRcvInfoWrappedConn(conn *SCTPConn) *SCTPSndRcvInfoWrappedConn { + conn.SubscribeEvents(SCTP_EVENT_DATA_IO) + return &SCTPSndRcvInfoWrappedConn{conn} +} + +func (c *SCTPSndRcvInfoWrappedConn) Write(b []byte) (int, error) { + if len(b) < int(sndRcvInfoSize) { + return 0, syscall.EINVAL + } + info := (*SndRcvInfo)(unsafe.Pointer(&b[0])) + n, err := c.conn.SCTPWrite(b[sndRcvInfoSize:], info) + return n + int(sndRcvInfoSize), err +} + +func (c *SCTPSndRcvInfoWrappedConn) Read(b []byte) (int, error) { + if len(b) < int(sndRcvInfoSize) { + return 0, syscall.EINVAL + } + n, info, err := c.conn.SCTPRead(b[sndRcvInfoSize:]) + if err != nil { + return n, err + } + copy(b, toBuf(info)) + return n + int(sndRcvInfoSize), err +} + +func (c *SCTPSndRcvInfoWrappedConn) Close() error { + return c.conn.Close() +} + +func (c *SCTPSndRcvInfoWrappedConn) LocalAddr() net.Addr { + return c.conn.LocalAddr() +} + +func (c *SCTPSndRcvInfoWrappedConn) RemoteAddr() net.Addr { + return c.conn.RemoteAddr() +} + +func (c *SCTPSndRcvInfoWrappedConn) SetDeadline(t time.Time) error { + return c.conn.SetDeadline(t) +} + +func (c *SCTPSndRcvInfoWrappedConn) SetReadDeadline(t time.Time) error { + return c.conn.SetReadDeadline(t) +} + +func (c *SCTPSndRcvInfoWrappedConn) SetWriteDeadline(t time.Time) error { + return c.conn.SetWriteDeadline(t) +} diff --git a/vendor/github.com/ishidawataru/sctp/sctp_linux.go b/vendor/github.com/ishidawataru/sctp/sctp_linux.go new file mode 100644 index 000000000000..f93ab8622ac9 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/sctp_linux.go @@ -0,0 +1,227 @@ +// +build linux,!386 + +package sctp + +import ( + "fmt" + "io" + "net" + "sync/atomic" + "syscall" + "unsafe" +) + +func setsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { + // FIXME: syscall.SYS_SETSOCKOPT is undefined on 386 + r0, r1, errno := syscall.Syscall6(syscall.SYS_SETSOCKOPT, + uintptr(fd), + SOL_SCTP, + optname, + optval, + optlen, + 0) + if errno != 0 { + return r0, r1, errno + } + return r0, r1, nil +} + +func getsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { + // FIXME: syscall.SYS_GETSOCKOPT is undefined on 386 + r0, r1, errno := syscall.Syscall6(syscall.SYS_GETSOCKOPT, + uintptr(fd), + SOL_SCTP, + optname, + optval, + optlen, + 0) + if errno != 0 { + return r0, r1, errno + } + return r0, r1, nil +} + +func (c *SCTPConn) SCTPWrite(b []byte, info *SndRcvInfo) (int, error) { + var cbuf []byte + if info != nil { + cmsgBuf := toBuf(info) + hdr := &syscall.Cmsghdr{ + Level: syscall.IPPROTO_SCTP, + Type: SCTP_CMSG_SNDRCV, + } + + // bitwidth of hdr.Len is platform-specific, + // so we use hdr.SetLen() rather than directly setting hdr.Len + hdr.SetLen(syscall.CmsgSpace(len(cmsgBuf))) + cbuf = append(toBuf(hdr), cmsgBuf...) + } + return syscall.SendmsgN(c.fd(), b, cbuf, nil, 0) +} + +func parseSndRcvInfo(b []byte) (*SndRcvInfo, error) { + msgs, err := syscall.ParseSocketControlMessage(b) + if err != nil { + return nil, err + } + for _, m := range msgs { + if m.Header.Level == syscall.IPPROTO_SCTP { + switch m.Header.Type { + case SCTP_CMSG_SNDRCV: + return (*SndRcvInfo)(unsafe.Pointer(&m.Data[0])), nil + } + } + } + return nil, nil +} + +func (c *SCTPConn) SCTPRead(b []byte) (int, *SndRcvInfo, error) { + oob := make([]byte, 254) + for { + n, oobn, recvflags, _, err := syscall.Recvmsg(c.fd(), b, oob, 0) + if err != nil { + return n, nil, err + } + + if n == 0 && oobn == 0 { + return 0, nil, io.EOF + } + + if recvflags&MSG_NOTIFICATION > 0 && c.notificationHandler != nil { + if err := c.notificationHandler(b[:n]); err != nil { + return 0, nil, err + } + } else { + var info *SndRcvInfo + if oobn > 0 { + info, err = parseSndRcvInfo(oob[:oobn]) + } + return n, info, err + } + } +} + +func (c *SCTPConn) Close() error { + if c != nil { + fd := atomic.SwapInt32(&c._fd, -1) + if fd > 0 { + info := &SndRcvInfo{ + Flags: SCTP_EOF, + } + c.SCTPWrite(nil, info) + syscall.Shutdown(int(fd), syscall.SHUT_RDWR) + return syscall.Close(int(fd)) + } + } + return syscall.EBADF +} + +func ListenSCTP(net string, laddr *SCTPAddr) (*SCTPListener, error) { + af := syscall.AF_INET + switch net { + case "sctp": + hasv6 := func(addr *SCTPAddr) bool { + if addr == nil { + return false + } + for _, ip := range addr.IP { + if ip.To4() == nil { + return true + } + } + return false + } + if hasv6(laddr) { + af = syscall.AF_INET6 + } + case "sctp4": + case "sctp6": + af = syscall.AF_INET6 + default: + return nil, fmt.Errorf("invalid net: %s", net) + } + + sock, err := syscall.Socket( + af, + syscall.SOCK_STREAM, + syscall.IPPROTO_SCTP, + ) + if err != nil { + return nil, err + } + err = setNumOstreams(sock, SCTP_MAX_STREAM) + if err != nil { + return nil, err + } + if laddr != nil && len(laddr.IP) != 0 { + err := SCTPBind(sock, laddr, SCTP_BINDX_ADD_ADDR) + if err != nil { + return nil, err + } + } + err = syscall.Listen(sock, syscall.SOMAXCONN) + if err != nil { + return nil, err + } + return &SCTPListener{ + fd: sock, + }, nil +} + +func (ln *SCTPListener) Accept() (net.Conn, error) { + fd, _, err := syscall.Accept4(ln.fd, 0) + return NewSCTPConn(fd, nil), err +} + +func (ln *SCTPListener) Close() error { + syscall.Shutdown(ln.fd, syscall.SHUT_RDWR) + return syscall.Close(ln.fd) +} + +func DialSCTP(net string, laddr, raddr *SCTPAddr) (*SCTPConn, error) { + af := syscall.AF_INET + switch net { + case "sctp": + hasv6 := func(addr *SCTPAddr) bool { + if addr == nil { + return false + } + for _, ip := range addr.IP { + if ip.To4() == nil { + return true + } + } + return false + } + if hasv6(laddr) || hasv6(raddr) { + af = syscall.AF_INET6 + } + case "sctp4": + case "sctp6": + af = syscall.AF_INET6 + default: + return nil, fmt.Errorf("invalid net: %s", net) + } + sock, err := syscall.Socket( + af, + syscall.SOCK_STREAM, + syscall.IPPROTO_SCTP, + ) + if err != nil { + return nil, err + } + err = setNumOstreams(sock, SCTP_MAX_STREAM) + if err != nil { + return nil, err + } + if laddr != nil { + err := SCTPBind(sock, laddr, SCTP_BINDX_ADD_ADDR) + if err != nil { + return nil, err + } + } + _, err = SCTPConnect(sock, raddr) + if err != nil { + return nil, err + } + return NewSCTPConn(sock, nil), nil +} diff --git a/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go b/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go new file mode 100644 index 000000000000..adcbf78b46fb --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go @@ -0,0 +1,47 @@ +// +build !linux linux,386 + +package sctp + +import ( + "errors" + "net" + "runtime" +) + +var ErrUnsupported = errors.New("SCTP is unsupported on " + runtime.GOOS + "/" + runtime.GOARCH) + +func setsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { + return 0, 0, ErrUnsupported +} + +func getsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { + return 0, 0, ErrUnsupported +} + +func (c *SCTPConn) SCTPWrite(b []byte, info *SndRcvInfo) (int, error) { + return 0, ErrUnsupported +} + +func (c *SCTPConn) SCTPRead(b []byte) (int, *SndRcvInfo, error) { + return 0, nil, ErrUnsupported +} + +func (c *SCTPConn) Close() error { + return ErrUnsupported +} + +func ListenSCTP(net string, laddr *SCTPAddr) (*SCTPListener, error) { + return nil, ErrUnsupported +} + +func (ln *SCTPListener) Accept() (net.Conn, error) { + return nil, ErrUnsupported +} + +func (ln *SCTPListener) Close() error { + return ErrUnsupported +} + +func DialSCTP(net string, laddr, raddr *SCTPAddr) (*SCTPConn, error) { + return nil, ErrUnsupported +} diff --git a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go index 7781c5361735..f32698cab22e 100644 --- a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go +++ b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go @@ -503,6 +503,8 @@ type WindowsNetwork struct { DNSSearchList []string `json:"DNSSearchList,omitempty"` // Name (ID) of the container that we will share with the network stack. NetworkSharedContainerName string `json:"networkSharedContainerName,omitempty"` + // name (ID) of the network namespace that will be used for the container. + NetworkNamespace string `json:"networkNamespace,omitempty"` } // WindowsHyperV contains information for configuring a container to run with Hyper-V isolation. diff --git a/vendor/github.com/boltdb/bolt/LICENSE b/vendor/go.etcd.io/bbolt/LICENSE similarity index 100% rename from vendor/github.com/boltdb/bolt/LICENSE rename to vendor/go.etcd.io/bbolt/LICENSE diff --git a/vendor/github.com/boltdb/bolt/README.md b/vendor/go.etcd.io/bbolt/README.md similarity index 90% rename from vendor/github.com/boltdb/bolt/README.md rename to vendor/go.etcd.io/bbolt/README.md index d7f80e97747a..6546fc01ac5a 100644 --- a/vendor/github.com/boltdb/bolt/README.md +++ b/vendor/go.etcd.io/bbolt/README.md @@ -1,5 +1,18 @@ -Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg) -==== +bbolt +===== + +[![Go Report Card](https://goreportcard.com/badge/github.com/etcd-io/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/etcd-io/bbolt) +[![Coverage](https://codecov.io/gh/etcd-io/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/etcd-io/bbolt) +[![Build Status Travis](https://img.shields.io/travis/etcd-io/bboltlabs.svg?style=flat-square&&branch=master)](https://travis-ci.com/etcd-io/bbolt) +[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/etcd-io/bbolt) +[![Releases](https://img.shields.io/github/release/etcd-io/bbolt/all.svg?style=flat-square)](https://github.com/etcd-io/bbolt/releases) +[![LICENSE](https://img.shields.io/github/license/etcd-io/bbolt.svg?style=flat-square)](https://github.com/etcd-io/bbolt/blob/master/LICENSE) + +bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value +store. The purpose of this fork is to provide the Go community with an active +maintenance and development target for Bolt; the goal is improved reliability +and stability. bbolt includes bug fixes, performance enhancements, and features +not found in Bolt while preserving backwards compatibility with the Bolt API. Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] [LMDB project][lmdb]. The goal of the project is to provide a simple, @@ -10,6 +23,8 @@ Since Bolt is meant to be used as such a low-level piece of functionality, simplicity is key. The API will be small and only focus on getting values and setting values. That's it. +[gh_ben]: https://github.com/benbjohnson +[bolt]: https://github.com/boltdb/bolt [hyc_symas]: https://twitter.com/hyc_symas [lmdb]: http://symas.com/mdb/ @@ -21,36 +36,42 @@ consistency and thread safety. Bolt is currently used in high-load production environments serving databases as large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed services every day. +## Project versioning + +bbolt uses [semantic versioning](http://semver.org). +API should not change between patch and minor releases. +New minor versions may add additional features to the API. + ## Table of Contents -- [Getting Started](#getting-started) - - [Installing](#installing) - - [Opening a database](#opening-a-database) - - [Transactions](#transactions) - - [Read-write transactions](#read-write-transactions) - - [Read-only transactions](#read-only-transactions) - - [Batch read-write transactions](#batch-read-write-transactions) - - [Managing transactions manually](#managing-transactions-manually) - - [Using buckets](#using-buckets) - - [Using key/value pairs](#using-keyvalue-pairs) - - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) - - [Iterating over keys](#iterating-over-keys) - - [Prefix scans](#prefix-scans) - - [Range scans](#range-scans) - - [ForEach()](#foreach) - - [Nested buckets](#nested-buckets) - - [Database backups](#database-backups) - - [Statistics](#statistics) - - [Read-Only Mode](#read-only-mode) - - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) -- [Resources](#resources) -- [Comparison with other databases](#comparison-with-other-databases) - - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) - - [LevelDB, RocksDB](#leveldb-rocksdb) - - [LMDB](#lmdb) -- [Caveats & Limitations](#caveats--limitations) -- [Reading the Source](#reading-the-source) -- [Other Projects Using Bolt](#other-projects-using-bolt) + - [Getting Started](#getting-started) + - [Installing](#installing) + - [Opening a database](#opening-a-database) + - [Transactions](#transactions) + - [Read-write transactions](#read-write-transactions) + - [Read-only transactions](#read-only-transactions) + - [Batch read-write transactions](#batch-read-write-transactions) + - [Managing transactions manually](#managing-transactions-manually) + - [Using buckets](#using-buckets) + - [Using key/value pairs](#using-keyvalue-pairs) + - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) + - [Iterating over keys](#iterating-over-keys) + - [Prefix scans](#prefix-scans) + - [Range scans](#range-scans) + - [ForEach()](#foreach) + - [Nested buckets](#nested-buckets) + - [Database backups](#database-backups) + - [Statistics](#statistics) + - [Read-Only Mode](#read-only-mode) + - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) + - [Resources](#resources) + - [Comparison with other databases](#comparison-with-other-databases) + - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) + - [LevelDB, RocksDB](#leveldb-rocksdb) + - [LMDB](#lmdb) + - [Caveats & Limitations](#caveats--limitations) + - [Reading the Source](#reading-the-source) + - [Other Projects Using Bolt](#other-projects-using-bolt) ## Getting Started @@ -59,13 +80,28 @@ Shopify and Heroku use Bolt-backed services every day. To start using Bolt, install Go and run `go get`: ```sh -$ go get github.com/boltdb/bolt/... +$ go get go.etcd.io/bbolt/... ``` This will retrieve the library and install the `bolt` command line utility into your `$GOBIN` path. +### Importing bbolt + +To use bbolt as an embedded key-value store, import as: + +```go +import bolt "go.etcd.io/bbolt" + +db, err := bolt.Open(path, 0666, nil) +if err != nil { + return err +} +defer db.Close() +``` + + ### Opening a database The top-level object in Bolt is a `DB`. It is represented as a single file on @@ -79,7 +115,7 @@ package main import ( "log" - "github.com/boltdb/bolt" + bolt "go.etcd.io/bbolt" ) func main() { @@ -522,7 +558,7 @@ this from a read-only transaction, it will perform a hot backup and not block your other database reads and writes. By default, it will use a regular file handle which will utilize the operating -system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx) +system's page cache. See the [`Tx`](https://godoc.org/go.etcd.io/bbolt#Tx) documentation for information about optimizing for larger-than-RAM datasets. One common use case is to backup over HTTP so you can use tools like `cURL` to @@ -811,7 +847,7 @@ Here are a few things to note when evaluating and using Bolt: ## Reading the Source -Bolt is a relatively small code base (<3KLOC) for an embedded, serializable, +Bolt is a relatively small code base (<5KLOC) for an embedded, serializable, transactional key/value database so it can be a good starting point for people interested in how databases work. @@ -863,53 +899,55 @@ them via pull request. Below is a list of public, open source projects that use Bolt: -* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. -* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. -* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. -* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. -* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. -* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. +* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. +* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support. +* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB +* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. +* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. +* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. +* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. +* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining + simple tx and key scans. +* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. * [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations. -* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. -* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". +* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. +* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. +* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. +* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. * [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka. +* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. +* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. +* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains +* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". +* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. +* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. * [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed. -* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. -* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. +* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies +* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. * [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage. -* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. -* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. -* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. -* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. -* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. -* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. -* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. +* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. +* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. +* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. +* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. * [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system. -* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. -* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. -* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. +* [reef-pi](https://github.com/reef-pi/reef-pi) - reef-pi is an award winning, modular, DIY reef tank controller using easy to learn electronics based on a Raspberry Pi. +* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service +* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. * [stow](https://github.com/djherbis/stow) - a persistence manager for objects backed by boltdb. -* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining - simple tx and key scans. -* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. -* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service -* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. -* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. -* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. * [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. -* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. * [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. -* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. -* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. -* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. +* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. +* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. * [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. -* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains -* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. -* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. -* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. -* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies -* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB +* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/boltdb/bolt/bolt_386.go b/vendor/go.etcd.io/bbolt/bolt_386.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_386.go rename to vendor/go.etcd.io/bbolt/bolt_386.go index 820d533c15fa..4d35ee7cf3d2 100644 --- a/vendor/github.com/boltdb/bolt/bolt_386.go +++ b/vendor/go.etcd.io/bbolt/bolt_386.go @@ -1,4 +1,4 @@ -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB diff --git a/vendor/github.com/boltdb/bolt/bolt_amd64.go b/vendor/go.etcd.io/bbolt/bolt_amd64.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_amd64.go rename to vendor/go.etcd.io/bbolt/bolt_amd64.go index 98fafdb47d86..60a52dad56b2 100644 --- a/vendor/github.com/boltdb/bolt/bolt_amd64.go +++ b/vendor/go.etcd.io/bbolt/bolt_amd64.go @@ -1,4 +1,4 @@ -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_arm.go b/vendor/go.etcd.io/bbolt/bolt_arm.go similarity index 98% rename from vendor/github.com/boltdb/bolt/bolt_arm.go rename to vendor/go.etcd.io/bbolt/bolt_arm.go index 7e5cb4b94128..105d27ddb7dd 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm.go +++ b/vendor/go.etcd.io/bbolt/bolt_arm.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import "unsafe" diff --git a/vendor/github.com/boltdb/bolt/bolt_arm64.go b/vendor/go.etcd.io/bbolt/bolt_arm64.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_arm64.go rename to vendor/go.etcd.io/bbolt/bolt_arm64.go index b26d84f91ba7..f5aa2a5ee248 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm64.go +++ b/vendor/go.etcd.io/bbolt/bolt_arm64.go @@ -1,6 +1,6 @@ // +build arm64 -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_linux.go b/vendor/go.etcd.io/bbolt/bolt_linux.go similarity index 91% rename from vendor/github.com/boltdb/bolt/bolt_linux.go rename to vendor/go.etcd.io/bbolt/bolt_linux.go index 2b6766614090..7707bcacf03a 100644 --- a/vendor/github.com/boltdb/bolt/bolt_linux.go +++ b/vendor/go.etcd.io/bbolt/bolt_linux.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "syscall" diff --git a/vendor/go.etcd.io/bbolt/bolt_mips64x.go b/vendor/go.etcd.io/bbolt/bolt_mips64x.go new file mode 100644 index 000000000000..baeb289fd94b --- /dev/null +++ b/vendor/go.etcd.io/bbolt/bolt_mips64x.go @@ -0,0 +1,12 @@ +// +build mips64 mips64le + +package bbolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x8000000000 // 512GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_mipsx.go b/vendor/go.etcd.io/bbolt/bolt_mipsx.go new file mode 100644 index 000000000000..2d9b1a91f364 --- /dev/null +++ b/vendor/go.etcd.io/bbolt/bolt_mipsx.go @@ -0,0 +1,12 @@ +// +build mips mipsle + +package bbolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x40000000 // 1GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_openbsd.go b/vendor/go.etcd.io/bbolt/bolt_openbsd.go similarity index 97% rename from vendor/github.com/boltdb/bolt/bolt_openbsd.go rename to vendor/go.etcd.io/bbolt/bolt_openbsd.go index 7058c3d734ef..d7f50358ef14 100644 --- a/vendor/github.com/boltdb/bolt/bolt_openbsd.go +++ b/vendor/go.etcd.io/bbolt/bolt_openbsd.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "syscall" diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc.go b/vendor/go.etcd.io/bbolt/bolt_ppc.go similarity index 69% rename from vendor/github.com/boltdb/bolt/bolt_ppc.go rename to vendor/go.etcd.io/bbolt/bolt_ppc.go index 645ddc3edc2d..69804714aaef 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc.go @@ -1,9 +1,12 @@ // +build ppc -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64.go b/vendor/go.etcd.io/bbolt/bolt_ppc64.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_ppc64.go rename to vendor/go.etcd.io/bbolt/bolt_ppc64.go index 9331d9771eb3..356590857607 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc64.go @@ -1,6 +1,6 @@ // +build ppc64 -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go b/vendor/go.etcd.io/bbolt/bolt_ppc64le.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_ppc64le.go rename to vendor/go.etcd.io/bbolt/bolt_ppc64le.go index 8c143bc5d194..422c7c69d667 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc64le.go @@ -1,6 +1,6 @@ // +build ppc64le -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_s390x.go b/vendor/go.etcd.io/bbolt/bolt_s390x.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_s390x.go rename to vendor/go.etcd.io/bbolt/bolt_s390x.go index d7c39af92539..6d3fcb825d34 100644 --- a/vendor/github.com/boltdb/bolt/bolt_s390x.go +++ b/vendor/go.etcd.io/bbolt/bolt_s390x.go @@ -1,6 +1,6 @@ // +build s390x -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_unix.go b/vendor/go.etcd.io/bbolt/bolt_unix.go similarity index 71% rename from vendor/github.com/boltdb/bolt/bolt_unix.go rename to vendor/go.etcd.io/bbolt/bolt_unix.go index cad62dda1e38..5f2bb5145199 100644 --- a/vendor/github.com/boltdb/bolt/bolt_unix.go +++ b/vendor/go.etcd.io/bbolt/bolt_unix.go @@ -1,41 +1,43 @@ // +build !windows,!plan9,!solaris -package bolt +package bbolt import ( "fmt" - "os" "syscall" "time" "unsafe" ) // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + fd := db.file.Fd() + flag := syscall.LOCK_NB + if exclusive { + flag |= syscall.LOCK_EX + } else { + flag |= syscall.LOCK_SH + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } - flag := syscall.LOCK_SH - if exclusive { - flag = syscall.LOCK_EX - } - - // Otherwise attempt to obtain an exclusive lock. - err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB) + // Attempt to obtain an exclusive lock. + err := syscall.Flock(int(fd), flag) if err == nil { return nil } else if err != syscall.EWOULDBLOCK { return err } + // If we timed out then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } @@ -53,7 +55,9 @@ func mmap(db *DB, sz int) error { } // Advise the kernel that the mmap is accessed randomly. - if err := madvise(b, syscall.MADV_RANDOM); err != nil { + err = madvise(b, syscall.MADV_RANDOM) + if err != nil && err != syscall.ENOSYS { + // Ignore not implemented error in kernel because it still works. return fmt.Errorf("madvise: %s", err) } diff --git a/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go b/vendor/go.etcd.io/bbolt/bolt_unix_solaris.go similarity index 70% rename from vendor/github.com/boltdb/bolt/bolt_unix_solaris.go rename to vendor/go.etcd.io/bbolt/bolt_unix_solaris.go index 307bf2b3ee97..babad657862b 100644 --- a/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go +++ b/vendor/go.etcd.io/bbolt/bolt_unix_solaris.go @@ -1,8 +1,7 @@ -package bolt +package bbolt import ( "fmt" - "os" "syscall" "time" "unsafe" @@ -11,36 +10,35 @@ import ( ) // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + fd := db.file.Fd() + var lockType int16 + if exclusive { + lockType = syscall.F_WRLCK + } else { + lockType = syscall.F_RDLCK + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } - var lock syscall.Flock_t - lock.Start = 0 - lock.Len = 0 - lock.Pid = 0 - lock.Whence = 0 - lock.Pid = 0 - if exclusive { - lock.Type = syscall.F_WRLCK - } else { - lock.Type = syscall.F_RDLCK - } - err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock) + // Attempt to obtain an exclusive lock. + lock := syscall.Flock_t{Type: lockType} + err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock) if err == nil { return nil } else if err != syscall.EAGAIN { return err } + // If we timed out then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } diff --git a/vendor/github.com/boltdb/bolt/bolt_windows.go b/vendor/go.etcd.io/bbolt/bolt_windows.go similarity index 76% rename from vendor/github.com/boltdb/bolt/bolt_windows.go rename to vendor/go.etcd.io/bbolt/bolt_windows.go index b00fb0720a42..fca178bd2908 100644 --- a/vendor/github.com/boltdb/bolt/bolt_windows.go +++ b/vendor/go.etcd.io/bbolt/bolt_windows.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -16,8 +16,6 @@ var ( ) const ( - lockExt = ".lock" - // see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx flagLockExclusive = 2 flagLockFailImmediately = 1 @@ -48,48 +46,47 @@ func fdatasync(db *DB) error { } // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { - // Create a separate lock file on windows because a process - // cannot share an exclusive lock on the same file. This is - // needed during Tx.WriteTo(). - f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode) - if err != nil { - return err - } - db.lockfile = f - +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + var flag uint32 = flagLockFailImmediately + if exclusive { + flag |= flagLockExclusive + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } - - var flag uint32 = flagLockFailImmediately - if exclusive { - flag |= flagLockExclusive - } + // Fix for https://github.com/etcd-io/bbolt/issues/121. Use byte-range + // -1..0 as the lock on the database file. + var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 + err := lockFileEx(syscall.Handle(db.file.Fd()), flag, 0, 1, 0, &syscall.Overlapped{ + Offset: m1, + OffsetHigh: m1, + }) - err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{}) if err == nil { return nil } else if err != errLockViolation { return err } + // If we timed oumercit then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } // funlock releases an advisory lock on a file descriptor. func funlock(db *DB) error { - err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{}) - db.lockfile.Close() - os.Remove(db.path + lockExt) + var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 + err := unlockFileEx(syscall.Handle(db.file.Fd()), 0, 1, 0, &syscall.Overlapped{ + Offset: m1, + OffsetHigh: m1, + }) return err } diff --git a/vendor/github.com/boltdb/bolt/boltsync_unix.go b/vendor/go.etcd.io/bbolt/boltsync_unix.go similarity index 91% rename from vendor/github.com/boltdb/bolt/boltsync_unix.go rename to vendor/go.etcd.io/bbolt/boltsync_unix.go index f50442523c38..9587afefee40 100644 --- a/vendor/github.com/boltdb/bolt/boltsync_unix.go +++ b/vendor/go.etcd.io/bbolt/boltsync_unix.go @@ -1,6 +1,6 @@ // +build !windows,!plan9,!linux,!openbsd -package bolt +package bbolt // fdatasync flushes written data to a file descriptor. func fdatasync(db *DB) error { diff --git a/vendor/github.com/boltdb/bolt/bucket.go b/vendor/go.etcd.io/bbolt/bucket.go similarity index 99% rename from vendor/github.com/boltdb/bolt/bucket.go rename to vendor/go.etcd.io/bbolt/bucket.go index 0c5bf27463e1..84bfd4d6a283 100644 --- a/vendor/github.com/boltdb/bolt/bucket.go +++ b/vendor/go.etcd.io/bbolt/bucket.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -14,13 +14,6 @@ const ( MaxValueSize = (1 << 31) - 2 ) -const ( - maxUint = ^uint(0) - minUint = 0 - maxInt = int(^uint(0) >> 1) - minInt = -maxInt - 1 -) - const bucketHeaderSize = int(unsafe.Sizeof(bucket{})) const ( @@ -323,7 +316,12 @@ func (b *Bucket) Delete(key []byte) error { // Move cursor to correct position. c := b.Cursor() - _, _, flags := c.seek(key) + k, _, flags := c.seek(key) + + // Return nil if the key doesn't exist. + if !bytes.Equal(key, k) { + return nil + } // Return an error if there is already existing bucket value. if (flags & bucketLeafFlag) != 0 { diff --git a/vendor/github.com/boltdb/bolt/cursor.go b/vendor/go.etcd.io/bbolt/cursor.go similarity index 99% rename from vendor/github.com/boltdb/bolt/cursor.go rename to vendor/go.etcd.io/bbolt/cursor.go index 1be9f35e3ef8..3000aced6c4c 100644 --- a/vendor/github.com/boltdb/bolt/cursor.go +++ b/vendor/go.etcd.io/bbolt/cursor.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -157,12 +157,6 @@ func (c *Cursor) seek(seek []byte) (key []byte, value []byte, flags uint32) { // Start from root page/node and traverse to correct page. c.stack = c.stack[:0] c.search(seek, c.bucket.root) - ref := &c.stack[len(c.stack)-1] - - // If the cursor is pointing to the end of page/node then return nil. - if ref.index >= ref.count() { - return nil, nil, 0 - } // If this is a bucket then return a nil value. return c.keyValue() @@ -339,6 +333,8 @@ func (c *Cursor) nsearch(key []byte) { // keyValue returns the key and value of the current leaf element. func (c *Cursor) keyValue() ([]byte, []byte, uint32) { ref := &c.stack[len(c.stack)-1] + + // If the cursor is pointing to the end of page/node then return nil. if ref.count() == 0 || ref.index >= ref.count() { return nil, nil, 0 } diff --git a/vendor/github.com/boltdb/bolt/db.go b/vendor/go.etcd.io/bbolt/db.go similarity index 84% rename from vendor/github.com/boltdb/bolt/db.go rename to vendor/go.etcd.io/bbolt/db.go index f352ff14fe40..d91dcf813a14 100644 --- a/vendor/github.com/boltdb/bolt/db.go +++ b/vendor/go.etcd.io/bbolt/db.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "errors" @@ -7,8 +7,7 @@ import ( "log" "os" "runtime" - "runtime/debug" - "strings" + "sort" "sync" "time" "unsafe" @@ -23,6 +22,8 @@ const version = 2 // Represents a marker value to indicate that a file is a Bolt DB. const magic uint32 = 0xED0CDAED +const pgidNoFreelist pgid = 0xffffffffffffffff + // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when // syncing changes to a file. This is required as some operating systems, // such as OpenBSD, do not have a unified buffer cache (UBC) and writes @@ -39,6 +40,9 @@ const ( // default page size for db is set to the OS page size. var defaultPageSize = os.Getpagesize() +// The time elapsed between consecutive file locking attempts. +const flockRetryTimeout = 50 * time.Millisecond + // DB represents a collection of buckets persisted to a file on disk. // All data access is performed through transactions which can be obtained through the DB. // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called. @@ -61,6 +65,11 @@ type DB struct { // THIS IS UNSAFE. PLEASE USE WITH CAUTION. NoSync bool + // When true, skips syncing freelist to disk. This improves the database + // write performance under normal operation, but requires a full database + // re-sync during recovery. + NoFreelistSync bool + // When true, skips the truncate call when growing the database. // Setting this to true is only safe on non-ext3/ext4 systems. // Skipping truncation avoids preallocation of hard drive space and @@ -96,8 +105,7 @@ type DB struct { path string file *os.File - lockfile *os.File // windows only - dataref []byte // mmap'ed readonly, write throws SEGV + dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int filesz int // current on disk file size @@ -107,9 +115,11 @@ type DB struct { opened bool rwtx *Tx txs []*Tx - freelist *freelist stats Stats + freelist *freelist + freelistLoad sync.Once + pagePool sync.Pool batchMu sync.Mutex @@ -148,14 +158,17 @@ func (db *DB) String() string { // If the file does not exist then it will be created automatically. // Passing in nil options will cause Bolt to open the database with the default options. func Open(path string, mode os.FileMode, options *Options) (*DB, error) { - var db = &DB{opened: true} - + db := &DB{ + opened: true, + } // Set default options if no options are provided. if options == nil { options = DefaultOptions } + db.NoSync = options.NoSync db.NoGrowSync = options.NoGrowSync db.MmapFlags = options.MmapFlags + db.NoFreelistSync = options.NoFreelistSync // Set default values for later DB operations. db.MaxBatchSize = DefaultMaxBatchSize @@ -183,7 +196,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // if !options.ReadOnly. // The database file is locked using the shared lock (more than one process may // hold a lock at the same time) otherwise (options.ReadOnly is set). - if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil { + if err := flock(db, !db.readOnly, options.Timeout); err != nil { _ = db.close() return nil, err } @@ -191,31 +204,41 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // Default values for test hooks db.ops.writeAt = db.file.WriteAt + if db.pageSize = options.PageSize; db.pageSize == 0 { + // Set the default page size to the OS page size. + db.pageSize = defaultPageSize + } + // Initialize the database if it doesn't exist. if info, err := db.file.Stat(); err != nil { + _ = db.close() return nil, err } else if info.Size() == 0 { // Initialize new files with meta pages. if err := db.init(); err != nil { + // clean up file descriptor on initialization fail + _ = db.close() return nil, err } } else { // Read the first meta page to determine the page size. var buf [0x1000]byte - if _, err := db.file.ReadAt(buf[:], 0); err == nil { - m := db.pageInBuffer(buf[:], 0).meta() - if err := m.validate(); err != nil { - // If we can't read the page size, we can assume it's the same - // as the OS -- since that's how the page size was chosen in the - // first place. - // - // If the first page is invalid and this OS uses a different - // page size than what the database was created with then we - // are out of luck and cannot access the database. - db.pageSize = os.Getpagesize() - } else { + // If we can't read the page size, but can read a page, assume + // it's the same as the OS or one given -- since that's how the + // page size was chosen in the first place. + // + // If the first page is invalid and this OS uses a different + // page size than what the database was created with then we + // are out of luck and cannot access the database. + // + // TODO: scan for next page + if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) { + if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil { db.pageSize = int(m.pageSize) } + } else { + _ = db.close() + return nil, ErrInvalid } } @@ -232,14 +255,50 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { return nil, err } - // Read in the freelist. - db.freelist = newFreelist() - db.freelist.read(db.page(db.meta().freelist)) + if db.readOnly { + return db, nil + } + + db.loadFreelist() + + // Flush freelist when transitioning from no sync to sync so + // NoFreelistSync unaware boltdb can open the db later. + if !db.NoFreelistSync && !db.hasSyncedFreelist() { + tx, err := db.Begin(true) + if tx != nil { + err = tx.Commit() + } + if err != nil { + _ = db.close() + return nil, err + } + } // Mark the database as opened and return. return db, nil } +// loadFreelist reads the freelist if it is synced, or reconstructs it +// by scanning the DB if it is not synced. It assumes there are no +// concurrent accesses being made to the freelist. +func (db *DB) loadFreelist() { + db.freelistLoad.Do(func() { + db.freelist = newFreelist() + if !db.hasSyncedFreelist() { + // Reconstruct free list by scanning the DB. + db.freelist.readIDs(db.freepages()) + } else { + // Read free list from freelist page. + db.freelist.read(db.page(db.meta().freelist)) + } + db.stats.FreePageN = len(db.freelist.ids) + }) +} + +func (db *DB) hasSyncedFreelist() bool { + return db.meta().freelist != pgidNoFreelist +} + // mmap opens the underlying memory-mapped file and initializes the meta references. // minsz is the minimum size that the new mmap can be. func (db *DB) mmap(minsz int) error { @@ -341,9 +400,6 @@ func (db *DB) mmapSize(size int) (int, error) { // init creates a new database file and initializes its meta pages. func (db *DB) init() error { - // Set the page size to the OS page size. - db.pageSize = os.Getpagesize() - // Create two meta pages on a buffer. buf := make([]byte, db.pageSize*4) for i := 0; i < 2; i++ { @@ -387,7 +443,8 @@ func (db *DB) init() error { } // Close releases all database resources. -// All transactions must be closed before closing the database. +// It will block waiting for any open transactions to finish +// before closing the database and returning. func (db *DB) Close() error { db.rwlock.Lock() defer db.rwlock.Unlock() @@ -395,8 +452,8 @@ func (db *DB) Close() error { db.metalock.Lock() defer db.metalock.Unlock() - db.mmaplock.RLock() - defer db.mmaplock.RUnlock() + db.mmaplock.Lock() + defer db.mmaplock.Unlock() return db.close() } @@ -526,21 +583,36 @@ func (db *DB) beginRWTx() (*Tx, error) { t := &Tx{writable: true} t.init(db) db.rwtx = t + db.freePages() + return t, nil +} - // Free any pages associated with closed read-only transactions. - var minid txid = 0xFFFFFFFFFFFFFFFF - for _, t := range db.txs { - if t.meta.txid < minid { - minid = t.meta.txid - } +// freePages releases any pages associated with closed read-only transactions. +func (db *DB) freePages() { + // Free all pending pages prior to earliest open transaction. + sort.Sort(txsById(db.txs)) + minid := txid(0xFFFFFFFFFFFFFFFF) + if len(db.txs) > 0 { + minid = db.txs[0].meta.txid } if minid > 0 { db.freelist.release(minid - 1) } - - return t, nil + // Release unused txid extents. + for _, t := range db.txs { + db.freelist.releaseRange(minid, t.meta.txid-1) + minid = t.meta.txid + 1 + } + db.freelist.releaseRange(minid, txid(0xFFFFFFFFFFFFFFFF)) + // Any page both allocated and freed in an extent is safe to release. } +type txsById []*Tx + +func (t txsById) Len() int { return len(t) } +func (t txsById) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t txsById) Less(i, j int) bool { return t[i].meta.txid < t[j].meta.txid } + // removeTx removes a transaction from the database. func (db *DB) removeTx(tx *Tx) { // Release the read lock on the mmap. @@ -633,11 +705,7 @@ func (db *DB) View(fn func(*Tx) error) error { return err } - if err := t.Rollback(); err != nil { - return err - } - - return nil + return t.Rollback() } // Batch calls fn as part of a batch. It behaves similar to Update, @@ -737,9 +805,7 @@ retry: // pass success, or bolt internal errors, to all callers for _, c := range b.calls { - if c.err != nil { - c.err <- err - } + c.err <- err } break retry } @@ -826,7 +892,7 @@ func (db *DB) meta() *meta { } // allocate returns a contiguous block of memory starting at a given page. -func (db *DB) allocate(count int) (*page, error) { +func (db *DB) allocate(txid txid, count int) (*page, error) { // Allocate a temporary buffer for the page. var buf []byte if count == 1 { @@ -838,7 +904,7 @@ func (db *DB) allocate(count int) (*page, error) { p.overflow = uint32(count - 1) // Use pages from the freelist if they are available. - if p.id = db.freelist.allocate(count); p.id != 0 { + if p.id = db.freelist.allocate(txid, count); p.id != 0 { return p, nil } @@ -893,6 +959,38 @@ func (db *DB) IsReadOnly() bool { return db.readOnly } +func (db *DB) freepages() []pgid { + tx, err := db.beginTx() + defer func() { + err = tx.Rollback() + if err != nil { + panic("freepages: failed to rollback tx") + } + }() + if err != nil { + panic("freepages: failed to open read only tx") + } + + reachable := make(map[pgid]*page) + nofreed := make(map[pgid]bool) + ech := make(chan error) + go func() { + for e := range ech { + panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e)) + } + }() + tx.checkBucket(&tx.root, reachable, nofreed, ech) + close(ech) + + var fids []pgid + for i := pgid(2); i < db.meta().pgid; i++ { + if _, ok := reachable[i]; !ok { + fids = append(fids, i) + } + } + return fids +} + // Options represents the options that can be set when opening a database. type Options struct { // Timeout is the amount of time to wait to obtain a file lock. @@ -903,6 +1001,10 @@ type Options struct { // Sets the DB.NoGrowSync flag before memory mapping the file. NoGrowSync bool + // Do not sync freelist to disk. This improves the database write performance + // under normal operation, but requires a full database re-sync during recovery. + NoFreelistSync bool + // Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to // grab a shared lock (UNIX). ReadOnly bool @@ -919,6 +1021,14 @@ type Options struct { // If initialMmapSize is smaller than the previous database size, // it takes no effect. InitialMmapSize int + + // PageSize overrides the default OS page size. + PageSize int + + // NoSync sets the initial value of DB.NoSync. Normally this can just be + // set directly on the DB itself when returned from Open(), but this option + // is useful in APIs which expose Options but not the underlying DB. + NoSync bool } // DefaultOptions represent the options used if nil options are passed into Open(). @@ -960,10 +1070,6 @@ func (s *Stats) Sub(other *Stats) Stats { return diff } -func (s *Stats) add(other *Stats) { - s.TxStats.add(&other.TxStats) -} - type Info struct { Data uintptr PageSize int @@ -1002,7 +1108,8 @@ func (m *meta) copy(dest *meta) { func (m *meta) write(p *page) { if m.root.root >= m.pgid { panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid)) - } else if m.freelist >= m.pgid { + } else if m.freelist >= m.pgid && m.freelist != pgidNoFreelist { + // TODO: reject pgidNoFreeList if !NoFreelistSync panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid)) } @@ -1029,11 +1136,3 @@ func _assert(condition bool, msg string, v ...interface{}) { panic(fmt.Sprintf("assertion failed: "+msg, v...)) } } - -func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } -func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) } - -func printstack() { - stack := strings.Join(strings.Split(string(debug.Stack()), "\n")[2:], "\n") - fmt.Fprintln(os.Stderr, stack) -} diff --git a/vendor/github.com/boltdb/bolt/doc.go b/vendor/go.etcd.io/bbolt/doc.go similarity index 94% rename from vendor/github.com/boltdb/bolt/doc.go rename to vendor/go.etcd.io/bbolt/doc.go index cc937845dba0..95f25f01c6a4 100644 --- a/vendor/github.com/boltdb/bolt/doc.go +++ b/vendor/go.etcd.io/bbolt/doc.go @@ -1,5 +1,5 @@ /* -Package bolt implements a low-level key/value store in pure Go. It supports +package bbolt implements a low-level key/value store in pure Go. It supports fully serializable transactions, ACID semantics, and lock-free MVCC with multiple readers and a single writer. Bolt can be used for projects that want a simple data store without the need to add large dependencies such as @@ -41,4 +41,4 @@ point to different data or can point to invalid memory which will cause a panic. */ -package bolt +package bbolt diff --git a/vendor/github.com/boltdb/bolt/errors.go b/vendor/go.etcd.io/bbolt/errors.go similarity index 99% rename from vendor/github.com/boltdb/bolt/errors.go rename to vendor/go.etcd.io/bbolt/errors.go index a3620a3ebb29..48758ca5770b 100644 --- a/vendor/github.com/boltdb/bolt/errors.go +++ b/vendor/go.etcd.io/bbolt/errors.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import "errors" diff --git a/vendor/github.com/boltdb/bolt/freelist.go b/vendor/go.etcd.io/bbolt/freelist.go similarity index 65% rename from vendor/github.com/boltdb/bolt/freelist.go rename to vendor/go.etcd.io/bbolt/freelist.go index aba48f58c628..e4bcb2dcf959 100644 --- a/vendor/github.com/boltdb/bolt/freelist.go +++ b/vendor/go.etcd.io/bbolt/freelist.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -6,18 +6,28 @@ import ( "unsafe" ) +// txPending holds a list of pgids and corresponding allocation txns +// that are pending to be freed. +type txPending struct { + ids []pgid + alloctx []txid // txids allocating the ids + lastReleaseBegin txid // beginning txid of last matching releaseRange +} + // freelist represents a list of all pages that are available for allocation. // It also tracks pages that have been freed but are still in use by open transactions. type freelist struct { - ids []pgid // all free and available free page ids. - pending map[txid][]pgid // mapping of soon-to-be free page ids by tx. - cache map[pgid]bool // fast lookup of all free and pending page ids. + ids []pgid // all free and available free page ids. + allocs map[pgid]txid // mapping of txid that allocated a pgid. + pending map[txid]*txPending // mapping of soon-to-be free page ids by tx. + cache map[pgid]bool // fast lookup of all free and pending page ids. } // newFreelist returns an empty, initialized freelist. func newFreelist() *freelist { return &freelist{ - pending: make(map[txid][]pgid), + allocs: make(map[pgid]txid), + pending: make(map[txid]*txPending), cache: make(map[pgid]bool), } } @@ -45,8 +55,8 @@ func (f *freelist) free_count() int { // pending_count returns count of pending pages func (f *freelist) pending_count() int { var count int - for _, list := range f.pending { - count += len(list) + for _, txp := range f.pending { + count += len(txp.ids) } return count } @@ -55,8 +65,8 @@ func (f *freelist) pending_count() int { // f.count returns the minimum length required for dst. func (f *freelist) copyall(dst []pgid) { m := make(pgids, 0, f.pending_count()) - for _, list := range f.pending { - m = append(m, list...) + for _, txp := range f.pending { + m = append(m, txp.ids...) } sort.Sort(m) mergepgids(dst, f.ids, m) @@ -64,7 +74,7 @@ func (f *freelist) copyall(dst []pgid) { // allocate returns the starting page id of a contiguous list of pages of a given size. // If a contiguous block cannot be found then 0 is returned. -func (f *freelist) allocate(n int) pgid { +func (f *freelist) allocate(txid txid, n int) pgid { if len(f.ids) == 0 { return 0 } @@ -97,7 +107,7 @@ func (f *freelist) allocate(n int) pgid { for i := pgid(0); i < pgid(n); i++ { delete(f.cache, initial+i) } - + f.allocs[initial] = txid return initial } @@ -114,28 +124,73 @@ func (f *freelist) free(txid txid, p *page) { } // Free page and all its overflow pages. - var ids = f.pending[txid] + txp := f.pending[txid] + if txp == nil { + txp = &txPending{} + f.pending[txid] = txp + } + allocTxid, ok := f.allocs[p.id] + if ok { + delete(f.allocs, p.id) + } else if (p.flags & freelistPageFlag) != 0 { + // Freelist is always allocated by prior tx. + allocTxid = txid - 1 + } + for id := p.id; id <= p.id+pgid(p.overflow); id++ { // Verify that page is not already free. if f.cache[id] { panic(fmt.Sprintf("page %d already freed", id)) } - // Add to the freelist and cache. - ids = append(ids, id) + txp.ids = append(txp.ids, id) + txp.alloctx = append(txp.alloctx, allocTxid) f.cache[id] = true } - f.pending[txid] = ids } // release moves all page ids for a transaction id (or older) to the freelist. func (f *freelist) release(txid txid) { m := make(pgids, 0) - for tid, ids := range f.pending { + for tid, txp := range f.pending { if tid <= txid { // Move transaction's pending pages to the available freelist. // Don't remove from the cache since the page is still free. - m = append(m, ids...) + m = append(m, txp.ids...) + delete(f.pending, tid) + } + } + sort.Sort(m) + f.ids = pgids(f.ids).merge(m) +} + +// releaseRange moves pending pages allocated within an extent [begin,end] to the free list. +func (f *freelist) releaseRange(begin, end txid) { + if begin > end { + return + } + var m pgids + for tid, txp := range f.pending { + if tid < begin || tid > end { + continue + } + // Don't recompute freed pages if ranges haven't updated. + if txp.lastReleaseBegin == begin { + continue + } + for i := 0; i < len(txp.ids); i++ { + if atx := txp.alloctx[i]; atx < begin || atx > end { + continue + } + m = append(m, txp.ids[i]) + txp.ids[i] = txp.ids[len(txp.ids)-1] + txp.ids = txp.ids[:len(txp.ids)-1] + txp.alloctx[i] = txp.alloctx[len(txp.alloctx)-1] + txp.alloctx = txp.alloctx[:len(txp.alloctx)-1] + i-- + } + txp.lastReleaseBegin = begin + if len(txp.ids) == 0 { delete(f.pending, tid) } } @@ -146,12 +201,29 @@ func (f *freelist) release(txid txid) { // rollback removes the pages from a given pending tx. func (f *freelist) rollback(txid txid) { // Remove page ids from cache. - for _, id := range f.pending[txid] { - delete(f.cache, id) + txp := f.pending[txid] + if txp == nil { + return } - - // Remove pages from pending list. + var m pgids + for i, pgid := range txp.ids { + delete(f.cache, pgid) + tx := txp.alloctx[i] + if tx == 0 { + continue + } + if tx != txid { + // Pending free aborted; restore page back to alloc list. + f.allocs[pgid] = tx + } else { + // Freed page was allocated by this txn; OK to throw away. + m = append(m, pgid) + } + } + // Remove pages from pending list and mark as free if allocated by txid. delete(f.pending, txid) + sort.Sort(m) + f.ids = pgids(f.ids).merge(m) } // freed returns whether a given page is in the free list. @@ -161,6 +233,9 @@ func (f *freelist) freed(pgid pgid) bool { // read initializes the freelist from a freelist page. func (f *freelist) read(p *page) { + if (p.flags & freelistPageFlag) == 0 { + panic(fmt.Sprintf("invalid freelist page: %d, page type is %s", p.id, p.typ())) + } // If the page.count is at the max uint16 value (64k) then it's considered // an overflow and the size of the freelist is stored as the first element. idx, count := 0, int(p.count) @@ -173,7 +248,7 @@ func (f *freelist) read(p *page) { if count == 0 { f.ids = nil } else { - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count] f.ids = make([]pgid, len(ids)) copy(f.ids, ids) @@ -185,6 +260,12 @@ func (f *freelist) read(p *page) { f.reindex() } +// read initializes the freelist from a given list of ids. +func (f *freelist) readIDs(ids []pgid) { + f.ids = ids + f.reindex() +} + // write writes the page ids onto a freelist page. All free and pending ids are // saved to disk since in the event of a program crash, all pending ids will // become free. @@ -217,8 +298,8 @@ func (f *freelist) reload(p *page) { // Build a cache of only pending pages. pcache := make(map[pgid]bool) - for _, pendingIDs := range f.pending { - for _, pendingID := range pendingIDs { + for _, txp := range f.pending { + for _, pendingID := range txp.ids { pcache[pendingID] = true } } @@ -244,8 +325,8 @@ func (f *freelist) reindex() { for _, id := range f.ids { f.cache[id] = true } - for _, pendingIDs := range f.pending { - for _, pendingID := range pendingIDs { + for _, txp := range f.pending { + for _, pendingID := range txp.ids { f.cache[pendingID] = true } } diff --git a/vendor/github.com/boltdb/bolt/node.go b/vendor/go.etcd.io/bbolt/node.go similarity index 99% rename from vendor/github.com/boltdb/bolt/node.go rename to vendor/go.etcd.io/bbolt/node.go index 159318b229cb..6c3fa553ea68 100644 --- a/vendor/github.com/boltdb/bolt/node.go +++ b/vendor/go.etcd.io/bbolt/node.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -365,7 +365,7 @@ func (n *node) spill() error { } // Allocate contiguous space for the node. - p, err := tx.allocate((node.size() / tx.db.pageSize) + 1) + p, err := tx.allocate((node.size() + tx.db.pageSize - 1) / tx.db.pageSize) if err != nil { return err } diff --git a/vendor/github.com/boltdb/bolt/page.go b/vendor/go.etcd.io/bbolt/page.go similarity index 99% rename from vendor/github.com/boltdb/bolt/page.go rename to vendor/go.etcd.io/bbolt/page.go index cde403ae86de..bca9615f0fd6 100644 --- a/vendor/github.com/boltdb/bolt/page.go +++ b/vendor/go.etcd.io/bbolt/page.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" diff --git a/vendor/github.com/boltdb/bolt/tx.go b/vendor/go.etcd.io/bbolt/tx.go similarity index 94% rename from vendor/github.com/boltdb/bolt/tx.go rename to vendor/go.etcd.io/bbolt/tx.go index 6700308a2903..f50864142762 100644 --- a/vendor/github.com/boltdb/bolt/tx.go +++ b/vendor/go.etcd.io/bbolt/tx.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -126,10 +126,7 @@ func (tx *Tx) DeleteBucket(name []byte) error { // the error is returned to the caller. func (tx *Tx) ForEach(fn func(name []byte, b *Bucket) error) error { return tx.root.ForEach(func(k, v []byte) error { - if err := fn(k, tx.root.Bucket(k)); err != nil { - return err - } - return nil + return fn(k, tx.root.Bucket(k)) }) } @@ -169,28 +166,18 @@ func (tx *Tx) Commit() error { // Free the old root bucket. tx.meta.root.root = tx.root.root - opgid := tx.meta.pgid - - // Free the freelist and allocate new pages for it. This will overestimate - // the size of the freelist but not underestimate the size (which would be bad). - tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) - p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1) - if err != nil { - tx.rollback() - return err - } - if err := tx.db.freelist.write(p); err != nil { - tx.rollback() - return err + // Free the old freelist because commit writes out a fresh freelist. + if tx.meta.freelist != pgidNoFreelist { + tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) } - tx.meta.freelist = p.id - // If the high water mark has moved up then attempt to grow the database. - if tx.meta.pgid > opgid { - if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { - tx.rollback() + if !tx.db.NoFreelistSync { + err := tx.commitFreelist() + if err != nil { return err } + } else { + tx.meta.freelist = pgidNoFreelist } // Write dirty pages to disk. @@ -235,6 +222,31 @@ func (tx *Tx) Commit() error { return nil } +func (tx *Tx) commitFreelist() error { + // Allocate new pages for the new free list. This will overestimate + // the size of the freelist but not underestimate the size (which would be bad). + opgid := tx.meta.pgid + p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1) + if err != nil { + tx.rollback() + return err + } + if err := tx.db.freelist.write(p); err != nil { + tx.rollback() + return err + } + tx.meta.freelist = p.id + // If the high water mark has moved up then attempt to grow the database. + if tx.meta.pgid > opgid { + if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { + tx.rollback() + return err + } + } + + return nil +} + // Rollback closes the transaction and ignores all previous updates. Read-only // transactions must be rolled back and not committed. func (tx *Tx) Rollback() error { @@ -291,7 +303,9 @@ func (tx *Tx) close() { } // Copy writes the entire database to a writer. -// This function exists for backwards compatibility. Use WriteTo() instead. +// This function exists for backwards compatibility. +// +// Deprecated; Use WriteTo() instead. func (tx *Tx) Copy(w io.Writer) error { _, err := tx.WriteTo(w) return err @@ -305,7 +319,11 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { if err != nil { return 0, err } - defer func() { _ = f.Close() }() + defer func() { + if cerr := f.Close(); err == nil { + err = cerr + } + }() // Generate a meta page. We use the same page data for both meta pages. buf := make([]byte, tx.db.pageSize) @@ -333,7 +351,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { } // Move past the meta pages in the file. - if _, err := f.Seek(int64(tx.db.pageSize*2), os.SEEK_SET); err != nil { + if _, err := f.Seek(int64(tx.db.pageSize*2), io.SeekStart); err != nil { return n, fmt.Errorf("seek: %s", err) } @@ -344,7 +362,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { return n, err } - return n, f.Close() + return n, nil } // CopyFile copies the entire database to file at the given path. @@ -379,6 +397,9 @@ func (tx *Tx) Check() <-chan error { } func (tx *Tx) check(ch chan error) { + // Force loading free list if opened in ReadOnly mode. + tx.db.loadFreelist() + // Check if any pages are double freed. freed := make(map[pgid]bool) all := make([]pgid, tx.db.freelist.count()) @@ -394,8 +415,10 @@ func (tx *Tx) check(ch chan error) { reachable := make(map[pgid]*page) reachable[0] = tx.page(0) // meta0 reachable[1] = tx.page(1) // meta1 - for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ { - reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist) + if tx.meta.freelist != pgidNoFreelist { + for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ { + reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist) + } } // Recursively check buckets. @@ -453,7 +476,7 @@ func (tx *Tx) checkBucket(b *Bucket, reachable map[pgid]*page, freed map[pgid]bo // allocate returns a contiguous block of memory starting at a given page. func (tx *Tx) allocate(count int) (*page, error) { - p, err := tx.db.allocate(count) + p, err := tx.db.allocate(tx.meta.txid, count) if err != nil { return nil, err } @@ -462,7 +485,7 @@ func (tx *Tx) allocate(count int) (*page, error) { tx.pages[p.id] = p // Update statistics. - tx.stats.PageCount++ + tx.stats.PageCount += count tx.stats.PageAlloc += count * tx.db.pageSize return p, nil diff --git a/worker/runc/runc.go b/worker/runc/runc.go index 1389525a3a02..404c44df2830 100644 --- a/worker/runc/runc.go +++ b/worker/runc/runc.go @@ -6,7 +6,6 @@ import ( "path/filepath" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/content/local" "github.com/containerd/containerd/diff/apply" "github.com/containerd/containerd/diff/walking" @@ -22,6 +21,7 @@ import ( "github.com/moby/buildkit/worker/base" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" ) // SnapshotterFactory instantiates a snapshotter