From 6489084a517c44e4461e3a5508768d1c31c4a062 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Thu, 10 Aug 2017 14:43:29 -0700 Subject: [PATCH 1/2] vendor: coreos/bbolt v1.3.1-coreos.0, add others in glide.yaml --- cmd/vendor/github.com/coreos/bbolt/db.go | 28 +++++++-- .../github.com/coreos/bbolt/freelist.go | 3 +- cmd/vendor/github.com/coreos/bbolt/tx.go | 15 +++-- .../github.com/russross/blackfriday/html.go | 1 + .../russross/blackfriday/smartypants.go | 58 ++++++++++++++----- glide.lock | 12 ++-- glide.yaml | 15 ++++- 7 files changed, 99 insertions(+), 33 deletions(-) diff --git a/cmd/vendor/github.com/coreos/bbolt/db.go b/cmd/vendor/github.com/coreos/bbolt/db.go index a6ab73c9dc8..c9d800fcc2c 100644 --- a/cmd/vendor/github.com/coreos/bbolt/db.go +++ b/cmd/vendor/github.com/coreos/bbolt/db.go @@ -24,6 +24,8 @@ const version = 2 // Represents a marker value to indicate that a file is a Bolt DB. const magic uint32 = 0xED0CDAED +const pgidNoFreelist pgid = 0xffffffffffffffff + // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when // syncing changes to a file. This is required as some operating systems, // such as OpenBSD, do not have a unified buffer cache (UBC) and writes @@ -239,14 +241,29 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { return nil, err } - if db.NoFreelistSync { - db.freelist = newFreelist() + db.freelist = newFreelist() + noFreeList := db.meta().freelist == pgidNoFreelist + if noFreeList { + // Reconstruct free list by scanning the DB. db.freelist.readIDs(db.freepages()) } else { - // Read in the freelist. - db.freelist = newFreelist() + // Read free list from freelist page. db.freelist.read(db.page(db.meta().freelist)) } + db.stats.FreePageN = len(db.freelist.ids) + + // Flush freelist when transitioning from no sync to sync so + // NoFreelistSync unaware boltdb can open the db later. + if !db.NoFreelistSync && noFreeList && ((mode & 0222) != 0) { + tx, err := db.Begin(true) + if tx != nil { + err = tx.Commit() + } + if err != nil { + _ = db.close() + return nil, err + } + } // Mark the database as opened and return. return db, nil @@ -1065,7 +1082,8 @@ func (m *meta) copy(dest *meta) { func (m *meta) write(p *page) { if m.root.root >= m.pgid { panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid)) - } else if m.freelist >= m.pgid { + } else if m.freelist >= m.pgid && m.freelist != pgidNoFreelist { + // TODO: reject pgidNoFreeList if !NoFreelistSync panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid)) } diff --git a/cmd/vendor/github.com/coreos/bbolt/freelist.go b/cmd/vendor/github.com/coreos/bbolt/freelist.go index 8ac0d61de33..13ce5166600 100644 --- a/cmd/vendor/github.com/coreos/bbolt/freelist.go +++ b/cmd/vendor/github.com/coreos/bbolt/freelist.go @@ -6,7 +6,6 @@ import ( "unsafe" ) - // txPending holds a list of pgids and corresponding allocation txns // that are pending to be freed. type txPending struct { @@ -246,7 +245,7 @@ func (f *freelist) read(p *page) { if count == 0 { f.ids = nil } else { - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:idx+count] f.ids = make([]pgid, len(ids)) copy(f.ids, ids) diff --git a/cmd/vendor/github.com/coreos/bbolt/tx.go b/cmd/vendor/github.com/coreos/bbolt/tx.go index 5370e5fe19e..6b2fa283491 100644 --- a/cmd/vendor/github.com/coreos/bbolt/tx.go +++ b/cmd/vendor/github.com/coreos/bbolt/tx.go @@ -169,11 +169,18 @@ func (tx *Tx) Commit() error { // Free the old root bucket. tx.meta.root.root = tx.root.root + // Free the old freelist because commit writes out a fresh freelist. + if tx.meta.freelist != pgidNoFreelist { + tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) + } + if !tx.db.NoFreelistSync { err := tx.commitFreelist() if err != nil { return err } + } else { + tx.meta.freelist = pgidNoFreelist } // Write dirty pages to disk. @@ -219,11 +226,9 @@ func (tx *Tx) Commit() error { } func (tx *Tx) commitFreelist() error { - opgid := tx.meta.pgid - - // Free the freelist and allocate new pages for it. This will overestimate + // Allocate new pages for the new free list. This will overestimate // the size of the freelist but not underestimate the size (which would be bad). - tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) + opgid := tx.meta.pgid p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1) if err != nil { tx.rollback() @@ -404,7 +409,7 @@ func (tx *Tx) check(ch chan error) { reachable := make(map[pgid]*page) reachable[0] = tx.page(0) // meta0 reachable[1] = tx.page(1) // meta1 - if !tx.DB().NoFreelistSync { + if tx.meta.freelist != pgidNoFreelist { for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ { reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist) } diff --git a/cmd/vendor/github.com/russross/blackfriday/html.go b/cmd/vendor/github.com/russross/blackfriday/html.go index 74e67ee82bd..2f0ad3b1e3e 100644 --- a/cmd/vendor/github.com/russross/blackfriday/html.go +++ b/cmd/vendor/github.com/russross/blackfriday/html.go @@ -42,6 +42,7 @@ const ( HTML_SMARTYPANTS_DASHES // enable smart dashes (with HTML_USE_SMARTYPANTS) HTML_SMARTYPANTS_LATEX_DASHES // enable LaTeX-style dashes (with HTML_USE_SMARTYPANTS and HTML_SMARTYPANTS_DASHES) HTML_SMARTYPANTS_ANGLED_QUOTES // enable angled double quotes (with HTML_USE_SMARTYPANTS) for double quotes rendering + HTML_SMARTYPANTS_QUOTES_NBSP // enable "French guillemets" (with HTML_USE_SMARTYPANTS) HTML_FOOTNOTE_RETURN_LINKS // generate a link at the end of a footnote to return to the source ) diff --git a/cmd/vendor/github.com/russross/blackfriday/smartypants.go b/cmd/vendor/github.com/russross/blackfriday/smartypants.go index eeffa5e1e1a..f25bd07d95b 100644 --- a/cmd/vendor/github.com/russross/blackfriday/smartypants.go +++ b/cmd/vendor/github.com/russross/blackfriday/smartypants.go @@ -39,7 +39,7 @@ func isdigit(c byte) bool { return c >= '0' && c <= '9' } -func smartQuoteHelper(out *bytes.Buffer, previousChar byte, nextChar byte, quote byte, isOpen *bool) bool { +func smartQuoteHelper(out *bytes.Buffer, previousChar byte, nextChar byte, quote byte, isOpen *bool, addNBSP bool) bool { // edge of the buffer is likely to be a tag that we don't get to see, // so we treat it like text sometimes @@ -96,6 +96,12 @@ func smartQuoteHelper(out *bytes.Buffer, previousChar byte, nextChar byte, quote *isOpen = false } + // Note that with the limited lookahead, this non-breaking + // space will also be appended to single double quotes. + if addNBSP && !*isOpen { + out.WriteString(" ") + } + out.WriteByte('&') if *isOpen { out.WriteByte('l') @@ -104,6 +110,11 @@ func smartQuoteHelper(out *bytes.Buffer, previousChar byte, nextChar byte, quote } out.WriteByte(quote) out.WriteString("quo;") + + if addNBSP && *isOpen { + out.WriteString(" ") + } + return true } @@ -116,7 +127,7 @@ func smartSingleQuote(out *bytes.Buffer, smrt *smartypantsData, previousChar byt if len(text) >= 3 { nextChar = text[2] } - if smartQuoteHelper(out, previousChar, nextChar, 'd', &smrt.inDoubleQuote) { + if smartQuoteHelper(out, previousChar, nextChar, 'd', &smrt.inDoubleQuote, false) { return 1 } } @@ -141,7 +152,7 @@ func smartSingleQuote(out *bytes.Buffer, smrt *smartypantsData, previousChar byt if len(text) > 1 { nextChar = text[1] } - if smartQuoteHelper(out, previousChar, nextChar, 's', &smrt.inSingleQuote) { + if smartQuoteHelper(out, previousChar, nextChar, 's', &smrt.inSingleQuote, false) { return 0 } @@ -205,13 +216,13 @@ func smartDashLatex(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, return 0 } -func smartAmpVariant(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte, quote byte) int { +func smartAmpVariant(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte, quote byte, addNBSP bool) int { if bytes.HasPrefix(text, []byte(""")) { nextChar := byte(0) if len(text) >= 7 { nextChar = text[6] } - if smartQuoteHelper(out, previousChar, nextChar, quote, &smrt.inDoubleQuote) { + if smartQuoteHelper(out, previousChar, nextChar, quote, &smrt.inDoubleQuote, addNBSP) { return 5 } } @@ -224,12 +235,15 @@ func smartAmpVariant(out *bytes.Buffer, smrt *smartypantsData, previousChar byte return 0 } -func smartAmp(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int { - return smartAmpVariant(out, smrt, previousChar, text, 'd') -} +func smartAmp(angledQuotes, addNBSP bool) func(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int { + var quote byte = 'd' + if angledQuotes { + quote = 'a' + } -func smartAmpAngledQuote(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int { - return smartAmpVariant(out, smrt, previousChar, text, 'a') + return func(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int { + return smartAmpVariant(out, smrt, previousChar, text, quote, addNBSP) + } } func smartPeriod(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int { @@ -253,7 +267,7 @@ func smartBacktick(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, if len(text) >= 3 { nextChar = text[2] } - if smartQuoteHelper(out, previousChar, nextChar, 'd', &smrt.inDoubleQuote) { + if smartQuoteHelper(out, previousChar, nextChar, 'd', &smrt.inDoubleQuote, false) { return 1 } } @@ -337,7 +351,7 @@ func smartDoubleQuoteVariant(out *bytes.Buffer, smrt *smartypantsData, previousC if len(text) > 1 { nextChar = text[1] } - if !smartQuoteHelper(out, previousChar, nextChar, quote, &smrt.inDoubleQuote) { + if !smartQuoteHelper(out, previousChar, nextChar, quote, &smrt.inDoubleQuote, false) { out.WriteString(""") } @@ -367,14 +381,30 @@ type smartCallback func(out *bytes.Buffer, smrt *smartypantsData, previousChar b type smartypantsRenderer [256]smartCallback +var ( + smartAmpAngled = smartAmp(true, false) + smartAmpAngledNBSP = smartAmp(true, true) + smartAmpRegular = smartAmp(false, false) + smartAmpRegularNBSP = smartAmp(false, true) +) + func smartypants(flags int) *smartypantsRenderer { r := new(smartypantsRenderer) + addNBSP := flags&HTML_SMARTYPANTS_QUOTES_NBSP != 0 if flags&HTML_SMARTYPANTS_ANGLED_QUOTES == 0 { r['"'] = smartDoubleQuote - r['&'] = smartAmp + if !addNBSP { + r['&'] = smartAmpRegular + } else { + r['&'] = smartAmpRegularNBSP + } } else { r['"'] = smartAngledDoubleQuote - r['&'] = smartAmpAngledQuote + if !addNBSP { + r['&'] = smartAmpAngled + } else { + r['&'] = smartAmpAngledNBSP + } } r['\''] = smartSingleQuote r['('] = smartParens diff --git a/glide.lock b/glide.lock index da5169dbd61..7cfd49b1e78 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: a75247f755ba076f6b7663e6ddc2e7458fc4fd497b4fe691861b2b32afd4bbff -updated: 2017-07-27T10:45:21.070753552-07:00 +hash: 6b16200008cc5fd50a370c50676225fbc6af0d79bd3ec0fdb94847c55d99a1a7 +updated: 2017-08-10T14:48:42.863537562-07:00 imports: - name: github.com/beorn7/perks version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 @@ -10,7 +10,7 @@ imports: - name: github.com/cockroachdb/cmux version: 112f0506e7743d64a6eb8fedbcff13d9979bbf92 - name: github.com/coreos/bbolt - version: ad39960eb40bb33c9bda31bed2eaf4fdda15efe6 + version: 12923fe56c105bca6efbbcc258cd762b4258333d - name: github.com/coreos/go-semver version: 8ab6407b697782a06568d4b7f1db25550ec2e4c6 subpackages: @@ -96,7 +96,7 @@ imports: subpackages: - xfs - name: github.com/russross/blackfriday - version: 067529f716f4c3f5e37c8c95ddd59df1007290ae + version: 4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c - name: github.com/spf13/cobra version: 1c44ec8d3f1552cac48999f9306da23c4d8a288b - name: github.com/spf13/pflag @@ -129,7 +129,7 @@ imports: subpackages: - unix - name: golang.org/x/text - version: 9e2f80a6ba7ed4ba13e0cd4b1f094bf916875735 + version: b19bf474d317b857955b12035d2c5acb57ce8b01 subpackages: - secure/bidirule - transform @@ -140,7 +140,7 @@ imports: subpackages: - rate - name: google.golang.org/genproto - version: b0a3dcfcd1a9bd48e63634bd8802960804cf8315 + version: 09f6ed296fc66555a25fe4ce95173148778dfa85 subpackages: - googleapis/rpc/status - name: google.golang.org/grpc diff --git a/glide.yaml b/glide.yaml index 9af52511171..5e9a8d5e726 100644 --- a/glide.yaml +++ b/glide.yaml @@ -3,7 +3,7 @@ import: - package: github.com/bgentry/speakeasy version: v0.1.0 - package: github.com/coreos/bbolt - version: ad39960eb40bb33c9bda31bed2eaf4fdda15efe6 + version: v1.3.1-coreos.0 - package: github.com/cockroachdb/cmux version: 112f0506e7743d64a6eb8fedbcff13d9979bbf92 - package: github.com/coreos/go-semver @@ -115,5 +115,18 @@ import: version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b - package: github.com/dgrijalva/jwt-go version: v3.0.0 +- package: google.golang.org/genproto + version: 09f6ed296fc66555a25fe4ce95173148778dfa85 + subpackages: + - googleapis/rpc/status +- package: golang.org/x/text + version: b19bf474d317b857955b12035d2c5acb57ce8b01 + subpackages: + - secure/bidirule + - transform + - unicode/bidi + - unicode/norm +- package: github.com/russross/blackfriday + version: 4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c ignore: - google.golang.org/appengine From 2513e8c9ce6c2dc7410a5a22a1673192ffa272e2 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Thu, 10 Aug 2017 16:39:51 -0700 Subject: [PATCH 2/2] integration: increase numPuts to write more than 1 page For ppc64. Reference: https://github.com/coreos/bbolt/issues/15#issuecomment-321700834. Signed-off-by: Gyu-Ho Lee --- integration/metrics_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/metrics_test.go b/integration/metrics_test.go index 8e7d60a537c..bd7850cf2c4 100644 --- a/integration/metrics_test.go +++ b/integration/metrics_test.go @@ -50,7 +50,7 @@ func TestMetricDbSizeDefrag(t *testing.T) { mc := toGRPC(clus.Client(0)).Maintenance // expand the db size - numPuts := 10 + numPuts := 25 // large enough to write more than 1 page putreq := &pb.PutRequest{Key: []byte("k"), Value: make([]byte, 4096)} for i := 0; i < numPuts; i++ { if _, err := kvc.Put(context.TODO(), putreq); err != nil {