From 959f3c36bb84a1317cdb8b7bc31831cbaeff8f7a Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Tue, 19 Mar 2019 00:22:17 +0900 Subject: [PATCH] Support Badger (#38) * Support Badger * Update CHANGES.md --- CHANGES.md | 1 + Dockerfile | 2 +- Gopkg.lock | 9 ++ README.md | 13 +- config/config_badger.go | 21 +++ index/raft_server.go | 1 + vendor/github.com/mosuka/bbadger/README.md | 32 ++++ vendor/github.com/mosuka/bbadger/bleve.go | 17 +++ vendor/github.com/mosuka/bbadger/go.mod | 22 +++ vendor/github.com/mosuka/bbadger/go.sum | 139 ++++++++++++++++++ .../mosuka/bbadger/prefix_iterator.go | 57 +++++++ .../mosuka/bbadger/range_iterator.go | 70 +++++++++ vendor/github.com/mosuka/bbadger/reader.go | 55 +++++++ vendor/github.com/mosuka/bbadger/store.go | 108 ++++++++++++++ vendor/github.com/mosuka/bbadger/writer.go | 75 ++++++++++ 15 files changed, 614 insertions(+), 8 deletions(-) create mode 100644 config/config_badger.go create mode 100644 vendor/github.com/mosuka/bbadger/README.md create mode 100644 vendor/github.com/mosuka/bbadger/bleve.go create mode 100644 vendor/github.com/mosuka/bbadger/go.mod create mode 100644 vendor/github.com/mosuka/bbadger/go.sum create mode 100644 vendor/github.com/mosuka/bbadger/prefix_iterator.go create mode 100644 vendor/github.com/mosuka/bbadger/range_iterator.go create mode 100644 vendor/github.com/mosuka/bbadger/reader.go create mode 100644 vendor/github.com/mosuka/bbadger/store.go create mode 100644 vendor/github.com/mosuka/bbadger/writer.go diff --git a/CHANGES.md b/CHANGES.md index b3216e0..abefecc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added +- Support Badger #38 - Add index stats #37 - Add Wikipedia example #35 - Support cznicb and leveldb #34 diff --git a/Dockerfile b/Dockerfile index 5a53d08..2117524 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && \ GOOS=linux \ GOARCH=amd64 \ CGO_ENABLED=1 \ - BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \ + BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb badger" \ VERSION="${VERSION}" \ make build diff --git a/Gopkg.lock b/Gopkg.lock index ff949ba..1dbaf1f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -420,6 +420,14 @@ revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" version = "v1.0.1" +[[projects]] + branch = "master" + digest = "1:fe91944cf2a46eecf510f0691eff66ba08c9152759defbc322f054ccca312e1d" + name = "github.com/mosuka/bbadger" + packages = ["."] + pruneopts = "UT" + revision = "fdf72caa1e806ce4d7d8a9ef9ea144ee2f0eff25" + [[projects]] branch = "master" digest = "1:c22fdadfe2de0e1df75987cac34f1c721157dc24f180c76b4ef66a3c9637a799" @@ -681,6 +689,7 @@ "github.com/hashicorp/raft", "github.com/hashicorp/raft-boltdb", "github.com/mash/go-accesslog", + "github.com/mosuka/bbadger", "github.com/natefinch/lumberjack", "github.com/prometheus/client_golang/prometheus", "github.com/prometheus/client_golang/prometheus/promhttp", diff --git a/README.md b/README.md index 74dd947..078153a 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ You can enable supported Bleve Extensions for like following: ```bash $ make \ GOOS=linux \ - BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \ + BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb badger" \ CGO_ENABLED=1 \ build ``` @@ -131,14 +131,13 @@ $ make \ ```bash $ make GOOS=darwin \ GOOS=darwin \ - BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \ + BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb badger" \ CGO_ENABLED=1 \ CGO_LDFLAGS="-L/usr/local/opt/icu4c/lib" \ CGO_CFLAGS="-I/usr/local/opt/icu4c/include" \ build ``` - Please refer to the following table for details of Bleve Extensions: | BUILD_TAGS | CGO_ENABLED | Description | @@ -149,7 +148,7 @@ Please refer to the following table for details of Bleve Extensions: | libstemmer | 1 | Enable Language Stemmer (Danish, German, English, Spanish, Finnish, French, Hungarian, Italian, Dutch, Norwegian, Portuguese, Romanian, Russian, Swedish, Turkish) | | cznicb | 0 | Enable cznicb KV store | | leveldb | 1 | Enable LevelDB | - +| badger | 0 | Enable Badger | You can see the binary file when build successful like so: @@ -164,7 +163,7 @@ blast-index Running a Blast index node is easy. Start Blast data node like so: ```bash -$ ./bin/blast-index start --node-id=index1 --data-dir=/tmp/blast/index1 --bind-addr=:6060 --grpc-addr=:5050 --http-addr=:8080 --index-mapping-file ./example/index_mapping.json +$ ./bin/blast-index start --node-id=index1 --data-dir=/tmp/blast/index1 --bind-addr=:6060 --grpc-addr=:5050 --http-addr=:8080 --index-mapping-file=./example/index_mapping.json ``` Please refer to following document for details of index mapping: @@ -433,8 +432,8 @@ $ curl -X DELETE 'http://127.0.0.1:8080/documents/enwiki_1' Blast is easy to bring up the cluster. Blast data node is already running, but that is not fault tolerant. If you need to increase the fault tolerance, bring up 2 more data nodes like so: ```bash -$ ./bin/blast-index start --node-id=index2 --data-dir=/tmp/blast/index2 --bind-addr=:6061 --grpc-addr=:5051 --http-addr=:8081 --index-mapping-file ./example/index_mapping.json --join-addr=:5050 -$ ./bin/blast-index start --node-id=index3 --data-dir=/tmp/blast/index3 --bind-addr=:6062 --grpc-addr=:5052 --http-addr=:8082 --index-mapping-file ./example/index_mapping.json --join-addr=:5050 +$ ./bin/blast-index start --node-id=index2 --data-dir=/tmp/blast/index2 --bind-addr=:6061 --grpc-addr=:5051 --http-addr=:8081 --index-mapping-file=./example/index_mapping.json --join-addr=:5050 +$ ./bin/blast-index start --node-id=index3 --data-dir=/tmp/blast/index3 --bind-addr=:6062 --grpc-addr=:5052 --http-addr=:8082 --index-mapping-file=./example/index_mapping.json --join-addr=:5050 ``` _Above example shows each Blast node running on the same host, so each node must listen on different ports. This would not be necessary if each node ran on a different host._ diff --git a/config/config_badger.go b/config/config_badger.go new file mode 100644 index 0000000..247bdf0 --- /dev/null +++ b/config/config_badger.go @@ -0,0 +1,21 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build badger full + +package config + +import ( + _ "github.com/mosuka/bbadger" +) diff --git a/index/raft_server.go b/index/raft_server.go index 7a9c476..0b980fa 100644 --- a/index/raft_server.go +++ b/index/raft_server.go @@ -27,6 +27,7 @@ import ( "github.com/golang/protobuf/ptypes/any" "github.com/hashicorp/raft" raftboltdb "github.com/hashicorp/raft-boltdb" + _ "github.com/mosuka/blast/config" "github.com/mosuka/blast/errors" "github.com/mosuka/blast/protobuf" "github.com/mosuka/blast/protobuf/index" diff --git a/vendor/github.com/mosuka/bbadger/README.md b/vendor/github.com/mosuka/bbadger/README.md new file mode 100644 index 0000000..8aee238 --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/README.md @@ -0,0 +1,32 @@ +Bleve Badger Backend +===================== +> [Blevesearch](https://github.com/blevesearch/bleve) kvstore implementation based on [Badger](https://github.com/dgraph-io/badger) forked from [https://github.com/akhenakh/bleve/tree/badger](https://github.com/akhenakh/bleve/tree/badger) with alot of improvements and fixes. + +Usage +========== +> `➜ go get github.com/alash3al/bbadger` . + +```go +package main + +import ( + "fmt" + + "github.com/alash3al/bbadger" + "github.com/blevesearch/bleve" +) + +func main() { + // create/open bleveIndex + index, err := bbadger.BleveIndex("/tmp/badger/indexName", bleve.NewIndexMapping()) + + // index some data + err = index.Index(identifier, your_data) + + // search for some text + query := bleve.NewMatchQuery("text") + search := bleve.NewSearchRequest(query) + searchResults, err := index.Search(search) +} + +``` diff --git a/vendor/github.com/mosuka/bbadger/bleve.go b/vendor/github.com/mosuka/bbadger/bleve.go new file mode 100644 index 0000000..c5e20eb --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/bleve.go @@ -0,0 +1,17 @@ +package bbadger + +import ( + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" +) + +// BleveIndex a helper function that open (creates if not exists a new) bleve index +func BleveIndex(path string, mapping mapping.IndexMapping) (bleve.Index, error) { + index, err := bleve.NewUsing(path, mapping, bleve.Config.DefaultIndexType, Name, nil) + + if err != nil && err == bleve.ErrorIndexPathExists { + index, err = bleve.Open(path) + } + + return index, err +} diff --git a/vendor/github.com/mosuka/bbadger/go.mod b/vendor/github.com/mosuka/bbadger/go.mod new file mode 100644 index 0000000..1f77a05 --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/go.mod @@ -0,0 +1,22 @@ +module github.com/mosuka/bbadger + +go 1.12 + +require ( + github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 // indirect + github.com/RoaringBitmap/roaring v0.4.17 // indirect + github.com/Smerity/govarint v0.0.0-20150407073650-7265e41f48f1 // indirect + github.com/blevesearch/bleve v0.7.0 + github.com/blevesearch/go-porterstemmer v1.0.2 // indirect + github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f // indirect + github.com/boltdb/bolt v1.3.1 // indirect + github.com/couchbase/vellum v0.0.0-20190111184608-e91b68ff3efe // indirect + github.com/dgraph-io/badger v1.5.4 + github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect + github.com/edsrzf/mmap-go v1.0.0 // indirect + github.com/golang/protobuf v1.3.1 // indirect + github.com/pkg/errors v0.8.1 // indirect + github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect + golang.org/x/net v0.0.0-20190313220215-9f648a60d977 // indirect + golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f // indirect +) diff --git a/vendor/github.com/mosuka/bbadger/go.sum b/vendor/github.com/mosuka/bbadger/go.sum new file mode 100644 index 0000000..65a3eec --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/go.sum @@ -0,0 +1,139 @@ +github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo= +github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/RoaringBitmap/roaring v0.4.16 h1:NholfewybRLOwACgfqfzn/N5xa6keKNs4fP00t0cwLo= +github.com/RoaringBitmap/roaring v0.4.16/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= +github.com/RoaringBitmap/roaring v0.4.17 h1:oCYFIFEMSQZrLHpywH7919esI1VSrQZ0pJXkZPGIJ78= +github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= +github.com/Smerity/govarint v0.0.0-20150407073650-7265e41f48f1 h1:G/NOANWMQev0CftoyxQwtRakdyNNNMB3qxkt/tj1HGs= +github.com/Smerity/govarint v0.0.0-20150407073650-7265e41f48f1/go.mod h1:o80NPAib/LOl8Eysqppjj7kkGkqz++eqzYGlvROpDcQ= +github.com/blevesearch/bleve v0.7.0 h1:znyZ3zjsh2Scr60vszs7rbF29TU6i1q9bfnZf1vh0Ac= +github.com/blevesearch/bleve v0.7.0/go.mod h1:Y2lmIkzV6mcNfAnAdOd+ZxHkHchhBfU/xroGIp61wfw= +github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3 h1:U6vnxZrTfItfiUiYx0lf/LgHjRSfaKK5QHSom3lEbnA= +github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ= +github.com/blevesearch/go-porterstemmer v1.0.1 h1:+ZjIF3K4U+LxqMybaE4hxyMuMvdX1Fq17CyzOXxJaiM= +github.com/blevesearch/go-porterstemmer v1.0.1/go.mod h1:haWQqFT3RdOGz7PJuM3or/pWNJS1pKkoZJWCkWu0DVA= +github.com/blevesearch/go-porterstemmer v1.0.2 h1:qe7n69gBd1OLY5sHKnxQHIbzn0LNJA4hpAf+5XDxV2I= +github.com/blevesearch/go-porterstemmer v1.0.2/go.mod h1:haWQqFT3RdOGz7PJuM3or/pWNJS1pKkoZJWCkWu0DVA= +github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f h1:kqbi9lqXLLs+zfWlgo1PIiRQ86n33K1JKotjj4rSYOg= +github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f/go.mod h1:IInt5XRvpiGE09KOk9mmCMLjHhydIhNPKPPFLFBB7L8= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/couchbase/vellum v0.0.0-20180910213445-01d5c56e6095 h1:dh7mqP7LS9voSd1Wx515giC2lPjPVduBpujISaftHrc= +github.com/couchbase/vellum v0.0.0-20180910213445-01d5c56e6095/go.mod h1:prYTC8EgTu3gwbqJihkud9zRXISvyulAplQ6exdCo1g= +github.com/couchbase/vellum v0.0.0-20190111184608-e91b68ff3efe h1:2o6Y7KMjJNsuMTF8f2H2eTKRhqH7+bQbjr+D+LnhE5M= +github.com/couchbase/vellum v0.0.0-20190111184608-e91b68ff3efe/go.mod h1:prYTC8EgTu3gwbqJihkud9zRXISvyulAplQ6exdCo1g= +github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk= +github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= +github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7 h1:y+DH9ARrWiiNBV+6waYP2IPcsRbxdU1qsnycPfShF4c= +github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= +github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 h1:0rkFMAbn5KBKNpJyHQ6Prb95vIKanmAe62KxsrN+sqA= +github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg= +github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= +github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102 h1:afESQBXJEnj3fu+34X//E8Wg3nEbMJxJkwSc0tPePK0= +github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8= +github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= +github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= +github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd h1:r04MMPyLHj/QwZuMJ5+7tJcBr1AQjpiAK/rZWRrQT7o= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5iLFSdbEVA/idR6Q2WhCm+deb7ir2CcM= +github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmhodges/levigo v0.0.0-20161115193449-c42d9e0ca023 h1:y5P5G9cANJZt3MXlMrgELo5mNLZPXH8aGFFFG7IzPU0= +github.com/jmhodges/levigo v0.0.0-20161115193449-c42d9e0ca023/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= +github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo= +github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= +github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s= +github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/syndtr/goleveldb v0.0.0-20181102132633-a4119e27a65d h1:wnS/zl2nFbhU9r6hlfqYEy9qJ0gG89VJGigG6dq7+Eo= +github.com/syndtr/goleveldb v0.0.0-20181102132633-a4119e27a65d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 h1:HOxvxvnntLiPn123Fk+twfUhCQdMDaqmb0cclArW0T0= +github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= +github.com/tinylib/msgp v1.0.2 h1:DfdQrzQa7Yh2es9SuLkixqxuXS2SxsdYn0KbdrOGWD8= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/willf/bitset v1.1.9 h1:GBtFynGY9ZWZmEC9sWuu41/7VBXPFCOAbCbqTflOg9c= +github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc h1:ZMCWScCvS2fUVFw8LOpxyUUW5qiviqr4Dg5NdjLeiLU= +golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc h1:SdCq5U4J+PpbSDIl9bM0V1e1Ug1jsnBkAFvTs1htn7U= +golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f h1:yCrMx/EeIue0+Qca57bWZS7VX6ymEoypmhWyPhz0NHM= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/mosuka/bbadger/prefix_iterator.go b/vendor/github.com/mosuka/bbadger/prefix_iterator.go new file mode 100644 index 0000000..9c81fee --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/prefix_iterator.go @@ -0,0 +1,57 @@ +package bbadger + +import ( + "bytes" + + "github.com/dgraph-io/badger" +) + +// PrefixIterator blevesearch prefix-iterator implementation +type PrefixIterator struct { + iterator *badger.Iterator + prefix []byte +} + +// Seek advance the iterator to the specified key +func (i *PrefixIterator) Seek(key []byte) { + if bytes.Compare(key, i.prefix) < 0 { + i.iterator.Seek(i.prefix) + return + } + i.iterator.Seek(key) +} + +// Next advance the iterator to the next step +func (i *PrefixIterator) Next() { + i.iterator.Next() +} + +// Current returns the key & value of the current step +func (i *PrefixIterator) Current() ([]byte, []byte, bool) { + if i.Valid() { + return i.Key(), i.Value(), true + } + return nil, nil, false +} + +// Key return the key of the current step +func (i *PrefixIterator) Key() []byte { + return i.iterator.Item().KeyCopy(nil) +} + +// Value returns the value of the current step +func (i *PrefixIterator) Value() []byte { + v, _ := i.iterator.Item().ValueCopy(nil) + return v +} + +// Valid whether the current iterator step is valid or not +func (i *PrefixIterator) Valid() bool { + return i.iterator.ValidForPrefix(i.prefix) +} + +// Close closes the current iterator and commit its transaction +func (i *PrefixIterator) Close() error { + i.iterator.Close() + return nil +} diff --git a/vendor/github.com/mosuka/bbadger/range_iterator.go b/vendor/github.com/mosuka/bbadger/range_iterator.go new file mode 100644 index 0000000..210bf60 --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/range_iterator.go @@ -0,0 +1,70 @@ +package bbadger + +import ( + "bytes" + + "github.com/dgraph-io/badger" +) + +// RangeIterator implements blevesearch store iterator +type RangeIterator struct { + iterator *badger.Iterator + start []byte + stop []byte +} + +// Seek advance the iterator to the specified key +func (i *RangeIterator) Seek(key []byte) { + if bytes.Compare(key, i.start) < 0 { + i.iterator.Seek(i.start) + return + } + i.iterator.Seek(key) +} + +// Next advance the iterator to the next step +func (i *RangeIterator) Next() { + i.iterator.Next() +} + +// Current returns the key & value of the current step +func (i *RangeIterator) Current() ([]byte, []byte, bool) { + if i.Valid() { + return i.Key(), i.Value(), true + } + return nil, nil, false +} + +// Key return the key of the current step +func (i *RangeIterator) Key() []byte { + return i.iterator.Item().KeyCopy(nil) +} + +// Value returns the value of the current step +func (i *RangeIterator) Value() []byte { + v, _ := i.iterator.Item().ValueCopy(nil) + + return v +} + +// Valid whether the current iterator step is valid or not +func (i *RangeIterator) Valid() bool { + if !i.iterator.Valid() { + return false + } + + if i.stop == nil || len(i.stop) == 0 { + return true + } + + if bytes.Compare(i.stop, i.iterator.Item().Key()) <= 0 { + return false + } + return true +} + +// Close closes the current iterator and commit its transaction +func (i *RangeIterator) Close() error { + i.iterator.Close() + return nil +} diff --git a/vendor/github.com/mosuka/bbadger/reader.go b/vendor/github.com/mosuka/bbadger/reader.go new file mode 100644 index 0000000..3e2d1b9 --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/reader.go @@ -0,0 +1,55 @@ +package bbadger + +import ( + "github.com/blevesearch/bleve/index/store" + "github.com/dgraph-io/badger" +) + +// Reader implements bleve/Store/Reader interface +type Reader struct { + itrOpts badger.IteratorOptions + s *Store + txn *badger.Txn +} + +// Get fetch the value of the specified key from the store +func (r *Reader) Get(k []byte) ([]byte, error) { + item, err := r.txn.Get(k) + if err != nil { + return nil, nil + } + return item.ValueCopy(nil) +} + +// MultiGet returns multiple values for the specified keys +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +// PrefixIterator initialize a new prefix iterator +func (r *Reader) PrefixIterator(k []byte) store.KVIterator { + itr := r.txn.NewIterator(r.itrOpts) + rv := PrefixIterator{ + iterator: itr, + prefix: k, + } + rv.iterator.Seek(k) + return &rv +} + +// RangeIterator initialize a new range iterator +func (r *Reader) RangeIterator(start, end []byte) store.KVIterator { + itr := r.txn.NewIterator(r.itrOpts) + rv := RangeIterator{ + iterator: itr, + start: start, + stop: end, + } + rv.iterator.Seek(start) + return &rv +} + +// Close closes the current reader and do some cleanup +func (r *Reader) Close() error { + return r.txn.Commit(nil) +} diff --git a/vendor/github.com/mosuka/bbadger/store.go b/vendor/github.com/mosuka/bbadger/store.go new file mode 100644 index 0000000..36ff23c --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/store.go @@ -0,0 +1,108 @@ +// MIT LICENSE +// +// Copyright (c) 2017 Fabrice Aneche +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package bbadger + +import ( + "os" + + "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/registry" + "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/options" +) + +const ( + // Name is the name of this engine in blevesearch + Name = "badger" +) + +// Store implements blevesearch store +type Store struct { + path string + db *badger.DB + mo store.MergeOperator +} + +// New creates a new store instance +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + path, ok := config["path"].(string) + if !ok { + return nil, os.ErrInvalid + } + if path == "" { + return nil, os.ErrInvalid + } + + opt := badger.DefaultOptions + opt.Dir = path + opt.ValueDir = path + opt.ReadOnly = false + opt.Truncate = true + opt.TableLoadingMode = options.LoadToRAM + opt.ValueLogLoadingMode = options.MemoryMap + + if _, err := os.Stat(path); os.IsNotExist(err) { + err := os.MkdirAll(path, os.FileMode(0700)) + if err != nil { + return nil, err + } + } + + db, err := badger.Open(opt) + if err != nil { + return nil, err + } + + rv := Store{ + path: path, + db: db, + mo: mo, + } + return &rv, nil +} + +// Close cleanup and close the current store +func (s *Store) Close() error { + return s.db.Close() +} + +// Reader initialize a new store.Reader +func (s *Store) Reader() (store.KVReader, error) { + return &Reader{ + itrOpts: badger.DefaultIteratorOptions, + s: s, + txn: s.db.NewTransaction(false), + }, nil +} + +// Writer initialize a new store.Writer +func (s *Store) Writer() (store.KVWriter, error) { + return &Writer{ + s: s, + }, nil +} + +// init add the engine name to blevesearch +func init() { + registry.RegisterKVStore(Name, New) +} diff --git a/vendor/github.com/mosuka/bbadger/writer.go b/vendor/github.com/mosuka/bbadger/writer.go new file mode 100644 index 0000000..e9377b9 --- /dev/null +++ b/vendor/github.com/mosuka/bbadger/writer.go @@ -0,0 +1,75 @@ +package bbadger + +import ( + "fmt" + + "github.com/blevesearch/bleve/index/store" +) + +// Writer bleve.search/store/Writer implementation +// I (alash3al) adopted it from bleve/store/boltdb +type Writer struct { + s *Store +} + +// NewBatch implements NewBatch +func (w *Writer) NewBatch() store.KVBatch { + return store.NewEmulatedBatch(w.s.mo) +} + +// NewBatchEx implements bleve NewBatchEx +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +// ExecuteBatch implements bleve ExecuteBatch +func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) { + emulatedBatch, ok := batch.(*store.EmulatedBatch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + txn := w.s.db.NewTransaction(true) + + defer (func() { + txn.Commit(nil) + })() + + for k, mergeOps := range emulatedBatch.Merger.Merges { + kb := []byte(k) + item, err := txn.Get(kb) + existingVal := []byte{} + if err == nil { + existingVal, _ = item.ValueCopy(nil) + } + mergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps) + if !fullMergeOk { + return fmt.Errorf("merge operator returned failure") + } + err = txn.Set(kb, mergedVal) + if err != nil { + return err + } + } + + for _, op := range emulatedBatch.Ops { + if op.V != nil { + err = txn.Set(op.K, op.V) + if err != nil { + return err + } + } else { + err = txn.Delete(op.K) + if err != nil { + return err + } + } + } + + return nil +} + +// Close closes the current writer +func (w *Writer) Close() error { + return nil +}