diff --git a/Dockerfile b/Dockerfile index b741ccc..5a53d08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2018 Minoru Osuka +# Copyright (c) 2019 Minoru Osuka # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ RUN apt-get update && \ golang \ libicu-dev \ libstemmer-dev \ + libleveldb-dev \ gcc-4.8 \ g++-4.8 \ build-essential && \ @@ -44,7 +45,7 @@ RUN apt-get update && \ GOOS=linux \ GOARCH=amd64 \ CGO_ENABLED=1 \ - BUILD_TAGS="kagome icu libstemmer cld2" \ + BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \ VERSION="${VERSION}" \ make build @@ -55,7 +56,8 @@ MAINTAINER Minoru Osuka "minoru.osuka@gmail.com" RUN apt-get update && \ apt-get install -y \ libicu-dev \ - libstemmer-dev && \ + libstemmer-dev \ + libleveldb-dev && \ apt-get clean COPY --from=0 /go/src/github.com/blevesearch/cld2/cld2/internal/*.so /usr/local/lib/ diff --git a/README.md b/README.md index 7d79002..6040c11 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Blast requires some C/C++ libraries. The following sections are instructions for $ sudo apt-get install -y \ libicu-dev \ libstemmer-dev \ + libleveldb-dev \ gcc-4.8 \ g++-4.8 \ build-essential @@ -72,7 +73,8 @@ $ sudo cp *.so /usr/local/lib ```bash $ brew install \ - icu4c + icu4c \ + leveldb $ go get -u -v github.com/blevesearch/cld2 $ cd ${GOPATH}/src/github.com/blevesearch/cld2 @@ -118,7 +120,7 @@ You can enable supported Bleve Extensions for like following: ```bash $ make \ GOOS=linux \ - BUILD_TAGS="kagome icu libstemmer cld2" \ + BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \ CGO_ENABLED=1 \ build ``` @@ -129,7 +131,7 @@ $ make \ ```bash $ make GOOS=darwin \ GOOS=darwin \ - BUILD_TAGS="kagome icu libstemmer cld2" \ + BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \ CGO_ENABLED=1 \ CGO_LDFLAGS="-L/usr/local/opt/icu4c/lib" \ CGO_CFLAGS="-I/usr/local/opt/icu4c/include" \ @@ -145,6 +147,9 @@ Please refer to the following table for details of Bleve Extensions: | kagome | 0 | Enable Japanese Language Analyser | | icu | 1 | Enable ICU Tokenizer, Thai Language Analyser | | libstemmer | 1 | Enable Language Stemmer (Danish, German, English, Spanish, Finnish, French, Hungarian, Italian, Dutch, Norwegian, Portuguese, Romanian, Russian, Swedish, Turkish) | +| cznicb | 0 | Enable cznicb KV store | +| leveldb | 1 | Enable LevelDB | + You can see the binary file when build successful like so: @@ -564,7 +569,8 @@ $ docker run --rm --name blast-index1 \ --grpc-addr=:5050 \ --http-addr=:8080 \ --data-dir=/tmp/blast/index1 \ - --index-mapping-file=/opt/blast/example/index_mapping.json + --index-mapping-file=/opt/blast/example/index_mapping.json \ + --index-storage-type=leveldb ``` You can execute the command in docker container as follows: diff --git a/cmd/blast-index/main.go b/cmd/blast-index/main.go index 42d7a12..63d76c3 100644 --- a/cmd/blast-index/main.go +++ b/cmd/blast-index/main.go @@ -74,6 +74,11 @@ func main() { Value: "", Usage: "Path to a file containing a JSON representation of an index mapping to use", }, + cli.StringFlag{ + Name: "index-storage-type, s", + Value: "boltdb", + Usage: "Index storage type to use", + }, cli.StringFlag{ Name: "log-level, L", Value: "INFO", diff --git a/cmd/blast-index/start.go b/cmd/blast-index/start.go index b76e30b..a96ce78 100644 --- a/cmd/blast-index/start.go +++ b/cmd/blast-index/start.go @@ -35,6 +35,7 @@ func execStart(c *cli.Context) error { joinAddr := c.String("join-addr") indexMappingFile := c.String("index-mapping-file") + indexStorageType := c.String("index-storage-type") logLevel := c.String("log-level") logFilename := c.String("log-file") @@ -70,7 +71,7 @@ func execStart(c *cli.Context) error { httpAccessLogCompress, ) - svr, err := index.NewServer(nodeId, bindAddr, grpcAddr, httpAddr, dataDir, joinAddr, indexMappingFile, logger, httpAccessLogger) + svr, err := index.NewServer(nodeId, bindAddr, grpcAddr, httpAddr, dataDir, joinAddr, indexMappingFile, indexStorageType, logger, httpAccessLogger) if err != nil { return err } diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index cbe4bc1..1cf687a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2018 Minoru Osuka +# Copyright (c) 2019 Minoru Osuka # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/index/index.go b/index/index.go index 8f3c87d..c87cebd 100644 --- a/index/index.go +++ b/index/index.go @@ -33,14 +33,14 @@ type Index struct { logger *log.Logger } -func NewIndex(dir string, indexMapping *mapping.IndexMappingImpl, logger *log.Logger) (*Index, error) { +func NewIndex(dir string, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*Index, error) { bleve.SetLog(logger) var index bleve.Index _, err := os.Stat(dir) if os.IsNotExist(err) { // create new index - index, err = bleve.NewUsing(dir, indexMapping, bleve.Config.DefaultIndexType, bleve.Config.DefaultKVStore, nil) + index, err = bleve.NewUsing(dir, indexMapping, bleve.Config.DefaultIndexType, indexStorageType, nil) if err != nil { return nil, err } diff --git a/index/raft_fsm.go b/index/raft_fsm.go index f7df6db..008071e 100644 --- a/index/raft_fsm.go +++ b/index/raft_fsm.go @@ -38,8 +38,8 @@ type RaftFSM struct { logger *log.Logger } -func NewRaftFSM(path string, indexMapping *mapping.IndexMappingImpl, logger *log.Logger) (*RaftFSM, error) { - index, err := NewIndex(path, indexMapping, logger) +func NewRaftFSM(path string, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*RaftFSM, error) { + index, err := NewIndex(path, indexMapping, indexStorageType, logger) if err != nil { return nil, err } diff --git a/index/raft_server.go b/index/raft_server.go index 6f8aba3..b5c233c 100644 --- a/index/raft_server.go +++ b/index/raft_server.go @@ -43,8 +43,8 @@ type RaftServer struct { logger *log.Logger } -func NewRaftServer(node *blastraft.Node, bootstrap bool, indexMapping *mapping.IndexMappingImpl, logger *log.Logger) (*RaftServer, error) { - fsm, err := NewRaftFSM(filepath.Join(node.DataDir, "index"), indexMapping, logger) +func NewRaftServer(node *blastraft.Node, bootstrap bool, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*RaftServer, error) { + fsm, err := NewRaftFSM(filepath.Join(node.DataDir, "index"), indexMapping, indexStorageType, logger) if err != nil { return nil, err } diff --git a/index/server.go b/index/server.go index 6e8d7a9..5512be7 100644 --- a/index/server.go +++ b/index/server.go @@ -41,7 +41,7 @@ type Server struct { httpLogger *log.Logger } -func NewServer(nodeId string, bindAddr string, grpcAddr string, httpAddr string, dataDir string, joinAddr string, indexMappingPath string, logger *log.Logger, httpLogger *log.Logger) (*Server, error) { +func NewServer(nodeId string, bindAddr string, grpcAddr string, httpAddr string, dataDir string, joinAddr string, indexMappingPath string, indexStorageType string, logger *log.Logger, httpLogger *log.Logger) (*Server, error) { var err error server := &Server{ @@ -94,7 +94,7 @@ func NewServer(nodeId string, bindAddr string, grpcAddr string, httpAddr string, } // create raft server - server.raftServer, err = NewRaftServer(server.node, server.bootstrap, indexMapping, server.logger) + server.raftServer, err = NewRaftServer(server.node, server.bootstrap, indexMapping, indexStorageType, server.logger) if err != nil { return nil, err }