Support cznicb and leveldb (#34)

Dockerfile

@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Minoru Osuka
+# Copyright (c) 2019 Minoru Osuka
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@ RUN apt-get update && \
       golang \
       libicu-dev \
       libstemmer-dev \
+      libleveldb-dev \
       gcc-4.8 \
       g++-4.8 \
       build-essential && \
@@ -44,7 +45,7 @@ RUN apt-get update && \
     GOOS=linux \
       GOARCH=amd64 \
       CGO_ENABLED=1 \
-      BUILD_TAGS="kagome icu libstemmer cld2" \
+      BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \
       VERSION="${VERSION}" \
       make build
 
@@ -55,7 +56,8 @@ MAINTAINER Minoru Osuka "minoru.osuka@gmail.com"
 RUN apt-get update && \
     apt-get install -y \
       libicu-dev \
-      libstemmer-dev && \
+      libstemmer-dev \
+      libleveldb-dev && \
     apt-get clean
 
 COPY --from=0 /go/src/github.com/blevesearch/cld2/cld2/internal/*.so /usr/local/lib/

README.md

@@ -49,6 +49,7 @@ Blast requires some C/C++ libraries. The following sections are instructions for
 $ sudo apt-get install -y \
     libicu-dev \
     libstemmer-dev \
+    libleveldb-dev \
     gcc-4.8 \
     g++-4.8 \
     build-essential
@@ -72,7 +73,8 @@ $ sudo cp *.so /usr/local/lib
 
 ```bash
 $ brew install \
-    icu4c
+    icu4c \
+    leveldb
 
 $ go get -u -v github.com/blevesearch/cld2
 $ cd ${GOPATH}/src/github.com/blevesearch/cld2
@@ -118,7 +120,7 @@ You can enable supported Bleve Extensions for like following:
 ```bash
 $ make \
     GOOS=linux \
-    BUILD_TAGS="kagome icu libstemmer cld2" \
+    BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \
     CGO_ENABLED=1 \
     build
 ```
@@ -129,7 +131,7 @@ $ make \
 ```bash
 $ make GOOS=darwin \
     GOOS=darwin \
-    BUILD_TAGS="kagome icu libstemmer cld2" \
+    BUILD_TAGS="kagome icu libstemmer cld2 cznicb leveldb" \
     CGO_ENABLED=1 \
     CGO_LDFLAGS="-L/usr/local/opt/icu4c/lib" \
     CGO_CFLAGS="-I/usr/local/opt/icu4c/include" \
@@ -145,6 +147,9 @@ Please refer to the following table for details of Bleve Extensions:
 | kagome                     | 0           | Enable Japanese Language Analyser            |
 | icu                        | 1           | Enable ICU Tokenizer, Thai Language Analyser |
 | libstemmer                 | 1           | Enable Language Stemmer (Danish, German, English, Spanish, Finnish, French, Hungarian, Italian, Dutch, Norwegian, Portuguese, Romanian, Russian, Swedish, Turkish) |
+| cznicb                     | 0           | Enable cznicb KV store                       |
+| leveldb                    | 1           | Enable LevelDB                               |
+
 
 You can see the binary file when build successful like so:
 
@@ -564,7 +569,8 @@ $ docker run --rm --name blast-index1 \
       --grpc-addr=:5050 \
       --http-addr=:8080 \
       --data-dir=/tmp/blast/index1 \
-      --index-mapping-file=/opt/blast/example/index_mapping.json
+      --index-mapping-file=/opt/blast/example/index_mapping.json \
+      --index-storage-type=leveldb
 ```
 
 You can execute the command in docker container as follows:

cmd/blast-index/main.go

@@ -74,6 +74,11 @@ func main() {
 					Value: "",
 					Usage: "Path to a file containing a JSON representation of an index mapping to use",
 				},
+				cli.StringFlag{
+					Name:  "index-storage-type, s",
+					Value: "boltdb",
+					Usage: "Index storage type to use",
+				},
 				cli.StringFlag{
 					Name:  "log-level, L",
 					Value: "INFO",

cmd/blast-index/start.go

@@ -35,6 +35,7 @@ func execStart(c *cli.Context) error {
 	joinAddr := c.String("join-addr")
 
 	indexMappingFile := c.String("index-mapping-file")
+	indexStorageType := c.String("index-storage-type")
 
 	logLevel := c.String("log-level")
 	logFilename := c.String("log-file")
@@ -70,7 +71,7 @@ func execStart(c *cli.Context) error {
 		httpAccessLogCompress,
 	)
 
-	svr, err := index.NewServer(nodeId, bindAddr, grpcAddr, httpAddr, dataDir, joinAddr, indexMappingFile, logger, httpAccessLogger)
+	svr, err := index.NewServer(nodeId, bindAddr, grpcAddr, httpAddr, dataDir, joinAddr, indexMappingFile, indexStorageType, logger, httpAccessLogger)
 	if err != nil {
 		return err
 	}

docker-entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2018 Minoru Osuka
+# Copyright (c) 2019 Minoru Osuka
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

index/index.go

@@ -33,14 +33,14 @@ type Index struct {
 	logger *log.Logger
 }
 
-func NewIndex(dir string, indexMapping *mapping.IndexMappingImpl, logger *log.Logger) (*Index, error) {
+func NewIndex(dir string, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*Index, error) {
 	bleve.SetLog(logger)
 
 	var index bleve.Index
 	_, err := os.Stat(dir)
 	if os.IsNotExist(err) {
 		// create new index
-		index, err = bleve.NewUsing(dir, indexMapping, bleve.Config.DefaultIndexType, bleve.Config.DefaultKVStore, nil)
+		index, err = bleve.NewUsing(dir, indexMapping, bleve.Config.DefaultIndexType, indexStorageType, nil)
 		if err != nil {
 			return nil, err
 		}

index/raft_fsm.go

@@ -38,8 +38,8 @@ type RaftFSM struct {
 	logger *log.Logger
 }
 
-func NewRaftFSM(path string, indexMapping *mapping.IndexMappingImpl, logger *log.Logger) (*RaftFSM, error) {
-	index, err := NewIndex(path, indexMapping, logger)
+func NewRaftFSM(path string, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*RaftFSM, error) {
+	index, err := NewIndex(path, indexMapping, indexStorageType, logger)
 	if err != nil {
 		return nil, err
 	}

index/raft_server.go

@@ -43,8 +43,8 @@ type RaftServer struct {
 	logger *log.Logger
 }
 
-func NewRaftServer(node *blastraft.Node, bootstrap bool, indexMapping *mapping.IndexMappingImpl, logger *log.Logger) (*RaftServer, error) {
-	fsm, err := NewRaftFSM(filepath.Join(node.DataDir, "index"), indexMapping, logger)
+func NewRaftServer(node *blastraft.Node, bootstrap bool, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*RaftServer, error) {
+	fsm, err := NewRaftFSM(filepath.Join(node.DataDir, "index"), indexMapping, indexStorageType, logger)
 	if err != nil {
 		return nil, err
 	}

index/server.go

@@ -41,7 +41,7 @@ type Server struct {
 	httpLogger *log.Logger
 }
 
-func NewServer(nodeId string, bindAddr string, grpcAddr string, httpAddr string, dataDir string, joinAddr string, indexMappingPath string, logger *log.Logger, httpLogger *log.Logger) (*Server, error) {
+func NewServer(nodeId string, bindAddr string, grpcAddr string, httpAddr string, dataDir string, joinAddr string, indexMappingPath string, indexStorageType string, logger *log.Logger, httpLogger *log.Logger) (*Server, error) {
 	var err error
 
 	server := &Server{
@@ -94,7 +94,7 @@ func NewServer(nodeId string, bindAddr string, grpcAddr string, httpAddr string,
 	}
 
 	// create raft server
-	server.raftServer, err = NewRaftServer(server.node, server.bootstrap, indexMapping, server.logger)
+	server.raftServer, err = NewRaftServer(server.node, server.bootstrap, indexMapping, indexStorageType, server.logger)
 	if err != nil {
 		return nil, err
 	}