Make Bleve Extensions available. (#10)

* Support cld2 cznicb icu leveldb libstemmer rocksdb

* Update Makefile

* Update README.md

* Update Gopkg.toml

* Update README.md

* Update README.md

* Update Makefile

* Enable Bleve Extensions

* Delete unnecessary comments

* Update README.md

.gitignore

@@ -20,6 +20,6 @@
 .idea/
 
 # Blast
-*.log
-bin
-data
+bin/
+data/
+dist/

Dockerfile

@@ -12,24 +12,72 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.10.3
+FROM ubuntu:18.10
 
 ARG VERSION=
-ARG BUILD_TAGS=
 
-COPY . /go/src/github.com/mosuka/blast
+ENV GOPATH /go
 
-RUN cd /go/src/github.com/mosuka/blast && \
-    make GOOS=linux GOARCH=amd64 VERSION=${VERSION} BUILD_TAG=${BUILD_TAGS} build
+COPY . ${GOPATH}/src/github.com/mosuka/blast
 
+RUN apt-get update && \
+    apt-get install -y git \
+                       golang \
+                       libicu-dev \
+                       libleveldb-dev \
+                       libstemmer-dev \
+                       libgflags-dev \
+                       libsnappy-dev \
+                       zlib1g-dev \
+                       libbz2-dev \
+                       liblz4-dev \
+                       libzstd-dev \
+                       librocksdb-dev \
+                       gcc-4.8 \
+                       g++-4.8 \
+                       build-essential && \
+    apt-get clean && \
 
-FROM alpine:3.7
-#FROM scratch
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 80 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 90 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 90 && \
+
+    go get -u -v github.com/blevesearch/cld2 && \
+    cd ${GOPATH}/src/github.com/blevesearch/cld2 && \
+    git clone https://github.com/CLD2Owners/cld2.git && \
+    cd cld2/internal && \
+    ./compile_libs.sh && \
+    cp *.so /usr/local/lib && \
+
+    cd ${GOPATH}/src/github.com/mosuka/blast && \
+    GOOS=linux \
+    GOARCH=amd64 \
+    CGO_ENABLED=1 \
+    CGO_CFLAGS="-I/usr/include/rocksdb" \
+    CGO_LDFLAGS="-L/usr/lib -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" \
+    BUILD_TAGS=full \
+    VERSION=${VERSION} \
+    make build
+
+FROM ubuntu:18.10
 
 MAINTAINER Minoru Osuka "minoru.osuka@gmail.com"
 
-RUN apk --no-cache update
+RUN apt-get update && \
+    apt-get install -y libicu-dev \
+                       libleveldb-dev \
+                       libstemmer-dev \
+                       libgflags-dev \
+                       libsnappy-dev \
+                       zlib1g-dev \
+                       libbz2-dev \
+                       liblz4-dev \
+                       libzstd-dev \
+                       librocksdb-dev && \
+    apt-get clean
 
+COPY --from=0 /go/src/github.com/blevesearch/cld2/cld2/internal/*.so /usr/local/lib/
 COPY --from=0 /go/src/github.com/mosuka/blast/bin/blast /usr/bin/blast
 
 EXPOSE 10000 10001 10002

Makefile

@@ -13,12 +13,16 @@
 # limitations under the License.
 
 VERSION ?=
-GOOS = linux
-GOARCH = amd64
-BUILD_TAGS = kagome
-BIN_EXT =
+GOOS ?= linux
+GOARCH ?= amd64
+BUILD_TAGS ?=
+CGO_ENABLED ?= 0
+CGO_CFLAGS ?=
+CGO_LDFLAGS ?=
+GO15VENDOREXPERIMENT ?= 1
+BIN_EXT ?=
 
-GO := CGO_ENABLED=0 GO15VENDOREXPERIMENT=1 go
+GO := CGO_ENABLED=$(CGO_ENABLED) GO15VENDOREXPERIMENT=$(CGO_ENABLED) go
 
 PACKAGES = $(shell $(GO) list ./... | grep -v '/vendor/')
 
@@ -29,7 +33,7 @@ TARGET_PACKAGES = $(shell find . -name 'main.go' -print0 | xargs -0 -n1 dirname
 ifeq ($(VERSION),)
   VERSION = $(shell cat ./Versionfile)
 endif
-LDFLAGS = -ldflags "-X \"github.com/mosuka/blast/version.Version=${VERSION}\""
+LDFLAGS = -ldflags "-X \"github.com/mosuka/blast/version.Version=$(VERSION)\""
 
 ifeq ($(GOOS),windows)
   BIN_EXT = .exe
@@ -65,48 +69,60 @@ format:
 .PHONY: test
 test:
 	@echo ">> testing all packages"
-	@echo "   VERSION    = $(VERSION)"
-	@echo "   BUILD_TAGS = $(BUILD_TAGS)"
-	@$(GO) test -v -tags=${BUILD_TAGS} ${LDFLAGS} $(PACKAGES)
+	@echo "   VERSION     = $(VERSION)"
+	@echo "   CGO_ENABLED = $(CGO_ENABLED)"
+	@echo "   CGO_CFLAGS  = $(CGO_CFLAGS)"
+	@echo "   CGO_LDFLAGS = $(CGO_LDFLAGS)"
+	@echo "   BUILD_TAGS  = $(BUILD_TAGS)"
+	@$(GO) test -v -tags="$(BUILD_TAGS)" $(LDFLAGS) $(PACKAGES)
 
 .PHONY: build
 build:
 	@echo ">> building binaries"
-	@echo "   VERSION    = $(VERSION)"
-	@echo "   GOOS       = $(GOOS)"
-	@echo "   GOARCH     = $(GOARCH)"
-	@echo "   BUILD_TAGS = $(BUILD_TAGS)"
-	@for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; GOARCH=$(GOARCH) GOOS=$(GOOS) $(GO) build -tags=${BUILD_TAGS} ${LDFLAGS} -o ./bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done
+	@echo "   VERSION     = $(VERSION)"
+	@echo "   GOOS        = $(GOOS)"
+	@echo "   GOARCH      = $(GOARCH)"
+	@echo "   CGO_ENABLED = $(CGO_ENABLED)"
+	@echo "   CGO_CFLAGS  = $(CGO_CFLAGS)"
+	@echo "   CGO_LDFLAGS = $(CGO_LDFLAGS)"
+	@echo "   BUILD_TAGS  = $(BUILD_TAGS)"
+	@for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; $(GO) build -tags="$(BUILD_TAGS)" $(LDFLAGS) -o ./bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done
 
 .PHONY: install
 install:
 	@echo ">> installing binaries"
-	@echo "   VERSION    = $(VERSION)"
-	@echo "   GOOS       = $(GOOS)"
-	@echo "   GOARCH     = $(GOARCH)"
-	@echo "   BUILD_TAGS = $(BUILD_TAGS)"
-	@for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; GOARCH=$(GOARCH) GOOS=$(GOOS) $(GO) install -tags=${BUILD_TAGS} ${LDFLAGS} $$target_pkg || exit 1; done
+	@echo "   VERSION     = $(VERSION)"
+	@echo "   GOOS        = $(GOOS)"
+	@echo "   GOARCH      = $(GOARCH)"
+	@echo "   CGO_ENABLED = $(CGO_ENABLED)"
+	@echo "   CGO_CFLAGS  = $(CGO_CFLAGS)"
+	@echo "   CGO_LDFLAGS = $(CGO_LDFLAGS)"
+	@echo "   BUILD_TAGS  = $(BUILD_TAGS)"
+	@for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; $(GO) install -tags="$(BUILD_TAGS)" $(LDFLAGS) $$target_pkg || exit 1; done
 
 .PHONY: dist
 dist:
 	@echo ">> packaging binaries"
-	@echo "   VERSION    = $(VERSION)"
-	@echo "   GOOS       = $(GOOS)"
-	@echo "   GOARCH     = $(GOARCH)"
-	@echo "   BUILD_TAGS = $(BUILD_TAGS)"
+	@echo "   VERSION     = $(VERSION)"
+	@echo "   GOOS        = $(GOOS)"
+	@echo "   GOARCH      = $(GOARCH)"
+	@echo "   CGO_ENABLED = $(CGO_ENABLED)"
+	@echo "   CGO_CFLAGS  = $(CGO_CFLAGS)"
+	@echo "   CGO_LDFLAGS = $(CGO_LDFLAGS)"
+	@echo "   BUILD_TAGS  = $(BUILD_TAGS)"
 	mkdir -p ./dist/$(GOOS)-$(GOARCH)/bin
-	@for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; GOARCH=$(GOARCH) GOOS=$(GOOS) $(GO) build -tags=${BUILD_TAGS} ${LDFLAGS} -o ./dist/$(GOOS)-$(GOARCH)/bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done
+	@for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; $(GO) build -tags="$(BUILD_TAGS)" $(LDFLAGS) -o ./dist/$(GOOS)-$(GOARCH)/bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done
 	(cd ./dist/$(GOOS)-$(GOARCH); tar zcfv ../blast-${VERSION}.$(GOOS)-$(GOARCH).tar.gz .)
 
 .PHONY: docker
 docker:
 	@echo ">> building docker container image"
-	@echo "   VERSION      = $(VERSION)"
-	@echo "   BUILD_TAGS   = $(BUILD_TAGS)"
-	docker build -t mosuka/blast:v${VERSION} --build-arg VERSION=${VERSION} --build-arg BUILD_TAGS=${BUILD_TAGS} .
+	@echo "   VERSION     = $(VERSION)"
+	docker build -t mosuka/blast:v${VERSION} --build-arg VERSION=${VERSION} .
 
 .PHONY: clean
 clean:
 	@echo ">> cleaning binaries"
 	rm -rf ./bin
+	rm -rf ./data
 	rm -rf ./dist

README.md

@@ -15,23 +15,125 @@ Blast makes it easy for programmers to develop search applications with advanced
 - Index replication
 - An easy-to-use HTTP API
 - CLI is also available
+- Docker container image is also available
 
 ## Building Blast
 
-Building Blast requires Go 1.9 or later. To build Blast on Linux like so:
+Blast requires Bleve and [Bleve Extensions](https://github.com/blevesearch/blevex). Some Bleve Extensions requires C/C++ libraries. The following sections are instructions for satisfying dependencies on particular platforms.
+
+### Requirement
+
+- Go
+- Git
+
+### Ubuntu 18.10
+
+```
+$ sudo apt-get install -y libicu-dev \
+                          libleveldb-dev \
+                          libstemmer-dev \
+                          libgflags-dev \
+                          libsnappy-dev \
+                          zlib1g-dev \
+                          libbz2-dev \
+                          liblz4-dev \
+                          libzstd-dev \
+                          librocksdb-dev \
+                          gcc-4.8 \
+                          g++-4.8 \
+                          build-essential
+
+$ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80
+$ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 80
+$ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 90
+$ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 90
+
+$ export GOPATH=${HOME}/go
+$ go get -u -v github.com/blevesearch/cld2
+$ cd ${GOPATH}/src/github.com/blevesearch/cld2
+$ git clone https://github.com/CLD2Owners/cld2.git
+$ cd cld2/internal
+$ ./compile_libs.sh
+$ sudo cp *.so /usr/local/lib
+```
+
+### macOS High Sierra Version 10.13.6
+
+```bash
+$ brew install icu4c \
+               leveldb \
+               rocksdb \
+               zstd
+
+$ CGO_LDFLAGS="-L/usr/local/opt/icu4c/lib" \
+  CGO_CFLAGS="-I/usr/local/opt/icu4c/include" \
+  go get -u -v github.com/blevesearch/cld2
+$ cd ${GOPATH}/src/github.com/blevesearch/cld2
+$ git clone https://github.com/CLD2Owners/cld2.git
+$ cd cld2/internal
+$ perl -p -i -e 's/soname=/install_name,/' compile_libs.sh
+$ ./compile_libs.sh
+$ sudo cp *.so /usr/local/lib
+```
+
+### Build Blast
+
+Build Blast for Linux as following:
 
 ```bash
 $ git clone git@github.com:mosuka/blast.git
 $ cd blast
 $ make build
 ```
 
-If you want to build Blast other platform, please set `GOOS`、`GOARCH` like following:
+If you want to build for other platform, set `GOOS`, `GOARCH`. For example, build for macOS like following:
 
 ```bash
-$ make GOOS=darwin build
+$ GOOS=darwin \
+  make build
 ```
 
+If you want to build Blast with Bleve and Bleve Extentions (blevex), please set `CGO_LDFLAGS`, `CGO_CFLAGS`, `CGO_ENABLED` and `BUILD_TAGS`. For example, enable Japanese Language Analyzer like following:
+
+```bash
+$ BUILD_TAGS=kagome \
+  make build
+```
+
+You can enable all Bleve Extensions for macOS like following:
+
+```bash
+$ GOOS=darwin \
+  CGO_LDFLAGS="-L/usr/local/opt/icu4c/lib -L/usr/local/opt/rocksdb/lib -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" \
+  CGO_CFLAGS="-I/usr/local/opt/icu4c/include -I/usr/local/opt/rocksdb/include" \
+  CGO_ENABLED=1 \
+  BUILD_TAGS="full" \
+  make build
+```
+
+Also, you can enable all Bleve Extensions for Linux like following:
+
+```bash
+$ GOOS=linux \
+  CGO_LDFLAGS="-L/usr/lib -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" \
+  CGO_CFLAGS="-I/usr/include/rocksdb" \
+  CGO_ENABLED=1 \
+  BUILD_TAGS=full \
+  make build
+```
+
+Please refer to the following table for details of Bleve Extensions:
+
+| | CGO_ENABLED | BUILD_TAGS |
+| --- | --- | --- |
+| Compact Language Detector | 1 | cld2 |
+| cznicb KV store |  | cznicb |
+| Thai Language Analyser | 1 | icu |
+| Japanese Language Analyser |  | kagome |
+| LevelDB | 1 | leveldb |
+| Danish, German, English, Spanish, Finnish, French, Hungarian, Italian, Dutch, Norwegian, Portuguese, Romanian, Russian, Swedish, Turkish Language Stemmer | 1 | libstemmer |
+| RocksDB | 1 | rocksdb |
+
 You can see the binary file when build successful like so:
 
 ```bash
@@ -796,11 +898,11 @@ $ docker run --rm --name blast1 \
     -p 10000:10000 \
     -p 10001:10001 \
     -p 10002:10002 \
-    mosuka/blast:v0.2.0 start \
-    --bind-addr=:10000 \
-    --grpc-addr=:10001 \
-    --http-addr=:10002 \
-    --node-id=node1
+    mosuka/blast:v0.3.0 start \
+      --bind-addr=:10000 \
+      --grpc-addr=:10001 \
+      --http-addr=:10002 \
+      --node-id=node1
 ```
 
 ### Running Blast cluster on Docker Compose

Versionfile

@@ -1 +1 @@
-0.2.0
+0.3.0

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2'
 services:
   blast1:
-    image: mosuka/blast:v0.2.0
+    image: mosuka/blast:v0.3.0
     restart: always
     ports:
     - 10000:10000
@@ -15,7 +15,7 @@ services:
     command: start
 
   blast2:
-    image: mosuka/blast:v0.2.0
+    image: mosuka/blast:v0.3.0
     restart: always
     ports:
     - 11000:10000
@@ -30,7 +30,7 @@ services:
     command: start
 
   blast3:
-    image: mosuka/blast:v0.2.0
+    image: mosuka/blast:v0.3.0
     restart: always
     ports:
     - 12000:10000