diff --git a/.gitignore b/.gitignore index c078dd8..3c8ff32 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,6 @@ .idea/ # Blast -*.log -bin -data +bin/ +data/ +dist/ diff --git a/Dockerfile b/Dockerfile index da11b3b..3366e18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,24 +12,72 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.10.3 +FROM ubuntu:18.10 ARG VERSION= -ARG BUILD_TAGS= -COPY . /go/src/github.com/mosuka/blast +ENV GOPATH /go -RUN cd /go/src/github.com/mosuka/blast && \ - make GOOS=linux GOARCH=amd64 VERSION=${VERSION} BUILD_TAG=${BUILD_TAGS} build +COPY . ${GOPATH}/src/github.com/mosuka/blast +RUN apt-get update && \ + apt-get install -y git \ + golang \ + libicu-dev \ + libleveldb-dev \ + libstemmer-dev \ + libgflags-dev \ + libsnappy-dev \ + zlib1g-dev \ + libbz2-dev \ + liblz4-dev \ + libzstd-dev \ + librocksdb-dev \ + gcc-4.8 \ + g++-4.8 \ + build-essential && \ + apt-get clean && \ -FROM alpine:3.7 -#FROM scratch + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 80 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 90 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 90 && \ + + go get -u -v github.com/blevesearch/cld2 && \ + cd ${GOPATH}/src/github.com/blevesearch/cld2 && \ + git clone https://github.com/CLD2Owners/cld2.git && \ + cd cld2/internal && \ + ./compile_libs.sh && \ + cp *.so /usr/local/lib && \ + + cd ${GOPATH}/src/github.com/mosuka/blast && \ + GOOS=linux \ + GOARCH=amd64 \ + CGO_ENABLED=1 \ + CGO_CFLAGS="-I/usr/include/rocksdb" \ + CGO_LDFLAGS="-L/usr/lib -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" \ + BUILD_TAGS=full \ + VERSION=${VERSION} \ + make build + +FROM ubuntu:18.10 MAINTAINER Minoru Osuka "minoru.osuka@gmail.com" -RUN apk --no-cache update +RUN apt-get update && \ + apt-get install -y libicu-dev \ + libleveldb-dev \ + libstemmer-dev \ + libgflags-dev \ + libsnappy-dev \ + zlib1g-dev \ + libbz2-dev \ + liblz4-dev \ + libzstd-dev \ + librocksdb-dev && \ + apt-get clean +COPY --from=0 /go/src/github.com/blevesearch/cld2/cld2/internal/*.so /usr/local/lib/ COPY --from=0 /go/src/github.com/mosuka/blast/bin/blast /usr/bin/blast EXPOSE 10000 10001 10002 diff --git a/Makefile b/Makefile index d1d5e78..6b69f19 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,16 @@ # limitations under the License. VERSION ?= -GOOS = linux -GOARCH = amd64 -BUILD_TAGS = kagome -BIN_EXT = +GOOS ?= linux +GOARCH ?= amd64 +BUILD_TAGS ?= +CGO_ENABLED ?= 0 +CGO_CFLAGS ?= +CGO_LDFLAGS ?= +GO15VENDOREXPERIMENT ?= 1 +BIN_EXT ?= -GO := CGO_ENABLED=0 GO15VENDOREXPERIMENT=1 go +GO := CGO_ENABLED=$(CGO_ENABLED) GO15VENDOREXPERIMENT=$(CGO_ENABLED) go PACKAGES = $(shell $(GO) list ./... | grep -v '/vendor/') @@ -29,7 +33,7 @@ TARGET_PACKAGES = $(shell find . -name 'main.go' -print0 | xargs -0 -n1 dirname ifeq ($(VERSION),) VERSION = $(shell cat ./Versionfile) endif -LDFLAGS = -ldflags "-X \"github.com/mosuka/blast/version.Version=${VERSION}\"" +LDFLAGS = -ldflags "-X \"github.com/mosuka/blast/version.Version=$(VERSION)\"" ifeq ($(GOOS),windows) BIN_EXT = .exe @@ -65,48 +69,60 @@ format: .PHONY: test test: @echo ">> testing all packages" - @echo " VERSION = $(VERSION)" - @echo " BUILD_TAGS = $(BUILD_TAGS)" - @$(GO) test -v -tags=${BUILD_TAGS} ${LDFLAGS} $(PACKAGES) + @echo " VERSION = $(VERSION)" + @echo " CGO_ENABLED = $(CGO_ENABLED)" + @echo " CGO_CFLAGS = $(CGO_CFLAGS)" + @echo " CGO_LDFLAGS = $(CGO_LDFLAGS)" + @echo " BUILD_TAGS = $(BUILD_TAGS)" + @$(GO) test -v -tags="$(BUILD_TAGS)" $(LDFLAGS) $(PACKAGES) .PHONY: build build: @echo ">> building binaries" - @echo " VERSION = $(VERSION)" - @echo " GOOS = $(GOOS)" - @echo " GOARCH = $(GOARCH)" - @echo " BUILD_TAGS = $(BUILD_TAGS)" - @for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; GOARCH=$(GOARCH) GOOS=$(GOOS) $(GO) build -tags=${BUILD_TAGS} ${LDFLAGS} -o ./bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done + @echo " VERSION = $(VERSION)" + @echo " GOOS = $(GOOS)" + @echo " GOARCH = $(GOARCH)" + @echo " CGO_ENABLED = $(CGO_ENABLED)" + @echo " CGO_CFLAGS = $(CGO_CFLAGS)" + @echo " CGO_LDFLAGS = $(CGO_LDFLAGS)" + @echo " BUILD_TAGS = $(BUILD_TAGS)" + @for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; $(GO) build -tags="$(BUILD_TAGS)" $(LDFLAGS) -o ./bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done .PHONY: install install: @echo ">> installing binaries" - @echo " VERSION = $(VERSION)" - @echo " GOOS = $(GOOS)" - @echo " GOARCH = $(GOARCH)" - @echo " BUILD_TAGS = $(BUILD_TAGS)" - @for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; GOARCH=$(GOARCH) GOOS=$(GOOS) $(GO) install -tags=${BUILD_TAGS} ${LDFLAGS} $$target_pkg || exit 1; done + @echo " VERSION = $(VERSION)" + @echo " GOOS = $(GOOS)" + @echo " GOARCH = $(GOARCH)" + @echo " CGO_ENABLED = $(CGO_ENABLED)" + @echo " CGO_CFLAGS = $(CGO_CFLAGS)" + @echo " CGO_LDFLAGS = $(CGO_LDFLAGS)" + @echo " BUILD_TAGS = $(BUILD_TAGS)" + @for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; $(GO) install -tags="$(BUILD_TAGS)" $(LDFLAGS) $$target_pkg || exit 1; done .PHONY: dist dist: @echo ">> packaging binaries" - @echo " VERSION = $(VERSION)" - @echo " GOOS = $(GOOS)" - @echo " GOARCH = $(GOARCH)" - @echo " BUILD_TAGS = $(BUILD_TAGS)" + @echo " VERSION = $(VERSION)" + @echo " GOOS = $(GOOS)" + @echo " GOARCH = $(GOARCH)" + @echo " CGO_ENABLED = $(CGO_ENABLED)" + @echo " CGO_CFLAGS = $(CGO_CFLAGS)" + @echo " CGO_LDFLAGS = $(CGO_LDFLAGS)" + @echo " BUILD_TAGS = $(BUILD_TAGS)" mkdir -p ./dist/$(GOOS)-$(GOARCH)/bin - @for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; GOARCH=$(GOARCH) GOOS=$(GOOS) $(GO) build -tags=${BUILD_TAGS} ${LDFLAGS} -o ./dist/$(GOOS)-$(GOARCH)/bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done + @for target_pkg in $(TARGET_PACKAGES); do echo $$target_pkg; $(GO) build -tags="$(BUILD_TAGS)" $(LDFLAGS) -o ./dist/$(GOOS)-$(GOARCH)/bin/`basename $$target_pkg`$(BIN_EXT) $$target_pkg || exit 1; done (cd ./dist/$(GOOS)-$(GOARCH); tar zcfv ../blast-${VERSION}.$(GOOS)-$(GOARCH).tar.gz .) .PHONY: docker docker: @echo ">> building docker container image" - @echo " VERSION = $(VERSION)" - @echo " BUILD_TAGS = $(BUILD_TAGS)" - docker build -t mosuka/blast:v${VERSION} --build-arg VERSION=${VERSION} --build-arg BUILD_TAGS=${BUILD_TAGS} . + @echo " VERSION = $(VERSION)" + docker build -t mosuka/blast:v${VERSION} --build-arg VERSION=${VERSION} . .PHONY: clean clean: @echo ">> cleaning binaries" rm -rf ./bin + rm -rf ./data rm -rf ./dist diff --git a/README.md b/README.md index 2af1305..1fe2e89 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,70 @@ Blast makes it easy for programmers to develop search applications with advanced - Index replication - An easy-to-use HTTP API - CLI is also available +- Docker container image is also available ## Building Blast -Building Blast requires Go 1.9 or later. To build Blast on Linux like so: +Blast requires Bleve and [Bleve Extensions](https://github.com/blevesearch/blevex). Some Bleve Extensions requires C/C++ libraries. The following sections are instructions for satisfying dependencies on particular platforms. + +### Requirement + +- Go +- Git + +### Ubuntu 18.10 + +``` +$ sudo apt-get install -y libicu-dev \ + libleveldb-dev \ + libstemmer-dev \ + libgflags-dev \ + libsnappy-dev \ + zlib1g-dev \ + libbz2-dev \ + liblz4-dev \ + libzstd-dev \ + librocksdb-dev \ + gcc-4.8 \ + g++-4.8 \ + build-essential + +$ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 +$ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 80 +$ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 90 +$ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 90 + +$ export GOPATH=${HOME}/go +$ go get -u -v github.com/blevesearch/cld2 +$ cd ${GOPATH}/src/github.com/blevesearch/cld2 +$ git clone https://github.com/CLD2Owners/cld2.git +$ cd cld2/internal +$ ./compile_libs.sh +$ sudo cp *.so /usr/local/lib +``` + +### macOS High Sierra Version 10.13.6 + +```bash +$ brew install icu4c \ + leveldb \ + rocksdb \ + zstd + +$ CGO_LDFLAGS="-L/usr/local/opt/icu4c/lib" \ + CGO_CFLAGS="-I/usr/local/opt/icu4c/include" \ + go get -u -v github.com/blevesearch/cld2 +$ cd ${GOPATH}/src/github.com/blevesearch/cld2 +$ git clone https://github.com/CLD2Owners/cld2.git +$ cd cld2/internal +$ perl -p -i -e 's/soname=/install_name,/' compile_libs.sh +$ ./compile_libs.sh +$ sudo cp *.so /usr/local/lib +``` + +### Build Blast + +Build Blast for Linux as following: ```bash $ git clone git@github.com:mosuka/blast.git @@ -26,12 +86,54 @@ $ cd blast $ make build ``` -If you want to build Blast other platform, please set `GOOS`、`GOARCH` like following: +If you want to build for other platform, set `GOOS`, `GOARCH`. For example, build for macOS like following: ```bash -$ make GOOS=darwin build +$ GOOS=darwin \ + make build ``` +If you want to build Blast with Bleve and Bleve Extentions (blevex), please set `CGO_LDFLAGS`, `CGO_CFLAGS`, `CGO_ENABLED` and `BUILD_TAGS`. For example, enable Japanese Language Analyzer like following: + +```bash +$ BUILD_TAGS=kagome \ + make build +``` + +You can enable all Bleve Extensions for macOS like following: + +```bash +$ GOOS=darwin \ + CGO_LDFLAGS="-L/usr/local/opt/icu4c/lib -L/usr/local/opt/rocksdb/lib -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" \ + CGO_CFLAGS="-I/usr/local/opt/icu4c/include -I/usr/local/opt/rocksdb/include" \ + CGO_ENABLED=1 \ + BUILD_TAGS="full" \ + make build +``` + +Also, you can enable all Bleve Extensions for Linux like following: + +```bash +$ GOOS=linux \ + CGO_LDFLAGS="-L/usr/lib -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" \ + CGO_CFLAGS="-I/usr/include/rocksdb" \ + CGO_ENABLED=1 \ + BUILD_TAGS=full \ + make build +``` + +Please refer to the following table for details of Bleve Extensions: + +| | CGO_ENABLED | BUILD_TAGS | +| --- | --- | --- | +| Compact Language Detector | 1 | cld2 | +| cznicb KV store | | cznicb | +| Thai Language Analyser | 1 | icu | +| Japanese Language Analyser | | kagome | +| LevelDB | 1 | leveldb | +| Danish, German, English, Spanish, Finnish, French, Hungarian, Italian, Dutch, Norwegian, Portuguese, Romanian, Russian, Swedish, Turkish Language Stemmer | 1 | libstemmer | +| RocksDB | 1 | rocksdb | + You can see the binary file when build successful like so: ```bash @@ -796,11 +898,11 @@ $ docker run --rm --name blast1 \ -p 10000:10000 \ -p 10001:10001 \ -p 10002:10002 \ - mosuka/blast:v0.2.0 start \ - --bind-addr=:10000 \ - --grpc-addr=:10001 \ - --http-addr=:10002 \ - --node-id=node1 + mosuka/blast:v0.3.0 start \ + --bind-addr=:10000 \ + --grpc-addr=:10001 \ + --http-addr=:10002 \ + --node-id=node1 ``` ### Running Blast cluster on Docker Compose diff --git a/Versionfile b/Versionfile index 0ea3a94..0d91a54 100644 --- a/Versionfile +++ b/Versionfile @@ -1 +1 @@ -0.2.0 +0.3.0 diff --git a/docker-compose.yml b/docker-compose.yml index 3a77708..14974a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: blast1: - image: mosuka/blast:v0.2.0 + image: mosuka/blast:v0.3.0 restart: always ports: - 10000:10000 @@ -15,7 +15,7 @@ services: command: start blast2: - image: mosuka/blast:v0.2.0 + image: mosuka/blast:v0.3.0 restart: always ports: - 11000:10000 @@ -30,7 +30,7 @@ services: command: start blast3: - image: mosuka/blast:v0.2.0 + image: mosuka/blast:v0.3.0 restart: always ports: - 12000:10000