From 517ffa07b435b311bb48a23c209258d31c523b5f Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Tue, 23 May 2023 22:49:29 +0200 Subject: [PATCH 1/6] build against libre2-dev if found --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 334e738c8ce..f4fc765741e 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Bu endif export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ - -trimpath -tags netgo,osusergo,sqlite_omit_load_extension + -trimpath -tags netgo,osusergo,sqlite_omit_load_extension$(RE2_TAG) ifneq (,$(TEST_COVERAGE)) LD_OPTS += -cover @@ -76,6 +76,13 @@ clean: testclean @$(RM) ./$(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_BIN) $(WIN_IGNORE_ERR) @$(RM) ./$(DUMMY_PLUGIN_FOLDER)/$(DUMMY_PLUGIN_BIN) $(WIN_IGNORE_ERR) +RE2_TAG := $(shell echo "int main() { return 0; }" | $(CC) -x c - -o /dev/null -lre2 >/dev/null 2>&1 && echo ,re2_cgo) + +ifneq (,$(RE2_TAG)) +$(info Using C++ regexp library) +else +$(info Using WebAssembly regexp library) +endif cscli: goversion @$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS) From abc65977fdf42a20fc28acd069b07629f2bd5dd1 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Fri, 2 Jun 2023 14:35:30 +0200 Subject: [PATCH 2/6] moved stuff in the makefile, make it possible to choose c++ or wasm --- Makefile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index f4fc765741e..917e21d1b5a 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,16 @@ ifdef BUILD_STATIC $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) endif +# override with "make RE2_TAG=" to use the WebAssembly regexp library +# override with "make RE2_TAG=,re2_cgo" to use the C++ regexp library +RE2_TAG ?= $(shell echo "int main() { return 0; }" | $(CC) -x c - -o /dev/null -lre2 >/dev/null 2>&1 && echo ,re2_cgo) + +ifneq (,$(RE2_TAG)) +$(info Using C++ regexp library) +else +$(info Using WebAssembly regexp library) +endif + export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ -trimpath -tags netgo,osusergo,sqlite_omit_load_extension$(RE2_TAG) @@ -76,14 +86,6 @@ clean: testclean @$(RM) ./$(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_BIN) $(WIN_IGNORE_ERR) @$(RM) ./$(DUMMY_PLUGIN_FOLDER)/$(DUMMY_PLUGIN_BIN) $(WIN_IGNORE_ERR) -RE2_TAG := $(shell echo "int main() { return 0; }" | $(CC) -x c - -o /dev/null -lre2 >/dev/null 2>&1 && echo ,re2_cgo) - -ifneq (,$(RE2_TAG)) -$(info Using C++ regexp library) -else -$(info Using WebAssembly regexp library) -endif - cscli: goversion @$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS) From c132d5c07acdd2d70b03b36cd7558bdbdfd0e0f0 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Fri, 2 Jun 2023 15:27:32 +0200 Subject: [PATCH 3/6] use wasm by default in windows, override with RE2_TAG --- Makefile | 6 +----- mk/platform/unix_common.mk | 4 ++++ mk/platform/windows.mk | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 917e21d1b5a..df2895b1680 100644 --- a/Makefile +++ b/Makefile @@ -38,10 +38,6 @@ ifdef BUILD_STATIC $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) endif -# override with "make RE2_TAG=" to use the WebAssembly regexp library -# override with "make RE2_TAG=,re2_cgo" to use the C++ regexp library -RE2_TAG ?= $(shell echo "int main() { return 0; }" | $(CC) -x c - -o /dev/null -lre2 >/dev/null 2>&1 && echo ,re2_cgo) - ifneq (,$(RE2_TAG)) $(info Using C++ regexp library) else @@ -49,7 +45,7 @@ $(info Using WebAssembly regexp library) endif export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ - -trimpath -tags netgo,osusergo,sqlite_omit_load_extension$(RE2_TAG) + -trimpath -tags netgo,osusergo,sqlite_omit_load_extension$(addprefix ,$(RE2_TAG)) ifneq (,$(TEST_COVERAGE)) LD_OPTS += -cover diff --git a/mk/platform/unix_common.mk b/mk/platform/unix_common.mk index f611693f48b..e0456da7d15 100644 --- a/mk/platform/unix_common.mk +++ b/mk/platform/unix_common.mk @@ -12,3 +12,7 @@ BUILD_VERSION?=$(shell git describe --tags) BUILD_TIMESTAMP=$(shell date +%F"_"%T) DEFAULT_CONFIGDIR?=/etc/crowdsec DEFAULT_DATADIR?=/var/lib/crowdsec/data + +# override with "make RE2_TAG=" to use the WebAssembly regexp library +# override with "make RE2_TAG=re2_cgo" to use the C++ regexp library +RE2_TAG ?= $(shell echo "int main() { return 0; }" | $(CC) -x c - -o /dev/null -lre2 >/dev/null 2>&1 && echo re2_cgo) diff --git a/mk/platform/windows.mk b/mk/platform/windows.mk index e4c5e8a1234..0e984c80b9a 100644 --- a/mk/platform/windows.mk +++ b/mk/platform/windows.mk @@ -19,4 +19,8 @@ CPR=Copy-Item -Recurse MKDIR=New-Item -ItemType directory WIN_IGNORE_ERR=; exit 0 +# Link to the C++ re2 library with "make RE2_TAG=re2_cgo" +# or webassembly by default +RE2_TAG ?= + $(info Building for windows) From 531a426e89f2c47aba5ec35f3e82ab00b4b56fd8 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Tue, 6 Jun 2023 09:11:54 +0200 Subject: [PATCH 4/6] fix addprefix with comma --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index df2895b1680..048e912d760 100644 --- a/Makefile +++ b/Makefile @@ -44,8 +44,10 @@ else $(info Using WebAssembly regexp library) endif +COMMA=, + export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ - -trimpath -tags netgo,osusergo,sqlite_omit_load_extension$(addprefix ,$(RE2_TAG)) + -trimpath -tags netgo,osusergo,sqlite_omit_load_extension$(addprefix $(COMMA),$(RE2_TAG)) ifneq (,$(TEST_COVERAGE)) LD_OPTS += -cover From 84bfb9962e6f2c5cc04ab4f135d2a91604a36de5 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Tue, 6 Jun 2023 11:40:49 +0200 Subject: [PATCH 5/6] Detect re2 with pkg-config, add Libre2 variable to cwversion --- Makefile | 51 +++++++++++++++++++++++++------------- mk/platform/unix_common.mk | 4 --- mk/platform/windows.mk | 4 --- pkg/cwversion/version.go | 2 ++ 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 82647be6a4c..d6f1b95f237 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ include mk/platform.mk BUILD_REQUIRE_GO_MAJOR ?= 1 BUILD_REQUIRE_GO_MINOR ?= 20 +GOCMD = go +GOTEST = $(GOCMD) test + BUILD_CODENAME ?= alphaga CROWDSEC_FOLDER = ./cmd/crowdsec @@ -14,8 +17,21 @@ PLUGINS_DIR = ./plugins/notifications CROWDSEC_BIN = crowdsec$(EXT) CSCLI_BIN = cscli$(EXT) +# Directory for the release files +RELDIR = crowdsec-$(BUILD_VERSION) + GO_MODULE_NAME = github.com/crowdsecurity/crowdsec +# see if we have libre2-dev installed for C++ optimizations +RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null) + +#-------------------------------------- +# +# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/ and plugins/ +# + +MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)" + LD_OPTS_VARS= \ -X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Version=$(BUILD_VERSION)' \ -X 'github.com/crowdsecurity/go-cs-lib/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \ @@ -28,38 +44,37 @@ ifneq (,$(DOCKER_BUILD)) LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.System=docker' endif -ifdef BUILD_STATIC -$(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) -endif +GO_TAGS := netgo,osusergo,sqlite_omit_load_extension -ifneq (,$(RE2_TAG)) -$(info Using C++ regexp library) -else -$(info Using WebAssembly regexp library) +ifneq (,$(RE2_CHECK)) +# += adds a space that we don't want +GO_TAGS := $(GO_TAGS),re2_cgo +LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++' endif -COMMA=, - export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ - -trimpath -tags netgo,osusergo,sqlite_omit_load_extension$(addprefix $(COMMA),$(RE2_TAG)) + -trimpath -tags $(GO_TAGS) ifneq (,$(TEST_COVERAGE)) LD_OPTS += -cover endif -GOCMD = go -GOTEST = $(GOCMD) test - -RELDIR = crowdsec-$(BUILD_VERSION) - -# flags for sub-makefiles -MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)" +#-------------------------------------- .PHONY: build build: pre-build goversion crowdsec cscli plugins +.PHONY: pre-build pre-build: +ifdef BUILD_STATIC + $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) +endif $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH)) +ifneq (,$(RE2_CHECK)) + $(info Using C++ regexp library) +else + $(info Fallback to WebAssembly regexp library. To use the C++ version, make sure you have installed libre2-dev and pkg-config.) +endif $(info ) .PHONY: all @@ -83,9 +98,11 @@ clean: testclean $(MAKE) -C $(PLUGINS_DIR)/$(plugin) clean $(MAKE_FLAGS); \ ) +.PHONY: cscli cscli: goversion @$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS) +.PHONY: crowdsec crowdsec: goversion @$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS) diff --git a/mk/platform/unix_common.mk b/mk/platform/unix_common.mk index e0456da7d15..f611693f48b 100644 --- a/mk/platform/unix_common.mk +++ b/mk/platform/unix_common.mk @@ -12,7 +12,3 @@ BUILD_VERSION?=$(shell git describe --tags) BUILD_TIMESTAMP=$(shell date +%F"_"%T) DEFAULT_CONFIGDIR?=/etc/crowdsec DEFAULT_DATADIR?=/var/lib/crowdsec/data - -# override with "make RE2_TAG=" to use the WebAssembly regexp library -# override with "make RE2_TAG=re2_cgo" to use the C++ regexp library -RE2_TAG ?= $(shell echo "int main() { return 0; }" | $(CC) -x c - -o /dev/null -lre2 >/dev/null 2>&1 && echo re2_cgo) diff --git a/mk/platform/windows.mk b/mk/platform/windows.mk index a617cf34df1..8e2cdf19b40 100644 --- a/mk/platform/windows.mk +++ b/mk/platform/windows.mk @@ -18,7 +18,3 @@ CP=Copy-Item CPR=Copy-Item -Recurse MKDIR=New-Item -ItemType directory WIN_IGNORE_ERR=; exit 0 - -# Link to the C++ re2 library with "make RE2_TAG=re2_cgo" -# or webassembly by default -RE2_TAG ?= diff --git a/pkg/cwversion/version.go b/pkg/cwversion/version.go index f910159fa9e..aeac6f2f22c 100644 --- a/pkg/cwversion/version.go +++ b/pkg/cwversion/version.go @@ -20,6 +20,7 @@ var ( Constraint_scenario = ">= 1.0, < 3.0" Constraint_api = "v1" Constraint_acquis = ">= 1.0, < 2.0" + Libre2 = "WebAssembly" ) func ShowStr() string { @@ -38,6 +39,7 @@ func Show() { log.Printf("BuildDate: %s", version.BuildDate) log.Printf("GoVersion: %s", version.GoVersion) log.Printf("Platform: %s\n", System) + log.Printf("libre2: %s\n", Libre2) log.Printf("Constraint_parser: %s", Constraint_parser) log.Printf("Constraint_scenario: %s", Constraint_scenario) log.Printf("Constraint_api: %s", Constraint_api) From 3c2e3b24479cec9fccd76df63f27db3afad61e10 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Tue, 6 Jun 2023 12:35:03 +0200 Subject: [PATCH 6/6] re2 for the docker version --- Dockerfile | 12 +++++++++++- Dockerfile.debian | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b3b958896b2..da1c3ab06a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,9 +7,19 @@ WORKDIR /go/src/crowdsec COPY . . +# Alpine does not ship a static version of re2, we can build it ourselves +# Later versions require 'abseil', which is likewise not available in its static form +ENV RE2_VERSION=2023-03-01 + # wizard.sh requires GNU coreutils -RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \ +RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \ + wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ + tar -xzf ${RE2_VERSION}.tar.gz && \ + cd re2-${RE2_VERSION} && \ + make && \ + make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ + cd - && \ make clean release DOCKER_BUILD=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ diff --git a/Dockerfile.debian b/Dockerfile.debian index b4cc4c9ec1a..10b06befdf0 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -12,7 +12,7 @@ ENV DEBCONF_NOWARNINGS="yes" # wizard.sh requires GNU coreutils RUN apt-get update && \ - apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \ + apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ make clean release DOCKER_BUILD=1 && \ cd crowdsec-v* && \