Remove TinyGo artifacts from repo but maintain codepaths (#161)

wasilibs · Nov 28, 2024 · 1bb9793 · 1bb9793
1 parent 4ec0a93
commit 1bb9793
Show file tree

Hide file tree

Showing 22 changed files with 26 additions and 171 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -16,20 +16,12 @@ jobs:
       matrix:
         mode:
           - cgo
-          - tinygo
           - wazero
         os:
           - macos-13
           - macos-14
           - ubuntu-22.04
           - windows-2022
-        exclude:
-          # https://github.com/tinygo-org/tinygo/issues/3594
-          - os: windows-2022
-            mode: tinygo
-          # No available binary
-          - os: macos-14
-            mode: tinygo
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
       - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
@@ -54,15 +46,6 @@ jobs:
           # This version of go is not sync with setup-go, but this is probably the best we can do
           pacboy: gcc:p re2:p go:p pkg-config:p diffutils:p
 
-      - name: setup tinygo
-        if: ${{ matrix.mode == 'tinygo' }}
-        uses: acifani/setup-tinygo@b2ba42b249c7d3efdfe94166ec0f48b3191404f7 # v1
-        with:
-          tinygo-version: 0.33.0
-      - name: setup wasmtime for tinygo
-        if: ${{ matrix.mode == 'tinygo' }}
-        run: go install github.com/wasilibs/tools/cmd/wasmtime@875fe73f677c58d467ee373a9e00e6cb66b268f3
-
       - name: run checks
         run: go run ./build ${{ startsWith(matrix.os, 'ubuntu-') && 'check' || 'test' }}
         if: ${{ !startsWith(matrix.os, 'windows-') || matrix.mode != 'cgo' }}

diff --git a/README.md b/README.md
@@ -5,14 +5,10 @@ go-re2 is a drop-in replacement for the standard library [regexp][1] package whi
 re2 is packaged as a WebAssembly module and accessed with the pure Go runtime, [wazero][3].
 This means that it is compatible with any Go application, regardless of availability of cgo.
 
-The library can also be used in a TinyGo application being compiled to WebAssembly. Currently,
-`regexp` when compiled with TinyGo always has very slow performance and sometimes fails to
-compile expressions completely.
-
 Note that if your regular expressions or input are small, this library is slower than the
 standard library. You will generally "know" if your application requires high performance for
 complex regular expressions, for example in security filtering software. If you do not know
-your app has such needs and are not using TinyGo, you should turn away now.
+your app has such needs, you should turn away now.
 
 ## Behavior differences
 
@@ -51,7 +47,7 @@ provide any guarantee of API stability even across minor version updates.
 ## Usage
 
 go-re2 is a standard Go library package and can be added to a go.mod file. It will work fine in
-Go or TinyGo projects.
+any Go project. See [below](#tinygo) for notes about TinyGo support.
 
 ```
 go get github.com/wasilibs/go-re2
@@ -109,6 +105,14 @@ On Mac start by installing [homebrew][9] including installation of the command l
 brew install re2
 ```
 
+### TinyGo
+
+This project began as a way to use re2 with TinyGo WASI projects. However, recent versions of re2 have reworked
+their build, notably depending on absl which requires threads support and breaks compatibility with TinyGo programs.
+To stay up-to-date with re2, after much time this project has removed building of TinyGo-specific Wasm artifacts.
+Build tags to enable cgo codepaths for TinyGo are kept to provide best-effort support with projects bringing their
+own Wasm, with the only known usage currently in [coraza-wasilibs][10].
+
 ## Performance
 
 Benchmarks are run against every commit in the [bench][4] workflow. GitHub action runners are highly
@@ -259,3 +263,4 @@ performance in real world use cases.
 [6]: https://github.com/corazawaf/coraza
 [8]: https://www.msys2.org/
 [9]: https://brew.sh/
+[10]: https://github.com/corazawaf/coraza-wasilibs
diff --git a/build/main.go b/build/main.go
@@ -26,21 +26,16 @@ func main() {
 		Usage: "Runs Go tests.",
 		Action: func(a *goyek.A) {
 			mode := strings.ToLower(os.Getenv("RE2_TEST_MODE"))
-			if mode != "tinygo" {
-				race := "-race"
-				if os.Getenv("TEST_NORACE") != "" {
-					race = ""
-				}
-				cmd.Exec(a, fmt.Sprintf(`go test -v -timeout=20m %s -tags "%s" ./...`, race, strings.Join(tags, ",")))
-				if mode == "" {
-					cmd.Exec(a, fmt.Sprintf("go build -o %s ./internal/e2e", filepath.Join("out", "test.wasm")), cmd.Env("GOOS", "wasip1"), cmd.Env("GOARCH", "wasm"))
-					// Could invoke wazero directly but the CLI has a simpler entry point.
-					cmd.Exec(a, fmt.Sprintf("go run github.com/tetratelabs/wazero/cmd/wazero@v1.7.1 run %s", filepath.Join("out", "test.wasm")))
-				}
-				return
+			race := "-race"
+			if os.Getenv("TEST_NORACE") != "" {
+				race = ""
+			}
+			cmd.Exec(a, fmt.Sprintf(`go test -v -timeout=20m %s -tags "%s" ./...`, race, strings.Join(tags, ",")))
+			if mode == "" {
+				cmd.Exec(a, fmt.Sprintf("go build -o %s ./internal/e2e", filepath.Join("out", "test.wasm")), cmd.Env("GOOS", "wasip1"), cmd.Env("GOARCH", "wasm"))
+				// Could invoke wazero directly but the CLI has a simpler entry point.
+				cmd.Exec(a, fmt.Sprintf("go run github.com/tetratelabs/wazero/cmd/wazero@v1.7.1 run %s", filepath.Join("out", "test.wasm")))
 			}
-
-			cmd.Exec(a, fmt.Sprintf(`tinygo test -scheduler=none -gc=custom -target=wasip1 -v -tags "%s" ./...`, strings.Join(tags, ",")))
 		},
 	}))
 
@@ -63,11 +58,8 @@ func buildTags() []string {
 	exhaustive := os.Getenv("RE2_TEST_EXHAUSTIVE") == "1"
 
 	var tags []string
-	switch mode {
-	case "cgo":
+	if mode == "cgo" {
 		tags = append(tags, "re2_cgo")
-	case "tinygo":
-		tags = append(tags, "custommalloc")
 	}
 	if exhaustive {
 		tags = append(tags, "re2_test_exhaustive")

diff --git a/buildtools/re2/Dockerfile b/buildtools/re2/Dockerfile
@@ -1,32 +1,9 @@
-# Copyright 2022 The OWASP Coraza contributors
-# SPDX-License-Identifier: Apache-2.0
-
 FROM ghcr.io/webassembly/wasi-sdk:wasi-sdk-22 AS base
 
 RUN apt-get update && apt-get install -y binaryen curl
 
-RUN mkdir -p /re2 && curl -L https://github.com/google/re2/archive/refs/tags/2023-03-01.tar.gz | tar -xz --strip-components 1 -C /re2
 WORKDIR /re2
-
-# We need to build twice independently, once without threads for tinygo, and once with for wazero.
-
-FROM base AS tinygo
-
-ENV CXXFLAGS -fno-exceptions -O3 ${CXXFLAGS}
-ENV RE2_CXXFLAGS -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -I. -DRE2_NO_THREADS
-ENV LDFLAGS $CFLAGS -mexec-model=reactor
-
-RUN make obj/libre2.a
-
-WORKDIR /cre2
-ADD internal/cre2/cre2.cpp /cre2
-ADD internal/cre2/cre2.h /cre2
-# Just one source file so not worth running make
-RUN $CXX -c cre2.cpp -o cre2.o -I. -I/re2 $CXXFLAGS \
-  && $AR cru libcre2.a cre2.o \
-  && $RANLIB libcre2.a
-
-FROM base AS wazero
+RUN curl -L https://github.com/google/re2/archive/refs/tags/2023-03-01.tar.gz | tar -xz --strip-components 1 -C /re2
 
 ENV CFLAGS --target=wasm32-wasi-threads --sysroot=/wasi-sysroot -pthread -O3 -D__USE_ISOC11 -msimd128
 ENV CXXFLAGS -fno-exceptions $CFLAGS
@@ -74,17 +51,4 @@ RUN $CXX -o libcre2-noopt.so -Wl,--global-base=1024 $LDFLAGS \
 
 RUN wasm-opt -o libcre2.so --low-memory-unused --flatten --rereloop --converge -O3 libcre2-noopt.so
 
-FROM ghcr.io/webassembly/wasi-sdk:wasi-sdk-22
-
-COPY --from=tinygo /re2/obj/libre2.a libre2.a
-COPY --from=tinygo /cre2/libcre2.a libcre2.a
-COPY --from=wazero /cre2/libcre2.so libcre2.so
-
-CMD ["cp", \
-  "libre2.a", \
-  "libcre2.a", \
-  "libcre2.so", \
-  "/wasi-sysroot/lib/wasm32-wasi/libc++.a", \
-  "/wasi-sysroot/lib/wasm32-wasi/libc++abi.a", \
-  "/usr/lib/llvm-17/lib/clang/17/lib/wasi/libclang_rt.builtins-wasm32.a", \
-  "/out/"]
+CMD ["cp", "libcre2.so", "/out/"]
diff --git a/concurrency_test.go b/concurrency_test.go
@@ -1,5 +1,3 @@
-//go:build !tinygo.wasm
-
 package re2
 
 import (

diff --git a/exec2_test.go b/exec2_test.go
@@ -1,6 +1,4 @@
-// Disable on tinygo.wasm for now since due to the poor performance of the default GC,
-// these tests don't complete.
-//go:build re2_test_exhaustive && !tinygo.wasm
+//go:build re2_test_exhaustive
 
 package re2
 

diff --git a/exec_test.go b/exec_test.go
@@ -1,6 +1,4 @@
-// Currently these tests run too slow with TinyGo, likely due to low default GC performance so
-// disable by default there.
-//go:build !tinygo.wasm || re2_test_exhaustive
+//go:build re2_test_exhaustive
 
 package re2
 

diff --git a/experimental/init_tinygowasm_test.go b/experimental/init_tinygowasm_test.go
diff --git a/go.mod b/go.mod
@@ -4,11 +4,7 @@ go 1.21
 
 require (
 	github.com/tetratelabs/wazero v1.8.2
-	github.com/wasilibs/nottinygc v0.7.1
 	github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52
 )
 
-require (
-	github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a // indirect
-	golang.org/x/sys v0.21.0 // indirect
-)
+require golang.org/x/sys v0.21.0 // indirect
diff --git a/go.sum b/go.sum
@@ -1,15 +1,11 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a h1:tdPcGgyiH0K+SbsJBBm2oPyEIOTAvLBwD9TuUwVtZho=
-github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
 github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
-github.com/wasilibs/nottinygc v0.7.1 h1:rKu19+SFniRNuSo5NX7/wxpSpXmMUmkcyt/YiWLJg8w=
-github.com/wasilibs/nottinygc v0.7.1/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
 github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4=
 github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
 golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=

diff --git a/init_tinygowasm.go b/init_tinygowasm.go
diff --git a/init_tinygowasm_test.go b/init_tinygowasm_test.go
diff --git a/internal/e2e/go.sum b/internal/e2e/go.sum
@@ -1,15 +1,11 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a h1:tdPcGgyiH0K+SbsJBBm2oPyEIOTAvLBwD9TuUwVtZho=
-github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
 github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
-github.com/wasilibs/nottinygc v0.7.1 h1:rKu19+SFniRNuSo5NX7/wxpSpXmMUmkcyt/YiWLJg8w=
-github.com/wasilibs/nottinygc v0.7.1/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
 github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4=
 github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
 golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=

diff --git a/internal/re2_tinygowasm.go → internal/re2_re2_cgo.go b/internal/re2_tinygowasm.go → internal/re2_re2_cgo.go
diff --git a/internal/wasm/libc++.a b/internal/wasm/libc++.a
diff --git a/internal/wasm/libc++abi.a b/internal/wasm/libc++abi.a
diff --git a/internal/wasm/libclang_rt.builtins-wasm32.a b/internal/wasm/libclang_rt.builtins-wasm32.a
diff --git a/internal/wasm/libcre2.a b/internal/wasm/libcre2.a
diff --git a/internal/wasm/libre2.a b/internal/wasm/libre2.a
diff --git a/malloc_tinygowasm.go b/malloc_tinygowasm.go
diff --git a/stress_test.go b/stress_test.go
@@ -1,5 +1,3 @@
-//go:build !tinygo.wasm
-
 package re2
 
 import (

diff --git a/wafbench/go.sum b/wafbench/go.sum
@@ -302,8 +302,6 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/wasilibs/nottinygc v0.7.1 h1:rKu19+SFniRNuSo5NX7/wxpSpXmMUmkcyt/YiWLJg8w=
-github.com/wasilibs/nottinygc v0.7.1/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
 github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4=
 github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
 github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=