From 79942cf34b055c813bbca5804ba965cde2d4de8e Mon Sep 17 00:00:00 2001 From: Morgan Date: Wed, 15 May 2024 11:34:00 +0200 Subject: [PATCH] feat(pkg/db): remove cleveldb, fsdb and rocksdb (#1714) Split from #1602. Context: > @zivkovicmilos:[^1] Unpopular opinion: we should probably just drop these DBs entirely from the codebase, they are never gonna be used in place of LevelDB in the near future > > @ajnavarro: IMHO the needed database is defined by the application needs, not from any other external usecases. Sticking to one of the existing key/value databases, we can use their special capabilities to make it more performant for our use case (yes, even if they are all key/value DBs, they differ in a lot of low-level functionality). \ > I would say to use one and add support in the future for more if needed. After a discussion in the review meeting w/ @jaekwon, this PR is now changed to maintain another reference implementation with boltdb. It also makes sure that it is selectable, when available, from the tm2 node configuration. After this PR, there are no databases in the codebase using cgo. [^1]: https://github.com/gnolang/gno/pull/1602#discussion_r1490862277 --- .github/dependabot.yml | 5 - .github/workflows/lint_template.yml | 2 - .github/workflows/test_template.yml | 2 - .github/workflows/tm2.yml | 4 +- contribs/gnodev/go.mod | 1 + contribs/gnodev/go.sum | 6 +- contribs/gnokeykc/go.sum | 4 - gno.land/pkg/gnoland/app.go | 1 + go.mod | 3 - go.sum | 16 +- tm2/Makefile | 4 +- tm2/pkg/bft/config/config.go | 16 +- tm2/pkg/bft/config/config_test.go | 3 + tm2/pkg/bft/state/state_test.go | 4 +- tm2/pkg/db/_all/all.go | 5 - tm2/pkg/db/_all/all_cgo.go | 10 - tm2/pkg/db/_tags/cleveldb.go | 5 - tm2/pkg/db/_tags/fsdb.go | 5 - tm2/pkg/db/_tags/rocksdb.go | 5 - tm2/pkg/db/cleveldb/c_level_db.go | 326 ----------------------- tm2/pkg/db/cleveldb/c_level_db_test.go | 113 -------- tm2/pkg/db/cleveldb/no_cgo_test.go | 9 - tm2/pkg/db/db.go | 12 - tm2/pkg/db/fsdb/fsdb.go | 276 -------------------- tm2/pkg/db/rocksdb/grocks_db.go | 342 ------------------------- tm2/pkg/db/rocksdb/grocks_db_test.go | 34 --- tm2/pkg/db/rocksdb/no_cgo_test.go | 9 - tm2/pkg/iavl/benchmarks/bench_test.go | 14 +- tm2/pkg/iavl/with_gcc_test.go | 19 -- 29 files changed, 23 insertions(+), 1232 deletions(-) delete mode 100644 tm2/pkg/db/_all/all_cgo.go delete mode 100644 tm2/pkg/db/_tags/cleveldb.go delete mode 100644 tm2/pkg/db/_tags/fsdb.go delete mode 100644 tm2/pkg/db/_tags/rocksdb.go delete mode 100644 tm2/pkg/db/cleveldb/c_level_db.go delete mode 100644 tm2/pkg/db/cleveldb/c_level_db_test.go delete mode 100644 tm2/pkg/db/cleveldb/no_cgo_test.go delete mode 100644 tm2/pkg/db/fsdb/fsdb.go delete mode 100644 tm2/pkg/db/rocksdb/grocks_db.go delete mode 100644 tm2/pkg/db/rocksdb/grocks_db_test.go delete mode 100644 tm2/pkg/db/rocksdb/no_cgo_test.go delete mode 100644 tm2/pkg/iavl/with_gcc_test.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f5f686570e2..6a2a2966370 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,11 +28,6 @@ updates: everything-else: patterns: - "*" - exclude-patterns: - # NOTE: grocksdb should be updated manually, to match the version - # available on Ubuntu's latest LTS release; and updated in conjunction - # with the ubuntu version on .github/workflows/db-tests.yml - - "github.com/linxGnu/grocksdb" open-pull-requests-limit: 10 pull-request-branch-name: separator: "-" diff --git a/.github/workflows/lint_template.yml b/.github/workflows/lint_template.yml index 1ed755c0fab..c1330d0a3d0 100644 --- a/.github/workflows/lint_template.yml +++ b/.github/workflows/lint_template.yml @@ -19,8 +19,6 @@ jobs: go-version: ${{ inputs.go-version }} - name: Checkout code uses: actions/checkout@v4 - - name: Install database dependencies - run: sudo apt-get install -y libleveldb-dev librocksdb-dev - name: Lint uses: golangci/golangci-lint-action@v5 with: diff --git a/.github/workflows/test_template.yml b/.github/workflows/test_template.yml index b813acd3281..c6b7207bc53 100644 --- a/.github/workflows/test_template.yml +++ b/.github/workflows/test_template.yml @@ -24,8 +24,6 @@ jobs: go-version: ${{ inputs.go-version }} - name: Checkout code uses: actions/checkout@v4 - - name: Install database dependencies - run: sudo apt-get install -y libleveldb-dev librocksdb-dev - name: Go test run: go test -coverprofile coverage.out -covermode=atomic -timeout ${{ inputs.tests-timeout }} -v ./... working-directory: ${{ inputs.modulepath }} diff --git a/.github/workflows/tm2.yml b/.github/workflows/tm2.yml index 30811de9039..7471fd74cb7 100644 --- a/.github/workflows/tm2.yml +++ b/.github/workflows/tm2.yml @@ -5,7 +5,7 @@ on: branches: - master workflow_dispatch: - pull_request: + pull_request: paths: - "tm2/**" - ".github/**" @@ -17,4 +17,4 @@ jobs: with: modulepath: "tm2" secrets: - codecov-token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + codecov-token: ${{ secrets.CODECOV_TOKEN }} diff --git a/contribs/gnodev/go.mod b/contribs/gnodev/go.mod index df5236bc4a3..4741a5d7326 100644 --- a/contribs/gnodev/go.mod +++ b/contribs/gnodev/go.mod @@ -53,6 +53,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect + go.etcd.io/bbolt v1.3.9 // indirect go.opentelemetry.io/otel v1.25.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.25.0 // indirect go.opentelemetry.io/otel/metric v1.25.0 // indirect diff --git a/contribs/gnodev/go.sum b/contribs/gnodev/go.sum index dc5528a4be8..a2c9f154560 100644 --- a/contribs/gnodev/go.sum +++ b/contribs/gnodev/go.sum @@ -100,8 +100,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYp github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -112,8 +110,6 @@ github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/linxGnu/grocksdb v1.6.20 h1:C0SNv12/OBr/zOdGw6reXS+mKpIdQGb/AkZWjHYnO64= -github.com/linxGnu/grocksdb v1.6.20/go.mod h1:IbTMGpmWg/1pg2hcG9LlxkqyqiJymdCweaUrzsLRFmg= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= @@ -207,6 +203,8 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/contribs/gnokeykc/go.sum b/contribs/gnokeykc/go.sum index d7bda688d4f..4027483eb1c 100644 --- a/contribs/gnokeykc/go.sum +++ b/contribs/gnokeykc/go.sum @@ -85,8 +85,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYp github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -95,8 +93,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/linxGnu/grocksdb v1.6.20 h1:C0SNv12/OBr/zOdGw6reXS+mKpIdQGb/AkZWjHYnO64= -github.com/linxGnu/grocksdb v1.6.20/go.mod h1:IbTMGpmWg/1pg2hcG9LlxkqyqiJymdCweaUrzsLRFmg= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= diff --git a/gno.land/pkg/gnoland/app.go b/gno.land/pkg/gnoland/app.go index 03222c7e672..cd0c946f907 100644 --- a/gno.land/pkg/gnoland/app.go +++ b/gno.land/pkg/gnoland/app.go @@ -20,6 +20,7 @@ import ( "github.com/gnolang/gno/tm2/pkg/store/iavl" // Only goleveldb is supported for now. + _ "github.com/gnolang/gno/tm2/pkg/db/_tags" _ "github.com/gnolang/gno/tm2/pkg/db/goleveldb" "github.com/gnolang/gno/tm2/pkg/db/memdb" ) diff --git a/go.mod b/go.mod index 0ad00bb21de..b5ef5021fac 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.1 github.com/gotuna/gotuna v0.6.0 - github.com/jmhodges/levigo v1.0.0 github.com/libp2p/go-buffer-pool v0.1.0 - github.com/linxGnu/grocksdb v1.6.20 github.com/mattn/go-runewidth v0.0.15 github.com/pelletier/go-toml v1.9.5 github.com/peterbourgon/ff/v3 v3.4.0 @@ -66,7 +64,6 @@ require ( github.com/zondax/ledger-go v0.14.3 // indirect go.opentelemetry.io/otel/trace v1.25.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect - golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect diff --git a/go.sum b/go.sum index 0b1d1b203f2..e4d728a106d 100644 --- a/go.sum +++ b/go.sum @@ -43,12 +43,6 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeC github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -102,8 +96,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYp github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -114,8 +106,6 @@ github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/linxGnu/grocksdb v1.6.20 h1:C0SNv12/OBr/zOdGw6reXS+mKpIdQGb/AkZWjHYnO64= -github.com/linxGnu/grocksdb v1.6.20/go.mod h1:IbTMGpmWg/1pg2hcG9LlxkqyqiJymdCweaUrzsLRFmg= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= @@ -151,7 +141,6 @@ github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -213,8 +202,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -279,7 +268,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/tm2/Makefile b/tm2/Makefile index f841b989b77..0aaa63e5285 100644 --- a/tm2/Makefile +++ b/tm2/Makefile @@ -62,6 +62,4 @@ _test.flappy: _test.pkg.others:; go test $(GOTEST_FLAGS) `go list ./pkg/... | grep -Ev 'pkg/(amino|bft|db|iavl/benchmarks)(/|$$)'` _test.pkg.amino:; go test $(GOTEST_FLAGS) ./pkg/amino/... _test.pkg.bft:; go test $(GOTEST_FLAGS) ./pkg/bft/... - -_test.pkg.db: - CGO_ENABLED=$(CGO_ENABLED) go test $(GOTEST_FLAGS) ./pkg/db/... ./pkg/iavl/benchmarks/... +_test.pkg.db:; go test $(GOTEST_FLAGS) ./pkg/db/... ./pkg/iavl/benchmarks/... diff --git a/tm2/pkg/bft/config/config.go b/tm2/pkg/bft/config/config.go index e863e1ebcb6..a35157c2a4f 100644 --- a/tm2/pkg/bft/config/config.go +++ b/tm2/pkg/bft/config/config.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "dario.cat/mergo" abci "github.com/gnolang/gno/tm2/pkg/bft/abci/types" @@ -256,19 +257,15 @@ type BaseConfig struct { // and verifying their commits FastSyncMode bool `toml:"fast_sync" comment:"If this node is many blocks behind the tip of the chain, FastSync\n allows them to catchup quickly by downloading blocks in parallel\n and verifying their commits"` - // Database backend: goleveldb | cleveldb | boltdb + // Database backend: goleveldb | boltdb // * goleveldb (github.com/syndtr/goleveldb - most popular implementation) // - pure go // - stable - // * cleveldb (uses levigo wrapper) - // - fast - // - requires gcc - // - use cleveldb build tag (go build -tags cleveldb) // * boltdb (uses etcd's fork of bolt - go.etcd.io/bbolt) // - EXPERIMENTAL // - may be faster is some use-cases (random reads - indexer) // - use boltdb build tag (go build -tags boltdb) - DBBackend string `toml:"db_backend" comment:"Database backend: goleveldb | cleveldb | boltdb\n * goleveldb (github.com/syndtr/goleveldb - most popular implementation)\n - pure go\n - stable\n * cleveldb (uses levigo wrapper)\n - fast\n - requires gcc\n - use cleveldb build tag (go build -tags cleveldb)\n * boltdb (uses etcd's fork of bolt - go.etcd.io/bbolt)\n - EXPERIMENTAL\n - may be faster is some use-cases (random reads - indexer)\n - use boltdb build tag (go build -tags boltdb)"` + DBBackend string `toml:"db_backend" comment:"Database backend: goleveldb | boltdb\n * goleveldb (github.com/syndtr/goleveldb - most popular implementation)\n - pure go\n - stable\n* boltdb (uses etcd's fork of bolt - go.etcd.io/bbolt)\n - EXPERIMENTAL\n - may be faster is some use-cases (random reads - indexer)\n - use boltdb build tag (go build -tags boltdb)"` // Database directory DBPath string `toml:"db_dir" comment:"Database directory"` @@ -369,9 +366,10 @@ func (cfg BaseConfig) ValidateBasic() error { } // Verify the DB backend - if cfg.DBBackend != db.GoLevelDBBackend.String() && - cfg.DBBackend != db.CLevelDBBackend.String() && - cfg.DBBackend != db.BoltDBBackend.String() { + // This will reject also any databases that haven't been added with build tags. + // always reject memdb, as it shouldn't be used as a real-life database. + if cfg.DBBackend == "memdb" || + !slices.Contains(db.BackendList(), db.BackendType(cfg.DBBackend)) { return errInvalidDBBackend } diff --git a/tm2/pkg/bft/config/config_test.go b/tm2/pkg/bft/config/config_test.go index 541b5591985..77f7c0d5e16 100644 --- a/tm2/pkg/bft/config/config_test.go +++ b/tm2/pkg/bft/config/config_test.go @@ -6,6 +6,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + // allows the default config to have a valid DB + _ "github.com/gnolang/gno/tm2/pkg/db/goleveldb" ) func TestConfig_LoadOrMakeConfigWithOptions(t *testing.T) { diff --git a/tm2/pkg/bft/state/state_test.go b/tm2/pkg/bft/state/state_test.go index 15c87a41589..469e44fed1f 100644 --- a/tm2/pkg/bft/state/state_test.go +++ b/tm2/pkg/bft/state/state_test.go @@ -16,9 +16,7 @@ import ( "github.com/gnolang/gno/tm2/pkg/bft/types" "github.com/gnolang/gno/tm2/pkg/crypto/ed25519" dbm "github.com/gnolang/gno/tm2/pkg/db" - _ "github.com/gnolang/gno/tm2/pkg/db/_tags" - _ "github.com/gnolang/gno/tm2/pkg/db/goleveldb" - _ "github.com/gnolang/gno/tm2/pkg/db/memdb" + _ "github.com/gnolang/gno/tm2/pkg/db/_all" "github.com/gnolang/gno/tm2/pkg/random" ) diff --git a/tm2/pkg/db/_all/all.go b/tm2/pkg/db/_all/all.go index f654efd53c7..115ba351a4d 100644 --- a/tm2/pkg/db/_all/all.go +++ b/tm2/pkg/db/_all/all.go @@ -1,13 +1,8 @@ // Package all imports all available databases. It is useful mostly in tests. -// -// cgo databases (rocksdb, cleveldb) will be excluded if CGO_ENABLED=0. package all import ( - // Keep in sync with list of non-cgo backends. - // Add cgo backends in all_cgo.go. _ "github.com/gnolang/gno/tm2/pkg/db/boltdb" - _ "github.com/gnolang/gno/tm2/pkg/db/fsdb" _ "github.com/gnolang/gno/tm2/pkg/db/goleveldb" _ "github.com/gnolang/gno/tm2/pkg/db/memdb" ) diff --git a/tm2/pkg/db/_all/all_cgo.go b/tm2/pkg/db/_all/all_cgo.go deleted file mode 100644 index 293361de713..00000000000 --- a/tm2/pkg/db/_all/all_cgo.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build cgo - -package all - -import ( - // Keep in sync with list of cgo backends. - // Add non-cgo backends in all.go. - _ "github.com/gnolang/gno/tm2/pkg/db/cleveldb" - _ "github.com/gnolang/gno/tm2/pkg/db/rocksdb" -) diff --git a/tm2/pkg/db/_tags/cleveldb.go b/tm2/pkg/db/_tags/cleveldb.go deleted file mode 100644 index cd4f4fc2521..00000000000 --- a/tm2/pkg/db/_tags/cleveldb.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:build cleveldb - -package tags - -import _ "github.com/gnolang/gno/tm2/pkg/db/cleveldb" diff --git a/tm2/pkg/db/_tags/fsdb.go b/tm2/pkg/db/_tags/fsdb.go deleted file mode 100644 index 134a831010b..00000000000 --- a/tm2/pkg/db/_tags/fsdb.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:build fsdb - -package tags - -import _ "github.com/gnolang/gno/tm2/pkg/db/fsdb" diff --git a/tm2/pkg/db/_tags/rocksdb.go b/tm2/pkg/db/_tags/rocksdb.go deleted file mode 100644 index f1e19ef31c9..00000000000 --- a/tm2/pkg/db/_tags/rocksdb.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:build rocksdb - -package tags - -import _ "github.com/gnolang/gno/tm2/pkg/db/rocksdb" diff --git a/tm2/pkg/db/cleveldb/c_level_db.go b/tm2/pkg/db/cleveldb/c_level_db.go deleted file mode 100644 index 51d567b2628..00000000000 --- a/tm2/pkg/db/cleveldb/c_level_db.go +++ /dev/null @@ -1,326 +0,0 @@ -//go:build cgo - -package cleveldb - -import ( - "bytes" - "fmt" - "path/filepath" - - "github.com/gnolang/gno/tm2/pkg/db" - "github.com/gnolang/gno/tm2/pkg/db/internal" - "github.com/jmhodges/levigo" -) - -func init() { - dbCreator := func(name string, dir string) (db.DB, error) { - return NewCLevelDB(name, dir) - } - db.InternalRegisterDBCreator(db.CLevelDBBackend, dbCreator, false) -} - -var _ db.DB = (*CLevelDB)(nil) - -type CLevelDB struct { - db *levigo.DB - ro *levigo.ReadOptions - wo *levigo.WriteOptions - woSync *levigo.WriteOptions -} - -func NewCLevelDB(name string, dir string) (*CLevelDB, error) { - dbPath := filepath.Join(dir, name+".db") - - opts := levigo.NewOptions() - opts.SetCache(levigo.NewLRUCache(1 << 30)) - opts.SetCreateIfMissing(true) - db, err := levigo.Open(dbPath, opts) - if err != nil { - return nil, err - } - ro := levigo.NewReadOptions() - wo := levigo.NewWriteOptions() - woSync := levigo.NewWriteOptions() - woSync.SetSync(true) - database := &CLevelDB{ - db: db, - ro: ro, - wo: wo, - woSync: woSync, - } - return database, nil -} - -// Implements DB. -func (db *CLevelDB) Get(key []byte) []byte { - key = internal.NonNilBytes(key) - res, err := db.db.Get(db.ro, key) - if err != nil { - panic(err) - } - return res -} - -// Implements DB. -func (db *CLevelDB) Has(key []byte) bool { - return db.Get(key) != nil -} - -// Implements DB. -func (db *CLevelDB) Set(key []byte, value []byte) { - key = internal.NonNilBytes(key) - value = internal.NonNilBytes(value) - err := db.db.Put(db.wo, key, value) - if err != nil { - panic(err) - } -} - -// Implements DB. -func (db *CLevelDB) SetSync(key []byte, value []byte) { - key = internal.NonNilBytes(key) - value = internal.NonNilBytes(value) - err := db.db.Put(db.woSync, key, value) - if err != nil { - panic(err) - } -} - -// Implements DB. -func (db *CLevelDB) Delete(key []byte) { - key = internal.NonNilBytes(key) - err := db.db.Delete(db.wo, key) - if err != nil { - panic(err) - } -} - -// Implements DB. -func (db *CLevelDB) DeleteSync(key []byte) { - key = internal.NonNilBytes(key) - err := db.db.Delete(db.woSync, key) - if err != nil { - panic(err) - } -} - -func (db *CLevelDB) DB() *levigo.DB { - return db.db -} - -// Implements DB. -func (db *CLevelDB) Close() { - db.db.Close() - db.ro.Close() - db.wo.Close() - db.woSync.Close() -} - -// Implements DB. -func (db *CLevelDB) Print() { - itr := db.Iterator(nil, nil) - defer itr.Close() - for ; itr.Valid(); itr.Next() { - key := itr.Key() - value := itr.Value() - fmt.Printf("[%X]:\t[%X]\n", key, value) - } -} - -// Implements DB. -func (db *CLevelDB) Stats() map[string]string { - keys := []string{ - "leveldb.aliveiters", - "leveldb.alivesnaps", - "leveldb.blockpool", - "leveldb.cachedblock", - "leveldb.num-files-at-level{n}", - "leveldb.openedtables", - "leveldb.sstables", - "leveldb.stats", - } - - stats := make(map[string]string, len(keys)) - for _, key := range keys { - str := db.db.PropertyValue(key) - stats[key] = str - } - return stats -} - -//---------------------------------------- -// Batch - -// Implements DB. -func (db *CLevelDB) NewBatch() db.Batch { - batch := levigo.NewWriteBatch() - return &cLevelDBBatch{db, batch} -} - -type cLevelDBBatch struct { - db *CLevelDB - batch *levigo.WriteBatch -} - -// Implements Batch. -func (mBatch *cLevelDBBatch) Set(key, value []byte) { - mBatch.batch.Put(key, value) -} - -// Implements Batch. -func (mBatch *cLevelDBBatch) Delete(key []byte) { - mBatch.batch.Delete(key) -} - -// Implements Batch. -func (mBatch *cLevelDBBatch) Write() { - err := mBatch.db.db.Write(mBatch.db.wo, mBatch.batch) - if err != nil { - panic(err) - } -} - -// Implements Batch. -func (mBatch *cLevelDBBatch) WriteSync() { - err := mBatch.db.db.Write(mBatch.db.woSync, mBatch.batch) - if err != nil { - panic(err) - } -} - -// Implements Batch. -func (mBatch *cLevelDBBatch) Close() { - mBatch.batch.Close() -} - -//---------------------------------------- -// Iterator -// NOTE This is almost identical to db/go_level_db.Iterator -// Before creating a third version, refactor. - -func (db *CLevelDB) Iterator(start, end []byte) db.Iterator { - itr := db.db.NewIterator(db.ro) - return newCLevelDBIterator(itr, start, end, false) -} - -func (db *CLevelDB) ReverseIterator(start, end []byte) db.Iterator { - itr := db.db.NewIterator(db.ro) - return newCLevelDBIterator(itr, start, end, true) -} - -var _ db.Iterator = (*cLevelDBIterator)(nil) - -type cLevelDBIterator struct { - source *levigo.Iterator - start, end []byte - isReverse bool - isInvalid bool -} - -func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse bool) *cLevelDBIterator { - if isReverse { - if end == nil { - source.SeekToLast() - } else { - source.Seek(end) - if source.Valid() { - eoakey := source.Key() // end or after key - if bytes.Compare(end, eoakey) <= 0 { - source.Prev() - } - } else { - source.SeekToLast() - } - } - } else { - if start == nil { - source.SeekToFirst() - } else { - source.Seek(start) - } - } - return &cLevelDBIterator{ - source: source, - start: start, - end: end, - isReverse: isReverse, - isInvalid: false, - } -} - -func (itr cLevelDBIterator) Domain() ([]byte, []byte) { - return itr.start, itr.end -} - -func (itr cLevelDBIterator) Valid() bool { - // Once invalid, forever invalid. - if itr.isInvalid { - return false - } - - // Panic on DB error. No way to recover. - itr.assertNoError() - - // If source is invalid, invalid. - if !itr.source.Valid() { - itr.isInvalid = true - return false - } - - // If key is end or past it, invalid. - start := itr.start - end := itr.end - key := itr.source.Key() - if itr.isReverse { - if start != nil && bytes.Compare(key, start) < 0 { - itr.isInvalid = true - return false - } - } else { - if end != nil && bytes.Compare(end, key) <= 0 { - itr.isInvalid = true - return false - } - } - - // It's valid. - return true -} - -func (itr cLevelDBIterator) Key() []byte { - itr.assertNoError() - itr.assertIsValid() - return itr.source.Key() -} - -func (itr cLevelDBIterator) Value() []byte { - itr.assertNoError() - itr.assertIsValid() - return itr.source.Value() -} - -func (itr cLevelDBIterator) Next() { - itr.assertNoError() - itr.assertIsValid() - if itr.isReverse { - itr.source.Prev() - } else { - itr.source.Next() - } -} - -func (itr cLevelDBIterator) Close() { - itr.source.Close() -} - -func (itr cLevelDBIterator) assertNoError() { - if err := itr.source.GetError(); err != nil { - panic(err) - } -} - -func (itr cLevelDBIterator) assertIsValid() { - if !itr.Valid() { - panic("cLevelDBIterator is invalid") - } -} diff --git a/tm2/pkg/db/cleveldb/c_level_db_test.go b/tm2/pkg/db/cleveldb/c_level_db_test.go deleted file mode 100644 index a6a95b42dcb..00000000000 --- a/tm2/pkg/db/cleveldb/c_level_db_test.go +++ /dev/null @@ -1,113 +0,0 @@ -//go:build cgo - -package cleveldb - -import ( - "bytes" - "encoding/binary" - "fmt" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/gnolang/gno/tm2/pkg/db" - "github.com/gnolang/gno/tm2/pkg/db/internal" -) - -func BenchmarkRandomReadsWrites2(b *testing.B) { - b.StopTimer() - - numItems := int64(1000000) - mrand := map[int64]int64{} - for i := 0; i < int(numItems); i++ { - mrand[int64(i)] = int64(0) - } - db, err := NewCLevelDB(fmt.Sprintf("test_%x", internal.RandStr(12)), "") - if err != nil { - b.Fatal(err.Error()) - return - } - - fmt.Println("ok, starting") - b.StartTimer() - - for i := 0; i < b.N; i++ { - // Write something - { - idx := (int64(rand.Int()) % numItems) - mrand[idx]++ - val := mrand[idx] - idxBytes := int642Bytes(idx) - valBytes := int642Bytes(val) - // fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) - db.Set( - idxBytes, - valBytes, - ) - } - // Read something - { - idx := (int64(rand.Int()) % numItems) - val := mrand[idx] - idxBytes := int642Bytes(idx) - valBytes := db.Get(idxBytes) - // fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) - if val == 0 { - if !bytes.Equal(valBytes, nil) { - b.Errorf("Expected %v for %v, got %X", - nil, idx, valBytes) - break - } - } else { - if len(valBytes) != 8 { - b.Errorf("Expected length 8 for %v, got %X", - idx, valBytes) - break - } - valGot := bytes2Int64(valBytes) - if val != valGot { - b.Errorf("Expected %v for %v, got %v", - val, idx, valGot) - break - } - } - } - } - - db.Close() -} - -func int642Bytes(i int64) []byte { - buf := make([]byte, 8) - binary.BigEndian.PutUint64(buf, uint64(i)) - return buf -} - -func bytes2Int64(buf []byte) int64 { - return int64(binary.BigEndian.Uint64(buf)) -} - -func TestCLevelDBBackend(t *testing.T) { - t.Parallel() - - name := fmt.Sprintf("test_%x", internal.RandStr(12)) - // Can't use "" (current directory) or "./" here because levigo.Open returns: - // "Error initializing DB: IO error: test_XXX.db: Invalid argument" - db, err := db.NewDB(name, db.CLevelDBBackend, t.TempDir()) - require.NoError(t, err) - - _, ok := db.(*CLevelDB) - assert.True(t, ok) -} - -func TestCLevelDBStats(t *testing.T) { - t.Parallel() - - name := fmt.Sprintf("test_%x", internal.RandStr(12)) - db, err := db.NewDB(name, db.CLevelDBBackend, t.TempDir()) - require.NoError(t, err) - - assert.NotEmpty(t, db.Stats()) -} diff --git a/tm2/pkg/db/cleveldb/no_cgo_test.go b/tm2/pkg/db/cleveldb/no_cgo_test.go deleted file mode 100644 index 71c1b83bf18..00000000000 --- a/tm2/pkg/db/cleveldb/no_cgo_test.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !cgo - -package rocksdb - -import "testing" - -func TestSkip(t *testing.T) { - t.Skip("This package requires cgo to compile and test") -} diff --git a/tm2/pkg/db/db.go b/tm2/pkg/db/db.go index 13a9ad6e77f..20ac969b9b3 100644 --- a/tm2/pkg/db/db.go +++ b/tm2/pkg/db/db.go @@ -39,26 +39,14 @@ const ( // popular implementation) // - stable GoLevelDBBackend BackendType = "goleveldb" - // CLevelDBBackend represents cleveldb (uses levigo wrapper) - // - fast - // - requires gcc - CLevelDBBackend BackendType = "cleveldb" // MemDBBackend represents in-memory key value store, which is mostly used // for testing. MemDBBackend BackendType = "memdb" - // FSDBBackend represents filesystem database - // - EXPERIMENTAL - // - slow - FSDBBackend BackendType = "fsdb" // BoltDBBackend represents bolt (uses etcd's fork of bolt - // go.etcd.io/bbolt) // - EXPERIMENTAL // - may be faster is some use-cases (random reads - indexer) BoltDBBackend BackendType = "boltdb" - // RocksDBBackend represents rocksdb (uses github.com/linuxGnu/grocksdb) - // - EXPERIMENTAL - // - requires gcc - RocksDBBackend BackendType = "rocksdb" ) type dbCreator func(name string, dir string) (DB, error) diff --git a/tm2/pkg/db/fsdb/fsdb.go b/tm2/pkg/db/fsdb/fsdb.go deleted file mode 100644 index ddb2439b0f6..00000000000 --- a/tm2/pkg/db/fsdb/fsdb.go +++ /dev/null @@ -1,276 +0,0 @@ -package fsdb - -import ( - "fmt" - "io" - "net/url" - "os" - "path/filepath" - "sort" - "sync" - - "github.com/gnolang/gno/tm2/pkg/db" - "github.com/gnolang/gno/tm2/pkg/db/internal" - "github.com/gnolang/gno/tm2/pkg/errors" -) - -const ( - keyPerm = os.FileMode(0o600) - dirPerm = os.FileMode(0o700) -) - -func init() { - db.InternalRegisterDBCreator(db.FSDBBackend, func(name, dir string) (db.DB, error) { - dbPath := filepath.Join(dir, name+".db") - return NewFSDB(dbPath), nil - }, false) -} - -var _ db.DB = (*FSDB)(nil) - -// It's slow. -type FSDB struct { - mtx sync.Mutex - dir string -} - -func NewFSDB(dir string) *FSDB { - err := os.MkdirAll(dir, dirPerm) - if err != nil { - panic(errors.Wrap(err, "Creating FSDB dir "+dir)) - } - database := &FSDB{ - dir: dir, - } - return database -} - -func (db *FSDB) Get(key []byte) []byte { - db.mtx.Lock() - defer db.mtx.Unlock() - key = escapeKey(key) - - path := db.nameToPath(key) - value, err := read(path) - if os.IsNotExist(err) { - return nil - } else if err != nil { - panic(errors.Wrap(err, "Getting key %s (0x%X)", string(key), key)) - } - return value -} - -func (db *FSDB) Has(key []byte) bool { - db.mtx.Lock() - defer db.mtx.Unlock() - key = escapeKey(key) - - path := db.nameToPath(key) - return fileExists(path) -} - -func fileExists(filePath string) bool { - _, err := os.Stat(filePath) - return !os.IsNotExist(err) -} - -func (db *FSDB) Set(key []byte, value []byte) { - db.mtx.Lock() - defer db.mtx.Unlock() - - db.SetNoLock(key, value) -} - -func (db *FSDB) SetSync(key []byte, value []byte) { - db.mtx.Lock() - defer db.mtx.Unlock() - - db.SetNoLock(key, value) -} - -// NOTE: Implements atomicSetDeleter. -func (db *FSDB) SetNoLock(key []byte, value []byte) { - key = escapeKey(key) - value = internal.NonNilBytes(value) - path := db.nameToPath(key) - err := write(path, value) - if err != nil { - panic(errors.Wrap(err, "Setting key %s (0x%X)", string(key), key)) - } -} - -func (db *FSDB) Delete(key []byte) { - db.mtx.Lock() - defer db.mtx.Unlock() - - db.DeleteNoLock(key) -} - -func (db *FSDB) DeleteSync(key []byte) { - db.mtx.Lock() - defer db.mtx.Unlock() - - db.DeleteNoLock(key) -} - -// NOTE: Implements atomicSetDeleter. -func (db *FSDB) DeleteNoLock(key []byte) { - key = escapeKey(key) - path := db.nameToPath(key) - err := remove(path) - if os.IsNotExist(err) { - return - } else if err != nil { - panic(errors.Wrap(err, "Removing key %s (0x%X)", string(key), key)) - } -} - -func (db *FSDB) Close() { - // Nothing to do. -} - -func (db *FSDB) Print() { - db.mtx.Lock() - defer db.mtx.Unlock() - - panic("FSDB.Print not yet implemented") -} - -func (db *FSDB) Stats() map[string]string { - db.mtx.Lock() - defer db.mtx.Unlock() - - panic("FSDB.Stats not yet implemented") -} - -func (db *FSDB) NewBatch() db.Batch { - db.mtx.Lock() - defer db.mtx.Unlock() - - // Not sure we would ever want to try... - // It doesn't seem easy for general filesystems. - panic("FSDB.NewBatch not yet implemented") -} - -func (db *FSDB) Mutex() *sync.Mutex { - return &(db.mtx) -} - -func (db *FSDB) Iterator(start, end []byte) db.Iterator { - return db.MakeIterator(start, end, false) -} - -func (db *FSDB) MakeIterator(start, end []byte, isReversed bool) db.Iterator { - db.mtx.Lock() - defer db.mtx.Unlock() - - // We need a copy of all of the keys. - // Not the best, but probably not a bottleneck depending. - keys, err := list(db.dir, start, end) - if err != nil { - panic(errors.Wrap(err, "Listing keys in %s", db.dir)) - } - if isReversed { - sort.Sort(sort.Reverse(sort.StringSlice(keys))) - } else { - sort.Strings(keys) - } - return internal.NewMemIterator(db, keys, start, end) -} - -func (db *FSDB) ReverseIterator(start, end []byte) db.Iterator { - return db.MakeIterator(start, end, true) -} - -func (db *FSDB) nameToPath(name []byte) string { - n := url.PathEscape(string(name)) - return filepath.Join(db.dir, n) -} - -// Read some bytes to a file. -// CONTRACT: returns os errors directly without wrapping. -func read(path string) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - d, err := io.ReadAll(f) - if err != nil { - return nil, err - } - return d, nil -} - -// Write some bytes from a file. -// CONTRACT: returns os errors directly without wrapping. -func write(path string, d []byte) error { - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, keyPerm) - if err != nil { - return err - } - defer f.Close() - // fInfo, err := f.Stat() - // if err != nil { - // return err - // } - // if fInfo.Mode() != keyPerm { - // return tmerrors.NewErrPermissionsChanged(f.Name(), keyPerm, fInfo.Mode()) - // } - _, err = f.Write(d) - if err != nil { - return err - } - err = f.Sync() - return err -} - -// Remove a file. -// CONTRACT: returns os errors directly without wrapping. -func remove(path string) error { - return os.Remove(path) -} - -// List keys in a directory, stripping of escape sequences and dir portions. -// CONTRACT: returns os errors directly without wrapping. -func list(dirPath string, start, end []byte) ([]string, error) { - dir, err := os.Open(dirPath) - if err != nil { - return nil, err - } - defer dir.Close() - - names, err := dir.Readdirnames(0) - if err != nil { - return nil, err - } - var keys []string - for _, name := range names { - n, err := url.PathUnescape(name) - if err != nil { - return nil, fmt.Errorf("failed to unescape %s while listing", name) - } - key := unescapeKey([]byte(n)) - if db.IsKeyInDomain(key, start, end) { - keys = append(keys, string(key)) - } - } - return keys, nil -} - -// To support empty or nil keys, while the file system doesn't allow empty -// filenames. -func escapeKey(key []byte) []byte { - return []byte("k_" + string(key)) -} - -func unescapeKey(escKey []byte) []byte { - if len(escKey) < 2 { - panic(fmt.Sprintf("Invalid esc key: %x", escKey)) - } - if string(escKey[:2]) != "k_" { - panic(fmt.Sprintf("Invalid esc key: %x", escKey)) - } - return escKey[2:] -} diff --git a/tm2/pkg/db/rocksdb/grocks_db.go b/tm2/pkg/db/rocksdb/grocks_db.go deleted file mode 100644 index 4733c0ba8d9..00000000000 --- a/tm2/pkg/db/rocksdb/grocks_db.go +++ /dev/null @@ -1,342 +0,0 @@ -//go:build cgo - -package rocksdb - -import ( - "bytes" - "fmt" - "path/filepath" - "runtime" - - "github.com/gnolang/gno/tm2/pkg/db" - "github.com/gnolang/gno/tm2/pkg/db/internal" - "github.com/linxGnu/grocksdb" -) - -func init() { - dbCreator := func(name string, dir string) (db.DB, error) { - return NewRocksDB(name, dir) - } - db.InternalRegisterDBCreator(db.RocksDBBackend, dbCreator, false) -} - -var _ db.DB = (*RocksDB)(nil) - -type RocksDB struct { - db *grocksdb.DB - ro *grocksdb.ReadOptions - wo *grocksdb.WriteOptions - woSync *grocksdb.WriteOptions -} - -func NewRocksDB(name string, dir string) (*RocksDB, error) { - // default rocksdb option, good enough for most cases, including heavy workloads. - // 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). - // compression: snappy as default, need to -lsnappy to enable. - bbto := grocksdb.NewDefaultBlockBasedTableOptions() - bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30)) - bbto.SetFilterPolicy(grocksdb.NewBloomFilter(10)) - - opts := grocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.IncreaseParallelism(runtime.NumCPU()) - // 1.5GB maximum memory use for writebuffer. - opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) - return NewRocksDBWithOptions(name, dir, opts) -} - -func NewRocksDBWithOptions(name string, dir string, opts *grocksdb.Options) (*RocksDB, error) { - dbPath := filepath.Join(dir, name+".db") - db, err := grocksdb.OpenDb(opts, dbPath) - if err != nil { - return nil, err - } - ro := grocksdb.NewDefaultReadOptions() - wo := grocksdb.NewDefaultWriteOptions() - woSync := grocksdb.NewDefaultWriteOptions() - woSync.SetSync(true) - database := &RocksDB{ - db: db, - ro: ro, - wo: wo, - woSync: woSync, - } - return database, nil -} - -// Implements DB. -func (db *RocksDB) Get(key []byte) []byte { - key = internal.NonNilBytes(key) - res, err := db.db.Get(db.ro, key) - if err != nil { - panic(err) - } - return moveSliceToBytes(res) -} - -// Implements DB. -func (db *RocksDB) Has(key []byte) bool { - return db.Get(key) != nil -} - -// Implements DB. -func (db *RocksDB) Set(key []byte, value []byte) { - key = internal.NonNilBytes(key) - value = internal.NonNilBytes(value) - err := db.db.Put(db.wo, key, value) - if err != nil { - panic(err) - } -} - -// Implements DB. -func (db *RocksDB) SetSync(key []byte, value []byte) { - key = internal.NonNilBytes(key) - value = internal.NonNilBytes(value) - err := db.db.Put(db.woSync, key, value) - if err != nil { - panic(err) - } -} - -// Implements DB. -func (db *RocksDB) Delete(key []byte) { - key = internal.NonNilBytes(key) - err := db.db.Delete(db.wo, key) - if err != nil { - panic(err) - } -} - -// Implements DB. -func (db *RocksDB) DeleteSync(key []byte) { - key = internal.NonNilBytes(key) - err := db.db.Delete(db.woSync, key) - if err != nil { - panic(err) - } -} - -func (db *RocksDB) DB() *grocksdb.DB { - return db.db -} - -// Implements DB. -func (db *RocksDB) Close() { - db.ro.Destroy() - db.wo.Destroy() - db.woSync.Destroy() - db.db.Close() -} - -// Implements DB. -func (db *RocksDB) Print() { - itr := db.Iterator(nil, nil) - defer itr.Close() - for ; itr.Valid(); itr.Next() { - key := itr.Key() - value := itr.Value() - fmt.Printf("[%X]:\t[%X]\n", key, value) - } -} - -// Implements DB. -func (db *RocksDB) Stats() map[string]string { - keys := []string{"rocksdb.stats"} - stats := make(map[string]string, len(keys)) - for _, key := range keys { - stats[key] = db.db.GetProperty(key) - } - return stats -} - -//---------------------------------------- -// Batch - -// Implements DB. -func (db *RocksDB) NewBatch() db.Batch { - batch := grocksdb.NewWriteBatch() - return &rocksDBBatch{db, batch} -} - -type rocksDBBatch struct { - db *RocksDB - batch *grocksdb.WriteBatch -} - -// Implements Batch. -func (mBatch *rocksDBBatch) Set(key, value []byte) { - mBatch.batch.Put(key, value) -} - -// Implements Batch. -func (mBatch *rocksDBBatch) Delete(key []byte) { - mBatch.batch.Delete(key) -} - -// Implements Batch. -func (mBatch *rocksDBBatch) Write() { - err := mBatch.db.db.Write(mBatch.db.wo, mBatch.batch) - if err != nil { - panic(err) - } -} - -// Implements Batch. -func (mBatch *rocksDBBatch) WriteSync() { - err := mBatch.db.db.Write(mBatch.db.woSync, mBatch.batch) - if err != nil { - panic(err) - } -} - -// Implements Batch. -func (mBatch *rocksDBBatch) Close() { - mBatch.batch.Destroy() -} - -//---------------------------------------- -// Iterator -// NOTE This is almost identical to db/go_level_db.Iterator -// Before creating a third version, refactor. - -func (db *RocksDB) Iterator(start, end []byte) db.Iterator { - itr := db.db.NewIterator(db.ro) - return newRocksDBIterator(itr, start, end, false) -} - -func (db *RocksDB) ReverseIterator(start, end []byte) db.Iterator { - itr := db.db.NewIterator(db.ro) - return newRocksDBIterator(itr, start, end, true) -} - -var _ db.Iterator = (*rocksDBIterator)(nil) - -type rocksDBIterator struct { - source *grocksdb.Iterator - start, end []byte - isReverse bool - isInvalid bool -} - -func newRocksDBIterator(source *grocksdb.Iterator, start, end []byte, isReverse bool) *rocksDBIterator { - if isReverse { - if end == nil { - source.SeekToLast() - } else { - source.Seek(end) - if source.Valid() { - eoakey := moveSliceToBytes(source.Key()) // end or after key - if bytes.Compare(end, eoakey) <= 0 { - source.Prev() - } - } else { - source.SeekToLast() - } - } - } else { - if start == nil { - source.SeekToFirst() - } else { - source.Seek(start) - } - } - return &rocksDBIterator{ - source: source, - start: start, - end: end, - isReverse: isReverse, - isInvalid: false, - } -} - -func (itr rocksDBIterator) Domain() ([]byte, []byte) { - return itr.start, itr.end -} - -func (itr rocksDBIterator) Valid() bool { - // Once invalid, forever invalid. - if itr.isInvalid { - return false - } - - // Panic on DB error. No way to recover. - itr.assertNoError() - - // If source is invalid, invalid. - if !itr.source.Valid() { - itr.isInvalid = true - return false - } - - // If key is end or past it, invalid. - start := itr.start - end := itr.end - key := moveSliceToBytes(itr.source.Key()) - if itr.isReverse { - if start != nil && bytes.Compare(key, start) < 0 { - itr.isInvalid = true - return false - } - } else { - if end != nil && bytes.Compare(end, key) <= 0 { - itr.isInvalid = true - return false - } - } - - // It's valid. - return true -} - -func (itr rocksDBIterator) Key() []byte { - itr.assertNoError() - itr.assertIsValid() - return moveSliceToBytes(itr.source.Key()) -} - -func (itr rocksDBIterator) Value() []byte { - itr.assertNoError() - itr.assertIsValid() - return moveSliceToBytes(itr.source.Value()) -} - -func (itr rocksDBIterator) Next() { - itr.assertNoError() - itr.assertIsValid() - if itr.isReverse { - itr.source.Prev() - } else { - itr.source.Next() - } -} - -func (itr rocksDBIterator) Close() { - itr.source.Close() -} - -func (itr rocksDBIterator) assertNoError() { - if err := itr.source.Err(); err != nil { - panic(err) - } -} - -func (itr rocksDBIterator) assertIsValid() { - if !itr.Valid() { - panic("rocksDBIterator is invalid") - } -} - -// moveSliceToBytes will free the slice and copy out a go []byte -// This function can be applied on *Slice returned from Key() and Value() -// of an Iterator, because they are marked as freed. -func moveSliceToBytes(s *grocksdb.Slice) []byte { - defer s.Free() - if !s.Exists() { - return nil - } - v := make([]byte, len(s.Data())) - copy(v, s.Data()) - return v -} diff --git a/tm2/pkg/db/rocksdb/grocks_db_test.go b/tm2/pkg/db/rocksdb/grocks_db_test.go deleted file mode 100644 index 8dfb7e4b528..00000000000 --- a/tm2/pkg/db/rocksdb/grocks_db_test.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build cgo - -package rocksdb - -import ( - "fmt" - "testing" - - "github.com/gnolang/gno/tm2/pkg/db" - "github.com/gnolang/gno/tm2/pkg/db/internal" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGRocksDBBackend(t *testing.T) { - t.Parallel() - - name := fmt.Sprintf("test_%x", internal.RandStr(12)) - db, err := db.NewDB(name, db.RocksDBBackend, t.TempDir()) - require.NoError(t, err) - - _, ok := db.(*RocksDB) - assert.True(t, ok) -} - -func TestGRocksDBStats(t *testing.T) { - t.Parallel() - - name := fmt.Sprintf("test_%x", internal.RandStr(12)) - db, err := db.NewDB(name, db.RocksDBBackend, t.TempDir()) - require.NoError(t, err) - - assert.NotEmpty(t, db.Stats()) -} diff --git a/tm2/pkg/db/rocksdb/no_cgo_test.go b/tm2/pkg/db/rocksdb/no_cgo_test.go deleted file mode 100644 index 71c1b83bf18..00000000000 --- a/tm2/pkg/db/rocksdb/no_cgo_test.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !cgo - -package rocksdb - -import "testing" - -func TestSkip(t *testing.T) { - t.Skip("This package requires cgo to compile and test") -} diff --git a/tm2/pkg/iavl/benchmarks/bench_test.go b/tm2/pkg/iavl/benchmarks/bench_test.go index aaf28d1fbd1..88de3634b7a 100644 --- a/tm2/pkg/iavl/benchmarks/bench_test.go +++ b/tm2/pkg/iavl/benchmarks/bench_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "os" "runtime" - "slices" "testing" "github.com/stretchr/testify/require" @@ -152,15 +151,8 @@ type benchmark struct { keyLen, dataLen int } -func backendList() []db.BackendType { - return slices.DeleteFunc(db.BackendList(), func(s db.BackendType) bool { - // fsdb doesn't support batch ops, and it's slow anyways, so let's skip. - return s == db.FSDBBackend - }) -} - func BenchmarkSmall(b *testing.B) { - ls := backendList() + ls := db.BackendList() bs := make([]benchmark, 0, len(ls)) for _, backend := range ls { bs = append(bs, benchmark{backend, 1_000, 100, 16, 40}) @@ -169,7 +161,7 @@ func BenchmarkSmall(b *testing.B) { } func BenchmarkMedium(b *testing.B) { - ls := backendList() + ls := db.BackendList() bs := make([]benchmark, 0, len(ls)) for _, backend := range ls { bs = append(bs, benchmark{backend, 100_000, 100, 16, 40}) @@ -178,7 +170,7 @@ func BenchmarkMedium(b *testing.B) { } func BenchmarkLarge(b *testing.B) { - ls := backendList() + ls := db.BackendList() bs := make([]benchmark, 0, len(ls)) for _, backend := range ls { bs = append(bs, benchmark{backend, 1_000_000, 100, 16, 40}) diff --git a/tm2/pkg/iavl/with_gcc_test.go b/tm2/pkg/iavl/with_gcc_test.go deleted file mode 100644 index 83ffea740ba..00000000000 --- a/tm2/pkg/iavl/with_gcc_test.go +++ /dev/null @@ -1,19 +0,0 @@ -//go:build gcc - -// This file exists because some of the DBs e.g CLevelDB -// require gcc as the compiler before they can ran otherwise -// we'll encounter crashes such as in https://github.com/tendermint/merkleeyes/issues/39 - -package iavl - -import ( - "testing" - - "github.com/gnolang/gno/tm2/pkg/db" - _ "github.com/gnolang/gno/tm2/pkg/db/cleveldb" -) - -func BenchmarkImmutableAvlTreeCLevelDB(b *testing.B) { - db := db.NewDB("test", db.CLevelDBBackendStr, "./") - benchmarkImmutableAvlTreeWithDB(b, db) -}