From 48ebd1330c17c3767a27403efaac7bdf1b2d07dc Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Tue, 11 Jul 2023 17:20:09 -0500 Subject: [PATCH 01/36] feat/clusterdb --- .circleci/config.yml | 6 + .../en/default-lotus-miner-config.toml | 40 +++ go.mod | 12 +- go.sum | 24 +- itests/clusterdb_test.go | 188 +++++++++++++ itests/kit/ensemble.go | 5 + itests/kit/node_miner.go | 3 + lib/sturdy/clusterdb/clusterdb.go | 251 ++++++++++++++++++ lib/sturdy/clusterdb/doc.go | 35 +++ lib/sturdy/clusterdb/metrics.go | 70 +++++ lib/sturdy/clusterdb/sql/20230706.sql | 6 + lib/sturdy/clusterdb/userfuncs.go | 149 +++++++++++ metrics/metrics.go | 79 +++--- node/builder_miner.go | 2 + node/config/def.go | 7 + node/config/doc_gen.go | 46 ++++ node/config/types.go | 24 ++ 17 files changed, 906 insertions(+), 41 deletions(-) create mode 100644 itests/clusterdb_test.go create mode 100644 lib/sturdy/clusterdb/clusterdb.go create mode 100644 lib/sturdy/clusterdb/doc.go create mode 100644 lib/sturdy/clusterdb/metrics.go create mode 100644 lib/sturdy/clusterdb/sql/20230706.sql create mode 100644 lib/sturdy/clusterdb/userfuncs.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fcb831454c..0d633daf1ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -557,6 +557,12 @@ workflows: - build suite: itest-cli target: "./itests/cli_test.go" + - test: + name: test-itest-clusterdb + requires: + - build + suite: itest-clusterdb + target: "./itests/clusterdb_test.go" - test: name: test-itest-deadlines requires: diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index c0a204bf1db..1ff115b0746 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -894,3 +894,43 @@ #GCInterval = "1m0s" +[ClusterDB] + # HOSTS is a list of hostnames to nodes running YugabyteDB + # in a cluster. Only 1 is required + # + # type: []string + # env var: LOTUS_CLUSTERDB_HOSTS + #Hosts = ["127.0.0.1"] + + # The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default. + # + # type: string + # env var: LOTUS_CLUSTERDB_USERNAME + #Username = "yugabyte" + + # The password for the related username. Blank for default. + # + # type: string + # env var: LOTUS_CLUSTERDB_PASSWORD + #Password = "yugabyte" + + # The database (logical partition) within Yugabyte. Blank for default. + # + # type: string + # env var: LOTUS_CLUSTERDB_DATABASE + #Database = "yugabyte" + + # The port to find Yugabyte. Blank for default. + # + # type: string + # env var: LOTUS_CLUSTERDB_PORT + #Port = "5433" + + # ITest is for optimized integration testing and not + # for production. Blank for default production configuration. + # + # type: string + # env var: LOTUS_CLUSTERDB_ITEST + #ITest = "" + + diff --git a/go.mod b/go.mod index ab1f60b6e3b..d3105cb831c 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/gdamore/tcell/v2 v2.2.0 + github.com/georgysavva/scany/v2 v2.0.0 github.com/go-openapi/spec v0.19.11 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 @@ -102,8 +103,10 @@ require ( github.com/ipld/go-ipld-selector-text-lite v0.0.1 github.com/ipni/go-libipni v0.0.8 github.com/ipni/index-provider v0.12.0 + github.com/jackc/pgx/v5 v5.4.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/koalacxr/quantile v0.0.1 + github.com/kr/pretty v0.3.1 github.com/libp2p/go-buffer-pool v0.1.0 github.com/libp2p/go-libp2p v0.27.6 github.com/libp2p/go-libp2p-consensus v0.0.1 @@ -127,7 +130,7 @@ require ( github.com/multiformats/go-varint v0.0.7 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/polydawn/refmt v0.89.0 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.16.0 github.com/puzpuzpuz/xsync/v2 v2.4.0 github.com/raulk/clock v1.1.0 github.com/raulk/go-watchdog v1.3.0 @@ -243,6 +246,9 @@ require ( github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect @@ -255,6 +261,7 @@ require ( github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/koron/go-ssdp v0.0.4 // indirect + github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect @@ -291,7 +298,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.3.2 // indirect @@ -299,6 +306,7 @@ require ( github.com/quic-go/quic-go v0.33.0 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/rivo/uniseg v0.1.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/cors v1.7.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v2.18.12+incompatible // indirect diff --git a/go.sum b/go.sum index 71341203d8d..ab18517ee0b 100644 --- a/go.sum +++ b/go.sum @@ -169,6 +169,7 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -391,6 +392,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U4= github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU= +github.com/georgysavva/scany/v2 v2.0.0 h1:RGXqxDv4row7/FYoK8MRXAZXqoWF/NM+NP0q50k3DKU= +github.com/georgysavva/scany/v2 v2.0.0/go.mod h1:sigOdh+0qb/+aOs3TVhehVT10p8qJL7K/Zhyz8vWo38= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -465,6 +468,7 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -867,6 +871,14 @@ github.com/ipni/index-provider v0.12.0 h1:R3F6dxxKNv4XkE4GJZNLOG0bDEbBQ/S5iztXwS github.com/ipni/index-provider v0.12.0/go.mod h1:GhyrADJp7n06fqoc1djzkvL4buZYHzV8SoWrlxEo5F4= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE= +github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= +github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk= +github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -950,6 +962,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= @@ -958,6 +971,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= @@ -1398,6 +1412,7 @@ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1426,8 +1441,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1461,8 +1476,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Eznag= @@ -1489,6 +1504,7 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= diff --git a/itests/clusterdb_test.go b/itests/clusterdb_test.go new file mode 100644 index 00000000000..e844912112e --- /dev/null +++ b/itests/clusterdb_test.go @@ -0,0 +1,188 @@ +package itests + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/filecoin-project/lotus/itests/kit" + "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/node" + "github.com/filecoin-project/lotus/node/config" +) + +func staticConfig() config.ClusterDB { + return config.ClusterDB{ + Hosts: []string{"127.0.0.1"}, + Username: "yugabyte", + Password: "yugabyte", + Database: "yugabyte", + Port: "5433", + ITest: clusterdb.ITestNewID(), + } +} +func withSetup(t *testing.T, f func(*kit.TestMiner)) { + _, miner, _ := kit.EnsembleMinimal(t, + kit.LatestActorsAt(-1), + kit.MockProofs(), + kit.ConstructorOpts( + node.Override(new(config.ClusterDB), func() config.ClusterDB { + return staticConfig() + }), + node.Override(new(*clusterdb.DB), clusterdb.NewFromConfig), //Why does this not work? + ), + ) + + var err error + miner.ClusterDB, err = clusterdb.NewFromConfig(staticConfig()) + if err != nil { + t.Fatal("Yugabyte connection error:", err) + } + + defer miner.ClusterDB.ITestDeleteAll() + f(miner) +} + +func TestCrud(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + withSetup(t, func(miner *kit.TestMiner) { + err := miner.ClusterDB.Exec(ctx, ` + INSERT INTO + itest_scratch (some_int, content) + VALUES + (11, 'cows'), + (5, 'cats') + `) + if err != nil { + t.Fatal("Could not insert: ", err) + } + var ints []struct { + Count int `db:"some_int"` + Animal string `db:"content"` + Unpopulated int + } + err = miner.ClusterDB.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch") + if err != nil { + t.Fatal("Could not select: ", err) + } + if len(ints) != 2 { + t.Fatal("unexpected count of returns. Want 2, Got ", len(ints)) + } + if ints[0].Count != 11 || ints[1].Count != 5 { + t.Fatal("expected [11,5] got ", ints) + } + if ints[0].Animal != "cows" || ints[1].Animal != "cats" { + t.Fatal("expected, [cows, cats] ", ints) + } + fmt.Println("test completed") + }) +} + +func TestTransaction(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + withSetup(t, func(miner *kit.TestMiner) { + if err := miner.ClusterDB.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { + t.Fatal("E0", err) + } + miner.ClusterDB.BeginTransaction(ctx, func(tx *clusterdb.Transaction) (commit bool) { + if err := tx.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil { + t.Fatal("E1", err) + } + + // sum1 is read from OUTSIDE the transaction so it's the old value + var sum1 int + if err := miner.ClusterDB.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum1); err != nil { + t.Fatal("E2", err) + } + if sum1 != 4+5+6 { + t.Fatal("Expected 15, got ", sum1) + } + + // sum2 is from INSIDE the transaction, so the updated value. + var sum2 int + if err := tx.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum2); err != nil { + t.Fatal("E3", err) + } + if sum2 != 4+5+6+7+8+9 { + t.Fatal("Expected 39, got ", sum2) + } + return false // rollback + }) + + var sum2 int + // Query() example (yes, QueryRow would be preferred here) + q, err := miner.ClusterDB.Query(ctx, "SELECT SUM(some_int) FROM itest_scratch") + if err != nil { + t.Fatal("E4", err) + } + defer q.Close() + var rowCt int + for q.Next() { + q.Scan(&sum2) + rowCt++ + } + if sum2 != 4+5+6 { + t.Fatal("Expected 15, got ", sum2) + } + if rowCt != 1 { + t.Fatal("unexpected count of rows") + } + }) +} + +func TestPartialWalk(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + withSetup(t, func(miner *kit.TestMiner) { + if err := miner.ClusterDB.Exec(ctx, ` + INSERT INTO + itest_scratch (content, some_int) + VALUES + ('andy was here', 5), + ('lotus is awesome', 6), + ('hello world', 7), + ('3rd integration test', 8), + ('fiddlesticks', 9) + `); err != nil { + t.Fatal("e1", err) + } + + // TASK: FIND THE ID of the string with a specific SHA256 + needle := "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + q, err := miner.ClusterDB.Query(ctx, `SELECT id, content FROM itest_scratch`) + if err != nil { + t.Fatal("e2", err) + } + defer q.Close() + + var tmp struct { + Src string `db:"content"` + ID int + } + + var done bool + for q.Next() { + + if err := q.StructScan(&tmp); err != nil { + t.Fatal("") + } + + bSha := sha256.Sum256([]byte(tmp.Src)) + if hex.EncodeToString(bSha[:]) == needle { + done = true + break + } + } + if !done { + t.Fatal("We didn't find it.") + } + // Answer: tmp.ID + }) +} diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 19cc163af5a..250caa64039 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -49,6 +49,7 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" "github.com/filecoin-project/lotus/gateway" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" "github.com/filecoin-project/lotus/markets/idxprov" "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test" lotusminer "github.com/filecoin-project/lotus/miner" @@ -763,6 +764,10 @@ func (n *Ensemble) Start() *Ensemble { ) } + opts = append(opts, node.Override(new(*clusterdb.DB), func(cfg config.ClusterDB) (*clusterdb.DB, error) { + return clusterdb.NewFromConfig(cfg) + })) + // start node stop, err := node.New(ctx, opts...) require.NoError(n.t, err) diff --git a/itests/kit/node_miner.go b/itests/kit/node_miner.go index 4b81c9df0bd..cdcc03b139a 100644 --- a/itests/kit/node_miner.go +++ b/itests/kit/node_miner.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/wallet/key" + "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" "github.com/filecoin-project/lotus/miner" sealing "github.com/filecoin-project/lotus/storage/pipeline" "github.com/filecoin-project/lotus/storage/sealer/storiface" @@ -87,6 +88,8 @@ type TestMiner struct { RemoteListener net.Listener options nodeOpts + + ClusterDB *clusterdb.DB } func (tm *TestMiner) PledgeSectors(ctx context.Context, n, existing int, blockNotif <-chan struct{}) { diff --git a/lib/sturdy/clusterdb/clusterdb.go b/lib/sturdy/clusterdb/clusterdb.go new file mode 100644 index 00000000000..4c323c78eb3 --- /dev/null +++ b/lib/sturdy/clusterdb/clusterdb.go @@ -0,0 +1,251 @@ +package clusterdb + +// TODO 2 - build integration tests + +import ( + "context" + "embed" + "errors" + "fmt" + "math/rand" + "regexp" + "sort" + "strconv" + "strings" + "time" + + logging "github.com/ipfs/go-log/v2" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/filecoin-project/lotus/node/config" +) + +// ItestNewID see ITestWithID doc +func ITestNewID() string { + return strconv.Itoa(rand.Intn(99999)) +} + +type DB struct { + pgx *pgxpool.Pool + cfg *pgxpool.Config + schema string + hostnames []string + log func(string) +} + +var logger = logging.Logger("sturdydb") + +// NewFromConfig is a convenience function. +// In usage: +// +// db, err := NewFromConfig(config.SturdyDB) // in binary init +func NewFromConfig(cfg config.ClusterDB) (*DB, error) { + return New( + cfg.Hosts, + cfg.Username, + cfg.Password, + cfg.Database, + cfg.Port, + cfg.ITest, + func(s string) { logger.Error(s) }, + ) +} + +// New is to be called once per binary to establish the pool. +// log() is for errors. It returns an upgraded database's connection. +// This entry point serves both production and integration tests, so it's more DI. +func New(hosts []string, username, password, database, port, itest string, log func(string)) (*DB, error) { + connString := "" + if len(hosts) > 0 { + connString = "host=" + hosts[0] + " " + } + for k, v := range map[string]string{"user": username, "password": password, "dbname": database, "port": port} { + if strings.TrimSpace(v) != "" { + connString += k + "=" + v + " " + } + } + + schema := "lotus" + if itest != "" { + schema = "itest_" + itest + } + + if err := ensureSchemaExists(connString, schema); err != nil { + return nil, err + } + cfg, err := pgxpool.ParseConfig(connString + "search_path=" + schema) + if err != nil { + return nil, err + } + + // enable multiple fallback hosts. + for _, h := range hosts[1:] { + cfg.ConnConfig.Fallbacks = append(cfg.ConnConfig.Fallbacks, &pgconn.FallbackConfig{Host: h}) + } + + cfg.ConnConfig.OnNotice = func(conn *pgconn.PgConn, n *pgconn.Notice) { + log("database notice: " + n.Message + ": " + n.Detail) + DBMeasures.Errors.M(1) + } + + db := DB{cfg: cfg, schema: schema, hostnames: hosts, log: log} // pgx populated in AddStatsAndConnect + if err := db.addStatsAndConnect(); err != nil { + return nil, err + } + + return &db, db.upgrade() +} + +type tracer struct { +} + +type ctxkey string + +var sqlStart = ctxkey("sqlStart") + +func (t tracer) TraceQueryStart(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + return context.WithValue(ctx, sqlStart, time.Now()) +} +func (t tracer) TraceQueryEnd(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryEndData) { + DBMeasures.Hits.M(1) + ms := time.Since(ctx.Value(sqlStart).(time.Time)).Milliseconds() + DBMeasures.TotalWait.M(ms) + DBMeasures.Waits.Observe(float64(ms)) + if data.Err != nil { + DBMeasures.Errors.M(1) + } + // Can log what type of query it is, but not what tables + // Can log rows affected. +} + +// addStatsAndConnect connects a prometheus logger. Be sure to run this before using the DB. +func (db *DB) addStatsAndConnect() error { + + db.cfg.ConnConfig.Tracer = tracer{} + + hostnameToIndex := map[string]float64{} + for i, h := range db.hostnames { + hostnameToIndex[h] = float64(i) + } + db.cfg.AfterConnect = func(ctx context.Context, c *pgx.Conn) error { + s := db.pgx.Stat() + DBMeasures.OpenConnections.M(int64(s.TotalConns())) + DBMeasures.WhichHost.Observe(hostnameToIndex[c.Config().Host]) + + //FUTURE place for any connection seasoning + return nil + } + + var err error + db.pgx, err = pgxpool.NewWithConfig(context.Background(), db.cfg) + if err != nil { + db.log(fmt.Sprintf("Unable to connect to database: %v\n", err)) + return err + } + return nil +} + +// ITestDeleteAll will delete everything created for "this" integration test. +// This must be called at the end of each integration test. +func (db *DB) ITestDeleteAll() { + if !strings.HasPrefix(db.schema, "itest_") { + fmt.Println("Warning: this should never be called on anything but an itest schema.") + return + } + defer db.pgx.Close() + _, err := db.pgx.Exec(context.Background(), "DROP SCHEMA "+db.schema+" CASCADE") + if err != nil { + fmt.Println("warning: unclean itest shutdown: cannot delete schema: " + err.Error()) + return + } +} + +var schemaREString = "^[A-Za-z0-9_]+$" +var schemaRE = regexp.MustCompile(schemaREString) + +func ensureSchemaExists(connString, schema string) error { + // FUTURE allow using fallback DBs for start-up. + p, err := pgx.Connect(context.Background(), connString) + if err != nil { + return err + } + defer p.Close(context.Background()) + + if len(schema) < 5 || !schemaRE.MatchString(schema) { + return errors.New("schema must be of the form " + schemaREString + "\n Got: " + schema) + } + _, err = p.Exec(context.Background(), "CREATE SCHEMA IF NOT EXISTS "+schema) + if err != nil { + return fmt.Errorf("cannot create schema: %w", err) + } + return nil +} + +//go:embed sql +var fs embed.FS + +func (db *DB) upgrade() error { + // Does the version table exist? if not, make it. + // NOTE: This cannot change except via the next sql file. + err := db.Exec(context.Background(), `CREATE TABLE IF NOT EXISTS base ( + id SERIAL PRIMARY KEY, + entry CHAR(12), + applied TIMESTAMP DEFAULT current_timestamp + )`) + if err != nil { + db.log("Upgrade failed.") + return err + } + + // __Run scripts in order.__ + + landed := map[string]bool{} + { + var landedEntries []struct{ Entry string } + err = db.Select(context.Background(), &landedEntries, "SELECT entry FROM base") + if err != nil { + db.log("Cannot read entries: " + err.Error()) + return err + } + for _, l := range landedEntries { + landed[l.Entry] = true + } + } + dir, err := fs.ReadDir("sql") + if err != nil { + db.log("Cannot read fs entries: " + err.Error()) + return err + } + sort.Slice(dir, func(i, j int) bool { return dir[i].Name() < dir[j].Name() }) + for _, e := range dir { + name := e.Name() + if landed[name] || !strings.HasSuffix(name, ".sql") { + continue + } + file, err := fs.ReadFile("sql/" + name) + if err != nil { + db.log("weird embed file read err") + return err + } + for _, s := range strings.Split(string(file), ";") { // Implement the changes. + if len(strings.TrimSpace(s)) == 0 { + continue + } + _, err = db.pgx.Exec(context.Background(), s) + if err != nil { + db.log(fmt.Sprintf("Could not upgrade! File %s, Query: %s, Returned: %s", name, s, err.Error())) + return err + } + } + + // Mark Completed. + err = db.Exec(context.Background(), "INSERT INTO base (entry) VALUES ($1)", name) + if err != nil { + db.log("Cannot update base: " + err.Error()) + return fmt.Errorf("cannot insert into base: %w", err) + } + } + return nil +} diff --git a/lib/sturdy/clusterdb/doc.go b/lib/sturdy/clusterdb/doc.go new file mode 100644 index 00000000000..6e86aacd73a --- /dev/null +++ b/lib/sturdy/clusterdb/doc.go @@ -0,0 +1,35 @@ +/* +# Clusterdb provides database abstractions over SP-wide Postgres-compatible instance(s). + +# Features + + Rolling to secondary database servers on connection failure + Convenience features for Go + SQL + Prevention of SQL injection vulnerabilities + Monitors behavior via Prometheus stats and logging of errors. + +# Usage + +Processes should use New() to instantiate a *DB and keep it. +Consumers can use this *DB concurrently. +Creating and changing tables & views should happen in ./sql/ folder. +Name the file "today's date" in the format: YYYYMMDD.sql (ex: 20231231.sql for the year's last day) + + a. CREATE TABLE should NOT have a schema: + GOOD: CREATE TABLE foo (); + BAD: CREATE TABLE me.foo (); + b. Schema is managed for you. It provides isolation for integraton tests & multi-use. + c. Git Merges: All run once, so old-after-new is OK when there are no deps. + d. NEVER change shipped sql files. Have later files make corrections. + e. Anything not ran will be ran, so an older date making it to master is OK. + +Write SQL with context, raw strings, and args: + + name := "Alice" + var ID int + err := QueryRow(ctx, "SELECT id from people where first_name=?", name).Scan(&ID) + fmt.Println(ID) + +Note: Scan() is column-oriented, while Select() & StructScan() is field name/tag oriented. +*/ +package clusterdb diff --git a/lib/sturdy/clusterdb/metrics.go b/lib/sturdy/clusterdb/metrics.go new file mode 100644 index 00000000000..430de54f209 --- /dev/null +++ b/lib/sturdy/clusterdb/metrics.go @@ -0,0 +1,70 @@ +package clusterdb + +import ( + "github.com/prometheus/client_golang/prometheus" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + + "github.com/filecoin-project/lotus/metrics" +) + +var ( + dbTag, _ = tag.NewKey("db_name") + pre = "clusterdb/base/" + waitsBuckets = []float64{0, 10, 20, 30, 50, 80, 130, 210, 340, 550, 890} + whichHostBuckets = []float64{0, 1, 2, 3, 4, 5} +) + +// DBMeasures groups all db metrics. +var DBMeasures = struct { + Hits *stats.Int64Measure + TotalWait *stats.Int64Measure + Waits prometheus.Histogram + OpenConnections *stats.Int64Measure + Errors *stats.Int64Measure + WhichHost prometheus.Histogram +}{ + Hits: stats.Int64(pre+"hits", "Total number of uses.", stats.UnitDimensionless), + TotalWait: stats.Int64(pre+"total_wait", "Total delay. A numerator over hits to get average wait.", stats.UnitMilliseconds), + Waits: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: pre + "waits", + Buckets: waitsBuckets, + Help: "The histogram of waits for query completions.", + }), + OpenConnections: stats.Int64(pre+"open_connections", "Total connection count.", stats.UnitDimensionless), + Errors: stats.Int64(pre+"errors", "Total error count.", stats.UnitDimensionless), + WhichHost: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: pre + "which_host", + Buckets: whichHostBuckets, + Help: "The index of the hostname being used", + }), +} + +// CacheViews groups all cache-related default views. +func init() { + metrics.RegisterViews( + &view.View{ + Measure: DBMeasures.Hits, + Aggregation: view.Sum(), + TagKeys: []tag.Key{dbTag}, + }, + &view.View{ + Measure: DBMeasures.TotalWait, + Aggregation: view.Sum(), + TagKeys: []tag.Key{dbTag}, + }, + &view.View{ + Measure: DBMeasures.OpenConnections, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{dbTag}, + }, + &view.View{ + Measure: DBMeasures.Errors, + Aggregation: view.Sum(), + TagKeys: []tag.Key{dbTag}, + }, + ) + prometheus.Register(DBMeasures.Waits) + prometheus.Register(DBMeasures.WhichHost) +} diff --git a/lib/sturdy/clusterdb/sql/20230706.sql b/lib/sturdy/clusterdb/sql/20230706.sql new file mode 100644 index 00000000000..b45aca7fa71 --- /dev/null +++ b/lib/sturdy/clusterdb/sql/20230706.sql @@ -0,0 +1,6 @@ +CREATE TABLE itest_scratch ( + id SERIAL PRIMARY KEY, + content TEXT, + some_int INTEGER, + update_time TIMESTAMP DEFAULT current_timestamp +) \ No newline at end of file diff --git a/lib/sturdy/clusterdb/userfuncs.go b/lib/sturdy/clusterdb/userfuncs.go new file mode 100644 index 00000000000..e9cd84bd021 --- /dev/null +++ b/lib/sturdy/clusterdb/userfuncs.go @@ -0,0 +1,149 @@ +package clusterdb + +import ( + "context" + + "github.com/georgysavva/scany/v2/pgxscan" + "github.com/jackc/pgx/v5" +) + +type Intf interface { + ITestDeleteAll() + Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error + Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) + QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row + Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error + BeginTransaction(ctx context.Context, f func(t *Transaction) (commit bool)) (retErr error) +} + +// rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries. +// In any package, raw strings will satisfy compilation. Ex: +// +// sturdydb.Exec("INSERT INTO version (number) VALUES (1)") +// +// This prevents SQL injection attacks where the input contains query fragments. +type rawStringOnly string + +// Exec executes changes (INSERT, DELETE, or UPDATE). +// Note, for CREATE & DROP please keep these permanent and express +// them in the ./sql/ files (next number). +func (db *DB) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error { + _, err := db.pgx.Exec(ctx, string(sql), arguments...) + return err +} + +type Qry interface { + Next() bool + Err() error + Close() + Scan(...any) error + Values() ([]any, error) +} + +// Query offers Next/Err/Close/Scan/Values/StructScan +type Query struct { + Qry +} + +// Query allows iterating returned values to save memory consumption +// with the downside of needing to `defer q.Close()`. For a simpler interface, +// try Select() +// Next() must be called to advance the row cursor, including the first time: +// Ex: +// q, err := db.Query(ctx, "SELECT id, name FROM users") +// handleError(err) +// defer q.Close() +// +// for q.Next() { +// var id int +// var name string +// handleError(q.Scan(&id, &name)) +// fmt.Println(id, name) +// } +func (db *DB) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) { + q, err := db.pgx.Query(ctx, string(sql), arguments...) + return &Query{q}, err +} +func (q *Query) StructScan(s any) error { + return pgxscan.ScanRow(s, q.Qry.(pgx.Rows)) +} + +type Row interface { + Scan(...any) error +} + +// QueryRow gets 1 row using column order matching. +// This is a timesaver for the special case of wanting the first row returned only. +// EX: +// +// var name, pet string +// var ID = 123 +// err := db.QueryRow(ctx, "SELECT name, pet FROM users WHERE ID=?", ID).Scan(&name, &pet) +func (db *DB) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row { + return db.pgx.QueryRow(ctx, string(sql), arguments...) +} + +/* +Select multiple rows into a slice using name matching +Ex: + + type user struct { + Name string + ID int + Number string `db:"tel_no"` + } + + var users []user + pet := "cat" + err := db.Select(ctx, &users, "SELECT name, id, tel_no FROM customers WHERE pet=?", pet) +*/ +func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error { + return pgxscan.Select(ctx, db.pgx, sliceOfStructPtr, string(sql), arguments...) +} + +type Transaction struct { + pgx.Tx +} + +// BeginTransaction is how you can access transactions using this library. +// The entire transaction happens in the function passed in. +// The return must be true or a rollback will occur. +func (db *DB) BeginTransaction(ctx context.Context, f func(t *Transaction) (commit bool)) (retErr error) { + tx, err := db.pgx.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return err + } + var commit bool + defer func() { // Panic clean-up. + if !commit { + retErr = tx.Rollback(ctx) + } + }() + commit = f(&Transaction{tx}) + if commit { + return tx.Commit(ctx) + } + return nil +} + +// Exec in a transaction. +func (t *Transaction) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error { + _, err := t.Tx.Exec(ctx, string(sql), arguments...) + return err +} + +// Query in a transaction. +func (t *Transaction) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) { + q, err := t.Tx.Query(ctx, string(sql), arguments...) + return &Query{q}, err +} + +// QueryRow in a transaction. +func (t *Transaction) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row { + return t.Tx.QueryRow(ctx, string(sql), arguments...) +} + +// Select in a transaction. +func (t *Transaction) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error { + return pgxscan.Select(ctx, t.Tx, sliceOfStructPtr, string(sql), arguments...) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index ee7bd8695cc..b1c241b2134 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -695,46 +695,55 @@ var ( } ) +var views = []*view.View{ + InfoView, + PeerCountView, + APIRequestDurationView, + + GraphsyncReceivingPeersCountView, + GraphsyncReceivingActiveCountView, + GraphsyncReceivingCountCountView, + GraphsyncReceivingTotalMemoryAllocatedView, + GraphsyncReceivingTotalPendingAllocationsView, + GraphsyncReceivingPeersPendingView, + GraphsyncSendingPeersCountView, + GraphsyncSendingActiveCountView, + GraphsyncSendingCountCountView, + GraphsyncSendingTotalMemoryAllocatedView, + GraphsyncSendingTotalPendingAllocationsView, + GraphsyncSendingPeersPendingView, + + RcmgrAllowConnView, + RcmgrBlockConnView, + RcmgrAllowStreamView, + RcmgrBlockStreamView, + RcmgrAllowPeerView, + RcmgrBlockPeerView, + RcmgrAllowProtoView, + RcmgrBlockProtoView, + RcmgrBlockProtoPeerView, + RcmgrAllowSvcView, + RcmgrBlockSvcView, + RcmgrBlockSvcPeerView, + RcmgrAllowMemView, + RcmgrBlockMemView, +} + // DefaultViews is an array of OpenCensus views for metric gathering purposes var DefaultViews = func() []*view.View { - views := []*view.View{ - InfoView, - PeerCountView, - APIRequestDurationView, - - GraphsyncReceivingPeersCountView, - GraphsyncReceivingActiveCountView, - GraphsyncReceivingCountCountView, - GraphsyncReceivingTotalMemoryAllocatedView, - GraphsyncReceivingTotalPendingAllocationsView, - GraphsyncReceivingPeersPendingView, - GraphsyncSendingPeersCountView, - GraphsyncSendingActiveCountView, - GraphsyncSendingCountCountView, - GraphsyncSendingTotalMemoryAllocatedView, - GraphsyncSendingTotalPendingAllocationsView, - GraphsyncSendingPeersPendingView, - - RcmgrAllowConnView, - RcmgrBlockConnView, - RcmgrAllowStreamView, - RcmgrBlockStreamView, - RcmgrAllowPeerView, - RcmgrBlockPeerView, - RcmgrAllowProtoView, - RcmgrBlockProtoView, - RcmgrBlockProtoPeerView, - RcmgrAllowSvcView, - RcmgrBlockSvcView, - RcmgrBlockSvcPeerView, - RcmgrAllowMemView, - RcmgrBlockMemView, - } - views = append(views, blockstore.DefaultViews...) - views = append(views, rpcmetrics.DefaultViews...) return views }() +// RegisterViews adds views to the default list without modifying this file. +func RegisterViews(v ...*view.View) { + views = append(views, v...) +} + +func init() { + RegisterViews(blockstore.DefaultViews...) + RegisterViews(rpcmetrics.DefaultViews...) +} + var ChainNodeViews = append([]*view.View{ ChainNodeHeightView, ChainNodeHeightExpectedView, diff --git a/node/builder_miner.go b/node/builder_miner.go index d9c92422d00..69d567da80f 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" + "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/dealfilter" "github.com/filecoin-project/lotus/markets/idxprov" @@ -231,6 +232,7 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(config.SealerConfig), cfg.Storage), Override(new(config.ProvingConfig), cfg.Proving), Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), + Override(new(*clusterdb.DB), clusterdb.NewFromConfig), ) } diff --git a/node/config/def.go b/node/config/def.go index 42b035c66ae..5075355dc8f 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -269,6 +269,13 @@ func DefaultStorageMiner() *StorageMiner { MaxConcurrentUnseals: 5, GCInterval: Duration(1 * time.Minute), }, + ClusterDB: ClusterDB{ + Hosts: []string{"127.0.0.1"}, + Username: "yugabyte", + Password: "yugabyte", + Database: "yugabyte", + Port: "5433", + }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 5361b2d6c59..9c3617a6918 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -117,6 +117,46 @@ without existing payment channels with available funds will fail instead of automatically performing on-chain operations.`, }, }, + "ClusterDB": []DocField{ + { + Name: "Hosts", + Type: "[]string", + + Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB +in a cluster. Only 1 is required`, + }, + { + Name: "Username", + Type: "string", + + Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`, + }, + { + Name: "Password", + Type: "string", + + Comment: `The password for the related username. Blank for default.`, + }, + { + Name: "Database", + Type: "string", + + Comment: `The database (logical partition) within Yugabyte. Blank for default.`, + }, + { + Name: "Port", + Type: "string", + + Comment: `The port to find Yugabyte. Blank for default.`, + }, + { + Name: "ITest", + Type: "string", + + Comment: `ITest is for optimized integration testing and not +for production. Blank for default production configuration.`, + }, + }, "Common": []DocField{ { Name: "API", @@ -1386,6 +1426,12 @@ HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`, Name: "DAGStore", Type: "DAGStoreConfig", + Comment: ``, + }, + { + Name: "ClusterDB", + Type: "ClusterDB", + Comment: ``, }, }, diff --git a/node/config/types.go b/node/config/types.go index c89e8f70bad..74630d9a3cd 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -61,6 +61,8 @@ type StorageMiner struct { Fees MinerFeeConfig Addresses MinerAddressConfig DAGStore DAGStoreConfig + + ClusterDB ClusterDB } type DAGStoreConfig struct { @@ -732,3 +734,25 @@ type IndexConfig struct { // EnableMsgIndex enables indexing of messages on chain. EnableMsgIndex bool } + +type ClusterDB struct { + // HOSTS is a list of hostnames to nodes running YugabyteDB + // in a cluster. Only 1 is required + Hosts []string + + // The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default. + Username string + + // The password for the related username. Blank for default. + Password string + + // The database (logical partition) within Yugabyte. Blank for default. + Database string + + // The port to find Yugabyte. Blank for default. + Port string + + // ITest is for optimized integration testing and not + // for production. Blank for default production configuration. + ITest string +} From 3212a237414d037625808815a61ad51423e7ea65 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Tue, 11 Jul 2023 21:50:03 -0500 Subject: [PATCH 02/36] renames --- itests/clusterdb_test.go | 3 --- lib/sturdy/clusterdb/clusterdb.go | 4 ++-- lib/sturdy/clusterdb/userfuncs.go | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/itests/clusterdb_test.go b/itests/clusterdb_test.go index e844912112e..e1b29f691d0 100644 --- a/itests/clusterdb_test.go +++ b/itests/clusterdb_test.go @@ -28,9 +28,6 @@ func withSetup(t *testing.T, f func(*kit.TestMiner)) { kit.LatestActorsAt(-1), kit.MockProofs(), kit.ConstructorOpts( - node.Override(new(config.ClusterDB), func() config.ClusterDB { - return staticConfig() - }), node.Override(new(*clusterdb.DB), clusterdb.NewFromConfig), //Why does this not work? ), ) diff --git a/lib/sturdy/clusterdb/clusterdb.go b/lib/sturdy/clusterdb/clusterdb.go index 4c323c78eb3..7b77b6365f7 100644 --- a/lib/sturdy/clusterdb/clusterdb.go +++ b/lib/sturdy/clusterdb/clusterdb.go @@ -35,12 +35,12 @@ type DB struct { log func(string) } -var logger = logging.Logger("sturdydb") +var logger = logging.Logger("clusterdb") // NewFromConfig is a convenience function. // In usage: // -// db, err := NewFromConfig(config.SturdyDB) // in binary init +// db, err := NewFromConfig(config.ClusterDB) // in binary init func NewFromConfig(cfg config.ClusterDB) (*DB, error) { return New( cfg.Hosts, diff --git a/lib/sturdy/clusterdb/userfuncs.go b/lib/sturdy/clusterdb/userfuncs.go index e9cd84bd021..98ceddb0883 100644 --- a/lib/sturdy/clusterdb/userfuncs.go +++ b/lib/sturdy/clusterdb/userfuncs.go @@ -19,7 +19,7 @@ type Intf interface { // rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries. // In any package, raw strings will satisfy compilation. Ex: // -// sturdydb.Exec("INSERT INTO version (number) VALUES (1)") +// clusterDB.Exec("INSERT INTO version (number) VALUES (1)") // // This prevents SQL injection attacks where the input contains query fragments. type rawStringOnly string From c89e9ca44f7b98a27666340afe70e00eb0ef8b14 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 11:02:23 -0500 Subject: [PATCH 03/36] SturdyConfig Fixed: ITestID --- itests/clusterdb_test.go | 45 ++++++++++++------------------- itests/kit/ensemble.go | 5 ---- itests/kit/node_miner.go | 3 --- lib/sturdy/clusterdb/clusterdb.go | 27 ++++++++++++++----- node/builder_miner.go | 3 ++- node/impl/storminer.go | 3 +++ 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/itests/clusterdb_test.go b/itests/clusterdb_test.go index e1b29f691d0..9a2ddb2e1de 100644 --- a/itests/clusterdb_test.go +++ b/itests/clusterdb_test.go @@ -10,35 +10,21 @@ import ( "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" "github.com/filecoin-project/lotus/node" - "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/impl" ) -func staticConfig() config.ClusterDB { - return config.ClusterDB{ - Hosts: []string{"127.0.0.1"}, - Username: "yugabyte", - Password: "yugabyte", - Database: "yugabyte", - Port: "5433", - ITest: clusterdb.ITestNewID(), - } -} func withSetup(t *testing.T, f func(*kit.TestMiner)) { _, miner, _ := kit.EnsembleMinimal(t, kit.LatestActorsAt(-1), kit.MockProofs(), kit.ConstructorOpts( - node.Override(new(*clusterdb.DB), clusterdb.NewFromConfig), //Why does this not work? + node.Override(new(clusterdb.ITestID), func() clusterdb.ITestID { + return clusterdb.ITestNewID() + }), ), ) - var err error - miner.ClusterDB, err = clusterdb.NewFromConfig(staticConfig()) - if err != nil { - t.Fatal("Yugabyte connection error:", err) - } - - defer miner.ClusterDB.ITestDeleteAll() + defer miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB.ITestDeleteAll() f(miner) } @@ -47,7 +33,8 @@ func TestCrud(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - err := miner.ClusterDB.Exec(ctx, ` + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + err := cdb.Exec(ctx, ` INSERT INTO itest_scratch (some_int, content) VALUES @@ -62,7 +49,7 @@ func TestCrud(t *testing.T) { Animal string `db:"content"` Unpopulated int } - err = miner.ClusterDB.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch") + err = cdb.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch") if err != nil { t.Fatal("Could not select: ", err) } @@ -84,17 +71,18 @@ func TestTransaction(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - if err := miner.ClusterDB.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + if err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { t.Fatal("E0", err) } - miner.ClusterDB.BeginTransaction(ctx, func(tx *clusterdb.Transaction) (commit bool) { + cdb.BeginTransaction(ctx, func(tx *clusterdb.Transaction) (commit bool) { if err := tx.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil { t.Fatal("E1", err) } // sum1 is read from OUTSIDE the transaction so it's the old value var sum1 int - if err := miner.ClusterDB.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum1); err != nil { + if err := cdb.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum1); err != nil { t.Fatal("E2", err) } if sum1 != 4+5+6 { @@ -114,7 +102,7 @@ func TestTransaction(t *testing.T) { var sum2 int // Query() example (yes, QueryRow would be preferred here) - q, err := miner.ClusterDB.Query(ctx, "SELECT SUM(some_int) FROM itest_scratch") + q, err := cdb.Query(ctx, "SELECT SUM(some_int) FROM itest_scratch") if err != nil { t.Fatal("E4", err) } @@ -138,7 +126,8 @@ func TestPartialWalk(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - if err := miner.ClusterDB.Exec(ctx, ` + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + if err := cdb.Exec(ctx, ` INSERT INTO itest_scratch (content, some_int) VALUES @@ -153,7 +142,7 @@ func TestPartialWalk(t *testing.T) { // TASK: FIND THE ID of the string with a specific SHA256 needle := "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" - q, err := miner.ClusterDB.Query(ctx, `SELECT id, content FROM itest_scratch`) + q, err := cdb.Query(ctx, `SELECT id, content FROM itest_scratch`) if err != nil { t.Fatal("e2", err) } @@ -168,7 +157,7 @@ func TestPartialWalk(t *testing.T) { for q.Next() { if err := q.StructScan(&tmp); err != nil { - t.Fatal("") + t.Fatal("structscan err " + err.Error()) } bSha := sha256.Sum256([]byte(tmp.Src)) diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 250caa64039..19cc163af5a 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -49,7 +49,6 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" "github.com/filecoin-project/lotus/gateway" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" "github.com/filecoin-project/lotus/markets/idxprov" "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test" lotusminer "github.com/filecoin-project/lotus/miner" @@ -764,10 +763,6 @@ func (n *Ensemble) Start() *Ensemble { ) } - opts = append(opts, node.Override(new(*clusterdb.DB), func(cfg config.ClusterDB) (*clusterdb.DB, error) { - return clusterdb.NewFromConfig(cfg) - })) - // start node stop, err := node.New(ctx, opts...) require.NoError(n.t, err) diff --git a/itests/kit/node_miner.go b/itests/kit/node_miner.go index cdcc03b139a..4b81c9df0bd 100644 --- a/itests/kit/node_miner.go +++ b/itests/kit/node_miner.go @@ -24,7 +24,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/wallet/key" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" "github.com/filecoin-project/lotus/miner" sealing "github.com/filecoin-project/lotus/storage/pipeline" "github.com/filecoin-project/lotus/storage/sealer/storiface" @@ -88,8 +87,6 @@ type TestMiner struct { RemoteListener net.Listener options nodeOpts - - ClusterDB *clusterdb.DB } func (tm *TestMiner) PledgeSectors(ctx context.Context, n, existing int, blockNotif <-chan struct{}) { diff --git a/lib/sturdy/clusterdb/clusterdb.go b/lib/sturdy/clusterdb/clusterdb.go index 7b77b6365f7..13a5db20982 100644 --- a/lib/sturdy/clusterdb/clusterdb.go +++ b/lib/sturdy/clusterdb/clusterdb.go @@ -1,7 +1,5 @@ package clusterdb -// TODO 2 - build integration tests - import ( "context" "embed" @@ -22,9 +20,11 @@ import ( "github.com/filecoin-project/lotus/node/config" ) +type ITestID string + // ItestNewID see ITestWithID doc -func ITestNewID() string { - return strconv.Itoa(rand.Intn(99999)) +func ITestNewID() ITestID { + return ITestID(strconv.Itoa(rand.Intn(99999))) } type DB struct { @@ -48,15 +48,30 @@ func NewFromConfig(cfg config.ClusterDB) (*DB, error) { cfg.Password, cfg.Database, cfg.Port, - cfg.ITest, + "", func(s string) { logger.Error(s) }, ) } +func NewFromConfigWithITestID(cfg config.ClusterDB) func(id ITestID) (*DB, error) { + return func(id ITestID) (*DB, error) { + return New( + cfg.Hosts, + cfg.Username, + cfg.Password, + cfg.Database, + cfg.Port, + id, + func(s string) { logger.Error(s) }, + ) + } +} + // New is to be called once per binary to establish the pool. // log() is for errors. It returns an upgraded database's connection. // This entry point serves both production and integration tests, so it's more DI. -func New(hosts []string, username, password, database, port, itest string, log func(string)) (*DB, error) { +func New(hosts []string, username, password, database, port string, itestID ITestID, log func(string)) (*DB, error) { + itest := string(itestID) connString := "" if len(hosts) > 0 { connString = "host=" + hosts[0] + " " diff --git a/node/builder_miner.go b/node/builder_miner.go index 69d567da80f..de777f34f92 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -232,7 +232,8 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(config.SealerConfig), cfg.Storage), Override(new(config.ProvingConfig), cfg.Proving), Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), - Override(new(*clusterdb.DB), clusterdb.NewFromConfig), + Override(new(*clusterdb.DB), clusterdb.NewFromConfigWithITestID(cfg.ClusterDB)), + Override(new(clusterdb.ITestID), ""), ) } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index e4fa41c788a..89837baffb9 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -45,6 +45,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" @@ -122,6 +123,8 @@ type StorageMinerAPI struct { GetSealingConfigFunc dtypes.GetSealingConfigFunc `optional:"true"` GetExpectedSealDurationFunc dtypes.GetExpectedSealDurationFunc `optional:"true"` SetExpectedSealDurationFunc dtypes.SetExpectedSealDurationFunc `optional:"true"` + + ClusterDB *clusterdb.DB } var _ api.StorageMiner = &StorageMinerAPI{} From 76489c2e04b3e628444ea8a436a085a591654c6c Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 11:25:44 -0500 Subject: [PATCH 04/36] Clusterdb ITests: default to unique test id --- itests/clusterdb_test.go | 6 ------ itests/kit/ensemble.go | 8 ++++++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/itests/clusterdb_test.go b/itests/clusterdb_test.go index 9a2ddb2e1de..05846af7242 100644 --- a/itests/clusterdb_test.go +++ b/itests/clusterdb_test.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" - "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" ) @@ -17,11 +16,6 @@ func withSetup(t *testing.T, f func(*kit.TestMiner)) { _, miner, _ := kit.EnsembleMinimal(t, kit.LatestActorsAt(-1), kit.MockProofs(), - kit.ConstructorOpts( - node.Override(new(clusterdb.ITestID), func() clusterdb.ITestID { - return clusterdb.ITestNewID() - }), - ), ) defer miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB.ITestDeleteAll() diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 19cc163af5a..d00077274cf 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -49,6 +49,7 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" "github.com/filecoin-project/lotus/gateway" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" "github.com/filecoin-project/lotus/markets/idxprov" "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test" lotusminer "github.com/filecoin-project/lotus/miner" @@ -357,6 +358,8 @@ func (n *Ensemble) Start() *Ensemble { n.mn = mocknet.New() } + sharedITestID := clusterdb.ITestNewID() + // --------------------- // FULL NODES // --------------------- @@ -722,6 +725,8 @@ func (n *Ensemble) Start() *Ensemble { // upgrades node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule), + + node.Override(new(clusterdb.ITestID), sharedITestID), } if m.options.subsystems.Has(SMarkets) { @@ -824,6 +829,8 @@ func (n *Ensemble) Start() *Ensemble { auth := http.Header(nil) + // FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).ClusterDB to setup. + remote := paths.NewRemote(localStore, m.MinerNode, auth, 20, &paths.DefaultPartialFileHandler{}) store := m.options.workerStorageOpt(remote) @@ -853,6 +860,7 @@ func (n *Ensemble) Start() *Ensemble { require.NoError(n.t, err) n.active.workers = append(n.active.workers, m) + } // If we are here, we have processed all inactive workers and moved them From 27111da6beab2c3ecce6dae683fdbc0ae56eebc1 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 14:44:05 -0500 Subject: [PATCH 05/36] ITestID not string --- extern/filecoin-ffi | 2 +- itests/clusterdb_test.go | 1 - itests/kit/ensemble.go | 4 ++++ node/builder_miner.go | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index de34caff946..a458f638e3c 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit de34caff946d598edb299566d951b44b9b7f7dd4 +Subproject commit a458f638e3c8603c9b5a9ed9847c3af4597e46d4 diff --git a/itests/clusterdb_test.go b/itests/clusterdb_test.go index 05846af7242..dd9dd8f5bbf 100644 --- a/itests/clusterdb_test.go +++ b/itests/clusterdb_test.go @@ -18,7 +18,6 @@ func withSetup(t *testing.T, f func(*kit.TestMiner)) { kit.MockProofs(), ) - defer miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB.ITestDeleteAll() f(miner) } diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index d00077274cf..da9df08d1ac 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -55,6 +55,7 @@ import ( lotusminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" testing2 "github.com/filecoin-project/lotus/node/modules/testing" @@ -773,6 +774,9 @@ func (n *Ensemble) Start() *Ensemble { require.NoError(n.t, err) n.t.Cleanup(func() { _ = stop(context.Background()) }) + n.t.Cleanup(func() { + m.StorageMiner.(*impl.StorageMinerAPI).ClusterDB.ITestDeleteAll() + }) m.BaseAPI = m.StorageMiner diff --git a/node/builder_miner.go b/node/builder_miner.go index de777f34f92..366c89fa4ee 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -233,7 +233,7 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(config.ProvingConfig), cfg.Proving), Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), Override(new(*clusterdb.DB), clusterdb.NewFromConfigWithITestID(cfg.ClusterDB)), - Override(new(clusterdb.ITestID), ""), + Override(new(clusterdb.ITestID), clusterdb.ITestID("")), ) } From 4b5c417bfd57b1e71d18cc7b190a9c8fea08f9c1 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 16:08:14 -0500 Subject: [PATCH 06/36] circleci postgres try1 --- .circleci/config.yml | 93 ++++++++++++++++++++++++++++++++++++++++++ .circleci/template.yml | 12 ++++++ 2 files changed, 105 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d633daf1ab..049855bb051 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -165,6 +165,14 @@ jobs: default: unit description: Test suite name to report to CircleCI. executor: << parameters.executor >> + docker: + - image: circleci/postgres:12 + environment: + PGHOST: localhost + PGUSER: yugabyte + POSTGRES_USER: yugabyte + POSTGRES_DB: yugabyte + POSTGRES_HOST_AUTH_METHOD: trust steps: - install-ubuntu-deps - attach_workspace: @@ -173,6 +181,9 @@ jobs: condition: << parameters.get-params >> steps: - download-params + - run: + name: Wait for db to run + command: dockerize -wait tcp://localhost:5433 -timeout 1m - run: name: go test environment: @@ -537,54 +548,63 @@ workflows: name: test-itest-api requires: - build + - postgres suite: itest-api target: "./itests/api_test.go" - test: name: test-itest-batch_deal requires: - build + - postgres suite: itest-batch_deal target: "./itests/batch_deal_test.go" - test: name: test-itest-ccupgrade requires: - build + - postgres suite: itest-ccupgrade target: "./itests/ccupgrade_test.go" - test: name: test-itest-cli requires: - build + - postgres suite: itest-cli target: "./itests/cli_test.go" - test: name: test-itest-clusterdb requires: - build + - postgres suite: itest-clusterdb target: "./itests/clusterdb_test.go" - test: name: test-itest-deadlines requires: - build + - postgres suite: itest-deadlines target: "./itests/deadlines_test.go" - test: name: test-itest-deals_512mb requires: - build + - postgres suite: itest-deals_512mb target: "./itests/deals_512mb_test.go" - test: name: test-itest-deals_anycid requires: - build + - postgres suite: itest-deals_anycid target: "./itests/deals_anycid_test.go" - test: name: test-itest-deals_concurrent requires: - build + - postgres suite: itest-deals_concurrent target: "./itests/deals_concurrent_test.go" executor: golang-2xl @@ -592,420 +612,490 @@ workflows: name: test-itest-deals_invalid_utf8_label requires: - build + - postgres suite: itest-deals_invalid_utf8_label target: "./itests/deals_invalid_utf8_label_test.go" - test: name: test-itest-deals_max_staging_deals requires: - build + - postgres suite: itest-deals_max_staging_deals target: "./itests/deals_max_staging_deals_test.go" - test: name: test-itest-deals_offline requires: - build + - postgres suite: itest-deals_offline target: "./itests/deals_offline_test.go" - test: name: test-itest-deals_padding requires: - build + - postgres suite: itest-deals_padding target: "./itests/deals_padding_test.go" - test: name: test-itest-deals_partial_retrieval_dm-level requires: - build + - postgres suite: itest-deals_partial_retrieval_dm-level target: "./itests/deals_partial_retrieval_dm-level_test.go" - test: name: test-itest-deals_partial_retrieval requires: - build + - postgres suite: itest-deals_partial_retrieval target: "./itests/deals_partial_retrieval_test.go" - test: name: test-itest-deals_power requires: - build + - postgres suite: itest-deals_power target: "./itests/deals_power_test.go" - test: name: test-itest-deals_pricing requires: - build + - postgres suite: itest-deals_pricing target: "./itests/deals_pricing_test.go" - test: name: test-itest-deals_publish requires: - build + - postgres suite: itest-deals_publish target: "./itests/deals_publish_test.go" - test: name: test-itest-deals_remote_retrieval requires: - build + - postgres suite: itest-deals_remote_retrieval target: "./itests/deals_remote_retrieval_test.go" - test: name: test-itest-deals_retry_deal_no_funds requires: - build + - postgres suite: itest-deals_retry_deal_no_funds target: "./itests/deals_retry_deal_no_funds_test.go" - test: name: test-itest-deals requires: - build + - postgres suite: itest-deals target: "./itests/deals_test.go" - test: name: test-itest-decode_params requires: - build + - postgres suite: itest-decode_params target: "./itests/decode_params_test.go" - test: name: test-itest-dup_mpool_messages requires: - build + - postgres suite: itest-dup_mpool_messages target: "./itests/dup_mpool_messages_test.go" - test: name: test-itest-eth_account_abstraction requires: - build + - postgres suite: itest-eth_account_abstraction target: "./itests/eth_account_abstraction_test.go" - test: name: test-itest-eth_api requires: - build + - postgres suite: itest-eth_api target: "./itests/eth_api_test.go" - test: name: test-itest-eth_balance requires: - build + - postgres suite: itest-eth_balance target: "./itests/eth_balance_test.go" - test: name: test-itest-eth_block_hash requires: - build + - postgres suite: itest-eth_block_hash target: "./itests/eth_block_hash_test.go" - test: name: test-itest-eth_bytecode requires: - build + - postgres suite: itest-eth_bytecode target: "./itests/eth_bytecode_test.go" - test: name: test-itest-eth_config requires: - build + - postgres suite: itest-eth_config target: "./itests/eth_config_test.go" - test: name: test-itest-eth_conformance requires: - build + - postgres suite: itest-eth_conformance target: "./itests/eth_conformance_test.go" - test: name: test-itest-eth_deploy requires: - build + - postgres suite: itest-eth_deploy target: "./itests/eth_deploy_test.go" - test: name: test-itest-eth_fee_history requires: - build + - postgres suite: itest-eth_fee_history target: "./itests/eth_fee_history_test.go" - test: name: test-itest-eth_filter requires: - build + - postgres suite: itest-eth_filter target: "./itests/eth_filter_test.go" - test: name: test-itest-eth_hash_lookup requires: - build + - postgres suite: itest-eth_hash_lookup target: "./itests/eth_hash_lookup_test.go" - test: name: test-itest-eth_transactions requires: - build + - postgres suite: itest-eth_transactions target: "./itests/eth_transactions_test.go" - test: name: test-itest-fevm_address requires: - build + - postgres suite: itest-fevm_address target: "./itests/fevm_address_test.go" - test: name: test-itest-fevm_events requires: - build + - postgres suite: itest-fevm_events target: "./itests/fevm_events_test.go" - test: name: test-itest-fevm requires: - build + - postgres suite: itest-fevm target: "./itests/fevm_test.go" - test: name: test-itest-gas_estimation requires: - build + - postgres suite: itest-gas_estimation target: "./itests/gas_estimation_test.go" - test: name: test-itest-gateway requires: - build + - postgres suite: itest-gateway target: "./itests/gateway_test.go" - test: name: test-itest-get_messages_in_ts requires: - build + - postgres suite: itest-get_messages_in_ts target: "./itests/get_messages_in_ts_test.go" - test: name: test-itest-lite_migration requires: - build + - postgres suite: itest-lite_migration target: "./itests/lite_migration_test.go" - test: name: test-itest-lookup_robust_address requires: - build + - postgres suite: itest-lookup_robust_address target: "./itests/lookup_robust_address_test.go" - test: name: test-itest-mempool requires: - build + - postgres suite: itest-mempool target: "./itests/mempool_test.go" - test: name: test-itest-migration requires: - build + - postgres suite: itest-migration target: "./itests/migration_test.go" - test: name: test-itest-mpool_msg_uuid requires: - build + - postgres suite: itest-mpool_msg_uuid target: "./itests/mpool_msg_uuid_test.go" - test: name: test-itest-mpool_push_with_uuid requires: - build + - postgres suite: itest-mpool_push_with_uuid target: "./itests/mpool_push_with_uuid_test.go" - test: name: test-itest-msgindex requires: - build + - postgres suite: itest-msgindex target: "./itests/msgindex_test.go" - test: name: test-itest-multisig requires: - build + - postgres suite: itest-multisig target: "./itests/multisig_test.go" - test: name: test-itest-net requires: - build + - postgres suite: itest-net target: "./itests/net_test.go" - test: name: test-itest-nonce requires: - build + - postgres suite: itest-nonce target: "./itests/nonce_test.go" - test: name: test-itest-path_detach_redeclare requires: - build + - postgres suite: itest-path_detach_redeclare target: "./itests/path_detach_redeclare_test.go" - test: name: test-itest-path_type_filters requires: - build + - postgres suite: itest-path_type_filters target: "./itests/path_type_filters_test.go" - test: name: test-itest-paych_api requires: - build + - postgres suite: itest-paych_api target: "./itests/paych_api_test.go" - test: name: test-itest-paych_cli requires: - build + - postgres suite: itest-paych_cli target: "./itests/paych_cli_test.go" - test: name: test-itest-pending_deal_allocation requires: - build + - postgres suite: itest-pending_deal_allocation target: "./itests/pending_deal_allocation_test.go" - test: name: test-itest-raft_messagesigner requires: - build + - postgres suite: itest-raft_messagesigner target: "./itests/raft_messagesigner_test.go" - test: name: test-itest-remove_verifreg_datacap requires: - build + - postgres suite: itest-remove_verifreg_datacap target: "./itests/remove_verifreg_datacap_test.go" - test: name: test-itest-sdr_upgrade requires: - build + - postgres suite: itest-sdr_upgrade target: "./itests/sdr_upgrade_test.go" - test: name: test-itest-sealing_resources requires: - build + - postgres suite: itest-sealing_resources target: "./itests/sealing_resources_test.go" - test: name: test-itest-sector_finalize_early requires: - build + - postgres suite: itest-sector_finalize_early target: "./itests/sector_finalize_early_test.go" - test: name: test-itest-sector_import_full requires: - build + - postgres suite: itest-sector_import_full target: "./itests/sector_import_full_test.go" - test: name: test-itest-sector_import_simple requires: - build + - postgres suite: itest-sector_import_simple target: "./itests/sector_import_simple_test.go" - test: name: test-itest-sector_make_cc_avail requires: - build + - postgres suite: itest-sector_make_cc_avail target: "./itests/sector_make_cc_avail_test.go" - test: name: test-itest-sector_miner_collateral requires: - build + - postgres suite: itest-sector_miner_collateral target: "./itests/sector_miner_collateral_test.go" - test: name: test-itest-sector_numassign requires: - build + - postgres suite: itest-sector_numassign target: "./itests/sector_numassign_test.go" - test: name: test-itest-sector_pledge requires: - build + - postgres suite: itest-sector_pledge target: "./itests/sector_pledge_test.go" - test: name: test-itest-sector_prefer_no_upgrade requires: - build + - postgres suite: itest-sector_prefer_no_upgrade target: "./itests/sector_prefer_no_upgrade_test.go" - test: name: test-itest-sector_revert_available requires: - build + - postgres suite: itest-sector_revert_available target: "./itests/sector_revert_available_test.go" - test: name: test-itest-sector_terminate requires: - build + - postgres suite: itest-sector_terminate target: "./itests/sector_terminate_test.go" - test: name: test-itest-sector_unseal requires: - build + - postgres suite: itest-sector_unseal target: "./itests/sector_unseal_test.go" - test: name: test-itest-self_sent_txn requires: - build + - postgres suite: itest-self_sent_txn target: "./itests/self_sent_txn_test.go" - test: name: test-itest-splitstore requires: - build + - postgres suite: itest-splitstore target: "./itests/splitstore_test.go" - test: name: test-itest-tape requires: - build + - postgres suite: itest-tape target: "./itests/tape_test.go" - test: name: test-itest-verifreg requires: - build + - postgres suite: itest-verifreg target: "./itests/verifreg_test.go" - test: name: test-itest-wdpost_config requires: - build + - postgres suite: itest-wdpost_config target: "./itests/wdpost_config_test.go" - test: name: test-itest-wdpost_dispute requires: - build + - postgres suite: itest-wdpost_dispute target: "./itests/wdpost_dispute_test.go" - test: name: test-itest-wdpost_no_miner_storage requires: - build + - postgres suite: itest-wdpost_no_miner_storage target: "./itests/wdpost_no_miner_storage_test.go" - test: name: test-itest-wdpost requires: - build + - postgres suite: itest-wdpost target: "./itests/wdpost_test.go" get-params: true @@ -1014,6 +1104,7 @@ workflows: name: test-itest-wdpost_worker_config requires: - build + - postgres suite: itest-wdpost_worker_config target: "./itests/wdpost_worker_config_test.go" executor: golang-2xl @@ -1021,6 +1112,7 @@ workflows: name: test-itest-worker requires: - build + - postgres suite: itest-worker target: "./itests/worker_test.go" executor: golang-2xl @@ -1028,6 +1120,7 @@ workflows: name: test-itest-worker_upgrade requires: - build + - postgres suite: itest-worker_upgrade target: "./itests/worker_upgrade_test.go" - test: diff --git a/.circleci/template.yml b/.circleci/template.yml index cd8aeb663c9..96210b7efeb 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -165,6 +165,14 @@ jobs: default: unit description: Test suite name to report to CircleCI. executor: << parameters.executor >> + docker: + - image: circleci/postgres:12 + environment: + PGHOST: localhost + PGUSER: yugabyte + POSTGRES_USER: yugabyte + POSTGRES_DB: yugabyte + POSTGRES_HOST_AUTH_METHOD: trust steps: - install-ubuntu-deps - attach_workspace: @@ -173,6 +181,9 @@ jobs: condition: << parameters.get-params >> steps: - download-params + - run: + name: Wait for db to run + command: dockerize -wait tcp://localhost:5433 -timeout 1m - run: name: go test environment: @@ -540,6 +551,7 @@ workflows: name: test-itest-[[ $name ]] requires: - build + - postgres suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]] From 6e0683fa13e2bf6683f7ac4d321dca3d31652746 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 16:10:09 -0500 Subject: [PATCH 07/36] circleci try2 --- .circleci/template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index 96210b7efeb..ec5e359b6f3 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -551,7 +551,6 @@ workflows: name: test-itest-[[ $name ]] requires: - build - - postgres suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]] From 1b491d7c4ae00a6578e93b8f417b8f9dc8dfbf04 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 16:13:36 -0500 Subject: [PATCH 08/36] circleci try 3 --- .circleci/config.yml | 82 -------------------------------------------- 1 file changed, 82 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 049855bb051..8d0b61b8748 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -548,63 +548,54 @@ workflows: name: test-itest-api requires: - build - - postgres suite: itest-api target: "./itests/api_test.go" - test: name: test-itest-batch_deal requires: - build - - postgres suite: itest-batch_deal target: "./itests/batch_deal_test.go" - test: name: test-itest-ccupgrade requires: - build - - postgres suite: itest-ccupgrade target: "./itests/ccupgrade_test.go" - test: name: test-itest-cli requires: - build - - postgres suite: itest-cli target: "./itests/cli_test.go" - test: name: test-itest-clusterdb requires: - build - - postgres suite: itest-clusterdb target: "./itests/clusterdb_test.go" - test: name: test-itest-deadlines requires: - build - - postgres suite: itest-deadlines target: "./itests/deadlines_test.go" - test: name: test-itest-deals_512mb requires: - build - - postgres suite: itest-deals_512mb target: "./itests/deals_512mb_test.go" - test: name: test-itest-deals_anycid requires: - build - - postgres suite: itest-deals_anycid target: "./itests/deals_anycid_test.go" - test: name: test-itest-deals_concurrent requires: - build - - postgres suite: itest-deals_concurrent target: "./itests/deals_concurrent_test.go" executor: golang-2xl @@ -612,490 +603,420 @@ workflows: name: test-itest-deals_invalid_utf8_label requires: - build - - postgres suite: itest-deals_invalid_utf8_label target: "./itests/deals_invalid_utf8_label_test.go" - test: name: test-itest-deals_max_staging_deals requires: - build - - postgres suite: itest-deals_max_staging_deals target: "./itests/deals_max_staging_deals_test.go" - test: name: test-itest-deals_offline requires: - build - - postgres suite: itest-deals_offline target: "./itests/deals_offline_test.go" - test: name: test-itest-deals_padding requires: - build - - postgres suite: itest-deals_padding target: "./itests/deals_padding_test.go" - test: name: test-itest-deals_partial_retrieval_dm-level requires: - build - - postgres suite: itest-deals_partial_retrieval_dm-level target: "./itests/deals_partial_retrieval_dm-level_test.go" - test: name: test-itest-deals_partial_retrieval requires: - build - - postgres suite: itest-deals_partial_retrieval target: "./itests/deals_partial_retrieval_test.go" - test: name: test-itest-deals_power requires: - build - - postgres suite: itest-deals_power target: "./itests/deals_power_test.go" - test: name: test-itest-deals_pricing requires: - build - - postgres suite: itest-deals_pricing target: "./itests/deals_pricing_test.go" - test: name: test-itest-deals_publish requires: - build - - postgres suite: itest-deals_publish target: "./itests/deals_publish_test.go" - test: name: test-itest-deals_remote_retrieval requires: - build - - postgres suite: itest-deals_remote_retrieval target: "./itests/deals_remote_retrieval_test.go" - test: name: test-itest-deals_retry_deal_no_funds requires: - build - - postgres suite: itest-deals_retry_deal_no_funds target: "./itests/deals_retry_deal_no_funds_test.go" - test: name: test-itest-deals requires: - build - - postgres suite: itest-deals target: "./itests/deals_test.go" - test: name: test-itest-decode_params requires: - build - - postgres suite: itest-decode_params target: "./itests/decode_params_test.go" - test: name: test-itest-dup_mpool_messages requires: - build - - postgres suite: itest-dup_mpool_messages target: "./itests/dup_mpool_messages_test.go" - test: name: test-itest-eth_account_abstraction requires: - build - - postgres suite: itest-eth_account_abstraction target: "./itests/eth_account_abstraction_test.go" - test: name: test-itest-eth_api requires: - build - - postgres suite: itest-eth_api target: "./itests/eth_api_test.go" - test: name: test-itest-eth_balance requires: - build - - postgres suite: itest-eth_balance target: "./itests/eth_balance_test.go" - test: name: test-itest-eth_block_hash requires: - build - - postgres suite: itest-eth_block_hash target: "./itests/eth_block_hash_test.go" - test: name: test-itest-eth_bytecode requires: - build - - postgres suite: itest-eth_bytecode target: "./itests/eth_bytecode_test.go" - test: name: test-itest-eth_config requires: - build - - postgres suite: itest-eth_config target: "./itests/eth_config_test.go" - test: name: test-itest-eth_conformance requires: - build - - postgres suite: itest-eth_conformance target: "./itests/eth_conformance_test.go" - test: name: test-itest-eth_deploy requires: - build - - postgres suite: itest-eth_deploy target: "./itests/eth_deploy_test.go" - test: name: test-itest-eth_fee_history requires: - build - - postgres suite: itest-eth_fee_history target: "./itests/eth_fee_history_test.go" - test: name: test-itest-eth_filter requires: - build - - postgres suite: itest-eth_filter target: "./itests/eth_filter_test.go" - test: name: test-itest-eth_hash_lookup requires: - build - - postgres suite: itest-eth_hash_lookup target: "./itests/eth_hash_lookup_test.go" - test: name: test-itest-eth_transactions requires: - build - - postgres suite: itest-eth_transactions target: "./itests/eth_transactions_test.go" - test: name: test-itest-fevm_address requires: - build - - postgres suite: itest-fevm_address target: "./itests/fevm_address_test.go" - test: name: test-itest-fevm_events requires: - build - - postgres suite: itest-fevm_events target: "./itests/fevm_events_test.go" - test: name: test-itest-fevm requires: - build - - postgres suite: itest-fevm target: "./itests/fevm_test.go" - test: name: test-itest-gas_estimation requires: - build - - postgres suite: itest-gas_estimation target: "./itests/gas_estimation_test.go" - test: name: test-itest-gateway requires: - build - - postgres suite: itest-gateway target: "./itests/gateway_test.go" - test: name: test-itest-get_messages_in_ts requires: - build - - postgres suite: itest-get_messages_in_ts target: "./itests/get_messages_in_ts_test.go" - test: name: test-itest-lite_migration requires: - build - - postgres suite: itest-lite_migration target: "./itests/lite_migration_test.go" - test: name: test-itest-lookup_robust_address requires: - build - - postgres suite: itest-lookup_robust_address target: "./itests/lookup_robust_address_test.go" - test: name: test-itest-mempool requires: - build - - postgres suite: itest-mempool target: "./itests/mempool_test.go" - test: name: test-itest-migration requires: - build - - postgres suite: itest-migration target: "./itests/migration_test.go" - test: name: test-itest-mpool_msg_uuid requires: - build - - postgres suite: itest-mpool_msg_uuid target: "./itests/mpool_msg_uuid_test.go" - test: name: test-itest-mpool_push_with_uuid requires: - build - - postgres suite: itest-mpool_push_with_uuid target: "./itests/mpool_push_with_uuid_test.go" - test: name: test-itest-msgindex requires: - build - - postgres suite: itest-msgindex target: "./itests/msgindex_test.go" - test: name: test-itest-multisig requires: - build - - postgres suite: itest-multisig target: "./itests/multisig_test.go" - test: name: test-itest-net requires: - build - - postgres suite: itest-net target: "./itests/net_test.go" - test: name: test-itest-nonce requires: - build - - postgres suite: itest-nonce target: "./itests/nonce_test.go" - test: name: test-itest-path_detach_redeclare requires: - build - - postgres suite: itest-path_detach_redeclare target: "./itests/path_detach_redeclare_test.go" - test: name: test-itest-path_type_filters requires: - build - - postgres suite: itest-path_type_filters target: "./itests/path_type_filters_test.go" - test: name: test-itest-paych_api requires: - build - - postgres suite: itest-paych_api target: "./itests/paych_api_test.go" - test: name: test-itest-paych_cli requires: - build - - postgres suite: itest-paych_cli target: "./itests/paych_cli_test.go" - test: name: test-itest-pending_deal_allocation requires: - build - - postgres suite: itest-pending_deal_allocation target: "./itests/pending_deal_allocation_test.go" - test: name: test-itest-raft_messagesigner requires: - build - - postgres suite: itest-raft_messagesigner target: "./itests/raft_messagesigner_test.go" - test: name: test-itest-remove_verifreg_datacap requires: - build - - postgres suite: itest-remove_verifreg_datacap target: "./itests/remove_verifreg_datacap_test.go" - test: name: test-itest-sdr_upgrade requires: - build - - postgres suite: itest-sdr_upgrade target: "./itests/sdr_upgrade_test.go" - test: name: test-itest-sealing_resources requires: - build - - postgres suite: itest-sealing_resources target: "./itests/sealing_resources_test.go" - test: name: test-itest-sector_finalize_early requires: - build - - postgres suite: itest-sector_finalize_early target: "./itests/sector_finalize_early_test.go" - test: name: test-itest-sector_import_full requires: - build - - postgres suite: itest-sector_import_full target: "./itests/sector_import_full_test.go" - test: name: test-itest-sector_import_simple requires: - build - - postgres suite: itest-sector_import_simple target: "./itests/sector_import_simple_test.go" - test: name: test-itest-sector_make_cc_avail requires: - build - - postgres suite: itest-sector_make_cc_avail target: "./itests/sector_make_cc_avail_test.go" - test: name: test-itest-sector_miner_collateral requires: - build - - postgres suite: itest-sector_miner_collateral target: "./itests/sector_miner_collateral_test.go" - test: name: test-itest-sector_numassign requires: - build - - postgres suite: itest-sector_numassign target: "./itests/sector_numassign_test.go" - test: name: test-itest-sector_pledge requires: - build - - postgres suite: itest-sector_pledge target: "./itests/sector_pledge_test.go" - test: name: test-itest-sector_prefer_no_upgrade requires: - build - - postgres suite: itest-sector_prefer_no_upgrade target: "./itests/sector_prefer_no_upgrade_test.go" - test: name: test-itest-sector_revert_available requires: - build - - postgres suite: itest-sector_revert_available target: "./itests/sector_revert_available_test.go" - test: name: test-itest-sector_terminate requires: - build - - postgres suite: itest-sector_terminate target: "./itests/sector_terminate_test.go" - test: name: test-itest-sector_unseal requires: - build - - postgres suite: itest-sector_unseal target: "./itests/sector_unseal_test.go" - test: name: test-itest-self_sent_txn requires: - build - - postgres suite: itest-self_sent_txn target: "./itests/self_sent_txn_test.go" - test: name: test-itest-splitstore requires: - build - - postgres suite: itest-splitstore target: "./itests/splitstore_test.go" - test: name: test-itest-tape requires: - build - - postgres suite: itest-tape target: "./itests/tape_test.go" - test: name: test-itest-verifreg requires: - build - - postgres suite: itest-verifreg target: "./itests/verifreg_test.go" - test: name: test-itest-wdpost_config requires: - build - - postgres suite: itest-wdpost_config target: "./itests/wdpost_config_test.go" - test: name: test-itest-wdpost_dispute requires: - build - - postgres suite: itest-wdpost_dispute target: "./itests/wdpost_dispute_test.go" - test: name: test-itest-wdpost_no_miner_storage requires: - build - - postgres suite: itest-wdpost_no_miner_storage target: "./itests/wdpost_no_miner_storage_test.go" - test: name: test-itest-wdpost requires: - build - - postgres suite: itest-wdpost target: "./itests/wdpost_test.go" get-params: true @@ -1104,7 +1025,6 @@ workflows: name: test-itest-wdpost_worker_config requires: - build - - postgres suite: itest-wdpost_worker_config target: "./itests/wdpost_worker_config_test.go" executor: golang-2xl @@ -1112,7 +1032,6 @@ workflows: name: test-itest-worker requires: - build - - postgres suite: itest-worker target: "./itests/worker_test.go" executor: golang-2xl @@ -1120,7 +1039,6 @@ workflows: name: test-itest-worker_upgrade requires: - build - - postgres suite: itest-worker_upgrade target: "./itests/worker_upgrade_test.go" - test: From c6bc59767692f956f91247b662a42aed709df5c7 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 16:21:14 -0500 Subject: [PATCH 09/36] circleci try 4 --- .circleci/config.yml | 1 + .circleci/template.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d0b61b8748..cf3c00fa2fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,6 +166,7 @@ jobs: description: Test suite name to report to CircleCI. executor: << parameters.executor >> docker: + - image: cimg/base - image: circleci/postgres:12 environment: PGHOST: localhost diff --git a/.circleci/template.yml b/.circleci/template.yml index ec5e359b6f3..379f347da0c 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -166,6 +166,7 @@ jobs: description: Test suite name to report to CircleCI. executor: << parameters.executor >> docker: + - image: cimg/base - image: circleci/postgres:12 environment: PGHOST: localhost From e156ad92e032860ba4860782f8b3018c5ad027f2 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 16:28:30 -0500 Subject: [PATCH 10/36] circleci try5 --- .circleci/config.yml | 2 +- .circleci/template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf3c00fa2fc..12d5dd426a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,7 +166,7 @@ jobs: description: Test suite name to report to CircleCI. executor: << parameters.executor >> docker: - - image: cimg/base + - image: ubuntu:20.04 - image: circleci/postgres:12 environment: PGHOST: localhost diff --git a/.circleci/template.yml b/.circleci/template.yml index 379f347da0c..0fe505120a7 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -166,7 +166,7 @@ jobs: description: Test suite name to report to CircleCI. executor: << parameters.executor >> docker: - - image: cimg/base + - image: ubuntu:20.04 - image: circleci/postgres:12 environment: PGHOST: localhost From 3675277b27df5d1734e624f9eb818826ad5cc6ab Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 12 Jul 2023 17:02:36 -0500 Subject: [PATCH 11/36] circleci try6 --- .circleci/config.yml | 17 +++++++---------- .circleci/template.yml | 17 +++++++---------- go.mod | 3 --- go.sum | 3 --- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12d5dd426a4..24edaf673f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,8 +96,14 @@ commands: git fetch --all install-ubuntu-deps: steps: + - run: sudo apt install curl ca-certificates gnupg + - run: curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev postgresql-15 postgresql-contrib-15 + - run: sudo "port = 5433" >>/etc/postgresql/15.0/main/postgresql.conf + - run: sudo systemctl start postgresql.service + - run: sudo -u postgres createuser yugabyte PASSWORD yugabyte + - run: sudo -u postgres createdb yugabyte check-go-version: steps: - run: | @@ -165,15 +171,6 @@ jobs: default: unit description: Test suite name to report to CircleCI. executor: << parameters.executor >> - docker: - - image: ubuntu:20.04 - - image: circleci/postgres:12 - environment: - PGHOST: localhost - PGUSER: yugabyte - POSTGRES_USER: yugabyte - POSTGRES_DB: yugabyte - POSTGRES_HOST_AUTH_METHOD: trust steps: - install-ubuntu-deps - attach_workspace: diff --git a/.circleci/template.yml b/.circleci/template.yml index 0fe505120a7..c811fee3a34 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -96,8 +96,14 @@ commands: git fetch --all install-ubuntu-deps: steps: + - run: sudo apt install curl ca-certificates gnupg + - run: curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev postgresql-15 postgresql-contrib-15 + - run: sudo "port = 5433" >>/etc/postgresql/15.0/main/postgresql.conf + - run: sudo systemctl start postgresql.service + - run: sudo -u postgres createuser yugabyte PASSWORD yugabyte + - run: sudo -u postgres createdb yugabyte check-go-version: steps: - run: | @@ -165,15 +171,6 @@ jobs: default: unit description: Test suite name to report to CircleCI. executor: << parameters.executor >> - docker: - - image: ubuntu:20.04 - - image: circleci/postgres:12 - environment: - PGHOST: localhost - PGUSER: yugabyte - POSTGRES_USER: yugabyte - POSTGRES_DB: yugabyte - POSTGRES_HOST_AUTH_METHOD: trust steps: - install-ubuntu-deps - attach_workspace: diff --git a/go.mod b/go.mod index d3105cb831c..2da784ad601 100644 --- a/go.mod +++ b/go.mod @@ -106,7 +106,6 @@ require ( github.com/jackc/pgx/v5 v5.4.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/koalacxr/quantile v0.0.1 - github.com/kr/pretty v0.3.1 github.com/libp2p/go-buffer-pool v0.1.0 github.com/libp2p/go-libp2p v0.27.6 github.com/libp2p/go-libp2p-consensus v0.0.1 @@ -261,7 +260,6 @@ require ( github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/koron/go-ssdp v0.0.4 // indirect - github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect @@ -306,7 +304,6 @@ require ( github.com/quic-go/quic-go v0.33.0 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/rivo/uniseg v0.1.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/cors v1.7.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v2.18.12+incompatible // indirect diff --git a/go.sum b/go.sum index ab18517ee0b..ebbc4dcc8a2 100644 --- a/go.sum +++ b/go.sum @@ -962,7 +962,6 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= @@ -1412,7 +1411,6 @@ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1504,7 +1502,6 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= From 633b65d392bb7648b103ab9ee63ab56ce6c97f81 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 12:19:32 -0500 Subject: [PATCH 12/36] circleci try 7 --- .circleci/template.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index c811fee3a34..6b28f17f3c8 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -97,13 +97,12 @@ commands: install-ubuntu-deps: steps: - run: sudo apt install curl ca-certificates gnupg - - run: curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev postgresql-15 postgresql-contrib-15 - - run: sudo "port = 5433" >>/etc/postgresql/15.0/main/postgresql.conf - - run: sudo systemctl start postgresql.service - - run: sudo -u postgres createuser yugabyte PASSWORD yugabyte - - run: sudo -u postgres createdb yugabyte + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev python-is-python3 + intall-yugabyte: + steps: + - run: curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz + - run: yugabytedb/bin/yugabyted start check-go-version: steps: - run: | @@ -173,15 +172,13 @@ jobs: executor: << parameters.executor >> steps: - install-ubuntu-deps + - install-yugabyte - attach_workspace: at: ~/ - when: condition: << parameters.get-params >> steps: - download-params - - run: - name: Wait for db to run - command: dockerize -wait tcp://localhost:5433 -timeout 1m - run: name: go test environment: From 0811965046c858fcfb396b0bfb3857ca334eb5fe Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 12:49:10 -0500 Subject: [PATCH 13/36] circleci try 8 --- .circleci/config.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 24edaf673f9..4ccb89b6c17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,13 +97,12 @@ commands: install-ubuntu-deps: steps: - run: sudo apt install curl ca-certificates gnupg - - run: curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev postgresql-15 postgresql-contrib-15 - - run: sudo "port = 5433" >>/etc/postgresql/15.0/main/postgresql.conf - - run: sudo systemctl start postgresql.service - - run: sudo -u postgres createuser yugabyte PASSWORD yugabyte - - run: sudo -u postgres createdb yugabyte + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev python-is-python3 + intall-yugabyte: + steps: + - run: curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz + - run: yugabytedb/bin/yugabyted start check-go-version: steps: - run: | @@ -173,15 +172,13 @@ jobs: executor: << parameters.executor >> steps: - install-ubuntu-deps + - install-yugabyte - attach_workspace: at: ~/ - when: condition: << parameters.get-params >> steps: - download-params - - run: - name: Wait for db to run - command: dockerize -wait tcp://localhost:5433 -timeout 1m - run: name: go test environment: From 3705989c2990378b681d1c76c255e0fbb8ab4258 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 12:53:59 -0500 Subject: [PATCH 14/36] circleci try 9 --- .circleci/template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index 6b28f17f3c8..7c3de46ddcd 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -99,7 +99,7 @@ commands: - run: sudo apt install curl ca-certificates gnupg - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev python-is-python3 - intall-yugabyte: + install-yugabyte: steps: - run: curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz - run: yugabytedb/bin/yugabyted start From b7a491c10a4e423ebdfc02392466e4284e67325f Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 12:54:29 -0500 Subject: [PATCH 15/36] circleci try 10 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ccb89b6c17..69176bf2d6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,7 +99,7 @@ commands: - run: sudo apt install curl ca-certificates gnupg - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev python-is-python3 - intall-yugabyte: + install-yugabyte: steps: - run: curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz - run: yugabytedb/bin/yugabyted start From 3cf920c750b5b4dff01205b8a3b9d92ce31481d8 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 21:33:56 -0500 Subject: [PATCH 16/36] circleci try 10 --- .circleci/config.yml | 6 ++++-- .circleci/template.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 69176bf2d6f..44635fe6b33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,8 +101,10 @@ commands: - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev python-is-python3 install-yugabyte: steps: - - run: curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz - - run: yugabytedb/bin/yugabyted start + - run: | + curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz + yugabyte-2.19.0.0/bin/post_install.sh + yugabyte-2.19.0.0/bin/yugabyted start check-go-version: steps: - run: | diff --git a/.circleci/template.yml b/.circleci/template.yml index 7c3de46ddcd..f46eb1286e3 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -101,8 +101,10 @@ commands: - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev python-is-python3 install-yugabyte: steps: - - run: curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz - - run: yugabytedb/bin/yugabyted start + - run: | + curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz + yugabyte-2.19.0.0/bin/post_install.sh + yugabyte-2.19.0.0/bin/yugabyted start check-go-version: steps: - run: | From fb30289dad74384d71c376b0798f9061525da011 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 22:08:07 -0500 Subject: [PATCH 17/36] Updated config.yml --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 44635fe6b33..5d14f78ac66 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,6 +17,9 @@ executors: ubuntu: docker: - image: ubuntu:20.04 + yugabyte: + docker: + - image: yugabytedb/yugabyte:latest commands: build-platform-specific: From 9acd6c7e339eebeda487d965272c5ac0f6a24082 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 22:49:44 -0500 Subject: [PATCH 18/36] circleci try 11 --- .circleci/config.yml | 53 ++++++++++++++++++------------------------ .circleci/gen.go | 4 ++++ .circleci/template.yml | 51 ++++++++++++++++++---------------------- 3 files changed, 49 insertions(+), 59 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 44635fe6b33..0b63f0da7a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,17 +6,9 @@ orbs: executors: golang: docker: - # Must match GO_VERSION_MIN in project root + # Must match GO_VERSION_MIN in project root. Change in gen.go - image: cimg/go:1.19.7 resource_class: medium+ - golang-2xl: - docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.19.7 - resource_class: 2xlarge - ubuntu: - docker: - - image: ubuntu:20.04 commands: build-platform-specific: @@ -98,13 +90,7 @@ commands: steps: - run: sudo apt install curl ca-certificates gnupg - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev python-is-python3 - install-yugabyte: - steps: - - run: | - curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz - yugabyte-2.19.0.0/bin/post_install.sh - yugabyte-2.19.0.0/bin/yugabyted start + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev check-go-version: steps: - run: | @@ -150,9 +136,9 @@ jobs: Run tests with gotestsum. working_directory: ~/lotus parameters: &test-params - executor: - type: executor - default: golang + resource_class: + type: resource_class + default: medium+ go-test-flags: type: string default: "-timeout 20m" @@ -171,10 +157,12 @@ jobs: type: string default: unit description: Test suite name to report to CircleCI. - executor: << parameters.executor >> + docker: + - image: cimg/go:1.19.7 + resource_class: << parameters.resource_class >> + - image: yugabytedb/yugabyte:latest steps: - install-ubuntu-deps - - install-yugabyte - attach_workspace: at: ~/ - when: @@ -188,6 +176,7 @@ jobs: SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | + dockerize -wait tcp://localhost:5433 -timeout 1m mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts gotestsum \ @@ -217,7 +206,9 @@ jobs: Branch on github.com/filecoin-project/test-vectors to checkout and test with. If empty (the default) the commit defined by the git submodule is used. - executor: << parameters.executor >> + docker: + - image: cimg/go:1.19.7 + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -404,15 +395,17 @@ jobs: Run golangci-lint. working_directory: ~/lotus parameters: - executor: - type: executor - default: golang + resource_class: + type: resource_class + default: medium+ args: type: string default: '' description: | Arguments to pass to golangci-lint - executor: << parameters.executor >> + docker: + - image: cimg/go:1.19.7 + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -595,7 +588,7 @@ workflows: - build suite: itest-deals_concurrent target: "./itests/deals_concurrent_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-deals_invalid_utf8_label requires: @@ -1024,14 +1017,14 @@ workflows: - build suite: itest-wdpost_worker_config target: "./itests/wdpost_worker_config_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-worker requires: - build suite: itest-worker target: "./itests/worker_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-worker_upgrade requires: @@ -1058,7 +1051,7 @@ workflows: - build suite: utest-unit-rest target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..." - executor: golang-2xl + resource_class: 2xlarge - test: name: test-unit-storage requires: diff --git a/.circleci/gen.go b/.circleci/gen.go index 5d951027a0c..d85b15d90fa 100644 --- a/.circleci/gen.go +++ b/.circleci/gen.go @@ -10,6 +10,8 @@ import ( "text/template" ) +const GoVersion = "1.19.7" + //go:generate go run ./gen.go .. //go:embed template.yml @@ -109,6 +111,7 @@ func main() { Networks []string ItestFiles []string UnitSuites map[string]string + GoVersion string } in := data{ Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"}, @@ -123,6 +126,7 @@ func main() { } return ret }(), + GoVersion: GoVersion, } out, err := os.Create("./config.yml") diff --git a/.circleci/template.yml b/.circleci/template.yml index f46eb1286e3..3acbc687545 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -6,17 +6,9 @@ orbs: executors: golang: docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.19.7 + # Must match GO_VERSION_MIN in project root. Change in gen.go + - image: cimg/go:[[ .GoVersion]] resource_class: medium+ - golang-2xl: - docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.19.7 - resource_class: 2xlarge - ubuntu: - docker: - - image: ubuntu:20.04 commands: build-platform-specific: @@ -98,13 +90,7 @@ commands: steps: - run: sudo apt install curl ca-certificates gnupg - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev python-is-python3 - install-yugabyte: - steps: - - run: | - curl https://downloads.yugabyte.com/releases/2.19.0.0/yugabyte-2.19.0.0-b190-linux-x86_64.tar.gz | tar -xz - yugabyte-2.19.0.0/bin/post_install.sh - yugabyte-2.19.0.0/bin/yugabyted start + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev check-go-version: steps: - run: | @@ -150,9 +136,9 @@ jobs: Run tests with gotestsum. working_directory: ~/lotus parameters: &test-params - executor: - type: executor - default: golang + resource_class: + type: resource_class + default: medium+ go-test-flags: type: string default: "-timeout 20m" @@ -171,10 +157,12 @@ jobs: type: string default: unit description: Test suite name to report to CircleCI. - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + resource_class: << parameters.resource_class >> + - image: yugabytedb/yugabyte:latest steps: - install-ubuntu-deps - - install-yugabyte - attach_workspace: at: ~/ - when: @@ -188,6 +176,7 @@ jobs: SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | + dockerize -wait tcp://localhost:5433 -timeout 1m mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts gotestsum \ @@ -217,7 +206,9 @@ jobs: Branch on github.com/filecoin-project/test-vectors to checkout and test with. If empty (the default) the commit defined by the git submodule is used. - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -404,15 +395,17 @@ jobs: Run golangci-lint. working_directory: ~/lotus parameters: - executor: - type: executor - default: golang + resource_class: + type: resource_class + default: medium+ args: type: string default: '' description: | Arguments to pass to golangci-lint - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -551,7 +544,7 @@ workflows: suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]] - executor: golang-2xl + resource_class: 2xlarge [[- end]] [[- if (eq $name "wdpost")]] get-params: true @@ -566,7 +559,7 @@ workflows: suite: utest-[[ $suite ]] target: "[[ $pkgs ]]" [[if eq $suite "unit-cli"]]get-params: true[[end]] - [[- if eq $suite "unit-rest"]]executor: golang-2xl[[end]] + [[- if eq $suite "unit-rest"]]resource_class: 2xlarge[[end]] [[- end]] - test: go-test-flags: "-run=TestMulticoreSDR" From b393e058c5c97042e65069d3c2f4474470f25879 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 22:57:36 -0500 Subject: [PATCH 19/36] circleci try 12 --- .circleci/config.yml | 5 +---- .circleci/template.yml | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b63f0da7a3..3a81a49b1b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -395,9 +395,6 @@ jobs: Run golangci-lint. working_directory: ~/lotus parameters: - resource_class: - type: resource_class - default: medium+ args: type: string default: '' @@ -405,7 +402,7 @@ jobs: Arguments to pass to golangci-lint docker: - image: cimg/go:1.19.7 - resource_class: << parameters.resource_class >> + resource_class: medium+ steps: - install-ubuntu-deps - attach_workspace: diff --git a/.circleci/template.yml b/.circleci/template.yml index 3acbc687545..3c87b9eaf21 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -395,9 +395,6 @@ jobs: Run golangci-lint. working_directory: ~/lotus parameters: - resource_class: - type: resource_class - default: medium+ args: type: string default: '' @@ -405,7 +402,7 @@ jobs: Arguments to pass to golangci-lint docker: - image: cimg/go:[[ .GoVersion]] - resource_class: << parameters.resource_class >> + resource_class: medium+ steps: - install-ubuntu-deps - attach_workspace: From 32f884e39b9164d2ed54d3fa4a7b1e9f987b3981 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 23:10:45 -0500 Subject: [PATCH 20/36] circleci try 13 --- .circleci/config.yml | 6 +++--- .circleci/template.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a81a49b1b3..0ecfc00848f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,8 +159,8 @@ jobs: description: Test suite name to report to CircleCI. docker: - image: cimg/go:1.19.7 - resource_class: << parameters.resource_class >> - image: yugabytedb/yugabyte:latest + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -208,7 +208,7 @@ jobs: submodule is used. docker: - image: cimg/go:1.19.7 - resource_class: << parameters.resource_class >> + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -402,7 +402,7 @@ jobs: Arguments to pass to golangci-lint docker: - image: cimg/go:1.19.7 - resource_class: medium+ + resource_class: medium+ steps: - install-ubuntu-deps - attach_workspace: diff --git a/.circleci/template.yml b/.circleci/template.yml index 3c87b9eaf21..26058ce85a4 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -159,8 +159,8 @@ jobs: description: Test suite name to report to CircleCI. docker: - image: cimg/go:[[ .GoVersion]] - resource_class: << parameters.resource_class >> - image: yugabytedb/yugabyte:latest + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -208,7 +208,7 @@ jobs: submodule is used. docker: - image: cimg/go:[[ .GoVersion]] - resource_class: << parameters.resource_class >> + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -402,7 +402,7 @@ jobs: Arguments to pass to golangci-lint docker: - image: cimg/go:[[ .GoVersion]] - resource_class: medium+ + resource_class: medium+ steps: - install-ubuntu-deps - attach_workspace: From cf28802612134af250a5a14849b3664782f5904d Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 23:12:44 -0500 Subject: [PATCH 21/36] circleci try 14 --- .circleci/config.yml | 2 +- .circleci/template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ecfc00848f..9b570872e7f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,7 +137,7 @@ jobs: working_directory: ~/lotus parameters: &test-params resource_class: - type: resource_class + type: string default: medium+ go-test-flags: type: string diff --git a/.circleci/template.yml b/.circleci/template.yml index 26058ce85a4..d51c45ded8c 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -137,7 +137,7 @@ jobs: working_directory: ~/lotus parameters: &test-params resource_class: - type: resource_class + type: string default: medium+ go-test-flags: type: string From 37318e70c86aba3099dfc637bc82b5506e1ef7eb Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 13 Jul 2023 23:29:16 -0500 Subject: [PATCH 22/36] circleci try 14 --- .circleci/config.yml | 1 + .circleci/template.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b570872e7f..c9a438f0d20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,6 +160,7 @@ jobs: docker: - image: cimg/go:1.19.7 - image: yugabytedb/yugabyte:latest + command: bin/yugabyted start resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps diff --git a/.circleci/template.yml b/.circleci/template.yml index d51c45ded8c..7d190676c98 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -160,6 +160,7 @@ jobs: docker: - image: cimg/go:[[ .GoVersion]] - image: yugabytedb/yugabyte:latest + command: bin/yugabyted start resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps From 2baf8e6dfd8055676ec5cf2f57655caf806dd3a9 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 09:35:21 -0500 Subject: [PATCH 23/36] circleci try 15 --- .circleci/config.yml | 2 +- .circleci/template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9a438f0d20..69962fabb17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,7 +160,7 @@ jobs: docker: - image: cimg/go:1.19.7 - image: yugabytedb/yugabyte:latest - command: bin/yugabyted start + command: bin/yugabyted start && bash resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps diff --git a/.circleci/template.yml b/.circleci/template.yml index 7d190676c98..427ba3d538d 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -160,7 +160,7 @@ jobs: docker: - image: cimg/go:[[ .GoVersion]] - image: yugabytedb/yugabyte:latest - command: bin/yugabyted start + command: bin/yugabyted start && bash resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps From a56a6c14c34b2f0e155fe3fc3a980122ff861d3b Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 09:46:03 -0500 Subject: [PATCH 24/36] circleci try 16 --- .circleci/config.yml | 2 +- .circleci/template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 69962fabb17..a6604d6ae6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,7 +160,7 @@ jobs: docker: - image: cimg/go:1.19.7 - image: yugabytedb/yugabyte:latest - command: bin/yugabyted start && bash + command: bin/yugabyted start --daemon=false resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps diff --git a/.circleci/template.yml b/.circleci/template.yml index 427ba3d538d..19a04383758 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -160,7 +160,7 @@ jobs: docker: - image: cimg/go:[[ .GoVersion]] - image: yugabytedb/yugabyte:latest - command: bin/yugabyted start && bash + command: bin/yugabyted start --daemon=false resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps From bde2dbdc142281978831002a9c889ccb36861506 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 10:55:07 -0500 Subject: [PATCH 25/36] circleci test --- .circleci/config.yml | 3 ++- .circleci/template.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6604d6ae6b..81714ae26da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -161,6 +161,7 @@ jobs: - image: cimg/go:1.19.7 - image: yugabytedb/yugabyte:latest command: bin/yugabyted start --daemon=false + name: yugabyte resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps @@ -177,7 +178,7 @@ jobs: SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | - dockerize -wait tcp://localhost:5433 -timeout 1m + dockerize -wait tcp://yugabyte:5433 -timeout 1m mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts gotestsum \ diff --git a/.circleci/template.yml b/.circleci/template.yml index 19a04383758..2b24cbf01ba 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -161,6 +161,7 @@ jobs: - image: cimg/go:[[ .GoVersion]] - image: yugabytedb/yugabyte:latest command: bin/yugabyted start --daemon=false + name: yugabyte resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps @@ -177,7 +178,7 @@ jobs: SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | - dockerize -wait tcp://localhost:5433 -timeout 1m + dockerize -wait tcp://yugabyte:5433 -timeout 1m mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts gotestsum \ From b9c615ace5605b83c9d9c6b8820f150c8cc2c93b Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 11:22:01 -0500 Subject: [PATCH 26/36] harmony codename and long itest timeout --- .circleci/config.yml | 14 ++-- .circleci/template.yml | 2 +- .../en/default-lotus-miner-config.toml | 14 ++-- .../{clusterdb_test.go => harmonydb_test.go} | 10 +-- itests/kit/ensemble.go | 10 +-- .../clusterdb => harmony/harmonydb}/doc.go | 4 +- .../harmonydb/harmonydb.go} | 10 +-- .../harmonydb}/metrics.go | 4 +- .../harmonydb}/sql/20230706.sql | 0 .../harmonydb}/userfuncs.go | 4 +- node/builder_miner.go | 6 +- node/config/def.go | 2 +- node/config/doc_gen.go | 84 +++++++++---------- node/config/types.go | 4 +- node/impl/storminer.go | 4 +- 15 files changed, 86 insertions(+), 86 deletions(-) rename itests/{clusterdb_test.go => harmonydb_test.go} (92%) rename lib/{sturdy/clusterdb => harmony/harmonydb}/doc.go (93%) rename lib/{sturdy/clusterdb/clusterdb.go => harmony/harmonydb/harmonydb.go} (96%) rename lib/{sturdy/clusterdb => harmony/harmonydb}/metrics.go (97%) rename lib/{sturdy/clusterdb => harmony/harmonydb}/sql/20230706.sql (100%) rename lib/{sturdy/clusterdb => harmony/harmonydb}/userfuncs.go (98%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 81714ae26da..8508d819626 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -178,9 +178,9 @@ jobs: SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | - dockerize -wait tcp://yugabyte:5433 -timeout 1m mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts + dockerize -wait tcp://yugabyte:5433 -timeout 3m gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ @@ -557,12 +557,6 @@ workflows: - build suite: itest-cli target: "./itests/cli_test.go" - - test: - name: test-itest-clusterdb - requires: - - build - suite: itest-clusterdb - target: "./itests/clusterdb_test.go" - test: name: test-itest-deadlines requires: @@ -780,6 +774,12 @@ workflows: - build suite: itest-get_messages_in_ts target: "./itests/get_messages_in_ts_test.go" + - test: + name: test-itest-harmonydb + requires: + - build + suite: itest-harmonydb + target: "./itests/harmonydb_test.go" - test: name: test-itest-lite_migration requires: diff --git a/.circleci/template.yml b/.circleci/template.yml index 2b24cbf01ba..8e340f70a9a 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -178,9 +178,9 @@ jobs: SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | - dockerize -wait tcp://yugabyte:5433 -timeout 1m mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts + dockerize -wait tcp://yugabyte:5433 -timeout 3m gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 1ff115b0746..0efb22ff003 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -894,43 +894,43 @@ #GCInterval = "1m0s" -[ClusterDB] +[HarmonyDB] # HOSTS is a list of hostnames to nodes running YugabyteDB # in a cluster. Only 1 is required # # type: []string - # env var: LOTUS_CLUSTERDB_HOSTS + # env var: LOTUS_HARMONYDB_HOSTS #Hosts = ["127.0.0.1"] # The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default. # # type: string - # env var: LOTUS_CLUSTERDB_USERNAME + # env var: LOTUS_HARMONYDB_USERNAME #Username = "yugabyte" # The password for the related username. Blank for default. # # type: string - # env var: LOTUS_CLUSTERDB_PASSWORD + # env var: LOTUS_HARMONYDB_PASSWORD #Password = "yugabyte" # The database (logical partition) within Yugabyte. Blank for default. # # type: string - # env var: LOTUS_CLUSTERDB_DATABASE + # env var: LOTUS_HARMONYDB_DATABASE #Database = "yugabyte" # The port to find Yugabyte. Blank for default. # # type: string - # env var: LOTUS_CLUSTERDB_PORT + # env var: LOTUS_HARMONYDB_PORT #Port = "5433" # ITest is for optimized integration testing and not # for production. Blank for default production configuration. # # type: string - # env var: LOTUS_CLUSTERDB_ITEST + # env var: LOTUS_HARMONYDB_ITEST #ITest = "" diff --git a/itests/clusterdb_test.go b/itests/harmonydb_test.go similarity index 92% rename from itests/clusterdb_test.go rename to itests/harmonydb_test.go index dd9dd8f5bbf..150b2bd9e09 100644 --- a/itests/clusterdb_test.go +++ b/itests/harmonydb_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/filecoin-project/lotus/itests/kit" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/node/impl" ) @@ -26,7 +26,7 @@ func TestCrud(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB err := cdb.Exec(ctx, ` INSERT INTO itest_scratch (some_int, content) @@ -64,11 +64,11 @@ func TestTransaction(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB if err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { t.Fatal("E0", err) } - cdb.BeginTransaction(ctx, func(tx *clusterdb.Transaction) (commit bool) { + cdb.BeginTransaction(ctx, func(tx *harmonydb.Transaction) (commit bool) { if err := tx.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil { t.Fatal("E1", err) } @@ -119,7 +119,7 @@ func TestPartialWalk(t *testing.T) { defer cancel() withSetup(t, func(miner *kit.TestMiner) { - cdb := miner.BaseAPI.(*impl.StorageMinerAPI).ClusterDB + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB if err := cdb.Exec(ctx, ` INSERT INTO itest_scratch (content, some_int) diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index da9df08d1ac..dfbc412a1dc 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -49,7 +49,7 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" "github.com/filecoin-project/lotus/gateway" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/markets/idxprov" "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test" lotusminer "github.com/filecoin-project/lotus/miner" @@ -359,7 +359,7 @@ func (n *Ensemble) Start() *Ensemble { n.mn = mocknet.New() } - sharedITestID := clusterdb.ITestNewID() + sharedITestID := harmonydb.ITestNewID() // --------------------- // FULL NODES @@ -727,7 +727,7 @@ func (n *Ensemble) Start() *Ensemble { // upgrades node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule), - node.Override(new(clusterdb.ITestID), sharedITestID), + node.Override(new(harmonydb.ITestID), sharedITestID), } if m.options.subsystems.Has(SMarkets) { @@ -775,7 +775,7 @@ func (n *Ensemble) Start() *Ensemble { n.t.Cleanup(func() { _ = stop(context.Background()) }) n.t.Cleanup(func() { - m.StorageMiner.(*impl.StorageMinerAPI).ClusterDB.ITestDeleteAll() + m.StorageMiner.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll() }) m.BaseAPI = m.StorageMiner @@ -833,7 +833,7 @@ func (n *Ensemble) Start() *Ensemble { auth := http.Header(nil) - // FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).ClusterDB to setup. + // FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).HarmonyDB to setup. remote := paths.NewRemote(localStore, m.MinerNode, auth, 20, &paths.DefaultPartialFileHandler{}) store := m.options.workerStorageOpt(remote) diff --git a/lib/sturdy/clusterdb/doc.go b/lib/harmony/harmonydb/doc.go similarity index 93% rename from lib/sturdy/clusterdb/doc.go rename to lib/harmony/harmonydb/doc.go index 6e86aacd73a..ac60a02602a 100644 --- a/lib/sturdy/clusterdb/doc.go +++ b/lib/harmony/harmonydb/doc.go @@ -1,5 +1,5 @@ /* -# Clusterdb provides database abstractions over SP-wide Postgres-compatible instance(s). +# HarmonyDB provides database abstractions over SP-wide Postgres-compatible instance(s). # Features @@ -32,4 +32,4 @@ Write SQL with context, raw strings, and args: Note: Scan() is column-oriented, while Select() & StructScan() is field name/tag oriented. */ -package clusterdb +package harmonydb diff --git a/lib/sturdy/clusterdb/clusterdb.go b/lib/harmony/harmonydb/harmonydb.go similarity index 96% rename from lib/sturdy/clusterdb/clusterdb.go rename to lib/harmony/harmonydb/harmonydb.go index 13a5db20982..fdf57b84642 100644 --- a/lib/sturdy/clusterdb/clusterdb.go +++ b/lib/harmony/harmonydb/harmonydb.go @@ -1,4 +1,4 @@ -package clusterdb +package harmonydb import ( "context" @@ -35,13 +35,13 @@ type DB struct { log func(string) } -var logger = logging.Logger("clusterdb") +var logger = logging.Logger("harmonydb") // NewFromConfig is a convenience function. // In usage: // -// db, err := NewFromConfig(config.ClusterDB) // in binary init -func NewFromConfig(cfg config.ClusterDB) (*DB, error) { +// db, err := NewFromConfig(config.HarmonyDB) // in binary init +func NewFromConfig(cfg config.HarmonyDB) (*DB, error) { return New( cfg.Hosts, cfg.Username, @@ -53,7 +53,7 @@ func NewFromConfig(cfg config.ClusterDB) (*DB, error) { ) } -func NewFromConfigWithITestID(cfg config.ClusterDB) func(id ITestID) (*DB, error) { +func NewFromConfigWithITestID(cfg config.HarmonyDB) func(id ITestID) (*DB, error) { return func(id ITestID) (*DB, error) { return New( cfg.Hosts, diff --git a/lib/sturdy/clusterdb/metrics.go b/lib/harmony/harmonydb/metrics.go similarity index 97% rename from lib/sturdy/clusterdb/metrics.go rename to lib/harmony/harmonydb/metrics.go index 430de54f209..fde11279f36 100644 --- a/lib/sturdy/clusterdb/metrics.go +++ b/lib/harmony/harmonydb/metrics.go @@ -1,4 +1,4 @@ -package clusterdb +package harmonydb import ( "github.com/prometheus/client_golang/prometheus" @@ -11,7 +11,7 @@ import ( var ( dbTag, _ = tag.NewKey("db_name") - pre = "clusterdb/base/" + pre = "harmonydb/base/" waitsBuckets = []float64{0, 10, 20, 30, 50, 80, 130, 210, 340, 550, 890} whichHostBuckets = []float64{0, 1, 2, 3, 4, 5} ) diff --git a/lib/sturdy/clusterdb/sql/20230706.sql b/lib/harmony/harmonydb/sql/20230706.sql similarity index 100% rename from lib/sturdy/clusterdb/sql/20230706.sql rename to lib/harmony/harmonydb/sql/20230706.sql diff --git a/lib/sturdy/clusterdb/userfuncs.go b/lib/harmony/harmonydb/userfuncs.go similarity index 98% rename from lib/sturdy/clusterdb/userfuncs.go rename to lib/harmony/harmonydb/userfuncs.go index 98ceddb0883..8eebdd60731 100644 --- a/lib/sturdy/clusterdb/userfuncs.go +++ b/lib/harmony/harmonydb/userfuncs.go @@ -1,4 +1,4 @@ -package clusterdb +package harmonydb import ( "context" @@ -19,7 +19,7 @@ type Intf interface { // rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries. // In any package, raw strings will satisfy compilation. Ex: // -// clusterDB.Exec("INSERT INTO version (number) VALUES (1)") +// harmonydb.Exec("INSERT INTO version (number) VALUES (1)") // // This prevents SQL injection attacks where the input contains query fragments. type rawStringOnly string diff --git a/node/builder_miner.go b/node/builder_miner.go index 366c89fa4ee..3b3fa8d4843 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -19,7 +19,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/dealfilter" "github.com/filecoin-project/lotus/markets/idxprov" @@ -232,8 +232,8 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(config.SealerConfig), cfg.Storage), Override(new(config.ProvingConfig), cfg.Proving), Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), - Override(new(*clusterdb.DB), clusterdb.NewFromConfigWithITestID(cfg.ClusterDB)), - Override(new(clusterdb.ITestID), clusterdb.ITestID("")), + Override(new(*harmonydb.DB), harmonydb.NewFromConfigWithITestID(cfg.HarmonyDB)), + Override(new(harmonydb.ITestID), harmonydb.ITestID("")), ) } diff --git a/node/config/def.go b/node/config/def.go index 5075355dc8f..47d314fc04b 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -269,7 +269,7 @@ func DefaultStorageMiner() *StorageMiner { MaxConcurrentUnseals: 5, GCInterval: Duration(1 * time.Minute), }, - ClusterDB: ClusterDB{ + HarmonyDB: HarmonyDB{ Hosts: []string{"127.0.0.1"}, Username: "yugabyte", Password: "yugabyte", diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 9c3617a6918..c513eecbc9e 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -117,46 +117,6 @@ without existing payment channels with available funds will fail instead of automatically performing on-chain operations.`, }, }, - "ClusterDB": []DocField{ - { - Name: "Hosts", - Type: "[]string", - - Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB -in a cluster. Only 1 is required`, - }, - { - Name: "Username", - Type: "string", - - Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`, - }, - { - Name: "Password", - Type: "string", - - Comment: `The password for the related username. Blank for default.`, - }, - { - Name: "Database", - Type: "string", - - Comment: `The database (logical partition) within Yugabyte. Blank for default.`, - }, - { - Name: "Port", - Type: "string", - - Comment: `The port to find Yugabyte. Blank for default.`, - }, - { - Name: "ITest", - Type: "string", - - Comment: `ITest is for optimized integration testing and not -for production. Blank for default production configuration.`, - }, - }, "Common": []DocField{ { Name: "API", @@ -508,6 +468,46 @@ Set to 0 to keep all mappings`, Comment: ``, }, }, + "HarmonyDB": []DocField{ + { + Name: "Hosts", + Type: "[]string", + + Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB +in a cluster. Only 1 is required`, + }, + { + Name: "Username", + Type: "string", + + Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`, + }, + { + Name: "Password", + Type: "string", + + Comment: `The password for the related username. Blank for default.`, + }, + { + Name: "Database", + Type: "string", + + Comment: `The database (logical partition) within Yugabyte. Blank for default.`, + }, + { + Name: "Port", + Type: "string", + + Comment: `The port to find Yugabyte. Blank for default.`, + }, + { + Name: "ITest", + Type: "string", + + Comment: `ITest is for optimized integration testing and not +for production. Blank for default production configuration.`, + }, + }, "IndexConfig": []DocField{ { Name: "EnableMsgIndex", @@ -1429,8 +1429,8 @@ HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`, Comment: ``, }, { - Name: "ClusterDB", - Type: "ClusterDB", + Name: "HarmonyDB", + Type: "HarmonyDB", Comment: ``, }, diff --git a/node/config/types.go b/node/config/types.go index 74630d9a3cd..e2d1e655df5 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -62,7 +62,7 @@ type StorageMiner struct { Addresses MinerAddressConfig DAGStore DAGStoreConfig - ClusterDB ClusterDB + HarmonyDB HarmonyDB } type DAGStoreConfig struct { @@ -735,7 +735,7 @@ type IndexConfig struct { EnableMsgIndex bool } -type ClusterDB struct { +type HarmonyDB struct { // HOSTS is a list of hostnames to nodes running YugabyteDB // in a cluster. Only 1 is required Hosts []string diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 89837baffb9..4932e0504a6 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -45,7 +45,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sturdy/clusterdb" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" @@ -124,7 +124,7 @@ type StorageMinerAPI struct { GetExpectedSealDurationFunc dtypes.GetExpectedSealDurationFunc `optional:"true"` SetExpectedSealDurationFunc dtypes.SetExpectedSealDurationFunc `optional:"true"` - ClusterDB *clusterdb.DB + HarmonyDB *harmonydb.DB } var _ api.StorageMiner = &StorageMinerAPI{} From 3fed2b207b5469c99cfdb5a7b7b00d2559c440be Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 15:07:51 -0500 Subject: [PATCH 27/36] circleci test 17 --- .circleci/config.yml | 2 ++ .circleci/template.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8508d819626..b6a9a19e944 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,6 +159,8 @@ jobs: description: Test suite name to report to CircleCI. docker: - image: cimg/go:1.19.7 + environment: + LOTUS_HARMONYDB_HOSTS=yugabyte - image: yugabytedb/yugabyte:latest command: bin/yugabyted start --daemon=false name: yugabyte diff --git a/.circleci/template.yml b/.circleci/template.yml index 8e340f70a9a..d3eb1d2cdec 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -159,6 +159,8 @@ jobs: description: Test suite name to report to CircleCI. docker: - image: cimg/go:[[ .GoVersion]] + environment: + LOTUS_HARMONYDB_HOSTS=yugabyte - image: yugabytedb/yugabyte:latest command: bin/yugabyted start --daemon=false name: yugabyte From cb4645b344f74eb0a0a85ee80d995ce9366bac34 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 15:23:42 -0500 Subject: [PATCH 28/36] circleci try 18 --- .circleci/config.yml | 4 ++-- .circleci/template.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b6a9a19e944..24494d35a5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,8 +159,7 @@ jobs: description: Test suite name to report to CircleCI. docker: - image: cimg/go:1.19.7 - environment: - LOTUS_HARMONYDB_HOSTS=yugabyte + LOTUS_HARMONYDB_HOSTS: yugabyte - image: yugabytedb/yugabyte:latest command: bin/yugabyted start --daemon=false name: yugabyte @@ -183,6 +182,7 @@ jobs: mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts dockerize -wait tcp://yugabyte:5433 -timeout 3m + env gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ diff --git a/.circleci/template.yml b/.circleci/template.yml index d3eb1d2cdec..4fd460aefe8 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -159,8 +159,7 @@ jobs: description: Test suite name to report to CircleCI. docker: - image: cimg/go:[[ .GoVersion]] - environment: - LOTUS_HARMONYDB_HOSTS=yugabyte + LOTUS_HARMONYDB_HOSTS: yugabyte - image: yugabytedb/yugabyte:latest command: bin/yugabyted start --daemon=false name: yugabyte @@ -183,6 +182,7 @@ jobs: mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts dockerize -wait tcp://yugabyte:5433 -timeout 3m + env gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ From 7cd274167c9750db1dbc80e022efad534795f93a Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 15:26:14 -0500 Subject: [PATCH 29/36] circleci try 19 --- .circleci/config.yml | 1 + .circleci/template.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 24494d35a5c..7abb9d5fc00 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,6 +159,7 @@ jobs: description: Test suite name to report to CircleCI. docker: - image: cimg/go:1.19.7 + environment: LOTUS_HARMONYDB_HOSTS: yugabyte - image: yugabytedb/yugabyte:latest command: bin/yugabyted start --daemon=false diff --git a/.circleci/template.yml b/.circleci/template.yml index 4fd460aefe8..d8eeb6048cf 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -159,6 +159,7 @@ jobs: description: Test suite name to report to CircleCI. docker: - image: cimg/go:[[ .GoVersion]] + environment: LOTUS_HARMONYDB_HOSTS: yugabyte - image: yugabytedb/yugabyte:latest command: bin/yugabyted start --daemon=false From 19df71fc5ca33971024dbc7f7737e61ff3524f8f Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 16:00:19 -0500 Subject: [PATCH 30/36] circleci try 20 --- itests/harmonydb_test.go | 10 ++++++++-- itests/kit/ensemble.go | 19 ++++++++++++++++++- lib/harmony/harmonydb/harmonydb.go | 2 +- lib/harmony/harmonydb/metrics.go | 11 +++++++++-- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/itests/harmonydb_test.go b/itests/harmonydb_test.go index 150b2bd9e09..9e09691da92 100644 --- a/itests/harmonydb_test.go +++ b/itests/harmonydb_test.go @@ -68,7 +68,7 @@ func TestTransaction(t *testing.T) { if err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { t.Fatal("E0", err) } - cdb.BeginTransaction(ctx, func(tx *harmonydb.Transaction) (commit bool) { + err := cdb.BeginTransaction(ctx, func(tx *harmonydb.Transaction) (commit bool) { if err := tx.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil { t.Fatal("E1", err) } @@ -92,6 +92,9 @@ func TestTransaction(t *testing.T) { } return false // rollback }) + if err != nil { + t.Fatal("ET", err) + } var sum2 int // Query() example (yes, QueryRow would be preferred here) @@ -102,7 +105,10 @@ func TestTransaction(t *testing.T) { defer q.Close() var rowCt int for q.Next() { - q.Scan(&sum2) + err := q.Scan(&sum2) + if err != nil { + t.Fatal("error scanning ", err) + } rowCt++ } if sum2 != 4+5+6 { diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index dfbc412a1dc..2adec26a01a 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -728,6 +728,15 @@ func (n *Ensemble) Start() *Ensemble { node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule), node.Override(new(harmonydb.ITestID), sharedITestID), + node.Override(new(config.HarmonyDB), func() config.HarmonyDB { + return config.HarmonyDB{ + Hosts: []string{envElse("LOTUS_HARMONYDB_HOSTS", "127.0.0.1")}, + Database: "yugabyte", + Username: "yugabyte", + Password: "yugabyte", + Port: "5433", + } + }), } if m.options.subsystems.Has(SMarkets) { @@ -774,8 +783,9 @@ func (n *Ensemble) Start() *Ensemble { require.NoError(n.t, err) n.t.Cleanup(func() { _ = stop(context.Background()) }) + mCopy := m n.t.Cleanup(func() { - m.StorageMiner.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll() + mCopy.StorageMiner.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll() }) m.BaseAPI = m.StorageMiner @@ -1077,3 +1087,10 @@ func importPreSealMeta(ctx context.Context, meta genesis.Miner, mds dtypes.Metad size := binary.PutUvarint(buf, uint64(maxSectorID)) return mds.Put(ctx, datastore.NewKey(pipeline.StorageCounterDSPrefix), buf[:size]) } + +func envElse(env, els string) string { + if v := os.Getenv(env); v != "" { + return v + } + return els +} diff --git a/lib/harmony/harmonydb/harmonydb.go b/lib/harmony/harmonydb/harmonydb.go index fdf57b84642..702a97681dc 100644 --- a/lib/harmony/harmonydb/harmonydb.go +++ b/lib/harmony/harmonydb/harmonydb.go @@ -186,7 +186,7 @@ func ensureSchemaExists(connString, schema string) error { if err != nil { return err } - defer p.Close(context.Background()) + defer func() { _ = p.Close(context.Background()) }() if len(schema) < 5 || !schemaRE.MatchString(schema) { return errors.New("schema must be of the form " + schemaREString + "\n Got: " + schema) diff --git a/lib/harmony/harmonydb/metrics.go b/lib/harmony/harmonydb/metrics.go index fde11279f36..444b32f6295 100644 --- a/lib/harmony/harmonydb/metrics.go +++ b/lib/harmony/harmonydb/metrics.go @@ -65,6 +65,13 @@ func init() { TagKeys: []tag.Key{dbTag}, }, ) - prometheus.Register(DBMeasures.Waits) - prometheus.Register(DBMeasures.WhichHost) + err := prometheus.Register(DBMeasures.Waits) + if err != nil { + panic(err) + } + + err = prometheus.Register(DBMeasures.WhichHost) + if err != nil { + panic(err) + } } From 15a8e79f13bcef035c2b90de05f84770e5747f5d Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 16:12:36 -0500 Subject: [PATCH 31/36] harmonydb metrics oops --- lib/harmony/harmonydb/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/harmony/harmonydb/metrics.go b/lib/harmony/harmonydb/metrics.go index 444b32f6295..b29a76ad56a 100644 --- a/lib/harmony/harmonydb/metrics.go +++ b/lib/harmony/harmonydb/metrics.go @@ -11,7 +11,7 @@ import ( var ( dbTag, _ = tag.NewKey("db_name") - pre = "harmonydb/base/" + pre = "harmonydb_base_" waitsBuckets = []float64{0, 10, 20, 30, 50, 80, 130, 210, 340, 550, 890} whichHostBuckets = []float64{0, 1, 2, 3, 4, 5} ) From a22b7cf8692aa0b238315d241b1c5cdbc217fb8d Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 17:05:40 -0500 Subject: [PATCH 32/36] circleci try 21 use the test's config --- node/builder_miner.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/node/builder_miner.go b/node/builder_miner.go index 3b3fa8d4843..bd81f426562 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -231,9 +231,12 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(config.SealerConfig), cfg.Storage), Override(new(config.ProvingConfig), cfg.Proving), - Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), - Override(new(*harmonydb.DB), harmonydb.NewFromConfigWithITestID(cfg.HarmonyDB)), + Override(new(config.HarmonyDB), cfg.HarmonyDB), Override(new(harmonydb.ITestID), harmonydb.ITestID("")), + Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), + Override(new(*harmonydb.DB), func(cfg config.HarmonyDB, id harmonydb.ITestID) (*harmonydb.DB, error) { + return harmonydb.NewFromConfigWithITestID(cfg)(id) + }), ) } From e9f7c9c3ec7b1403964bb6387e9b72987b03c89c Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 18:00:05 -0500 Subject: [PATCH 33/36] cast cleanup --- itests/kit/ensemble.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 2adec26a01a..04a02cf6164 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -785,7 +785,7 @@ func (n *Ensemble) Start() *Ensemble { n.t.Cleanup(func() { _ = stop(context.Background()) }) mCopy := m n.t.Cleanup(func() { - mCopy.StorageMiner.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll() + mCopy.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll() }) m.BaseAPI = m.StorageMiner From d17bfb14f8cc2571e20fb8c7eca4bfd2f8f0d514 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 14 Jul 2023 18:05:49 -0500 Subject: [PATCH 34/36] feat: miner deps: harmonydb --- .circleci/config.yml | 53 ++-- .circleci/gen.go | 4 + .circleci/template.yml | 45 +-- build/openrpc/full.json.gz | Bin 33969 -> 33969 bytes build/openrpc/gateway.json.gz | Bin 10425 -> 10425 bytes build/openrpc/miner.json.gz | Bin 15939 -> 15939 bytes build/openrpc/worker.json.gz | Bin 5246 -> 5246 bytes build/version.go | 2 +- documentation/en/cli-lotus-miner.md | 2 +- documentation/en/cli-lotus-worker.md | 2 +- documentation/en/cli-lotus.md | 2 +- .../en/default-lotus-miner-config.toml | 40 +++ extern/filecoin-ffi | 2 +- go.mod | 9 +- go.sum | 21 +- itests/harmonydb_test.go | 173 ++++++++++++ itests/kit/ensemble.go | 29 ++ lib/harmony/harmonydb/doc.go | 35 +++ lib/harmony/harmonydb/harmonydb.go | 266 ++++++++++++++++++ lib/harmony/harmonydb/metrics.go | 77 +++++ lib/harmony/harmonydb/sql/20230706.sql | 6 + lib/harmony/harmonydb/userfuncs.go | 149 ++++++++++ metrics/metrics.go | 79 +++--- node/builder_miner.go | 6 + node/config/def.go | 7 + node/config/doc_gen.go | 46 +++ node/config/types.go | 24 ++ node/impl/storminer.go | 3 + 28 files changed, 993 insertions(+), 89 deletions(-) create mode 100644 itests/harmonydb_test.go create mode 100644 lib/harmony/harmonydb/doc.go create mode 100644 lib/harmony/harmonydb/harmonydb.go create mode 100644 lib/harmony/harmonydb/metrics.go create mode 100644 lib/harmony/harmonydb/sql/20230706.sql create mode 100644 lib/harmony/harmonydb/userfuncs.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fcb831454c..7abb9d5fc00 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,17 +6,9 @@ orbs: executors: golang: docker: - # Must match GO_VERSION_MIN in project root + # Must match GO_VERSION_MIN in project root. Change in gen.go - image: cimg/go:1.19.7 resource_class: medium+ - golang-2xl: - docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.19.7 - resource_class: 2xlarge - ubuntu: - docker: - - image: ubuntu:20.04 commands: build-platform-specific: @@ -96,6 +88,7 @@ commands: git fetch --all install-ubuntu-deps: steps: + - run: sudo apt install curl ca-certificates gnupg - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev check-go-version: @@ -143,9 +136,9 @@ jobs: Run tests with gotestsum. working_directory: ~/lotus parameters: &test-params - executor: - type: executor - default: golang + resource_class: + type: string + default: medium+ go-test-flags: type: string default: "-timeout 20m" @@ -164,7 +157,14 @@ jobs: type: string default: unit description: Test suite name to report to CircleCI. - executor: << parameters.executor >> + docker: + - image: cimg/go:1.19.7 + environment: + LOTUS_HARMONYDB_HOSTS: yugabyte + - image: yugabytedb/yugabyte:latest + command: bin/yugabyted start --daemon=false + name: yugabyte + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -182,6 +182,8 @@ jobs: command: | mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts + dockerize -wait tcp://yugabyte:5433 -timeout 3m + env gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ @@ -209,7 +211,9 @@ jobs: Branch on github.com/filecoin-project/test-vectors to checkout and test with. If empty (the default) the commit defined by the git submodule is used. - executor: << parameters.executor >> + docker: + - image: cimg/go:1.19.7 + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -396,15 +400,14 @@ jobs: Run golangci-lint. working_directory: ~/lotus parameters: - executor: - type: executor - default: golang args: type: string default: '' description: | Arguments to pass to golangci-lint - executor: << parameters.executor >> + docker: + - image: cimg/go:1.19.7 + resource_class: medium+ steps: - install-ubuntu-deps - attach_workspace: @@ -581,7 +584,7 @@ workflows: - build suite: itest-deals_concurrent target: "./itests/deals_concurrent_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-deals_invalid_utf8_label requires: @@ -774,6 +777,12 @@ workflows: - build suite: itest-get_messages_in_ts target: "./itests/get_messages_in_ts_test.go" + - test: + name: test-itest-harmonydb + requires: + - build + suite: itest-harmonydb + target: "./itests/harmonydb_test.go" - test: name: test-itest-lite_migration requires: @@ -1010,14 +1019,14 @@ workflows: - build suite: itest-wdpost_worker_config target: "./itests/wdpost_worker_config_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-worker requires: - build suite: itest-worker target: "./itests/worker_test.go" - executor: golang-2xl + resource_class: 2xlarge - test: name: test-itest-worker_upgrade requires: @@ -1044,7 +1053,7 @@ workflows: - build suite: utest-unit-rest target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..." - executor: golang-2xl + resource_class: 2xlarge - test: name: test-unit-storage requires: diff --git a/.circleci/gen.go b/.circleci/gen.go index 5d951027a0c..d85b15d90fa 100644 --- a/.circleci/gen.go +++ b/.circleci/gen.go @@ -10,6 +10,8 @@ import ( "text/template" ) +const GoVersion = "1.19.7" + //go:generate go run ./gen.go .. //go:embed template.yml @@ -109,6 +111,7 @@ func main() { Networks []string ItestFiles []string UnitSuites map[string]string + GoVersion string } in := data{ Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"}, @@ -123,6 +126,7 @@ func main() { } return ret }(), + GoVersion: GoVersion, } out, err := os.Create("./config.yml") diff --git a/.circleci/template.yml b/.circleci/template.yml index cd8aeb663c9..d8eeb6048cf 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -6,17 +6,9 @@ orbs: executors: golang: docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.19.7 + # Must match GO_VERSION_MIN in project root. Change in gen.go + - image: cimg/go:[[ .GoVersion]] resource_class: medium+ - golang-2xl: - docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.19.7 - resource_class: 2xlarge - ubuntu: - docker: - - image: ubuntu:20.04 commands: build-platform-specific: @@ -96,6 +88,7 @@ commands: git fetch --all install-ubuntu-deps: steps: + - run: sudo apt install curl ca-certificates gnupg - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev check-go-version: @@ -143,9 +136,9 @@ jobs: Run tests with gotestsum. working_directory: ~/lotus parameters: &test-params - executor: - type: executor - default: golang + resource_class: + type: string + default: medium+ go-test-flags: type: string default: "-timeout 20m" @@ -164,7 +157,14 @@ jobs: type: string default: unit description: Test suite name to report to CircleCI. - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + environment: + LOTUS_HARMONYDB_HOSTS: yugabyte + - image: yugabytedb/yugabyte:latest + command: bin/yugabyted start --daemon=false + name: yugabyte + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -182,6 +182,8 @@ jobs: command: | mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts + dockerize -wait tcp://yugabyte:5433 -timeout 3m + env gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ @@ -209,7 +211,9 @@ jobs: Branch on github.com/filecoin-project/test-vectors to checkout and test with. If empty (the default) the commit defined by the git submodule is used. - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + resource_class: << parameters.resource_class >> steps: - install-ubuntu-deps - attach_workspace: @@ -396,15 +400,14 @@ jobs: Run golangci-lint. working_directory: ~/lotus parameters: - executor: - type: executor - default: golang args: type: string default: '' description: | Arguments to pass to golangci-lint - executor: << parameters.executor >> + docker: + - image: cimg/go:[[ .GoVersion]] + resource_class: medium+ steps: - install-ubuntu-deps - attach_workspace: @@ -543,7 +546,7 @@ workflows: suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]] - executor: golang-2xl + resource_class: 2xlarge [[- end]] [[- if (eq $name "wdpost")]] get-params: true @@ -558,7 +561,7 @@ workflows: suite: utest-[[ $suite ]] target: "[[ $pkgs ]]" [[if eq $suite "unit-cli"]]get-params: true[[end]] - [[- if eq $suite "unit-rest"]]executor: golang-2xl[[end]] + [[- if eq $suite "unit-rest"]]resource_class: 2xlarge[[end]] [[- end]] - test: go-test-flags: "-run=TestMulticoreSDR" diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 6a25cd677273192c7b42c6211511056b637a1a70..340f840afe6d62811f6c44896e95603e0a9adbe3 100644 GIT binary patch delta 23 fcmdnk$+WSPX+j61+s00X77o_d-S$_murUAtZ+!^x delta 23 ecmdnk$+WSPX+j5M+s00X7LJxa|IjN}*cbqH8VOF){nv*?+kixt<_^>0049{33~tl diff --git a/build/version.go b/build/version.go index 2b7cbeda27b..56a2cc0ab7c 100644 --- a/build/version.go +++ b/build/version.go @@ -37,7 +37,7 @@ func BuildTypeString() string { } // BuildVersion is the local build version -const BuildVersion = "1.23.2-dev" +const BuildVersion = "1.23.3-dev" func UserVersion() string { if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" { diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index a6ea6756254..ee8f4583771 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -7,7 +7,7 @@ USAGE: lotus-miner [global options] command [command options] [arguments...] VERSION: - 1.23.2-dev + 1.23.3-dev COMMANDS: init Initialize a lotus miner repo diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index 9c749f44aea..bdf992e58a3 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -7,7 +7,7 @@ USAGE: lotus-worker [global options] command [command options] [arguments...] VERSION: - 1.23.2-dev + 1.23.3-dev COMMANDS: run Start lotus worker diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 751182a4341..fe63acbc6c5 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,7 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: - 1.23.2-dev + 1.23.3-dev COMMANDS: daemon Start a lotus daemon process diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index c0a204bf1db..0efb22ff003 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -894,3 +894,43 @@ #GCInterval = "1m0s" +[HarmonyDB] + # HOSTS is a list of hostnames to nodes running YugabyteDB + # in a cluster. Only 1 is required + # + # type: []string + # env var: LOTUS_HARMONYDB_HOSTS + #Hosts = ["127.0.0.1"] + + # The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default. + # + # type: string + # env var: LOTUS_HARMONYDB_USERNAME + #Username = "yugabyte" + + # The password for the related username. Blank for default. + # + # type: string + # env var: LOTUS_HARMONYDB_PASSWORD + #Password = "yugabyte" + + # The database (logical partition) within Yugabyte. Blank for default. + # + # type: string + # env var: LOTUS_HARMONYDB_DATABASE + #Database = "yugabyte" + + # The port to find Yugabyte. Blank for default. + # + # type: string + # env var: LOTUS_HARMONYDB_PORT + #Port = "5433" + + # ITest is for optimized integration testing and not + # for production. Blank for default production configuration. + # + # type: string + # env var: LOTUS_HARMONYDB_ITEST + #ITest = "" + + diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index de34caff946..a458f638e3c 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit de34caff946d598edb299566d951b44b9b7f7dd4 +Subproject commit a458f638e3c8603c9b5a9ed9847c3af4597e46d4 diff --git a/go.mod b/go.mod index ab1f60b6e3b..2da784ad601 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/gdamore/tcell/v2 v2.2.0 + github.com/georgysavva/scany/v2 v2.0.0 github.com/go-openapi/spec v0.19.11 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 @@ -102,6 +103,7 @@ require ( github.com/ipld/go-ipld-selector-text-lite v0.0.1 github.com/ipni/go-libipni v0.0.8 github.com/ipni/index-provider v0.12.0 + github.com/jackc/pgx/v5 v5.4.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/koalacxr/quantile v0.0.1 github.com/libp2p/go-buffer-pool v0.1.0 @@ -127,7 +129,7 @@ require ( github.com/multiformats/go-varint v0.0.7 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/polydawn/refmt v0.89.0 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.16.0 github.com/puzpuzpuz/xsync/v2 v2.4.0 github.com/raulk/clock v1.1.0 github.com/raulk/go-watchdog v1.3.0 @@ -243,6 +245,9 @@ require ( github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect @@ -291,7 +296,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.3.2 // indirect diff --git a/go.sum b/go.sum index 71341203d8d..ebbc4dcc8a2 100644 --- a/go.sum +++ b/go.sum @@ -169,6 +169,7 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -391,6 +392,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U4= github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU= +github.com/georgysavva/scany/v2 v2.0.0 h1:RGXqxDv4row7/FYoK8MRXAZXqoWF/NM+NP0q50k3DKU= +github.com/georgysavva/scany/v2 v2.0.0/go.mod h1:sigOdh+0qb/+aOs3TVhehVT10p8qJL7K/Zhyz8vWo38= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -465,6 +468,7 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -867,6 +871,14 @@ github.com/ipni/index-provider v0.12.0 h1:R3F6dxxKNv4XkE4GJZNLOG0bDEbBQ/S5iztXwS github.com/ipni/index-provider v0.12.0/go.mod h1:GhyrADJp7n06fqoc1djzkvL4buZYHzV8SoWrlxEo5F4= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE= +github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= +github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk= +github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -958,6 +970,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= @@ -1426,8 +1439,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1461,8 +1474,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Eznag= diff --git a/itests/harmonydb_test.go b/itests/harmonydb_test.go new file mode 100644 index 00000000000..9e09691da92 --- /dev/null +++ b/itests/harmonydb_test.go @@ -0,0 +1,173 @@ +package itests + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/filecoin-project/lotus/itests/kit" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" + "github.com/filecoin-project/lotus/node/impl" +) + +func withSetup(t *testing.T, f func(*kit.TestMiner)) { + _, miner, _ := kit.EnsembleMinimal(t, + kit.LatestActorsAt(-1), + kit.MockProofs(), + ) + + f(miner) +} + +func TestCrud(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + withSetup(t, func(miner *kit.TestMiner) { + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB + err := cdb.Exec(ctx, ` + INSERT INTO + itest_scratch (some_int, content) + VALUES + (11, 'cows'), + (5, 'cats') + `) + if err != nil { + t.Fatal("Could not insert: ", err) + } + var ints []struct { + Count int `db:"some_int"` + Animal string `db:"content"` + Unpopulated int + } + err = cdb.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch") + if err != nil { + t.Fatal("Could not select: ", err) + } + if len(ints) != 2 { + t.Fatal("unexpected count of returns. Want 2, Got ", len(ints)) + } + if ints[0].Count != 11 || ints[1].Count != 5 { + t.Fatal("expected [11,5] got ", ints) + } + if ints[0].Animal != "cows" || ints[1].Animal != "cats" { + t.Fatal("expected, [cows, cats] ", ints) + } + fmt.Println("test completed") + }) +} + +func TestTransaction(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + withSetup(t, func(miner *kit.TestMiner) { + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB + if err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { + t.Fatal("E0", err) + } + err := cdb.BeginTransaction(ctx, func(tx *harmonydb.Transaction) (commit bool) { + if err := tx.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil { + t.Fatal("E1", err) + } + + // sum1 is read from OUTSIDE the transaction so it's the old value + var sum1 int + if err := cdb.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum1); err != nil { + t.Fatal("E2", err) + } + if sum1 != 4+5+6 { + t.Fatal("Expected 15, got ", sum1) + } + + // sum2 is from INSIDE the transaction, so the updated value. + var sum2 int + if err := tx.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum2); err != nil { + t.Fatal("E3", err) + } + if sum2 != 4+5+6+7+8+9 { + t.Fatal("Expected 39, got ", sum2) + } + return false // rollback + }) + if err != nil { + t.Fatal("ET", err) + } + + var sum2 int + // Query() example (yes, QueryRow would be preferred here) + q, err := cdb.Query(ctx, "SELECT SUM(some_int) FROM itest_scratch") + if err != nil { + t.Fatal("E4", err) + } + defer q.Close() + var rowCt int + for q.Next() { + err := q.Scan(&sum2) + if err != nil { + t.Fatal("error scanning ", err) + } + rowCt++ + } + if sum2 != 4+5+6 { + t.Fatal("Expected 15, got ", sum2) + } + if rowCt != 1 { + t.Fatal("unexpected count of rows") + } + }) +} + +func TestPartialWalk(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + withSetup(t, func(miner *kit.TestMiner) { + cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB + if err := cdb.Exec(ctx, ` + INSERT INTO + itest_scratch (content, some_int) + VALUES + ('andy was here', 5), + ('lotus is awesome', 6), + ('hello world', 7), + ('3rd integration test', 8), + ('fiddlesticks', 9) + `); err != nil { + t.Fatal("e1", err) + } + + // TASK: FIND THE ID of the string with a specific SHA256 + needle := "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + q, err := cdb.Query(ctx, `SELECT id, content FROM itest_scratch`) + if err != nil { + t.Fatal("e2", err) + } + defer q.Close() + + var tmp struct { + Src string `db:"content"` + ID int + } + + var done bool + for q.Next() { + + if err := q.StructScan(&tmp); err != nil { + t.Fatal("structscan err " + err.Error()) + } + + bSha := sha256.Sum256([]byte(tmp.Src)) + if hex.EncodeToString(bSha[:]) == needle { + done = true + break + } + } + if !done { + t.Fatal("We didn't find it.") + } + // Answer: tmp.ID + }) +} diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 19cc163af5a..04a02cf6164 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -49,11 +49,13 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" "github.com/filecoin-project/lotus/gateway" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/markets/idxprov" "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test" lotusminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" testing2 "github.com/filecoin-project/lotus/node/modules/testing" @@ -357,6 +359,8 @@ func (n *Ensemble) Start() *Ensemble { n.mn = mocknet.New() } + sharedITestID := harmonydb.ITestNewID() + // --------------------- // FULL NODES // --------------------- @@ -722,6 +726,17 @@ func (n *Ensemble) Start() *Ensemble { // upgrades node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule), + + node.Override(new(harmonydb.ITestID), sharedITestID), + node.Override(new(config.HarmonyDB), func() config.HarmonyDB { + return config.HarmonyDB{ + Hosts: []string{envElse("LOTUS_HARMONYDB_HOSTS", "127.0.0.1")}, + Database: "yugabyte", + Username: "yugabyte", + Password: "yugabyte", + Port: "5433", + } + }), } if m.options.subsystems.Has(SMarkets) { @@ -768,6 +783,10 @@ func (n *Ensemble) Start() *Ensemble { require.NoError(n.t, err) n.t.Cleanup(func() { _ = stop(context.Background()) }) + mCopy := m + n.t.Cleanup(func() { + mCopy.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll() + }) m.BaseAPI = m.StorageMiner @@ -824,6 +843,8 @@ func (n *Ensemble) Start() *Ensemble { auth := http.Header(nil) + // FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).HarmonyDB to setup. + remote := paths.NewRemote(localStore, m.MinerNode, auth, 20, &paths.DefaultPartialFileHandler{}) store := m.options.workerStorageOpt(remote) @@ -853,6 +874,7 @@ func (n *Ensemble) Start() *Ensemble { require.NoError(n.t, err) n.active.workers = append(n.active.workers, m) + } // If we are here, we have processed all inactive workers and moved them @@ -1065,3 +1087,10 @@ func importPreSealMeta(ctx context.Context, meta genesis.Miner, mds dtypes.Metad size := binary.PutUvarint(buf, uint64(maxSectorID)) return mds.Put(ctx, datastore.NewKey(pipeline.StorageCounterDSPrefix), buf[:size]) } + +func envElse(env, els string) string { + if v := os.Getenv(env); v != "" { + return v + } + return els +} diff --git a/lib/harmony/harmonydb/doc.go b/lib/harmony/harmonydb/doc.go new file mode 100644 index 00000000000..ac60a02602a --- /dev/null +++ b/lib/harmony/harmonydb/doc.go @@ -0,0 +1,35 @@ +/* +# HarmonyDB provides database abstractions over SP-wide Postgres-compatible instance(s). + +# Features + + Rolling to secondary database servers on connection failure + Convenience features for Go + SQL + Prevention of SQL injection vulnerabilities + Monitors behavior via Prometheus stats and logging of errors. + +# Usage + +Processes should use New() to instantiate a *DB and keep it. +Consumers can use this *DB concurrently. +Creating and changing tables & views should happen in ./sql/ folder. +Name the file "today's date" in the format: YYYYMMDD.sql (ex: 20231231.sql for the year's last day) + + a. CREATE TABLE should NOT have a schema: + GOOD: CREATE TABLE foo (); + BAD: CREATE TABLE me.foo (); + b. Schema is managed for you. It provides isolation for integraton tests & multi-use. + c. Git Merges: All run once, so old-after-new is OK when there are no deps. + d. NEVER change shipped sql files. Have later files make corrections. + e. Anything not ran will be ran, so an older date making it to master is OK. + +Write SQL with context, raw strings, and args: + + name := "Alice" + var ID int + err := QueryRow(ctx, "SELECT id from people where first_name=?", name).Scan(&ID) + fmt.Println(ID) + +Note: Scan() is column-oriented, while Select() & StructScan() is field name/tag oriented. +*/ +package harmonydb diff --git a/lib/harmony/harmonydb/harmonydb.go b/lib/harmony/harmonydb/harmonydb.go new file mode 100644 index 00000000000..702a97681dc --- /dev/null +++ b/lib/harmony/harmonydb/harmonydb.go @@ -0,0 +1,266 @@ +package harmonydb + +import ( + "context" + "embed" + "errors" + "fmt" + "math/rand" + "regexp" + "sort" + "strconv" + "strings" + "time" + + logging "github.com/ipfs/go-log/v2" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/filecoin-project/lotus/node/config" +) + +type ITestID string + +// ItestNewID see ITestWithID doc +func ITestNewID() ITestID { + return ITestID(strconv.Itoa(rand.Intn(99999))) +} + +type DB struct { + pgx *pgxpool.Pool + cfg *pgxpool.Config + schema string + hostnames []string + log func(string) +} + +var logger = logging.Logger("harmonydb") + +// NewFromConfig is a convenience function. +// In usage: +// +// db, err := NewFromConfig(config.HarmonyDB) // in binary init +func NewFromConfig(cfg config.HarmonyDB) (*DB, error) { + return New( + cfg.Hosts, + cfg.Username, + cfg.Password, + cfg.Database, + cfg.Port, + "", + func(s string) { logger.Error(s) }, + ) +} + +func NewFromConfigWithITestID(cfg config.HarmonyDB) func(id ITestID) (*DB, error) { + return func(id ITestID) (*DB, error) { + return New( + cfg.Hosts, + cfg.Username, + cfg.Password, + cfg.Database, + cfg.Port, + id, + func(s string) { logger.Error(s) }, + ) + } +} + +// New is to be called once per binary to establish the pool. +// log() is for errors. It returns an upgraded database's connection. +// This entry point serves both production and integration tests, so it's more DI. +func New(hosts []string, username, password, database, port string, itestID ITestID, log func(string)) (*DB, error) { + itest := string(itestID) + connString := "" + if len(hosts) > 0 { + connString = "host=" + hosts[0] + " " + } + for k, v := range map[string]string{"user": username, "password": password, "dbname": database, "port": port} { + if strings.TrimSpace(v) != "" { + connString += k + "=" + v + " " + } + } + + schema := "lotus" + if itest != "" { + schema = "itest_" + itest + } + + if err := ensureSchemaExists(connString, schema); err != nil { + return nil, err + } + cfg, err := pgxpool.ParseConfig(connString + "search_path=" + schema) + if err != nil { + return nil, err + } + + // enable multiple fallback hosts. + for _, h := range hosts[1:] { + cfg.ConnConfig.Fallbacks = append(cfg.ConnConfig.Fallbacks, &pgconn.FallbackConfig{Host: h}) + } + + cfg.ConnConfig.OnNotice = func(conn *pgconn.PgConn, n *pgconn.Notice) { + log("database notice: " + n.Message + ": " + n.Detail) + DBMeasures.Errors.M(1) + } + + db := DB{cfg: cfg, schema: schema, hostnames: hosts, log: log} // pgx populated in AddStatsAndConnect + if err := db.addStatsAndConnect(); err != nil { + return nil, err + } + + return &db, db.upgrade() +} + +type tracer struct { +} + +type ctxkey string + +var sqlStart = ctxkey("sqlStart") + +func (t tracer) TraceQueryStart(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + return context.WithValue(ctx, sqlStart, time.Now()) +} +func (t tracer) TraceQueryEnd(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryEndData) { + DBMeasures.Hits.M(1) + ms := time.Since(ctx.Value(sqlStart).(time.Time)).Milliseconds() + DBMeasures.TotalWait.M(ms) + DBMeasures.Waits.Observe(float64(ms)) + if data.Err != nil { + DBMeasures.Errors.M(1) + } + // Can log what type of query it is, but not what tables + // Can log rows affected. +} + +// addStatsAndConnect connects a prometheus logger. Be sure to run this before using the DB. +func (db *DB) addStatsAndConnect() error { + + db.cfg.ConnConfig.Tracer = tracer{} + + hostnameToIndex := map[string]float64{} + for i, h := range db.hostnames { + hostnameToIndex[h] = float64(i) + } + db.cfg.AfterConnect = func(ctx context.Context, c *pgx.Conn) error { + s := db.pgx.Stat() + DBMeasures.OpenConnections.M(int64(s.TotalConns())) + DBMeasures.WhichHost.Observe(hostnameToIndex[c.Config().Host]) + + //FUTURE place for any connection seasoning + return nil + } + + var err error + db.pgx, err = pgxpool.NewWithConfig(context.Background(), db.cfg) + if err != nil { + db.log(fmt.Sprintf("Unable to connect to database: %v\n", err)) + return err + } + return nil +} + +// ITestDeleteAll will delete everything created for "this" integration test. +// This must be called at the end of each integration test. +func (db *DB) ITestDeleteAll() { + if !strings.HasPrefix(db.schema, "itest_") { + fmt.Println("Warning: this should never be called on anything but an itest schema.") + return + } + defer db.pgx.Close() + _, err := db.pgx.Exec(context.Background(), "DROP SCHEMA "+db.schema+" CASCADE") + if err != nil { + fmt.Println("warning: unclean itest shutdown: cannot delete schema: " + err.Error()) + return + } +} + +var schemaREString = "^[A-Za-z0-9_]+$" +var schemaRE = regexp.MustCompile(schemaREString) + +func ensureSchemaExists(connString, schema string) error { + // FUTURE allow using fallback DBs for start-up. + p, err := pgx.Connect(context.Background(), connString) + if err != nil { + return err + } + defer func() { _ = p.Close(context.Background()) }() + + if len(schema) < 5 || !schemaRE.MatchString(schema) { + return errors.New("schema must be of the form " + schemaREString + "\n Got: " + schema) + } + _, err = p.Exec(context.Background(), "CREATE SCHEMA IF NOT EXISTS "+schema) + if err != nil { + return fmt.Errorf("cannot create schema: %w", err) + } + return nil +} + +//go:embed sql +var fs embed.FS + +func (db *DB) upgrade() error { + // Does the version table exist? if not, make it. + // NOTE: This cannot change except via the next sql file. + err := db.Exec(context.Background(), `CREATE TABLE IF NOT EXISTS base ( + id SERIAL PRIMARY KEY, + entry CHAR(12), + applied TIMESTAMP DEFAULT current_timestamp + )`) + if err != nil { + db.log("Upgrade failed.") + return err + } + + // __Run scripts in order.__ + + landed := map[string]bool{} + { + var landedEntries []struct{ Entry string } + err = db.Select(context.Background(), &landedEntries, "SELECT entry FROM base") + if err != nil { + db.log("Cannot read entries: " + err.Error()) + return err + } + for _, l := range landedEntries { + landed[l.Entry] = true + } + } + dir, err := fs.ReadDir("sql") + if err != nil { + db.log("Cannot read fs entries: " + err.Error()) + return err + } + sort.Slice(dir, func(i, j int) bool { return dir[i].Name() < dir[j].Name() }) + for _, e := range dir { + name := e.Name() + if landed[name] || !strings.HasSuffix(name, ".sql") { + continue + } + file, err := fs.ReadFile("sql/" + name) + if err != nil { + db.log("weird embed file read err") + return err + } + for _, s := range strings.Split(string(file), ";") { // Implement the changes. + if len(strings.TrimSpace(s)) == 0 { + continue + } + _, err = db.pgx.Exec(context.Background(), s) + if err != nil { + db.log(fmt.Sprintf("Could not upgrade! File %s, Query: %s, Returned: %s", name, s, err.Error())) + return err + } + } + + // Mark Completed. + err = db.Exec(context.Background(), "INSERT INTO base (entry) VALUES ($1)", name) + if err != nil { + db.log("Cannot update base: " + err.Error()) + return fmt.Errorf("cannot insert into base: %w", err) + } + } + return nil +} diff --git a/lib/harmony/harmonydb/metrics.go b/lib/harmony/harmonydb/metrics.go new file mode 100644 index 00000000000..b29a76ad56a --- /dev/null +++ b/lib/harmony/harmonydb/metrics.go @@ -0,0 +1,77 @@ +package harmonydb + +import ( + "github.com/prometheus/client_golang/prometheus" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + + "github.com/filecoin-project/lotus/metrics" +) + +var ( + dbTag, _ = tag.NewKey("db_name") + pre = "harmonydb_base_" + waitsBuckets = []float64{0, 10, 20, 30, 50, 80, 130, 210, 340, 550, 890} + whichHostBuckets = []float64{0, 1, 2, 3, 4, 5} +) + +// DBMeasures groups all db metrics. +var DBMeasures = struct { + Hits *stats.Int64Measure + TotalWait *stats.Int64Measure + Waits prometheus.Histogram + OpenConnections *stats.Int64Measure + Errors *stats.Int64Measure + WhichHost prometheus.Histogram +}{ + Hits: stats.Int64(pre+"hits", "Total number of uses.", stats.UnitDimensionless), + TotalWait: stats.Int64(pre+"total_wait", "Total delay. A numerator over hits to get average wait.", stats.UnitMilliseconds), + Waits: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: pre + "waits", + Buckets: waitsBuckets, + Help: "The histogram of waits for query completions.", + }), + OpenConnections: stats.Int64(pre+"open_connections", "Total connection count.", stats.UnitDimensionless), + Errors: stats.Int64(pre+"errors", "Total error count.", stats.UnitDimensionless), + WhichHost: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: pre + "which_host", + Buckets: whichHostBuckets, + Help: "The index of the hostname being used", + }), +} + +// CacheViews groups all cache-related default views. +func init() { + metrics.RegisterViews( + &view.View{ + Measure: DBMeasures.Hits, + Aggregation: view.Sum(), + TagKeys: []tag.Key{dbTag}, + }, + &view.View{ + Measure: DBMeasures.TotalWait, + Aggregation: view.Sum(), + TagKeys: []tag.Key{dbTag}, + }, + &view.View{ + Measure: DBMeasures.OpenConnections, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{dbTag}, + }, + &view.View{ + Measure: DBMeasures.Errors, + Aggregation: view.Sum(), + TagKeys: []tag.Key{dbTag}, + }, + ) + err := prometheus.Register(DBMeasures.Waits) + if err != nil { + panic(err) + } + + err = prometheus.Register(DBMeasures.WhichHost) + if err != nil { + panic(err) + } +} diff --git a/lib/harmony/harmonydb/sql/20230706.sql b/lib/harmony/harmonydb/sql/20230706.sql new file mode 100644 index 00000000000..b45aca7fa71 --- /dev/null +++ b/lib/harmony/harmonydb/sql/20230706.sql @@ -0,0 +1,6 @@ +CREATE TABLE itest_scratch ( + id SERIAL PRIMARY KEY, + content TEXT, + some_int INTEGER, + update_time TIMESTAMP DEFAULT current_timestamp +) \ No newline at end of file diff --git a/lib/harmony/harmonydb/userfuncs.go b/lib/harmony/harmonydb/userfuncs.go new file mode 100644 index 00000000000..8eebdd60731 --- /dev/null +++ b/lib/harmony/harmonydb/userfuncs.go @@ -0,0 +1,149 @@ +package harmonydb + +import ( + "context" + + "github.com/georgysavva/scany/v2/pgxscan" + "github.com/jackc/pgx/v5" +) + +type Intf interface { + ITestDeleteAll() + Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error + Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) + QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row + Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error + BeginTransaction(ctx context.Context, f func(t *Transaction) (commit bool)) (retErr error) +} + +// rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries. +// In any package, raw strings will satisfy compilation. Ex: +// +// harmonydb.Exec("INSERT INTO version (number) VALUES (1)") +// +// This prevents SQL injection attacks where the input contains query fragments. +type rawStringOnly string + +// Exec executes changes (INSERT, DELETE, or UPDATE). +// Note, for CREATE & DROP please keep these permanent and express +// them in the ./sql/ files (next number). +func (db *DB) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error { + _, err := db.pgx.Exec(ctx, string(sql), arguments...) + return err +} + +type Qry interface { + Next() bool + Err() error + Close() + Scan(...any) error + Values() ([]any, error) +} + +// Query offers Next/Err/Close/Scan/Values/StructScan +type Query struct { + Qry +} + +// Query allows iterating returned values to save memory consumption +// with the downside of needing to `defer q.Close()`. For a simpler interface, +// try Select() +// Next() must be called to advance the row cursor, including the first time: +// Ex: +// q, err := db.Query(ctx, "SELECT id, name FROM users") +// handleError(err) +// defer q.Close() +// +// for q.Next() { +// var id int +// var name string +// handleError(q.Scan(&id, &name)) +// fmt.Println(id, name) +// } +func (db *DB) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) { + q, err := db.pgx.Query(ctx, string(sql), arguments...) + return &Query{q}, err +} +func (q *Query) StructScan(s any) error { + return pgxscan.ScanRow(s, q.Qry.(pgx.Rows)) +} + +type Row interface { + Scan(...any) error +} + +// QueryRow gets 1 row using column order matching. +// This is a timesaver for the special case of wanting the first row returned only. +// EX: +// +// var name, pet string +// var ID = 123 +// err := db.QueryRow(ctx, "SELECT name, pet FROM users WHERE ID=?", ID).Scan(&name, &pet) +func (db *DB) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row { + return db.pgx.QueryRow(ctx, string(sql), arguments...) +} + +/* +Select multiple rows into a slice using name matching +Ex: + + type user struct { + Name string + ID int + Number string `db:"tel_no"` + } + + var users []user + pet := "cat" + err := db.Select(ctx, &users, "SELECT name, id, tel_no FROM customers WHERE pet=?", pet) +*/ +func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error { + return pgxscan.Select(ctx, db.pgx, sliceOfStructPtr, string(sql), arguments...) +} + +type Transaction struct { + pgx.Tx +} + +// BeginTransaction is how you can access transactions using this library. +// The entire transaction happens in the function passed in. +// The return must be true or a rollback will occur. +func (db *DB) BeginTransaction(ctx context.Context, f func(t *Transaction) (commit bool)) (retErr error) { + tx, err := db.pgx.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return err + } + var commit bool + defer func() { // Panic clean-up. + if !commit { + retErr = tx.Rollback(ctx) + } + }() + commit = f(&Transaction{tx}) + if commit { + return tx.Commit(ctx) + } + return nil +} + +// Exec in a transaction. +func (t *Transaction) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error { + _, err := t.Tx.Exec(ctx, string(sql), arguments...) + return err +} + +// Query in a transaction. +func (t *Transaction) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) { + q, err := t.Tx.Query(ctx, string(sql), arguments...) + return &Query{q}, err +} + +// QueryRow in a transaction. +func (t *Transaction) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row { + return t.Tx.QueryRow(ctx, string(sql), arguments...) +} + +// Select in a transaction. +func (t *Transaction) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error { + return pgxscan.Select(ctx, t.Tx, sliceOfStructPtr, string(sql), arguments...) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index ee7bd8695cc..b1c241b2134 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -695,46 +695,55 @@ var ( } ) +var views = []*view.View{ + InfoView, + PeerCountView, + APIRequestDurationView, + + GraphsyncReceivingPeersCountView, + GraphsyncReceivingActiveCountView, + GraphsyncReceivingCountCountView, + GraphsyncReceivingTotalMemoryAllocatedView, + GraphsyncReceivingTotalPendingAllocationsView, + GraphsyncReceivingPeersPendingView, + GraphsyncSendingPeersCountView, + GraphsyncSendingActiveCountView, + GraphsyncSendingCountCountView, + GraphsyncSendingTotalMemoryAllocatedView, + GraphsyncSendingTotalPendingAllocationsView, + GraphsyncSendingPeersPendingView, + + RcmgrAllowConnView, + RcmgrBlockConnView, + RcmgrAllowStreamView, + RcmgrBlockStreamView, + RcmgrAllowPeerView, + RcmgrBlockPeerView, + RcmgrAllowProtoView, + RcmgrBlockProtoView, + RcmgrBlockProtoPeerView, + RcmgrAllowSvcView, + RcmgrBlockSvcView, + RcmgrBlockSvcPeerView, + RcmgrAllowMemView, + RcmgrBlockMemView, +} + // DefaultViews is an array of OpenCensus views for metric gathering purposes var DefaultViews = func() []*view.View { - views := []*view.View{ - InfoView, - PeerCountView, - APIRequestDurationView, - - GraphsyncReceivingPeersCountView, - GraphsyncReceivingActiveCountView, - GraphsyncReceivingCountCountView, - GraphsyncReceivingTotalMemoryAllocatedView, - GraphsyncReceivingTotalPendingAllocationsView, - GraphsyncReceivingPeersPendingView, - GraphsyncSendingPeersCountView, - GraphsyncSendingActiveCountView, - GraphsyncSendingCountCountView, - GraphsyncSendingTotalMemoryAllocatedView, - GraphsyncSendingTotalPendingAllocationsView, - GraphsyncSendingPeersPendingView, - - RcmgrAllowConnView, - RcmgrBlockConnView, - RcmgrAllowStreamView, - RcmgrBlockStreamView, - RcmgrAllowPeerView, - RcmgrBlockPeerView, - RcmgrAllowProtoView, - RcmgrBlockProtoView, - RcmgrBlockProtoPeerView, - RcmgrAllowSvcView, - RcmgrBlockSvcView, - RcmgrBlockSvcPeerView, - RcmgrAllowMemView, - RcmgrBlockMemView, - } - views = append(views, blockstore.DefaultViews...) - views = append(views, rpcmetrics.DefaultViews...) return views }() +// RegisterViews adds views to the default list without modifying this file. +func RegisterViews(v ...*view.View) { + views = append(views, v...) +} + +func init() { + RegisterViews(blockstore.DefaultViews...) + RegisterViews(rpcmetrics.DefaultViews...) +} + var ChainNodeViews = append([]*view.View{ ChainNodeHeightView, ChainNodeHeightExpectedView, diff --git a/node/builder_miner.go b/node/builder_miner.go index d9c92422d00..bd81f426562 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/dealfilter" "github.com/filecoin-project/lotus/markets/idxprov" @@ -230,7 +231,12 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(config.SealerConfig), cfg.Storage), Override(new(config.ProvingConfig), cfg.Proving), + Override(new(config.HarmonyDB), cfg.HarmonyDB), + Override(new(harmonydb.ITestID), harmonydb.ITestID("")), Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)), + Override(new(*harmonydb.DB), func(cfg config.HarmonyDB, id harmonydb.ITestID) (*harmonydb.DB, error) { + return harmonydb.NewFromConfigWithITestID(cfg)(id) + }), ) } diff --git a/node/config/def.go b/node/config/def.go index 42b035c66ae..47d314fc04b 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -269,6 +269,13 @@ func DefaultStorageMiner() *StorageMiner { MaxConcurrentUnseals: 5, GCInterval: Duration(1 * time.Minute), }, + HarmonyDB: HarmonyDB{ + Hosts: []string{"127.0.0.1"}, + Username: "yugabyte", + Password: "yugabyte", + Database: "yugabyte", + Port: "5433", + }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 5361b2d6c59..c513eecbc9e 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -468,6 +468,46 @@ Set to 0 to keep all mappings`, Comment: ``, }, }, + "HarmonyDB": []DocField{ + { + Name: "Hosts", + Type: "[]string", + + Comment: `HOSTS is a list of hostnames to nodes running YugabyteDB +in a cluster. Only 1 is required`, + }, + { + Name: "Username", + Type: "string", + + Comment: `The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.`, + }, + { + Name: "Password", + Type: "string", + + Comment: `The password for the related username. Blank for default.`, + }, + { + Name: "Database", + Type: "string", + + Comment: `The database (logical partition) within Yugabyte. Blank for default.`, + }, + { + Name: "Port", + Type: "string", + + Comment: `The port to find Yugabyte. Blank for default.`, + }, + { + Name: "ITest", + Type: "string", + + Comment: `ITest is for optimized integration testing and not +for production. Blank for default production configuration.`, + }, + }, "IndexConfig": []DocField{ { Name: "EnableMsgIndex", @@ -1386,6 +1426,12 @@ HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`, Name: "DAGStore", Type: "DAGStoreConfig", + Comment: ``, + }, + { + Name: "HarmonyDB", + Type: "HarmonyDB", + Comment: ``, }, }, diff --git a/node/config/types.go b/node/config/types.go index c89e8f70bad..e2d1e655df5 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -61,6 +61,8 @@ type StorageMiner struct { Fees MinerFeeConfig Addresses MinerAddressConfig DAGStore DAGStoreConfig + + HarmonyDB HarmonyDB } type DAGStoreConfig struct { @@ -732,3 +734,25 @@ type IndexConfig struct { // EnableMsgIndex enables indexing of messages on chain. EnableMsgIndex bool } + +type HarmonyDB struct { + // HOSTS is a list of hostnames to nodes running YugabyteDB + // in a cluster. Only 1 is required + Hosts []string + + // The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default. + Username string + + // The password for the related username. Blank for default. + Password string + + // The database (logical partition) within Yugabyte. Blank for default. + Database string + + // The port to find Yugabyte. Blank for default. + Port string + + // ITest is for optimized integration testing and not + // for production. Blank for default production configuration. + ITest string +} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index e4fa41c788a..4932e0504a6 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -45,6 +45,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" @@ -122,6 +123,8 @@ type StorageMinerAPI struct { GetSealingConfigFunc dtypes.GetSealingConfigFunc `optional:"true"` GetExpectedSealDurationFunc dtypes.GetExpectedSealDurationFunc `optional:"true"` SetExpectedSealDurationFunc dtypes.SetExpectedSealDurationFunc `optional:"true"` + + HarmonyDB *harmonydb.DB } var _ api.StorageMiner = &StorageMinerAPI{} From d39e699e1fc0c90483cf780f5ecb92f029152289 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Tue, 18 Jul 2023 14:51:26 -0700 Subject: [PATCH 35/36] harmonydb: better API --- itests/harmonydb_test.go | 12 +++---- lib/harmony/harmonydb/harmonydb.go | 4 +-- lib/harmony/harmonydb/userfuncs.go | 50 ++++++++++++++---------------- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/itests/harmonydb_test.go b/itests/harmonydb_test.go index 9e09691da92..b52a2aa8f79 100644 --- a/itests/harmonydb_test.go +++ b/itests/harmonydb_test.go @@ -27,7 +27,7 @@ func TestCrud(t *testing.T) { withSetup(t, func(miner *kit.TestMiner) { cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB - err := cdb.Exec(ctx, ` + _, err := cdb.Exec(ctx, ` INSERT INTO itest_scratch (some_int, content) VALUES @@ -65,11 +65,11 @@ func TestTransaction(t *testing.T) { withSetup(t, func(miner *kit.TestMiner) { cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB - if err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { + if _, err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil { t.Fatal("E0", err) } - err := cdb.BeginTransaction(ctx, func(tx *harmonydb.Transaction) (commit bool) { - if err := tx.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil { + _, err := cdb.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool) { + if _, err := tx.Exec("INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil { t.Fatal("E1", err) } @@ -84,7 +84,7 @@ func TestTransaction(t *testing.T) { // sum2 is from INSIDE the transaction, so the updated value. var sum2 int - if err := tx.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum2); err != nil { + if err := tx.QueryRow("SELECT SUM(some_int) FROM itest_scratch").Scan(&sum2); err != nil { t.Fatal("E3", err) } if sum2 != 4+5+6+7+8+9 { @@ -126,7 +126,7 @@ func TestPartialWalk(t *testing.T) { withSetup(t, func(miner *kit.TestMiner) { cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB - if err := cdb.Exec(ctx, ` + if _, err := cdb.Exec(ctx, ` INSERT INTO itest_scratch (content, some_int) VALUES diff --git a/lib/harmony/harmonydb/harmonydb.go b/lib/harmony/harmonydb/harmonydb.go index 702a97681dc..fd31e7a1325 100644 --- a/lib/harmony/harmonydb/harmonydb.go +++ b/lib/harmony/harmonydb/harmonydb.go @@ -204,7 +204,7 @@ var fs embed.FS func (db *DB) upgrade() error { // Does the version table exist? if not, make it. // NOTE: This cannot change except via the next sql file. - err := db.Exec(context.Background(), `CREATE TABLE IF NOT EXISTS base ( + _, err := db.Exec(context.Background(), `CREATE TABLE IF NOT EXISTS base ( id SERIAL PRIMARY KEY, entry CHAR(12), applied TIMESTAMP DEFAULT current_timestamp @@ -256,7 +256,7 @@ func (db *DB) upgrade() error { } // Mark Completed. - err = db.Exec(context.Background(), "INSERT INTO base (entry) VALUES ($1)", name) + _, err = db.Exec(context.Background(), "INSERT INTO base (entry) VALUES ($1)", name) if err != nil { db.log("Cannot update base: " + err.Error()) return fmt.Errorf("cannot insert into base: %w", err) diff --git a/lib/harmony/harmonydb/userfuncs.go b/lib/harmony/harmonydb/userfuncs.go index 8eebdd60731..4d35fd8ca7b 100644 --- a/lib/harmony/harmonydb/userfuncs.go +++ b/lib/harmony/harmonydb/userfuncs.go @@ -7,15 +7,6 @@ import ( "github.com/jackc/pgx/v5" ) -type Intf interface { - ITestDeleteAll() - Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error - Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) - QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row - Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error - BeginTransaction(ctx context.Context, f func(t *Transaction) (commit bool)) (retErr error) -} - // rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries. // In any package, raw strings will satisfy compilation. Ex: // @@ -27,9 +18,9 @@ type rawStringOnly string // Exec executes changes (INSERT, DELETE, or UPDATE). // Note, for CREATE & DROP please keep these permanent and express // them in the ./sql/ files (next number). -func (db *DB) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error { - _, err := db.pgx.Exec(ctx, string(sql), arguments...) - return err +func (db *DB) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) (count int, err error) { + res, err := db.pgx.Exec(ctx, string(sql), arguments...) + return int(res.RowsAffected()), err } type Qry interface { @@ -101,17 +92,18 @@ func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnl return pgxscan.Select(ctx, db.pgx, sliceOfStructPtr, string(sql), arguments...) } -type Transaction struct { +type Tx struct { pgx.Tx + ctx context.Context } // BeginTransaction is how you can access transactions using this library. // The entire transaction happens in the function passed in. // The return must be true or a rollback will occur. -func (db *DB) BeginTransaction(ctx context.Context, f func(t *Transaction) (commit bool)) (retErr error) { +func (db *DB) BeginTransaction(ctx context.Context, f func(*Tx) (commit bool)) (didCommit bool, retErr error) { tx, err := db.pgx.BeginTx(ctx, pgx.TxOptions{}) if err != nil { - return err + return false, err } var commit bool defer func() { // Panic clean-up. @@ -119,31 +111,35 @@ func (db *DB) BeginTransaction(ctx context.Context, f func(t *Transaction) (comm retErr = tx.Rollback(ctx) } }() - commit = f(&Transaction{tx}) + commit = f(&Tx{tx, ctx}) if commit { - return tx.Commit(ctx) + err := tx.Commit(ctx) + if err != nil { + return false, err + } + return true, nil } - return nil + return false, nil } // Exec in a transaction. -func (t *Transaction) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) error { - _, err := t.Tx.Exec(ctx, string(sql), arguments...) - return err +func (t *Tx) Exec(sql rawStringOnly, arguments ...any) (count int, err error) { + res, err := t.Tx.Exec(t.ctx, string(sql), arguments...) + return int(res.RowsAffected()), err } // Query in a transaction. -func (t *Transaction) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) { - q, err := t.Tx.Query(ctx, string(sql), arguments...) +func (t *Tx) Query(sql rawStringOnly, arguments ...any) (*Query, error) { + q, err := t.Tx.Query(t.ctx, string(sql), arguments...) return &Query{q}, err } // QueryRow in a transaction. -func (t *Transaction) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row { - return t.Tx.QueryRow(ctx, string(sql), arguments...) +func (t *Tx) QueryRow(sql rawStringOnly, arguments ...any) Row { + return t.Tx.QueryRow(t.ctx, string(sql), arguments...) } // Select in a transaction. -func (t *Transaction) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error { - return pgxscan.Select(ctx, t.Tx, sliceOfStructPtr, string(sql), arguments...) +func (t *Tx) Select(sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error { + return pgxscan.Select(t.ctx, t.Tx, sliceOfStructPtr, string(sql), arguments...) } From 0a57b848986ad0b8c588be854009c491a35fc840 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 27 Jul 2023 18:42:16 -0500 Subject: [PATCH 36/36] cleanup: itest removed from harmonydb --- documentation/en/default-lotus-miner-config.toml | 7 ------- node/config/doc_gen.go | 7 ------- node/config/types.go | 4 ---- 3 files changed, 18 deletions(-) diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 0efb22ff003..9c3f3afb298 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -926,11 +926,4 @@ # env var: LOTUS_HARMONYDB_PORT #Port = "5433" - # ITest is for optimized integration testing and not - # for production. Blank for default production configuration. - # - # type: string - # env var: LOTUS_HARMONYDB_ITEST - #ITest = "" - diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index c513eecbc9e..d8976d56084 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -500,13 +500,6 @@ in a cluster. Only 1 is required`, Comment: `The port to find Yugabyte. Blank for default.`, }, - { - Name: "ITest", - Type: "string", - - Comment: `ITest is for optimized integration testing and not -for production. Blank for default production configuration.`, - }, }, "IndexConfig": []DocField{ { diff --git a/node/config/types.go b/node/config/types.go index e2d1e655df5..c0b6c97b7fc 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -751,8 +751,4 @@ type HarmonyDB struct { // The port to find Yugabyte. Blank for default. Port string - - // ITest is for optimized integration testing and not - // for production. Blank for default production configuration. - ITest string }