diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000000..f3f88a1ae129 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,15 @@ +name: checks + +on: [pull_request] + +jobs: + linter-check: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: "1.18" + check-latest: true + - uses: actions/checkout@v2 + - name: Run linter + run: go run build/ci.go lint diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml new file mode 100644 index 000000000000..2b05dcda5f8f --- /dev/null +++ b/.github/workflows/on-pr.yml @@ -0,0 +1,7 @@ +name: Build and test + +on: [pull_request] + +jobs: + run-tests: + uses: ./.github/workflows/tests.yml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 000000000000..610f8afbff82 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,61 @@ +name: Publish geth to release +on: + release: + types: [published] +jobs: + run-tests: + uses: ./.github/workflows/tests.yml + build: + name: Run docker build and publish + needs: run-tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker build + run: docker build -t vulcanize/go-ethereum -f Dockerfile . + - name: Get the version + id: vars + run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + - name: Tag docker image + run: docker tag vulcanize/go-ethereum docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} + - name: Docker Login + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Push + run: docker push docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} + push_to_registries: + name: Publish assets to Release + runs-on: ubuntu-latest + needs: build + steps: + - name: Get the version + id: vars + run: | + echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/}) + - name: Docker Login to Github Registry + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Pull + run: docker pull docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} + - name: Copy ethereum binary file + run: docker run --rm --entrypoint cat docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} /usr/local/bin/geth > geth-linux-amd64 + - name: Docker Login to Docker Registry + run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin + - name: Tag docker image + run: docker tag docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} vulcanize/vdb-geth:${{steps.vars.outputs.tag}} + - name: Docker Push to Docker Hub + run: docker push vulcanize/vdb-geth:${{steps.vars.outputs.tag}} + - name: Get release + id: get_release + uses: bruceadams/get-release@v1.2.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Release Asset + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.get_release.outputs.upload_url }} + asset_path: geth-linux-amd64 + asset_name: geth-linux-amd64 + asset_content_type: application/octet-stream diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000000..c2de4fc02f8f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,142 @@ +name: Tests for Geth that are used in multiple jobs. + +on: + workflow_call: + +env: + stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || 'f2fd766f5400fcb9eb47b50675d2e3b1f2753702'}} + ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || 'be345e0733d2c025e4082c5154e441317ae94cf7' }} + GOPATH: /tmp/go + +jobs: + build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker build + run: docker build -t vulcanize/go-ethereum . + + geth-unit-test: + name: Run geth unit test + runs-on: ubuntu-latest + env: + GO111MODULE: on + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/setup-go@v3 + with: + go-version: "1.18" + check-latest: true + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Run unit tests + run: | + make test + + statediff-unit-test: + name: Run state diff unit test + runs-on: ubuntu-latest + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/setup-go@v3 + with: + go-version: "1.18" + check-latest: true + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Run docker compose + run: | + docker-compose up -d + + - name: Give the migration a few seconds + run: sleep 30; + + - name: Run unit tests + run: make statedifftest + + private-network-test: + name: Start Geth in a private network. + runs-on: ubuntu-latest + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/setup-go@v3 + with: + go-version: "1.18" + check-latest: true + + - name: Checkout code + uses: actions/checkout@v3 + with: + path: "./go-ethereum" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.stack-orchestrator-ref }} + path: "./stack-orchestrator/" + repository: vulcanize/stack-orchestrator + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-eth-db-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > $GITHUB_WORKSPACE/config.sh + echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> $GITHUB_WORKSPACE/config.sh + echo db_write=true >> $GITHUB_WORKSPACE/config.sh + echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> $GITHUB_WORKSPACE/config.sh + cat $GITHUB_WORKSPACE/config.sh + + - name: Compile Geth + run: | + cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts + ./compile-geth.sh -e docker -p $GITHUB_WORKSPACE/config.sh + cd - + + - name: Run docker compose + run: | + docker-compose \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \ + --env-file $GITHUB_WORKSPACE/config.sh \ + up -d --build + + - name: Make sure the /root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS exists within a certain time frame. + shell: bash + run: | + COUNT=0 + ATTEMPTS=15 + docker ps + until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" cp go-ethereum:/root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS ./STATEFUL_TEST_DEPLOYED_ADDRESS) || [[ $COUNT -eq $ATTEMPTS ]]; do echo -e "$(( COUNT++ ))... \c"; sleep 10; done + [[ $COUNT -eq $ATTEMPTS ]] && echo "Could not find the successful contract deployment" && (exit 1) + cat ./STATEFUL_TEST_DEPLOYED_ADDRESS + sleep 15; + + - name: Create a new transaction. + shell: bash + run: | + docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec go-ethereum /bin/bash /root/transaction_info/NEW_TRANSACTION + echo $? + + - name: Make sure we see entries in the header table + shell: bash + run: | + rows=$(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec ipld-eth-db psql -U vdbm -d vulcanize_testing -AXqtc "SELECT COUNT(*) FROM eth.header_cids") + [[ "$rows" -lt "1" ]] && echo "We could not find any rows in postgres table." && (exit 1) + echo $rows + docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec ipld-eth-db psql -U vdbm -d vulcanize_testing -AXqtc "SELECT * FROM eth.header_cids" diff --git a/.gitignore b/.gitignore index 1ee8b83022ef..78894cb66ecd 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,15 @@ profile.cov /dashboard/assets/package-lock.json **/yarn-error.log +foundry/deployments/local-private-network/geth-linux-amd64 +foundry/projects/local-private-network/geth-linux-amd64 + +# Helpful repos +related-repositories/foundry-test/** +related-repositories/hive/** +related-repositories/ipld-eth-db/** +statediff/indexer/database/sql/statediffing_test_file.sql +statediff/statediffing_test_file.sql +statediff/known_gaps.sql +related-repositories/foundry-test/ +related-repositories/ipld-eth-db/ diff --git a/.gitmodules b/.gitmodules index 241c169c4772..2171d3b2c097 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,4 +5,4 @@ [submodule "evm-benchmarks"] path = tests/evm-benchmarks url = https://github.com/ipsilon/evm-benchmarks - shallow = true + shallow = true \ No newline at end of file diff --git a/Dockerfile.amd64 b/Dockerfile.amd64 new file mode 100644 index 000000000000..7a35376c9710 --- /dev/null +++ b/Dockerfile.amd64 @@ -0,0 +1,7 @@ +# Build Geth in a stock Go builder container +FROM golang:1.15.5 as builder + +#RUN apk add --no-cache make gcc musl-dev linux-headers git + +ADD . /go-ethereum +RUN cd /go-ethereum && make geth diff --git a/Makefile b/Makefile index e97acbef23e6..706477c4257c 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,31 @@ .PHONY: geth android ios evm all test clean +BIN = $(GOPATH)/bin + +## Migration tool +GOOSE = $(BIN)/goose +$(BIN)/goose: + go get -u github.com/pressly/goose/cmd/goose + GOBIN = ./build/bin GO ?= latest GORUN = env GO111MODULE=on go run +#Database +HOST_NAME = localhost +PORT = 5432 +USER = vdbm +PASSWORD = password + +# Set env variable +# `PGPASSWORD` is used by `createdb` and `dropdb` +export PGPASSWORD=$(PASSWORD) + +#Test +TEST_DB = vulcanize_public +TEST_CONNECT_STRING = postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable + geth: $(GORUN) build/ci.go install ./cmd/geth @echo "Done building." @@ -48,3 +69,13 @@ devtools: env GOBIN= go install ./cmd/abigen @type "solc" 2> /dev/null || echo 'Please install solc' @type "protoc" 2> /dev/null || echo 'Please install protoc' + +.PHONY: statedifftest +statedifftest: | $(GOOSE) + GO111MODULE=on go get github.com/stretchr/testify/assert@v1.7.0 + GO111MODULE=on MODE=statediff go test -p 1 ./statediff/... -v + +.PHONY: statediff_filewriting_test +statediff_filetest: | $(GOOSE) + GO111MODULE=on go get github.com/stretchr/testify/assert@v1.7.0 + GO111MODULE=on MODE=statediff STATEDIFF_DB=file go test -p 1 ./statediff/... -v diff --git a/README.md b/README.md index b20eb5b748b4..585f94b67292 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ Official Golang implementation of the Ethereum protocol. -[![API Reference]( -https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 -)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) +[![API Reference](https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) [![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum) [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) @@ -12,6 +10,21 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874 Automated builds are available for stable releases and the unstable master branch. Binary archives are published at https://geth.ethereum.org/downloads/. +## Vulcanize Specific + +This section captures components specific to vulcanize. + +### Branching Structure + +We currently follow the following branching structure. + +1. Create a branch: `v1.10.18-statediff-vX` --> feature/some-feature` +2. Create a PR upstream: `feature/some-feature` --> `v1.10.18-statediff-vX` +3. When a release is ready, create a release branch: `v1.10.18-statediff-vX` --> `v1.10.18-statediff-X.Y.Z` +4. When `v1.10.18-statediff-vX` is stable, merge it to `statediff`. + +This process is subject to change. + ## Building the source For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth). @@ -34,16 +47,16 @@ make all The go-ethereum project comes with several wrappers/executables found in the `cmd` directory. -| Command | Description | -| :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | -| `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | -| `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | -| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | -| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | -| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | -| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | +| Command | Description | +| :--------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | +| `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | +| `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | +| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | +| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | +| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | +| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | +| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | ## Running `geth` @@ -56,17 +69,17 @@ on how you can run your own `geth` instance. Minimum: -* CPU with 2+ cores -* 4GB RAM -* 1TB free storage space to sync the Mainnet -* 8 MBit/sec download Internet service +- CPU with 2+ cores +- 4GB RAM +- 1TB free storage space to sync the Mainnet +- 8 MBit/sec download Internet service Recommended: -* Fast CPU with 4+ cores -* 16GB+ RAM -* High Performance SSD with at least 1TB free space -* 25+ MBit/sec download Internet service +- Fast CPU with 4+ cores +- 16GB+ RAM +- High Performance SSD with at least 1TB free space +- 25+ MBit/sec download Internet service ### Full node on the main Ethereum network @@ -80,15 +93,16 @@ $ geth console ``` This command will: - * Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag), - causing it to download more data in exchange for avoiding processing the entire history - of the Ethereum network, which is very CPU intensive. - * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), - (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://github.com/ChainSafe/web3.js/blob/0.20.7/DOCUMENTATION.md) - (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), - as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). - This tool is optional and if you leave it out you can always attach to an already running - `geth` instance with `geth attach`. + +- Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag), + causing it to download more data in exchange for avoiding processing the entire history + of the Ethereum network, which is very CPU intensive. +- Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), + (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://github.com/ChainSafe/web3.js/blob/0.20.7/DOCUMENTATION.md) + (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), + as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). + This tool is optional and if you leave it out you can always attach to an already running + `geth` instance with `geth attach`. ### A Full node on the Görli test network @@ -107,27 +121,27 @@ useful on the testnet too. Please, see above for their explanations if you've sk Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit: - * Instead of connecting the main Ethereum network, the client will connect to the Görli - test network, which uses different P2P bootnodes, different network IDs and genesis - states. - * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` - will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on - Linux). Note, on OSX and Linux this also means that attaching to a running testnet node - requires the use of a custom endpoint since `geth attach` will try to attach to a - production node endpoint by default, e.g., - `geth attach /goerli/geth.ipc`. Windows users are not affected by - this. - -*Note: Although there are some internal protective measures to prevent transactions from +- Instead of connecting the main Ethereum network, the client will connect to the Görli + test network, which uses different P2P bootnodes, different network IDs and genesis + states. +- Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` + will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on + Linux). Note, on OSX and Linux this also means that attaching to a running testnet node + requires the use of a custom endpoint since `geth attach` will try to attach to a + production node endpoint by default, e.g., + `geth attach /goerli/geth.ipc`. Windows users are not affected by + this. + +_Note: Although there are some internal protective measures to prevent transactions from crossing over between the main network and test network, you should make sure to always use separate accounts for play-money and real-money. Unless you manually move accounts, `geth` will by default correctly separate the two networks and will not make any -accounts available between them.* +accounts available between them._ ### Full node on the Rinkeby test network Go Ethereum also supports connecting to the older proof-of-authority based test network -called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community. +called [_Rinkeby_](https://www.rinkeby.io) which is operated by members of the community. ```shell $ geth --rinkeby console @@ -144,7 +158,7 @@ network's low difficulty/security. $ geth --ropsten console ``` -*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.* +_Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory._ ### Configuration @@ -162,7 +176,7 @@ export your existing configuration: $ geth --your-favourite-flags dumpconfig ``` -*Note: This works only with `geth` v1.6.0 and above.* +_Note: This works only with `geth` v1.6.0 and above._ #### Docker quick start @@ -176,7 +190,7 @@ docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ ``` This will start `geth` in snap-sync mode with a DB memory allowance of 1GB just as the -above command does. It will also create a persistent volume in your home directory for +above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image. @@ -302,8 +316,8 @@ that other nodes can use to connect to it and exchange peer information. Make su replace the displayed IP address information (most probably `[::]`) with your externally accessible IP to get the actual `enode` URL. -*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less -recommended way.* +_Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less +recommended way._ #### Starting up your member nodes @@ -317,8 +331,8 @@ do also specify a custom `--datadir` flag. $ geth --datadir=path/to/custom/data/folder --bootnodes= ``` -*Note: Since your network will be completely cut off from the main and test networks, you'll -also need to configure a miner to process transactions and create new blocks for you.* +_Note: Since your network will be completely cut off from the main and test networks, you'll +also need to configure a miner to process transactions and create new blocks for you._ #### Running a private miner @@ -356,13 +370,13 @@ and merge procedures quick and simple. Please make sure your contributions adhere to our coding guidelines: - * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) - guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). - * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) - guidelines. - * Pull requests need to be based on and opened against the `master` branch. - * Commit messages should be prefixed with the package(s) they modify. - * E.g. "eth, rpc: make trace configs optional" +- Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) + guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). +- Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) + guidelines. +- Pull requests need to be based on and opened against the `master` branch. +- Commit messages should be prefixed with the package(s) they modify. + - E.g. "eth, rpc: make trace configs optional" Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide) for more details on configuring your environment, managing project dependencies, and diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 30565fda6185..7c1bb2dc5d94 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -18,10 +18,12 @@ package main import ( "bufio" + "context" "errors" "fmt" "os" "reflect" + "time" "unicode" "github.com/urfave/cli/v2" @@ -39,6 +41,12 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff" + dumpdb "github.com/ethereum/go-ethereum/statediff/indexer/database/dump" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/naoina/toml" ) @@ -149,6 +157,9 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { cfg.Ethstats.URL = ctx.String(utils.EthStatsURLFlag.Name) } applyMetricConfig(ctx, &cfg) + if ctx.Bool(utils.StateDiffFlag.Name) { + cfg.Eth.Diffing = true + } return stack, cfg } @@ -187,7 +198,105 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { // Configure log filter RPC API. filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) - // Configure GraphQL if requested. + if ctx.Bool(utils.StateDiffFlag.Name) { + var indexerConfig interfaces.Config + var clientName, nodeID string + if ctx.IsSet(utils.StateDiffWritingFlag.Name) { + clientName = ctx.String(utils.StateDiffDBClientNameFlag.Name) + if ctx.IsSet(utils.StateDiffDBNodeIDFlag.Name) { + nodeID = ctx.String(utils.StateDiffDBNodeIDFlag.Name) + } else { + utils.Fatalf("Must specify node ID for statediff DB output") + } + + dbTypeStr := ctx.String(utils.StateDiffDBTypeFlag.Name) + dbType, err := shared.ResolveDBType(dbTypeStr) + if err != nil { + utils.Fatalf("%v", err) + } + switch dbType { + case shared.FILE: + fileModeStr := ctx.String(utils.StateDiffFileMode.Name) + fileMode, err := file.ResolveFileMode(fileModeStr) + if err != nil { + utils.Fatalf("%v", err) + } + + indexerConfig = file.Config{ + Mode: fileMode, + OutputDir: ctx.String(utils.StateDiffFileCsvDir.Name), + FilePath: ctx.String(utils.StateDiffFilePath.Name), + WatchedAddressesFilePath: ctx.String(utils.StateDiffWatchedAddressesFilePath.Name), + } + case shared.POSTGRES: + driverTypeStr := ctx.String(utils.StateDiffDBDriverTypeFlag.Name) + driverType, err := postgres.ResolveDriverType(driverTypeStr) + if err != nil { + utils.Fatalf("%v", err) + } + pgConfig := postgres.Config{ + Hostname: ctx.String(utils.StateDiffDBHostFlag.Name), + Port: ctx.Int(utils.StateDiffDBPortFlag.Name), + DatabaseName: ctx.String(utils.StateDiffDBNameFlag.Name), + Username: ctx.String(utils.StateDiffDBUserFlag.Name), + Password: ctx.String(utils.StateDiffDBPasswordFlag.Name), + ID: nodeID, + ClientName: clientName, + Driver: driverType, + } + if ctx.IsSet(utils.StateDiffDBMinConns.Name) { + pgConfig.MinConns = ctx.Int(utils.StateDiffDBMinConns.Name) + } + if ctx.IsSet(utils.StateDiffDBMaxConns.Name) { + pgConfig.MaxConns = ctx.Int(utils.StateDiffDBMaxConns.Name) + } + if ctx.IsSet(utils.StateDiffDBMaxIdleConns.Name) { + pgConfig.MaxIdle = ctx.Int(utils.StateDiffDBMaxIdleConns.Name) + } + if ctx.IsSet(utils.StateDiffDBMaxConnLifetime.Name) { + pgConfig.MaxConnLifetime = time.Duration(ctx.Duration(utils.StateDiffDBMaxConnLifetime.Name).Seconds()) + } + if ctx.IsSet(utils.StateDiffDBMaxConnIdleTime.Name) { + pgConfig.MaxConnIdleTime = time.Duration(ctx.Duration(utils.StateDiffDBMaxConnIdleTime.Name).Seconds()) + } + if ctx.IsSet(utils.StateDiffDBConnTimeout.Name) { + pgConfig.ConnTimeout = time.Duration(ctx.Duration(utils.StateDiffDBConnTimeout.Name).Seconds()) + } + indexerConfig = pgConfig + case shared.DUMP: + dumpTypeStr := ctx.String(utils.StateDiffDBDumpDst.Name) + dumpType, err := dumpdb.ResolveDumpType(dumpTypeStr) + if err != nil { + utils.Fatalf("%v", err) + } + switch dumpType { + case dumpdb.STDERR: + indexerConfig = dumpdb.Config{Dump: os.Stdout} + case dumpdb.STDOUT: + indexerConfig = dumpdb.Config{Dump: os.Stderr} + case dumpdb.DISCARD: + indexerConfig = dumpdb.Config{Dump: dumpdb.NewDiscardWriterCloser()} + default: + utils.Fatalf("unrecognized dump destination: %s", dumpType) + } + default: + utils.Fatalf("unrecognized database type: %s", dbType) + } + } + p := statediff.Config{ + IndexerConfig: indexerConfig, + KnownGapsFilePath: ctx.String(utils.StateDiffKnownGapsFilePath.Name), + ID: nodeID, + ClientName: clientName, + Context: context.Background(), + EnableWriteLoop: ctx.Bool(utils.StateDiffWritingFlag.Name), + NumWorkers: ctx.Uint(utils.StateDiffWorkersFlag.Name), + WaitForSync: ctx.Bool(utils.StateDiffWaitForSync.Name), + } + utils.RegisterStateDiffService(stack, eth, &cfg.Eth, p, backend) + } + + // Configure GraphQL if requested if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node) } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index b9e3ed31e813..c267d122ce4c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -154,6 +154,31 @@ var ( utils.GpoIgnoreGasPriceFlag, utils.MinerNotifyFullFlag, utils.IgnoreLegacyReceiptsFlag, + utils.StateDiffFlag, + utils.StateDiffDBTypeFlag, + utils.StateDiffDBDriverTypeFlag, + utils.StateDiffDBDumpDst, + utils.StateDiffDBNameFlag, + utils.StateDiffDBPasswordFlag, + utils.StateDiffDBUserFlag, + utils.StateDiffDBHostFlag, + utils.StateDiffDBPortFlag, + utils.StateDiffDBMaxConnLifetime, + utils.StateDiffDBMaxConnIdleTime, + utils.StateDiffDBMaxConns, + utils.StateDiffDBMinConns, + utils.StateDiffDBMaxIdleConns, + utils.StateDiffDBConnTimeout, + utils.StateDiffDBNodeIDFlag, + utils.StateDiffDBClientNameFlag, + utils.StateDiffWritingFlag, + utils.StateDiffWorkersFlag, + utils.StateDiffFileMode, + utils.StateDiffFileCsvDir, + utils.StateDiffFilePath, + utils.StateDiffKnownGapsFilePath, + utils.StateDiffWaitForSync, + utils.StateDiffWatchedAddressesFilePath, configFileFlag, }, utils.NetworkFlags, utils.DatabasePathFlags) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 9e95193343a9..f7477c71097e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -66,6 +66,8 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" "github.com/urfave/cli/v2" @@ -977,6 +979,118 @@ var ( Value: metrics.DefaultConfig.InfluxDBOrganization, Category: flags.MetricsCategory, } + + StateDiffFlag = &cli.BoolFlag{ + Name: "statediff", + Usage: "Enables the processing of state diffs between each block", + Category: flags.MiscCategory, + } + StateDiffDBTypeFlag = &cli.StringFlag{ + Name: "statediff.db.type", + Usage: "Statediff database type (current options: postgres, file, dump)", + Value: "postgres", + } + StateDiffDBDriverTypeFlag = &cli.StringFlag{ + Name: "statediff.db.driver", + Usage: "Statediff database driver type", + Value: "pgx", + } + StateDiffDBDumpDst = &cli.StringFlag{ + Name: "statediff.dump.dst", + Usage: "Statediff database dump destination (default is stdout)", + Value: "stdout", + } + StateDiffDBHostFlag = &cli.StringFlag{ + Name: "statediff.db.host", + Usage: "Statediff database hostname/ip", + Value: "localhost", + } + StateDiffDBPortFlag = &cli.IntFlag{ + Name: "statediff.db.port", + Usage: "Statediff database port", + Value: 5432, + } + StateDiffDBNameFlag = &cli.StringFlag{ + Name: "statediff.db.name", + Usage: "Statediff database name", + } + StateDiffDBPasswordFlag = &cli.StringFlag{ + Name: "statediff.db.password", + Usage: "Statediff database password", + } + StateDiffDBUserFlag = &cli.StringFlag{ + Name: "statediff.db.user", + Usage: "Statediff database username", + Value: "postgres", + } + StateDiffDBMaxConnLifetime = &cli.DurationFlag{ + Name: "statediff.db.maxconnlifetime", + Usage: "Statediff database maximum connection lifetime (in seconds)", + } + StateDiffDBMaxConnIdleTime = &cli.DurationFlag{ + Name: "statediff.db.maxconnidletime", + Usage: "Statediff database maximum connection idle time (in seconds)", + } + StateDiffDBMaxConns = &cli.IntFlag{ + Name: "statediff.db.maxconns", + Usage: "Statediff database maximum connections", + } + StateDiffDBMinConns = &cli.IntFlag{ + Name: "statediff.db.minconns", + Usage: "Statediff database minimum connections", + } + StateDiffDBMaxIdleConns = &cli.IntFlag{ + Name: "statediff.db.maxidleconns", + Usage: "Statediff database maximum idle connections", + } + StateDiffDBConnTimeout = &cli.DurationFlag{ + Name: "statediff.db.conntimeout", + Usage: "Statediff database connection timeout (in seconds)", + } + StateDiffDBNodeIDFlag = &cli.StringFlag{ + Name: "statediff.db.nodeid", + Usage: "Node ID to use when writing state diffs to database", + } + StateDiffFileMode = &cli.StringFlag{ + Name: "statediff.file.mode", + Usage: "Statediff file writing mode (current options: csv, sql)", + Value: "csv", + } + StateDiffFileCsvDir = &cli.StringFlag{ + Name: "statediff.file.csvdir", + Usage: "Full path of output directory to write statediff data out to when operating in csv file mode", + } + StateDiffFilePath = &cli.StringFlag{ + Name: "statediff.file.path", + Usage: "Full path (including filename) to write statediff data out to when operating in sql file mode", + } + StateDiffKnownGapsFilePath = &cli.StringFlag{ + Name: "statediff.knowngapsfile.path", + Usage: "Full path (including filename) to write knownGaps statements when the DB is unavailable.", + Value: "./known_gaps.sql", + } + StateDiffWatchedAddressesFilePath = &cli.StringFlag{ + Name: "statediff.file.wapath", + Usage: "Full path (including filename) to write statediff watched addresses out to when operating in file mode", + } + StateDiffDBClientNameFlag = &cli.StringFlag{ + Name: "statediff.db.clientname", + Usage: "Client name to use when writing state diffs to database", + Value: "go-ethereum", + } + StateDiffWritingFlag = &cli.BoolFlag{ + Name: "statediff.writing", + Usage: "Activates progressive writing of state diffs to database as new block are synced", + } + StateDiffWorkersFlag = &cli.UintFlag{ + Name: "statediff.workers", + Usage: "Number of concurrent workers to use during statediff processing (default 1)", + Value: 1, + } + StateDiffWaitForSync = &cli.BoolFlag{ + Name: "statediff.waitforsync", + Usage: "Should the statediff service wait for geth to catch up to the head of the chain?", + } ) var ( @@ -1238,6 +1352,10 @@ func setWS(ctx *cli.Context, cfg *node.Config) { if ctx.IsSet(WSPathPrefixFlag.Name) { cfg.WSPathPrefix = ctx.String(WSPathPrefixFlag.Name) } + + if ctx.Bool(StateDiffFlag.Name) { + cfg.WSModules = append(cfg.WSModules, "statediff") + } } // setIPC creates an IPC path configuration from the set command line flags, @@ -2016,6 +2134,15 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend return backend.APIBackend, backend } +// RegisterLesEthService adds an Ethereum les client to the stack. +func RegisterLesEthService(stack *node.Node, cfg *eth.Config) *les.LightEthereum { + backend, err := les.New(stack, cfg) + if err != nil { + Fatalf("Failed to register the Ethereum service: %v", err) + } + return backend +} + // RegisterEthStatsService configures the Ethereum Stats daemon and adds it to the node. func RegisterEthStatsService(stack *node.Node, backend ethapi.Backend, url string) { if err := ethstats.New(stack, backend, backend.Engine(), url); err != nil { @@ -2044,6 +2171,13 @@ func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconf return filterSystem } +// RegisterStateDiffService configures and registers a service to stream state diff data over RPC +func RegisterStateDiffService(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params statediff.Config, backend ethapi.Backend) { + if err := statediff.New(stack, ethServ, cfg, params, backend); err != nil { + Fatalf("Failed to register the Statediff service: %v", err) + } +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/core/blockchain.go b/core/blockchain.go index a98c3b4dbeb3..cd215cce3abc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -27,6 +27,8 @@ import ( "sync/atomic" "time" + lru "github.com/hashicorp/golang-lru" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/common/prque" @@ -43,7 +45,6 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" - lru "github.com/hashicorp/golang-lru" ) var ( @@ -133,6 +134,7 @@ type CacheConfig struct { Preimages bool // Whether to store preimage of trie key to the disk SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it + StateDiffing bool // Whether or not the statediffing service is running } // defaultCacheConfig are the default caching values if none are specified by the @@ -213,6 +215,10 @@ type BlockChain struct { processor Processor // Block transaction processor interface forker *ForkChoice vmConfig vm.Config + + // Locked roots and their mutex + trieLock sync.Mutex + lockedRoots map[common.Hash]bool } // NewBlockChain returns a fully initialised block chain using information @@ -249,6 +255,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par futureBlocks: futureBlocks, engine: engine, vmConfig: vmConfig, + lockedRoots: make(map[common.Hash]bool), } bc.forker = NewForkChoice(bc, shouldPreserve) bc.validator = NewBlockValidator(chainConfig, bc, engine) @@ -887,7 +894,10 @@ func (bc *BlockChain) Stop() { } } for !bc.triegc.Empty() { - triedb.Dereference(bc.triegc.PopItem().(common.Hash)) + pruneRoot := bc.triegc.PopItem().(common.Hash) + if !bc.TrieLocked(pruneRoot) { + triedb.Dereference(pruneRoot) + } } if size, _ := triedb.Size(); size != 0 { log.Error("Dangling trie nodes after full cleanup") @@ -1284,6 +1294,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) + // If we are statediffing, lock the trie until the statediffing service is done using it + if bc.cacheConfig.StateDiffing { + bc.LockTrie(root) + } + if current := block.NumberU64(); current > TriesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( @@ -1322,7 +1337,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.triegc.Push(root, number) break } - triedb.Dereference(root.(common.Hash)) + pruneRoot := root.(common.Hash) + if !bc.TrieLocked(pruneRoot) { + log.Debug("Dereferencing", "root", root.(common.Hash).Hex()) + triedb.Dereference(pruneRoot) + } } } } @@ -2421,3 +2440,28 @@ func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Pro bc.validator = v bc.processor = p } + +// TrieLocked returns whether the trie associated with the provided root is locked for use +func (bc *BlockChain) TrieLocked(root common.Hash) bool { + bc.trieLock.Lock() + locked, ok := bc.lockedRoots[root] + bc.trieLock.Unlock() + if !ok { + return false + } + return locked +} + +// LockTrie prevents dereferencing of the provided root +func (bc *BlockChain) LockTrie(root common.Hash) { + bc.trieLock.Lock() + bc.lockedRoots[root] = true + bc.trieLock.Unlock() +} + +// UnlockTrie allows dereferencing of the provided root- provided it was previously locked +func (bc *BlockChain) UnlockTrie(root common.Hash) { + bc.trieLock.Lock() + bc.lockedRoots[root] = false + bc.trieLock.Unlock() +} diff --git a/core/types/receipt.go b/core/types/receipt.go index bdf48451473c..e42caf34e5a6 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -69,6 +69,7 @@ type Receipt struct { BlockHash common.Hash `json:"blockHash,omitempty"` BlockNumber *big.Int `json:"blockNumber,omitempty"` TransactionIndex uint `json:"transactionIndex"` + LogRoot common.Hash `json:"logRoot"` } type receiptMarshaling struct { @@ -135,6 +136,9 @@ func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt { // EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt // into an RLP stream. If no post state is present, byzantium fork is assumed. +// For a legacy Receipt this returns RLP([PostStateOrStatus, CumulativeGasUsed, Bloom, Logs]) +// For a EIP-2718 Receipt this returns RLP(TxType || ReceiptPayload) +// For a EIP-2930 Receipt, TxType == 0x01 and ReceiptPayload == RLP([PostStateOrStatus, CumulativeGasUsed, Bloom, Logs]) func (r *Receipt) EncodeRLP(w io.Writer) error { data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs} if r.Type == LegacyTxType { diff --git a/core/types/transaction.go b/core/types/transaction.go index 715ede15db2e..8bd7bac0b65c 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -88,6 +88,9 @@ type TxData interface { } // EncodeRLP implements rlp.Encoder +// For a legacy Transaction this returns RLP([AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, V, R, S]) +// For a EIP-2718 Transaction this returns RLP(TxType || TxPayload) +// For a EIP-2930 Transaction, TxType == 0x01 and TxPayload == RLP([ChainID, AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, AccessList, V, R, S] func (tx *Transaction) EncodeRLP(w io.Writer) error { if tx.Type() == LegacyTxType { return rlp.Encode(w, tx.inner) @@ -108,9 +111,10 @@ func (tx *Transaction) encodeTyped(w *bytes.Buffer) error { return rlp.Encode(w, tx.inner) } -// MarshalBinary returns the canonical encoding of the transaction. -// For legacy transactions, it returns the RLP encoding. For EIP-2718 typed -// transactions, it returns the type and payload. +// MarshalBinary returns the canonical consensus encoding of the transaction. +// For a legacy Transaction this returns RLP([AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, V, R, S]) +// For a EIP-2718 Transaction this returns TxType || TxPayload +// For a EIP-2930 Transaction, TxType == 0x01 and TxPayload == RLP([ChainID, AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, AccessList, V, R, S] func (tx *Transaction) MarshalBinary() ([]byte, error) { if tx.Type() == LegacyTxType { return rlp.EncodeToBytes(tx.inner) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000000..62a81b0aa70e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + migrations: + restart: on-failure + depends_on: + - ipld-eth-db + image: vulcanize/ipld-eth-db:v4.2.1-alpha + environment: + DATABASE_USER: "vdbm" + DATABASE_NAME: "vulcanize_testing" + DATABASE_PASSWORD: "password" + DATABASE_HOSTNAME: "ipld-eth-db" + DATABASE_PORT: 5432 + + ipld-eth-db: + image: timescale/timescaledb:latest-pg14 + restart: always + command: ["postgres", "-c", "log_statement=all"] + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_testing" + POSTGRES_PASSWORD: "password" + ports: + - "127.0.0.1:8077:5432" + volumes: + - ./statediff/indexer/database/file:/file_indexer diff --git a/eth/backend.go b/eth/backend.go index 778207636344..4d4c92d6f919 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -199,6 +199,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieTimeLimit: config.TrieTimeout, SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, + StateDiffing: config.Diffing, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 5690366421dc..08077e397c91 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -214,6 +214,10 @@ type Config struct { // OverrideTerminalTotalDifficultyPassed (TODO: remove after the fork) OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` + + // Signify whether or not we are producing statediffs + // If we are, do not dereference state roots until the statediffing service is done with them + Diffing bool } // CreateConsensusEngine creates a consensus engine for the given chain configuration. diff --git a/go.mod b/go.mod index 4a769c7a2dca..7d44b3eaac72 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,8 @@ require ( github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff + github.com/georgysavva/scany v0.2.9 + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-stack/stack v1.8.0 github.com/golang-jwt/jwt/v4 v4.3.0 github.com/golang/protobuf v1.5.2 @@ -37,26 +39,42 @@ require ( github.com/huin/goupnp v1.0.3 github.com/influxdata/influxdb v1.8.3 github.com/influxdata/influxdb-client-go/v2 v2.4.0 + github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect + github.com/ipfs/go-block-format v0.0.3 + github.com/ipfs/go-cid v0.2.0 + github.com/ipfs/go-ipfs-blockstore v1.2.0 + github.com/ipfs/go-ipfs-ds-help v1.1.0 + github.com/ipfs/go-ipld-format v0.4.0 + github.com/jackc/pgconn v1.10.0 + github.com/jackc/pgx/v4 v4.13.0 github.com/jackpal/go-nat-pmp v1.0.2 - github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e + github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b + github.com/jmoiron/sqlx v1.2.0 github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.2 + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lib/pq v1.10.6 github.com/mattn/go-colorable v0.1.8 github.com/mattn/go-isatty v0.0.12 + github.com/multiformats/go-multihash v0.1.0 + github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 - github.com/prometheus/tsdb v0.7.1 + github.com/pganalyze/pg_query_go/v2 v2.1.0 + github.com/prometheus/tsdb v0.10.0 github.com/rjeczalik/notify v0.9.1 github.com/rs/cors v1.7.0 - github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible + github.com/shirou/gopsutil v3.21.11+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 - github.com/stretchr/testify v1.7.2 - github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 + github.com/stretchr/testify v1.7.0 + github.com/supranational/blst v0.3.8 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + github.com/thoas/go-funk v0.9.2 + github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef github.com/urfave/cli/v2 v2.10.2 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a @@ -70,7 +88,6 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect - github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 // indirect @@ -82,27 +99,50 @@ require ( github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect - github.com/go-logfmt/logfmt v0.4.0 // indirect - github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect - github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect - github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/gogo/protobuf v1.3.1 // indirect + github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-datastore v0.5.0 // indirect + github.com/ipfs/go-ipfs-util v0.0.2 // indirect + github.com/ipfs/go-log v0.0.1 // indirect + github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.1.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.8.1 // indirect + github.com/jackc/puddle v1.1.3 // indirect + github.com/jbenet/goprocess v0.1.4 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect - github.com/naoina/go-stringutil v0.1.0 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.0.3 // indirect + github.com/multiformats/go-base36 v0.1.0 // indirect + github.com/multiformats/go-multibase v0.0.3 // indirect + github.com/multiformats/go-varint v0.0.6 // indirect github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/objx v0.2.0 // indirect github.com/tklauser/numcpus v0.2.2 // indirect + github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.uber.org/atomic v1.6.0 // indirect golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.1.6 // indirect + ) diff --git a/go.sum b/go.sum index 4b27867fbc79..ce34b8198778 100644 --- a/go.sum +++ b/go.sum @@ -28,9 +28,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -80,12 +79,19 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0 h1:gFqGlGl/5f9UGXAaKapCGUfaTCgRKKnzu2VvzMZlOFA= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cockroach-go/v2 v2.0.3 h1:ZA346ACHIZctef6trOTwBAEvPVm1k0uLm/bb2Atc+S8= +github.com/cockroachdb/cockroach-go/v2 v2.0.3/go.mod h1:hAuDgiVgDVkfirP9JnhXEfcXEPRKBpYdGz+l7mvYSzw= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f h1:C43yEtQ6NIf4ftFXD/V55gnGFgPbMQobd//YlnLjUJ8= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -101,6 +107,7 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeC github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -119,6 +126,7 @@ github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= @@ -127,13 +135,14 @@ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlK github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/georgysavva/scany v0.2.9 h1:Xt6rjYpHnMClTm/g+oZTnoSxUwiln5GqMNU+QeLNHQU= +github.com/georgysavva/scany v0.2.9/go.mod h1:yeOeC1BdIdl6hOwy8uefL2WNSlseFzbhlG/frrh65SA= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -144,24 +153,34 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -183,6 +202,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -195,6 +215,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -204,8 +225,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -216,10 +237,13 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -231,7 +255,6 @@ github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= @@ -247,12 +270,122 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= +github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= +github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= +github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-datastore v0.5.0 h1:rQicVCEacWyk4JZ6G5bD9TKR7lZEG1MWcG7UdWYrFAU= +github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= +github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= +github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= +github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= +github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= +github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78= +github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7JMeA= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U= -github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b h1:ZGiXF8sz7PDk6RgkP+A/SFfUD0ZR/AgG6SpRNEDKZy8= +github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b/go.mod h1:hQmNrgofl+IY/8L+n20H6E6PWBBTokdsv+q49j0QhsU= +github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -264,19 +397,26 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -288,15 +428,27 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= +github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -304,9 +456,18 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= +github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= @@ -314,35 +475,45 @@ github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8oh github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= +github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA= +github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -351,6 +522,8 @@ github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChl github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/pganalyze/pg_query_go/v2 v2.1.0 h1:donwPZ4G/X+kMs7j5eYtKjdziqyOLVp3pkUrzb9lDl8= +github.com/pganalyze/pg_query_go/v2 v2.1.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -365,32 +538,44 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= +github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v0.0.0-20200419222939-1884f454f8ea/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -398,24 +583,21 @@ github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57N github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 h1:m+8fKfQwCAy1QjzINvKe/pYtLjo2dl59x2w9YSEJxuY= -github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/supranational/blst v0.3.8 h1:glwLF4oBRSJOTr05lRBgNwGQST0ndP2wg29fSeTRKCY= +github.com/supranational/blst v0.3.8/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= -github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= -github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= -github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/thoas/go-funk v0.9.2 h1:oKlNYv0AY5nyf9g+/GhMgS/UO2ces0QRdPKwkhY3VCk= +github.com/thoas/go-funk v0.9.2/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -429,30 +611,57 @@ github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhA github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -474,6 +683,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= @@ -492,6 +702,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -500,6 +711,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -509,11 +721,10 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -533,19 +744,23 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -560,17 +775,15 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -591,6 +804,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -602,14 +816,18 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -617,11 +835,13 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191126055441-b0650ceb63d9/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -677,10 +897,12 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -703,5 +925,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= +lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/mobile/android_test.go b/mobile/android_test.go index 2ddf5d9d91ed..71ab8d9a4d3e 100644 --- a/mobile/android_test.go +++ b/mobile/android_test.go @@ -154,6 +154,7 @@ public class AndroidTest extends InstrumentationTestCase { // // This method has been adapted from golang.org/x/mobile/bind/java/seq_test.go/runTest func TestAndroid(t *testing.T) { + t.Skip("Skipping this test for statediff as this is not relevant") // Skip tests on Windows altogether if runtime.GOOS == "windows" { t.Skip("cannot test Android bindings on Windows, skipping") diff --git a/params/version.go b/params/version.go index 5f24b41f2852..234c4e339f60 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 23 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 23 // Patch version component of the current release + VersionMeta = "statediff-4.2.0-alpha" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/rpc/http.go b/rpc/http.go index 858d80858652..d2e3e6eb13db 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -31,7 +31,7 @@ import ( ) const ( - maxRequestContentLength = 1024 * 1024 * 5 + maxRequestContentLength = 1024 * 1024 * 12 contentType = "application/json" ) diff --git a/scripts/run_unit_test.sh b/scripts/run_unit_test.sh new file mode 100755 index 000000000000..b449090f678e --- /dev/null +++ b/scripts/run_unit_test.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +mkdir -p out + +# Remove existing docker-tsdb directory +rm -rf out/docker-tsdb/ + +# Copy over files to setup TimescaleDB +ID=$(docker create vulcanize/ipld-eth-db:v4.1.1-alpha) +docker cp $ID:/app/docker-tsdb out/docker-tsdb/ +docker rm -v $ID + +# Spin up TimescaleDB +docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml up ipld-eth-db +sleep 45 + +# Run unit tests +go clean -testcache +make statedifftest + +# Clean up +docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes +rm -rf out/docker-tsdb/ diff --git a/statediff/README.md b/statediff/README.md new file mode 100644 index 000000000000..f262d7a8ecb4 --- /dev/null +++ b/statediff/README.md @@ -0,0 +1,318 @@ +# Statediff + +This package provides an auxiliary service that asynchronously processes state diff objects from chain events, +either relaying the state objects to RPC subscribers or writing them directly to Postgres as IPLD objects. + +It also exposes RPC endpoints for fetching or writing to Postgres the state diff at a specific block height +or for a specific block hash, this operates on historical block and state data and so depends on a complete state archive. + +Data is emitted in this differential format in order to make it feasible to IPLD-ize and index the _entire_ Ethereum state +(including intermediate state and storage trie nodes). If this state diff process is ran continuously from genesis, +the entire state at any block can be materialized from the cumulative differentials up to that point. + +## Statediff object + +A state diff `StateObject` is the collection of all the state and storage trie nodes that have been updated in a given block. +For convenience, we also associate these nodes with the block number and hash, and optionally the set of code hashes and code for any +contracts deployed in this block. + +A complete state diff `StateObject` will include all state and storage intermediate nodes, which is necessary for generating proofs and for +traversing the tries. + +```go +// StateObject is a collection of state (and linked storage nodes) as well as the associated block number, block hash, +// and a set of code hashes and their code +type StateObject struct { + BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Nodes []StateNode `json:"nodes" gencodec:"required"` + CodeAndCodeHashes []CodeAndCodeHash `json:"codeMapping"` +} + +// StateNode holds the data for a single state diff node +type StateNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + StorageNodes []StorageNode `json:"storage"` + LeafKey []byte `json:"leafKey"` +} + +// StorageNode holds the data for a single storage diff node +type StorageNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + LeafKey []byte `json:"leafKey"` +} + +// CodeAndCodeHash struct for holding codehash => code mappings +// we can't use an actual map because they are not rlp serializable +type CodeAndCodeHash struct { + Hash common.Hash `json:"codeHash"` + Code []byte `json:"code"` +} +``` + +These objects are packed into a `Payload` structure which can additionally associate the `StateObject` +with the block (header, uncles, and transactions), receipts, and total difficulty. +This `Payload` encapsulates all of the differential data at a given block, and allows us to index the entire Ethereum data structure +as hash-linked IPLD objects. + +```go +// Payload packages the data to send to state diff subscriptions +type Payload struct { + BlockRlp []byte `json:"blockRlp"` + TotalDifficulty *big.Int `json:"totalDifficulty"` + ReceiptsRlp []byte `json:"receiptsRlp"` + StateObjectRlp []byte `json:"stateObjectRlp" gencodec:"required"` + + encoded []byte + err error +} +``` + +## Usage + +This state diffing service runs as an auxiliary service concurrent to the regular syncing process of the geth node. + +### CLI configuration + +This service introduces a CLI flag namespace `statediff` + +`--statediff` flag is used to turn on the service + +`--statediff.writing` is used to tell the service to write state diff objects it produces from synced ChainEvents directly to a configured Postgres database + +`--statediff.workers` is used to set the number of concurrent workers to process state diff objects and write them into the database + +`--statediff.db.type` is the type of database we write out to (current options: postgres, dump, file) + +`--statediff.dump.dst` is the destination to write to when operating in database dump mode (stdout, stderr, discard) + +`--statediff.db.driver` is the specific driver to use for the database (current options for postgres: pgx and sqlx) + +`--statediff.db.host` is the hostname/ip to dial to connect to the database + +`--statediff.db.port` is the port to dial to connect to the database + +`--statediff.db.name` is the name of the database to connect to + +`--statediff.db.user` is the user to connect to the database as + +`--statediff.db.password` is the password to use to connect to the database + +`--statediff.db.conntimeout` is the connection timeout (in seconds) + +`--statediff.db.maxconns` is the maximum number of database connections + +`--statediff.db.minconns` is the minimum number of database connections + +`--statediff.db.maxidleconns` is the maximum number of idle connections + +`--statediff.db.maxconnidletime` is the maximum lifetime for an idle connection (in seconds) + +`--statediff.db.maxconnlifetime` is the maximum lifetime for a connection (in seconds) + +`--statediff.db.nodeid` is the node id to use in the Postgres database + +`--statediff.db.clientname` is the client name to use in the Postgres database + +`--statediff.file.path` full path (including filename) to write statediff data out to when operating in file mode + +`--statediff.file.wapath` full path (including filename) to write statediff watched addresses out to when operating in file mode + +The service can only operate in full sync mode (`--syncmode=full`), but only the historical RPC endpoints require an archive node (`--gcmode=archive`) + +e.g. +`./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db.type=postgres --statediff.db.driver=sqlx --statediff.db.host=localhost --statediff.db.port=5432 --statediff.db.name=vulcanize_test --statediff.db.user=postgres --statediff.db.nodeid=nodeid --statediff.db.clientname=clientname` + +When operating in `--statediff.db.type=file` mode, the service will write SQL statements out to the file designated by +`--statediff.file.path`. Please note that it writes out SQL statements with all `ON CONFLICT` constraint checks dropped. +This is done so that we can scale out the production of the SQL statements horizontally, merge the separate SQL files produced, +de-duplicate using unix tools (`sort statediff.sql | uniq` or `sort -u statediff.sql`), bulk load using psql +(`psql db_name --set ON_ERROR_STOP=on -f statediff.sql`), and then add our primary and foreign key constraints and indexes +back afterwards. + +### RPC endpoints + +The state diffing service exposes both a WS subscription endpoint, and a number of HTTP unary endpoints. + +Each of these endpoints requires a set of parameters provided by the caller + +```go +// Params is used to carry in parameters from subscribing/requesting clients configuration +type Params struct { + IntermediateStateNodes bool + IntermediateStorageNodes bool + IncludeBlock bool + IncludeReceipts bool + IncludeTD bool + IncludeCode bool + WatchedAddresses []common.Address +} +``` + +Using these params we can tell the service whether to include state and/or storage intermediate nodes; whether +to include the associated block (header, uncles, and transactions); whether to include the associated receipts; +whether to include the total difficulty for this block; whether to include the set of code hashes and code for +contracts deployed in this block; whether to limit the diffing process to a list of specific addresses. + +#### Subscription endpoint + +A websocket supporting RPC endpoint is exposed for subscribing to state diff `StateObjects` that come off the head of the chain while the geth node syncs. + +```go +// Stream is a subscription endpoint that fires off state diff payloads as they are created +Stream(ctx context.Context, params Params) (*rpc.Subscription, error) +``` + +To expose this endpoint the node needs to have the websocket server turned on (`--ws`), +and the `statediff` namespace exposed (`--ws.api=statediff`). + +Go code subscriptions to this endpoint can be created using the `rpc.Client.Subscribe()` method, +with the "statediff" namespace, a `statediff.Payload` channel, and the name of the statediff api's rpc method: "stream". + +e.g. + +```go + +cli, err := rpc.Dial("ipcPathOrWsURL") +if err != nil { + // handle error +} +stateDiffPayloadChan := make(chan statediff.Payload, 20000) +methodName := "stream" +params := statediff.Params{ + IncludeBlock: true, + IncludeTD: true, + IncludeReceipts: true, + IntermediateStorageNodes: true, + IntermediateStateNodes: true, +} +rpcSub, err := cli.Subscribe(context.Background(), statediff.APIName, stateDiffPayloadChan, methodName, params) +if err != nil { + // handle error +} +for { + select { + case stateDiffPayload := <- stateDiffPayloadChan: + // process the payload + case err := <- rpcSub.Err(): + // handle rpc subscription error + } +} +``` + +#### Unary endpoints + +The service also exposes unary RPC endpoints for retrieving the state diff `StateObject` for a specific block height/hash. + +```go +// StateDiffAt returns a state diff payload at the specific blockheight +StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) + +// StateDiffFor returns a state diff payload for the specific blockhash +StateDiffFor(ctx context.Context, blockHash common.Hash, params Params) (*Payload, error) +``` + +To expose this endpoint the node needs to have the HTTP server turned on (`--http`), +and the `statediff` namespace exposed (`--http.api=statediff`). + +### Direct indexing into Postgres + +If `--statediff.writing` is set, the service will convert the state diff `StateObject` data into IPLD objects, persist them directly to Postgres, +and generate secondary indexes around the IPLD data. + +The schema and migrations for this Postgres database are provided in `statediff/db/`. + +#### Postgres setup + +We use [pressly/goose](https://github.com/pressly/goose) as our Postgres migration manager. +You can also load the Postgres schema directly into a database using + +`psql database_name < schema.sql` + +This will only work on a version 12.4 Postgres database. + +#### Schema overview + +Our Postgres schemas are built around a single IPFS backing Postgres IPLD blockstore table (`public.blocks`) that conforms with [go-ds-sql](https://github.com/ipfs/go-ds-sql/blob/master/postgres/postgres.go). +All IPLD objects are stored in this table, where `key` is the blockstore-prefixed multihash key for the IPLD object and `data` contains +the bytes for the IPLD block (in the case of all Ethereum IPLDs, this is the RLP byte encoding of the Ethereum object). + +The IPLD objects in this table can be traversed using an IPLD DAG interface, but since this table only maps multihash to raw IPLD object +it is not particularly useful for searching through the data by looking up Ethereum objects by their constituent fields +(e.g. by block number, tx source/recipient, state/storage trie node path). To improve the accessibility of these objects +we create an Ethereum [advanced data layout](https://github.com/ipld/specs#schemas-and-advanced-data-layouts) (ADL) by generating secondary +indexes on top of the raw IPLDs in other Postgres tables. + +These secondary index tables fall under the `eth` schema and follow an `{objectType}_cids` naming convention. +These tables provide a view into individual fields of the underlying Ethereum IPLD objects, allowing lookups on these fields, and reference the raw IPLD objects stored in `public.blocks` +by foreign keys to their multihash keys. +Additionally, these tables maintain the hash-linked nature of Ethereum objects to one another. E.g. a storage trie node entry in the `storage_cids` +table contains a `state_id` foreign key which references the `id` for the `state_cids` entry that contains the state leaf node for the contract that storage node belongs to, +and in turn that `state_cids` entry contains a `header_id` foreign key which references the `id` of the `header_cids` entry that contains the header for the block these state and storage nodes were updated (diffed). + +### Optimization + +On mainnet this process is extremely IO intensive and requires significant resources to allow it to keep up with the head of the chain. +The state diff processing time for a specific block is dependent on the number and complexity of the state changes that occur in a block and +the number of updated state nodes that are available in the in-memory cache vs must be retrieved from disc. + +If memory permits, one means of improving the efficiency of this process is to increase the in-memory trie cache allocation. +This can be done by increasing the overall `--cache` allocation and/or by increasing the % of the cache allocated to trie +usage with `--cache.trie`. + +## Versioning, Branches, Rebasing, and Releasing + +Internal tagged releases are maintained for building the latest version of statediffing geth or using it as a go mod dependency. +When a new core go-ethereum version is released, statediffing geth is rebased onto and adjusted to work with the new tag. + +We want to maintain a complete record of our git history, but in order to make frequent and timely rebases feasible we also +need to be able to squash our work before performing a rebase. To this end we retain multiple branches with partial incremental history that culminate in +the full incremental history. + +### Versioning + +Example: `v1.10.16-statediff-3.0.2` + +- The first section, `v1.10.16`, corresponds to the release of the root branch this version is rebased onto (e.g., [](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16)[https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16)) +- The second section, `3.0.2`, corresponds to the version of our statediffing code. The major version here (3) should always correspond with the major version of the `ipld-eth-db` schema version it works with (e.g., [](https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6)[https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6](https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6)); it is only bumped when we bump the major version of the schema. + - The major version of the schema is only bumped when a breaking change is made to the schema. + - The minor version is bumped when a new feature is added, or a fix is performed that breaks or updates the statediffing API or CLI in some way. + - The patch version is bumped whenever minor fixes/patches/features are done that don’t change/break API/CLI compatibility. +- We are very strict about the first section and the major version of the statediffing code, but some discretion is required when deciding to bump minor versus patch version of the statediffing code. + +The statediff version is included in the `VersionMeta` in params/version.go + +### Branches + +We maintain two official kinds of branches: + +Major Branch: `{Root Version}-statediff` +Major branches retain the cumulative state of all changes made before the latest root version rebase and track the full incremental history of changes made between the latest root version rebase and the next. +Aside from creating the branch by performing the rebase described in the section below, these branches are never worked off of or committed to directly. + +Feature Branch: `{Root Version}-statediff-{Statediff Version}` +Feature branches are checked out from a major branch in order to work on a new feature or fix for the statediffing code. +The statediff version of a feature branch is the new version it affects on the major branch when merged. Internal tagged releases +are cut against these branches after they are merged back to the major branch. + +If a developer is unsure what version their patch should affect, they should remain working on an unofficial branch. From there +they can open a PR against the targeted root branch and be directed to the appropriate feature version and branch. + +### Rebasing + +When a new root tagged release comes out we rebase our statediffing code on top of the new tag using the following process: + +1. Checkout a new major branch for the tag from the current major branch +2. On the new major branch, squash all our commits since the last major rebase +3. On the new major branch, perform the rebase against the new tag +4. Push the new major branch to the remote +5. From the new major branch, checkout a new feature branch based on the new major version and the last statediff version +6. On this new feature branch, add the new major branch to the .github/workflows/on-master.yml list of "on push" branches +7. On this new feature branch, make any fixes/adjustments required for all statediffing geth tests to pass +8. PR this feature branch into the new major branch, this PR will trigger CI tests and builds. +9. After merging PR, rebase feature branch onto major branch +10. Cut a new release targeting the feature branch, this release should have the new root version but the same statediff version as the last release diff --git a/statediff/api.go b/statediff/api.go new file mode 100644 index 000000000000..0a7c5bba8cc7 --- /dev/null +++ b/statediff/api.go @@ -0,0 +1,156 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "context" + + "github.com/ethereum/go-ethereum/statediff/types" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +// APIName is the namespace used for the state diffing service API +const APIName = "statediff" + +// APIVersion is the version of the state diffing service API +const APIVersion = "0.0.1" + +// PublicStateDiffAPI provides an RPC subscription interface +// that can be used to stream out state diffs as they +// are produced by a full node +type PublicStateDiffAPI struct { + sds IService +} + +// NewPublicStateDiffAPI creates an rpc subscription interface for the underlying statediff service +func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { + return &PublicStateDiffAPI{ + sds: sds, + } +} + +// Stream is the public method to setup a subscription that fires off statediff service payloads as they are created +func (api *PublicStateDiffAPI) Stream(ctx context.Context, params Params) (*rpc.Subscription, error) { + // ensure that the RPC connection supports subscriptions + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return nil, rpc.ErrNotificationsUnsupported + } + + // create subscription and start waiting for events + rpcSub := notifier.CreateSubscription() + + go func() { + // subscribe to events from the statediff service + payloadChannel := make(chan Payload, chainEventChanSize) + quitChan := make(chan bool, 1) + api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan, params) + // loop and await payloads and relay them to the subscriber with the notifier + for { + select { + case payload := <-payloadChannel: + if err := notifier.Notify(rpcSub.ID, payload); err != nil { + log.Error("Failed to send state diff packet; error: " + err.Error()) + if err := api.sds.Unsubscribe(rpcSub.ID); err != nil { + log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error()) + } + return + } + case err := <-rpcSub.Err(): + if err != nil { + log.Error("State diff service rpcSub error: " + err.Error()) + err = api.sds.Unsubscribe(rpcSub.ID) + if err != nil { + log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error()) + } + return + } + case <-quitChan: + // don't need to unsubscribe, service does so before sending the quit signal + return + } + } + }() + + return rpcSub, nil +} + +// StateDiffAt returns a state diff payload at the specific blockheight +func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) { + return api.sds.StateDiffAt(blockNumber, params) +} + +// StateDiffFor returns a state diff payload for the specific blockhash +func (api *PublicStateDiffAPI) StateDiffFor(ctx context.Context, blockHash common.Hash, params Params) (*Payload, error) { + return api.sds.StateDiffFor(blockHash, params) +} + +// StateTrieAt returns a state trie payload at the specific blockheight +func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) { + return api.sds.StateTrieAt(blockNumber, params) +} + +// StreamCodeAndCodeHash writes all of the codehash=>code pairs out to a websocket channel +func (api *PublicStateDiffAPI) StreamCodeAndCodeHash(ctx context.Context, blockNumber uint64) (*rpc.Subscription, error) { + // ensure that the RPC connection supports subscriptions + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return nil, rpc.ErrNotificationsUnsupported + } + + // create subscription and start waiting for events + rpcSub := notifier.CreateSubscription() + payloadChan := make(chan types.CodeAndCodeHash, chainEventChanSize) + quitChan := make(chan bool) + api.sds.StreamCodeAndCodeHash(blockNumber, payloadChan, quitChan) + go func() { + for { + select { + case payload := <-payloadChan: + if err := notifier.Notify(rpcSub.ID, payload); err != nil { + log.Error("Failed to send code and codehash packet", "err", err) + return + } + case err := <-rpcSub.Err(): + log.Error("State diff service rpcSub error", "err", err) + return + case <-quitChan: + return + } + } + }() + + return rpcSub, nil +} + +// WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight +func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params Params) error { + return api.sds.WriteStateDiffAt(blockNumber, params) +} + +// WriteStateDiffFor writes a state diff object directly to DB for the specific block hash +func (api *PublicStateDiffAPI) WriteStateDiffFor(ctx context.Context, blockHash common.Hash, params Params) error { + return api.sds.WriteStateDiffFor(blockHash, params) +} + +// WatchAddress changes the list of watched addresses to which the direct indexing is restricted according to given operation +func (api *PublicStateDiffAPI) WatchAddress(operation types.OperationType, args []types.WatchAddressArg) error { + return api.sds.WatchAddress(operation, args) +} diff --git a/statediff/builder.go b/statediff/builder.go new file mode 100644 index 000000000000..c3ecbf11794c --- /dev/null +++ b/statediff/builder.go @@ -0,0 +1,892 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/trie_helpers" + types2 "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" +) + +var ( + nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") + emptyNode, _ = rlp.EncodeToBytes(&[]byte{}) + emptyContractRoot = crypto.Keccak256Hash(emptyNode) + nullCodeHash = crypto.Keccak256Hash([]byte{}).Bytes() +) + +// Builder interface exposes the method for building a state diff between two blocks +type Builder interface { + BuildStateDiffObject(args Args, params Params) (types2.StateObject, error) + BuildStateTrieObject(current *types.Block) (types2.StateObject, error) + WriteStateDiffObject(args types2.StateRoots, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error +} + +type StateDiffBuilder struct { + StateCache state.Database +} + +type IterPair struct { + Older, Newer trie.NodeIterator +} + +// convenience +func StateNodeAppender(nodes *[]types2.StateNode) types2.StateNodeSink { + return func(node types2.StateNode) error { + *nodes = append(*nodes, node) + return nil + } +} +func StorageNodeAppender(nodes *[]types2.StorageNode) types2.StorageNodeSink { + return func(node types2.StorageNode) error { + *nodes = append(*nodes, node) + return nil + } +} +func CodeMappingAppender(codeAndCodeHashes *[]types2.CodeAndCodeHash) types2.CodeSink { + return func(c types2.CodeAndCodeHash) error { + *codeAndCodeHashes = append(*codeAndCodeHashes, c) + return nil + } +} + +// NewBuilder is used to create a statediff builder +func NewBuilder(stateCache state.Database) Builder { + return &StateDiffBuilder{ + StateCache: stateCache, // state cache is safe for concurrent reads + } +} + +// BuildStateTrieObject builds a state trie object from the provided block +func (sdb *StateDiffBuilder) BuildStateTrieObject(current *types.Block) (types2.StateObject, error) { + currentTrie, err := sdb.StateCache.OpenTrie(current.Root()) + if err != nil { + return types2.StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err) + } + it := currentTrie.NodeIterator([]byte{}) + stateNodes, codeAndCodeHashes, err := sdb.buildStateTrie(it) + if err != nil { + return types2.StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err) + } + return types2.StateObject{ + BlockNumber: current.Number(), + BlockHash: current.Hash(), + Nodes: stateNodes, + CodeAndCodeHashes: codeAndCodeHashes, + }, nil +} + +func (sdb *StateDiffBuilder) buildStateTrie(it trie.NodeIterator) ([]types2.StateNode, []types2.CodeAndCodeHash, error) { + stateNodes := make([]types2.StateNode, 0) + codeAndCodeHashes := make([]types2.CodeAndCodeHash, 0) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) + if err != nil { + return nil, nil, err + } + switch node.NodeType { + case types2.Leaf: + var account types.StateAccount + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + node.LeafKey = leafKey + if !bytes.Equal(account.CodeHash, nullCodeHash) { + var storageNodes []types2.StorageNode + err := sdb.buildStorageNodesEventual(account.Root, true, StorageNodeAppender(&storageNodes)) + if err != nil { + return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err) + } + node.StorageNodes = storageNodes + // emit codehash => code mappings for cod + codeHash := common.BytesToHash(account.CodeHash) + code, err := sdb.StateCache.ContractCode(common.Hash{}, codeHash) + if err != nil { + return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) + } + codeAndCodeHashes = append(codeAndCodeHashes, types2.CodeAndCodeHash{ + Hash: codeHash, + Code: code, + }) + } + stateNodes = append(stateNodes, node) + case types2.Extension, types2.Branch: + stateNodes = append(stateNodes, node) + default: + return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return stateNodes, codeAndCodeHashes, it.Error() +} + +// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters +func (sdb *StateDiffBuilder) BuildStateDiffObject(args Args, params Params) (types2.StateObject, error) { + var stateNodes []types2.StateNode + var codeAndCodeHashes []types2.CodeAndCodeHash + err := sdb.WriteStateDiffObject( + types2.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot}, + params, StateNodeAppender(&stateNodes), CodeMappingAppender(&codeAndCodeHashes)) + if err != nil { + return types2.StateObject{}, err + } + return types2.StateObject{ + BlockHash: args.BlockHash, + BlockNumber: args.BlockNumber, + Nodes: stateNodes, + CodeAndCodeHashes: codeAndCodeHashes, + }, nil +} + +// WriteStateDiffObject writes a statediff object to output callback +func (sdb *StateDiffBuilder) WriteStateDiffObject(args types2.StateRoots, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error { + // Load tries for old and new states + oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot) + if err != nil { + return fmt.Errorf("error creating trie for oldStateRoot: %v", err) + } + newTrie, err := sdb.StateCache.OpenTrie(args.NewStateRoot) + if err != nil { + return fmt.Errorf("error creating trie for newStateRoot: %v", err) + } + + // we do two state trie iterations: + // one for new/updated nodes, + // one for deleted/updated nodes; + // prepare 2 iterator instances for each task + iterPairs := []IterPair{ + { + Older: oldTrie.NodeIterator([]byte{}), + Newer: newTrie.NodeIterator([]byte{}), + }, + { + Older: oldTrie.NodeIterator([]byte{}), + Newer: newTrie.NodeIterator([]byte{}), + }, + } + + if !params.IntermediateStateNodes { + return sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs, params, output, codeOutput) + } else { + return sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs, params, output, codeOutput) + } +} + +func (sdb *StateDiffBuilder) BuildStateDiffWithIntermediateStateNodes(iterPairs []IterPair, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error { + // collect a slice of all the nodes that were touched and exist at B (B-A) + // a map of their leafkey to all the accounts that were touched and exist at B + // and a slice of all the paths for the nodes in both of the above sets + diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStateWithIntermediateNodes( + iterPairs[0].Older, iterPairs[0].Newer, params.watchedAddressesLeafPaths, output) + if err != nil { + return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) + } + + // collect a slice of all the nodes that existed at a path in A that doesn't exist in B + // a map of their leafkey to all the accounts that were touched and exist at A + diffAccountsAtA, err := sdb.deletedOrUpdatedState( + iterPairs[1].Older, iterPairs[1].Newer, + diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafPaths, + params.IntermediateStateNodes, params.IntermediateStorageNodes, output) + if err != nil { + return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) + } + + // collect and sort the leafkey keys for both account mappings into a slice + createKeys := trie_helpers.SortKeys(diffAccountsAtB) + deleteKeys := trie_helpers.SortKeys(diffAccountsAtA) + + // and then find the intersection of these keys + // these are the leafkeys for the accounts which exist at both A and B but are different + // this also mutates the passed in createKeys and deleteKeys, removing the intersection keys + // and leaving the truly created or deleted keys in place + updatedKeys := trie_helpers.FindIntersection(createKeys, deleteKeys) + + // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two + err = sdb.buildAccountUpdates( + diffAccountsAtB, diffAccountsAtA, updatedKeys, + params.IntermediateStorageNodes, output) + if err != nil { + return fmt.Errorf("error building diff for updated accounts: %v", err) + } + // build the diff nodes for created accounts + err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput) + if err != nil { + return fmt.Errorf("error building diff for created accounts: %v", err) + } + return nil +} + +func (sdb *StateDiffBuilder) BuildStateDiffWithoutIntermediateStateNodes(iterPairs []IterPair, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error { + // collect a map of their leafkey to all the accounts that were touched and exist at B + // and a slice of all the paths for the nodes in both of the above sets + diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState( + iterPairs[0].Older, iterPairs[0].Newer, + params.watchedAddressesLeafPaths) + if err != nil { + return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) + } + + // collect a slice of all the nodes that existed at a path in A that doesn't exist in B + // a map of their leafkey to all the accounts that were touched and exist at A + diffAccountsAtA, err := sdb.deletedOrUpdatedState( + iterPairs[1].Older, iterPairs[1].Newer, + diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafPaths, + params.IntermediateStateNodes, params.IntermediateStorageNodes, output) + if err != nil { + return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) + } + + // collect and sort the leafkeys for both account mappings into a slice + createKeys := trie_helpers.SortKeys(diffAccountsAtB) + deleteKeys := trie_helpers.SortKeys(diffAccountsAtA) + + // and then find the intersection of these keys + // these are the leafkeys for the accounts which exist at both A and B but are different + // this also mutates the passed in createKeys and deleteKeys, removing in intersection keys + // and leaving the truly created or deleted keys in place + updatedKeys := trie_helpers.FindIntersection(createKeys, deleteKeys) + + // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two + err = sdb.buildAccountUpdates( + diffAccountsAtB, diffAccountsAtA, updatedKeys, + params.IntermediateStorageNodes, output) + if err != nil { + return fmt.Errorf("error building diff for updated accounts: %v", err) + } + // build the diff nodes for created accounts + err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput) + if err != nil { + return fmt.Errorf("error building diff for created accounts: %v", err) + } + return nil +} + +// createdAndUpdatedState returns +// a mapping of their leafkeys to all the accounts that exist in a different state at B than A +// and a slice of the paths for all of the nodes included in both +func (sdb *StateDiffBuilder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddressesLeafPaths [][]byte) (types2.AccountMap, map[string]bool, error) { + diffPathsAtB := make(map[string]bool) + diffAccountsAtB := make(types2.AccountMap) + watchingAddresses := len(watchedAddressesLeafPaths) > 0 + + it, _ := trie.NewDifferenceIterator(a, b) + for it.Next(true) { + // ignore node if it is not along paths of interest + if watchingAddresses && !isValidPrefixPath(watchedAddressesLeafPaths, it.Path()) { + continue + } + + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) + if err != nil { + return nil, nil, err + } + if node.NodeType == types2.Leaf { + // created vs updated is important for leaf nodes since we need to diff their storage + // so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey + var account types.StateAccount + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + + // ignore leaf node if it is not a watched address + if !isWatchedAddress(watchedAddressesLeafPaths, valueNodePath) { + continue + } + + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + diffAccountsAtB[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + Account: &account, + } + } + // add both intermediate and leaf node paths to the list of diffPathsAtB + diffPathsAtB[common.Bytes2Hex(node.Path)] = true + } + return diffAccountsAtB, diffPathsAtB, it.Error() +} + +// createdAndUpdatedStateWithIntermediateNodes returns +// a slice of all the intermediate nodes that exist in a different state at B than A +// a mapping of their leafkeys to all the accounts that exist in a different state at B than A +// and a slice of the paths for all of the nodes included in both +func (sdb *StateDiffBuilder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, watchedAddressesLeafPaths [][]byte, output types2.StateNodeSink) (types2.AccountMap, map[string]bool, error) { + diffPathsAtB := make(map[string]bool) + diffAccountsAtB := make(types2.AccountMap) + watchingAddresses := len(watchedAddressesLeafPaths) > 0 + + it, _ := trie.NewDifferenceIterator(a, b) + for it.Next(true) { + // ignore node if it is not along paths of interest + if watchingAddresses && !isValidPrefixPath(watchedAddressesLeafPaths, it.Path()) { + continue + } + + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) + if err != nil { + return nil, nil, err + } + switch node.NodeType { + case types2.Leaf: + // created vs updated is important for leaf nodes since we need to diff their storage + // so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey + var account types.StateAccount + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + + // ignore leaf node if it is not a watched address + if !isWatchedAddress(watchedAddressesLeafPaths, valueNodePath) { + continue + } + + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + diffAccountsAtB[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + Account: &account, + } + case types2.Extension, types2.Branch: + // create a diff for any intermediate node that has changed at b + // created vs updated makes no difference for intermediate nodes since we do not need to diff storage + if err := output(types2.StateNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + }); err != nil { + return nil, nil, err + } + default: + return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType) + } + // add both intermediate and leaf node paths to the list of diffPathsAtB + diffPathsAtB[common.Bytes2Hex(node.Path)] = true + } + return diffAccountsAtB, diffPathsAtB, it.Error() +} + +// deletedOrUpdatedState returns a slice of all the pathes that are emptied at B +// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B +func (sdb *StateDiffBuilder) deletedOrUpdatedState(a, b trie.NodeIterator, diffAccountsAtB types2.AccountMap, diffPathsAtB map[string]bool, watchedAddressesLeafPaths [][]byte, intermediateStateNodes, intermediateStorageNodes bool, output types2.StateNodeSink) (types2.AccountMap, error) { + diffAccountAtA := make(types2.AccountMap) + watchingAddresses := len(watchedAddressesLeafPaths) > 0 + + it, _ := trie.NewDifferenceIterator(b, a) + for it.Next(true) { + // ignore node if it is not along paths of interest + if watchingAddresses && !isValidPrefixPath(watchedAddressesLeafPaths, it.Path()) { + continue + } + + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) + if err != nil { + return nil, err + } + switch node.NodeType { + case types2.Leaf: + // map all different accounts at A to their leafkey + var account types.StateAccount + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + + // ignore leaf node if it is not a watched address + if !isWatchedAddress(watchedAddressesLeafPaths, valueNodePath) { + continue + } + + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + diffAccountAtA[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + Account: &account, + } + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted (or moved) in B + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + var diff types2.StateNode + // if this node's leaf key also did not show up in diffAccountsAtB + // that means the node was deleted + // in that case, emit an empty "removed" diff state node + // include empty "removed" diff storage nodes for all the storage slots + if _, ok := diffAccountsAtB[common.Bytes2Hex(leafKey)]; !ok { + diff = types2.StateNode{ + NodeType: types2.Removed, + Path: node.Path, + LeafKey: leafKey, + NodeValue: []byte{}, + } + + var storageDiffs []types2.StorageNode + err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, StorageNodeAppender(&storageDiffs)) + if err != nil { + return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err) + } + diff.StorageNodes = storageDiffs + } else { + // emit an empty "removed" diff with empty leaf key if the account was moved + diff = types2.StateNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + } + } + + if err := output(diff); err != nil { + return nil, err + } + } + case types2.Extension, types2.Branch: + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted (or moved) in B + // emit an empty "removed" diff to signify as such + if intermediateStateNodes { + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + if err := output(types2.StateNode{ + Path: node.Path, + NodeValue: []byte{}, + NodeType: types2.Removed, + }); err != nil { + return nil, err + } + } + } + // fall through, we did everything we need to do with these node types + default: + return nil, fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return diffAccountAtA, it.Error() +} + +// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys +// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states +// needs to be called before building account creations and deletions as this mutates +// those account maps to remove the accounts which were updated +func (sdb *StateDiffBuilder) buildAccountUpdates(creations, deletions types2.AccountMap, updatedKeys []string, intermediateStorageNodes bool, output types2.StateNodeSink) error { + var err error + for _, key := range updatedKeys { + createdAcc := creations[key] + deletedAcc := deletions[key] + var storageDiffs []types2.StorageNode + if deletedAcc.Account != nil && createdAcc.Account != nil { + oldSR := deletedAcc.Account.Root + newSR := createdAcc.Account.Root + err = sdb.buildStorageNodesIncremental( + oldSR, newSR, intermediateStorageNodes, + StorageNodeAppender(&storageDiffs)) + if err != nil { + return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err) + } + } + if err = output(types2.StateNode{ + NodeType: createdAcc.NodeType, + Path: createdAcc.Path, + NodeValue: createdAcc.NodeValue, + LeafKey: createdAcc.LeafKey, + StorageNodes: storageDiffs, + }); err != nil { + return err + } + delete(creations, key) + delete(deletions, key) + } + + return nil +} + +// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A +// it also returns the code and codehash for created contract accounts +func (sdb *StateDiffBuilder) buildAccountCreations(accounts types2.AccountMap, intermediateStorageNodes bool, output types2.StateNodeSink, codeOutput types2.CodeSink) error { + for _, val := range accounts { + diff := types2.StateNode{ + NodeType: val.NodeType, + Path: val.Path, + LeafKey: val.LeafKey, + NodeValue: val.NodeValue, + } + if !bytes.Equal(val.Account.CodeHash, nullCodeHash) { + // For contract creations, any storage node contained is a diff + var storageDiffs []types2.StorageNode + err := sdb.buildStorageNodesEventual(val.Account.Root, intermediateStorageNodes, StorageNodeAppender(&storageDiffs)) + if err != nil { + return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err) + } + diff.StorageNodes = storageDiffs + // emit codehash => code mappings for cod + codeHash := common.BytesToHash(val.Account.CodeHash) + code, err := sdb.StateCache.ContractCode(common.Hash{}, codeHash) + if err != nil { + return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) + } + if err := codeOutput(types2.CodeAndCodeHash{ + Hash: codeHash, + Code: code, + }); err != nil { + return err + } + } + if err := output(diff); err != nil { + return err + } + } + + return nil +} + +// buildStorageNodesEventual builds the storage diff node objects for a created account +// i.e. it returns all the storage nodes at this state, since there is no previous state +func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { + if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { + return nil + } + log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) + sTrie, err := sdb.StateCache.OpenTrie(sr) + if err != nil { + log.Info("error in build storage diff eventual", "error", err) + return err + } + it := sTrie.NodeIterator(make([]byte, 0)) + err = sdb.buildStorageNodesFromTrie(it, intermediateNodes, output) + if err != nil { + return err + } + return nil +} + +// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator +// including intermediate nodes can be turned on or off +func (sdb *StateDiffBuilder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) + if err != nil { + return err + } + switch node.NodeType { + case types2.Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if err := output(types2.StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + }); err != nil { + return err + } + case types2.Extension, types2.Branch: + if intermediateNodes { + if err := output(types2.StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + }); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return it.Error() +} + +// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account +func (sdb *StateDiffBuilder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { + if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { + return nil + } + log.Debug("Storage Root For Removed Diffs", "root", sr.Hex()) + sTrie, err := sdb.StateCache.OpenTrie(sr) + if err != nil { + log.Info("error in build removed account storage diffs", "error", err) + return err + } + it := sTrie.NodeIterator(make([]byte, 0)) + err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output) + if err != nil { + return err + } + return nil +} + +// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator +// including intermediate nodes can be turned on or off +func (sdb *StateDiffBuilder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) + if err != nil { + return err + } + switch node.NodeType { + case types2.Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + LeafKey: leafKey, + }); err != nil { + return err + } + case types2.Extension, types2.Branch: + if intermediateNodes { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } + } + default: + return fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return it.Error() +} + +// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A +func (sdb *StateDiffBuilder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { + if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) { + return nil + } + log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) + oldTrie, err := sdb.StateCache.OpenTrie(oldSR) + if err != nil { + return err + } + newTrie, err := sdb.StateCache.OpenTrie(newSR) + if err != nil { + return err + } + + diffSlotsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStorage( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + intermediateNodes, output) + if err != nil { + return err + } + err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + diffSlotsAtB, diffPathsAtB, intermediateNodes, output) + if err != nil { + return err + } + return nil +} + +func (sdb *StateDiffBuilder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) (map[string]bool, map[string]bool, error) { + diffPathsAtB := make(map[string]bool) + diffSlotsAtB := make(map[string]bool) + it, _ := trie.NewDifferenceIterator(a, b) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) + if err != nil { + return nil, nil, err + } + switch node.NodeType { + case types2.Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + diffSlotsAtB[common.Bytes2Hex(leafKey)] = true + if err := output(types2.StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + LeafKey: leafKey, + }); err != nil { + return nil, nil, err + } + case types2.Extension, types2.Branch: + if intermediateNodes { + if err := output(types2.StorageNode{ + NodeType: node.NodeType, + Path: node.Path, + NodeValue: node.NodeValue, + }); err != nil { + return nil, nil, err + } + } + default: + return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType) + } + diffPathsAtB[common.Bytes2Hex(node.Path)] = true + } + return diffSlotsAtB, diffPathsAtB, it.Error() +} + +func (sdb *StateDiffBuilder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB, diffPathsAtB map[string]bool, intermediateNodes bool, output types2.StorageNodeSink) error { + it, _ := trie.NewDifferenceIterator(b, a) + for it.Next(true) { + // skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) + if err != nil { + return err + } + + switch node.NodeType { + case types2.Leaf: + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted (or moved) in B + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + // if this node's leaf key also did not show up in diffSlotsAtB + // that means the node was deleted + // in that case, emit an empty "removed" diff storage node + if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + LeafKey: leafKey, + }); err != nil { + return err + } + } else { + // emit an empty "removed" diff with empty leaf key if the account was moved + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } + } + } + case types2.Extension, types2.Branch: + // if this node's path did not show up in diffPathsAtB + // that means the node at this path was deleted in B + // in that case, emit an empty "removed" diff storage node + if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { + if intermediateNodes { + if err := output(types2.StorageNode{ + NodeType: types2.Removed, + Path: node.Path, + NodeValue: []byte{}, + }); err != nil { + return err + } + } + } + default: + return fmt.Errorf("unexpected node type %s", node.NodeType) + } + } + return it.Error() +} + +// isValidPrefixPath is used to check if a node at currentPath is a parent | ancestor to one of the addresses the builder is configured to watch +func isValidPrefixPath(watchedAddressesLeafPaths [][]byte, currentPath []byte) bool { + for _, watchedAddressPath := range watchedAddressesLeafPaths { + if bytes.HasPrefix(watchedAddressPath, currentPath) { + return true + } + } + + return false +} + +// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch +func isWatchedAddress(watchedAddressesLeafPaths [][]byte, valueNodePath []byte) bool { + // If we aren't watching any specific addresses, we are watching everything + if len(watchedAddressesLeafPaths) == 0 { + return true + } + + for _, watchedAddressPath := range watchedAddressesLeafPaths { + if bytes.Equal(watchedAddressPath, valueNodePath) { + return true + } + } + + return false +} diff --git a/statediff/builder_test.go b/statediff/builder_test.go new file mode 100644 index 000000000000..4ff6ed9fce3f --- /dev/null +++ b/statediff/builder_test.go @@ -0,0 +1,2600 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "fmt" + "math/big" + "os" + "sort" + "testing" + + types2 "github.com/ethereum/go-ethereum/statediff/types" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/test_helpers" +) + +var ( + contractLeafKey []byte + emptyDiffs = make([]types2.StateNode, 0) + emptyStorage = make([]types2.StorageNode, 0) + block0, block1, block2, block3, block4, block5, block6 *types.Block + builder statediff.Builder + minerAddress = common.HexToAddress("0x0") + minerLeafKey = test_helpers.AddressToLeafKey(minerAddress) + + slot0 = common.HexToHash("0") + slot1 = common.HexToHash("1") + slot2 = common.HexToHash("2") + slot3 = common.HexToHash("3") + + slot0StorageKey = crypto.Keccak256Hash(slot0[:]) + slot1StorageKey = crypto.Keccak256Hash(slot1[:]) + slot2StorageKey = crypto.Keccak256Hash(slot2[:]) + slot3StorageKey = crypto.Keccak256Hash(slot3[:]) + + slot0StorageValue = common.Hex2Bytes("94703c4b2bd70c169f5717101caee543299fc946c7") // prefixed AccountAddr1 + slot1StorageValue = common.Hex2Bytes("01") + slot2StorageValue = common.Hex2Bytes("09") + slot3StorageValue = common.Hex2Bytes("03") + + slot0StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), + slot0StorageValue, + }) + slot1StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf6"), + slot1StorageValue, + }) + slot2StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("305787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace"), + slot2StorageValue, + }) + slot3StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("32575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b"), + slot3StorageValue, + }) + + contractAccountAtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block2StorageBranchRootNode), + }) + contractAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), + contractAccountAtBlock2, + }) + contractAccountAtBlock3, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block3StorageBranchRootNode), + }) + contractAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), + contractAccountAtBlock3, + }) + contractAccountAtBlock4, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block4StorageBranchRootNode), + }) + contractAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), + contractAccountAtBlock4, + }) + contractAccountAtBlock5, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block5StorageBranchRootNode), + }) + contractAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"), + contractAccountAtBlock5, + }) + + minerAccountAtBlock1, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(2000002625000000000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + minerAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), + minerAccountAtBlock1, + }) + minerAccountAtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(4000111203461610525), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + minerAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), + minerAccountAtBlock2, + }) + + account1AtBlock1, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: test_helpers.Block1Account1Balance, + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + account1AtBlock1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1AtBlock1, + }) + account1AtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 2, + Balance: big.NewInt(999555797000009000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + account1AtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1AtBlock2, + }) + account1AtBlock5, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 2, + Balance: big.NewInt(2999586469962854280), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + account1AtBlock5LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1AtBlock5, + }) + account1AtBlock6, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 3, + Balance: big.NewInt(2999557977962854280), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + account1AtBlock6LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1AtBlock6, + }) + + account2AtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(1000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + account2AtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), + account2AtBlock2, + }) + account2AtBlock3, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(2000013574009435976), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + account2AtBlock3LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), + account2AtBlock3, + }) + account2AtBlock4, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(4000048088163070348), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + account2AtBlock4LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), + account2AtBlock4, + }) + account2AtBlock6, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(6000063258066544204), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + account2AtBlock6LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"), + account2AtBlock6, + }) + + bankAccountAtBlock0, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(test_helpers.TestBankFunds.Int64()), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + bankAccountAtBlock0LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock0, + }) + + block1BankBalance = big.NewInt(test_helpers.TestBankFunds.Int64() - test_helpers.BalanceChange10000 - test_helpers.GasFees) + bankAccountAtBlock1, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 1, + Balance: block1BankBalance, + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + bankAccountAtBlock1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock1, + }) + + block2BankBalance = block1BankBalance.Int64() - test_helpers.BalanceChange1Ether - test_helpers.GasFees + bankAccountAtBlock2, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 2, + Balance: big.NewInt(block2BankBalance), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + bankAccountAtBlock2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock2, + }) + bankAccountAtBlock3, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 3, + Balance: big.NewInt(999914255999990000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + bankAccountAtBlock3LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock3, + }) + bankAccountAtBlock4, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 6, + Balance: big.NewInt(999826859999990000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + bankAccountAtBlock4LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock4, + }) + bankAccountAtBlock5, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 8, + Balance: big.NewInt(999761283999990000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + bankAccountAtBlock5LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock5, + }) + + block1BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + crypto.Keccak256(bankAccountAtBlock1LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock1LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account1AtBlock1LeafNode), + []byte{}, + []byte{}, + }) + block2BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + crypto.Keccak256(bankAccountAtBlock2LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + crypto.Keccak256(contractAccountAtBlock2LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock2LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock2LeafNode), + []byte{}, + []byte{}, + }) + block3BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + crypto.Keccak256(bankAccountAtBlock3LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + crypto.Keccak256(contractAccountAtBlock3LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock3LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock2LeafNode), + []byte{}, + []byte{}, + }) + block4BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + crypto.Keccak256(bankAccountAtBlock4LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + crypto.Keccak256(contractAccountAtBlock4LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock4LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock2LeafNode), + []byte{}, + []byte{}, + }) + block5BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + crypto.Keccak256(bankAccountAtBlock5LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + crypto.Keccak256(contractAccountAtBlock5LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock4LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock5LeafNode), + []byte{}, + []byte{}, + }) + block6BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + crypto.Keccak256(bankAccountAtBlock5LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(minerAccountAtBlock2LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(account2AtBlock6LeafNode), + []byte{}, + crypto.Keccak256(account1AtBlock6LeafNode), + []byte{}, + []byte{}, + }) + + block2StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot1StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block3StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot1StorageLeafNode), + crypto.Keccak256(slot3StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block4StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + crypto.Keccak256(slot2StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block5StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot0StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot3StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) +) + +func init() { + if os.Getenv("MODE") != "statediff" { + fmt.Println("Skipping statediff test") + os.Exit(0) + } +} + +func TestBuilder(t *testing.T) { + blocks, chain := test_helpers.MakeChain(3, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block0 = test_helpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + params := statediff.Params{} + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + { + "testEmptyDiff", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &types2.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock0", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: test_helpers.NullHash, + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &types2.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock0LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &types2.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: types2.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + // 1000 transferred from testBankAddress to account1Addr + // 1000 transferred from account1Addr to account2Addr + // account1addr creates a new contract + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &types2.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: types2.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []types2.CodeAndCodeHash{ + { + Hash: test_helpers.CodeHash, + Code: test_helpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &types2.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithIntermediateNodes(t *testing.T) { + blocks, chain := test_helpers.MakeChain(3, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block0 = test_helpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + blocks = append([]*types.Block{block0}, blocks...) + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + { + "testEmptyDiff", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &types2.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock0", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: test_helpers.NullHash, + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &types2.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock0LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &types2.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block1BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: types2.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + // 1000 transferred from testBankAddress to account1Addr + // 1000 transferred from account1Addr to account2Addr + // account1addr creates a new contract + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &types2.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block2BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: types2.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block2StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []types2.CodeAndCodeHash{ + { + Hash: test_helpers.CodeHash, + Code: test_helpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &types2.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block3BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block3StorageBranchRootNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for i, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + // Let's also confirm that our root state nodes form the state root hash in the headers + if i > 0 { + block := blocks[i-1] + expectedStateRoot := block.Root() + for _, node := range test.expected.Nodes { + if bytes.Equal(node.Path, []byte{}) { + stateRoot := crypto.Keccak256Hash(node.NodeValue) + if !bytes.Equal(expectedStateRoot.Bytes(), stateRoot.Bytes()) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual stateroot: %x\r\nexpected stateroot: %x", stateRoot.Bytes(), expectedStateRoot.Bytes()) + } + } + } + } + } +} + +func TestBuilderWithWatchedAddressList(t *testing.T) { + blocks, chain := test_helpers.MakeChain(3, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block0 = test_helpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr}, + } + params.ComputeWatchedAddressesLeafPaths() + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + { + "testEmptyDiff", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &types2.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock0", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: test_helpers.NullHash, + NewStateRoot: block0.Root(), + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + }, + &types2.StateObject{ + BlockNumber: block0.Number(), + BlockHash: block0.Hash(), + Nodes: emptyDiffs, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &types2.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block1BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &types2.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block2BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block2StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []types2.CodeAndCodeHash{ + { + Hash: test_helpers.CodeHash, + Code: test_helpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &types2.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block3BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block3StorageBranchRootNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { + blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block3 = blocks[2] + block4 = blocks[3] + block5 = blocks[4] + block6 = blocks[5] + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes + { + "testBlock4", + statediff.Args{ + OldStateRoot: block3.Root(), + NewStateRoot: block4.Root(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + }, + &types2.StateObject{ + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block4BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock4LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block4StorageBranchRootNode, + }, + { + Path: []byte{'\x04'}, + NodeType: types2.Leaf, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: slot2StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock5", + statediff.Args{ + OldStateRoot: block4.Root(), + NewStateRoot: block5.Root(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + }, + &types2.StateObject{ + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block5BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock5LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block5StorageBranchRootNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + { + Path: []byte{'\x04'}, + NodeType: types2.Removed, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock6", + statediff.Args{ + OldStateRoot: block5.Root(), + NewStateRoot: block6.Root(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + }, + &types2.StateObject{ + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block6BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Removed, + LeafKey: contractLeafKey, + NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing.T) { + blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block3 = blocks[2] + block4 = blocks[3] + block5 = blocks[4] + block6 = blocks[5] + params := statediff.Params{ + IntermediateStateNodes: false, + IntermediateStorageNodes: false, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + // blocks 0-3 are the same as in TestBuilderWithIntermediateNodes + { + "testBlock4", + statediff.Args{ + OldStateRoot: block3.Root(), + NewStateRoot: block4.Root(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + }, + &types2.StateObject{ + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock4LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x04'}, + NodeType: types2.Leaf, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: slot2StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock5", + statediff.Args{ + OldStateRoot: block4.Root(), + NewStateRoot: block5.Root(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + }, + &types2.StateObject{ + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock5LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + { + Path: []byte{'\x04'}, + NodeType: types2.Removed, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock6", + statediff.Args{ + OldStateRoot: block5.Root(), + NewStateRoot: block6.Root(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + }, + &types2.StateObject{ + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: types2.Removed, + LeafKey: contractLeafKey, + NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithRemovedNonWatchedAccount(t *testing.T) { + blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block3 = blocks[2] + block4 = blocks[3] + block5 = blocks[4] + block6 = blocks[5] + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.Account2Addr}, + } + params.ComputeWatchedAddressesLeafPaths() + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + { + "testBlock4", + statediff.Args{ + OldStateRoot: block3.Root(), + NewStateRoot: block4.Root(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + }, + &types2.StateObject{ + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block4BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock4LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock5", + statediff.Args{ + OldStateRoot: block4.Root(), + NewStateRoot: block5.Root(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + }, + &types2.StateObject{ + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block5BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock6", + statediff.Args{ + OldStateRoot: block5.Root(), + NewStateRoot: block6.Root(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + }, + &types2.StateObject{ + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block6BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithRemovedWatchedAccount(t *testing.T) { + blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block3 = blocks[2] + block4 = blocks[3] + block5 = blocks[4] + block6 = blocks[5] + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + WatchedAddresses: []common.Address{test_helpers.Account1Addr, test_helpers.ContractAddr}, + } + params.ComputeWatchedAddressesLeafPaths() + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + { + "testBlock4", + statediff.Args{ + OldStateRoot: block3.Root(), + NewStateRoot: block4.Root(), + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + }, + &types2.StateObject{ + BlockNumber: block4.Number(), + BlockHash: block4.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block4BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock4LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block4StorageBranchRootNode, + }, + { + Path: []byte{'\x04'}, + NodeType: types2.Leaf, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: slot2StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + }, + }, + }, + { + "testBlock5", + statediff.Args{ + OldStateRoot: block4.Root(), + NewStateRoot: block5.Root(), + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + }, + &types2.StateObject{ + BlockNumber: block5.Number(), + BlockHash: block5.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block5BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock5LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block5StorageBranchRootNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + { + Path: []byte{'\x04'}, + NodeType: types2.Removed, + LeafKey: slot2StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock5LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock6", + statediff.Args{ + OldStateRoot: block5.Root(), + NewStateRoot: block6.Root(), + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + }, + &types2.StateObject{ + BlockNumber: block6.Number(), + BlockHash: block6.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block6BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Removed, + LeafKey: contractLeafKey, + NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Removed, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Removed, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock6LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +var ( + slot00StorageValue = common.Hex2Bytes("9471562b71999873db5b286df957af199ec94617f7") // prefixed TestBankAddress + + slot00StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"), + slot00StorageValue, + }) + + contractAccountAtBlock01, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(), + Root: crypto.Keccak256Hash(block01StorageBranchRootNode), + }) + contractAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3cb2583748c26e89ef19c2a8529b05a270f735553b4d44b6f2a1894987a71c8b"), + contractAccountAtBlock01, + }) + + bankAccountAtBlock01, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(3999629697375000000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + bankAccountAtBlock01LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock01, + }) + bankAccountAtBlock02, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 2, + Balance: big.NewInt(5999607323457344852), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + bankAccountAtBlock02LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccountAtBlock02, + }) + + block01BranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + crypto.Keccak256Hash(bankAccountAtBlock01LeafNode), + crypto.Keccak256Hash(contractAccountAtBlock01LeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + + block01StorageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{ + []byte{}, + []byte{}, + crypto.Keccak256(slot00StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + crypto.Keccak256(slot1StorageLeafNode), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) +) + +func TestBuilderWithMovedAccount(t *testing.T) { + blocks, chain := test_helpers.MakeChain(2, test_helpers.Genesis, test_helpers.TestSelfDestructChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block0 = test_helpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + params := statediff.Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + { + "testBlock1", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &types2.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block01BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock01LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x01'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock01LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block01StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot00StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + }, + CodeAndCodeHashes: []types2.CodeAndCodeHash{ + { + Hash: test_helpers.CodeHash, + Code: test_helpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock2", + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &types2.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock02LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x01'}, + NodeType: types2.Removed, + LeafKey: contractLeafKey, + NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Removed, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + }, + }, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Removed, + NodeValue: []byte{}, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { + blocks, chain := test_helpers.MakeChain(2, test_helpers.Genesis, test_helpers.TestSelfDestructChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block0 = test_helpers.Genesis + block1 = blocks[0] + block2 = blocks[1] + params := statediff.Params{ + IntermediateStateNodes: false, + IntermediateStorageNodes: false, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *types2.StateObject + }{ + { + "testBlock1", + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &types2.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock01LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x01'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock01LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot00StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + }, + CodeAndCodeHashes: []types2.CodeAndCodeHash{ + { + Hash: test_helpers.CodeHash, + Code: test_helpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock2", + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &types2.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock02LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x01'}, + NodeType: types2.Removed, + LeafKey: contractLeafKey, + NodeValue: []byte{}, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{'\x02'}, + NodeType: types2.Removed, + LeafKey: slot0StorageKey.Bytes(), + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Removed, + LeafKey: slot1StorageKey.Bytes(), + }, + }, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Removed, + NodeValue: []byte{}, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\r\n\r\n\r\nexpected state diff: %+v", diff, test.expected) + } + } +} + +func TestBuildStateTrie(t *testing.T) { + blocks, chain := test_helpers.MakeChain(3, test_helpers.Genesis, test_helpers.TestChainGen) + contractLeafKey = test_helpers.AddressToLeafKey(test_helpers.ContractAddr) + defer chain.Stop() + block1 = blocks[0] + block2 = blocks[1] + block3 = blocks[2] + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + block *types.Block + expected *types2.StateObject + }{ + { + "testBlock1", + block1, + &types2.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block1BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: types2.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock1LeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + block2, + &types2.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block2BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: types2.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock2LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block2StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []types2.CodeAndCodeHash{ + { + Hash: test_helpers.CodeHash, + Code: test_helpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + { + "testBlock3", + block3, + &types2.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []types2.StateNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block3BranchRootNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountAtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x05'}, + NodeType: types2.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountAtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1AtBlock2LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x06'}, + NodeType: types2.Leaf, + LeafKey: contractLeafKey, + NodeValue: contractAccountAtBlock3LeafNode, + StorageNodes: []types2.StorageNode{ + { + Path: []byte{}, + NodeType: types2.Branch, + NodeValue: block3StorageBranchRootNode, + }, + { + Path: []byte{'\x02'}, + NodeType: types2.Leaf, + LeafKey: slot0StorageKey.Bytes(), + NodeValue: slot0StorageLeafNode, + }, + { + Path: []byte{'\x0b'}, + NodeType: types2.Leaf, + LeafKey: slot1StorageKey.Bytes(), + NodeValue: slot1StorageLeafNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: slot3StorageKey.Bytes(), + NodeValue: slot3StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: types2.Leaf, + LeafKey: test_helpers.Account2LeafKey, + NodeValue: account2AtBlock3LeafNode, + StorageNodes: emptyStorage, + }, + }, + CodeAndCodeHashes: []types2.CodeAndCodeHash{ + { + Hash: test_helpers.CodeHash, + Code: test_helpers.ByteCodeAfterDeployment, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateTrieObject(test.block) + if err != nil { + t.Error(err) + } + receivedStateTrieRlp, err := rlp.EncodeToBytes(&diff) + if err != nil { + t.Error(err) + } + expectedStateTrieRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateTrieRlp, func(i, j int) bool { return receivedStateTrieRlp[i] < receivedStateTrieRlp[j] }) + sort.Slice(expectedStateTrieRlp, func(i, j int) bool { return expectedStateTrieRlp[i] < expectedStateTrieRlp[j] }) + if !bytes.Equal(receivedStateTrieRlp, expectedStateTrieRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state trie: %+v\r\n\r\n\r\nexpected state trie: %+v", diff, test.expected) + } + } +} + +/* +pragma solidity ^0.5.10; + +contract test { + address payable owner; + + modifier onlyOwner { + require( + msg.sender == owner, + "Only owner can call this function." + ); + _; + } + + uint256[100] data; + + constructor() public { + owner = msg.sender; + data = [1]; + } + + function Put(uint256 addr, uint256 value) public { + data[addr] = value; + } + + function close() public onlyOwner { //onlyOwner is custom modifier + selfdestruct(owner); // `owner` is the owners address + } +} +*/ diff --git a/statediff/config.go b/statediff/config.go new file mode 100644 index 000000000000..c6b305b88e03 --- /dev/null +++ b/statediff/config.go @@ -0,0 +1,91 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package statediff + +import ( + "context" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" +) + +// Config contains instantiation parameters for the state diffing service +type Config struct { + // The configuration used for the stateDiff Indexer + IndexerConfig interfaces.Config + // The filepath to write knownGaps insert statements if we can't connect to the DB. + KnownGapsFilePath string + // A unique ID used for this service + ID string + // Name for the client this service is running + ClientName string + // Whether to enable writing state diffs directly to track blockchain head + EnableWriteLoop bool + // Size of the worker pool + NumWorkers uint + // Should the statediff service wait until geth has synced to the head of the blockchain? + WaitForSync bool + // Context + Context context.Context +} + +// Params contains config parameters for the state diff builder +type Params struct { + IntermediateStateNodes bool + IntermediateStorageNodes bool + IncludeBlock bool + IncludeReceipts bool + IncludeTD bool + IncludeCode bool + WatchedAddresses []common.Address + watchedAddressesLeafPaths [][]byte +} + +// ComputeWatchedAddressesLeafPaths populates a slice with paths (hex_encoding(Keccak256)) of each of the WatchedAddresses +func (p *Params) ComputeWatchedAddressesLeafPaths() { + p.watchedAddressesLeafPaths = make([][]byte, len(p.WatchedAddresses)) + for i, address := range p.WatchedAddresses { + p.watchedAddressesLeafPaths[i] = keybytesToHex(crypto.Keccak256(address.Bytes())) + } +} + +// ParamsWithMutex allows to lock the parameters while they are being updated | read from +type ParamsWithMutex struct { + Params + sync.RWMutex +} + +// Args bundles the arguments for the state diff builder +type Args struct { + OldStateRoot, NewStateRoot, BlockHash common.Hash + BlockNumber *big.Int +} + +// https://github.com/ethereum/go-ethereum/blob/master/trie/encoding.go#L97 +func keybytesToHex(str []byte) []byte { + l := len(str)*2 + 1 + var nibbles = make([]byte, l) + for i, b := range str { + nibbles[i*2] = b / 16 + nibbles[i*2+1] = b % 16 + } + nibbles[l-1] = 16 + return nibbles +} diff --git a/statediff/docs/KnownGaps.md b/statediff/docs/KnownGaps.md new file mode 100644 index 000000000000..72e712f472c1 --- /dev/null +++ b/statediff/docs/KnownGaps.md @@ -0,0 +1,17 @@ +# Overview + +This document will provide some insight into the `known_gaps` table, their use cases, and implementation. Please refer to the [following PR](https://github.com/vulcanize/go-ethereum/pull/217) and the [following epic](https://github.com/vulcanize/ops/issues/143) to grasp their inception. + +![known gaps](diagrams/KnownGapsProcess.png) + +# Use Cases + +The known gaps table is updated when the following events occur: + +1. At start up we check the latest block from the `eth.headers_cid` table. We compare the first block that we are processing with the latest block from the DB. If they are not one unit of expectedDifference away from each other, add the gap between the two blocks. +2. If there is any error in processing a block (db connection, deadlock, etc), add that block to the knownErrorBlocks slice, when the next block is successfully written, write this slice into the DB. + +# Glossary + +1. `expectedDifference (number)` - This number indicates what the difference between two blocks should be. If we are capturing all events on a geth node then this number would be `1`. But once we scale nodes, the `expectedDifference` might be `2` or greater. +2. `processingKey (number)` - This number can be used to keep track of different geth nodes and their specific `expectedDifference`. diff --git a/statediff/docs/README.md b/statediff/docs/README.md new file mode 100644 index 000000000000..51b63e20747a --- /dev/null +++ b/statediff/docs/README.md @@ -0,0 +1,3 @@ +# Overview + +This folder keeps tracks of random documents as they relate to the `statediff` service. diff --git a/statediff/docs/database.md b/statediff/docs/database.md new file mode 100644 index 000000000000..847bc8fa2469 --- /dev/null +++ b/statediff/docs/database.md @@ -0,0 +1,21 @@ +# Overview + +This document will go through some notes on the database component of the statediff service. + +# Components + +- Indexer: The indexer creates IPLD and DB models to insert to the Postgres DB. It performs the insert utilizing and atomic function. +- Builder: The builder constructs the statediff object that needs to be inserted. +- Known Gaps: Captures any gaps that might have occured and either writes them to the DB, local sql file, to prometeus, or a local error. + +# Making Code Changes + +## Adding a New Function to the Indexer + +If you want to implement a new feature for adding data to the database. Keep the following in mind: + +1. You need to handle `sql`, `file`, and `dump`. + 1. `sql` - Contains the code needed to write directly to the `sql` db. + 2. `file` - Contains all the code required to write the SQL statements to a file. + 3. `dump` - Contains all the code for outputting events to the console. +2. You will have to add it to the `interfaces.StateDiffIndexer` interface. diff --git a/statediff/docs/diagrams/KnownGapsProcess.png b/statediff/docs/diagrams/KnownGapsProcess.png new file mode 100644 index 000000000000..40ebaa80aad2 Binary files /dev/null and b/statediff/docs/diagrams/KnownGapsProcess.png differ diff --git a/statediff/helpers.go b/statediff/helpers.go new file mode 100644 index 000000000000..1557eb227e94 --- /dev/null +++ b/statediff/helpers.go @@ -0,0 +1,65 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package statediff + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/ethereum/go-ethereum/log" + + "github.com/ethereum/go-ethereum/params" +) + +// LoadConfig loads chain config from json file +func LoadConfig(chainConfigPath string) (*params.ChainConfig, error) { + file, err := os.Open(chainConfigPath) + if err != nil { + log.Error(fmt.Sprintf("Failed to read chain config file: %v", err)) + + return nil, err + } + defer file.Close() + + chainConfig := new(params.ChainConfig) + if err := json.NewDecoder(file).Decode(chainConfig); err != nil { + log.Error(fmt.Sprintf("invalid chain config file: %v", err)) + + return nil, err + } + + log.Info(fmt.Sprintf("Using chain config from %s file. Content %+v", chainConfigPath, chainConfig)) + + return chainConfig, nil +} + +// ChainConfig returns the appropriate ethereum chain config for the provided chain id +func ChainConfig(chainID uint64) (*params.ChainConfig, error) { + switch chainID { + case 1: + return params.MainnetChainConfig, nil + case 3: + return params.RopstenChainConfig, nil + case 4: + return params.RinkebyChainConfig, nil + case 5: + return params.GoerliChainConfig, nil + default: + return nil, fmt.Errorf("chain config for chainid %d not available", chainID) + } +} diff --git a/statediff/indexer/constructor.go b/statediff/indexer/constructor.go new file mode 100644 index 000000000000..1a4f640019c7 --- /dev/null +++ b/statediff/indexer/constructor.go @@ -0,0 +1,81 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "context" + "fmt" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff/indexer/database/dump" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +// NewStateDiffIndexer creates and returns an implementation of the StateDiffIndexer interface. +func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, nodeInfo node.Info, config interfaces.Config) (sql.Database, interfaces.StateDiffIndexer, error) { + switch config.Type() { + case shared.FILE: + log.Info("Starting statediff service in SQL file writing mode") + fc, ok := config.(file.Config) + if !ok { + return nil, nil, fmt.Errorf("file config is not the correct type: got %T, expected %T", config, file.Config{}) + } + fc.NodeInfo = nodeInfo + ind, err := file.NewStateDiffIndexer(ctx, chainConfig, fc) + return nil, ind, err + case shared.POSTGRES: + log.Info("Starting statediff service in Postgres writing mode") + pgc, ok := config.(postgres.Config) + if !ok { + return nil, nil, fmt.Errorf("postgres config is not the correct type: got %T, expected %T", config, postgres.Config{}) + } + var err error + var driver sql.Driver + switch pgc.Driver { + case postgres.PGX: + driver, err = postgres.NewPGXDriver(ctx, pgc, nodeInfo) + if err != nil { + return nil, nil, err + } + case postgres.SQLX: + driver, err = postgres.NewSQLXDriver(ctx, pgc, nodeInfo) + if err != nil { + return nil, nil, err + } + default: + return nil, nil, fmt.Errorf("unrecognized Postgres driver type: %s", pgc.Driver) + } + db := postgres.NewPostgresDB(driver) + ind, err := sql.NewStateDiffIndexer(ctx, chainConfig, db) + return db, ind, err + case shared.DUMP: + log.Info("Starting statediff service in data dump mode") + dumpc, ok := config.(dump.Config) + if !ok { + return nil, nil, fmt.Errorf("dump config is not the correct type: got %T, expected %T", config, dump.Config{}) + } + return nil, dump.NewStateDiffIndexer(chainConfig, dumpc), nil + default: + return nil, nil, fmt.Errorf("unrecognized database type: %s", config.Type()) + } +} diff --git a/statediff/indexer/database/dump/batch_tx.go b/statediff/indexer/database/dump/batch_tx.go new file mode 100644 index 000000000000..ee195a558be2 --- /dev/null +++ b/statediff/indexer/database/dump/batch_tx.go @@ -0,0 +1,97 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dump + +import ( + "fmt" + "io" + + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + + "github.com/ethereum/go-ethereum/statediff/indexer/models" + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + node "github.com/ipfs/go-ipld-format" +) + +// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration +type BatchTx struct { + BlockNumber string + dump io.Writer + quit chan struct{} + iplds chan models.IPLDModel + ipldCache models.IPLDBatch + + submit func(blockTx *BatchTx, err error) error +} + +// Submit satisfies indexer.AtomicTx +func (tx *BatchTx) Submit(err error) error { + return tx.submit(tx, err) +} + +func (tx *BatchTx) flush() error { + if _, err := fmt.Fprintf(tx.dump, "%+v\r\n", tx.ipldCache); err != nil { + return err + } + tx.ipldCache = models.IPLDBatch{} + return nil +} + +// run in background goroutine to synchronize concurrent appends to the ipldCache +func (tx *BatchTx) cache() { + for { + select { + case i := <-tx.iplds: + tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key) + tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data) + case <-tx.quit: + tx.ipldCache = models.IPLDBatch{} + return + } + } +} + +func (tx *BatchTx) cacheDirect(key string, value []byte) { + tx.iplds <- models.IPLDModel{ + BlockNumber: tx.BlockNumber, + Key: key, + Data: value, + } +} + +func (tx *BatchTx) cacheIPLD(i node.Node) { + tx.iplds <- models.IPLDModel{ + BlockNumber: tx.BlockNumber, + Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(), + Data: i.RawData(), + } +} + +func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error) { + c, err := ipld.RawdataToCid(codec, raw, mh) + if err != nil { + return "", "", err + } + prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String() + tx.iplds <- models.IPLDModel{ + BlockNumber: tx.BlockNumber, + Key: prefixedKey, + Data: raw, + } + return c.String(), prefixedKey, err +} diff --git a/statediff/indexer/database/dump/config.go b/statediff/indexer/database/dump/config.go new file mode 100644 index 000000000000..6fb1f0a9ed74 --- /dev/null +++ b/statediff/indexer/database/dump/config.go @@ -0,0 +1,79 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dump + +import ( + "fmt" + "io" + "strings" + + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +// DumpType to explicitly type the dump destination +type DumpType string + +const ( + STDOUT = "Stdout" + STDERR = "Stderr" + DISCARD = "Discard" + UNKNOWN = "Unknown" +) + +// ResolveDumpType resolves the dump type for the provided string +func ResolveDumpType(str string) (DumpType, error) { + switch strings.ToLower(str) { + case "stdout", "out", "std out": + return STDOUT, nil + case "stderr", "err", "std err": + return STDERR, nil + case "discard", "void", "devnull", "dev null": + return DISCARD, nil + default: + return UNKNOWN, fmt.Errorf("unrecognized dump type: %s", str) + } +} + +// Config for data dump +type Config struct { + Dump io.WriteCloser +} + +// Type satisfies interfaces.Config +func (c Config) Type() shared.DBType { + return shared.DUMP +} + +// NewDiscardWriterCloser returns a discardWrapper wrapping io.Discard +func NewDiscardWriterCloser() io.WriteCloser { + return discardWrapper{blackhole: io.Discard} +} + +// discardWrapper wraps io.Discard with io.Closer +type discardWrapper struct { + blackhole io.Writer +} + +// Write satisfies io.Writer +func (dw discardWrapper) Write(b []byte) (int, error) { + return dw.blackhole.Write(b) +} + +// Close satisfies io.Closer +func (dw discardWrapper) Close() error { + return nil +} diff --git a/statediff/indexer/database/dump/indexer.go b/statediff/indexer/database/dump/indexer.go new file mode 100644 index 000000000000..2cc7e2e0a466 --- /dev/null +++ b/statediff/indexer/database/dump/indexer.go @@ -0,0 +1,538 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dump + +import ( + "fmt" + "io" + "math/big" + "time" + + ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var _ interfaces.StateDiffIndexer = &StateDiffIndexer{} + +var ( + indexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry) +) + +// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of a void +type StateDiffIndexer struct { + dump io.WriteCloser + chainConfig *params.ChainConfig +} + +// NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer +func NewStateDiffIndexer(chainConfig *params.ChainConfig, config Config) *StateDiffIndexer { + return &StateDiffIndexer{ + dump: config.Dump, + chainConfig: chainConfig, + } +} + +// ReportDBMetrics has nothing to report for dump +func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {} + +// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts) +// Returns an initiated DB transaction which must be Closed via defer to commit or rollback +func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) { + start, t := time.Now(), time.Now() + blockHash := block.Hash() + blockHashStr := blockHash.String() + height := block.NumberU64() + traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr) + transactions := block.Transactions() + // Derive any missing fields + if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil { + return nil, err + } + + // Generate the block iplds + headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts) + if err != nil { + return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err) + } + + if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) { + return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs)) + } + if len(txTrieNodes) != len(rctTrieNodes) { + return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes)) + } + + // Calculate reward + var reward *big.Int + // in PoA networks block reward is 0 + if sdi.chainConfig.Clique != nil { + reward = big.NewInt(0) + } else { + reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts) + } + t = time.Now() + + blockTx := &BatchTx{ + BlockNumber: block.Number().String(), + dump: sdi.dump, + iplds: make(chan models.IPLDModel), + quit: make(chan struct{}), + ipldCache: models.IPLDBatch{}, + submit: func(self *BatchTx, err error) error { + close(self.quit) + close(self.iplds) + tDiff := time.Since(t) + indexerMetrics.tStateStoreCodeProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String()) + t = time.Now() + if err := self.flush(); err != nil { + traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String()) + log.Debug(traceMsg) + return err + } + tDiff = time.Since(t) + indexerMetrics.tPostgresCommit.Update(tDiff) + traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String()) + traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String()) + log.Debug(traceMsg) + return err + }, + } + go blockTx.cache() + + tDiff := time.Since(t) + indexerMetrics.tFreePostgres.Update(tDiff) + + traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String()) + t = time.Now() + + // Publish and index header, collect headerID + var headerID string + headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty) + if err != nil { + return nil, err + } + tDiff = time.Since(t) + indexerMetrics.tHeaderProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String()) + t = time.Now() + // Publish and index uncles + err = sdi.processUncles(blockTx, headerID, block.Number(), uncleNodes) + if err != nil { + return nil, err + } + tDiff = time.Since(t) + indexerMetrics.tUncleProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String()) + t = time.Now() + // Publish and index receipts and txs + err = sdi.processReceiptsAndTxs(blockTx, processArgs{ + headerID: headerID, + blockNumber: block.Number(), + receipts: receipts, + txs: transactions, + rctNodes: rctNodes, + rctTrieNodes: rctTrieNodes, + txNodes: txNodes, + txTrieNodes: txTrieNodes, + logTrieNodes: logTrieNodes, + logLeafNodeCIDs: logLeafNodeCIDs, + rctLeafNodeCIDs: rctLeafNodeCIDs, + }) + if err != nil { + return nil, err + } + tDiff = time.Since(t) + indexerMetrics.tTxAndRecProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String()) + t = time.Now() + + return blockTx, err +} + +// processHeader publishes and indexes a header IPLD in Postgres +// it returns the headerID +func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode node.Node, reward, td *big.Int) (string, error) { + tx.cacheIPLD(headerNode) + + headerID := header.Hash().String() + mod := models.HeaderModel{ + CID: headerNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(headerNode.Cid()), + ParentHash: header.ParentHash.String(), + BlockNumber: header.Number.String(), + BlockHash: headerID, + TotalDifficulty: td.String(), + Reward: reward.String(), + Bloom: header.Bloom.Bytes(), + StateRoot: header.Root.String(), + RctRoot: header.ReceiptHash.String(), + TxRoot: header.TxHash.String(), + UncleRoot: header.UncleHash.String(), + Timestamp: header.Time, + Coinbase: header.Coinbase.String(), + } + _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", mod) + return headerID, err +} + +// processUncles publishes and indexes uncle IPLDs in Postgres +func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, uncleNodes []*ipld2.EthHeader) error { + // publish and index uncles + for _, uncleNode := range uncleNodes { + tx.cacheIPLD(uncleNode) + var uncleReward *big.Int + // in PoA networks uncle reward is 0 + if sdi.chainConfig.Clique != nil { + uncleReward = big.NewInt(0) + } else { + uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncleNode.Number.Uint64()) + } + uncle := models.UncleModel{ + BlockNumber: blockNumber.String(), + HeaderID: headerID, + CID: uncleNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()), + ParentHash: uncleNode.ParentHash.String(), + BlockHash: uncleNode.Hash().String(), + Reward: uncleReward.String(), + } + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", uncle); err != nil { + return err + } + } + return nil +} + +// processArgs bundles arguments to processReceiptsAndTxs +type processArgs struct { + headerID string + blockNumber *big.Int + receipts types.Receipts + txs types.Transactions + rctNodes []*ipld2.EthReceipt + rctTrieNodes []*ipld2.EthRctTrie + txNodes []*ipld2.EthTx + txTrieNodes []*ipld2.EthTxTrie + logTrieNodes [][]node.Node + logLeafNodeCIDs [][]cid.Cid + rctLeafNodeCIDs []cid.Cid +} + +// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres +func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error { + // Process receipts and txs + signer := types.MakeSigner(sdi.chainConfig, args.blockNumber) + for i, receipt := range args.receipts { + for _, logTrieNode := range args.logTrieNodes[i] { + tx.cacheIPLD(logTrieNode) + } + txNode := args.txNodes[i] + tx.cacheIPLD(txNode) + + // Indexing + // index tx + trx := args.txs[i] + trxID := trx.Hash().String() + + var val string + if trx.Value() != nil { + val = trx.Value().String() + } + + // derive sender for the tx that corresponds with this receipt + from, err := types.Sender(signer, trx) + if err != nil { + return fmt.Errorf("error deriving tx sender: %v", err) + } + txModel := models.TxModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + Dst: shared.HandleZeroAddrPointer(trx.To()), + Src: shared.HandleZeroAddr(from), + TxHash: trxID, + Index: int64(i), + Data: trx.Data(), + CID: txNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(txNode.Cid()), + Type: trx.Type(), + Value: val, + } + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", txModel); err != nil { + return err + } + + // index access list if this is one + for j, accessListElement := range trx.AccessList() { + storageKeys := make([]string, len(accessListElement.StorageKeys)) + for k, storageKey := range accessListElement.StorageKeys { + storageKeys[k] = storageKey.Hex() + } + accessListElementModel := models.AccessListElementModel{ + BlockNumber: args.blockNumber.String(), + TxID: trxID, + Index: int64(j), + Address: accessListElement.Address.Hex(), + StorageKeys: storageKeys, + } + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", accessListElementModel); err != nil { + return err + } + } + + // this is the contract address if this receipt is for a contract creation tx + contract := shared.HandleZeroAddr(receipt.ContractAddress) + var contractHash string + if contract != "" { + contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + } + + // index the receipt + if !args.rctLeafNodeCIDs[i].Defined() { + return fmt.Errorf("invalid receipt leaf node cid") + } + + rctModel := &models.ReceiptModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + TxID: trxID, + Contract: contract, + ContractHash: contractHash, + LeafCID: args.rctLeafNodeCIDs[i].String(), + LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]), + LogRoot: args.rctNodes[i].LogRoot.String(), + } + if len(receipt.PostState) == 0 { + rctModel.PostStatus = receipt.Status + } else { + rctModel.PostState = common.Bytes2Hex(receipt.PostState) + } + + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", rctModel); err != nil { + return err + } + + logDataSet := make([]*models.LogsModel, len(receipt.Logs)) + for idx, l := range receipt.Logs { + topicSet := make([]string, 4) + for ti, topic := range l.Topics { + topicSet[ti] = topic.Hex() + } + + if !args.logLeafNodeCIDs[i][idx].Defined() { + return fmt.Errorf("invalid log cid") + } + + logDataSet[idx] = &models.LogsModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + ReceiptID: trxID, + Address: l.Address.String(), + Index: int64(l.Index), + Data: l.Data, + LeafCID: args.logLeafNodeCIDs[i][idx].String(), + LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]), + Topic0: topicSet[0], + Topic1: topicSet[1], + Topic2: topicSet[2], + Topic3: topicSet[3], + } + } + + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", logDataSet); err != nil { + return err + } + } + + // publish trie nodes, these aren't indexed directly + for i, n := range args.txTrieNodes { + tx.cacheIPLD(n) + tx.cacheIPLD(args.rctTrieNodes[i]) + } + + return nil +} + +// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql +func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { + tx, ok := batch.(*BatchTx) + if !ok { + return fmt.Errorf("dump: batch is expected to be of type %T, got %T", &BatchTx{}, batch) + } + // publish the state node + var stateModel models.StateNodeModel + if stateNode.NodeType == sdtypes.Removed { + // short circuit if it is a Removed node + // this assumes the db has been initialized and a public.blocks entry for the Removed node is present + stateModel = models.StateNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: shared.RemovedNodeStateCID, + MhKey: shared.RemovedNodeMhKey, + NodeType: stateNode.NodeType.Int(), + } + } else { + stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } + } + + // index the state node, collect the stateID to reference by FK + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel); err != nil { + return err + } + + // if we have a leaf, decode and index the account data + if stateNode.NodeType == sdtypes.Leaf { + var i []interface{} + if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil { + return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements") + } + var account types.StateAccount + if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil { + return fmt.Errorf("error decoding state account rlp: %s", err.Error()) + } + accountModel := models.StateAccountModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Balance: account.Balance.String(), + Nonce: account.Nonce, + CodeHash: account.CodeHash, + StorageRoot: account.Root.String(), + } + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", accountModel); err != nil { + return err + } + } + + // if there are any storage nodes associated with this node, publish and index them + for _, storageNode := range stateNode.StorageNodes { + if storageNode.NodeType == sdtypes.Removed { + // short circuit if it is a Removed node + // this assumes the db has been initialized and a public.blocks entry for the Removed node is present + storageModel := models.StorageNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).String(), + CID: shared.RemovedNodeStorageCID, + MhKey: shared.RemovedNodeMhKey, + NodeType: storageNode.NodeType.Int(), + } + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", storageModel); err != nil { + return err + } + continue + } + storageCIDStr, storageMhKey, err := tx.cacheRaw(ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err) + } + storageModel := models.StorageNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).String(), + CID: storageCIDStr, + MhKey: storageMhKey, + NodeType: storageNode.NodeType.Int(), + } + if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", storageModel); err != nil { + return err + } + } + + return nil +} + +// PushCodeAndCodeHash publishes code and codehash pairs to the ipld sql +func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error { + tx, ok := batch.(*BatchTx) + if !ok { + return fmt.Errorf("dump: batch is expected to be of type %T, got %T", &BatchTx{}, batch) + } + // codec doesn't matter since db key is multihash-based + mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash) + if err != nil { + return fmt.Errorf("error deriving multihash key from codehash: %v", err) + } + tx.cacheDirect(mhKey, codeAndCodeHash.Code) + return nil +} + +// Close satisfies io.Closer +func (sdi *StateDiffIndexer) Close() error { + return sdi.dump.Close() +} + +// LoadWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) { + return nil, nil +} + +// InsertWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + return nil +} + +// RemoveWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error { + return nil +} + +// SetWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + return nil +} + +// ClearWatchedAddresses satisfies the interfaces.StateDiffIndexer interface +func (sdi *StateDiffIndexer) ClearWatchedAddresses() error { + return nil +} diff --git a/statediff/indexer/database/dump/metrics.go b/statediff/indexer/database/dump/metrics.go new file mode 100644 index 000000000000..700e42dc0e96 --- /dev/null +++ b/statediff/indexer/database/dump/metrics.go @@ -0,0 +1,94 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dump + +import ( + "strings" + + "github.com/ethereum/go-ethereum/metrics" +) + +const ( + namespace = "statediff" +) + +// Build a fully qualified metric name +func metricName(subsystem, name string) string { + if name == "" { + return "" + } + parts := []string{namespace, name} + if subsystem != "" { + parts = []string{namespace, subsystem, name} + } + // Prometheus uses _ but geth metrics uses / and replaces + return strings.Join(parts, "/") +} + +type indexerMetricsHandles struct { + // The total number of processed blocks + blocks metrics.Counter + // The total number of processed transactions + transactions metrics.Counter + // The total number of processed receipts + receipts metrics.Counter + // The total number of processed logs + logs metrics.Counter + // The total number of access list entries processed + accessListEntries metrics.Counter + // Time spent waiting for free postgres tx + tFreePostgres metrics.Timer + // Postgres transaction commit duration + tPostgresCommit metrics.Timer + // Header processing time + tHeaderProcessing metrics.Timer + // Uncle processing time + tUncleProcessing metrics.Timer + // Tx and receipt processing time + tTxAndRecProcessing metrics.Timer + // State, storage, and code combined processing time + tStateStoreCodeProcessing metrics.Timer +} + +func RegisterIndexerMetrics(reg metrics.Registry) indexerMetricsHandles { + ctx := indexerMetricsHandles{ + blocks: metrics.NewCounter(), + transactions: metrics.NewCounter(), + receipts: metrics.NewCounter(), + logs: metrics.NewCounter(), + accessListEntries: metrics.NewCounter(), + tFreePostgres: metrics.NewTimer(), + tPostgresCommit: metrics.NewTimer(), + tHeaderProcessing: metrics.NewTimer(), + tUncleProcessing: metrics.NewTimer(), + tTxAndRecProcessing: metrics.NewTimer(), + tStateStoreCodeProcessing: metrics.NewTimer(), + } + subsys := "indexer" + reg.Register(metricName(subsys, "blocks"), ctx.blocks) + reg.Register(metricName(subsys, "transactions"), ctx.transactions) + reg.Register(metricName(subsys, "receipts"), ctx.receipts) + reg.Register(metricName(subsys, "logs"), ctx.logs) + reg.Register(metricName(subsys, "access_list_entries"), ctx.accessListEntries) + reg.Register(metricName(subsys, "t_free_postgres"), ctx.tFreePostgres) + reg.Register(metricName(subsys, "t_postgres_commit"), ctx.tPostgresCommit) + reg.Register(metricName(subsys, "t_header_processing"), ctx.tHeaderProcessing) + reg.Register(metricName(subsys, "t_uncle_processing"), ctx.tUncleProcessing) + reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.tTxAndRecProcessing) + reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.tStateStoreCodeProcessing) + return ctx +} diff --git a/statediff/indexer/database/file/batch_tx.go b/statediff/indexer/database/file/batch_tx.go new file mode 100644 index 000000000000..d38bd1211877 --- /dev/null +++ b/statediff/indexer/database/file/batch_tx.go @@ -0,0 +1,29 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file + +// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration +type BatchTx struct { + BlockNumber string + + submit func(blockTx *BatchTx, err error) error +} + +// Submit satisfies indexer.AtomicTx +func (tx *BatchTx) Submit(err error) error { + return tx.submit(tx, err) +} diff --git a/statediff/indexer/database/file/config.go b/statediff/indexer/database/file/config.go new file mode 100644 index 000000000000..a3623e0fa518 --- /dev/null +++ b/statediff/indexer/database/file/config.go @@ -0,0 +1,84 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file + +import ( + "fmt" + "strings" + + "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +// FileMode to explicitly type the mode of file writer we are using +type FileMode string + +const ( + CSV FileMode = "CSV" + SQL FileMode = "SQL" + Unknown FileMode = "Unknown" +) + +// ResolveFileMode resolves a FileMode from a provided string +func ResolveFileMode(str string) (FileMode, error) { + switch strings.ToLower(str) { + case "csv": + return CSV, nil + case "sql": + return SQL, nil + default: + return Unknown, fmt.Errorf("unrecognized file type string: %s", str) + } +} + +// Config holds params for writing out CSV or SQL files +type Config struct { + Mode FileMode + OutputDir string + FilePath string + WatchedAddressesFilePath string + NodeInfo node.Info +} + +// Type satisfies interfaces.Config +func (c Config) Type() shared.DBType { + return shared.FILE +} + +var nodeInfo = node.Info{ + GenesisBlock: "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", + NetworkID: "1", + ChainID: 1, + ID: "mockNodeID", + ClientName: "go-ethereum", +} + +// CSVTestConfig config for unit tests +var CSVTestConfig = Config{ + Mode: CSV, + OutputDir: "./statediffing_test", + WatchedAddressesFilePath: "./statediffing_watched_addresses_test_file.csv", + NodeInfo: nodeInfo, +} + +// SQLTestConfig config for unit tests +var SQLTestConfig = Config{ + Mode: SQL, + FilePath: "./statediffing_test_file.sql", + WatchedAddressesFilePath: "./statediffing_watched_addresses_test_file.sql", + NodeInfo: nodeInfo, +} diff --git a/statediff/indexer/database/file/csv_indexer_legacy_test.go b/statediff/indexer/database/file/csv_indexer_legacy_test.go new file mode 100644 index 000000000000..55350a912917 --- /dev/null +++ b/statediff/indexer/database/file/csv_indexer_legacy_test.go @@ -0,0 +1,118 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file_test + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file/types" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/test" + "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" +) + +const dbDirectory = "/file_indexer" +const pgCopyStatement = `COPY %s FROM '%s' CSV` + +func setupLegacyCSVIndexer(t *testing.T) { + if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) { + err := os.RemoveAll(file.CSVTestConfig.OutputDir) + require.NoError(t, err) + } + + ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.CSVTestConfig) + require.NoError(t, err) + + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } +} + +func setupLegacyCSV(t *testing.T) { + setupLegacyCSVIndexer(t) + test.SetupLegacyTestData(t, ind) +} + +func dumpCSVFileData(t *testing.T) { + outputDir := filepath.Join(dbDirectory, file.CSVTestConfig.OutputDir) + workingDir, err := os.Getwd() + require.NoError(t, err) + + localOutputDir := filepath.Join(workingDir, file.CSVTestConfig.OutputDir) + + for _, tbl := range file.Tables { + err := test_helpers.DedupFile(file.TableFilePath(localOutputDir, tbl.Name)) + require.NoError(t, err) + + var stmt string + varcharColumns := tbl.VarcharColumns() + if len(varcharColumns) > 0 { + stmt = fmt.Sprintf( + pgCopyStatement+" FORCE NOT NULL %s", + tbl.Name, + file.TableFilePath(outputDir, tbl.Name), + strings.Join(varcharColumns, ", "), + ) + } else { + stmt = fmt.Sprintf(pgCopyStatement, tbl.Name, file.TableFilePath(outputDir, tbl.Name)) + } + + _, err = db.Exec(context.Background(), stmt) + require.NoError(t, err) + } +} + +func resetAndDumpWatchedAddressesCSVFileData(t *testing.T) { + test_helpers.TearDownDB(t, db) + + outputFilePath := filepath.Join(dbDirectory, file.CSVTestConfig.WatchedAddressesFilePath) + stmt := fmt.Sprintf(pgCopyStatement, types.TableWatchedAddresses.Name, outputFilePath) + + _, err = db.Exec(context.Background(), stmt) + require.NoError(t, err) +} + +func tearDownCSV(t *testing.T) { + test_helpers.TearDownDB(t, db) + require.NoError(t, db.Close()) + + require.NoError(t, os.RemoveAll(file.CSVTestConfig.OutputDir)) + + if err := os.Remove(file.CSVTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) { + require.NoError(t, err) + } +} + +func TestLegacyCSVFileIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs", func(t *testing.T) { + setupLegacyCSV(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestLegacyIndexer(t, db) + }) +} diff --git a/statediff/indexer/database/file/csv_indexer_test.go b/statediff/indexer/database/file/csv_indexer_test.go new file mode 100644 index 000000000000..81f425acbc04 --- /dev/null +++ b/statediff/indexer/database/file/csv_indexer_test.go @@ -0,0 +1,255 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file_test + +import ( + "context" + "errors" + "math/big" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/test" +) + +func setupCSVIndexer(t *testing.T) { + file.CSVTestConfig.OutputDir = "./statediffing_test" + + if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) { + err := os.RemoveAll(file.CSVTestConfig.OutputDir) + require.NoError(t, err) + } + + if _, err := os.Stat(file.CSVTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) { + err := os.Remove(file.CSVTestConfig.WatchedAddressesFilePath) + require.NoError(t, err) + } + + ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.CSVTestConfig) + require.NoError(t, err) + + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } +} + +func setupCSV(t *testing.T) { + setupCSVIndexer(t) + test.SetupTestData(t, ind) +} + +func setupCSVNonCanonical(t *testing.T) { + setupCSVIndexer(t) + test.SetupTestDataNonCanonical(t, ind) +} + +func TestCSVFileIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { + setupCSV(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexHeaderIPLDs(t, db) + }) + + t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { + setupCSV(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexTransactionIPLDs(t, db) + }) + + t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) { + setupCSV(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexLogIPLDs(t, db) + }) + + t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) { + setupCSV(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexReceiptIPLDs(t, db) + }) + + t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) { + setupCSV(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexStateIPLDs(t, db) + }) + + t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { + setupCSV(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexStorageIPLDs(t, db) + }) +} + +func TestCSVFileIndexerNonCanonical(t *testing.T) { + t.Run("Publish and index header", func(t *testing.T) { + setupCSVNonCanonical(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexHeaderNonCanonical(t, db) + }) + + t.Run("Publish and index transactions", func(t *testing.T) { + setupCSVNonCanonical(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexTransactionsNonCanonical(t, db) + }) + + t.Run("Publish and index receipts", func(t *testing.T) { + setupCSVNonCanonical(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexReceiptsNonCanonical(t, db) + }) + + t.Run("Publish and index logs", func(t *testing.T) { + setupCSVNonCanonical(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexLogsNonCanonical(t, db) + }) + + t.Run("Publish and index state nodes", func(t *testing.T) { + setupCSVNonCanonical(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexStateNonCanonical(t, db) + }) + + t.Run("Publish and index storage nodes", func(t *testing.T) { + setupCSVNonCanonical(t) + dumpCSVFileData(t) + defer tearDownCSV(t) + + test.TestPublishAndIndexStorageNonCanonical(t, db) + }) +} + +func TestCSVFileWatchAddressMethods(t *testing.T) { + setupCSVIndexer(t) + defer tearDownCSV(t) + + t.Run("Load watched addresses (empty table)", func(t *testing.T) { + test.TestLoadEmptyWatchedAddresses(t, ind) + }) + + t.Run("Insert watched addresses", func(t *testing.T) { + args := mocks.GetInsertWatchedAddressesArgs() + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1))) + require.NoError(t, err) + + resetAndDumpWatchedAddressesCSVFileData(t) + + test.TestInsertWatchedAddresses(t, db) + }) + + t.Run("Insert watched addresses (some already watched)", func(t *testing.T) { + args := mocks.GetInsertAlreadyWatchedAddressesArgs() + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2))) + require.NoError(t, err) + + resetAndDumpWatchedAddressesCSVFileData(t) + + test.TestInsertAlreadyWatchedAddresses(t, db) + }) + + t.Run("Remove watched addresses", func(t *testing.T) { + args := mocks.GetRemoveWatchedAddressesArgs() + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + resetAndDumpWatchedAddressesCSVFileData(t) + + test.TestRemoveWatchedAddresses(t, db) + }) + + t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) { + args := mocks.GetRemoveNonWatchedAddressesArgs() + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + resetAndDumpWatchedAddressesCSVFileData(t) + + test.TestRemoveNonWatchedAddresses(t, db) + }) + + t.Run("Set watched addresses", func(t *testing.T) { + args := mocks.GetSetWatchedAddressesArgs() + err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2))) + require.NoError(t, err) + + resetAndDumpWatchedAddressesCSVFileData(t) + + test.TestSetWatchedAddresses(t, db) + }) + + t.Run("Set watched addresses (some already watched)", func(t *testing.T) { + args := mocks.GetSetAlreadyWatchedAddressesArgs() + err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3))) + require.NoError(t, err) + + resetAndDumpWatchedAddressesCSVFileData(t) + + test.TestSetAlreadyWatchedAddresses(t, db) + }) + + t.Run("Load watched addresses", func(t *testing.T) { + test.TestLoadWatchedAddresses(t, ind) + }) + + t.Run("Clear watched addresses", func(t *testing.T) { + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + resetAndDumpWatchedAddressesCSVFileData(t) + + test.TestClearWatchedAddresses(t, db) + }) + + t.Run("Clear watched addresses (empty table)", func(t *testing.T) { + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + resetAndDumpWatchedAddressesCSVFileData(t) + + test.TestClearEmptyWatchedAddresses(t, db) + }) +} diff --git a/statediff/indexer/database/file/csv_writer.go b/statediff/indexer/database/file/csv_writer.go new file mode 100644 index 000000000000..2d4d997e3e6c --- /dev/null +++ b/statediff/indexer/database/file/csv_writer.go @@ -0,0 +1,455 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file + +import ( + "encoding/csv" + "errors" + "fmt" + "math/big" + "os" + "path/filepath" + "strconv" + + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + node "github.com/ipfs/go-ipld-format" + "github.com/thoas/go-funk" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file/types" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var ( + Tables = []*types.Table{ + &types.TableIPLDBlock, + &types.TableNodeInfo, + &types.TableHeader, + &types.TableStateNode, + &types.TableStorageNode, + &types.TableUncle, + &types.TableTransaction, + &types.TableAccessListElement, + &types.TableReceipt, + &types.TableLog, + &types.TableStateAccount, + } +) + +type tableRow struct { + table types.Table + values []interface{} +} + +type CSVWriter struct { + // dir containing output files + dir string + + writers fileWriters + watchedAddressesWriter fileWriter + + rows chan tableRow + flushChan chan struct{} + flushFinished chan struct{} + quitChan chan struct{} + doneChan chan struct{} +} + +type fileWriter struct { + *csv.Writer + file *os.File +} + +// fileWriters wraps the file writers for each output table +type fileWriters map[string]fileWriter + +func newFileWriter(path string) (ret fileWriter, err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return + } + + ret = fileWriter{ + Writer: csv.NewWriter(file), + file: file, + } + + return +} + +func makeFileWriters(dir string, tables []*types.Table) (fileWriters, error) { + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + writers := fileWriters{} + for _, tbl := range tables { + w, err := newFileWriter(TableFilePath(dir, tbl.Name)) + if err != nil { + return nil, err + } + writers[tbl.Name] = w + } + return writers, nil +} + +func (tx fileWriters) write(tbl *types.Table, args ...interface{}) error { + row := tbl.ToCsvRow(args...) + return tx[tbl.Name].Write(row) +} + +func (tx fileWriters) close() error { + for _, w := range tx { + err := w.file.Close() + if err != nil { + return err + } + } + return nil +} + +func (tx fileWriters) flush() error { + for _, w := range tx { + w.Flush() + if err := w.Error(); err != nil { + return err + } + } + return nil +} + +func NewCSVWriter(path string, watchedAddressesFilePath string) (*CSVWriter, error) { + if err := os.MkdirAll(path, 0777); err != nil { + return nil, fmt.Errorf("unable to make MkdirAll for path: %s err: %s", path, err) + } + + writers, err := makeFileWriters(path, Tables) + if err != nil { + return nil, err + } + + watchedAddressesWriter, err := newFileWriter(watchedAddressesFilePath) + if err != nil { + return nil, err + } + + csvWriter := &CSVWriter{ + writers: writers, + watchedAddressesWriter: watchedAddressesWriter, + dir: path, + rows: make(chan tableRow), + flushChan: make(chan struct{}), + flushFinished: make(chan struct{}), + quitChan: make(chan struct{}), + doneChan: make(chan struct{}), + } + return csvWriter, nil +} + +func (csw *CSVWriter) Loop() { + go func() { + defer close(csw.doneChan) + for { + select { + case row := <-csw.rows: + err := csw.writers.write(&row.table, row.values...) + if err != nil { + panic(fmt.Sprintf("error writing csv buffer: %v", err)) + } + case <-csw.quitChan: + if err := csw.writers.flush(); err != nil { + panic(fmt.Sprintf("error writing csv buffer to file: %v", err)) + } + return + case <-csw.flushChan: + if err := csw.writers.flush(); err != nil { + panic(fmt.Sprintf("error writing csv buffer to file: %v", err)) + } + csw.flushFinished <- struct{}{} + } + } + }() +} + +// Flush sends a flush signal to the looping process +func (csw *CSVWriter) Flush() { + csw.flushChan <- struct{}{} + <-csw.flushFinished +} + +func TableFilePath(dir, name string) string { return filepath.Join(dir, name+".csv") } + +// Close satisfies io.Closer +func (csw *CSVWriter) Close() error { + close(csw.quitChan) + <-csw.doneChan + close(csw.rows) + close(csw.flushChan) + close(csw.flushFinished) + return csw.writers.close() +} + +func (csw *CSVWriter) upsertNode(node nodeinfo.Info) { + var values []interface{} + values = append(values, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID) + csw.rows <- tableRow{types.TableNodeInfo, values} +} + +func (csw *CSVWriter) upsertIPLD(ipld models.IPLDModel) { + var values []interface{} + values = append(values, ipld.BlockNumber, ipld.Key, ipld.Data) + csw.rows <- tableRow{types.TableIPLDBlock, values} +} + +func (csw *CSVWriter) upsertIPLDDirect(blockNumber, key string, value []byte) { + csw.upsertIPLD(models.IPLDModel{ + BlockNumber: blockNumber, + Key: key, + Data: value, + }) +} + +func (csw *CSVWriter) upsertIPLDNode(blockNumber string, i node.Node) { + csw.upsertIPLD(models.IPLDModel{ + BlockNumber: blockNumber, + Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(), + Data: i.RawData(), + }) +} + +func (csw *CSVWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error) { + c, err := ipld.RawdataToCid(codec, raw, mh) + if err != nil { + return "", "", err + } + prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String() + csw.upsertIPLD(models.IPLDModel{ + BlockNumber: blockNumber, + Key: prefixedKey, + Data: raw, + }) + return c.String(), prefixedKey, err +} + +func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) { + var values []interface{} + values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, + header.TotalDifficulty, header.NodeID, header.Reward, header.StateRoot, header.TxRoot, + header.RctRoot, header.UncleRoot, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.MhKey, 1, header.Coinbase) + csw.rows <- tableRow{types.TableHeader, values} + indexerMetrics.blocks.Inc(1) +} + +func (csw *CSVWriter) upsertUncleCID(uncle models.UncleModel) { + var values []interface{} + values = append(values, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, + uncle.Reward, uncle.MhKey) + csw.rows <- tableRow{types.TableUncle, values} +} + +func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) { + var values []interface{} + values = append(values, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, + transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value) + csw.rows <- tableRow{types.TableTransaction, values} + indexerMetrics.transactions.Inc(1) +} + +func (csw *CSVWriter) upsertAccessListElement(accessListElement models.AccessListElementModel) { + var values []interface{} + values = append(values, accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address, accessListElement.StorageKeys) + csw.rows <- tableRow{types.TableAccessListElement, values} + indexerMetrics.accessListEntries.Inc(1) +} + +func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) { + var values []interface{} + values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, + rct.PostState, rct.PostStatus, rct.LogRoot) + csw.rows <- tableRow{types.TableReceipt, values} + indexerMetrics.receipts.Inc(1) +} + +func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) { + for _, l := range logs { + var values []interface{} + values = append(values, l.BlockNumber, l.HeaderID, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0, + l.Topic1, l.Topic2, l.Topic3, l.Data) + csw.rows <- tableRow{types.TableLog, values} + indexerMetrics.logs.Inc(1) + } +} + +func (csw *CSVWriter) upsertStateCID(stateNode models.StateNodeModel) { + var stateKey string + if stateNode.StateKey != nullHash.String() { + stateKey = stateNode.StateKey + } + + var values []interface{} + values = append(values, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, + stateNode.NodeType, true, stateNode.MhKey) + csw.rows <- tableRow{types.TableStateNode, values} +} + +func (csw *CSVWriter) upsertStateAccount(stateAccount models.StateAccountModel) { + var values []interface{} + values = append(values, stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance, + strconv.FormatUint(stateAccount.Nonce, 10), stateAccount.CodeHash, stateAccount.StorageRoot) + csw.rows <- tableRow{types.TableStateAccount, values} +} + +func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) { + var storageKey string + if storageCID.StorageKey != nullHash.String() { + storageKey = storageCID.StorageKey + } + + var values []interface{} + values = append(values, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, + storageCID.Path, storageCID.NodeType, true, storageCID.MhKey) + csw.rows <- tableRow{types.TableStorageNode, values} +} + +// LoadWatchedAddresses loads watched addresses from a file +func (csw *CSVWriter) loadWatchedAddresses() ([]common.Address, error) { + watchedAddressesFilePath := csw.watchedAddressesWriter.file.Name() + // load csv rows from watched addresses file + rows, err := loadWatchedAddressesRows(watchedAddressesFilePath) + if err != nil { + return nil, err + } + + // extract addresses from the csv rows + watchedAddresses := funk.Map(rows, func(row []string) common.Address { + // first column is for address in eth_meta.watched_addresses + addressString := row[0] + + return common.HexToAddress(addressString) + }).([]common.Address) + + return watchedAddresses, nil +} + +// InsertWatchedAddresses inserts the given addresses in a file +func (csw *CSVWriter) insertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + // load csv rows from watched addresses file + watchedAddresses, err := csw.loadWatchedAddresses() + if err != nil { + return err + } + + // append rows for new addresses to existing csv file + for _, arg := range args { + // ignore if already watched + if funk.Contains(watchedAddresses, common.HexToAddress(arg.Address)) { + continue + } + + var values []interface{} + values = append(values, arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0") + row := types.TableWatchedAddresses.ToCsvRow(values...) + + // writing directly instead of using rows channel as it needs to be flushed immediately + err = csw.watchedAddressesWriter.Write(row) + if err != nil { + return err + } + } + + // watched addresses need to be flushed immediately to the file to keep them in sync with in-memory watched addresses + csw.watchedAddressesWriter.Flush() + err = csw.watchedAddressesWriter.Error() + if err != nil { + return err + } + + return nil +} + +// RemoveWatchedAddresses removes the given watched addresses from a file +func (csw *CSVWriter) removeWatchedAddresses(args []sdtypes.WatchAddressArg) error { + // load csv rows from watched addresses file + watchedAddressesFilePath := csw.watchedAddressesWriter.file.Name() + rows, err := loadWatchedAddressesRows(watchedAddressesFilePath) + if err != nil { + return err + } + + // get rid of rows having addresses to be removed + filteredRows := funk.Filter(rows, func(row []string) bool { + return !funk.Contains(args, func(arg sdtypes.WatchAddressArg) bool { + // Compare first column in table for address + return arg.Address == row[0] + }) + }).([][]string) + + return dumpWatchedAddressesRows(csw.watchedAddressesWriter, filteredRows) +} + +// SetWatchedAddresses clears and inserts the given addresses in a file +func (csw *CSVWriter) setWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + var rows [][]string + for _, arg := range args { + row := types.TableWatchedAddresses.ToCsvRow(arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0") + rows = append(rows, row) + } + + return dumpWatchedAddressesRows(csw.watchedAddressesWriter, rows) +} + +// loadCSVWatchedAddresses loads csv rows from the given file +func loadWatchedAddressesRows(filePath string) ([][]string, error) { + file, err := os.Open(filePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return [][]string{}, nil + } + + return nil, fmt.Errorf("error opening watched addresses file: %v", err) + } + + defer file.Close() + reader := csv.NewReader(file) + + return reader.ReadAll() +} + +// dumpWatchedAddressesRows dumps csv rows to the given file +func dumpWatchedAddressesRows(watchedAddressesWriter fileWriter, filteredRows [][]string) error { + file := watchedAddressesWriter.file + file.Close() + + file, err := os.Create(file.Name()) + if err != nil { + return fmt.Errorf("error creating watched addresses file: %v", err) + } + + watchedAddressesWriter.Writer = csv.NewWriter(file) + watchedAddressesWriter.file = file + + for _, row := range filteredRows { + watchedAddressesWriter.Write(row) + } + + watchedAddressesWriter.Flush() + + return nil +} diff --git a/statediff/indexer/database/file/helpers.go b/statediff/indexer/database/file/helpers.go new file mode 100644 index 000000000000..dc635110cb96 --- /dev/null +++ b/statediff/indexer/database/file/helpers.go @@ -0,0 +1,60 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file + +import "bytes" + +// formatPostgresStringArray parses an array of strings into the proper Postgres string representation of that array +func formatPostgresStringArray(a []string) string { + if a == nil { + return "" + } + + if n := len(a); n > 0 { + // There will be at least two curly brackets, 2*N bytes of quotes, + // and N-1 bytes of delimiters. + b := make([]byte, 1, 1+3*n) + b[0] = '{' + + b = appendArrayQuotedBytes(b, []byte(a[0])) + for i := 1; i < n; i++ { + b = append(b, ',') + b = appendArrayQuotedBytes(b, []byte(a[i])) + } + + return string(append(b, '}')) + } + + return "{}" +} + +func appendArrayQuotedBytes(b, v []byte) []byte { + b = append(b, '"') + for { + i := bytes.IndexAny(v, `"\`) + if i < 0 { + b = append(b, v...) + break + } + if i > 0 { + b = append(b, v[:i]...) + } + b = append(b, '\\', v[i]) + v = v[i+1:] + } + return append(b, '"') +} diff --git a/statediff/indexer/database/file/indexer.go b/statediff/indexer/database/file/indexer.go new file mode 100644 index 000000000000..8103a68f4323 --- /dev/null +++ b/statediff/indexer/database/file/indexer.go @@ -0,0 +1,577 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file + +import ( + "context" + "errors" + "fmt" + "math/big" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +const defaultCSVOutputDir = "./statediff_output" +const defaultSQLFilePath = "./statediff.sql" +const defaultWatchedAddressesCSVFilePath = "./statediff-watched-addresses.csv" +const defaultWatchedAddressesSQLFilePath = "./statediff-watched-addresses.sql" + +const watchedAddressesInsert = "INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ('%s', '%d', '%d') ON CONFLICT (address) DO NOTHING;" + +var _ interfaces.StateDiffIndexer = &StateDiffIndexer{} + +var ( + indexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry) +) + +// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of a void +type StateDiffIndexer struct { + fileWriter FileWriter + chainConfig *params.ChainConfig + nodeID string + wg *sync.WaitGroup + removedCacheFlag *uint32 +} + +// NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer +func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, config Config) (*StateDiffIndexer, error) { + var err error + var writer FileWriter + + watchedAddressesFilePath := config.WatchedAddressesFilePath + + switch config.Mode { + case CSV: + outputDir := config.OutputDir + if outputDir == "" { + outputDir = defaultCSVOutputDir + } + + if _, err := os.Stat(outputDir); !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("cannot create output directory, directory (%s) already exists", outputDir) + } + log.Info("Writing statediff CSV files to directory", "file", outputDir) + + if watchedAddressesFilePath == "" { + watchedAddressesFilePath = defaultWatchedAddressesCSVFilePath + } + log.Info("Writing watched addresses to file", "file", watchedAddressesFilePath) + + writer, err = NewCSVWriter(outputDir, watchedAddressesFilePath) + if err != nil { + return nil, err + } + case SQL: + filePath := config.FilePath + if filePath == "" { + filePath = defaultSQLFilePath + } + if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("cannot create file, file (%s) already exists", filePath) + } + file, err := os.Create(filePath) + if err != nil { + return nil, fmt.Errorf("unable to create file (%s), err: %v", filePath, err) + } + log.Info("Writing statediff SQL statements to file", "file", filePath) + + if watchedAddressesFilePath == "" { + watchedAddressesFilePath = defaultWatchedAddressesSQLFilePath + } + log.Info("Writing watched addresses to file", "file", watchedAddressesFilePath) + + writer = NewSQLWriter(file, watchedAddressesFilePath) + default: + return nil, fmt.Errorf("unrecognized file mode: %s", config.Mode) + } + + wg := new(sync.WaitGroup) + writer.Loop() + writer.upsertNode(config.NodeInfo) + + return &StateDiffIndexer{ + fileWriter: writer, + chainConfig: chainConfig, + nodeID: config.NodeInfo.ID, + wg: wg, + }, nil +} + +// ReportDBMetrics has nothing to report for dump +func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {} + +// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts) +// Returns an initiated DB transaction which must be Closed via defer to commit or rollback +func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) { + sdi.removedCacheFlag = new(uint32) + start, t := time.Now(), time.Now() + blockHash := block.Hash() + blockHashStr := blockHash.String() + height := block.NumberU64() + traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr) + transactions := block.Transactions() + // Derive any missing fields + if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil { + return nil, err + } + + // Generate the block iplds + headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts) + if err != nil { + return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err) + } + + if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) { + return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs)) + } + if len(txTrieNodes) != len(rctTrieNodes) { + return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes)) + } + + // Calculate reward + var reward *big.Int + // in PoA networks block reward is 0 + if sdi.chainConfig.Clique != nil { + reward = big.NewInt(0) + } else { + reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts) + } + t = time.Now() + + blockTx := &BatchTx{ + BlockNumber: block.Number().String(), + submit: func(self *BatchTx, err error) error { + tDiff := time.Since(t) + indexerMetrics.tStateStoreCodeProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String()) + t = time.Now() + sdi.fileWriter.Flush() + tDiff = time.Since(t) + indexerMetrics.tPostgresCommit.Update(tDiff) + traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String()) + traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String()) + log.Debug(traceMsg) + return err + }, + } + tDiff := time.Since(t) + indexerMetrics.tFreePostgres.Update(tDiff) + traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String()) + t = time.Now() + + // write header, collect headerID + headerID := sdi.processHeader(block.Header(), headerNode, reward, totalDifficulty) + tDiff = time.Since(t) + indexerMetrics.tHeaderProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String()) + t = time.Now() + + // write uncles + sdi.processUncles(headerID, block.Number(), uncleNodes) + tDiff = time.Since(t) + indexerMetrics.tUncleProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String()) + t = time.Now() + + // write receipts and txs + err = sdi.processReceiptsAndTxs(processArgs{ + headerID: headerID, + blockNumber: block.Number(), + receipts: receipts, + txs: transactions, + rctNodes: rctNodes, + rctTrieNodes: rctTrieNodes, + txNodes: txNodes, + txTrieNodes: txTrieNodes, + logTrieNodes: logTrieNodes, + logLeafNodeCIDs: logLeafNodeCIDs, + rctLeafNodeCIDs: rctLeafNodeCIDs, + }) + if err != nil { + return nil, err + } + tDiff = time.Since(t) + indexerMetrics.tTxAndRecProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String()) + t = time.Now() + + return blockTx, err +} + +// processHeader write a header IPLD insert SQL stmt to a file +// it returns the headerID +func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode node.Node, reward, td *big.Int) string { + sdi.fileWriter.upsertIPLDNode(header.Number.String(), headerNode) + + var baseFee *string + if header.BaseFee != nil { + baseFee = new(string) + *baseFee = header.BaseFee.String() + } + headerID := header.Hash().String() + sdi.fileWriter.upsertHeaderCID(models.HeaderModel{ + NodeID: sdi.nodeID, + CID: headerNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(headerNode.Cid()), + ParentHash: header.ParentHash.String(), + BlockNumber: header.Number.String(), + BlockHash: headerID, + TotalDifficulty: td.String(), + Reward: reward.String(), + Bloom: header.Bloom.Bytes(), + StateRoot: header.Root.String(), + RctRoot: header.ReceiptHash.String(), + TxRoot: header.TxHash.String(), + UncleRoot: header.UncleHash.String(), + Timestamp: header.Time, + Coinbase: header.Coinbase.String(), + }) + return headerID +} + +// processUncles writes uncle IPLD insert SQL stmts to a file +func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber *big.Int, uncleNodes []*ipld2.EthHeader) { + // publish and index uncles + for _, uncleNode := range uncleNodes { + sdi.fileWriter.upsertIPLDNode(blockNumber.String(), uncleNode) + var uncleReward *big.Int + // in PoA networks uncle reward is 0 + if sdi.chainConfig.Clique != nil { + uncleReward = big.NewInt(0) + } else { + uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncleNode.Number.Uint64()) + } + sdi.fileWriter.upsertUncleCID(models.UncleModel{ + BlockNumber: blockNumber.String(), + HeaderID: headerID, + CID: uncleNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()), + ParentHash: uncleNode.ParentHash.String(), + BlockHash: uncleNode.Hash().String(), + Reward: uncleReward.String(), + }) + } +} + +// processArgs bundles arguments to processReceiptsAndTxs +type processArgs struct { + headerID string + blockNumber *big.Int + receipts types.Receipts + txs types.Transactions + rctNodes []*ipld2.EthReceipt + rctTrieNodes []*ipld2.EthRctTrie + txNodes []*ipld2.EthTx + txTrieNodes []*ipld2.EthTxTrie + logTrieNodes [][]node.Node + logLeafNodeCIDs [][]cid.Cid + rctLeafNodeCIDs []cid.Cid +} + +// processReceiptsAndTxs writes receipt and tx IPLD insert SQL stmts to a file +func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error { + // Process receipts and txs + signer := types.MakeSigner(sdi.chainConfig, args.blockNumber) + for i, receipt := range args.receipts { + for _, logTrieNode := range args.logTrieNodes[i] { + sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), logTrieNode) + } + txNode := args.txNodes[i] + sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), txNode) + + // index tx + trx := args.txs[i] + txID := trx.Hash().String() + + var val string + if trx.Value() != nil { + val = trx.Value().String() + } + + // derive sender for the tx that corresponds with this receipt + from, err := types.Sender(signer, trx) + if err != nil { + return fmt.Errorf("error deriving tx sender: %v", err) + } + txModel := models.TxModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + Dst: shared.HandleZeroAddrPointer(trx.To()), + Src: shared.HandleZeroAddr(from), + TxHash: txID, + Index: int64(i), + Data: trx.Data(), + CID: txNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(txNode.Cid()), + Type: trx.Type(), + Value: val, + } + sdi.fileWriter.upsertTransactionCID(txModel) + + // index access list if this is one + for j, accessListElement := range trx.AccessList() { + storageKeys := make([]string, len(accessListElement.StorageKeys)) + for k, storageKey := range accessListElement.StorageKeys { + storageKeys[k] = storageKey.Hex() + } + accessListElementModel := models.AccessListElementModel{ + BlockNumber: args.blockNumber.String(), + TxID: txID, + Index: int64(j), + Address: accessListElement.Address.Hex(), + StorageKeys: storageKeys, + } + sdi.fileWriter.upsertAccessListElement(accessListElementModel) + } + + // this is the contract address if this receipt is for a contract creation tx + contract := shared.HandleZeroAddr(receipt.ContractAddress) + var contractHash string + if contract != "" { + contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + } + + // index receipt + if !args.rctLeafNodeCIDs[i].Defined() { + return fmt.Errorf("invalid receipt leaf node cid") + } + + rctModel := &models.ReceiptModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + TxID: txID, + Contract: contract, + ContractHash: contractHash, + LeafCID: args.rctLeafNodeCIDs[i].String(), + LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]), + LogRoot: args.rctNodes[i].LogRoot.String(), + } + if len(receipt.PostState) == 0 { + rctModel.PostStatus = receipt.Status + } else { + rctModel.PostState = common.Bytes2Hex(receipt.PostState) + } + sdi.fileWriter.upsertReceiptCID(rctModel) + + // index logs + logDataSet := make([]*models.LogsModel, len(receipt.Logs)) + for idx, l := range receipt.Logs { + topicSet := make([]string, 4) + for ti, topic := range l.Topics { + topicSet[ti] = topic.Hex() + } + + if !args.logLeafNodeCIDs[i][idx].Defined() { + return fmt.Errorf("invalid log cid") + } + + logDataSet[idx] = &models.LogsModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + ReceiptID: txID, + Address: l.Address.String(), + Index: int64(l.Index), + Data: l.Data, + LeafCID: args.logLeafNodeCIDs[i][idx].String(), + LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]), + Topic0: topicSet[0], + Topic1: topicSet[1], + Topic2: topicSet[2], + Topic3: topicSet[3], + } + } + sdi.fileWriter.upsertLogCID(logDataSet) + } + + // publish trie nodes, these aren't indexed directly + for i, n := range args.txTrieNodes { + sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), n) + sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), args.rctTrieNodes[i]) + } + + return nil +} + +// PushStateNode writes a state diff node object (including any child storage nodes) IPLD insert SQL stmt to a file +func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { + tx, ok := batch.(*BatchTx) + if !ok { + return fmt.Errorf("file: batch is expected to be of type %T, got %T", &BatchTx{}, batch) + } + // publish the state node + var stateModel models.StateNodeModel + if stateNode.NodeType == sdtypes.Removed { + if atomic.LoadUint32(sdi.removedCacheFlag) == 0 { + atomic.StoreUint32(sdi.removedCacheFlag, 1) + sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, shared.RemovedNodeMhKey, []byte{}) + } + stateModel = models.StateNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: shared.RemovedNodeStateCID, + MhKey: shared.RemovedNodeMhKey, + NodeType: stateNode.NodeType.Int(), + } + } else { + stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(tx.BlockNumber, ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } + } + + // index the state node + sdi.fileWriter.upsertStateCID(stateModel) + + // if we have a leaf, decode and index the account data + if stateNode.NodeType == sdtypes.Leaf { + var i []interface{} + if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil { + return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements") + } + var account types.StateAccount + if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil { + return fmt.Errorf("error decoding state account rlp: %s", err.Error()) + } + accountModel := models.StateAccountModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Balance: account.Balance.String(), + Nonce: account.Nonce, + CodeHash: account.CodeHash, + StorageRoot: account.Root.String(), + } + sdi.fileWriter.upsertStateAccount(accountModel) + } + + // if there are any storage nodes associated with this node, publish and index them + for _, storageNode := range stateNode.StorageNodes { + if storageNode.NodeType == sdtypes.Removed { + if atomic.LoadUint32(sdi.removedCacheFlag) == 0 { + atomic.StoreUint32(sdi.removedCacheFlag, 1) + sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, shared.RemovedNodeMhKey, []byte{}) + } + storageModel := models.StorageNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).String(), + CID: shared.RemovedNodeStorageCID, + MhKey: shared.RemovedNodeMhKey, + NodeType: storageNode.NodeType.Int(), + } + sdi.fileWriter.upsertStorageCID(storageModel) + continue + } + storageCIDStr, storageMhKey, err := sdi.fileWriter.upsertIPLDRaw(tx.BlockNumber, ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err) + } + storageModel := models.StorageNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).String(), + CID: storageCIDStr, + MhKey: storageMhKey, + NodeType: storageNode.NodeType.Int(), + } + sdi.fileWriter.upsertStorageCID(storageModel) + } + + return nil +} + +// PushCodeAndCodeHash writes code and codehash pairs insert SQL stmts to a file +func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error { + tx, ok := batch.(*BatchTx) + if !ok { + return fmt.Errorf("file: batch is expected to be of type %T, got %T", &BatchTx{}, batch) + } + // codec doesn't matter since db key is multihash-based + mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash) + if err != nil { + return fmt.Errorf("error deriving multihash key from codehash: %v", err) + } + sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, mhKey, codeAndCodeHash.Code) + return nil +} + +// Close satisfies io.Closer +func (sdi *StateDiffIndexer) Close() error { + return sdi.fileWriter.Close() +} + +// LoadWatchedAddresses loads watched addresses from a file +func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) { + return sdi.fileWriter.loadWatchedAddresses() +} + +// InsertWatchedAddresses inserts the given addresses in a file +func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + return sdi.fileWriter.insertWatchedAddresses(args, currentBlockNumber) +} + +// RemoveWatchedAddresses removes the given watched addresses from a file +func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error { + return sdi.fileWriter.removeWatchedAddresses(args) +} + +// SetWatchedAddresses clears and inserts the given addresses in a file +func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + return sdi.fileWriter.setWatchedAddresses(args, currentBlockNumber) +} + +// ClearWatchedAddresses clears all the watched addresses from a file +func (sdi *StateDiffIndexer) ClearWatchedAddresses() error { + return sdi.SetWatchedAddresses([]sdtypes.WatchAddressArg{}, big.NewInt(0)) +} diff --git a/statediff/indexer/database/file/interfaces.go b/statediff/indexer/database/file/interfaces.go new file mode 100644 index 000000000000..271257dce9a2 --- /dev/null +++ b/statediff/indexer/database/file/interfaces.go @@ -0,0 +1,60 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file + +import ( + "math/big" + + node "github.com/ipfs/go-ipld-format" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/types" +) + +// Writer interface required by the file indexer +type FileWriter interface { + // Methods used to control the writer + Loop() + Close() error + Flush() + + // Methods to upsert ethereum data model objects + upsertNode(node nodeinfo.Info) + upsertHeaderCID(header models.HeaderModel) + upsertUncleCID(uncle models.UncleModel) + upsertTransactionCID(transaction models.TxModel) + upsertAccessListElement(accessListElement models.AccessListElementModel) + upsertReceiptCID(rct *models.ReceiptModel) + upsertLogCID(logs []*models.LogsModel) + upsertStateCID(stateNode models.StateNodeModel) + upsertStateAccount(stateAccount models.StateAccountModel) + upsertStorageCID(storageCID models.StorageNodeModel) + upsertIPLD(ipld models.IPLDModel) + + // Methods to upsert IPLD in different ways + upsertIPLDDirect(blockNumber, key string, value []byte) + upsertIPLDNode(blockNumber string, i node.Node) + upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error) + + // Methods to read and write watched addresses + loadWatchedAddresses() ([]common.Address, error) + insertWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error + removeWatchedAddresses(args []types.WatchAddressArg) error + setWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error +} diff --git a/statediff/indexer/database/file/mainnet_tests/indexer_test.go b/statediff/indexer/database/file/mainnet_tests/indexer_test.go new file mode 100644 index 000000000000..392fb2ee37a0 --- /dev/null +++ b/statediff/indexer/database/file/mainnet_tests/indexer_test.go @@ -0,0 +1,112 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package mainnet_tests + +import ( + "context" + "errors" + "fmt" + "math/big" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/test" + "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" +) + +var ( + err error + db sql.Database + ind interfaces.StateDiffIndexer + chainConf = params.MainnetChainConfig +) + +func init() { + if os.Getenv("MODE") != "statediff" { + fmt.Println("Skipping statediff test") + os.Exit(0) + } + if os.Getenv("STATEDIFF_DB") != "file" { + fmt.Println("Skipping statediff .sql file writing mode test") + os.Exit(0) + } +} + +func TestPushBlockAndState(t *testing.T) { + conf := test_helpers.GetTestConfig() + + for _, blockNumber := range test_helpers.ProblemBlocks { + conf.BlockNumber = big.NewInt(blockNumber) + tb, trs, err := test_helpers.TestBlockAndReceipts(conf) + require.NoError(t, err) + + testPushBlockAndState(t, tb, trs) + } + + testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf) + require.NoError(t, err) + + testPushBlockAndState(t, testBlock, testReceipts) +} + +func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) { + t.Run("Test PushBlock and PushStateNode", func(t *testing.T) { + setupMainnetIndexer(t) + defer dumpData(t) + defer tearDown(t) + + test.TestBlock(t, ind, block, receipts) + }) +} + +func setupMainnetIndexer(t *testing.T) { + if _, err := os.Stat(file.CSVTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) { + err := os.Remove(file.CSVTestConfig.FilePath) + require.NoError(t, err) + } + + ind, err = file.NewStateDiffIndexer(context.Background(), chainConf, file.CSVTestConfig) + require.NoError(t, err) + + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } +} + +func dumpData(t *testing.T) { + sqlFileBytes, err := os.ReadFile(file.CSVTestConfig.FilePath) + require.NoError(t, err) + + _, err = db.Exec(context.Background(), string(sqlFileBytes)) + require.NoError(t, err) +} + +func tearDown(t *testing.T) { + test_helpers.TearDownDB(t, db) + require.NoError(t, db.Close()) + + require.NoError(t, os.Remove(file.CSVTestConfig.FilePath)) +} diff --git a/statediff/indexer/database/file/metrics.go b/statediff/indexer/database/file/metrics.go new file mode 100644 index 000000000000..ca6e88f2b88b --- /dev/null +++ b/statediff/indexer/database/file/metrics.go @@ -0,0 +1,94 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file + +import ( + "strings" + + "github.com/ethereum/go-ethereum/metrics" +) + +const ( + namespace = "statediff" +) + +// Build a fully qualified metric name +func metricName(subsystem, name string) string { + if name == "" { + return "" + } + parts := []string{namespace, name} + if subsystem != "" { + parts = []string{namespace, subsystem, name} + } + // Prometheus uses _ but geth metrics uses / and replaces + return strings.Join(parts, "/") +} + +type indexerMetricsHandles struct { + // The total number of processed blocks + blocks metrics.Counter + // The total number of processed transactions + transactions metrics.Counter + // The total number of processed receipts + receipts metrics.Counter + // The total number of processed logs + logs metrics.Counter + // The total number of access list entries processed + accessListEntries metrics.Counter + // Time spent waiting for free postgres tx + tFreePostgres metrics.Timer + // Postgres transaction commit duration + tPostgresCommit metrics.Timer + // Header processing time + tHeaderProcessing metrics.Timer + // Uncle processing time + tUncleProcessing metrics.Timer + // Tx and receipt processing time + tTxAndRecProcessing metrics.Timer + // State, storage, and code combined processing time + tStateStoreCodeProcessing metrics.Timer +} + +func RegisterIndexerMetrics(reg metrics.Registry) indexerMetricsHandles { + ctx := indexerMetricsHandles{ + blocks: metrics.NewCounter(), + transactions: metrics.NewCounter(), + receipts: metrics.NewCounter(), + logs: metrics.NewCounter(), + accessListEntries: metrics.NewCounter(), + tFreePostgres: metrics.NewTimer(), + tPostgresCommit: metrics.NewTimer(), + tHeaderProcessing: metrics.NewTimer(), + tUncleProcessing: metrics.NewTimer(), + tTxAndRecProcessing: metrics.NewTimer(), + tStateStoreCodeProcessing: metrics.NewTimer(), + } + subsys := "indexer" + reg.Register(metricName(subsys, "blocks"), ctx.blocks) + reg.Register(metricName(subsys, "transactions"), ctx.transactions) + reg.Register(metricName(subsys, "receipts"), ctx.receipts) + reg.Register(metricName(subsys, "logs"), ctx.logs) + reg.Register(metricName(subsys, "access_list_entries"), ctx.accessListEntries) + reg.Register(metricName(subsys, "t_free_postgres"), ctx.tFreePostgres) + reg.Register(metricName(subsys, "t_postgres_commit"), ctx.tPostgresCommit) + reg.Register(metricName(subsys, "t_header_processing"), ctx.tHeaderProcessing) + reg.Register(metricName(subsys, "t_uncle_processing"), ctx.tUncleProcessing) + reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.tTxAndRecProcessing) + reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.tStateStoreCodeProcessing) + return ctx +} diff --git a/statediff/indexer/database/file/sql_indexer_legacy_test.go b/statediff/indexer/database/file/sql_indexer_legacy_test.go new file mode 100644 index 000000000000..02ced177efc0 --- /dev/null +++ b/statediff/indexer/database/file/sql_indexer_legacy_test.go @@ -0,0 +1,101 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file_test + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/test" + "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" +) + +var ( + db sql.Database + err error + ind interfaces.StateDiffIndexer +) + +func setupLegacySQLIndexer(t *testing.T) { + if _, err := os.Stat(file.SQLTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) { + err := os.Remove(file.SQLTestConfig.FilePath) + require.NoError(t, err) + } + + ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.SQLTestConfig) + require.NoError(t, err) + + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } +} + +func setupLegacySQL(t *testing.T) { + setupLegacySQLIndexer(t) + test.SetupLegacyTestData(t, ind) +} + +func dumpFileData(t *testing.T) { + err := test_helpers.DedupFile(file.SQLTestConfig.FilePath) + require.NoError(t, err) + + sqlFileBytes, err := os.ReadFile(file.SQLTestConfig.FilePath) + require.NoError(t, err) + + _, err = db.Exec(context.Background(), string(sqlFileBytes)) + require.NoError(t, err) +} + +func resetAndDumpWatchedAddressesFileData(t *testing.T) { + test_helpers.TearDownDB(t, db) + + sqlFileBytes, err := os.ReadFile(file.SQLTestConfig.WatchedAddressesFilePath) + require.NoError(t, err) + + _, err = db.Exec(context.Background(), string(sqlFileBytes)) + require.NoError(t, err) +} + +func tearDown(t *testing.T) { + test_helpers.TearDownDB(t, db) + require.NoError(t, db.Close()) + + require.NoError(t, os.Remove(file.SQLTestConfig.FilePath)) + + if err := os.Remove(file.SQLTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) { + require.NoError(t, err) + } +} + +func TestLegacySQLFileIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs", func(t *testing.T) { + setupLegacySQL(t) + dumpFileData(t) + defer tearDown(t) + + test.TestLegacyIndexer(t, db) + }) +} diff --git a/statediff/indexer/database/file/sql_indexer_test.go b/statediff/indexer/database/file/sql_indexer_test.go new file mode 100644 index 000000000000..0a73a8c47df9 --- /dev/null +++ b/statediff/indexer/database/file/sql_indexer_test.go @@ -0,0 +1,253 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file_test + +import ( + "context" + "errors" + "math/big" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/test" +) + +func setupIndexer(t *testing.T) { + if _, err := os.Stat(file.SQLTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) { + err := os.Remove(file.SQLTestConfig.FilePath) + require.NoError(t, err) + } + + if _, err := os.Stat(file.SQLTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) { + err := os.Remove(file.SQLTestConfig.WatchedAddressesFilePath) + require.NoError(t, err) + } + + ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.SQLTestConfig) + require.NoError(t, err) + + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } +} + +func setup(t *testing.T) { + setupIndexer(t) + test.SetupTestData(t, ind) +} + +func setupSQLNonCanonical(t *testing.T) { + setupIndexer(t) + test.SetupTestDataNonCanonical(t, ind) +} + +func TestSQLFileIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { + setup(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexHeaderIPLDs(t, db) + }) + + t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { + setup(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexTransactionIPLDs(t, db) + }) + + t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) { + setup(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexLogIPLDs(t, db) + }) + + t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) { + setup(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexReceiptIPLDs(t, db) + }) + + t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) { + setup(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexStateIPLDs(t, db) + }) + + t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { + setup(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexStorageIPLDs(t, db) + }) +} + +func TestSQLFileIndexerNonCanonical(t *testing.T) { + t.Run("Publish and index header", func(t *testing.T) { + setupSQLNonCanonical(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexHeaderNonCanonical(t, db) + }) + + t.Run("Publish and index transactions", func(t *testing.T) { + setupSQLNonCanonical(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexTransactionsNonCanonical(t, db) + }) + + t.Run("Publish and index receipts", func(t *testing.T) { + setupSQLNonCanonical(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexReceiptsNonCanonical(t, db) + }) + + t.Run("Publish and index logs", func(t *testing.T) { + setupSQLNonCanonical(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexLogsNonCanonical(t, db) + }) + + t.Run("Publish and index state nodes", func(t *testing.T) { + setupSQLNonCanonical(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexStateNonCanonical(t, db) + }) + + t.Run("Publish and index storage nodes", func(t *testing.T) { + setupSQLNonCanonical(t) + dumpFileData(t) + defer tearDown(t) + + test.TestPublishAndIndexStorageNonCanonical(t, db) + }) +} + +func TestSQLFileWatchAddressMethods(t *testing.T) { + setupIndexer(t) + defer tearDown(t) + + t.Run("Load watched addresses (empty table)", func(t *testing.T) { + test.TestLoadEmptyWatchedAddresses(t, ind) + }) + + t.Run("Insert watched addresses", func(t *testing.T) { + args := mocks.GetInsertWatchedAddressesArgs() + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1))) + require.NoError(t, err) + + resetAndDumpWatchedAddressesFileData(t) + + test.TestInsertWatchedAddresses(t, db) + }) + + t.Run("Insert watched addresses (some already watched)", func(t *testing.T) { + args := mocks.GetInsertAlreadyWatchedAddressesArgs() + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2))) + require.NoError(t, err) + + resetAndDumpWatchedAddressesFileData(t) + + test.TestInsertAlreadyWatchedAddresses(t, db) + }) + + t.Run("Remove watched addresses", func(t *testing.T) { + args := mocks.GetRemoveWatchedAddressesArgs() + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + resetAndDumpWatchedAddressesFileData(t) + + test.TestRemoveWatchedAddresses(t, db) + }) + + t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) { + args := mocks.GetRemoveNonWatchedAddressesArgs() + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + resetAndDumpWatchedAddressesFileData(t) + + test.TestRemoveNonWatchedAddresses(t, db) + }) + + t.Run("Set watched addresses", func(t *testing.T) { + args := mocks.GetSetWatchedAddressesArgs() + err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2))) + require.NoError(t, err) + + resetAndDumpWatchedAddressesFileData(t) + + test.TestSetWatchedAddresses(t, db) + }) + + t.Run("Set watched addresses (some already watched)", func(t *testing.T) { + args := mocks.GetSetAlreadyWatchedAddressesArgs() + err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3))) + require.NoError(t, err) + + resetAndDumpWatchedAddressesFileData(t) + + test.TestSetAlreadyWatchedAddresses(t, db) + }) + + t.Run("Load watched addresses", func(t *testing.T) { + test.TestLoadWatchedAddresses(t, ind) + }) + + t.Run("Clear watched addresses", func(t *testing.T) { + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + resetAndDumpWatchedAddressesFileData(t) + + test.TestClearWatchedAddresses(t, db) + }) + + t.Run("Clear watched addresses (empty table)", func(t *testing.T) { + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + resetAndDumpWatchedAddressesFileData(t) + + test.TestClearEmptyWatchedAddresses(t, db) + }) +} diff --git a/statediff/indexer/database/file/sql_writer.go b/statediff/indexer/database/file/sql_writer.go new file mode 100644 index 000000000000..b947fada93a4 --- /dev/null +++ b/statediff/indexer/database/file/sql_writer.go @@ -0,0 +1,429 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package file + +import ( + "bufio" + "errors" + "fmt" + "io" + "math/big" + "os" + + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + node "github.com/ipfs/go-ipld-format" + pg_query "github.com/pganalyze/pg_query_go/v2" + "github.com/thoas/go-funk" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/types" +) + +var ( + nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") + pipeSize = 65336 // min(linuxPipeSize, macOSPipeSize) + writeBufferSize = pipeSize * 16 * 96 +) + +// SQLWriter writes sql statements to a file +type SQLWriter struct { + wc io.WriteCloser + stmts chan []byte + collatedStmt []byte + collationIndex int + + flushChan chan struct{} + flushFinished chan struct{} + quitChan chan struct{} + doneChan chan struct{} + + watchedAddressesFilePath string +} + +// NewSQLWriter creates a new pointer to a Writer +func NewSQLWriter(wc io.WriteCloser, watchedAddressesFilePath string) *SQLWriter { + return &SQLWriter{ + wc: wc, + stmts: make(chan []byte), + collatedStmt: make([]byte, writeBufferSize), + flushChan: make(chan struct{}), + flushFinished: make(chan struct{}), + quitChan: make(chan struct{}), + doneChan: make(chan struct{}), + watchedAddressesFilePath: watchedAddressesFilePath, + } +} + +// Loop enables concurrent writes to the underlying os.File +// since os.File does not buffer, it utilizes an internal buffer that is the size of a unix pipe +// by using copy() and tracking the index/size of the buffer, we require only the initial memory allocation +func (sqw *SQLWriter) Loop() { + sqw.collationIndex = 0 + go func() { + defer close(sqw.doneChan) + var l int + for { + select { + case stmt := <-sqw.stmts: + l = len(stmt) + if sqw.collationIndex+l > writeBufferSize { + if err := sqw.flush(); err != nil { + panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err)) + } + if l > writeBufferSize { + if _, err := sqw.wc.Write(stmt); err != nil { + panic(fmt.Sprintf("error writing large sql stmt to file: %v", err)) + } + continue + } + } + copy(sqw.collatedStmt[sqw.collationIndex:sqw.collationIndex+l], stmt) + sqw.collationIndex += l + case <-sqw.quitChan: + if err := sqw.flush(); err != nil { + panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err)) + } + return + case <-sqw.flushChan: + if err := sqw.flush(); err != nil { + panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err)) + } + sqw.flushFinished <- struct{}{} + } + } + }() +} + +// Close satisfies io.Closer +func (sqw *SQLWriter) Close() error { + close(sqw.quitChan) + <-sqw.doneChan + close(sqw.stmts) + close(sqw.flushChan) + close(sqw.flushFinished) + return sqw.wc.Close() +} + +// Flush sends a flush signal to the looping process +func (sqw *SQLWriter) Flush() { + sqw.flushChan <- struct{}{} + <-sqw.flushFinished +} + +func (sqw *SQLWriter) flush() error { + if _, err := sqw.wc.Write(sqw.collatedStmt[0:sqw.collationIndex]); err != nil { + return err + } + sqw.collationIndex = 0 + return nil +} + +const ( + nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES " + + "('%s', '%s', '%s', '%s', %d);\n" + + ipldInsert = "INSERT INTO public.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n" + + headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, " + + "state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES " + + "('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %d, '%s');\n" + + uncleInsert = "INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES " + + "('%s', '%s', '%s', '%s', '%s', '%s', '%s');\n" + + txInsert = "INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, " + + "value) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '\\x%x', %d, '%s');\n" + + alInsert = "INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES " + + "('%s', '%s', %d, '%s', '%s');\n" + + rctInsert = "INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, " + + "post_status, log_root) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s');\n" + + logInsert = "INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, " + + "topic3, log_data) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '\\x%x');\n" + + stateInsert = "INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) " + + "VALUES ('%s', '%s', '%s', '%s', '\\x%x', %d, %t, '%s');\n" + + accountInsert = "INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) " + + "VALUES ('%s', '%s', '\\x%x', '%s', %d, '\\x%x', '%s');\n" + + storageInsert = "INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, " + + "node_type, diff, mh_key) VALUES ('%s', '%s', '\\x%x', '%s', '%s', '\\x%x', %d, %t, '%s');\n" +) + +func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) { + sqw.stmts <- []byte(fmt.Sprintf(nodeInsert, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID)) +} + +func (sqw *SQLWriter) upsertIPLD(ipld models.IPLDModel) { + sqw.stmts <- []byte(fmt.Sprintf(ipldInsert, ipld.BlockNumber, ipld.Key, ipld.Data)) +} + +func (sqw *SQLWriter) upsertIPLDDirect(blockNumber, key string, value []byte) { + sqw.upsertIPLD(models.IPLDModel{ + BlockNumber: blockNumber, + Key: key, + Data: value, + }) +} + +func (sqw *SQLWriter) upsertIPLDNode(blockNumber string, i node.Node) { + sqw.upsertIPLD(models.IPLDModel{ + BlockNumber: blockNumber, + Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(), + Data: i.RawData(), + }) +} + +func (sqw *SQLWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error) { + c, err := ipld.RawdataToCid(codec, raw, mh) + if err != nil { + return "", "", err + } + prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String() + sqw.upsertIPLD(models.IPLDModel{ + BlockNumber: blockNumber, + Key: prefixedKey, + Data: raw, + }) + return c.String(), prefixedKey, err +} + +func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) { + stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, + header.TotalDifficulty, header.NodeID, header.Reward, header.StateRoot, header.TxRoot, + header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1, header.Coinbase) + sqw.stmts <- []byte(stmt) + indexerMetrics.blocks.Inc(1) +} + +func (sqw *SQLWriter) upsertUncleCID(uncle models.UncleModel) { + sqw.stmts <- []byte(fmt.Sprintf(uncleInsert, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, + uncle.Reward, uncle.MhKey)) +} + +func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) { + sqw.stmts <- []byte(fmt.Sprintf(txInsert, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, + transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value)) + indexerMetrics.transactions.Inc(1) +} + +func (sqw *SQLWriter) upsertAccessListElement(accessListElement models.AccessListElementModel) { + sqw.stmts <- []byte(fmt.Sprintf(alInsert, accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address, + formatPostgresStringArray(accessListElement.StorageKeys))) + indexerMetrics.accessListEntries.Inc(1) +} + +func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) { + sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, + rct.PostState, rct.PostStatus, rct.LogRoot)) + indexerMetrics.receipts.Inc(1) +} + +func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) { + for _, l := range logs { + sqw.stmts <- []byte(fmt.Sprintf(logInsert, l.BlockNumber, l.HeaderID, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0, + l.Topic1, l.Topic2, l.Topic3, l.Data)) + indexerMetrics.logs.Inc(1) + } +} + +func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) { + var stateKey string + if stateNode.StateKey != nullHash.String() { + stateKey = stateNode.StateKey + } + sqw.stmts <- []byte(fmt.Sprintf(stateInsert, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, + stateNode.NodeType, true, stateNode.MhKey)) +} + +func (sqw *SQLWriter) upsertStateAccount(stateAccount models.StateAccountModel) { + sqw.stmts <- []byte(fmt.Sprintf(accountInsert, stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance, + stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot)) +} + +func (sqw *SQLWriter) upsertStorageCID(storageCID models.StorageNodeModel) { + var storageKey string + if storageCID.StorageKey != nullHash.String() { + storageKey = storageCID.StorageKey + } + sqw.stmts <- []byte(fmt.Sprintf(storageInsert, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, + storageCID.Path, storageCID.NodeType, true, storageCID.MhKey)) +} + +// LoadWatchedAddresses loads watched addresses from a file +func (sqw *SQLWriter) loadWatchedAddresses() ([]common.Address, error) { + // load sql statements from watched addresses file + stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath) + if err != nil { + return nil, err + } + + // extract addresses from the sql statements + watchedAddresses := []common.Address{} + for _, stmt := range stmts { + addressString, err := parseWatchedAddressStatement(stmt) + if err != nil { + return nil, err + } + watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString)) + } + + return watchedAddresses, nil +} + +// InsertWatchedAddresses inserts the given addresses in a file +func (sqw *SQLWriter) insertWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error { + // load sql statements from watched addresses file + stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath) + if err != nil { + return err + } + + // get already watched addresses + var watchedAddresses []string + for _, stmt := range stmts { + addressString, err := parseWatchedAddressStatement(stmt) + if err != nil { + return err + } + + watchedAddresses = append(watchedAddresses, addressString) + } + + // append statements for new addresses to existing statements + for _, arg := range args { + // ignore if already watched + if funk.Contains(watchedAddresses, arg.Address) { + continue + } + + stmt := fmt.Sprintf(watchedAddressesInsert, arg.Address, arg.CreatedAt, currentBlockNumber.Uint64()) + stmts = append(stmts, stmt) + } + + return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, stmts) +} + +// RemoveWatchedAddresses removes the given watched addresses from a file +func (sqw *SQLWriter) removeWatchedAddresses(args []types.WatchAddressArg) error { + // load sql statements from watched addresses file + stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath) + if err != nil { + return err + } + + // get rid of statements having addresses to be removed + var filteredStmts []string + for _, stmt := range stmts { + addressString, err := parseWatchedAddressStatement(stmt) + if err != nil { + return err + } + + toRemove := funk.Contains(args, func(arg types.WatchAddressArg) bool { + return arg.Address == addressString + }) + + if !toRemove { + filteredStmts = append(filteredStmts, stmt) + } + } + + return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, filteredStmts) +} + +// SetWatchedAddresses clears and inserts the given addresses in a file +func (sqw *SQLWriter) setWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error { + var stmts []string + for _, arg := range args { + stmt := fmt.Sprintf(watchedAddressesInsert, arg.Address, arg.CreatedAt, currentBlockNumber.Uint64()) + stmts = append(stmts, stmt) + } + + return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, stmts) +} + +// loadWatchedAddressesStatements loads sql statements from the given file in a string slice +func loadWatchedAddressesStatements(filePath string) ([]string, error) { + file, err := os.Open(filePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return []string{}, nil + } + + return nil, fmt.Errorf("error opening watched addresses file: %v", err) + } + defer file.Close() + + stmts := []string{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + stmts = append(stmts, scanner.Text()) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error loading watched addresses: %v", err) + } + + return stmts, nil +} + +// dumpWatchedAddressesStatements dumps sql statements to the given file +func dumpWatchedAddressesStatements(filePath string, stmts []string) error { + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("error creating watched addresses file: %v", err) + } + defer file.Close() + + for _, stmt := range stmts { + _, err := file.Write([]byte(stmt + "\n")) + if err != nil { + return fmt.Errorf("error inserting watched_addresses entry: %v", err) + } + } + + return nil +} + +// parseWatchedAddressStatement parses given sql insert statement to extract the address argument +func parseWatchedAddressStatement(stmt string) (string, error) { + parseResult, err := pg_query.Parse(stmt) + if err != nil { + return "", fmt.Errorf("error parsing sql stmt: %v", err) + } + + // extract address argument from parse output for a SQL statement of form + // "INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) + // VALUES ('0xabc', '123', '130') ON CONFLICT (address) DO NOTHING;" + addressString := parseResult.Stmts[0].Stmt.GetInsertStmt(). + SelectStmt.GetSelectStmt(). + ValuesLists[0].GetList(). + Items[0].GetAConst(). + GetVal(). + GetString_(). + Str + + return addressString, nil +} diff --git a/statediff/indexer/database/file/types/schema.go b/statediff/indexer/database/file/types/schema.go new file mode 100644 index 000000000000..ea0daefd615f --- /dev/null +++ b/statediff/indexer/database/file/types/schema.go @@ -0,0 +1,186 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +var TableIPLDBlock = Table{ + `public.blocks`, + []column{ + {name: "block_number", dbType: bigint}, + {name: "key", dbType: text}, + {name: "data", dbType: bytea}, + }, +} + +var TableNodeInfo = Table{ + Name: `public.nodes`, + Columns: []column{ + {name: "genesis_block", dbType: varchar}, + {name: "network_id", dbType: varchar}, + {name: "node_id", dbType: varchar}, + {name: "client_name", dbType: varchar}, + {name: "chain_id", dbType: integer}, + }, +} + +var TableHeader = Table{ + "eth.header_cids", + []column{ + {name: "block_number", dbType: bigint}, + {name: "block_hash", dbType: varchar}, + {name: "parent_hash", dbType: varchar}, + {name: "cid", dbType: text}, + {name: "td", dbType: numeric}, + {name: "node_id", dbType: varchar}, + {name: "reward", dbType: numeric}, + {name: "state_root", dbType: varchar}, + {name: "tx_root", dbType: varchar}, + {name: "receipt_root", dbType: varchar}, + {name: "uncle_root", dbType: varchar}, + {name: "bloom", dbType: bytea}, + {name: "timestamp", dbType: numeric}, + {name: "mh_key", dbType: text}, + {name: "times_validated", dbType: integer}, + {name: "coinbase", dbType: varchar}, + }, +} + +var TableStateNode = Table{ + "eth.state_cids", + []column{ + {name: "block_number", dbType: bigint}, + {name: "header_id", dbType: varchar}, + {name: "state_leaf_key", dbType: varchar}, + {name: "cid", dbType: text}, + {name: "state_path", dbType: bytea}, + {name: "node_type", dbType: integer}, + {name: "diff", dbType: boolean}, + {name: "mh_key", dbType: text}, + }, +} + +var TableStorageNode = Table{ + "eth.storage_cids", + []column{ + {name: "block_number", dbType: bigint}, + {name: "header_id", dbType: varchar}, + {name: "state_path", dbType: bytea}, + {name: "storage_leaf_key", dbType: varchar}, + {name: "cid", dbType: text}, + {name: "storage_path", dbType: bytea}, + {name: "node_type", dbType: integer}, + {name: "diff", dbType: boolean}, + {name: "mh_key", dbType: text}, + }, +} + +var TableUncle = Table{ + "eth.uncle_cids", + []column{ + {name: "block_number", dbType: bigint}, + {name: "block_hash", dbType: varchar}, + {name: "header_id", dbType: varchar}, + {name: "parent_hash", dbType: varchar}, + {name: "cid", dbType: text}, + {name: "reward", dbType: numeric}, + {name: "mh_key", dbType: text}, + }, +} + +var TableTransaction = Table{ + "eth.transaction_cids", + []column{ + {name: "block_number", dbType: bigint}, + {name: "header_id", dbType: varchar}, + {name: "tx_hash", dbType: varchar}, + {name: "cid", dbType: text}, + {name: "dst", dbType: varchar}, + {name: "src", dbType: varchar}, + {name: "index", dbType: integer}, + {name: "mh_key", dbType: text}, + {name: "tx_data", dbType: bytea}, + {name: "tx_type", dbType: integer}, + {name: "value", dbType: numeric}, + }, +} + +var TableAccessListElement = Table{ + "eth.access_list_elements", + []column{ + {name: "block_number", dbType: bigint}, + {name: "tx_id", dbType: varchar}, + {name: "index", dbType: integer}, + {name: "address", dbType: varchar}, + {name: "storage_keys", dbType: varchar, isArray: true}, + }, +} + +var TableReceipt = Table{ + "eth.receipt_cids", + []column{ + {name: "block_number", dbType: bigint}, + {name: "header_id", dbType: varchar}, + {name: "tx_id", dbType: varchar}, + {name: "leaf_cid", dbType: text}, + {name: "contract", dbType: varchar}, + {name: "contract_hash", dbType: varchar}, + {name: "leaf_mh_key", dbType: text}, + {name: "post_state", dbType: varchar}, + {name: "post_status", dbType: integer}, + {name: "log_root", dbType: varchar}, + }, +} + +var TableLog = Table{ + "eth.log_cids", + []column{ + {name: "block_number", dbType: bigint}, + {name: "header_id", dbType: varchar}, + {name: "leaf_cid", dbType: text}, + {name: "leaf_mh_key", dbType: text}, + {name: "rct_id", dbType: varchar}, + {name: "address", dbType: varchar}, + {name: "index", dbType: integer}, + {name: "topic0", dbType: varchar}, + {name: "topic1", dbType: varchar}, + {name: "topic2", dbType: varchar}, + {name: "topic3", dbType: varchar}, + {name: "log_data", dbType: bytea}, + }, +} + +var TableStateAccount = Table{ + "eth.state_accounts", + []column{ + {name: "block_number", dbType: bigint}, + {name: "header_id", dbType: varchar}, + {name: "state_path", dbType: bytea}, + {name: "balance", dbType: numeric}, + {name: "nonce", dbType: bigint}, + {name: "code_hash", dbType: bytea}, + {name: "storage_root", dbType: varchar}, + }, +} + +var TableWatchedAddresses = Table{ + "eth_meta.watched_addresses", + []column{ + {name: "address", dbType: varchar}, + {name: "created_at", dbType: bigint}, + {name: "watched_at", dbType: bigint}, + {name: "last_filled_at", dbType: bigint}, + }, +} diff --git a/statediff/indexer/database/file/types/table.go b/statediff/indexer/database/file/types/table.go new file mode 100644 index 000000000000..d7fd5af6c5b8 --- /dev/null +++ b/statediff/indexer/database/file/types/table.go @@ -0,0 +1,104 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "fmt" + "strings" + + "github.com/thoas/go-funk" +) + +type colType int + +const ( + integer colType = iota + boolean + bigint + numeric + bytea + varchar + text +) + +type column struct { + name string + dbType colType + isArray bool +} +type Table struct { + Name string + Columns []column +} + +func (tbl *Table) ToCsvRow(args ...interface{}) []string { + var row []string + for i, col := range tbl.Columns { + value := col.dbType.formatter()(args[i]) + + if col.isArray { + valueList := funk.Map(args[i], col.dbType.formatter()).([]string) + value = fmt.Sprintf("{%s}", strings.Join(valueList, ",")) + } + + row = append(row, value) + } + return row +} + +func (tbl *Table) VarcharColumns() []string { + columns := funk.Filter(tbl.Columns, func(col column) bool { + return col.dbType == varchar + }).([]column) + + columnNames := funk.Map(columns, func(col column) string { + return col.name + }).([]string) + + return columnNames +} + +type colfmt = func(interface{}) string + +func sprintf(f string) colfmt { + return func(x interface{}) string { return fmt.Sprintf(f, x) } +} + +func (typ colType) formatter() colfmt { + switch typ { + case integer: + return sprintf("%d") + case boolean: + return func(x interface{}) string { + if x.(bool) { + return "t" + } + return "f" + } + case bigint: + return sprintf("%s") + case numeric: + return sprintf("%s") + case bytea: + return sprintf(`\x%x`) + case varchar: + return sprintf("%s") + case text: + return sprintf("%s") + } + panic("unreachable") +} diff --git a/statediff/indexer/database/sql/batch_tx.go b/statediff/indexer/database/sql/batch_tx.go new file mode 100644 index 000000000000..06bb49c9ed3f --- /dev/null +++ b/statediff/indexer/database/sql/batch_tx.go @@ -0,0 +1,125 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "context" + "sync/atomic" + + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + node "github.com/ipfs/go-ipld-format" + "github.com/lib/pq" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" +) + +const startingCacheCapacity = 1024 * 24 + +// BatchTx wraps a sql tx with the state necessary for building the tx concurrently during trie difference iteration +type BatchTx struct { + BlockNumber string + ctx context.Context + dbtx Tx + stm string + quit chan struct{} + iplds chan models.IPLDModel + ipldCache models.IPLDBatch + removedCacheFlag *uint32 + + submit func(blockTx *BatchTx, err error) error +} + +// Submit satisfies indexer.AtomicTx +func (tx *BatchTx) Submit(err error) error { + return tx.submit(tx, err) +} + +func (tx *BatchTx) flush() error { + _, err := tx.dbtx.Exec(tx.ctx, tx.stm, pq.Array(tx.ipldCache.BlockNumbers), pq.Array(tx.ipldCache.Keys), + pq.Array(tx.ipldCache.Values)) + if err != nil { + return err + } + tx.ipldCache = models.IPLDBatch{} + return nil +} + +// run in background goroutine to synchronize concurrent appends to the ipldCache +func (tx *BatchTx) cache() { + for { + select { + case i := <-tx.iplds: + tx.ipldCache.BlockNumbers = append(tx.ipldCache.BlockNumbers, i.BlockNumber) + tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key) + tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data) + case <-tx.quit: + tx.ipldCache = models.IPLDBatch{} + return + } + } +} + +func (tx *BatchTx) cacheDirect(key string, value []byte) { + tx.iplds <- models.IPLDModel{ + BlockNumber: tx.BlockNumber, + Key: key, + Data: value, + } +} + +func (tx *BatchTx) cacheIPLD(i node.Node) { + tx.iplds <- models.IPLDModel{ + BlockNumber: tx.BlockNumber, + Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(), + Data: i.RawData(), + } +} + +func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error) { + c, err := ipld.RawdataToCid(codec, raw, mh) + if err != nil { + return "", "", err + } + prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String() + tx.iplds <- models.IPLDModel{ + BlockNumber: tx.BlockNumber, + Key: prefixedKey, + Data: raw, + } + return c.String(), prefixedKey, err +} + +func (tx *BatchTx) cacheRemoved(key string, value []byte) { + if atomic.LoadUint32(tx.removedCacheFlag) == 0 { + atomic.StoreUint32(tx.removedCacheFlag, 1) + tx.iplds <- models.IPLDModel{ + BlockNumber: tx.BlockNumber, + Key: key, + Data: value, + } + } +} + +// rollback sql transaction and log any error +func rollback(ctx context.Context, tx Tx) { + if err := tx.Rollback(ctx); err != nil { + log.Error(err.Error()) + } +} diff --git a/statediff/indexer/database/sql/indexer.go b/statediff/indexer/database/sql/indexer.go new file mode 100644 index 000000000000..762107ee5e81 --- /dev/null +++ b/statediff/indexer/database/sql/indexer.go @@ -0,0 +1,687 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package sql provides an interface for pushing and indexing IPLD objects into a sql database +// Metrics for reporting processing and connection stats are defined in ./metrics.go + +package sql + +import ( + "context" + "fmt" + "math/big" + "time" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var _ interfaces.StateDiffIndexer = &StateDiffIndexer{} + +var ( + indexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry) + dbMetrics = RegisterDBMetrics(metrics.DefaultRegistry) +) + +// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of an SQL sql +type StateDiffIndexer struct { + ctx context.Context + chainConfig *params.ChainConfig + dbWriter *Writer +} + +// NewStateDiffIndexer creates a sql implementation of interfaces.StateDiffIndexer +func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, db Database) (*StateDiffIndexer, error) { + return &StateDiffIndexer{ + ctx: ctx, + chainConfig: chainConfig, + dbWriter: NewWriter(db), + }, nil +} + +// ReportDBMetrics is a reporting function to run as goroutine +func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) { + if !metrics.Enabled { + return + } + ticker := time.NewTicker(delay) + go func() { + for { + select { + case <-ticker.C: + dbMetrics.Update(sdi.dbWriter.db.Stats()) + case <-quit: + ticker.Stop() + return + } + } + }() +} + +// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts) +// Returns an initiated DB transaction which must be Closed via defer to commit or rollback +func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) { + start, t := time.Now(), time.Now() + blockHash := block.Hash() + blockHashStr := blockHash.String() + height := block.NumberU64() + traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr) + transactions := block.Transactions() + // Derive any missing fields + if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil { + return nil, err + } + + // Generate the block iplds + headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts) + if err != nil { + return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err) + } + + if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) { + return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs)) + } + if len(txTrieNodes) != len(rctTrieNodes) { + return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes)) + } + + // Calculate reward + var reward *big.Int + // in PoA networks block reward is 0 + if sdi.chainConfig.Clique != nil { + reward = big.NewInt(0) + } else { + reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts) + } + t = time.Now() + + // Begin new db tx for everything + tx, err := sdi.dbWriter.db.Begin(sdi.ctx) + if err != nil { + return nil, err + } + defer func() { + if p := recover(); p != nil { + rollback(sdi.ctx, tx) + panic(p) + } else if err != nil { + rollback(sdi.ctx, tx) + } + }() + blockTx := &BatchTx{ + removedCacheFlag: new(uint32), + ctx: sdi.ctx, + BlockNumber: block.Number().String(), + stm: sdi.dbWriter.db.InsertIPLDsStm(), + iplds: make(chan models.IPLDModel), + quit: make(chan struct{}), + ipldCache: models.IPLDBatch{ + BlockNumbers: make([]string, 0, startingCacheCapacity), + Keys: make([]string, 0, startingCacheCapacity), + Values: make([][]byte, 0, startingCacheCapacity), + }, + dbtx: tx, + // handle transaction commit or rollback for any return case + submit: func(self *BatchTx, err error) error { + defer func() { + close(self.quit) + close(self.iplds) + }() + if p := recover(); p != nil { + log.Info("panic detected before tx submission, rolling back the tx", "panic", p) + rollback(sdi.ctx, tx) + panic(p) + } else if err != nil { + log.Info("error detected before tx submission, rolling back the tx", "error", err) + rollback(sdi.ctx, tx) + } else { + tDiff := time.Since(t) + indexerMetrics.tStateStoreCodeProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String()) + t = time.Now() + if err := self.flush(); err != nil { + rollback(sdi.ctx, tx) + traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String()) + log.Debug(traceMsg) + return err + } + err = tx.Commit(sdi.ctx) + tDiff = time.Since(t) + indexerMetrics.tPostgresCommit.Update(tDiff) + traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String()) + } + traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String()) + log.Debug(traceMsg) + return err + }, + } + go blockTx.cache() + + tDiff := time.Since(t) + indexerMetrics.tFreePostgres.Update(tDiff) + + traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String()) + t = time.Now() + + // Publish and index header, collect headerID + var headerID string + headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty) + if err != nil { + return nil, err + } + tDiff = time.Since(t) + indexerMetrics.tHeaderProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String()) + t = time.Now() + // Publish and index uncles + err = sdi.processUncles(blockTx, headerID, block.Number(), uncleNodes) + if err != nil { + return nil, err + } + tDiff = time.Since(t) + indexerMetrics.tUncleProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String()) + t = time.Now() + // Publish and index receipts and txs + err = sdi.processReceiptsAndTxs(blockTx, processArgs{ + headerID: headerID, + blockNumber: block.Number(), + receipts: receipts, + txs: transactions, + rctNodes: rctNodes, + rctTrieNodes: rctTrieNodes, + txNodes: txNodes, + txTrieNodes: txTrieNodes, + logTrieNodes: logTrieNodes, + logLeafNodeCIDs: logLeafNodeCIDs, + rctLeafNodeCIDs: rctLeafNodeCIDs, + }) + if err != nil { + return nil, err + } + tDiff = time.Since(t) + indexerMetrics.tTxAndRecProcessing.Update(tDiff) + traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String()) + t = time.Now() + + return blockTx, err +} + +// processHeader publishes and indexes a header IPLD in Postgres +// it returns the headerID +func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode node.Node, reward, td *big.Int) (string, error) { + tx.cacheIPLD(headerNode) + + var baseFee *string + if header.BaseFee != nil { + baseFee = new(string) + *baseFee = header.BaseFee.String() + } + headerID := header.Hash().String() + // index header + return headerID, sdi.dbWriter.upsertHeaderCID(tx.dbtx, models.HeaderModel{ + CID: headerNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(headerNode.Cid()), + ParentHash: header.ParentHash.String(), + BlockNumber: header.Number.String(), + BlockHash: headerID, + TotalDifficulty: td.String(), + Reward: reward.String(), + Bloom: header.Bloom.Bytes(), + StateRoot: header.Root.String(), + RctRoot: header.ReceiptHash.String(), + TxRoot: header.TxHash.String(), + UncleRoot: header.UncleHash.String(), + Timestamp: header.Time, + Coinbase: header.Coinbase.String(), + }) +} + +// processUncles publishes and indexes uncle IPLDs in Postgres +func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, uncleNodes []*ipld2.EthHeader) error { + // publish and index uncles + for _, uncleNode := range uncleNodes { + tx.cacheIPLD(uncleNode) + var uncleReward *big.Int + // in PoA networks uncle reward is 0 + if sdi.chainConfig.Clique != nil { + uncleReward = big.NewInt(0) + } else { + uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncleNode.Number.Uint64()) + } + uncle := models.UncleModel{ + BlockNumber: blockNumber.String(), + HeaderID: headerID, + CID: uncleNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()), + ParentHash: uncleNode.ParentHash.String(), + BlockHash: uncleNode.Hash().String(), + Reward: uncleReward.String(), + } + if err := sdi.dbWriter.upsertUncleCID(tx.dbtx, uncle); err != nil { + return err + } + } + return nil +} + +// processArgs bundles arguments to processReceiptsAndTxs +type processArgs struct { + headerID string + blockNumber *big.Int + receipts types.Receipts + txs types.Transactions + rctNodes []*ipld2.EthReceipt + rctTrieNodes []*ipld2.EthRctTrie + txNodes []*ipld2.EthTx + txTrieNodes []*ipld2.EthTxTrie + logTrieNodes [][]node.Node + logLeafNodeCIDs [][]cid.Cid + rctLeafNodeCIDs []cid.Cid +} + +// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres +func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error { + // Process receipts and txs + signer := types.MakeSigner(sdi.chainConfig, args.blockNumber) + for i, receipt := range args.receipts { + for _, logTrieNode := range args.logTrieNodes[i] { + tx.cacheIPLD(logTrieNode) + } + txNode := args.txNodes[i] + tx.cacheIPLD(txNode) + + // index tx + trx := args.txs[i] + txID := trx.Hash().String() + + var val string + if trx.Value() != nil { + val = trx.Value().String() + } + + // derive sender for the tx that corresponds with this receipt + from, err := types.Sender(signer, trx) + if err != nil { + return fmt.Errorf("error deriving tx sender: %v", err) + } + txModel := models.TxModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + Dst: shared.HandleZeroAddrPointer(trx.To()), + Src: shared.HandleZeroAddr(from), + TxHash: txID, + Index: int64(i), + Data: trx.Data(), + CID: txNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(txNode.Cid()), + Type: trx.Type(), + Value: val, + } + if err := sdi.dbWriter.upsertTransactionCID(tx.dbtx, txModel); err != nil { + return err + } + + // index access list if this is one + for j, accessListElement := range trx.AccessList() { + storageKeys := make([]string, len(accessListElement.StorageKeys)) + for k, storageKey := range accessListElement.StorageKeys { + storageKeys[k] = storageKey.Hex() + } + accessListElementModel := models.AccessListElementModel{ + BlockNumber: args.blockNumber.String(), + TxID: txID, + Index: int64(j), + Address: accessListElement.Address.Hex(), + StorageKeys: storageKeys, + } + if err := sdi.dbWriter.upsertAccessListElement(tx.dbtx, accessListElementModel); err != nil { + return err + } + } + + // this is the contract address if this receipt is for a contract creation tx + contract := shared.HandleZeroAddr(receipt.ContractAddress) + var contractHash string + if contract != "" { + contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + } + + // index receipt + if !args.rctLeafNodeCIDs[i].Defined() { + return fmt.Errorf("invalid receipt leaf node cid") + } + + rctModel := &models.ReceiptModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + TxID: txID, + Contract: contract, + ContractHash: contractHash, + LeafCID: args.rctLeafNodeCIDs[i].String(), + LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]), + LogRoot: args.rctNodes[i].LogRoot.String(), + } + if len(receipt.PostState) == 0 { + rctModel.PostStatus = receipt.Status + } else { + rctModel.PostState = common.Bytes2Hex(receipt.PostState) + } + + if err := sdi.dbWriter.upsertReceiptCID(tx.dbtx, rctModel); err != nil { + return err + } + + // index logs + logDataSet := make([]*models.LogsModel, len(receipt.Logs)) + for idx, l := range receipt.Logs { + topicSet := make([]string, 4) + for ti, topic := range l.Topics { + topicSet[ti] = topic.Hex() + } + + if !args.logLeafNodeCIDs[i][idx].Defined() { + return fmt.Errorf("invalid log cid") + } + + logDataSet[idx] = &models.LogsModel{ + BlockNumber: args.blockNumber.String(), + HeaderID: args.headerID, + ReceiptID: txID, + Address: l.Address.String(), + Index: int64(l.Index), + Data: l.Data, + LeafCID: args.logLeafNodeCIDs[i][idx].String(), + LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]), + Topic0: topicSet[0], + Topic1: topicSet[1], + Topic2: topicSet[2], + Topic3: topicSet[3], + } + } + + if err := sdi.dbWriter.upsertLogCID(tx.dbtx, logDataSet); err != nil { + return err + } + } + + // publish trie nodes, these aren't indexed directly + for i, n := range args.txTrieNodes { + tx.cacheIPLD(n) + tx.cacheIPLD(args.rctTrieNodes[i]) + } + + return nil +} + +// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql +func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { + tx, ok := batch.(*BatchTx) + if !ok { + return fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch) + } + // publish the state node + var stateModel models.StateNodeModel + if stateNode.NodeType == sdtypes.Removed { + tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{}) + stateModel = models.StateNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: shared.RemovedNodeStateCID, + MhKey: shared.RemovedNodeMhKey, + NodeType: stateNode.NodeType.Int(), + } + } else { + stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) + } + stateModel = models.StateNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: stateMhKey, + NodeType: stateNode.NodeType.Int(), + } + } + + // index the state node + if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil { + return err + } + + // if we have a leaf, decode and index the account data + if stateNode.NodeType == sdtypes.Leaf { + var i []interface{} + if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil { + return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements") + } + var account types.StateAccount + if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil { + return fmt.Errorf("error decoding state account rlp: %s", err.Error()) + } + accountModel := models.StateAccountModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Balance: account.Balance.String(), + Nonce: account.Nonce, + CodeHash: account.CodeHash, + StorageRoot: account.Root.String(), + } + if err := sdi.dbWriter.upsertStateAccount(tx.dbtx, accountModel); err != nil { + return err + } + } + + // if there are any storage nodes associated with this node, publish and index them + for _, storageNode := range stateNode.StorageNodes { + if storageNode.NodeType == sdtypes.Removed { + tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{}) + storageModel := models.StorageNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).String(), + CID: shared.RemovedNodeStorageCID, + MhKey: shared.RemovedNodeMhKey, + NodeType: storageNode.NodeType.Int(), + } + if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil { + return err + } + continue + } + storageCIDStr, storageMhKey, err := tx.cacheRaw(ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue) + if err != nil { + return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err) + } + storageModel := models.StorageNodeModel{ + BlockNumber: tx.BlockNumber, + HeaderID: headerID, + StatePath: stateNode.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).String(), + CID: storageCIDStr, + MhKey: storageMhKey, + NodeType: storageNode.NodeType.Int(), + } + if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil { + return err + } + } + + return nil +} + +// PushCodeAndCodeHash publishes code and codehash pairs to the ipld sql +func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error { + tx, ok := batch.(*BatchTx) + if !ok { + return fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch) + } + // codec doesn't matter since db key is multihash-based + mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash) + if err != nil { + return fmt.Errorf("error deriving multihash key from codehash: %v", err) + } + tx.cacheDirect(mhKey, codeAndCodeHash.Code) + return nil +} + +// Close satisfies io.Closer +func (sdi *StateDiffIndexer) Close() error { + return sdi.dbWriter.Close() +} + +// Update the known gaps table with the gap information. + +// LoadWatchedAddresses reads watched addresses from the database +func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) { + addressStrings := make([]string, 0) + pgStr := "SELECT address FROM eth_meta.watched_addresses" + err := sdi.dbWriter.db.Select(sdi.ctx, &addressStrings, pgStr) + if err != nil { + return nil, fmt.Errorf("error loading watched addresses: %v", err) + } + + watchedAddresses := []common.Address{} + for _, addressString := range addressStrings { + watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString)) + } + + return watchedAddresses, nil +} + +// InsertWatchedAddresses inserts the given addresses in the database +func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + tx, err := sdi.dbWriter.db.Begin(sdi.ctx) + if err != nil { + return err + } + defer func() { + if p := recover(); p != nil { + rollback(sdi.ctx, tx) + panic(p) + } else if err != nil { + rollback(sdi.ctx, tx) + } else { + err = tx.Commit(sdi.ctx) + } + }() + + for _, arg := range args { + _, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`, + arg.Address, arg.CreatedAt, currentBlockNumber.Uint64()) + if err != nil { + return fmt.Errorf("error inserting watched_addresses entry: %v", err) + } + } + + return err +} + +// RemoveWatchedAddresses removes the given watched addresses from the database +func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error { + tx, err := sdi.dbWriter.db.Begin(sdi.ctx) + if err != nil { + return err + } + defer func() { + if p := recover(); p != nil { + rollback(sdi.ctx, tx) + panic(p) + } else if err != nil { + rollback(sdi.ctx, tx) + } else { + err = tx.Commit(sdi.ctx) + } + }() + + for _, arg := range args { + _, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses WHERE address = $1`, arg.Address) + if err != nil { + return fmt.Errorf("error removing watched_addresses entry: %v", err) + } + } + + return err +} + +// SetWatchedAddresses clears and inserts the given addresses in the database +func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + tx, err := sdi.dbWriter.db.Begin(sdi.ctx) + if err != nil { + return err + } + defer func() { + if p := recover(); p != nil { + rollback(sdi.ctx, tx) + panic(p) + } else if err != nil { + rollback(sdi.ctx, tx) + } else { + err = tx.Commit(sdi.ctx) + } + }() + + _, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`) + if err != nil { + return fmt.Errorf("error setting watched_addresses table: %v", err) + } + + for _, arg := range args { + _, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`, + arg.Address, arg.CreatedAt, currentBlockNumber.Uint64()) + if err != nil { + return fmt.Errorf("error setting watched_addresses table: %v", err) + } + } + + return err +} + +// ClearWatchedAddresses clears all the watched addresses from the database +func (sdi *StateDiffIndexer) ClearWatchedAddresses() error { + _, err := sdi.dbWriter.db.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`) + if err != nil { + return fmt.Errorf("error clearing watched_addresses table: %v", err) + } + + return nil +} diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go new file mode 100644 index 000000000000..13fd0c026c1e --- /dev/null +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -0,0 +1,28 @@ +package sql_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" +) + +var ( + db sql.Database + err error + ind interfaces.StateDiffIndexer +) + +func checkTxClosure(t *testing.T, idle, inUse, open int64) { + require.Equal(t, idle, db.Stats().Idle()) + require.Equal(t, inUse, db.Stats().InUse()) + require.Equal(t, open, db.Stats().Open()) +} + +func tearDown(t *testing.T) { + test_helpers.TearDownDB(t, db) + require.NoError(t, ind.Close()) +} diff --git a/statediff/indexer/database/sql/interfaces.go b/statediff/indexer/database/sql/interfaces.go new file mode 100644 index 000000000000..613a0925194d --- /dev/null +++ b/statediff/indexer/database/sql/interfaces.go @@ -0,0 +1,88 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "context" + "io" + "time" +) + +// Database interfaces required by the sql indexer +type Database interface { + Driver + Statements +} + +// Driver interface has all the methods required by a driver implementation to support the sql indexer +type Driver interface { + QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow + Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) + Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error + Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error + Begin(ctx context.Context) (Tx, error) + Stats() Stats + NodeID() string + Context() context.Context + io.Closer +} + +// Statements interface to accommodate different SQL query syntax +type Statements interface { + InsertHeaderStm() string + InsertUncleStm() string + InsertTxStm() string + InsertAccessListElementStm() string + InsertRctStm() string + InsertLogStm() string + InsertStateStm() string + InsertAccountStm() string + InsertStorageStm() string + InsertIPLDStm() string + InsertIPLDsStm() string + InsertKnownGapsStm() string +} + +// Tx interface to accommodate different concrete SQL transaction types +type Tx interface { + QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow + Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) + Commit(ctx context.Context) error + Rollback(ctx context.Context) error +} + +// ScannableRow interface to accommodate different concrete row types +type ScannableRow interface { + Scan(dest ...interface{}) error +} + +// Result interface to accommodate different concrete result types +type Result interface { + RowsAffected() (int64, error) +} + +// Stats interface to accommodate different concrete sql stats types +type Stats interface { + MaxOpen() int64 + Open() int64 + InUse() int64 + Idle() int64 + WaitCount() int64 + WaitDuration() time.Duration + MaxIdleClosed() int64 + MaxLifetimeClosed() int64 +} diff --git a/statediff/indexer/database/sql/mainnet_tests/indexer_test.go b/statediff/indexer/database/sql/mainnet_tests/indexer_test.go new file mode 100644 index 000000000000..ce57a74acc50 --- /dev/null +++ b/statediff/indexer/database/sql/mainnet_tests/indexer_test.go @@ -0,0 +1,95 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package mainnet_tests + +import ( + "context" + "fmt" + "math/big" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/test" + "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" +) + +var ( + err error + db sql.Database + ind interfaces.StateDiffIndexer + chainConf = params.MainnetChainConfig +) + +func init() { + if os.Getenv("MODE") != "statediff" { + fmt.Println("Skipping statediff test") + os.Exit(0) + } +} + +func TestMainnetIndexer(t *testing.T) { + conf := test_helpers.GetTestConfig() + + for _, blockNumber := range test_helpers.ProblemBlocks { + conf.BlockNumber = big.NewInt(blockNumber) + tb, trs, err := test_helpers.TestBlockAndReceipts(conf) + require.NoError(t, err) + + testPushBlockAndState(t, tb, trs) + } + + testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf) + require.NoError(t, err) + + testPushBlockAndState(t, testBlock, testReceipts) +} + +func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) { + t.Run("Test PushBlock and PushStateNode", func(t *testing.T) { + setupMainnetIndexer(t) + defer checkTxClosure(t, 0, 0, 0) + defer tearDown(t) + + test.TestBlock(t, ind, block, receipts) + }) +} + +func setupMainnetIndexer(t *testing.T) { + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } + ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, db) +} + +func checkTxClosure(t *testing.T, idle, inUse, open int64) { + require.Equal(t, idle, db.Stats().Idle()) + require.Equal(t, inUse, db.Stats().InUse()) + require.Equal(t, open, db.Stats().Open()) +} + +func tearDown(t *testing.T) { + test_helpers.TearDownDB(t, db) + require.NoError(t, ind.Close()) +} diff --git a/statediff/indexer/database/sql/metrics.go b/statediff/indexer/database/sql/metrics.go new file mode 100644 index 000000000000..b0946a722d9b --- /dev/null +++ b/statediff/indexer/database/sql/metrics.go @@ -0,0 +1,147 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "strings" + + "github.com/ethereum/go-ethereum/metrics" +) + +const ( + namespace = "statediff" +) + +// Build a fully qualified metric name +func metricName(subsystem, name string) string { + if name == "" { + return "" + } + parts := []string{namespace, name} + if subsystem != "" { + parts = []string{namespace, subsystem, name} + } + // Prometheus uses _ but geth metrics uses / and replaces + return strings.Join(parts, "/") +} + +type indexerMetricsHandles struct { + // The total number of processed blocks + blocks metrics.Counter + // The total number of processed transactions + transactions metrics.Counter + // The total number of processed receipts + receipts metrics.Counter + // The total number of processed logs + logs metrics.Counter + // The total number of access list entries processed + accessListEntries metrics.Counter + // Time spent waiting for free postgres tx + tFreePostgres metrics.Timer + // Postgres transaction commit duration + tPostgresCommit metrics.Timer + // Header processing time + tHeaderProcessing metrics.Timer + // Uncle processing time + tUncleProcessing metrics.Timer + // Tx and receipt processing time + tTxAndRecProcessing metrics.Timer + // State, storage, and code combined processing time + tStateStoreCodeProcessing metrics.Timer +} + +func RegisterIndexerMetrics(reg metrics.Registry) indexerMetricsHandles { + ctx := indexerMetricsHandles{ + blocks: metrics.NewCounter(), + transactions: metrics.NewCounter(), + receipts: metrics.NewCounter(), + logs: metrics.NewCounter(), + accessListEntries: metrics.NewCounter(), + tFreePostgres: metrics.NewTimer(), + tPostgresCommit: metrics.NewTimer(), + tHeaderProcessing: metrics.NewTimer(), + tUncleProcessing: metrics.NewTimer(), + tTxAndRecProcessing: metrics.NewTimer(), + tStateStoreCodeProcessing: metrics.NewTimer(), + } + subsys := "indexer" + reg.Register(metricName(subsys, "blocks"), ctx.blocks) + reg.Register(metricName(subsys, "transactions"), ctx.transactions) + reg.Register(metricName(subsys, "receipts"), ctx.receipts) + reg.Register(metricName(subsys, "logs"), ctx.logs) + reg.Register(metricName(subsys, "access_list_entries"), ctx.accessListEntries) + reg.Register(metricName(subsys, "t_free_postgres"), ctx.tFreePostgres) + reg.Register(metricName(subsys, "t_postgres_commit"), ctx.tPostgresCommit) + reg.Register(metricName(subsys, "t_header_processing"), ctx.tHeaderProcessing) + reg.Register(metricName(subsys, "t_uncle_processing"), ctx.tUncleProcessing) + reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.tTxAndRecProcessing) + reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.tStateStoreCodeProcessing) + return ctx +} + +type dbMetricsHandles struct { + // Maximum number of open connections to the sql + maxOpen metrics.Gauge + // The number of established connections both in use and idle + open metrics.Gauge + // The number of connections currently in use + inUse metrics.Gauge + // The number of idle connections + idle metrics.Gauge + // The total number of connections waited for + waitedFor metrics.Counter + // The total time blocked waiting for a new connection + blockedMilliseconds metrics.Counter + // The total number of connections closed due to SetMaxIdleConns + closedMaxIdle metrics.Counter + // The total number of connections closed due to SetConnMaxLifetime + closedMaxLifetime metrics.Counter +} + +func RegisterDBMetrics(reg metrics.Registry) dbMetricsHandles { + ctx := dbMetricsHandles{ + maxOpen: metrics.NewGauge(), + open: metrics.NewGauge(), + inUse: metrics.NewGauge(), + idle: metrics.NewGauge(), + waitedFor: metrics.NewCounter(), + blockedMilliseconds: metrics.NewCounter(), + closedMaxIdle: metrics.NewCounter(), + closedMaxLifetime: metrics.NewCounter(), + } + subsys := "connections" + reg.Register(metricName(subsys, "max_open"), ctx.maxOpen) + reg.Register(metricName(subsys, "open"), ctx.open) + reg.Register(metricName(subsys, "in_use"), ctx.inUse) + reg.Register(metricName(subsys, "idle"), ctx.idle) + reg.Register(metricName(subsys, "waited_for"), ctx.waitedFor) + reg.Register(metricName(subsys, "blocked_milliseconds"), ctx.blockedMilliseconds) + reg.Register(metricName(subsys, "closed_max_idle"), ctx.closedMaxIdle) + reg.Register(metricName(subsys, "closed_max_lifetime"), ctx.closedMaxLifetime) + return ctx +} + +func (met *dbMetricsHandles) Update(stats Stats) { + met.maxOpen.Update(stats.MaxOpen()) + met.open.Update(stats.Open()) + met.inUse.Update(stats.InUse()) + met.idle.Update(stats.Idle()) + met.waitedFor.Inc(stats.WaitCount()) + met.blockedMilliseconds.Inc(stats.WaitDuration().Milliseconds()) + met.closedMaxIdle.Inc(stats.MaxIdleClosed()) + met.closedMaxLifetime.Inc(stats.MaxLifetimeClosed()) +} diff --git a/statediff/indexer/database/sql/pgx_indexer_legacy_test.go b/statediff/indexer/database/sql/pgx_indexer_legacy_test.go new file mode 100644 index 000000000000..292548b75417 --- /dev/null +++ b/statediff/indexer/database/sql/pgx_indexer_legacy_test.go @@ -0,0 +1,52 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/test" +) + +func setupLegacyPGXIndexer(t *testing.T) { + db, err = postgres.SetupPGXDB() + if err != nil { + t.Fatal(err) + } + ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db) + require.NoError(t, err) +} + +func setupLegacyPGX(t *testing.T) { + setupLegacyPGXIndexer(t) + test.SetupLegacyTestData(t, ind) +} + +func TestLegacyPGXIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs", func(t *testing.T) { + setupLegacyPGX(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestLegacyIndexer(t, db) + }) +} diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go new file mode 100644 index 000000000000..1dbf2dfa0b3d --- /dev/null +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -0,0 +1,227 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql_test + +import ( + "context" + "math/big" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/test" +) + +func setupPGXIndexer(t *testing.T) { + db, err = postgres.SetupPGXDB() + if err != nil { + t.Fatal(err) + } + ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db) + require.NoError(t, err) +} + +func setupPGX(t *testing.T) { + setupPGXIndexer(t) + test.SetupTestData(t, ind) +} + +func setupPGXNonCanonical(t *testing.T) { + setupPGXIndexer(t) + test.SetupTestDataNonCanonical(t, ind) +} + +// Test indexer for a canonical block +func TestPGXIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { + setupPGX(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexHeaderIPLDs(t, db) + }) + + t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { + setupPGX(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexTransactionIPLDs(t, db) + }) + + t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) { + setupPGX(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexLogIPLDs(t, db) + }) + + t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) { + setupPGX(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexReceiptIPLDs(t, db) + }) + + t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) { + setupPGX(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexStateIPLDs(t, db) + }) + + t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { + setupPGX(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexStorageIPLDs(t, db) + }) +} + +// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1 +func TestPGXIndexerNonCanonical(t *testing.T) { + t.Run("Publish and index header", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexHeaderNonCanonical(t, db) + }) + + t.Run("Publish and index transactions", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexTransactionsNonCanonical(t, db) + }) + + t.Run("Publish and index receipts", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexReceiptsNonCanonical(t, db) + }) + + t.Run("Publish and index logs", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexLogsNonCanonical(t, db) + }) + + t.Run("Publish and index state nodes", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexStateNonCanonical(t, db) + }) + + t.Run("Publish and index storage nodes", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + test.TestPublishAndIndexStorageNonCanonical(t, db) + }) +} + +func TestPGXWatchAddressMethods(t *testing.T) { + setupPGXIndexer(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + t.Run("Load watched addresses (empty table)", func(t *testing.T) { + test.TestLoadEmptyWatchedAddresses(t, ind) + }) + + t.Run("Insert watched addresses", func(t *testing.T) { + args := mocks.GetInsertWatchedAddressesArgs() + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1))) + require.NoError(t, err) + + test.TestInsertWatchedAddresses(t, db) + }) + + t.Run("Insert watched addresses (some already watched)", func(t *testing.T) { + args := mocks.GetInsertAlreadyWatchedAddressesArgs() + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2))) + require.NoError(t, err) + + test.TestInsertAlreadyWatchedAddresses(t, db) + }) + + t.Run("Remove watched addresses", func(t *testing.T) { + args := mocks.GetRemoveWatchedAddressesArgs() + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + test.TestRemoveWatchedAddresses(t, db) + }) + + t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) { + args := mocks.GetRemoveNonWatchedAddressesArgs() + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + test.TestRemoveNonWatchedAddresses(t, db) + }) + + t.Run("Set watched addresses", func(t *testing.T) { + args := mocks.GetSetWatchedAddressesArgs() + err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2))) + require.NoError(t, err) + + test.TestSetWatchedAddresses(t, db) + }) + + t.Run("Set watched addresses (some already watched)", func(t *testing.T) { + args := mocks.GetSetAlreadyWatchedAddressesArgs() + err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3))) + require.NoError(t, err) + + test.TestSetAlreadyWatchedAddresses(t, db) + }) + + t.Run("Load watched addresses", func(t *testing.T) { + test.TestLoadWatchedAddresses(t, ind) + }) + + t.Run("Clear watched addresses", func(t *testing.T) { + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + test.TestClearWatchedAddresses(t, db) + }) + + t.Run("Clear watched addresses (empty table)", func(t *testing.T) { + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + test.TestClearEmptyWatchedAddresses(t, db) + }) +} diff --git a/statediff/indexer/database/sql/postgres/config.go b/statediff/indexer/database/sql/postgres/config.go new file mode 100644 index 000000000000..27f78f8f4434 --- /dev/null +++ b/statediff/indexer/database/sql/postgres/config.go @@ -0,0 +1,98 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "fmt" + "strings" + "time" + + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +// DriverType to explicitly type the kind of sql driver we are using +type DriverType string + +const ( + PGX DriverType = "PGX" + SQLX DriverType = "SQLX" + Unknown DriverType = "Unknown" +) + +// ResolveDriverType resolves a DriverType from a provided string +func ResolveDriverType(str string) (DriverType, error) { + switch strings.ToLower(str) { + case "pgx", "pgxpool": + return PGX, nil + case "sqlx": + return SQLX, nil + default: + return Unknown, fmt.Errorf("unrecognized driver type string: %s", str) + } +} + +// DefaultConfig are default parameters for connecting to a Postgres sql +var DefaultConfig = Config{ + Hostname: "localhost", + Port: 8077, + DatabaseName: "vulcanize_testing", + Username: "vdbm", + Password: "password", +} + +// Config holds params for a Postgres db +type Config struct { + // conn string params + Hostname string + Port int + DatabaseName string + Username string + Password string + + // conn settings + MaxConns int + MaxIdle int + MinConns int + MaxConnIdleTime time.Duration + MaxConnLifetime time.Duration + ConnTimeout time.Duration + + // node info params + ID string + ClientName string + + // driver type + Driver DriverType +} + +// Type satisfies interfaces.Config +func (c Config) Type() shared.DBType { + return shared.POSTGRES +} + +// DbConnectionString constructs and returns the connection string from the config +func (c Config) DbConnectionString() string { + if len(c.Username) > 0 && len(c.Password) > 0 { + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable", + c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName) + } + if len(c.Username) > 0 && len(c.Password) == 0 { + return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable", + c.Username, c.Hostname, c.Port, c.DatabaseName) + } + return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName) +} diff --git a/statediff/indexer/database/sql/postgres/database.go b/statediff/indexer/database/sql/postgres/database.go new file mode 100644 index 000000000000..5484ff8d813a --- /dev/null +++ b/statediff/indexer/database/sql/postgres/database.go @@ -0,0 +1,109 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + +var _ sql.Database = &DB{} + +const ( + createNodeStm = `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (node_id) DO NOTHING` +) + +// NewPostgresDB returns a postgres.DB using the provided driver +func NewPostgresDB(driver sql.Driver) *DB { + return &DB{driver} +} + +// DB implements sql.Database using a configured driver and Postgres statement syntax +type DB struct { + sql.Driver +} + +// InsertHeaderStm satisfies the sql.Statements interface +// Stm == Statement +func (db *DB) InsertHeaderStm() string { + return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) + ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)` +} + +// InsertUncleStm satisfies the sql.Statements interface +func (db *DB) InsertUncleStm() string { + return `INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (block_hash, block_number) DO NOTHING` +} + +// InsertTxStm satisfies the sql.Statements interface +func (db *DB) InsertTxStm() string { + return `INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING` +} + +// InsertAccessListElementStm satisfies the sql.Statements interface +func (db *DB) InsertAccessListElementStm() string { + return `INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (tx_id, index, block_number) DO NOTHING` +} + +// InsertRctStm satisfies the sql.Statements interface +func (db *DB) InsertRctStm() string { + return `INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (tx_id, header_id, block_number) DO NOTHING` +} + +// InsertLogStm satisfies the sql.Statements interface +func (db *DB) InsertLogStm() string { + return `INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING` +} + +// InsertStateStm satisfies the sql.Statements interface +func (db *DB) InsertStateStm() string { + return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (header_id, state_path, block_number) DO UPDATE SET (block_number, state_leaf_key, cid, node_type, diff, mh_key) = ($1, $3, $4, $6, $7, $8)` +} + +// InsertAccountStm satisfies the sql.Statements interface +func (db *DB) InsertAccountStm() string { + return `INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (header_id, state_path, block_number) DO NOTHING` +} + +// InsertStorageStm satisfies the sql.Statements interface +func (db *DB) InsertStorageStm() string { + return `INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (header_id, state_path, storage_path, block_number) DO UPDATE SET (block_number, storage_leaf_key, cid, node_type, diff, mh_key) = ($1, $4, $5, $7, $8, $9)` +} + +// InsertIPLDStm satisfies the sql.Statements interface +func (db *DB) InsertIPLDStm() string { + return `INSERT INTO public.blocks (block_number, key, data) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING` +} + +// InsertIPLDsStm satisfies the sql.Statements interface +func (db *DB) InsertIPLDsStm() string { + return `INSERT INTO public.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING` +} + +// InsertKnownGapsStm satisfies the sql.Statements interface +func (db *DB) InsertKnownGapsStm() string { + return `INSERT INTO eth_meta.known_gaps (starting_block_number, ending_block_number, checked_out, processing_key) VALUES ($1, $2, $3, $4) + ON CONFLICT (starting_block_number) DO UPDATE SET (ending_block_number, processing_key) = ($2, $4) + WHERE eth_meta.known_gaps.ending_block_number <= $2` +} diff --git a/statediff/indexer/database/sql/postgres/errors.go b/statediff/indexer/database/sql/postgres/errors.go new file mode 100644 index 000000000000..effa74aa125e --- /dev/null +++ b/statediff/indexer/database/sql/postgres/errors.go @@ -0,0 +1,38 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "fmt" +) + +const ( + DbConnectionFailedMsg = "db connection failed" + SettingNodeFailedMsg = "unable to set db node" +) + +func ErrDBConnectionFailed(connectErr error) error { + return formatError(DbConnectionFailedMsg, connectErr.Error()) +} + +func ErrUnableToSetNode(setErr error) error { + return formatError(SettingNodeFailedMsg, setErr.Error()) +} + +func formatError(msg, err string) error { + return fmt.Errorf("%s: %s", msg, err) +} diff --git a/statediff/indexer/database/sql/postgres/pgx.go b/statediff/indexer/database/sql/postgres/pgx.go new file mode 100644 index 000000000000..936a3765d9a0 --- /dev/null +++ b/statediff/indexer/database/sql/postgres/pgx.go @@ -0,0 +1,233 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "context" + "time" + + "github.com/georgysavva/scany/pgxscan" + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/node" +) + +// PGXDriver driver, implements sql.Driver +type PGXDriver struct { + ctx context.Context + pool *pgxpool.Pool + nodeInfo node.Info + nodeID string +} + +// NewPGXDriver returns a new pgx driver +// it initializes the connection pool and creates the node info table +func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) { + pgConf, err := MakeConfig(config) + if err != nil { + return nil, err + } + dbPool, err := pgxpool.ConnectConfig(ctx, pgConf) + if err != nil { + return nil, ErrDBConnectionFailed(err) + } + pg := &PGXDriver{ctx: ctx, pool: dbPool, nodeInfo: node} + nodeErr := pg.createNode() + if nodeErr != nil { + return &PGXDriver{}, ErrUnableToSetNode(nodeErr) + } + return pg, nil +} + +// MakeConfig creates a pgxpool.Config from the provided Config +func MakeConfig(config Config) (*pgxpool.Config, error) { + conf, err := pgxpool.ParseConfig("") + if err != nil { + return nil, err + } + + //conf.ConnConfig.BuildStatementCache = nil + conf.ConnConfig.Config.Host = config.Hostname + conf.ConnConfig.Config.Port = uint16(config.Port) + conf.ConnConfig.Config.Database = config.DatabaseName + conf.ConnConfig.Config.User = config.Username + conf.ConnConfig.Config.Password = config.Password + + if config.ConnTimeout != 0 { + conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout + } + if config.MaxConns != 0 { + conf.MaxConns = int32(config.MaxConns) + } + if config.MinConns != 0 { + conf.MinConns = int32(config.MinConns) + } + if config.MaxConnLifetime != 0 { + conf.MaxConnLifetime = config.MaxConnLifetime + } + if config.MaxConnIdleTime != 0 { + conf.MaxConnIdleTime = config.MaxConnIdleTime + } + return conf, nil +} + +func (pgx *PGXDriver) createNode() error { + _, err := pgx.pool.Exec( + pgx.ctx, + createNodeStm, + pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID, + pgx.nodeInfo.ID, pgx.nodeInfo.ClientName, + pgx.nodeInfo.ChainID) + if err != nil { + return ErrUnableToSetNode(err) + } + pgx.nodeID = pgx.nodeInfo.ID + return nil +} + +// QueryRow satisfies sql.Database +func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { + return pgx.pool.QueryRow(ctx, sql, args...) +} + +// Exec satisfies sql.Database +func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { + res, err := pgx.pool.Exec(ctx, sql, args...) + return resultWrapper{ct: res}, err +} + +// Select satisfies sql.Database +func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return pgxscan.Select(ctx, pgx.pool, dest, query, args...) +} + +// Get satisfies sql.Database +func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return pgxscan.Get(ctx, pgx.pool, dest, query, args...) +} + +// Begin satisfies sql.Database +func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) { + tx, err := pgx.pool.Begin(ctx) + if err != nil { + return nil, err + } + return pgxTxWrapper{tx: tx}, nil +} + +func (pgx *PGXDriver) Stats() sql.Stats { + stats := pgx.pool.Stat() + return pgxStatsWrapper{stats: stats} +} + +// NodeID satisfies sql.Database +func (pgx *PGXDriver) NodeID() string { + return pgx.nodeID +} + +// Close satisfies sql.Database/io.Closer +func (pgx *PGXDriver) Close() error { + pgx.pool.Close() + return nil +} + +// Context satisfies sql.Database +func (pgx *PGXDriver) Context() context.Context { + return pgx.ctx +} + +type resultWrapper struct { + ct pgconn.CommandTag +} + +// RowsAffected satisfies sql.Result +func (r resultWrapper) RowsAffected() (int64, error) { + return r.ct.RowsAffected(), nil +} + +type pgxStatsWrapper struct { + stats *pgxpool.Stat +} + +// MaxOpen satisfies sql.Stats +func (s pgxStatsWrapper) MaxOpen() int64 { + return int64(s.stats.MaxConns()) +} + +// Open satisfies sql.Stats +func (s pgxStatsWrapper) Open() int64 { + return int64(s.stats.TotalConns()) +} + +// InUse satisfies sql.Stats +func (s pgxStatsWrapper) InUse() int64 { + return int64(s.stats.AcquiredConns()) +} + +// Idle satisfies sql.Stats +func (s pgxStatsWrapper) Idle() int64 { + return int64(s.stats.IdleConns()) +} + +// WaitCount satisfies sql.Stats +func (s pgxStatsWrapper) WaitCount() int64 { + return s.stats.EmptyAcquireCount() +} + +// WaitDuration satisfies sql.Stats +func (s pgxStatsWrapper) WaitDuration() time.Duration { + return s.stats.AcquireDuration() +} + +// MaxIdleClosed satisfies sql.Stats +func (s pgxStatsWrapper) MaxIdleClosed() int64 { + // this stat isn't supported by pgxpool, but we don't want to panic + return 0 +} + +// MaxLifetimeClosed satisfies sql.Stats +func (s pgxStatsWrapper) MaxLifetimeClosed() int64 { + return s.stats.CanceledAcquireCount() +} + +type pgxTxWrapper struct { + tx pgx.Tx +} + +// QueryRow satisfies sql.Tx +func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { + return t.tx.QueryRow(ctx, sql, args...) +} + +// Exec satisfies sql.Tx +func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { + res, err := t.tx.Exec(ctx, sql, args...) + return resultWrapper{ct: res}, err +} + +// Commit satisfies sql.Tx +func (t pgxTxWrapper) Commit(ctx context.Context) error { + return t.tx.Commit(ctx) +} + +// Rollback satisfies sql.Tx +func (t pgxTxWrapper) Rollback(ctx context.Context) error { + return t.tx.Rollback(ctx) +} diff --git a/statediff/indexer/database/sql/postgres/pgx_test.go b/statediff/indexer/database/sql/postgres/pgx_test.go new file mode 100644 index 000000000000..043112e8d0a8 --- /dev/null +++ b/statediff/indexer/database/sql/postgres/pgx_test.go @@ -0,0 +1,121 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres_test + +import ( + "context" + "fmt" + "math/big" + "strings" + "testing" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/node" +) + +var ( + pgConfig, _ = postgres.MakeConfig(postgres.DefaultConfig) + ctx = context.Background() +) + +func expectContainsSubstring(t *testing.T, full string, sub string) { + if !strings.Contains(full, sub) { + t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub) + } +} + +func TestPostgresPGX(t *testing.T) { + t.Run("connects to the sql", func(t *testing.T) { + dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err) + } + if dbPool == nil { + t.Fatal("DB pool is nil") + } + dbPool.Close() + }) + + t.Run("serializes big.Int to db", func(t *testing.T) { + // postgres driver doesn't support go big.Int type + // various casts in golang uint64, int64, overflow for + // transaction value (in wei) even though + // postgres numeric can handle an arbitrary + // sized int, so use string representation of big.Int + // and cast on insert + + dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err) + } + defer dbPool.Close() + + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + require.Equal(t, "34940183920000000000", bi.String()) + + defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`) + _, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )") + if err != nil { + t.Fatal(err) + } + + sqlStatement := ` + INSERT INTO example (id, data) + VALUES (1, cast($1 AS NUMERIC))` + _, err = dbPool.Exec(ctx, sqlStatement, bi.String()) + if err != nil { + t.Fatal(err) + } + + var data string + err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, data, bi.String()) + actual := new(big.Int) + actual.SetString(data, 10) + require.Equal(t, bi, actual) + }) + + t.Run("throws error when can't connect to the database", func(t *testing.T) { + goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"} + _, err := postgres.NewPGXDriver(ctx, postgres.Config{}, goodInfo) + if err == nil { + t.Fatal("Expected an error") + } + + expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg) + }) + + t.Run("throws error when can't create node", func(t *testing.T) { + badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) + badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"} + + _, err := postgres.NewPGXDriver(ctx, postgres.DefaultConfig, badInfo) + if err == nil { + t.Fatal("Expected an error") + } + + expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg) + }) +} diff --git a/statediff/indexer/database/sql/postgres/postgres_suite_test.go b/statediff/indexer/database/sql/postgres/postgres_suite_test.go new file mode 100644 index 000000000000..a020e088e103 --- /dev/null +++ b/statediff/indexer/database/sql/postgres/postgres_suite_test.go @@ -0,0 +1,33 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres_test + +import ( + "fmt" + "os" + + "github.com/ethereum/go-ethereum/log" +) + +func init() { + if os.Getenv("MODE") != "statediff" { + fmt.Println("Skipping statediff test") + os.Exit(0) + } + + log.Root().SetHandler(log.DiscardHandler()) +} diff --git a/statediff/indexer/database/sql/postgres/sqlx.go b/statediff/indexer/database/sql/postgres/sqlx.go new file mode 100644 index 000000000000..9f1753e678c3 --- /dev/null +++ b/statediff/indexer/database/sql/postgres/sqlx.go @@ -0,0 +1,187 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "context" + coresql "database/sql" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/node" +) + +// SQLXDriver driver, implements sql.Driver +type SQLXDriver struct { + ctx context.Context + db *sqlx.DB + nodeInfo node.Info + nodeID string +} + +// NewSQLXDriver returns a new sqlx driver for Postgres +// it initializes the connection pool and creates the node info table +func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) { + db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString()) + if err != nil { + return &SQLXDriver{}, ErrDBConnectionFailed(err) + } + if config.MaxConns > 0 { + db.SetMaxOpenConns(config.MaxConns) + } + if config.MaxConnLifetime > 0 { + db.SetConnMaxLifetime(config.MaxConnLifetime) + } + db.SetMaxIdleConns(config.MaxIdle) + driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node} + if err := driver.createNode(); err != nil { + return &SQLXDriver{}, ErrUnableToSetNode(err) + } + return driver, nil +} + +func (driver *SQLXDriver) createNode() error { + _, err := driver.db.Exec( + createNodeStm, + driver.nodeInfo.GenesisBlock, driver.nodeInfo.NetworkID, + driver.nodeInfo.ID, driver.nodeInfo.ClientName, + driver.nodeInfo.ChainID) + if err != nil { + return ErrUnableToSetNode(err) + } + driver.nodeID = driver.nodeInfo.ID + return nil +} + +// QueryRow satisfies sql.Database +func (driver *SQLXDriver) QueryRow(_ context.Context, sql string, args ...interface{}) sql.ScannableRow { + return driver.db.QueryRowx(sql, args...) +} + +// Exec satisfies sql.Database +func (driver *SQLXDriver) Exec(_ context.Context, sql string, args ...interface{}) (sql.Result, error) { + return driver.db.Exec(sql, args...) +} + +// Select satisfies sql.Database +func (driver *SQLXDriver) Select(_ context.Context, dest interface{}, query string, args ...interface{}) error { + return driver.db.Select(dest, query, args...) +} + +// Get satisfies sql.Database +func (driver *SQLXDriver) Get(_ context.Context, dest interface{}, query string, args ...interface{}) error { + return driver.db.Get(dest, query, args...) +} + +// Begin satisfies sql.Database +func (driver *SQLXDriver) Begin(_ context.Context) (sql.Tx, error) { + tx, err := driver.db.Beginx() + if err != nil { + return nil, err + } + return sqlxTxWrapper{tx: tx}, nil +} + +func (driver *SQLXDriver) Stats() sql.Stats { + stats := driver.db.Stats() + return sqlxStatsWrapper{stats: stats} +} + +// NodeID satisfies sql.Database +func (driver *SQLXDriver) NodeID() string { + return driver.nodeID +} + +// Close satisfies sql.Database/io.Closer +func (driver *SQLXDriver) Close() error { + return driver.db.Close() +} + +// Context satisfies sql.Database +func (driver *SQLXDriver) Context() context.Context { + return driver.ctx +} + +type sqlxStatsWrapper struct { + stats coresql.DBStats +} + +// MaxOpen satisfies sql.Stats +func (s sqlxStatsWrapper) MaxOpen() int64 { + return int64(s.stats.MaxOpenConnections) +} + +// Open satisfies sql.Stats +func (s sqlxStatsWrapper) Open() int64 { + return int64(s.stats.OpenConnections) +} + +// InUse satisfies sql.Stats +func (s sqlxStatsWrapper) InUse() int64 { + return int64(s.stats.InUse) +} + +// Idle satisfies sql.Stats +func (s sqlxStatsWrapper) Idle() int64 { + return int64(s.stats.Idle) +} + +// WaitCount satisfies sql.Stats +func (s sqlxStatsWrapper) WaitCount() int64 { + return s.stats.WaitCount +} + +// WaitDuration satisfies sql.Stats +func (s sqlxStatsWrapper) WaitDuration() time.Duration { + return s.stats.WaitDuration +} + +// MaxIdleClosed satisfies sql.Stats +func (s sqlxStatsWrapper) MaxIdleClosed() int64 { + return s.stats.MaxIdleClosed +} + +// MaxLifetimeClosed satisfies sql.Stats +func (s sqlxStatsWrapper) MaxLifetimeClosed() int64 { + return s.stats.MaxLifetimeClosed +} + +type sqlxTxWrapper struct { + tx *sqlx.Tx +} + +// QueryRow satisfies sql.Tx +func (t sqlxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow { + return t.tx.QueryRowx(sql, args...) +} + +// Exec satisfies sql.Tx +func (t sqlxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { + return t.tx.Exec(sql, args...) +} + +// Commit satisfies sql.Tx +func (t sqlxTxWrapper) Commit(ctx context.Context) error { + return t.tx.Commit() +} + +// Rollback satisfies sql.Tx +func (t sqlxTxWrapper) Rollback(ctx context.Context) error { + return t.tx.Rollback() +} diff --git a/statediff/indexer/database/sql/postgres/sqlx_test.go b/statediff/indexer/database/sql/postgres/sqlx_test.go new file mode 100644 index 000000000000..40b9763146ee --- /dev/null +++ b/statediff/indexer/database/sql/postgres/sqlx_test.go @@ -0,0 +1,119 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres_test + +import ( + "fmt" + "math/big" + "strings" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/node" +) + +func TestPostgresSQLX(t *testing.T) { + var sqlxdb *sqlx.DB + + t.Run("connects to the database", func(t *testing.T) { + var err error + connStr := postgres.DefaultConfig.DbConnectionString() + + sqlxdb, err = sqlx.Connect("postgres", connStr) + if err != nil { + t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err) + } + if sqlxdb == nil { + t.Fatal("DB is nil") + } + err = sqlxdb.Close() + if err != nil { + t.Fatal(err) + } + }) + + t.Run("serializes big.Int to db", func(t *testing.T) { + // postgres driver doesn't support go big.Int type + // various casts in golang uint64, int64, overflow for + // transaction value (in wei) even though + // postgres numeric can handle an arbitrary + // sized int, so use string representation of big.Int + // and cast on insert + + connStr := postgres.DefaultConfig.DbConnectionString() + db, err := sqlx.Connect("postgres", connStr) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + require.Equal(t, "34940183920000000000", bi.String()) + + defer db.Exec(`DROP TABLE IF EXISTS example`) + _, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )") + if err != nil { + t.Fatal(err) + } + + sqlStatement := ` + INSERT INTO example (id, data) + VALUES (1, cast($1 AS NUMERIC))` + _, err = db.Exec(sqlStatement, bi.String()) + if err != nil { + t.Fatal(err) + } + + var data string + err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, data, bi.String()) + actual := new(big.Int) + actual.SetString(data, 10) + require.Equal(t, bi, actual) + }) + + t.Run("throws error when can't connect to the database", func(t *testing.T) { + goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"} + _, err := postgres.NewSQLXDriver(ctx, postgres.Config{}, goodInfo) + if err == nil { + t.Fatal("Expected an error") + } + + expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg) + }) + + t.Run("throws error when can't create node", func(t *testing.T) { + badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) + badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"} + + _, err := postgres.NewSQLXDriver(ctx, postgres.DefaultConfig, badInfo) + if err == nil { + t.Fatal("Expected an error") + } + + expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg) + }) +} diff --git a/statediff/indexer/database/sql/postgres/test_helpers.go b/statediff/indexer/database/sql/postgres/test_helpers.go new file mode 100644 index 000000000000..0eab778ae922 --- /dev/null +++ b/statediff/indexer/database/sql/postgres/test_helpers.go @@ -0,0 +1,44 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "context" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/node" +) + +// SetupSQLXDB is used to setup a sqlx db for tests +func SetupSQLXDB() (sql.Database, error) { + conf := DefaultConfig + conf.MaxIdle = 0 + driver, err := NewSQLXDriver(context.Background(), conf, node.Info{}) + if err != nil { + return nil, err + } + return NewPostgresDB(driver), nil +} + +// SetupPGXDB is used to setup a pgx db for tests +func SetupPGXDB() (sql.Database, error) { + driver, err := NewPGXDriver(context.Background(), DefaultConfig, node.Info{}) + if err != nil { + return nil, err + } + return NewPostgresDB(driver), nil +} diff --git a/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go b/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go new file mode 100644 index 000000000000..4a07b8a0e256 --- /dev/null +++ b/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go @@ -0,0 +1,52 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/test" +) + +func setupLegacySQLXIndexer(t *testing.T) { + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } + ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db) + require.NoError(t, err) +} + +func setupLegacySQLX(t *testing.T) { + setupLegacySQLXIndexer(t) + test.SetupLegacyTestData(t, ind) +} + +func TestLegacySQLXIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs", func(t *testing.T) { + setupLegacySQLX(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestLegacyIndexer(t, db) + }) +} diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go new file mode 100644 index 000000000000..fa8844655172 --- /dev/null +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -0,0 +1,227 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql_test + +import ( + "context" + "math/big" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/test" +) + +func setupSQLXIndexer(t *testing.T) { + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } + ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db) + require.NoError(t, err) +} + +func setupSQLX(t *testing.T) { + setupSQLXIndexer(t) + test.SetupTestData(t, ind) +} + +func setupSQLXNonCanonical(t *testing.T) { + setupSQLXIndexer(t) + test.SetupTestDataNonCanonical(t, ind) +} + +// Test indexer for a canonical block +func TestSQLXIndexer(t *testing.T) { + t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { + setupSQLX(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexHeaderIPLDs(t, db) + }) + + t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { + setupSQLX(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexTransactionIPLDs(t, db) + }) + + t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) { + setupSQLX(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexLogIPLDs(t, db) + }) + + t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) { + setupSQLX(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexReceiptIPLDs(t, db) + }) + + t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) { + setupSQLX(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexStateIPLDs(t, db) + }) + + t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { + setupSQLX(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexStorageIPLDs(t, db) + }) +} + +// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1 +func TestSQLXIndexerNonCanonical(t *testing.T) { + t.Run("Publish and index header", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexHeaderNonCanonical(t, db) + }) + + t.Run("Publish and index transactions", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexTransactionsNonCanonical(t, db) + }) + + t.Run("Publish and index receipts", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexReceiptsNonCanonical(t, db) + }) + + t.Run("Publish and index logs", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexLogsNonCanonical(t, db) + }) + + t.Run("Publish and index state nodes", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexStateNonCanonical(t, db) + }) + + t.Run("Publish and index storage nodes", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + test.TestPublishAndIndexStorageNonCanonical(t, db) + }) +} + +func TestSQLXWatchAddressMethods(t *testing.T) { + setupSQLXIndexer(t) + defer tearDown(t) + defer checkTxClosure(t, 0, 0, 0) + + t.Run("Load watched addresses (empty table)", func(t *testing.T) { + test.TestLoadEmptyWatchedAddresses(t, ind) + }) + + t.Run("Insert watched addresses", func(t *testing.T) { + args := mocks.GetInsertWatchedAddressesArgs() + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1))) + require.NoError(t, err) + + test.TestInsertWatchedAddresses(t, db) + }) + + t.Run("Insert watched addresses (some already watched)", func(t *testing.T) { + args := mocks.GetInsertAlreadyWatchedAddressesArgs() + err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2))) + require.NoError(t, err) + + test.TestInsertAlreadyWatchedAddresses(t, db) + }) + + t.Run("Remove watched addresses", func(t *testing.T) { + args := mocks.GetRemoveWatchedAddressesArgs() + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + test.TestRemoveWatchedAddresses(t, db) + }) + + t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) { + args := mocks.GetRemoveNonWatchedAddressesArgs() + err = ind.RemoveWatchedAddresses(args) + require.NoError(t, err) + + test.TestRemoveNonWatchedAddresses(t, db) + }) + + t.Run("Set watched addresses", func(t *testing.T) { + args := mocks.GetSetWatchedAddressesArgs() + err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2))) + require.NoError(t, err) + + test.TestSetWatchedAddresses(t, db) + }) + + t.Run("Set watched addresses (some already watched)", func(t *testing.T) { + args := mocks.GetSetAlreadyWatchedAddressesArgs() + err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3))) + require.NoError(t, err) + + test.TestSetAlreadyWatchedAddresses(t, db) + }) + + t.Run("Load watched addresses", func(t *testing.T) { + test.TestLoadWatchedAddresses(t, ind) + }) + + t.Run("Clear watched addresses", func(t *testing.T) { + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + test.TestClearWatchedAddresses(t, db) + }) + + t.Run("Clear watched addresses (empty table)", func(t *testing.T) { + err = ind.ClearWatchedAddresses() + require.NoError(t, err) + + test.TestClearEmptyWatchedAddresses(t, db) + }) +} diff --git a/statediff/indexer/database/sql/writer.go b/statediff/indexer/database/sql/writer.go new file mode 100644 index 000000000000..70d8ba45f11d --- /dev/null +++ b/statediff/indexer/database/sql/writer.go @@ -0,0 +1,187 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/indexer/models" +) + +var ( + nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") +) + +// Writer handles processing and writing of indexed IPLD objects to Postgres +type Writer struct { + db Database +} + +// NewWriter creates a new pointer to a Writer +func NewWriter(db Database) *Writer { + return &Writer{ + db: db, + } +} + +// Close satisfies io.Closer +func (w *Writer) Close() error { + return w.db.Close() +} + +/* +INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) +ON CONFLICT (block_hash, block_number) DO UPDATE SET (block_number, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($1, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16) +*/ +func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error { + _, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(), + header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, w.db.NodeID(), + header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UncleRoot, header.Bloom, + header.Timestamp, header.MhKey, 1, header.Coinbase) + if err != nil { + return fmt.Errorf("error upserting header_cids entry: %v", err) + } + indexerMetrics.blocks.Inc(1) + return nil +} + +/* +INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) +ON CONFLICT (block_hash, block_number) DO NOTHING +*/ +func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error { + _, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(), + uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey) + if err != nil { + return fmt.Errorf("error upserting uncle_cids entry: %v", err) + } + return nil +} + +/* +INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) +ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING +*/ +func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error { + _, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(), + transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, + transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value) + if err != nil { + return fmt.Errorf("error upserting transaction_cids entry: %v", err) + } + indexerMetrics.transactions.Inc(1) + return nil +} + +/* +INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4, $5) +ON CONFLICT (tx_id, index, block_number) DO NOTHING +*/ +func (w *Writer) upsertAccessListElement(tx Tx, accessListElement models.AccessListElementModel) error { + _, err := tx.Exec(w.db.Context(), w.db.InsertAccessListElementStm(), + accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address, + accessListElement.StorageKeys) + if err != nil { + return fmt.Errorf("error upserting access_list_element entry: %v", err) + } + indexerMetrics.accessListEntries.Inc(1) + return nil +} + +/* +INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) +ON CONFLICT (tx_id, header_id, block_number) DO NOTHING +*/ +func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error { + _, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(), + rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState, + rct.PostStatus, rct.LogRoot) + if err != nil { + return fmt.Errorf("error upserting receipt_cids entry: %w", err) + } + indexerMetrics.receipts.Inc(1) + return nil +} + +/* +INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) +ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING +*/ +func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error { + for _, log := range logs { + _, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(), + log.BlockNumber, log.HeaderID, log.LeafCID, log.LeafMhKey, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1, + log.Topic2, log.Topic3, log.Data) + if err != nil { + return fmt.Errorf("error upserting logs entry: %w", err) + } + indexerMetrics.logs.Inc(1) + } + return nil +} + +/* +INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +ON CONFLICT (header_id, state_path, block_number) DO UPDATE SET (block_number, state_leaf_key, cid, node_type, diff, mh_key) = ($1 $3, $4, $6, $7, $8) +*/ +func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error { + var stateKey string + if stateNode.StateKey != nullHash.String() { + stateKey = stateNode.StateKey + } + _, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(), + stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true, + stateNode.MhKey) + if err != nil { + return fmt.Errorf("error upserting state_cids entry: %v", err) + } + return nil +} + +/* +INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7) +ON CONFLICT (header_id, state_path, block_number) DO NOTHING +*/ +func (w *Writer) upsertStateAccount(tx Tx, stateAccount models.StateAccountModel) error { + _, err := tx.Exec(w.db.Context(), w.db.InsertAccountStm(), + stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance, + stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot) + if err != nil { + return fmt.Errorf("error upserting state_accounts entry: %v", err) + } + return nil +} + +/* +INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (header_id, state_path, storage_path, block_number) DO UPDATE SET (block_number, storage_leaf_key, cid, node_type, diff, mh_key) = ($1, $4, $5, $7, $8, $9) +*/ +func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error { + var storageKey string + if storageCID.StorageKey != nullHash.String() { + storageKey = storageCID.StorageKey + } + _, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(), + storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, storageCID.Path, + storageCID.NodeType, true, storageCID.MhKey) + if err != nil { + return fmt.Errorf("error upserting storage_cids entry: %v", err) + } + return nil +} diff --git a/statediff/indexer/interfaces/interfaces.go b/statediff/indexer/interfaces/interfaces.go new file mode 100644 index 000000000000..6910e3f4962a --- /dev/null +++ b/statediff/indexer/interfaces/interfaces.go @@ -0,0 +1,55 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package interfaces + +import ( + "io" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +// StateDiffIndexer interface required to index statediff data +type StateDiffIndexer interface { + PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (Batch, error) + PushStateNode(tx Batch, stateNode sdtypes.StateNode, headerID string) error + PushCodeAndCodeHash(tx Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error + ReportDBMetrics(delay time.Duration, quit <-chan bool) + + // Methods used by WatchAddress API/functionality + LoadWatchedAddresses() ([]common.Address, error) + InsertWatchedAddresses(addresses []sdtypes.WatchAddressArg, currentBlock *big.Int) error + RemoveWatchedAddresses(addresses []sdtypes.WatchAddressArg) error + SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error + ClearWatchedAddresses() error + + io.Closer +} + +// Batch required for indexing data atomically +type Batch interface { + Submit(err error) error +} + +// Config used to configure different underlying implementations +type Config interface { + Type() shared.DBType +} diff --git a/statediff/indexer/ipld/eip2930_test_data/eth-block-12252078 b/statediff/indexer/ipld/eip2930_test_data/eth-block-12252078 new file mode 100644 index 000000000000..baee170abf46 Binary files /dev/null and b/statediff/indexer/ipld/eip2930_test_data/eth-block-12252078 differ diff --git a/statediff/indexer/ipld/eip2930_test_data/eth-block-12365585 b/statediff/indexer/ipld/eip2930_test_data/eth-block-12365585 new file mode 100644 index 000000000000..6f7d87645cb4 Binary files /dev/null and b/statediff/indexer/ipld/eip2930_test_data/eth-block-12365585 differ diff --git a/statediff/indexer/ipld/eip2930_test_data/eth-block-12365586 b/statediff/indexer/ipld/eip2930_test_data/eth-block-12365586 new file mode 100644 index 000000000000..119215e01506 Binary files /dev/null and b/statediff/indexer/ipld/eip2930_test_data/eth-block-12365586 differ diff --git a/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12252078 b/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12252078 new file mode 100644 index 000000000000..b4dcfb65ccf1 Binary files /dev/null and b/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12252078 differ diff --git a/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365585 b/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365585 new file mode 100644 index 000000000000..3bce0cf370bc Binary files /dev/null and b/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365585 differ diff --git a/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365586 b/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365586 new file mode 100644 index 000000000000..c0dbec5a5df1 Binary files /dev/null and b/statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365586 differ diff --git a/statediff/indexer/ipld/eth_account.go b/statediff/indexer/ipld/eth_account.go new file mode 100644 index 000000000000..bd68968b8d62 --- /dev/null +++ b/statediff/indexer/ipld/eth_account.go @@ -0,0 +1,175 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "math/big" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" +) + +// EthAccountSnapshot (eth-account-snapshot codec 0x97) +// represents an ethereum account, i.e. a wallet address or +// a smart contract +type EthAccountSnapshot struct { + *EthAccount + + cid cid.Cid + rawdata []byte +} + +// EthAccount is the building block of EthAccountSnapshot. +// Or, is the former stripped of its cid and rawdata components. +type EthAccount struct { + Nonce uint64 + Balance *big.Int + Root []byte // This is the storage root trie + CodeHash []byte // This is the hash of the EVM code +} + +// Static (compile time) check that EthAccountSnapshot satisfies the +// node.Node interface. +var _ node.Node = (*EthAccountSnapshot)(nil) + +/* + INPUT +*/ + +// Input should be managed by EthStateTrie + +/* + OUTPUT +*/ + +// Output should be managed by EthStateTrie + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the account snapshot. +func (as *EthAccountSnapshot) RawData() []byte { + return as.rawdata +} + +// Cid returns the cid of the transaction. +func (as *EthAccountSnapshot) Cid() cid.Cid { + return as.cid +} + +// String is a helper for output +func (as *EthAccountSnapshot) String() string { + return fmt.Sprintf("", as.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (as *EthAccountSnapshot) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-account-snapshot", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return as, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + case "balance": + return as.Balance, nil, nil + case "codeHash": + return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil + case "nonce": + return as.Nonce, nil, nil + case "root": + return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil + default: + return nil, nil, ErrInvalidLink + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (as *EthAccountSnapshot) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"balance", "codeHash", "nonce", "root"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := as.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (as *EthAccountSnapshot) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Size() (uint64, error) { + return 0, nil +} + +/* + EthAccountSnapshot functions +*/ + +// MarshalJSON processes the transaction into readable JSON format. +func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "balance": as.Balance, + "codeHash": keccak256ToCid(RawBinary, as.CodeHash), + "nonce": as.Nonce, + "root": keccak256ToCid(MEthStorageTrie, as.Root), + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipld/eth_account_test.go b/statediff/indexer/ipld/eth_account_test.go new file mode 100644 index 000000000000..f7c5341a69c5 --- /dev/null +++ b/statediff/indexer/ipld/eth_account_test.go @@ -0,0 +1,297 @@ +package ipld + +import ( + "encoding/json" + "fmt" + "os" + "regexp" + "testing" +) + +/* + Block INTERFACE +*/ +func init() { + if os.Getenv("MODE") != "statediff" { + fmt.Println("Skipping statediff test") + os.Exit(0) + } +} + +func TestAccountSnapshotBlockElements(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + if fmt.Sprintf("%x", eas.RawData())[:10] != "f84e808a03" { + t.Fatal("Wrong Data") + } + + if eas.Cid().String() != + "baglqcgzasckx2alxk43cksshnztjvhfyvbbh6bkp376gtcndm5cg4fkrkhsa" { + t.Fatal("Wrong Cid") + } +} + +func TestAccountSnapshotString(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + if eas.String() != + "" { + t.Fatalf("Wrong String()") + } +} + +func TestAccountSnapshotLoggable(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + l := eas.Loggable() + if _, ok := l["type"]; !ok { + t.Fatal("Loggable map expected the field 'type'") + } + + if l["type"] != "eth-account-snapshot" { + t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-account-snapshot", l["type"]) + } +} + +/* + Node INTERFACE +*/ +func TestAccountSnapshotResolve(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + // Empty path + obj, rest, err := eas.Resolve([]string{}) + reas, ok := obj.(*EthAccountSnapshot) + if !ok { + t.Fatalf("Wrong type of returned object\r\nexpected %T\r\ngot %T", &EthAccountSnapshot{}, reas) + } + if reas.Cid() != eas.Cid() { + t.Fatalf("wrong returned CID\r\nexpected %s\r\ngot %s", eas.Cid().String(), reas.Cid().String()) + } + if rest != nil { + t.Fatal("rest should be nil") + } + if err != nil { + t.Fatal("err should be nil") + } + + // len(p) > 1 + badCases := [][]string{ + {"two", "elements"}, + {"here", "three", "elements"}, + {"and", "here", "four", "elements"}, + } + + for _, bc := range badCases { + obj, rest, err = eas.Resolve(bc) + if obj != nil { + t.Fatal("obj should be nil") + } + if rest != nil { + t.Fatal("rest should be nil") + } + if err.Error() != fmt.Sprintf("unexpected path elements past %s", bc[0]) { + t.Fatal("wrong error") + } + } + + moreBadCases := []string{ + "i", + "am", + "not", + "an", + "account", + "field", + } + for _, mbc := range moreBadCases { + obj, rest, err = eas.Resolve([]string{mbc}) + if obj != nil { + t.Fatal("obj should be nil") + } + if rest != nil { + t.Fatal("rest should be nil") + } + if err != ErrInvalidLink { + t.Fatal("wrong error") + } + } + + goodCases := []string{ + "balance", + "codeHash", + "nonce", + "root", + } + for _, gc := range goodCases { + _, _, err = eas.Resolve([]string{gc}) + if err != nil { + t.Fatalf("error should be nil %v", gc) + } + } +} + +func TestAccountSnapshotTree(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + // Bad cases + tree := eas.Tree("non-empty-string", 0) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + tree = eas.Tree("non-empty-string", 1) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + tree = eas.Tree("", 0) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + // Good cases + tree = eas.Tree("", 1) + lookupElements := map[string]interface{}{ + "balance": nil, + "codeHash": nil, + "nonce": nil, + "root": nil, + } + + if len(tree) != len(lookupElements) { + t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree)) + } + + for _, te := range tree { + if _, ok := lookupElements[te]; !ok { + t.Fatalf("Unexpected Element: %v", te) + } + } +} + +func TestAccountSnapshotResolveLink(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + // bad case + obj, rest, err := eas.ResolveLink([]string{"supercalifragilist"}) + if obj != nil { + t.Fatalf("Expected obj to be nil") + } + if rest != nil { + t.Fatal("Expected rest to be nil") + } + if err != ErrInvalidLink { + t.Fatal("Wrong error") + } + + // good case + obj, rest, err = eas.ResolveLink([]string{"nonce"}) + if obj != nil { + t.Fatalf("Expected obj to be nil") + } + if rest != nil { + t.Fatal("Expected rest to be nil") + } + if err.Error() != "resolved item was not a link" { + t.Fatal("Wrong error") + } +} + +func TestAccountSnapshotCopy(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + defer func() { + r := recover() + if r == nil { + t.Fatal("Expected panic") + } + if r != "implement me" { + t.Fatalf("Wrong panic message\r\n expected %s\r\ngot %s", "'implement me'", r) + } + }() + + _ = eas.Copy() +} + +func TestAccountSnapshotLinks(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + if eas.Links() != nil { + t.Fatal("Links() expected to return nil") + } +} + +func TestAccountSnapshotStat(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + obj, err := eas.Stat() + if obj == nil { + t.Fatal("Expected a not null object node.NodeStat") + } + + if err != nil { + t.Fatal("Expected a nil error") + } +} + +func TestAccountSnapshotSize(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + size, err := eas.Size() + if size != uint64(0) { + t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", 0, size) + } + + if err != nil { + t.Fatal("Expected a nil error") + } +} + +/* + EthAccountSnapshot functions +*/ + +func TestAccountSnapshotMarshalJSON(t *testing.T) { + eas := prepareEthAccountSnapshot(t) + + jsonOutput, err := eas.MarshalJSON() + checkError(err, t) + + var data map[string]interface{} + err = json.Unmarshal(jsonOutput, &data) + checkError(err, t) + + balanceExpression := regexp.MustCompile(`{"balance":16011846000000000000000,`) + if !balanceExpression.MatchString(string(jsonOutput)) { + t.Fatal("Balance expression not found") + } + + code, _ := data["codeHash"].(map[string]interface{}) + if fmt.Sprintf("%s", code["/"]) != + "bafkrwigf2jdadbxxem6je7t5wlomoa6a4ualmu6kqittw6723acf3bneoa" { + t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "bafkrwigf2jdadbxxem6je7t5wlomoa6a4ualmu6kqittw6723acf3bneoa", fmt.Sprintf("%s", code["/"])) + } + + if fmt.Sprintf("%v", data["nonce"]) != "0" { + t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "0", fmt.Sprintf("%v", data["nonce"])) + } + + root, _ := data["root"].(map[string]interface{}) + if fmt.Sprintf("%s", root["/"]) != + "bagmacgzak3ub6fy3zrk2n74dixtjfqhynznurya3tfwk3qabmix3ly3dwqqq" { + t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "bagmacgzak3ub6fy3zrk2n74dixtjfqhynznurya3tfwk3qabmix3ly3dwqqq", fmt.Sprintf("%s", root["/"])) + } +} + +/* + AUXILIARS +*/ +func prepareEthAccountSnapshot(t *testing.T) *EthAccountSnapshot { + fi, err := os.Open("test_data/eth-state-trie-rlp-c9070d") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + return output.elements[1].(*EthAccountSnapshot) +} diff --git a/statediff/indexer/ipld/eth_header.go b/statediff/indexer/ipld/eth_header.go new file mode 100644 index 000000000000..9bc3072773ee --- /dev/null +++ b/statediff/indexer/ipld/eth_header.go @@ -0,0 +1,293 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// EthHeader (eth-block, codec 0x90), represents an ethereum block header +type EthHeader struct { + *types.Header + + cid cid.Cid + rawdata []byte +} + +// Static (compile time) check that EthHeader satisfies the node.Node interface. +var _ node.Node = (*EthHeader)(nil) + +/* + INPUT +*/ + +// NewEthHeader converts a *types.Header into an EthHeader IPLD node +func NewEthHeader(header *types.Header) (*EthHeader, error) { + headerRLP, err := rlp.EncodeToBytes(header) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthHeader{ + Header: header, + cid: c, + rawdata: headerRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthHeader takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) { + h := new(types.Header) + if err := rlp.DecodeBytes(b, h); err != nil { + return nil, err + } + return &EthHeader{ + Header: h, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the block header. +func (b *EthHeader) RawData() []byte { + return b.rawdata +} + +// Cid returns the cid of the block header. +func (b *EthHeader) Cid() cid.Cid { + return b.cid +} + +// String is a helper for output +func (b *EthHeader) String() string { + return fmt.Sprintf("", b.cid) +} + +// Loggable returns a map the type of IPLD Link. +func (b *EthHeader) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-header", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return b, nil, nil + } + + first, rest := p[0], p[1:] + + switch first { + case "parent": + return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil + case "receipts": + return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil + case "root": + return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil + case "tx": + return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil + case "uncles": + return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil + } + + if len(p) != 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", first) + } + + switch first { + case "bloom": + return b.Bloom, nil, nil + case "coinbase": + return b.Coinbase, nil, nil + case "difficulty": + return b.Difficulty, nil, nil + case "extra": + // This is a []byte. By default they are marshalled into Base64. + return fmt.Sprintf("0x%x", b.Extra), nil, nil + case "gaslimit": + return b.GasLimit, nil, nil + case "gasused": + return b.GasUsed, nil, nil + case "mixdigest": + return b.MixDigest, nil, nil + case "nonce": + return b.Nonce, nil, nil + case "number": + return b.Number, nil, nil + case "time": + return b.Time, nil, nil + default: + return nil, nil, ErrInvalidLink + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (b *EthHeader) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + return []string{ + "time", + "bloom", + "coinbase", + "difficulty", + "extra", + "gaslimit", + "gasused", + "mixdigest", + "nonce", + "number", + "parent", + "receipts", + "root", + "tx", + "uncles", + } +} + +// ResolveLink is a helper function that allows easier traversal of links through blocks +func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := b.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (b *EthHeader) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +// HINT: Use `ipfs refs ` +func (b *EthHeader) Links() []*node.Link { + return []*node.Link{ + {Cid: commonHashToCid(MEthHeader, b.ParentHash)}, + {Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, + {Cid: commonHashToCid(MEthStateTrie, b.Root)}, + {Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, + {Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, + } +} + +// Stat will go away. It is here to comply with the Node interface. +func (b *EthHeader) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the Node interface. +func (b *EthHeader) Size() (uint64, error) { + return 0, nil +} + +/* + EthHeader functions +*/ + +// MarshalJSON processes the block header into readable JSON format, +// converting the right links into their cids, and keeping the original +// hex hash, allowing the user to simplify external queries. +func (b *EthHeader) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "time": b.Time, + "bloom": b.Bloom, + "coinbase": b.Coinbase, + "difficulty": b.Difficulty, + "extra": fmt.Sprintf("0x%x", b.Extra), + "gaslimit": b.GasLimit, + "gasused": b.GasUsed, + "mixdigest": b.MixDigest, + "nonce": b.Nonce, + "number": b.Number, + "parent": commonHashToCid(MEthHeader, b.ParentHash), + "receipts": commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash), + "root": commonHashToCid(MEthStateTrie, b.Root), + "tx": commonHashToCid(MEthTxTrie, b.TxHash), + "uncles": commonHashToCid(MEthHeaderList, b.UncleHash), + } + return json.Marshal(out) +} + +// objJSONHeader defines the output of the JSON RPC API for either +// "eth_BlockByHash" or "eth_BlockByHeader". +type objJSONHeader struct { + Result objJSONHeaderResult `json:"result"` +} + +// objJSONBLockResult is the nested struct that takes +// the contents of the JSON field "result". +type objJSONHeaderResult struct { + types.Header // Use its fields and unmarshaler + *objJSONHeaderResultExt // Add these fields to the parsing +} + +// objJSONBLockResultExt facilitates the composition +// of the field "result", adding to the +// `types.Header` fields, both ommers (their hashes) and transactions. +type objJSONHeaderResultExt struct { + OmmerHashes []common.Hash `json:"uncles"` + Transactions []*types.Transaction `json:"transactions"` +} + +// UnmarshalJSON overrides the function types.Header.UnmarshalJSON, allowing us +// to parse the fields of Header, plus ommer hashes and transactions. +// (yes, ommer hashes. You will need to "eth_getUncleCountByBlockHash" per each ommer) +func (o *objJSONHeaderResult) UnmarshalJSON(input []byte) error { + err := o.Header.UnmarshalJSON(input) + if err != nil { + return err + } + + o.objJSONHeaderResultExt = &objJSONHeaderResultExt{} + err = json.Unmarshal(input, o.objJSONHeaderResultExt) + return err +} diff --git a/statediff/indexer/ipld/eth_header_test.go b/statediff/indexer/ipld/eth_header_test.go new file mode 100644 index 000000000000..fa4806fbf28a --- /dev/null +++ b/statediff/indexer/ipld/eth_header_test.go @@ -0,0 +1,585 @@ +package ipld + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "runtime" + "strconv" + "testing" + + block "github.com/ipfs/go-block-format" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" +) + +func TestBlockBodyRlpParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-block-body-rlp-999999") + checkError(err, t) + + output, _, _, err := FromBlockRLP(fi) + checkError(err, t) + + testEthBlockFields(output, t) +} + +func TestBlockHeaderRlpParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-block-header-rlp-999999") + checkError(err, t) + + output, _, _, err := FromBlockRLP(fi) + checkError(err, t) + + testEthBlockFields(output, t) +} + +func TestBlockBodyJsonParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-block-body-json-999999") + checkError(err, t) + + output, _, _, err := FromBlockJSON(fi) + checkError(err, t) + + testEthBlockFields(output, t) +} + +func TestEthBlockProcessTransactionsError(t *testing.T) { + // Let's just change one byte in a field of one of these transactions. + fi, err := os.Open("test_data/error-tx-eth-block-body-json-999999") + checkError(err, t) + + _, _, _, err = FromBlockJSON(fi) + if err == nil { + t.Fatal("Expected an error") + } +} + +// TestDecodeBlockHeader should work for both inputs (block header and block body) +// as what we are storing is just the block header +func TestDecodeBlockHeader(t *testing.T) { + storedEthBlock := prepareStoredEthBlock("test_data/eth-block-header-rlp-999999", t) + + ethBlock, err := DecodeEthHeader(storedEthBlock.Cid(), storedEthBlock.RawData()) + checkError(err, t) + + testEthBlockFields(ethBlock, t) +} + +func TestEthBlockString(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + if ethBlock.String() != "" { + t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "", ethBlock.String()) + } +} + +func TestEthBlockLoggable(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + l := ethBlock.Loggable() + if _, ok := l["type"]; !ok { + t.Fatal("Loggable map expected the field 'type'") + } + + if l["type"] != "eth-header" { + t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-header", l["type"]) + } +} + +func TestEthBlockJSONMarshal(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + jsonOutput, err := ethBlock.MarshalJSON() + checkError(err, t) + + var data map[string]interface{} + err = json.Unmarshal(jsonOutput, &data) + checkError(err, t) + + // Testing all fields is boring, but can help us to avoid + // that dreaded regression + if data["bloom"].(string)[:10] != "0x00000000" { + t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0x00000000", data["bloom"].(string)[:10]) + t.Fatal("Wrong Bloom") + } + if data["coinbase"] != "0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5" { + t.Fatalf("Wrong coinbase\r\nexpected %s\r\ngot %s", "0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5", data["coinbase"]) + } + if parseFloat(data["difficulty"]) != "12555463106190" { + t.Fatalf("Wrong Difficulty\r\nexpected %s\r\ngot %s", "12555463106190", parseFloat(data["difficulty"])) + } + if data["extra"] != "0xd783010303844765746887676f312e342e32856c696e7578" { + t.Fatalf("Wrong Extra\r\nexpected %s\r\ngot %s", "0xd783010303844765746887676f312e342e32856c696e7578", data["extra"]) + } + if parseFloat(data["gaslimit"]) != "3141592" { + t.Fatalf("Wrong Gas limit\r\nexpected %s\r\ngot %s", "3141592", parseFloat(data["gaslimit"])) + } + if parseFloat(data["gasused"]) != "231000" { + t.Fatalf("Wrong Gas used\r\nexpected %s\r\ngot %s", "231000", parseFloat(data["gasused"])) + } + if data["mixdigest"] != "0x5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0" { + t.Fatalf("Wrong Mix digest\r\nexpected %s\r\ngot %s", "0x5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0", data["mixdigest"]) + } + if data["nonce"] != "0xf491f46b60fe04b3" { + t.Fatalf("Wrong nonce\r\nexpected %s\r\ngot %s", "0xf491f46b60fe04b3", data["nonce"]) + } + if parseFloat(data["number"]) != "999999" { + t.Fatalf("Wrong block number\r\nexpected %s\r\ngot %s", "999999", parseFloat(data["number"])) + } + if parseMapElement(data["parent"]) != "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a" { + t.Fatalf("Wrong Parent cid\r\nexpected %s\r\ngot %s", "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a", parseMapElement(data["parent"])) + } + if parseMapElement(data["receipts"]) != "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq" { + t.Fatalf("Wrong Receipt root cid\r\nexpected %s\r\ngot %s", "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq", parseMapElement(data["receipts"])) + } + if parseMapElement(data["root"]) != "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia" { + t.Fatalf("Wrong root hash cid\r\nexpected %s\r\ngot %s", "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia", parseMapElement(data["root"])) + } + if parseFloat(data["time"]) != "1455404037" { + t.Fatalf("Wrong Time\r\nexpected %s\r\ngot %s", "1455404037", parseFloat(data["time"])) + } + if parseMapElement(data["tx"]) != "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka" { + t.Fatalf("Wrong Tx root cid\r\nexpected %s\r\ngot %s", "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka", parseMapElement(data["tx"])) + } + if parseMapElement(data["uncles"]) != "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq" { + t.Fatalf("Wrong Uncle hash cid\r\nexpected %s\r\ngot %s", "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq", parseMapElement(data["uncles"])) + } +} + +func TestEthBlockLinks(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + links := ethBlock.Links() + if links[0].Cid.String() != "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a" { + t.Fatalf("Wrong cid for parent link\r\nexpected: %s\r\ngot %s", "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a", links[0].Cid.String()) + } + if links[1].Cid.String() != "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq" { + t.Fatalf("Wrong cid for receipt root link\r\nexpected: %s\r\ngot %s", "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq", links[1].Cid.String()) + } + if links[2].Cid.String() != "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia" { + t.Fatalf("Wrong cid for state root link\r\nexpected: %s\r\ngot %s", "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia", links[2].Cid.String()) + } + if links[3].Cid.String() != "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka" { + t.Fatalf("Wrong cid for tx root link\r\nexpected: %s\r\ngot %s", "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka", links[3].Cid.String()) + } + if links[4].Cid.String() != "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq" { + t.Fatalf("Wrong cid for uncles root link\r\nexpected: %s\r\ngot %s", "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq", links[4].Cid.String()) + } +} + +func TestEthBlockResolveEmptyPath(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + obj, rest, err := ethBlock.Resolve([]string{}) + checkError(err, t) + + if ethBlock != obj.(*EthHeader) { + t.Fatal("Should have returned the same eth-block object") + } + + if len(rest) != 0 { + t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest)) + } +} + +func TestEthBlockResolveNoSuchLink(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + _, _, err := ethBlock.Resolve([]string{"wewonthavethisfieldever"}) + if err == nil { + t.Fatal("Should have failed with unknown field") + } + + if err != ErrInvalidLink { + t.Fatalf("Wrong error message\r\nexpected %s\r\ngot %s", ErrInvalidLink, err.Error()) + } +} + +func TestEthBlockResolveBloom(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + obj, rest, err := ethBlock.Resolve([]string{"bloom"}) + checkError(err, t) + + // The marshaler of types.Bloom should output it as 0x + bloomInText := fmt.Sprintf("%x", obj.(types.Bloom)) + if bloomInText[:10] != "0000000000" { + t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0000000000", bloomInText[:10]) + } + + if len(rest) != 0 { + t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest)) + } +} + +func TestEthBlockResolveBloomExtraPathElements(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + obj, rest, err := ethBlock.Resolve([]string{"bloom", "unexpected", "extra", "elements"}) + if obj != nil { + t.Fatal("Returned obj should be nil") + } + + if rest != nil { + t.Fatal("Returned rest should be nil") + } + + if err.Error() != "unexpected path elements past bloom" { + t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "unexpected path elements past bloom", err.Error()) + } +} + +func TestEthBlockResolveNonLinkFields(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + testCases := map[string][]string{ + "coinbase": {"%x", "52bc44d5378309ee2abf1539bf71de1b7d7be3b5"}, + "difficulty": {"%s", "12555463106190"}, + "extra": {"%s", "0xd783010303844765746887676f312e342e32856c696e7578"}, + "gaslimit": {"%d", "3141592"}, + "gasused": {"%d", "231000"}, + "mixdigest": {"%x", "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0"}, + "nonce": {"%x", "f491f46b60fe04b3"}, + "number": {"%s", "999999"}, + "time": {"%d", "1455404037"}, + } + + for field, value := range testCases { + obj, rest, err := ethBlock.Resolve([]string{field}) + checkError(err, t) + + format := value[0] + result := value[1] + if fmt.Sprintf(format, obj) != result { + t.Fatalf("Wrong %v\r\nexpected %v\r\ngot %s", field, result, fmt.Sprintf(format, obj)) + } + + if len(rest) != 0 { + t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest)) + } + } +} + +func TestEthBlockResolveNonLinkFieldsExtraPathElements(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + testCases := []string{ + "coinbase", + "difficulty", + "extra", + "gaslimit", + "gasused", + "mixdigest", + "nonce", + "number", + "time", + } + + for _, field := range testCases { + obj, rest, err := ethBlock.Resolve([]string{field, "unexpected", "extra", "elements"}) + if obj != nil { + t.Fatal("Returned obj should be nil") + } + + if rest != nil { + t.Fatal("Returned rest should be nil") + } + + if err.Error() != "unexpected path elements past "+field { + t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "unexpected path elements past "+field, err.Error()) + } + } +} + +func TestEthBlockResolveLinkFields(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + testCases := map[string]string{ + "parent": "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a", + "receipts": "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq", + "root": "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia", + "tx": "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka", + "uncles": "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq", + } + + for field, result := range testCases { + obj, rest, err := ethBlock.Resolve([]string{field, "anything", "goes", "here"}) + checkError(err, t) + + lnk, ok := obj.(*node.Link) + if !ok { + t.Fatal("Returned object is not a link") + } + + if lnk.Cid.String() != result { + t.Fatalf("Wrong %s cid\r\nexpected %v\r\ngot %v", field, result, lnk.Cid.String()) + } + + for i, p := range []string{"anything", "goes", "here"} { + if rest[i] != p { + t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", p, rest[i]) + } + } + } +} + +func TestEthBlockTreeBadParams(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + tree := ethBlock.Tree("non-empty-string", 0) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + tree = ethBlock.Tree("non-empty-string", 1) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + tree = ethBlock.Tree("", 0) + if tree != nil { + t.Fatal("Expected nil to be returned") + } +} + +func TestEThBlockTree(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + tree := ethBlock.Tree("", 1) + lookupElements := map[string]interface{}{ + "bloom": nil, + "coinbase": nil, + "difficulty": nil, + "extra": nil, + "gaslimit": nil, + "gasused": nil, + "mixdigest": nil, + "nonce": nil, + "number": nil, + "parent": nil, + "receipts": nil, + "root": nil, + "time": nil, + "tx": nil, + "uncles": nil, + } + + if len(tree) != len(lookupElements) { + t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree)) + } + + for _, te := range tree { + if _, ok := lookupElements[te]; !ok { + t.Fatalf("Unexpected Element: %v", te) + } + } +} + +/* + The two functions above: TestEthBlockResolveNonLinkFields and + TestEthBlockResolveLinkFields did all the heavy lifting. Then, we will + just test two use cases. +*/ +func TestEthBlockResolveLinksBadLink(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + obj, rest, err := ethBlock.ResolveLink([]string{"supercalifragilist"}) + if obj != nil { + t.Fatalf("Expected obj to be nil") + } + if rest != nil { + t.Fatal("Expected rest to be nil") + } + + if err != ErrInvalidLink { + t.Fatalf("Expected error\r\nexpected %s\r\ngot %s", ErrInvalidLink, err) + } +} + +func TestEthBlockResolveLinksGoodLink(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + obj, rest, err := ethBlock.ResolveLink([]string{"tx", "0", "0", "0"}) + if obj == nil { + t.Fatalf("Expected valid *node.Link obj to be returned") + } + + if rest == nil { + t.Fatal("Expected rest to be returned") + } + for i, p := range []string{"0", "0", "0"} { + if rest[i] != p { + t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", p, rest[i]) + } + } + + if err != nil { + t.Fatal("Non error expected") + } +} + +/* + These functions below should go away + We are working on test coverage anyways... +*/ +func TestEthBlockCopy(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + defer func() { + r := recover() + if r == nil { + t.Fatal("Expected panic") + } + if r != "implement me" { + t.Fatalf("Wrong panic message\r\nexpected %s\r\ngot %s", "'implement me'", r) + } + }() + + _ = ethBlock.Copy() +} + +func TestEthBlockStat(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + obj, err := ethBlock.Stat() + if obj == nil { + t.Fatal("Expected a not null object node.NodeStat") + } + + if err != nil { + t.Fatal("Expected a nil error") + } +} + +func TestEthBlockSize(t *testing.T) { + ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t) + + size, err := ethBlock.Size() + if size != 0 { + t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", 0, size) + } + + if err != nil { + t.Fatal("Expected a nil error") + } +} + +/* + AUXILIARS +*/ + +// checkError makes 3 lines into 1. +func checkError(err error, t *testing.T) { + if err != nil { + _, fn, line, _ := runtime.Caller(1) + t.Fatalf("[%v:%v] %v", fn, line, err) + } +} + +// parseFloat is a convenience function to test json output +func parseFloat(v interface{}) string { + return strconv.FormatFloat(v.(float64), 'f', 0, 64) +} + +// parseMapElement is a convenience function to tets json output +func parseMapElement(v interface{}) string { + return v.(map[string]interface{})["/"].(string) +} + +// prepareStoredEthBlock reads the block from a file source to get its rawdata +// and computes its cid, for then, feeding it into a new IPLD block function. +// So we can pretend that we got this block from the datastore +func prepareStoredEthBlock(filepath string, t *testing.T) *block.BasicBlock { + // Prepare the "fetched block". This one is supposed to be in the datastore + // and given away by github.com/ipfs/go-ipfs/merkledag + fi, err := os.Open(filepath) + checkError(err, t) + + b, err := ioutil.ReadAll(fi) + checkError(err, t) + + c, err := RawdataToCid(MEthHeader, b, multihash.KECCAK_256) + checkError(err, t) + + // It's good to clarify that this one below is an IPLD block + storedEthBlock, err := block.NewBlockWithCid(b, c) + checkError(err, t) + + return storedEthBlock +} + +// prepareDecodedEthBlock is more complex than function above, as it stores a +// basic block and RLP-decodes it +func prepareDecodedEthBlock(filepath string, t *testing.T) *EthHeader { + // Get the block from the datastore and decode it. + storedEthBlock := prepareStoredEthBlock("test_data/eth-block-header-rlp-999999", t) + ethBlock, err := DecodeEthHeader(storedEthBlock.Cid(), storedEthBlock.RawData()) + checkError(err, t) + + return ethBlock +} + +// testEthBlockFields checks the fields of EthBlock one by one. +func testEthBlockFields(ethBlock *EthHeader, t *testing.T) { + // Was the cid calculated? + if ethBlock.Cid().String() != "bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a" { + t.Fatalf("Wrong cid\r\nexpected %s\r\ngot %s", "bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a", ethBlock.Cid().String()) + } + + // Do we have the rawdata available? + if fmt.Sprintf("%x", ethBlock.RawData()[:10]) != "f90218a0d33c9dde9fff" { + t.Fatalf("Wrong Rawdata\r\nexpected %s\r\ngot %s", "f90218a0d33c9dde9fff", fmt.Sprintf("%x", ethBlock.RawData()[:10])) + } + + // Proper Fields of types.Header + if fmt.Sprintf("%x", ethBlock.ParentHash) != "d33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4" { + t.Fatalf("Wrong ParentHash\r\nexpected %s\r\ngot %s", "d33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4", fmt.Sprintf("%x", ethBlock.ParentHash)) + } + if fmt.Sprintf("%x", ethBlock.UncleHash) != "1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" { + t.Fatalf("Wrong UncleHash field\r\nexpected %s\r\ngot %s", "1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", fmt.Sprintf("%x", ethBlock.UncleHash)) + } + if fmt.Sprintf("%x", ethBlock.Coinbase) != "52bc44d5378309ee2abf1539bf71de1b7d7be3b5" { + t.Fatalf("Wrong Coinbase\r\nexpected %s\r\ngot %s", "52bc44d5378309ee2abf1539bf71de1b7d7be3b5", fmt.Sprintf("%x", ethBlock.Coinbase)) + } + if fmt.Sprintf("%x", ethBlock.Root) != "ed98aa4b5b19c82fb35364f08508ae0a6dec665fa57663dca94c5d70554cde10" { + t.Fatalf("Wrong Root\r\nexpected %s\r\ngot %s", "ed98aa4b5b19c82fb35364f08508ae0a6dec665fa57663dca94c5d70554cde10", fmt.Sprintf("%x", ethBlock.Root)) + } + if fmt.Sprintf("%x", ethBlock.TxHash) != "447cbd8c48f498a6912b10831cdff59c7fbfcbbe735ca92883d4fa06dcd7ae54" { + t.Fatalf("Wrong TxHash\r\nexpected %s\r\ngot %s", "447cbd8c48f498a6912b10831cdff59c7fbfcbbe735ca92883d4fa06dcd7ae54", fmt.Sprintf("%x", ethBlock.TxHash)) + } + if fmt.Sprintf("%x", ethBlock.ReceiptHash) != "7fa0f6ca2a01823208d80801edad37e3e3a003b55c89319b45eb1f97862ad229" { + t.Fatalf("Wrong ReceiptHash\r\nexpected %s\r\ngot %s", "7fa0f6ca2a01823208d80801edad37e3e3a003b55c89319b45eb1f97862ad229", fmt.Sprintf("%x", ethBlock.ReceiptHash)) + } + if len(ethBlock.Bloom) != 256 { + t.Fatalf("Wrong Bloom Length\r\nexpected %d\r\ngot %d", 256, len(ethBlock.Bloom)) + } + if fmt.Sprintf("%x", ethBlock.Bloom[71:76]) != "0000000000" { // You wouldn't want me to print out the whole bloom field? + t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0000000000", fmt.Sprintf("%x", ethBlock.Bloom[71:76])) + } + if ethBlock.Difficulty.String() != "12555463106190" { + t.Fatalf("Wrong Difficulty\r\nexpected %s\r\ngot %s", "12555463106190", ethBlock.Difficulty.String()) + } + if ethBlock.Number.String() != "999999" { + t.Fatalf("Wrong Block Number\r\nexpected %s\r\ngot %s", "999999", ethBlock.Number.String()) + } + if ethBlock.GasLimit != uint64(3141592) { + t.Fatalf("Wrong Gas Limit\r\nexpected %d\r\ngot %d", 3141592, ethBlock.GasLimit) + } + if ethBlock.GasUsed != uint64(231000) { + t.Fatalf("Wrong Gas Used\r\nexpected %d\r\ngot %d", 231000, ethBlock.GasUsed) + } + if ethBlock.Time != uint64(1455404037) { + t.Fatalf("Wrong Time\r\nexpected %d\r\ngot %d", 1455404037, ethBlock.Time) + } + if fmt.Sprintf("%x", ethBlock.Extra) != "d783010303844765746887676f312e342e32856c696e7578" { + t.Fatalf("Wrong Extra\r\nexpected %s\r\ngot %s", "d783010303844765746887676f312e342e32856c696e7578", fmt.Sprintf("%x", ethBlock.Extra)) + } + if fmt.Sprintf("%x", ethBlock.Nonce) != "f491f46b60fe04b3" { + t.Fatalf("Wrong Nonce\r\nexpected %s\r\ngot %s", "f491f46b60fe04b3", fmt.Sprintf("%x", ethBlock.Nonce)) + } + if fmt.Sprintf("%x", ethBlock.MixDigest) != "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0" { + t.Fatalf("Wrong MixDigest\r\nexpected %s\r\ngot %s", "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0", fmt.Sprintf("%x", ethBlock.MixDigest)) + } +} diff --git a/statediff/indexer/ipld/eth_log.go b/statediff/indexer/ipld/eth_log.go new file mode 100644 index 000000000000..225c44117292 --- /dev/null +++ b/statediff/indexer/ipld/eth_log.go @@ -0,0 +1,158 @@ +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// EthLog (eth-log, codec 0x9a), represents an ethereum block header +type EthLog struct { + *types.Log + + rawData []byte + cid cid.Cid +} + +// Static (compile time) check that EthLog satisfies the node.Node interface. +var _ node.Node = (*EthLog)(nil) + +// NewLog create a new EthLog IPLD node +func NewLog(log *types.Log) (*EthLog, error) { + logRaw, err := rlp.EncodeToBytes(log) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthLog, logRaw, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthLog{ + Log: log, + cid: c, + rawData: logRaw, + }, nil +} + +// DecodeEthLogs takes a cid and its raw binary data +func DecodeEthLogs(c cid.Cid, b []byte) (*EthLog, error) { + l := new(types.Log) + if err := rlp.DecodeBytes(b, l); err != nil { + return nil, err + } + return &EthLog{ + Log: l, + cid: c, + rawData: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the log. +func (l *EthLog) RawData() []byte { + return l.rawData +} + +// Cid returns the cid of the receipt log. +func (l *EthLog) Cid() cid.Cid { + return l.cid +} + +// String is a helper for output +func (l *EthLog) String() string { + return fmt.Sprintf("", l.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (l *EthLog) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-log", + } +} + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (l *EthLog) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return l, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + case "address": + return l.Address, nil, nil + case "data": + // This is a []byte. By default they are marshalled into Base64. + return fmt.Sprintf("0x%x", l.Data), nil, nil + case "topics": + return l.Topics, nil, nil + case "logIndex": + return l.Index, nil, nil + case "removed": + return l.Removed, nil, nil + default: + return nil, nil, ErrInvalidLink + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (l *EthLog) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + return []string{ + "address", + "data", + "topics", + "logIndex", + "removed", + } +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (l *EthLog) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := l.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (l *EthLog) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (l *EthLog) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (l *EthLog) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (l *EthLog) Size() (uint64, error) { + return 0, nil +} diff --git a/statediff/indexer/ipld/eth_log_trie.go b/statediff/indexer/ipld/eth_log_trie.go new file mode 100644 index 000000000000..8e8af9c798f1 --- /dev/null +++ b/statediff/indexer/ipld/eth_log_trie.go @@ -0,0 +1,144 @@ +package ipld + +import ( + "fmt" + + node "github.com/ipfs/go-ipld-format" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// EthLogTrie (eth-tx-trie codec 0x9p) represents +// a node from the transaction trie in ethereum. +type EthLogTrie struct { + *TrieNode +} + +/* + OUTPUT +*/ + +// DecodeEthLogTrie returns an EthLogTrie object from its cid and rawdata. +func DecodeEthLogTrie(c cid.Cid, b []byte) (*EthLogTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthLogTrieLeaf) + if err != nil { + return nil, err + } + return &EthLogTrie{TrieNode: tn}, nil +} + +// decodeEthLogTrieLeaf parses a eth-log-trie leaf +// from decoded RLP elements +func decodeEthLogTrieLeaf(i []interface{}) ([]interface{}, error) { + l := new(types.Log) + if err := rlp.DecodeBytes(i[1].([]byte), l); err != nil { + return nil, err + } + c, err := RawdataToCid(MEthLogTrie, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + + return []interface{}{ + i[0].([]byte), + &EthLog{ + Log: l, + cid: c, + rawData: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthLogTrie) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthLogTrie) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthLogTrie) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthLogTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-log-trie", + } +} + +// logTrie wraps a localTrie for use on the receipt trie. +type logTrie struct { + *localTrie +} + +// newLogTrie initializes and returns a logTrie. +func newLogTrie() *logTrie { + return &logTrie{ + localTrie: newLocalTrie(), + } +} + +// getNodes invokes the localTrie, which computes the root hash of the +// log trie and returns its sql keys, to return a slice +// of EthLogTrie nodes. +func (rt *logTrie) getNodes() ([]node.Node, error) { + keys, err := rt.getKeys() + if err != nil { + return nil, err + } + + out := make([]node.Node, 0, len(keys)) + for _, k := range keys { + n, err := rt.getNodeFromDB(k) + if err != nil { + return nil, err + } + out = append(out, n) + } + + return out, nil +} + +func (rt *logTrie) getNodeFromDB(key []byte) (*EthLogTrie, error) { + rawdata, err := rt.db.Get(key) + if err != nil { + return nil, err + } + tn := &TrieNode{ + cid: keccak256ToCid(MEthLogTrie, key), + rawdata: rawdata, + } + return &EthLogTrie{TrieNode: tn}, nil +} + +// getLeafNodes invokes the localTrie, which returns a slice +// of EthLogTrie leaf nodes. +func (rt *logTrie) getLeafNodes() ([]*EthLogTrie, []*nodeKey, error) { + keys, err := rt.getLeafKeys() + if err != nil { + return nil, nil, err + } + out := make([]*EthLogTrie, 0, len(keys)) + for _, k := range keys { + n, err := rt.getNodeFromDB(k.dbKey) + if err != nil { + return nil, nil, err + } + out = append(out, n) + } + + return out, keys, nil +} diff --git a/statediff/indexer/ipld/eth_parser.go b/statediff/indexer/ipld/eth_parser.go new file mode 100644 index 000000000000..03061f828740 --- /dev/null +++ b/statediff/indexer/ipld/eth_parser.go @@ -0,0 +1,302 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// FromBlockRLP takes an RLP message representing +// an ethereum block header or body (header, ommers and txs) +// to return it as a set of IPLD nodes for further processing. +func FromBlockRLP(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) { + // We may want to use this stream several times + rawdata, err := ioutil.ReadAll(r) + if err != nil { + return nil, nil, nil, err + } + + // Let's try to decode the received element as a block body + var decodedBlock types.Block + err = rlp.Decode(bytes.NewBuffer(rawdata), &decodedBlock) + if err != nil { + if err.Error()[:41] != "rlp: expected input list for types.Header" { + return nil, nil, nil, err + } + + // Maybe it is just a header... (body sans ommers and txs) + var decodedHeader types.Header + err := rlp.Decode(bytes.NewBuffer(rawdata), &decodedHeader) + if err != nil { + return nil, nil, nil, err + } + + c, err := RawdataToCid(MEthHeader, rawdata, multihash.KECCAK_256) + if err != nil { + return nil, nil, nil, err + } + // It was a header + return &EthHeader{ + Header: &decodedHeader, + cid: c, + rawdata: rawdata, + }, nil, nil, nil + } + + // This is a block body (header + ommers + txs) + // We'll extract the header bits here + headerRawData := getRLP(decodedBlock.Header()) + c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256) + if err != nil { + return nil, nil, nil, err + } + ethBlock := &EthHeader{ + Header: decodedBlock.Header(), + cid: c, + rawdata: headerRawData, + } + + // Process the found eth-tx objects + ethTxNodes, ethTxTrieNodes, err := processTransactions(decodedBlock.Transactions(), + decodedBlock.Header().TxHash[:]) + if err != nil { + return nil, nil, nil, err + } + + return ethBlock, ethTxNodes, ethTxTrieNodes, nil +} + +// FromBlockJSON takes the output of an ethereum client JSON API +// (i.e. parity or geth) and returns a set of IPLD nodes. +func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) { + var obj objJSONHeader + dec := json.NewDecoder(r) + err := dec.Decode(&obj) + if err != nil { + return nil, nil, nil, err + } + + headerRawData := getRLP(&obj.Result.Header) + c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256) + if err != nil { + return nil, nil, nil, err + } + ethBlock := &EthHeader{ + Header: &obj.Result.Header, + cid: c, + rawdata: headerRawData, + } + + // Process the found eth-tx objects + ethTxNodes, ethTxTrieNodes, err := processTransactions(obj.Result.Transactions, + obj.Result.Header.TxHash[:]) + if err != nil { + return nil, nil, nil, err + } + + return ethBlock, ethTxNodes, ethTxTrieNodes, nil +} + +// FromBlockAndReceipts takes a block and processes it +// to return it a set of IPLD nodes for further processing. +func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) { + // Process the header + headerNode, err := NewEthHeader(block.Header()) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, nil, nil, err + } + + // Process the uncles + uncleNodes := make([]*EthHeader, len(block.Uncles())) + for i, uncle := range block.Uncles() { + uncleNode, err := NewEthHeader(uncle) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, nil, nil, err + } + uncleNodes[i] = uncleNode + } + + // Process the txs + txNodes, txTrieNodes, err := processTransactions(block.Transactions(), + block.Header().TxHash[:]) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, nil, nil, err + } + + // Process the receipts and logs + rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts, + block.Header().ReceiptHash[:]) + + return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err +} + +// processTransactions will take the found transactions in a parsed block body +// to return IPLD node slices for eth-tx and eth-tx-trie +func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*EthTx, []*EthTxTrie, error) { + var ethTxNodes []*EthTx + transactionTrie := newTxTrie() + + for idx, tx := range txs { + ethTx, err := NewEthTx(tx) + if err != nil { + return nil, nil, err + } + ethTxNodes = append(ethTxNodes, ethTx) + if err := transactionTrie.Add(idx, ethTx.RawData()); err != nil { + return nil, nil, err + } + } + + if !bytes.Equal(transactionTrie.rootHash(), expectedTxRoot) { + return nil, nil, fmt.Errorf("wrong transaction hash computed") + } + txTrieNodes, err := transactionTrie.getNodes() + return ethTxNodes, txTrieNodes, err +} + +// processReceiptsAndLogs will take in receipts +// to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie, eth-log-trie-CID, eth-rct-trie-CID +func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) { + // Pre allocating memory. + ethRctNodes := make([]*EthReceipt, 0, len(rcts)) + ethLogleafNodeCids := make([][]cid.Cid, 0, len(rcts)) + ethLogTrieAndLogNodes := make([][]node.Node, 0, len(rcts)) + + receiptTrie := NewRctTrie() + + for idx, rct := range rcts { + // Process logs for each receipt. + logTrieNodes, leafNodeCids, logTrieHash, err := processLogs(rct.Logs) + if err != nil { + return nil, nil, nil, nil, nil, err + } + rct.LogRoot = logTrieHash + ethLogTrieAndLogNodes = append(ethLogTrieAndLogNodes, logTrieNodes) + ethLogleafNodeCids = append(ethLogleafNodeCids, leafNodeCids) + + ethRct, err := NewReceipt(rct) + if err != nil { + return nil, nil, nil, nil, nil, err + } + + ethRctNodes = append(ethRctNodes, ethRct) + if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil { + return nil, nil, nil, nil, nil, err + } + } + + if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) { + return nil, nil, nil, nil, nil, fmt.Errorf("wrong receipt hash computed") + } + + rctTrieNodes, err := receiptTrie.GetNodes() + if err != nil { + return nil, nil, nil, nil, nil, err + } + + rctLeafNodes, keys, err := receiptTrie.GetLeafNodes() + if err != nil { + return nil, nil, nil, nil, nil, err + } + + ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes)) + for i, rln := range rctLeafNodes { + var idx uint + + r := bytes.NewReader(keys[i].TrieKey) + err = rlp.Decode(r, &idx) + if err != nil { + return nil, nil, nil, nil, nil, err + } + ethRctleafNodeCids[idx] = rln.Cid() + } + + return ethRctNodes, rctTrieNodes, ethLogTrieAndLogNodes, ethLogleafNodeCids, ethRctleafNodeCids, err +} + +const keccak256Length = 32 + +func processLogs(logs []*types.Log) ([]node.Node, []cid.Cid, common.Hash, error) { + logTr := newLogTrie() + shortLog := make(map[uint64]*EthLog, len(logs)) + for idx, log := range logs { + logRaw, err := rlp.EncodeToBytes(log) + if err != nil { + return nil, nil, common.Hash{}, err + } + // if len(logRaw) <= keccak256Length it is possible this value's "leaf node" + // will be stored in its parent branch but only if len(partialPathOfTheNode) + len(logRaw) <= keccak256Length + // But we can't tell what the partial path will be until the trie is Commit()-ed + // So wait until we collect all the leaf nodes, and if we are missing any at the indexes we note in shortLogCIDs + // we know that these "leaf nodes" were internalized into their parent branch node and we move forward with + // using the cid.Cid we cached in shortLogCIDs + if len(logRaw) <= keccak256Length { + logNode, err := NewLog(log) + if err != nil { + return nil, nil, common.Hash{}, err + } + shortLog[uint64(idx)] = logNode + } + if err = logTr.Add(idx, logRaw); err != nil { + return nil, nil, common.Hash{}, err + } + } + + logTrieNodes, err := logTr.getNodes() + if err != nil { + return nil, nil, common.Hash{}, err + } + + leafNodes, keys, err := logTr.getLeafNodes() + if err != nil { + return nil, nil, common.Hash{}, err + } + leafNodeCids := make([]cid.Cid, len(logs)) + for i, ln := range leafNodes { + var idx uint + + r := bytes.NewReader(keys[i].TrieKey) + err = rlp.Decode(r, &idx) + if err != nil { + return nil, nil, common.Hash{}, err + } + leafNodeCids[idx] = ln.Cid() + } + // this is where we check which logs <= keccak256Length were actually internalized into parent branch node + // and replace those that were with the cid.Cid for the raw log IPLD + for i, l := range shortLog { + if !leafNodeCids[i].Defined() { + leafNodeCids[i] = l.Cid() + // if the leaf node was internalized, we append an IPLD for log itself to the list of IPLDs we need to publish + logTrieNodes = append(logTrieNodes, l) + } + } + + return logTrieNodes, leafNodeCids, common.BytesToHash(logTr.rootHash()), err +} diff --git a/statediff/indexer/ipld/eth_parser_test.go b/statediff/indexer/ipld/eth_parser_test.go new file mode 100644 index 000000000000..bcf28efde148 --- /dev/null +++ b/statediff/indexer/ipld/eth_parser_test.go @@ -0,0 +1,108 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" +) + +type kind string + +const ( + legacy kind = "legacy" + eip1559 kind = "eip2930" +) + +var blockFileNames = []string{ + "eth-block-12252078", + "eth-block-12365585", + "eth-block-12365586", +} + +var receiptsFileNames = []string{ + "eth-receipts-12252078", + "eth-receipts-12365585", + "eth-receipts-12365586", +} + +var kinds = []kind{ + eip1559, + eip1559, + legacy, +} + +type testCase struct { + kind kind + block *types.Block + receipts types.Receipts +} + +func loadBlockData(t *testing.T) []testCase { + fileDir := "./eip2930_test_data" + testCases := make([]testCase, len(blockFileNames)) + for i, blockFileName := range blockFileNames { + blockRLP, err := ioutil.ReadFile(filepath.Join(fileDir, blockFileName)) + if err != nil { + t.Fatalf("failed to load blockRLP from file, err %v", err) + } + block := new(types.Block) + if err := rlp.DecodeBytes(blockRLP, block); err != nil { + t.Fatalf("failed to decode blockRLP, err %v", err) + } + receiptsFileName := receiptsFileNames[i] + receiptsRLP, err := ioutil.ReadFile(filepath.Join(fileDir, receiptsFileName)) + if err != nil { + t.Fatalf("failed to load receiptsRLP from file, err %s", err) + } + receipts := make(types.Receipts, 0) + if err := rlp.DecodeBytes(receiptsRLP, &receipts); err != nil { + t.Fatalf("failed to decode receiptsRLP, err %s", err) + } + testCases[i] = testCase{ + block: block, + receipts: receipts, + kind: kinds[i], + } + } + return testCases +} + +func TestFromBlockAndReceipts(t *testing.T) { + testCases := loadBlockData(t) + for _, tc := range testCases { + _, _, _, _, _, _, _, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts) + if err != nil { + t.Fatalf("error generating IPLDs from block and receipts, err %v, kind %s, block hash %s", err, tc.kind, tc.block.Hash()) + } + } +} + +func TestProcessLogs(t *testing.T) { + logs := []*types.Log{mocks.MockLog1, mocks.MockLog2} + nodes, cids, _, err := processLogs(logs) + require.NoError(t, err) + require.GreaterOrEqual(t, len(nodes), len(logs)) + require.Equal(t, len(logs), len(cids)) +} diff --git a/statediff/indexer/ipld/eth_receipt.go b/statediff/indexer/ipld/eth_receipt.go new file mode 100644 index 000000000000..ccd785515b8a --- /dev/null +++ b/statediff/indexer/ipld/eth_receipt.go @@ -0,0 +1,205 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" +) + +type EthReceipt struct { + *types.Receipt + + rawdata []byte + cid cid.Cid +} + +// Static (compile time) check that EthReceipt satisfies the node.Node interface. +var _ node.Node = (*EthReceipt)(nil) + +/* + INPUT +*/ + +// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node +func NewReceipt(receipt *types.Receipt) (*EthReceipt, error) { + rctRaw, err := receipt.MarshalBinary() + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTxReceipt, rctRaw, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthReceipt{ + Receipt: receipt, + cid: c, + rawdata: rctRaw, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthReceipt takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) { + r := new(types.Receipt) + if err := r.UnmarshalBinary(b); err != nil { + return nil, err + } + return &EthReceipt{ + Receipt: r, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the receipt. +func (r *EthReceipt) RawData() []byte { + return r.rawdata +} + +// Cid returns the cid of the receipt. +func (r *EthReceipt) Cid() cid.Cid { + return r.cid +} + +// String is a helper for output +func (r *EthReceipt) String() string { + return fmt.Sprintf("", r.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (r *EthReceipt) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-receipt", + } +} + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (r *EthReceipt) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return r, nil, nil + } + + first, rest := p[0], p[1:] + if first != "logs" && len(p) != 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", first) + } + + switch first { + case "logs": + return &node.Link{Cid: commonHashToCid(MEthLog, r.LogRoot)}, rest, nil + case "root": + return r.PostState, nil, nil + case "status": + return r.Status, nil, nil + case "cumulativeGasUsed": + return r.CumulativeGasUsed, nil, nil + case "logsBloom": + return r.Bloom, nil, nil + case "transactionHash": + return r.TxHash, nil, nil + case "contractAddress": + return r.ContractAddress, nil, nil + case "gasUsed": + return r.GasUsed, nil, nil + case "type": + return r.Type, nil, nil + default: + return nil, nil, ErrInvalidLink + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (r *EthReceipt) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"type", "root", "status", "cumulativeGasUsed", "logsBloom", "logs", "transactionHash", "contractAddress", "gasUsed"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (r *EthReceipt) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := r.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (r *EthReceipt) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (r *EthReceipt) Links() []*node.Link { + return []*node.Link{ + {Cid: commonHashToCid(MEthLog, r.LogRoot)}, + } +} + +// Stat will go away. It is here to comply with the interface. +func (r *EthReceipt) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (r *EthReceipt) Size() (uint64, error) { + return strconv.ParseUint(r.Receipt.Size().String(), 10, 64) +} + +/* + EthReceipt functions +*/ + +// MarshalJSON processes the receipt into readable JSON format. +func (r *EthReceipt) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "root": r.PostState, + "status": r.Status, + "cumulativeGasUsed": r.CumulativeGasUsed, + "logsBloom": r.Bloom, + "logs": r.Logs, + "transactionHash": r.TxHash, + "contractAddress": r.ContractAddress, + "gasUsed": r.GasUsed, + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipld/eth_receipt_trie.go b/statediff/indexer/ipld/eth_receipt_trie.go new file mode 100644 index 000000000000..75d40eedba77 --- /dev/null +++ b/statediff/indexer/ipld/eth_receipt_trie.go @@ -0,0 +1,175 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" +) + +// EthRctTrie (eth-tx-trie codec 0x92) represents +// a node from the transaction trie in ethereum. +type EthRctTrie struct { + *TrieNode +} + +// Static (compile time) check that EthRctTrie satisfies the node.Node interface. +var _ node.Node = (*EthRctTrie)(nil) + +/* + INPUT +*/ + +// To create a proper trie of the eth-tx-trie objects, it is required +// to input all transactions belonging to a forest in a single step. +// We are adding the transactions, and creating its trie on +// block body parsing time. + +/* + OUTPUT +*/ + +// DecodeEthRctTrie returns an EthRctTrie object from its cid and rawdata. +func DecodeEthRctTrie(c cid.Cid, b []byte) (*EthRctTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthRctTrieLeaf) + if err != nil { + return nil, err + } + return &EthRctTrie{TrieNode: tn}, nil +} + +// decodeEthRctTrieLeaf parses a eth-rct-trie leaf +//from decoded RLP elements +func decodeEthRctTrieLeaf(i []interface{}) ([]interface{}, error) { + r := new(types.Receipt) + if err := r.UnmarshalBinary(i[1].([]byte)); err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTxReceipt, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthReceipt{ + Receipt: r, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthRctTrie) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthRctTrie) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthRctTrie) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthRctTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-rct-trie", + } +} + +/* + EthRctTrie functions +*/ + +// rctTrie wraps a localTrie for use on the receipt trie. +type rctTrie struct { + *localTrie +} + +// NewRctTrie initializes and returns a rctTrie. +func NewRctTrie() *rctTrie { + return &rctTrie{ + localTrie: newLocalTrie(), + } +} + +// GetNodes invokes the localTrie, which computes the root hash of the +// transaction trie and returns its sql keys, to return a slice +// of EthRctTrie nodes. +func (rt *rctTrie) GetNodes() ([]*EthRctTrie, error) { + keys, err := rt.getKeys() + if err != nil { + return nil, err + } + var out []*EthRctTrie + + for _, k := range keys { + n, err := rt.getNodeFromDB(k) + if err != nil { + return nil, err + } + out = append(out, n) + } + + return out, nil +} + +// GetLeafNodes invokes the localTrie, which returns a slice +// of EthRctTrie leaf nodes. +func (rt *rctTrie) GetLeafNodes() ([]*EthRctTrie, []*nodeKey, error) { + keys, err := rt.getLeafKeys() + if err != nil { + return nil, nil, err + } + + out := make([]*EthRctTrie, 0, len(keys)) + for _, k := range keys { + n, err := rt.getNodeFromDB(k.dbKey) + if err != nil { + return nil, nil, err + } + out = append(out, n) + } + + return out, keys, nil +} + +func (rt *rctTrie) getNodeFromDB(key []byte) (*EthRctTrie, error) { + rawdata, err := rt.db.Get(key) + if err != nil { + return nil, err + } + tn := &TrieNode{ + cid: keccak256ToCid(MEthTxReceiptTrie, key), + rawdata: rawdata, + } + + return &EthRctTrie{TrieNode: tn}, nil +} diff --git a/statediff/indexer/ipld/eth_state.go b/statediff/indexer/ipld/eth_state.go new file mode 100644 index 000000000000..9a2c230e23a0 --- /dev/null +++ b/statediff/indexer/ipld/eth_state.go @@ -0,0 +1,126 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + "io" + "io/ioutil" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/rlp" +) + +// EthStateTrie (eth-state-trie, codec 0x96), represents +// a node from the satte trie in ethereum. +type EthStateTrie struct { + *TrieNode +} + +// Static (compile time) check that EthStateTrie satisfies the node.Node interface. +var _ node.Node = (*EthStateTrie)(nil) + +/* + INPUT +*/ + +// FromStateTrieRLPFile takes the RLP representation of an ethereum +// state trie node to return it as an IPLD node for further processing. +func FromStateTrieRLPFile(r io.Reader) (*EthStateTrie, error) { + raw, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + return FromStateTrieRLP(raw) +} + +// FromStateTrieRLP takes the RLP representation of an ethereum +// state trie node to return it as an IPLD node for further processing. +func FromStateTrieRLP(raw []byte) (*EthStateTrie, error) { + c, err := RawdataToCid(MEthStateTrie, raw, multihash.KECCAK_256) + if err != nil { + return nil, err + } + // Let's run the whole mile and process the nodeKind and + // its elements, in case somebody would need this function + // to parse an RLP element from the filesystem + return DecodeEthStateTrie(c, raw) +} + +/* + OUTPUT +*/ + +// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata. +func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf) + if err != nil { + return nil, err + } + return &EthStateTrie{TrieNode: tn}, nil +} + +// decodeEthStateTrieLeaf parses a eth-tx-trie leaf +// from decoded RLP elements +func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) { + var account EthAccount + err := rlp.DecodeBytes(i[1].([]byte), &account) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthAccountSnapshot, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthAccountSnapshot{ + EthAccount: &account, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the state trie node. +func (st *EthStateTrie) RawData() []byte { + return st.rawdata +} + +// Cid returns the cid of the state trie node. +func (st *EthStateTrie) Cid() cid.Cid { + return st.cid +} + +// String is a helper for output +func (st *EthStateTrie) String() string { + return fmt.Sprintf("", st.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (st *EthStateTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-state-trie", + } +} diff --git a/statediff/indexer/ipld/eth_state_test.go b/statediff/indexer/ipld/eth_state_test.go new file mode 100644 index 000000000000..20ff7767016b --- /dev/null +++ b/statediff/indexer/ipld/eth_state_test.go @@ -0,0 +1,326 @@ +package ipld + +import ( + "fmt" + "os" + "testing" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" +) + +/* + INPUT + OUTPUT +*/ + +func TestStateTrieNodeEvenExtensionParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + if output.nodeKind != "extension" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "extension", output.nodeKind) + } + + if len(output.elements) != 2 { + t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements)) + } + + if fmt.Sprintf("%x", output.elements[0]) != "0d08" { + t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "0d08", fmt.Sprintf("%x", output.elements[0])) + } + + if output.elements[1].(cid.Cid).String() != + "baglacgzalnzmhhnxudxtga6t3do2rctb6ycgyj6mjnycoamlnc733nnbkd6q" { + t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", "baglacgzalnzmhhnxudxtga6t3do2rctb6ycgyj6mjnycoamlnc733nnbkd6q", output.elements[1].(cid.Cid).String()) + } +} + +func TestStateTrieNodeOddExtensionParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-56864f") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + if output.nodeKind != "extension" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "extension", output.nodeKind) + } + + if len(output.elements) != 2 { + t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements)) + } + + if fmt.Sprintf("%x", output.elements[0]) != "02" { + t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "02", fmt.Sprintf("%x", output.elements[0])) + } + + if output.elements[1].(cid.Cid).String() != + "baglacgzaizf2czb7wztoox4lu23qkwkbfamqsdzcmejzr3rsszrvkaktpfeq" { + t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", "baglacgzaizf2czb7wztoox4lu23qkwkbfamqsdzcmejzr3rsszrvkaktpfeq", output.elements[1].(cid.Cid).String()) + } +} + +func TestStateTrieNodeEvenLeafParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-0e8b34") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + if output.nodeKind != "leaf" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "leaf", output.nodeKind) + } + + if len(output.elements) != 2 { + t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements)) + } + + // bd66f60e5b954e1af93ded1b02cb575ff0ed6d9241797eff7576b0bf0637 + if fmt.Sprintf("%x", output.elements[0].([]byte)[0:10]) != "0b0d06060f06000e050b" { + t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "0b0d06060f06000e050b", fmt.Sprintf("%x", output.elements[0].([]byte)[0:10])) + } + + if output.elements[1].(*EthAccountSnapshot).String() != + "" { + t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "", output.elements[1].(*EthAccountSnapshot).String()) + } +} + +func TestStateTrieNodeOddLeafParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-c9070d") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + if output.nodeKind != "leaf" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "leaf", output.nodeKind) + } + + if len(output.elements) != 2 { + t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements)) + } + + // 6c9db9bb545a03425e300f3ee72bae098110336dd3eaf48c20a2e5b6865fc + if fmt.Sprintf("%x", output.elements[0].([]byte)[0:10]) != "060c090d0b090b0b0504" { + t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "060c090d0b090b0b0504", fmt.Sprintf("%x", output.elements[0].([]byte)[0:10])) + } + + if output.elements[1].(*EthAccountSnapshot).String() != + "" { + t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "", output.elements[1].(*EthAccountSnapshot).String()) + } +} + +/* + Block INTERFACE +*/ +func TestStateTrieBlockElements(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-d7f897") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + if fmt.Sprintf("%x", output.RawData())[:10] != "f90211a090" { + t.Fatalf("Wrong Data\r\nexpected %s\r\ngot %s", "f90211a090", fmt.Sprintf("%x", output.RawData())[:10]) + } + + if output.Cid().String() != + "baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca" { + t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca", output.Cid().String()) + } +} + +func TestStateTrieString(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-d7f897") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + if output.String() != + "" { + t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "", output.String()) + } +} + +func TestStateTrieLoggable(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-d7f897") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + l := output.Loggable() + if _, ok := l["type"]; !ok { + t.Fatal("Loggable map expected the field 'type'") + } + + if l["type"] != "eth-state-trie" { + t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-state-trie", l["type"]) + } +} + +/* + TRIE NODE (Through EthStateTrie) + Node INTERFACE +*/ + +func TestTraverseStateTrieWithResolve(t *testing.T) { + var err error + + stMap := prepareStateTrieMap(t) + + // This is the cid of the root of the block 0 + // baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca + currentNode := stMap["baglacgza274jot5vvr4ntlajtonnkaml5xbm4cts3liye6qxbhndawapavca"] + + // This is the path we want to traverse + // The eth address is 0x5abfec25f74cd88437631a7731906932776356f9 + // Its keccak-256 is cdd3e25edec0a536a05f5e5ab90a5603624c0ed77453b2e8f955cf8b43d4d0fb + // We use the keccak-256(addr) to traverse the state trie in ethereum. + var traversePath []string + for _, s := range "cdd3e25edec0a536a05f5e5ab90a5603624c0ed77453b2e8f955cf8b43d4d0fb" { + traversePath = append(traversePath, string(s)) + } + traversePath = append(traversePath, "balance") + + var obj interface{} + for { + obj, traversePath, err = currentNode.Resolve(traversePath) + link, ok := obj.(*node.Link) + if !ok { + break + } + if err != nil { + t.Fatal("Error should be nil") + } + + currentNode = stMap[link.Cid.String()] + if currentNode == nil { + t.Fatal("state trie node not found in memory map") + } + } + + if fmt.Sprintf("%v", obj) != "11901484239480000000000000" { + t.Fatalf("Wrong balance value\r\nexpected %s\r\ngot %s", "11901484239480000000000000", fmt.Sprintf("%v", obj)) + } +} + +func TestStateTrieResolveLinks(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f") + checkError(err, t) + + stNode, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + // bad case + obj, rest, err := stNode.ResolveLink([]string{"supercalifragilist"}) + if obj != nil { + t.Fatalf("Expected obj to be nil") + } + if rest != nil { + t.Fatal("Expected rest to be nil") + } + if err.Error() != "invalid path element" { + t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "invalid path element", err.Error()) + } + + // good case + obj, rest, err = stNode.ResolveLink([]string{"d8"}) + if obj == nil { + t.Fatalf("Expected a not nil obj to be returned") + } + if rest != nil { + t.Fatal("Expected rest to be nil") + } + if err != nil { + t.Fatal("Expected error to be nil") + } +} + +func TestStateTrieCopy(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f") + checkError(err, t) + + stNode, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + defer func() { + r := recover() + if r == nil { + t.Fatal("Expected panic") + } + if r != "implement me" { + t.Fatalf("Wrong panic message\r\nexpected %s\r\ngot %s", "'implement me'", r) + } + }() + + _ = stNode.Copy() +} + +func TestStateTrieStat(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f") + checkError(err, t) + + stNode, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + obj, err := stNode.Stat() + if obj == nil { + t.Fatal("Expected a not null object node.NodeStat") + } + + if err != nil { + t.Fatal("Expected a nil error") + } +} + +func TestStateTrieSize(t *testing.T) { + fi, err := os.Open("test_data/eth-state-trie-rlp-eb2f5f") + checkError(err, t) + + stNode, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + size, err := stNode.Size() + if size != uint64(0) { + t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", 0, size) + } + + if err != nil { + t.Fatal("Expected a nil error") + } +} + +func prepareStateTrieMap(t *testing.T) map[string]*EthStateTrie { + filepaths := []string{ + "test_data/eth-state-trie-rlp-0e8b34", + "test_data/eth-state-trie-rlp-56864f", + "test_data/eth-state-trie-rlp-6fc2d7", + "test_data/eth-state-trie-rlp-727994", + "test_data/eth-state-trie-rlp-c9070d", + "test_data/eth-state-trie-rlp-d5be90", + "test_data/eth-state-trie-rlp-d7f897", + "test_data/eth-state-trie-rlp-eb2f5f", + } + + out := make(map[string]*EthStateTrie) + + for _, fp := range filepaths { + fi, err := os.Open(fp) + checkError(err, t) + + stateTrieNode, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + out[stateTrieNode.Cid().String()] = stateTrieNode + } + + return out +} diff --git a/statediff/indexer/ipld/eth_storage.go b/statediff/indexer/ipld/eth_storage.go new file mode 100644 index 000000000000..8b4d6234d45c --- /dev/null +++ b/statediff/indexer/ipld/eth_storage.go @@ -0,0 +1,112 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + "io" + "io/ioutil" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" +) + +// EthStorageTrie (eth-storage-trie, codec 0x98), represents +// a node from the storage trie in ethereum. +type EthStorageTrie struct { + *TrieNode +} + +// Static (compile time) check that EthStorageTrie satisfies the node.Node interface. +var _ node.Node = (*EthStorageTrie)(nil) + +/* + INPUT +*/ + +// FromStorageTrieRLPFile takes the RLP representation of an ethereum +// storage trie node to return it as an IPLD node for further processing. +func FromStorageTrieRLPFile(r io.Reader) (*EthStorageTrie, error) { + raw, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + return FromStorageTrieRLP(raw) +} + +// FromStorageTrieRLP takes the RLP representation of an ethereum +// storage trie node to return it as an IPLD node for further processing. +func FromStorageTrieRLP(raw []byte) (*EthStorageTrie, error) { + c, err := RawdataToCid(MEthStorageTrie, raw, multihash.KECCAK_256) + if err != nil { + return nil, err + } + + // Let's run the whole mile and process the nodeKind and + // its elements, in case somebody would need this function + // to parse an RLP element from the filesystem + return DecodeEthStorageTrie(c, raw) +} + +/* + OUTPUT +*/ + +// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata. +func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf) + if err != nil { + return nil, err + } + return &EthStorageTrie{TrieNode: tn}, nil +} + +// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf +// from decoded RLP elements +func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) { + return []interface{}{ + i[0].([]byte), + i[1].([]byte), + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the storage trie node. +func (st *EthStorageTrie) RawData() []byte { + return st.rawdata +} + +// Cid returns the cid of the storage trie node. +func (st *EthStorageTrie) Cid() cid.Cid { + return st.cid +} + +// String is a helper for output +func (st *EthStorageTrie) String() string { + return fmt.Sprintf("", st.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (st *EthStorageTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-storage-trie", + } +} diff --git a/statediff/indexer/ipld/eth_storage_test.go b/statediff/indexer/ipld/eth_storage_test.go new file mode 100644 index 000000000000..ac4b38691c21 --- /dev/null +++ b/statediff/indexer/ipld/eth_storage_test.go @@ -0,0 +1,140 @@ +package ipld + +import ( + "fmt" + "os" + "testing" + + "github.com/ipfs/go-cid" +) + +/* + INPUT + OUTPUT +*/ + +func TestStorageTrieNodeExtensionParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-storage-trie-rlp-113049") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + if output.nodeKind != "extension" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "extension", output.nodeKind) + } + + if len(output.elements) != 2 { + t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(output.elements)) + } + + if fmt.Sprintf("%x", output.elements[0]) != "0a" { + t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "0a", fmt.Sprintf("%x", output.elements[0])) + } + + if output.elements[1].(cid.Cid).String() != + "baglacgzautxeutufae7owyrezfvwpan2vusocmxgzwqhzrhjbwprp2texgsq" { + t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", "baglacgzautxeutufae7owyrezfvwpan2vusocmxgzwqhzrhjbwprp2texgsq", output.elements[1].(cid.Cid).String()) + } +} + +func TestStateTrieNodeLeafParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-storage-trie-rlp-ffbcad") + checkError(err, t) + + output, err := FromStorageTrieRLPFile(fi) + checkError(err, t) + + if output.nodeKind != "leaf" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "leaf", output.nodeKind) + } + + if len(output.elements) != 2 { + t.Fatalf("Wrong number of elements for an leaf node\r\nexpected %d\r\ngot %d", 2, len(output.elements)) + } + + // 2ee1ae9c502e48e0ed528b7b39ac569cef69d7844b5606841a7f3fe898a2 + if fmt.Sprintf("%x", output.elements[0].([]byte)[:10]) != "020e0e010a0e090c0500" { + t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "020e0e010a0e090c0500", fmt.Sprintf("%x", output.elements[0].([]byte)[:10])) + } + + if fmt.Sprintf("%x", output.elements[1]) != "89056c31f304b2530000" { + t.Fatalf("Wrong Value\r\nexpected %s\r\ngot %s", "89056c31f304b2530000", fmt.Sprintf("%x", output.elements[1])) + } +} + +func TestStateTrieNodeBranchParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-storage-trie-rlp-ffc25c") + checkError(err, t) + + output, err := FromStateTrieRLPFile(fi) + checkError(err, t) + + if output.nodeKind != "branch" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "branch", output.nodeKind) + } + + if len(output.elements) != 17 { + t.Fatalf("Wrong number of elements for an branch node\r\nexpected %d\r\ngot %d", 17, len(output.elements)) + } + + if fmt.Sprintf("%s", output.elements[4]) != + "baglacgzadqhbmlxrxtw5hplcq5jn74p4dceryzw664w3237ra52dnghbjpva" { + t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "baglacgzadqhbmlxrxtw5hplcq5jn74p4dceryzw664w3237ra52dnghbjpva", fmt.Sprintf("%s", output.elements[4])) + } + + if fmt.Sprintf("%s", output.elements[10]) != + "baglacgza77d37i2v6uhtzeeq4vngragjbgbwq3lylpoc3lihenvzimybzxmq" { + t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "baglacgza77d37i2v6uhtzeeq4vngragjbgbwq3lylpoc3lihenvzimybzxmq", fmt.Sprintf("%s", output.elements[10])) + } +} + +/* + Block INTERFACE +*/ +func TestStorageTrieBlockElements(t *testing.T) { + fi, err := os.Open("test_data/eth-storage-trie-rlp-ffbcad") + checkError(err, t) + + output, err := FromStorageTrieRLPFile(fi) + checkError(err, t) + + if fmt.Sprintf("%x", output.RawData())[:10] != "eb9f202ee1" { + t.Fatalf("Wrong Data\r\nexpected %s\r\ngot %s", "eb9f202ee1", fmt.Sprintf("%x", output.RawData())[:10]) + } + + if output.Cid().String() != + "bagmacgza766k3oprj2qxn36eycw55pogmu3dwtfay6zdh6ajrhvw3b2nqg5a" { + t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "bagmacgza766k3oprj2qxn36eycw55pogmu3dwtfay6zdh6ajrhvw3b2nqg5a", output.Cid().String()) + } +} + +func TestStorageTrieString(t *testing.T) { + fi, err := os.Open("test_data/eth-storage-trie-rlp-ffbcad") + checkError(err, t) + + output, err := FromStorageTrieRLPFile(fi) + checkError(err, t) + + if output.String() != + "" { + t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "", output.String()) + } +} + +func TestStorageTrieLoggable(t *testing.T) { + fi, err := os.Open("test_data/eth-storage-trie-rlp-ffbcad") + checkError(err, t) + + output, err := FromStorageTrieRLPFile(fi) + checkError(err, t) + + l := output.Loggable() + if _, ok := l["type"]; !ok { + t.Fatal("Loggable map expected the field 'type'") + } + + if l["type"] != "eth-storage-trie" { + t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-storage-trie", l["type"]) + } +} diff --git a/statediff/indexer/ipld/eth_tx.go b/statediff/indexer/ipld/eth_tx.go new file mode 100644 index 000000000000..99b1f9dbe2eb --- /dev/null +++ b/statediff/indexer/ipld/eth_tx.go @@ -0,0 +1,238 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" +) + +// EthTx (eth-tx codec 0x93) represents an ethereum transaction +type EthTx struct { + *types.Transaction + + cid cid.Cid + rawdata []byte +} + +// Static (compile time) check that EthTx satisfies the node.Node interface. +var _ node.Node = (*EthTx)(nil) + +/* + INPUT +*/ + +// NewEthTx converts a *types.Transaction to an EthTx IPLD node +func NewEthTx(tx *types.Transaction) (*EthTx, error) { + txRaw, err := tx.MarshalBinary() + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTx, txRaw, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthTx{ + Transaction: tx, + cid: c, + rawdata: txRaw, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthTx takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthTx(c cid.Cid, b []byte) (*EthTx, error) { + t := new(types.Transaction) + if err := t.UnmarshalBinary(b); err != nil { + return nil, err + } + return &EthTx{ + Transaction: t, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthTx) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthTx) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthTx) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthTx) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-tx", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (t *EthTx) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return t, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + case "type": + return t.Type(), nil, nil + case "gas": + return t.Gas(), nil, nil + case "gasPrice": + return t.GasPrice(), nil, nil + case "input": + return fmt.Sprintf("%x", t.Data()), nil, nil + case "nonce": + return t.Nonce(), nil, nil + case "r": + _, r, _ := t.RawSignatureValues() + return hexutil.EncodeBig(r), nil, nil + case "s": + _, _, s := t.RawSignatureValues() + return hexutil.EncodeBig(s), nil, nil + case "toAddress": + return t.To(), nil, nil + case "v": + v, _, _ := t.RawSignatureValues() + return hexutil.EncodeBig(v), nil, nil + case "value": + return hexutil.EncodeBig(t.Value()), nil, nil + default: + return nil, nil, ErrInvalidLink + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (t *EthTx) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"type", "gas", "gasPrice", "input", "nonce", "r", "s", "toAddress", "v", "value"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (t *EthTx) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := t.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the interface. +func (t *EthTx) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (t *EthTx) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (t *EthTx) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. It returns the byte size for the transaction +func (t *EthTx) Size() (uint64, error) { + spl := strings.Split(t.Transaction.Size().String(), " ") + size, units := spl[0], spl[1] + floatSize, err := strconv.ParseFloat(size, 64) + if err != nil { + return 0, err + } + var byteSize uint64 + switch units { + case "B": + byteSize = uint64(floatSize) + case "KB": + byteSize = uint64(floatSize * 1000) + case "MB": + byteSize = uint64(floatSize * 1000000) + case "GB": + byteSize = uint64(floatSize * 1000000000) + case "TB": + byteSize = uint64(floatSize * 1000000000000) + default: + return 0, fmt.Errorf("unreconginized units %s", units) + } + return byteSize, nil +} + +/* + EthTx functions +*/ + +// MarshalJSON processes the transaction into readable JSON format. +func (t *EthTx) MarshalJSON() ([]byte, error) { + v, r, s := t.RawSignatureValues() + + out := map[string]interface{}{ + "gas": t.Gas(), + "gasPrice": hexutil.EncodeBig(t.GasPrice()), + "input": fmt.Sprintf("%x", t.Data()), + "nonce": t.Nonce(), + "r": hexutil.EncodeBig(r), + "s": hexutil.EncodeBig(s), + "toAddress": t.To(), + "v": hexutil.EncodeBig(v), + "value": hexutil.EncodeBig(t.Value()), + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipld/eth_tx_test.go b/statediff/indexer/ipld/eth_tx_test.go new file mode 100644 index 000000000000..8b459621e587 --- /dev/null +++ b/statediff/indexer/ipld/eth_tx_test.go @@ -0,0 +1,411 @@ +package ipld + +import ( + "encoding/hex" + "fmt" + "os" + "strconv" + "strings" + "testing" + + block "github.com/ipfs/go-block-format" + "github.com/multiformats/go-multihash" +) + +/* + EthBlock + INPUT +*/ + +func TestTxInBlockBodyRlpParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-block-body-rlp-999999") + checkError(err, t) + + _, output, _, err := FromBlockRLP(fi) + checkError(err, t) + + if len(output) != 11 { + t.Fatalf("Wrong number of parsed txs\r\nexpected %d\r\ngot %d", 11, len(output)) + } + + // Oh, let's just grab the last element and one from the middle + testTx05Fields(output[5], t) + testTx10Fields(output[10], t) +} + +func TestTxInBlockHeaderRlpParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-block-header-rlp-999999") + checkError(err, t) + + _, output, _, err := FromBlockRLP(fi) + checkError(err, t) + + if len(output) != 0 { + t.Fatalf("Wrong number of txs\r\nexpected %d\r\ngot %d", 0, len(output)) + } +} + +func TestTxInBlockBodyJsonParsing(t *testing.T) { + fi, err := os.Open("test_data/eth-block-body-json-999999") + checkError(err, t) + + _, output, _, err := FromBlockJSON(fi) + checkError(err, t) + + if len(output) != 11 { + t.Fatalf("Wrong number of parsed txs\r\nexpected %d\r\ngot %d", 11, len(output)) + } + + testTx05Fields(output[5], t) + testTx10Fields(output[10], t) +} + +/* + OUTPUT +*/ + +func TestDecodeTransaction(t *testing.T) { + // Prepare the "fetched transaction". + // This one is supposed to be in the datastore already, + // and given away by github.com/ipfs/go-ipfs/merkledag + rawTransactionString := + "f86c34850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f25" + + "8512af0d4000801ba0e9a25c929c26d1a95232ba75aef419a91b470651eb77614695e16c" + + "5ba023e383a0679fb2fc0d0b0f3549967c0894ee7d947f07d238a83ef745bc3ced5143a4af36" + rawTransaction, err := hex.DecodeString(rawTransactionString) + checkError(err, t) + c, err := RawdataToCid(MEthTx, rawTransaction, multihash.KECCAK_256) + checkError(err, t) + + // Just to clarify: This `block` is an IPFS block + storedTransaction, err := block.NewBlockWithCid(rawTransaction, c) + checkError(err, t) + + // Now the proper test + ethTransaction, err := DecodeEthTx(storedTransaction.Cid(), storedTransaction.RawData()) + checkError(err, t) + + testTx05Fields(ethTransaction, t) +} + +/* + Block INTERFACE +*/ + +func TestEthTxLoggable(t *testing.T) { + txs := prepareParsedTxs(t) + + l := txs[0].Loggable() + if _, ok := l["type"]; !ok { + t.Fatal("Loggable map expected the field 'type'") + } + + if l["type"] != "eth-tx" { + t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-tx", l["type"]) + } +} + +/* + Node INTERFACE +*/ + +func TestEthTxResolve(t *testing.T) { + tx := prepareParsedTxs(t)[0] + + // Empty path + obj, rest, err := tx.Resolve([]string{}) + rtx, ok := obj.(*EthTx) + if !ok { + t.Fatal("Wrong type of returned object") + } + if rtx.Cid() != tx.Cid() { + t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", tx.Cid().String(), rtx.Cid().String()) + } + if rest != nil { + t.Fatal("est should be nil") + } + if err != nil { + t.Fatal("err should be nil") + } + + // len(p) > 1 + badCases := [][]string{ + {"two", "elements"}, + {"here", "three", "elements"}, + {"and", "here", "four", "elements"}, + } + + for _, bc := range badCases { + obj, rest, err = tx.Resolve(bc) + if obj != nil { + t.Fatal("obj should be nil") + } + if rest != nil { + t.Fatal("rest should be nil") + } + if err.Error() != fmt.Sprintf("unexpected path elements past %s", bc[0]) { + t.Fatalf("wrong error\r\nexpected %s\r\ngot %s", fmt.Sprintf("unexpected path elements past %s", bc[0]), err.Error()) + } + } + + moreBadCases := []string{ + "i", + "am", + "not", + "a", + "tx", + "field", + } + for _, mbc := range moreBadCases { + obj, rest, err = tx.Resolve([]string{mbc}) + if obj != nil { + t.Fatal("obj should be nil") + } + if rest != nil { + t.Fatal("rest should be nil") + } + + if err != ErrInvalidLink { + t.Fatalf("wrong error\r\nexpected %s\r\ngot %s", ErrInvalidLink, err) + } + } + + goodCases := []string{ + "gas", + "gasPrice", + "input", + "nonce", + "r", + "s", + "toAddress", + "v", + "value", + } + for _, gc := range goodCases { + _, _, err = tx.Resolve([]string{gc}) + if err != nil { + t.Fatalf("error should be nil %v", gc) + } + } +} + +func TestEthTxTree(t *testing.T) { + tx := prepareParsedTxs(t)[0] + _ = tx + + // Bad cases + tree := tx.Tree("non-empty-string", 0) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + tree = tx.Tree("non-empty-string", 1) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + tree = tx.Tree("", 0) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + // Good cases + tree = tx.Tree("", 1) + lookupElements := map[string]interface{}{ + "type": nil, + "gas": nil, + "gasPrice": nil, + "input": nil, + "nonce": nil, + "r": nil, + "s": nil, + "toAddress": nil, + "v": nil, + "value": nil, + } + + if len(tree) != len(lookupElements) { + t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree)) + } + + for _, te := range tree { + if _, ok := lookupElements[te]; !ok { + t.Fatalf("Unexpected Element: %v", te) + } + } +} + +func TestEthTxResolveLink(t *testing.T) { + tx := prepareParsedTxs(t)[0] + + // bad case + obj, rest, err := tx.ResolveLink([]string{"supercalifragilist"}) + if obj != nil { + t.Fatalf("Expected obj to be nil") + } + if rest != nil { + t.Fatal("Expected rest to be nil") + } + if err != ErrInvalidLink { + t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", ErrInvalidLink, err.Error()) + } + + // good case + obj, rest, err = tx.ResolveLink([]string{"nonce"}) + if obj != nil { + t.Fatalf("Expected obj to be nil") + } + if rest != nil { + t.Fatal("Expected rest to be nil") + } + if err.Error() != "resolved item was not a link" { + t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "resolved item was not a link", err.Error()) + } +} + +func TestEthTxCopy(t *testing.T) { + tx := prepareParsedTxs(t)[0] + + defer func() { + r := recover() + if r == nil { + t.Fatal("Expected panic") + } + if r != "implement me" { + t.Fatalf("Wrong panic message\r\nexpected %s\r\ngot %s", "'implement me'", r) + } + }() + + _ = tx.Copy() +} + +func TestEthTxLinks(t *testing.T) { + tx := prepareParsedTxs(t)[0] + + if tx.Links() != nil { + t.Fatal("Links() expected to return nil") + } +} + +func TestEthTxStat(t *testing.T) { + tx := prepareParsedTxs(t)[0] + + obj, err := tx.Stat() + if obj == nil { + t.Fatal("Expected a not null object node.NodeStat") + } + + if err != nil { + t.Fatal("Expected a nil error") + } +} + +func TestEthTxSize(t *testing.T) { + tx := prepareParsedTxs(t)[0] + + size, err := tx.Size() + checkError(err, t) + + spl := strings.Split(tx.Transaction.Size().String(), " ") + expectedSize, units := spl[0], spl[1] + floatSize, err := strconv.ParseFloat(expectedSize, 64) + checkError(err, t) + + var byteSize uint64 + switch units { + case "B": + byteSize = uint64(floatSize) + case "KB": + byteSize = uint64(floatSize * 1000) + case "MB": + byteSize = uint64(floatSize * 1000000) + case "GB": + byteSize = uint64(floatSize * 1000000000) + case "TB": + byteSize = uint64(floatSize * 1000000000000) + default: + t.Fatal("Unexpected size units") + } + if size != byteSize { + t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", byteSize, size) + } +} + +/* + AUXILIARS +*/ + +// prepareParsedTxs is a convenienve method +func prepareParsedTxs(t *testing.T) []*EthTx { + fi, err := os.Open("test_data/eth-block-body-rlp-999999") + checkError(err, t) + + _, output, _, err := FromBlockRLP(fi) + checkError(err, t) + + return output +} + +func testTx05Fields(ethTx *EthTx, t *testing.T) { + // Was the cid calculated? + if ethTx.Cid().String() != "bagjqcgzawhfnvdnpmpcfoug7d3tz53k2ht3cidr45pnw3y7snpd46azbpp2a" { + t.Fatalf("Wrong cid\r\nexpected %s\r\ngot %s\r\n", "bagjqcgzawhfnvdnpmpcfoug7d3tz53k2ht3cidr45pnw3y7snpd46azbpp2a", ethTx.Cid().String()) + } + + // Do we have the rawdata available? + if fmt.Sprintf("%x", ethTx.RawData()[:10]) != "f86c34850df847580082" { + t.Fatalf("Wrong Rawdata\r\nexpected %s\r\ngot %s", "f86c34850df847580082", fmt.Sprintf("%x", ethTx.RawData()[:10])) + } + + // Proper Fields of types.Transaction + if fmt.Sprintf("%x", ethTx.To()) != "32be343b94f860124dc4fee278fdcbd38c102d88" { + t.Fatalf("Wrong Recipient\r\nexpected %s\r\ngot %s", "32be343b94f860124dc4fee278fdcbd38c102d88", fmt.Sprintf("%x", ethTx.To())) + } + if len(ethTx.Data()) != 0 { + t.Fatalf("Wrong len of Data\r\nexpected %d\r\ngot %d", 0, len(ethTx.Data())) + } + if fmt.Sprintf("%v", ethTx.Gas()) != "21000" { + t.Fatalf("Wrong Gas\r\nexpected %s\r\ngot %s", "21000", fmt.Sprintf("%v", ethTx.Gas())) + } + if fmt.Sprintf("%v", ethTx.Value()) != "1091424800000000000" { + t.Fatalf("Wrong Value\r\nexpected %s\r\ngot %s", "1091424800000000000", fmt.Sprintf("%v", ethTx.Value())) + } + if fmt.Sprintf("%v", ethTx.Nonce()) != "52" { + t.Fatalf("Wrong Nonce\r\nexpected %s\r\ngot %s", "52", fmt.Sprintf("%v", ethTx.Nonce())) + } + if fmt.Sprintf("%v", ethTx.GasPrice()) != "60000000000" { + t.Fatalf("Wrong Gas Price\r\nexpected %s\r\ngot %s", "60000000000", fmt.Sprintf("%v", ethTx.GasPrice())) + } +} + +func testTx10Fields(ethTx *EthTx, t *testing.T) { + // Was the cid calculated? + if ethTx.Cid().String() != "bagjqcgzaykakwayoec6j55zmq62cbvmplgf5u5j67affge3ksi4ermgitjoa" { + t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "bagjqcgzaykakwayoec6j55zmq62cbvmplgf5u5j67affge3ksi4ermgitjoa", ethTx.Cid().String()) + } + + // Do we have the rawdata available? + if fmt.Sprintf("%x", ethTx.RawData()[:10]) != "f8708302a120850ba43b" { + t.Fatalf("Wrong Rawdata\r\nexpected %s\r\ngot %s", "f8708302a120850ba43b", fmt.Sprintf("%x", ethTx.RawData()[:10])) + } + + // Proper Fields of types.Transaction + if fmt.Sprintf("%x", ethTx.To()) != "1c51bf013add0857c5d9cf2f71a7f15ca93d4816" { + t.Fatalf("Wrong Recipient\r\nexpected %s\r\ngot %s", "1c51bf013add0857c5d9cf2f71a7f15ca93d4816", fmt.Sprintf("%x", ethTx.To())) + } + if len(ethTx.Data()) != 0 { + t.Fatalf("Wrong len of Data\r\nexpected %d\r\ngot %d", 0, len(ethTx.Data())) + } + if fmt.Sprintf("%v", ethTx.Gas()) != "90000" { + t.Fatalf("Wrong Gas\r\nexpected %s\r\ngot %s", "90000", fmt.Sprintf("%v", ethTx.Gas())) + } + if fmt.Sprintf("%v", ethTx.Value()) != "1049756850000000000" { + t.Fatalf("Wrong Value\r\nexpected %s\r\ngot %s", "1049756850000000000", fmt.Sprintf("%v", ethTx.Value())) + } + if fmt.Sprintf("%v", ethTx.Nonce()) != "172320" { + t.Fatalf("Wrong Nonce\r\nexpected %s\r\ngot %s", "172320", fmt.Sprintf("%v", ethTx.Nonce())) + } + if fmt.Sprintf("%v", ethTx.GasPrice()) != "50000000000" { + t.Fatalf("Wrong Gas Price\r\nexpected %s\r\ngot %s", "50000000000", fmt.Sprintf("%v", ethTx.GasPrice())) + } +} diff --git a/statediff/indexer/ipld/eth_tx_trie.go b/statediff/indexer/ipld/eth_tx_trie.go new file mode 100644 index 000000000000..bb4f66df06bb --- /dev/null +++ b/statediff/indexer/ipld/eth_tx_trie.go @@ -0,0 +1,146 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" +) + +// EthTxTrie (eth-tx-trie codec 0x92) represents +// a node from the transaction trie in ethereum. +type EthTxTrie struct { + *TrieNode +} + +// Static (compile time) check that EthTxTrie satisfies the node.Node interface. +var _ node.Node = (*EthTxTrie)(nil) + +/* + INPUT +*/ + +// To create a proper trie of the eth-tx-trie objects, it is required +// to input all transactions belonging to a forest in a single step. +// We are adding the transactions, and creating its trie on +// block body parsing time. + +/* + OUTPUT +*/ + +// DecodeEthTxTrie returns an EthTxTrie object from its cid and rawdata. +func DecodeEthTxTrie(c cid.Cid, b []byte) (*EthTxTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthTxTrieLeaf) + if err != nil { + return nil, err + } + return &EthTxTrie{TrieNode: tn}, nil +} + +// decodeEthTxTrieLeaf parses a eth-tx-trie leaf +//from decoded RLP elements +func decodeEthTxTrieLeaf(i []interface{}) ([]interface{}, error) { + t := new(types.Transaction) + if err := t.UnmarshalBinary(i[1].([]byte)); err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTx, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthTx{ + Transaction: t, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthTxTrie) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthTxTrie) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthTxTrie) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthTxTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-tx-trie", + } +} + +/* + EthTxTrie functions +*/ + +// txTrie wraps a localTrie for use on the transaction trie. +type txTrie struct { + *localTrie +} + +// newTxTrie initializes and returns a txTrie. +func newTxTrie() *txTrie { + return &txTrie{ + localTrie: newLocalTrie(), + } +} + +// getNodes invokes the localTrie, which computes the root hash of the +// transaction trie and returns its sql keys, to return a slice +// of EthTxTrie nodes. +func (tt *txTrie) getNodes() ([]*EthTxTrie, error) { + keys, err := tt.getKeys() + if err != nil { + return nil, err + } + var out []*EthTxTrie + + for _, k := range keys { + rawdata, err := tt.db.Get(k) + if err != nil { + return nil, err + } + tn := &TrieNode{ + cid: keccak256ToCid(MEthTxTrie, k), + rawdata: rawdata, + } + out = append(out, &EthTxTrie{TrieNode: tn}) + } + + return out, nil +} diff --git a/statediff/indexer/ipld/eth_tx_trie_test.go b/statediff/indexer/ipld/eth_tx_trie_test.go new file mode 100644 index 000000000000..b067d0ea4057 --- /dev/null +++ b/statediff/indexer/ipld/eth_tx_trie_test.go @@ -0,0 +1,503 @@ +package ipld + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "os" + "testing" + + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" +) + +/* + EthBlock +*/ + +func TestTxTriesInBlockBodyJSONParsing(t *testing.T) { + // HINT: 306 txs + // cat test_data/eth-block-body-json-4139497 | jsontool | grep transactionIndex | wc -l + // or, https://etherscan.io/block/4139497 + fi, err := os.Open("test_data/eth-block-body-json-4139497") + checkError(err, t) + + _, _, output, err := FromBlockJSON(fi) + checkError(err, t) + if len(output) != 331 { + t.Fatalf("Wrong number of obtained tx trie nodes\r\nexpected %d\r\n got %d", 331, len(output)) + } +} + +/* + OUTPUT +*/ + +func TestTxTrieDecodeExtension(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieExtension(t) + + if ethTxTrie.nodeKind != "extension" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "extension", ethTxTrie.nodeKind) + } + + if len(ethTxTrie.elements) != 2 { + t.Fatalf("Wrong number of elements for an extension node\r\nexpected %d\r\ngot %d", 2, len(ethTxTrie.elements)) + } + + if fmt.Sprintf("%x", ethTxTrie.elements[0].([]byte)) != "0001" { + t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "0001", fmt.Sprintf("%x", ethTxTrie.elements[0].([]byte))) + } + + if ethTxTrie.elements[1].(cid.Cid).String() != + "bagjacgzak6wdjvshdtb7lrvlteweyd7f5qjr3dmzmh7g2xpi4xrwoujsio2a" { + t.Fatalf("Wrong CID\r\nexpected %s\r\ngot %s", "bagjacgzak6wdjvshdtb7lrvlteweyd7f5qjr3dmzmh7g2xpi4xrwoujsio2a", ethTxTrie.elements[1].(cid.Cid).String()) + } +} + +func TestTxTrieDecodeLeaf(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieLeaf(t) + + if ethTxTrie.nodeKind != "leaf" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "leaf", ethTxTrie.nodeKind) + } + + if len(ethTxTrie.elements) != 2 { + t.Fatalf("Wrong number of elements for a leaf node\r\nexpected %d\r\ngot %d", 2, len(ethTxTrie.elements)) + } + + if fmt.Sprintf("%x", ethTxTrie.elements[0].([]byte)) != "" { + t.Fatalf("Wrong key\r\nexpected %s\r\ngot %s", "", fmt.Sprintf("%x", ethTxTrie.elements[0].([]byte))) + } + + if _, ok := ethTxTrie.elements[1].(*EthTx); !ok { + t.Fatal("Expected element to be an EthTx") + } + + if ethTxTrie.elements[1].(*EthTx).String() != + "" { + t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "", ethTxTrie.elements[1].(*EthTx).String()) + } +} + +func TestTxTrieDecodeBranch(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieBranch(t) + + if ethTxTrie.nodeKind != "branch" { + t.Fatalf("Wrong nodeKind\r\nexpected %s\r\ngot %s", "branch", ethTxTrie.nodeKind) + } + + if len(ethTxTrie.elements) != 17 { + t.Fatalf("Wrong number of elements for a branch node\r\nexpected %d\r\ngot %d", 17, len(ethTxTrie.elements)) + } + + for i, element := range ethTxTrie.elements { + switch { + case i < 9: + if _, ok := element.(cid.Cid); !ok { + t.Fatal("Expected element to be a cid") + } + continue + default: + if element != nil { + t.Fatal("Expected element to be a nil") + } + } + } +} + +/* + Block INTERFACE +*/ + +func TestEthTxTrieBlockElements(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieExtension(t) + + if fmt.Sprintf("%x", ethTxTrie.RawData())[:10] != "e4820001a0" { + t.Fatalf("Wrong Data\r\nexpected %s\r\ngot %s", "e4820001a0", fmt.Sprintf("%x", ethTxTrie.RawData())[:10]) + } + + if ethTxTrie.Cid().String() != + "bagjacgzaw6ccgrfc3qnrl6joodbjjiet4haufnt2xww725luwgfhijnmg36q" { + t.Fatalf("Wrong Cid\r\nexpected %s\r\ngot %s", "bagjacgzaw6ccgrfc3qnrl6joodbjjiet4haufnt2xww725luwgfhijnmg36q", ethTxTrie.Cid().String()) + } +} + +func TestEthTxTrieString(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieExtension(t) + + if ethTxTrie.String() != "" { + t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "", ethTxTrie.String()) + } +} + +func TestEthTxTrieLoggable(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieExtension(t) + l := ethTxTrie.Loggable() + if _, ok := l["type"]; !ok { + t.Fatal("Loggable map expected the field 'type'") + } + + if l["type"] != "eth-tx-trie" { + t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-tx-trie", l["type"]) + } +} + +/* + Node INTERFACE +*/ + +func TestTxTrieResolveExtension(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieExtension(t) + + _ = ethTxTrie +} + +func TestTxTrieResolveLeaf(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieLeaf(t) + + _ = ethTxTrie +} + +func TestTxTrieResolveBranch(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieBranch(t) + + indexes := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"} + + for j, index := range indexes { + obj, rest, err := ethTxTrie.Resolve([]string{index, "nonce"}) + + switch { + case j < 9: + _, ok := obj.(*node.Link) + if !ok { + t.Fatalf("Returned object is not a link (index: %d)", j) + } + + if rest[0] != "nonce" { + t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", "nonce", rest[0]) + } + + if err != nil { + t.Fatal("Error should be nil") + } + + default: + if obj != nil { + t.Fatalf("Returned object should have been nil") + } + + if rest != nil { + t.Fatalf("Rest of the path returned should be nil") + } + + if err.Error() != "no such link in this branch" { + t.Fatalf("Wrong error") + } + } + } + + otherSuccessCases := [][]string{ + {"0", "1", "banana"}, + {"1", "banana"}, + {"7bc", "def"}, + {"bc", "def"}, + } + + for i := 0; i < len(otherSuccessCases); i = i + 2 { + osc := otherSuccessCases[i] + expectedRest := otherSuccessCases[i+1] + + obj, rest, err := ethTxTrie.Resolve(osc) + _, ok := obj.(*node.Link) + if !ok { + t.Fatalf("Returned object is not a link") + } + + for j := range expectedRest { + if rest[j] != expectedRest[j] { + t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", expectedRest[j], rest[j]) + } + } + + if err != nil { + t.Fatal("Error should be nil") + } + } +} + +func TestTraverseTxTrieWithResolve(t *testing.T) { + var err error + + txMap := prepareTxTrieMap(t) + + // This is the cid of the tx root at the block 4,139,497 + currentNode := txMap["bagjacgzaqolvvlyflkdiylijcu4ts6myxczkb2y3ewxmln5oyrsrkfc4v7ua"] + + // This is the path we want to traverse + // the transaction id 256, which is RLP encoded to 820100 + var traversePath []string + for _, s := range "820100" { + traversePath = append(traversePath, string(s)) + } + traversePath = append(traversePath, "value") + + var obj interface{} + for { + obj, traversePath, err = currentNode.Resolve(traversePath) + link, ok := obj.(*node.Link) + if !ok { + break + } + if err != nil { + t.Fatal("Error should be nil") + } + + currentNode = txMap[link.Cid.String()] + if currentNode == nil { + t.Fatal("transaction trie node not found in memory map") + } + } + + if fmt.Sprintf("%v", obj) != "0xc495a958603400" { + t.Fatalf("Wrong value\r\nexpected %s\r\ngot %s", "0xc495a958603400", fmt.Sprintf("%v", obj)) + } +} + +func TestTxTrieTreeBadParams(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieBranch(t) + + tree := ethTxTrie.Tree("non-empty-string", 0) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + tree = ethTxTrie.Tree("non-empty-string", 1) + if tree != nil { + t.Fatal("Expected nil to be returned") + } + + tree = ethTxTrie.Tree("", 0) + if tree != nil { + t.Fatal("Expected nil to be returned") + } +} + +func TestTxTrieTreeExtension(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieExtension(t) + + tree := ethTxTrie.Tree("", -1) + + if len(tree) != 1 { + t.Fatalf("An extension should have one element") + } + + if tree[0] != "01" { + t.Fatalf("Wrong trie element\r\nexpected %s\r\ngot %s", "01", tree[0]) + } +} + +func TestTxTrieTreeBranch(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieBranch(t) + + tree := ethTxTrie.Tree("", -1) + + lookupElements := map[string]interface{}{ + "0": nil, + "1": nil, + "2": nil, + "3": nil, + "4": nil, + "5": nil, + "6": nil, + "7": nil, + "8": nil, + } + + if len(tree) != len(lookupElements) { + t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree)) + } + + for _, te := range tree { + if _, ok := lookupElements[te]; !ok { + t.Fatalf("Unexpected Element: %v", te) + } + } +} + +func TestTxTrieLinksBranch(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieBranch(t) + + desiredValues := []string{ + "bagjacgzakhtcfpja453ydiaqxgidqmxhh7jwmxujib663deebwfs3m2n3hoa", + "bagjacgza2p2fuqh4vumknq6x5w7i47usvtu5ixqins6qjjtcks4zge3vx3qq", + "bagjacgza4fkhn7et3ra66yjkzbtvbxjefuketda6jctlut6it7gfahxhywga", + "bagjacgzacnryeybs52xryrka5uxi4eg4hi2mh66esaghu7cetzu6fsukrynq", + "bagjacgzastu5tc7lwz4ap3gznjwkyyepswquub7gvhags5mgdyfynnwbi43a", + "bagjacgza5qgp76ovvorkydni2lchew6ieu5wb55w6hdliiu6vft7zlxtdhjq", + "bagjacgzafnssc4yvln6zxmks5roskw4ckngta5n4yfy2skhlu435ve4b575a", + "bagjacgzagkuei7qxfxefufme2d3xizxokkq4ad3rzl2x4dq2uao6dcr4va2a", + "bagjacgzaxpaehtananrdxjghwukh2wwkkzcqwveppf6xclkrtd26rm27kqwq", + } + + links := ethTxTrie.Links() + + for i, v := range desiredValues { + if links[i].Cid.String() != v { + t.Fatalf("Wrong cid for link %d\r\nexpected %s\r\ngot %s", i, v, links[i].Cid.String()) + } + } +} + +/* + EthTxTrie Functions +*/ + +func TestTxTrieJSONMarshalExtension(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieExtension(t) + + jsonOutput, err := ethTxTrie.MarshalJSON() + checkError(err, t) + + var data map[string]interface{} + err = json.Unmarshal(jsonOutput, &data) + checkError(err, t) + + if parseMapElement(data["01"]) != + "bagjacgzak6wdjvshdtb7lrvlteweyd7f5qjr3dmzmh7g2xpi4xrwoujsio2a" { + t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "bagjacgzak6wdjvshdtb7lrvlteweyd7f5qjr3dmzmh7g2xpi4xrwoujsio2a", parseMapElement(data["01"])) + } + + if data["type"] != "extension" { + t.Fatalf("Wrong node type\r\nexpected %s\r\ngot %s", "extension", data["type"]) + } +} + +func TestTxTrieJSONMarshalLeaf(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieLeaf(t) + + jsonOutput, err := ethTxTrie.MarshalJSON() + checkError(err, t) + + var data map[string]interface{} + err = json.Unmarshal(jsonOutput, &data) + checkError(err, t) + + if data["type"] != "leaf" { + t.Fatalf("Wrong node type\r\nexpected %s\r\ngot %s", "leaf", data["type"]) + } + + if fmt.Sprintf("%v", data[""].(map[string]interface{})["nonce"]) != + "40243" { + t.Fatalf("Wrong nonce value\r\nexepcted %s\r\ngot %s", "40243", fmt.Sprintf("%v", data[""].(map[string]interface{})["nonce"])) + } +} + +func TestTxTrieJSONMarshalBranch(t *testing.T) { + ethTxTrie := prepareDecodedEthTxTrieBranch(t) + + jsonOutput, err := ethTxTrie.MarshalJSON() + checkError(err, t) + + var data map[string]interface{} + err = json.Unmarshal(jsonOutput, &data) + checkError(err, t) + + desiredValues := map[string]string{ + "0": "bagjacgzakhtcfpja453ydiaqxgidqmxhh7jwmxujib663deebwfs3m2n3hoa", + "1": "bagjacgza2p2fuqh4vumknq6x5w7i47usvtu5ixqins6qjjtcks4zge3vx3qq", + "2": "bagjacgza4fkhn7et3ra66yjkzbtvbxjefuketda6jctlut6it7gfahxhywga", + "3": "bagjacgzacnryeybs52xryrka5uxi4eg4hi2mh66esaghu7cetzu6fsukrynq", + "4": "bagjacgzastu5tc7lwz4ap3gznjwkyyepswquub7gvhags5mgdyfynnwbi43a", + "5": "bagjacgza5qgp76ovvorkydni2lchew6ieu5wb55w6hdliiu6vft7zlxtdhjq", + "6": "bagjacgzafnssc4yvln6zxmks5roskw4ckngta5n4yfy2skhlu435ve4b575a", + "7": "bagjacgzagkuei7qxfxefufme2d3xizxokkq4ad3rzl2x4dq2uao6dcr4va2a", + "8": "bagjacgzaxpaehtananrdxjghwukh2wwkkzcqwveppf6xclkrtd26rm27kqwq", + } + + for k, v := range desiredValues { + if parseMapElement(data[k]) != v { + t.Fatalf("Wrong Marshaled Value %s\r\nexpected %s\r\ngot %s", k, v, parseMapElement(data[k])) + } + } + + for _, v := range []string{"a", "b", "c", "d", "e", "f"} { + if data[v] != nil { + t.Fatal("Expected value to be nil") + } + } + + if data["type"] != "branch" { + t.Fatalf("Wrong node type\r\nexpected %s\r\ngot %s", "branch", data["type"]) + } +} + +/* + AUXILIARS +*/ + +// prepareDecodedEthTxTrie simulates an IPLD block available in the datastore, +// checks the source RLP and tests for the absence of errors during the decoding fase. +func prepareDecodedEthTxTrie(branchDataRLP string, t *testing.T) *EthTxTrie { + b, err := hex.DecodeString(branchDataRLP) + checkError(err, t) + + c, err := RawdataToCid(MEthTxTrie, b, multihash.KECCAK_256) + checkError(err, t) + + storedEthTxTrie, err := block.NewBlockWithCid(b, c) + checkError(err, t) + + ethTxTrie, err := DecodeEthTxTrie(storedEthTxTrie.Cid(), storedEthTxTrie.RawData()) + checkError(err, t) + + return ethTxTrie +} + +func prepareDecodedEthTxTrieExtension(t *testing.T) *EthTxTrie { + extensionDataRLP := + "e4820001a057ac34d6471cc3f5c6ab992c4c0fe5ec131d8d9961fe6d5de8e5e367513243b4" + return prepareDecodedEthTxTrie(extensionDataRLP, t) +} + +func prepareDecodedEthTxTrieLeaf(t *testing.T) *EthTxTrie { + leafDataRLP := + "f87220b86ff86d829d3384ee6b280083015f9094e0e6c781b8cba08bc840" + + "7eac0101b668d1fa6f4987c495a9586034008026a0981b6223c9d3c31971" + + "6da3cf057da84acf0fef897f4003d8a362d7bda42247dba066be134c4bc4" + + "32125209b5056ef274b7423bcac7cc398cf60b83aaff7b95469f" + return prepareDecodedEthTxTrie(leafDataRLP, t) +} + +func prepareDecodedEthTxTrieBranch(t *testing.T) *EthTxTrie { + branchDataRLP := + "f90131a051e622bd20e77781a010b9903832e73fd3665e89407ded8c840d8b2db34dd9" + + "dca0d3f45a40fcad18a6c3d7edbe8e7e92ace9d45e086cbd04a66254b9931375bee1a0" + + "e15476fc93dc41ef612ac86750dd242d14498c1e48a6ba4fc89fcc501ee7c58ca01363" + + "826032eeaf1c4540ed2e8e10dc3a34c3fbc4900c7a7c449e69e2ca8a8e1ba094e9d98b" + + "ebb67807ecd96a6cac608f95a14a07e6a9c06975861e0b86b6c14736a0ec0cfff9d5ab" + + "a2ac0da8d2c4725bc8253b60f7b6f1c6b4229ea967fcaef319d3a02b652173155b7d9b" + + "b152ec5d255b82534d3075bcc171a928eba737da9381effaa032a8447e172dc85a1584" + + "d0f77466ee52a1c00f71caf57e0e1aa01de18a3ca834a0bbc043cc0d03623ba4c7b514" + + "7d5aca56450b548f797d712d5198f5e8b35f542d8080808080808080" + return prepareDecodedEthTxTrie(branchDataRLP, t) +} + +func prepareTxTrieMap(t *testing.T) map[string]*EthTxTrie { + fi, err := os.Open("test_data/eth-block-body-json-4139497") + checkError(err, t) + + _, _, txTrieNodes, err := FromBlockJSON(fi) + checkError(err, t) + + out := make(map[string]*EthTxTrie) + + for _, txTrieNode := range txTrieNodes { + decodedNode, err := DecodeEthTxTrie(txTrieNode.Cid(), txTrieNode.RawData()) + checkError(err, t) + out[txTrieNode.Cid().String()] = decodedNode + } + + return out +} diff --git a/statediff/indexer/ipld/shared.go b/statediff/indexer/ipld/shared.go new file mode 100644 index 000000000000..258f573be631 --- /dev/null +++ b/statediff/indexer/ipld/shared.go @@ -0,0 +1,212 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "bytes" + "errors" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" +) + +// IPLD Codecs for Ethereum +// See the authoritative document: +// https://github.com/multiformats/multicodec/blob/master/table.csv +const ( + RawBinary = 0x55 + MEthHeader = 0x90 + MEthHeaderList = 0x91 + MEthTxTrie = 0x92 + MEthTx = 0x93 + MEthTxReceiptTrie = 0x94 + MEthTxReceipt = 0x95 + MEthStateTrie = 0x96 + MEthAccountSnapshot = 0x97 + MEthStorageTrie = 0x98 + MEthLogTrie = 0x99 + MEthLog = 0x9a +) + +var ( + nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") + ErrInvalidLink = errors.New("no such link") +) + +// RawdataToCid takes the desired codec and a slice of bytes +// and returns the proper cid of the object. +func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) { + c, err := cid.Prefix{ + Codec: codec, + Version: 1, + MhType: multiHash, + MhLength: -1, + }.Sum(rawdata) + if err != nil { + return cid.Cid{}, err + } + return c, nil +} + +// keccak256ToCid takes a keccak256 hash and returns its cid based on +// the codec given. +func keccak256ToCid(codec uint64, h []byte) cid.Cid { + buf, err := mh.Encode(h, mh.KECCAK_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, mh.Multihash(buf)) +} + +// commonHashToCid takes a go-ethereum common.Hash and returns its +// cid based on the codec given, +func commonHashToCid(codec uint64, h common.Hash) cid.Cid { + mhash, err := mh.Encode(h[:], mh.KECCAK_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, mhash) +} + +// localTrie wraps a go-ethereum trie and its underlying memory db. +// It contributes to the creation of the trie node objects. +type localTrie struct { + db ethdb.Database + trieDB *trie.Database + trie *trie.Trie +} + +// newLocalTrie initializes and returns a localTrie object +func newLocalTrie() *localTrie { + var err error + lt := &localTrie{} + lt.db = rawdb.NewMemoryDatabase() + lt.trieDB = trie.NewDatabase(lt.db) + lt.trie, err = trie.New(common.Hash{}, common.Hash{}, lt.trieDB) + if err != nil { + panic(err) + } + return lt +} + +// Add receives the index of an object and its rawdata value +// and includes it into the localTrie +func (lt *localTrie) Add(idx int, rawdata []byte) error { + key, err := rlp.EncodeToBytes(uint(idx)) + if err != nil { + panic(err) + } + return lt.trie.TryUpdate(key, rawdata) +} + +// rootHash returns the computed trie root. +// Useful for sanity checks on parsed data. +func (lt *localTrie) rootHash() []byte { + return lt.trie.Hash().Bytes() +} + +func (lt *localTrie) commit() error { + // commit trie nodes to trieDB + ltHash, trieNodes, err := lt.trie.Commit(true) + if err != nil { + return err + } + //new trie.Commit method signature also requires Update with returned NodeSet + lt.trieDB.Update(trie.NewWithNodeSet(trieNodes)) + + // commit trieDB to the underlying ethdb.Database + if err := lt.trieDB.Commit(ltHash, false, nil); err != nil { + return err + } + return nil +} + +// getKeys returns the stored keys of the memory sql +// of the localTrie for further processing. +func (lt *localTrie) getKeys() ([][]byte, error) { + if err := lt.commit(); err != nil { + return nil, err + } + + // collect all of the node keys + it := lt.trie.NodeIterator([]byte{}) + keyBytes := make([][]byte, 0) + for it.Next(true) { + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + keyBytes = append(keyBytes, it.Hash().Bytes()) + } + return keyBytes, nil +} + +type nodeKey struct { + dbKey []byte + TrieKey []byte +} + +// getLeafKeys returns the stored leaf keys from the memory sql +// of the localTrie for further processing. +func (lt *localTrie) getLeafKeys() ([]*nodeKey, error) { + if err := lt.commit(); err != nil { + return nil, err + } + + it := lt.trie.NodeIterator([]byte{}) + leafKeys := make([]*nodeKey, 0) + for it.Next(true) { + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + + node, nodeElements, err := sdtrie.ResolveNode(it, lt.trieDB) + if err != nil { + return nil, err + } + + if node.NodeType != sdtypes.Leaf { + continue + } + + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + + leafKeys = append(leafKeys, &nodeKey{dbKey: it.Hash().Bytes(), TrieKey: leafKey}) + } + return leafKeys, nil +} + +// getRLP encodes the given object to RLP returning its bytes. +func getRLP(object interface{}) []byte { + buf := new(bytes.Buffer) + if err := rlp.Encode(buf, object); err != nil { + panic(err) + } + + return buf.Bytes() +} diff --git a/statediff/indexer/ipld/test_data/error-tx-eth-block-body-json-999999 b/statediff/indexer/ipld/test_data/error-tx-eth-block-body-json-999999 new file mode 100644 index 000000000000..8654b53a9f85 --- /dev/null +++ b/statediff/indexer/ipld/test_data/error-tx-eth-block-body-json-999999 @@ -0,0 +1 @@ +{"jsonrpc":"2.0","result":{"author":"0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5","difficulty":"0xb6b4beb1e8e","extraData":"0xd783010303844765746887676f312e342e32856c696e7578","gasLimit":"0x2fefd8","gasUsed":"0x38658","hash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5","mixHash":"0x5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0","nonce":"0xf491f46b60fe04b3","number":"0xf423f","parentHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","receiptsRoot":"0x7fa0f6ca2a01823208d80801edad37e3e3a003b55c89319b45eb1f97862ad229","sealFields":["0xa05b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0","0x88f491f46b60fe04b3"],"sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":"0x6e8","stateRoot":"0xed98aa4b5b19c82fb35364f08508ae0a6dec665fa57663dca94c5d70554cde10","timestamp":"0x56bfb405","totalDifficulty":"0x6305496c80ab5c3f","transactions":[{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0xc3665b8a9224ba8da9a20322f31d599cafa52c5c","gas":"0x5209","gasPrice":"0xdf8475800","hash":"0x22879e0bc9602fef59dc0602f9bc385f12632da5cb4eee4b813a0c27159c4d24","input":"0x","networkId":null,"nonce":"0x1d3","publicKey":"0xc3dbee74f1b2b8dbedc417244b7f5a134c6f7769faf9ffe784b3f0fdda7ca52cf914d3f2b3164c009bf939796b77f047ccb4cc113d3bde5b06555b781e0c7149","r":"0x43531017f1569ec692c0bf1ad710ddb5158b60505ea33fb7a21245738539e2d5","raw":"0xf86e8201d3850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d8888102363ac310a4000801ca043531017f1569ec692c0bf1ad710ddb5158b60505ea33fb7a21245738539e2d5a03856c6a1117ff71e9b769ccb6960674038a3326c3dd84c152fc83ada28145a07","s":"0x3856c6a1117ff71e9b769ccb6960674038a3326c3dd84c152fc83ada28145a07","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x0","v":"0x1c","value":"0x102363ac310a4000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x4ce758b0c8aa655b77c14f16bd0190b5715be75a","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x3c634bf5f09f6b5b5ea377df7abb483f422ae5d4ba389c395f14f833de25d362","input":"0x","networkId":null,"nonce":"0x9","publicKey":"0x75022ee25c702fc6a53853843e00e87877e737f9c631a9d831c11693d7e31877a1b09755ab3a5c112decf57339839364b8b9a3c23ada01761b1e3a044e297316","r":"0x8219a4f30cb8dd7d5e1163ac433f207b599d804b0d74ee54c8694014db647700","raw":"0xf86c09850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880ed350879ce50000801ba08219a4f30cb8dd7d5e1163ac433f207b599d804b0d74ee54c8694014db647700a03db2e806986a746d44d675fdbbd7594bb2856946ba257209abfffdd1628141af","s":"0x3db2e806986a746d44d675fdbbd7594bb2856946ba257209abfffdd1628141af","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x1","v":"0x1b","value":"0xed350879ce50000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x30906581413d556de1a018adbe6cc63c88d58512","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x59feccaad599e776cd6635e68b5e19254cca3b38e49437044f1e1d15d00b0576","input":"0x","networkId":null,"nonce":"0x59","publicKey":"0xccf6be26c1eb1c89d5fe958db0112a46e3ac23a95ac0f709ce84a49ae3f20bcf143909bfe67f685caaf362066e1c7e224899f57678bbcecb7a720175bcbb387d","r":"0x1ca26859a6eed116312010359c2e8351d126f31b078a0e2e19aae0acc98d9488","raw":"0xf86c59850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88882b0ca8b9f5f02000801ba01ca26859a6eed116312010359c2e8351d126f31b078a0e2e19aae0acc98d9488a0172c1a299737440a9063af6547d567ca7d269bfc2a9e81ec1de21aa8bd8e17b1","s":"0x172c1a299737440a9063af6547d567ca7d269bfc2a9e81ec1de21aa8bd8e17b1","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x2","v":"0x1b","value":"0x2b0ca8b9f5f02000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x8bec4e6fb1a28820eb1e8ec2d4eae4842ed2f923","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x98a03afa804e248ada5f26e9118ae927d4d3cb60e78c54938dced1cf25ee3567","input":"0x","networkId":null,"nonce":"0x2","publicKey":"0xbc8c89a85804c7859069c13561dbbd8d1d4739ec7d18514c42b3ffea64529cee522a5e20d93373d0074e94c4c7b6eba51c7d2f18ef7c64c37520342acb233795","r":"0xa5aca100a264a8da4a58bef77c5116a6dde42186ac249623c0edcb30189640a","raw":"0xf86c02850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880fd037ba87693800801ba00a5aca100a264a8da4a58bef77c5116a6dde42186ac249623c0edcb30189640aa0783e9439755023b919897574f94337aaac4a1ddc20217e3ac264a7edf813ffdd","s":"0x783e9439755023b919897574f94337aaac4a1ddc20217e3ac264a7edf813ffdd","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x3","v":"0x1b","value":"0xfd037ba87693800"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x4835a9626b02369546502d2949e16b0fda110b0c","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x18f1e6430334ad548bc36fc317016bc9f7a076d1fa50a89fe4e1d095ed3f9562","input":"0x","networkId":null,"nonce":"0xd9","publicKey":"0x91b3b4fe89d112cfc7308619e8aa7de86f14af3f6b6e4e92becb6e29e98207835bbe1a69109c16b14b0eb7285d2b952a9cde6007932afe95e81eefc183f75314","r":"0xb93c6f8dce800a1ec57d70813c4d35e3ffe25a6f1ae9057cf706636cf34d662","raw":"0xf86d81d9850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d888814bac05c835a5400801ba00b93c6f8dce800a1ec57d70813c4d35e3ffe25a6f1ae9057cf706636cf34d662a06d254a5557b7716ef01dd28aa84cc919f397c0a778f3a109a1ee9df2fc530ec0","s":"0x6d254a5557b7716ef01dd28aa84cc919f397c0a778f3a109a1ee9df2fc530ec0","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x4","v":"0x1b","value":"0x14bac05c835a5400"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x9cc72ebf3daaf12c72e48605e1e67b47c95a1911","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0xb1cada8daf63c45750df1ee79eed5a3cf6240e3cebdb6de3f26bc7cf03217bf4","input":"0x","networkId":null,"nonce":"0x34","publicKey":"0x90dff18c1c01d566e6d8bf0190e3e965f98e7f51ccbbe6040f9a9972e88f4ad19f1547406454fbc9e1ebcf4c5f2f1e2df9b9371028fe0a552ecca5f5f0aa4129","r":"0xe9a25c929c26d1a95232ba75aef419a91b470651eb77614695e16c5ba023e383","raw":"0xf86c34850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f258512af0d4000801ba0e9a25c929c26d1a95232ba75aef419a91b470651eb77614695e16c5ba023e383a0679fb2fc0d0b0f3549967c0894ee7d947f07d238a83ef745bc3ced5143a4af36","s":"0x679fb2fc0d0b0f3549967c0894ee7d947f07d238a83ef745bc3ced5143a4af36","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x5","v":"0x1b","value":"0xf258512af0d4000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x5c51467399bc655f0cc6db88df15946717534633","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x4fa879b491e0779fc035758ec77b93c4e51d528d65b64eb055c015a58deff103","input":"0x","networkId":null,"nonce":"0x6f","publicKey":"0x0b7e2532afc2daa33763002525aa6c7edc25ea97d63baeeb2c6f5094f18dca4a0212b52061f9a9091aad5c4380a6506f9a51ddd2d014e78742bf144a58d6ffa0","r":"0x9e0b8360a36d6d0320aef19bd811431b1a692504549da9f05f9b4d9e329993b9","raw":"0xf86c6f850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88881c54e302456eb400801ca09e0b8360a36d6d0320aef19bd811431b1a692504549da9f05f9b4d9e329993b9a05acff70bd8cf82d9d70b11d4e59dc5d54937475ec394ec846263495f61e5e6ee","s":"0x5acff70bd8cf82d9d70b11d4e59dc5d54937475ec394ec846263495f61e5e6ee","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x6","v":"0x1c","value":"0x1c54e302456eb400"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x055d9d7ec193d1e062c6ec4fa80ef89b5c1258f4","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x1bea59827ab153b20cee79890d221a80fa6a04e552d667504c592ed314fb6d76","input":"0x","networkId":null,"nonce":"0x46","publicKey":"0xfae19a0ac08d36f0229663d45d0c41ca52c4e295c7af82a1b39515a79025175293400d026e0d41767aac42f8b7e4a6687c5762161457d753f1fc0766614868f9","r":"0xb2803f1bfa237bda762d214f71a4c71a7306f55df2880c77d746024e81ccbaa2","raw":"0xf86c46850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f0447b1edca4000801ca0b2803f1bfa237bda762d214f71a4c71a7306f55df2880c77d746024e81ccbaa2a07aeed35c0cbfbe0ed6552fd55b3f57fdc054eeabd02fc61bf66d9a8843aa593a","s":"0x7aeed35c0cbfbe0ed6552fd55b3f57fdc054eeabd02fc61bf66d9a8843aa593a","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x7","v":"0x1c","value":"0xf0447b1edca4000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x8e68c0c9b5275fa684291304af9cafe6ceaf2772","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0x73e87db1108a2aa852f48e088ca1a2771f9b7c18af8d1bd77a3cdcc72a750c56","input":"0x","networkId":null,"nonce":"0x3","publicKey":"0xa5e423dfcbdbba1fdbb785367a88235fa2569061d72b6c715111ac21cbef8fc1db860acdef85f1408c760f34b28a4f07d950ac15c4b85d5e528e50f546a89b6d","r":"0x6dccb1349919662c40455aee04472ae307195580837510ecf2e6fc428876eb03","raw":"0xf86d03850ba43b740083015f909426016a2b5d872adc1b131a4cd9d4b18789d0d9eb88016345785d8a0000801ba06dccb1349919662c40455aee04472ae307195580837510ecf2e6fc428876eb03a03b84ea9c3c6462ac086a1d789a167c2735896a6b5a40e85a6e45da8884fe27de","s":"0x3b84ea9c3c6462ac086a1d789a167c2735896a6b5a40e85a6e45da8884fe27de","standardV":"0x0","to":"0x26016a2b5d872adc1b131a4cd9d4b18789d0d9eb","transactionIndex":"0x8","v":"0x1b","value":"0x16345785d8a0000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x2a65aca4d5fc5b5c859090a6c34d164135398226","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0x337a5e90b73f44ffebea73cb3d97738c524f63e1032b30735e43212cff731aee","input":"0x","networkId":null,"nonce":"0x2a11f","publicKey":"0x4c3eb5e19c71d8245eaaaba21ef8f94a70e9250848d10ade086f893a7a33a06d7063590e9e6ca88f918d7704840d903298fe802b6047fa7f6d09603eba690c39","r":"0xaa8909295ff178639df961126970f44b5d894326eb47cead161f6910799a98b8","raw":"0xf8708302a11f850ba43b740083015f90945275c3371ece4d4a5b1e14cf6dbfc2277d58ef92880e93ea6a35f2e000801ba0aa8909295ff178639df961126970f44b5d894326eb47cead161f6910799a98b8a0254d7742eccaf2f4c44bfe638378dcf42bdde9465f231b89003cc7927de5d46e","s":"0x254d7742eccaf2f4c44bfe638378dcf42bdde9465f231b89003cc7927de5d46e","standardV":"0x0","to":"0x5275c3371ece4d4a5b1e14cf6dbfc2277d58ef92","transactionIndex":"0x9","v":"0x1b","value":"0xe93ea6a35f2e000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x2a65aca4d5fc5b5c859090a6c34d164135398226","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0xc280ab030e20bc9ef72c87b420d58f598bda753ef80a53136a923848b0c89a5c","input":"0x","networkId":null,"nonce":"0x2a120","publicKey":"0x4c3eb5e19c71d8245eaaaba21ef8f94a70e9250848d10ade086f893a7a33a06d7063590e9e6ca88f918d7704840d903298fe802b6047fa7f6d09603eba690c39","r":"0xcfe3ad31d6612f8d787c45f115cc5b43fb22bcc210b62ae71dc7cbf0a6bea8df","raw":"0xf8708302a120850ba43b740083015f90941c51bf013add0857c5d9cf2f71a7f15ca93d4816880e917c4b10c87400801ca0cfe3ad31d6612f8d787c45f115cc5b43fb22bcc210b62ae71dc7cbf0a6bea8dfa057db8998114fae3c337e99dbd8573d4085691880f4576c6c1f6c5bbfe67d6cf0","s":"0x57db8998114fae3c337e99dbd8573d4085691880f4576c6c1f6c5bbfe67d6cf0","standardV":"0x1","to":"0x1c51bf013add0857c5d9cf2f71a7f15ca93d4816","transactionIndex":"0xa","v":"0x1c","value":"0xe917c4b10c87400"}],"transactionsRoot":"0x447cbd8c48f498a6912b10831cdff59c7fbfcbbe735ca92883d4fa06dcd7ae54","uncles":[]},"id":1} diff --git a/statediff/indexer/ipld/test_data/eth-block-body-json-0 b/statediff/indexer/ipld/test_data/eth-block-body-json-0 new file mode 100644 index 000000000000..e7dfbca84118 --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-block-body-json-0 @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":1,"result":{"author":"0x0000000000000000000000000000000000000000","difficulty":"0x400000000","extraData":"0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa","gasLimit":"0x1388","gasUsed":"0x0","hash":"0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x0000000000000000000000000000000000000000","mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000","nonce":"0x0000000000000042","number":"0x0","parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000","receiptsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","sealFields":["0x0000000000000000000000000000000000000000000000000000000000000000","0x0000000000000042"],"sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":"0x21c","stateRoot":"0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544","timestamp":"0x0","totalDifficulty":"0x400000000","transactions":[],"transactionsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","uncles":[]}} \ No newline at end of file diff --git a/statediff/indexer/ipld/test_data/eth-block-body-json-4139497 b/statediff/indexer/ipld/test_data/eth-block-body-json-4139497 new file mode 100644 index 000000000000..02ef395846ca --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-block-body-json-4139497 @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":1,"result":{"difficulty":"0x5c647cfc07f1a","extraData":"0x65746865726d696e652d6173696137","gasLimit":"0x668fd6","gasUsed":"0x655bf3","hash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","logsBloom":"0x00004000840000000004000400000006008800000000000000900000000000000002000000100000000000000000020000000000004000000000080000080000000000000200000000000008000000000001000000000000000000000800000000014000000200000000804040000200000000000100008004000110001000200000020400000000800200000008000000400080008000200000001040000100002000000000000002000000000000000000010000000010080000000000000010080002000000000000002001000000000000040000000120200000000000000100000100000000000000000000000000000000000000000000040800000000","miner":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","mixHash":"0x2a65887132d93df4ad543ea9ab69b2de12bf1ef0d9a5b9128fe557a7cf6e365c","nonce":"0x68b593b0029de941","number":"0x3f29e9","parentHash":"0xf8ef0dc32d00fe925c9ac3039f3fe202ac6988f37b3710840848ecf29a4905d9","receiptsRoot":"0xf17608f36b1fc813fefd9cbd1fd653195de20ab72f2efcc95f7e00c6576080d6","sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":"0x8a42","stateRoot":"0x3258ad3d8a73140be9d3895166f3f88b0f65a5575d8176f10dc2a6dddac36b64","timestamp":"0x598c1020","totalDifficulty":"0x23bcce551ec1d5055c","transactions":[{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0x55335d56e95151bce1635bce649175ea954aecee","gas":"0x2117a","gasPrice":"0xbdfd63e00","hash":"0x51f9d60ce19d4174224f91be402d4504553f127511a630a18a8735b4c1db072e","input":"0x0f2c9329000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98000000000000000000000000e592b0d8baa2cb677034389b76a71b0d1823e0d1","nonce":"0x1","to":"0xe94b04a0fed112f3664e45adb2b8915693dd5ff3","transactionIndex":"0x0","value":"0xb7ce92a6fa0400","v":"0x26","r":"0xaa97e8fb84036ed395fab0e05f4432e219e855539a17a73444e915a3f18d7f15","s":"0x117401fbe04f6c8316ba4c344b37de5d1b5a6fc252160a093e7270d6fd37c2c4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfbb1b73c4f0bda4f67dca266ce6ef42f520fbb98","gas":"0x2d9dc","gasPrice":"0xbdfd63e00","hash":"0x57a6c52559d193fef65f8b99fdd46f341f0739ba7d4a772a87d8fad89fc2cff5","input":"0xa9059cbb000000000000000000000000744346c50253300694aea6d7e03f55a3ea91f8a30000000000000000000000000000000000000000000000000000013061e0a9ab","nonce":"0xc104d","to":"0x41e5560054824ea6b0732e656e3ad64e20e94e45","transactionIndex":"0x1","value":"0x0","v":"0x25","r":"0xe925321edf5dc905fa0ebf9a08d8915e0ce90463d55c19e8bdf0dc8e5e6ddc73","s":"0x328a5099139ae2e3f3be2736dec30fd2b3240892b77575e588b8f84a0e11307b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xa624ceb708a1e9a3962de82c5a3c5850db0097f1","gas":"0x2117a","gasPrice":"0xbdfd63e00","hash":"0x616694b9e9aea8d913797a50958a9343e18451ccb2abffa1b10b2d06378c612f","input":"0x0f2c9329000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98000000000000000000000000e592b0d8baa2cb677034389b76a71b0d1823e0d1","nonce":"0x24","to":"0xe94b04a0fed112f3664e45adb2b8915693dd5ff3","transactionIndex":"0x2","value":"0xa9f1b6b74205400","v":"0x26","r":"0x4b6f583ee70f4aabad8da3c97a0b1d7bd18ef6463aa08fb730696b758abe255c","s":"0x1a13f3c8fef9b92c28151db22b03b9b9894b2d7ef103a38b204ac5ba970073fe"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xb083a0287b4e7f8319eee74b27e42bdd77da4e1a","gas":"0x2117a","gasPrice":"0xbdfd63e00","hash":"0x92a84244da41cd93c1c0ab7b7d13556453d3fd76317a71fa89ba129ad4c9d80e","input":"0x0f2c9329000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98000000000000000000000000e592b0d8baa2cb677034389b76a71b0d1823e0d1","nonce":"0x3","to":"0xe94b04a0fed112f3664e45adb2b8915693dd5ff3","transactionIndex":"0x3","value":"0xd51851e1dacc00","v":"0x26","r":"0xc8304a7acbaddcdd4ac10216697ea88d1b154c9d0de42fb75ad9a301fef38cc1","s":"0x76cdd85171fb9da403def3fbfafb8545835aebeb9a541e6207d9d373914e1e8d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xc348b6a2758fb408e5cce34d43feee1726692e0d","gas":"0x13880","gasPrice":"0x719f11100","hash":"0x164a9b95e7914ef6071b6228699635e8e8d58b4d60fd4736aabd87b5bcf8d5fb","input":"0x","nonce":"0x3b","to":"0x7727e5113d1d161373623e5f49fd568b4f543a9e","transactionIndex":"0x4","value":"0x18f7be6e64863700","v":"0x25","r":"0xfb14159445060e4a1809e7d959210da4151fe1535c8b9aa9158b5d7536b0fbac","s":"0x3563cf5da676135b36d9d2305f1ee133452280e2c1abe16bda50fe502557d1d9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xd3273eba07248020bf98a8b560ec1576a612102f","gas":"0x5208","gasPrice":"0x6fc23ac00","hash":"0x5d6f0ac462923b852080c3b96afa862bc93a4bc605e5feb9bda64780d6c89089","input":"0x","nonce":"0x67ac","to":"0xd66f7b11c7da581406d62a501fdee675466f4593","transactionIndex":"0x5","value":"0x5bd6662df2c3c400","v":"0x1c","r":"0xf042ec51b11a4c14cb7f48e50e3c4278965530f9e5c4a17926e47f83dbf09fe5","s":"0x5eee0c65eacdabfb60688656d108ab5dc74dce9ad79f661148bbba7694a5c191"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0x3b0bc51ab9de1e5b7b6e34e5b960285805c41736","gas":"0x5208","gasPrice":"0x6fc23ac00","hash":"0x8c951abf8f855e94f1059a0b9f9de8e23e12ffc7d4511e0dcbfe73060ff2e9ee","input":"0x","nonce":"0x6595","to":"0x7c402ca59a701f6b3f077f175b4c964122043221","transactionIndex":"0x6","value":"0x5bd6662df2c3c400","v":"0x1c","r":"0x36d4084792312a9aafd676e0570acc14b29b590bc3f38e0c643ff278653628ae","s":"0x4f25d719cd23e3fb88bd205e955d8127c819d208046e83f9ac9a47c35ec2a814"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0x093177dbaa25a001e3ee343d3ec492e71b9367aa","gas":"0x6271","gasPrice":"0x6c7fc3b40","hash":"0xecc2c35c2ca748c7eb2970d76288e34ab514a48c60670ba5fa04ec50d59be1f5","input":"0x","nonce":"0x2","to":"0xda1b2aeac0196d39658186604609fff185e1774d","transactionIndex":"0x7","value":"0x5b09cd3e5e90000","v":"0x26","r":"0xef0a0125e0984c9a59fbe475df19bed2fcbfbe02ced04ad9f5f25530e276a527","s":"0x7ce66b31396aaf02a34d966e87e03ba9f04ac021f56e8ca1cd6124434df61ab1"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xf04ad0c7eb4ed654c52477f8e756800bde9f2341","gas":"0x5208","gasPrice":"0x4e3b29200","hash":"0x7475e0a920d21ee08b85f0ec61b02ed646190ff23ae2805dfef4cfe81c59a46e","input":"0x","nonce":"0x427","to":"0x1e4f986d287bacf4283d35ca61fb342ca91674d6","transactionIndex":"0x8","value":"0x3d48c89a6020000","v":"0x26","r":"0xdab319aff51e0755b832a17fba0e4778895980eb6cb87a2aa4b35edd418163ef","s":"0x10dc3f986fe67347e293177acdd0dbfa7a910d64c9c484a0635221dd652a6191"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xb2930b35844a230f00e51431acae96fe543a0347","gas":"0x186a0","gasPrice":"0x4a817c800","hash":"0x070599a9b0a4e550cdb1b5068d0d3bfe3fc0d60302973d3b3abad3a4762ae81c","input":"0x","nonce":"0x569fe","to":"0x79d56207445e24f5eeb391358924a39c620dd1e0","transactionIndex":"0x9","value":"0x21c60092fff800","v":"0x26","r":"0x77ba2e5b7c617e6ad54a7d4ca14362837cdd3138648a0855436a6fef99033d4d","s":"0x6714b8b257a8c714b2395fca0a8bfd9299fa3d759da9c01a2582d7114a316f05"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xcbf44ffb74ae94a4b696e716964b1d69400c7749","gas":"0x11170","gasPrice":"0x4a817c800","hash":"0xa3031fce94886738b6666b8a58233e845e9fd4ced150f65c043738fc54ccc7bb","input":"0xa9059cbb000000000000000000000000e74db956a107baa7cadc1258a6f539f40fc4fec100000000000000000000000000000000000000000000000000000002caa8e180","nonce":"0x0","to":"0x93e682107d1e9defb0b5ee701c71707a4b2e46bc","transactionIndex":"0xa","value":"0x0","v":"0x26","r":"0xda99eedee485f9f789cc183307b139b63e0885c7135796fbcca1d20415fd884e","s":"0x5103dc4b2fe14ec65fe2c98c331cf9177ffee86a89ab5b8079a3ee285bdab7c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfc203c5f867be784726ef4198c0e8fc1313074db","gas":"0x5208","gasPrice":"0x4a817c800","hash":"0xaf654ac5eaecca624725c4236adcbee10a9b4c76f4bb71c893c373c659a4305a","input":"0x","nonce":"0x1","to":"0xa3da2a2f864a180297adedc48ad51e562d7a9f8a","transactionIndex":"0xb","value":"0x1e81bba24c058138c1","v":"0x25","r":"0x6e1989c52a8d07f84ad0701cc6eae4e9fbb2ca79476b03422098d03e52e6a594","s":"0x6d36b9c8ed63abab0ada4fd9c53541d4b948b23c79ae118cd2f205a010f2c0ee"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0xb2f6b98129aad387041bfe8710bc1bf363bb208f15d49a482b5d15bbd13d1cec","input":"0x","nonce":"0x2aaab9","to":"0xabcd334c3504100e6d26d895c8c658e35fe515f7","transactionIndex":"0xc","value":"0xaf069a8a72ee91","v":"0x25","r":"0x5b5bdfabd8a099a056af2ecef44bc142aa5bfe7623a14505fc0c6f3f059eee0","s":"0x336f76890622529392f3eabbd793be3ec6367b31b65737d6ea2ebedcc934f3d6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0x75814b803794e796a4b496765af343121020238e","gas":"0x5208","gasPrice":"0x4a817c800","hash":"0xcf257e096c2cd20debbb4608d00ca28b3c576b705de8109090caead53ccfab17","input":"0x","nonce":"0x1","to":"0xd0bcd02f598c2473395842d647011b6d1cdd0e5c","transactionIndex":"0xd","value":"0x1ee647737e6ec208c1","v":"0x26","r":"0xcd5de53b8c661068d31053854e4e562f276e8481cba387d6853910d415a8e213","s":"0x2d209a8658c9411087c389f3bdeaa9c2ff70eac8950f0b4db413fcc39a4fee2b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0x8f5aae245398626bc162b47b862fa09e49190b38","gas":"0xbb00","gasPrice":"0x2540be400","hash":"0xb16f7c1b61134c155cb820d8f51d77e93fa7212c8f46be42dbfc8a3767d176fb","input":"0xa9059cbb0000000000000000000000004f5151785e03b47d0c6641872bb6b29b6de1b77c00000000000000000000000000000000000000000000000bbc4849990fa54400","nonce":"0x0","to":"0x888666ca69e0f178ded6d75b5726cee99a87d698","transactionIndex":"0xe","value":"0x0","v":"0x1c","r":"0x25dca29942900fe444e2e3e27ea41648d6a22947a9d8a38e11ae367b0a064d0f","s":"0x52e06101618b2fe1f3a845f4f39a3092016e920855be9c9b447e3d6828e1b263"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0x22b84d5ffea8b801c0422afe752377a64aa738c2","gas":"0x186a0","gasPrice":"0x2540be400","hash":"0xf40a89152e66d51b54ae72df0712e08fd6c121fd1d58f7cbc38f63249a139963","input":"0x","nonce":"0x1af86","to":"0x444d80ab1f1540642d69b3eaeb790903cf4872bd","transactionIndex":"0xf","value":"0x53444835ec580000","v":"0x25","r":"0xebadeffaf6e5a8b53f482372e9b33db8c0380f4a21a388f499b0f0072e8e2afa","s":"0x455bd8083723fbb895f0fe62c02ad1882bc3daa76443e4a77494f984824e9c73"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x5ee4fb7764e28e71b9d0ce72741d6df027b4a79f969a71364db380de686cc1f1","input":"0x","nonce":"0x9c5c","to":"0x3c13a69380e27bfd16a5bc5528f4c1d6cc4993ac","transactionIndex":"0x10","value":"0xbec8544eceac00","v":"0x26","r":"0xe7eb23823262f600e33b526a953ac7e32dcc0cf86d9f1febbf8db30edea03b02","s":"0x595d98353ad032557caf00ebe14f21ebe66fab85394a421d8bbd9a47b3ae6627"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5","gas":"0xc350","gasPrice":"0xee6b2800","hash":"0xc0e565782181943c4697199214db1d21a535835b665b2ba771fbe4693ce52de0","input":"0x","nonce":"0x292ad7","to":"0x0fbb3c7bcac281b97f8a8a3292a026d67c3230f1","transactionIndex":"0x11","value":"0xb2e25606328960","v":"0x26","r":"0x837849bae28e40b752586ce7135cee1a4741eb3f68b089cb6ef4dfb4b6291738","s":"0x312d8f5e8a25836687d6eb69be151016074355ee5b580793111314daba9da1a6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfbb1b73c4f0bda4f67dca266ce6ef42f520fbb98","gas":"0x2d99c","gasPrice":"0xbdfd63e00","hash":"0xc092388bd2e7626c53e3c580b4a5d57de3442b28c97b34fe1ff68042b9026137","input":"0xa9059cbb000000000000000000000000cd2e8348d2f58f02f1859ecdef07d1ecf1f0ced9000000000000000000000000000000000000000000000000000000174867a5c0","nonce":"0xc104e","to":"0x41e5560054824ea6b0732e656e3ad64e20e94e45","transactionIndex":"0x12","value":"0x0","v":"0x26","r":"0xec60ffa5508b41567c20a68f26df77c3de22fa3b11fa853c7562f693df12cc03","s":"0x5fff6d9220b4da3f68358ead8b782e52de0f2ae2def4c07e5d547d513fcfe80"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x34ee80fe753728be177a1e6ed5541565b2c94da9ac8fb5d16e7cc757cea3692a","input":"0x","nonce":"0x2aaaba","to":"0xfd15c258b4191b73c7dde5df066f4732e4392f7f","transactionIndex":"0x13","value":"0xdee2eb356bf15a2","v":"0x25","r":"0xa5737391f905649e6ed6604db0b4040e94aec8bc6ad47afcbb1f1cbd934a7dc2","s":"0x5a52547c6fce0aaf436c26033f92ef7542629b8cfad92a5137979f072f6371af"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfc203c5f867be784726ef4198c0e8fc1313074db","gas":"0x5208","gasPrice":"0x4a817c800","hash":"0x8c9d7cbc1629acab3c2b0a6423a84025e5bc11f15eec3bcfe2e224a505bdd5d2","input":"0x","nonce":"0x2","to":"0x42bd724618c19fd396b95891621e267968707dd3","transactionIndex":"0x14","value":"0x17b2a64c0adf2a073f","v":"0x26","r":"0xe40d950eeef37b63fd058ad8e0e9510b858ba5a67e033d99f89c9023a6fa227e","s":"0x791f7b441d70533f9670b7db0b224921944fea8820b5dfb2f06704f75872bea5"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbe853e3717ddcec5f9d57ed55e6ec1dc6fa1e9545c901b52a156f7b1b9c9cd3b","input":"0x","nonce":"0x9c5d","to":"0x7cb1e28cf73698e0474bf1b7b98d01a8e71204b1","transactionIndex":"0x15","value":"0xf1591cc0b131a400","v":"0x25","r":"0xd96f474d79e265d9dc5bf6bd09c46b54a25627caff37ff549c726e0ea7812920","s":"0x7630e1a32cdd1e3eaee6c00b38d349dfa3048ccbc20431cf651a218c124c1ab3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5","gas":"0xc350","gasPrice":"0xee6b2800","hash":"0xfaacce929d5e0f054479cb584dab3490770c43e616c3bba0c2f8bfd0a074a603","input":"0x","nonce":"0x292ad8","to":"0xda6b3b1bd62b06ca13fb37f660e8daf848b60330","transactionIndex":"0x16","value":"0x2e7c5072cf1e9e0","v":"0x26","r":"0x7d51a1209d5475564a4df31fef6d0a09c8b8aa1cc6d1c87cda42f02a58db4da6","s":"0x5ab60244b91d00e7d588151bd9f51f4fec1349c5c146b2178c2bca94610cbe3d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfbb1b73c4f0bda4f67dca266ce6ef42f520fbb98","gas":"0x2d99c","gasPrice":"0xbdfd63e00","hash":"0xff1d6ee564b1be371792551a5b047ccdf519e74f3d5513da008318baf6915715","input":"0xa9059cbb00000000000000000000000091b1053eb9486b0b63d44a5cba021c324991027d0000000000000000000000000000000000000000000000000000005981122544","nonce":"0xc104f","to":"0x41e5560054824ea6b0732e656e3ad64e20e94e45","transactionIndex":"0x17","value":"0x0","v":"0x26","r":"0xe13fe6b5356d9abc156dffe6395f7b724a9b35ec58fc4026811241b03bad7a92","s":"0x33ae3ea46a35263b6d5e96574317c233affa15aea7d79209facbe88ce2eed013"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x4a2db708d569b49383b1d8abbff178b574affc87f879d57b5798904b52d0d4fb","input":"0x","nonce":"0x2aaabb","to":"0x029f13b14a1c4c65aa19f03fb12c0d761fc9e662","transactionIndex":"0x18","value":"0xb0297da2f04b2c","v":"0x26","r":"0xbad7b74d953063bb260fd27fc57c3ce40f46ab872fd44d62e30edd2a2da91e02","s":"0x7e513fa35422c73d96c51b455cfd09bd846ae5ea1f6047c6c84151cbfa68e6bd"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x5fa1fc424acb1df5a2efe579d9cf301ee4b7415b7086800fe48a1fd2f4127fee","input":"0x","nonce":"0x9c5e","to":"0x1f70dbf8b8c7a47dceea01ffe6749382245fa10f","transactionIndex":"0x19","value":"0x1a21d8eef282000","v":"0x25","r":"0xac50ff5d7c54b976fb08d24e235a1ba4e611a017332e20747818b1091cdf3a2f","s":"0x1523cdd85db8b8fd1e6dbc29bffef1583744f5a5ed278a97999fe44642e6b77f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfbb1b73c4f0bda4f67dca266ce6ef42f520fbb98","gas":"0x2d9dc","gasPrice":"0xbdfd63e00","hash":"0x231bcf683e12cb3cb50d2979154e5537822b30974a3bf08596a231ae7ffde4e2","input":"0xa9059cbb00000000000000000000000018e3dfeaebe76cfacc75fd724e2c6e4ba140d56a00000000000000000000000000000000000000000000000000000107a24798c0","nonce":"0xc1050","to":"0x41e5560054824ea6b0732e656e3ad64e20e94e45","transactionIndex":"0x1a","value":"0x0","v":"0x25","r":"0x942337149235dbe45a6fb9596ca5bfd47f3d48a49bf17980bb7a424203f48130","s":"0x673e537d0a7edc66bfb3bfd7918adc7541f1850cb5249113cd6af089d25a75d3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x2809c2c670b3a0a57ab0279e369f34972e8aa818743a7b462e6c3812b139aa85","input":"0x","nonce":"0x2aaabc","to":"0x54da15b491babc978b2a3fc31841911a12c5ca0b","transactionIndex":"0x1b","value":"0xae56830ea32b52","v":"0x25","r":"0x8d0aa2d9e685b918186da550d2b00c51a0c471fc78493f6c6427d69b5e25def0","s":"0x79e64a26af42c415adba3b41dd899d030d683bb0c2c18289f0024bec84a34189"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb38ef7a0d9f4ec185696f9328171e38586d5f0c0c725cb2b1adf8a5c8a32b33e","input":"0x","nonce":"0x9c5f","to":"0x55b840e722a5a73b34320a34c48463e67993c0e2","transactionIndex":"0x1c","value":"0xd923293ec5e400","v":"0x26","r":"0x71e3e7c505606dfd773f53badb0f2d081207cbea0000c288781a18bcd6b75c14","s":"0x264bb04158b749785485323ba868ba7ada155985af7951bf948c4fb35bdc0ae7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfbb1b73c4f0bda4f67dca266ce6ef42f520fbb98","gas":"0x2d99c","gasPrice":"0xbdfd63e00","hash":"0x7b3c92175534b96e35797ba00deb87f606edac372bd573f06ff6636140938f6e","input":"0xa9059cbb000000000000000000000000be69390fbf8871caf82e2b70a92a4f7a87d161c20000000000000000000000000000000000000000000000000000004b585bb7f3","nonce":"0xc1051","to":"0x41e5560054824ea6b0732e656e3ad64e20e94e45","transactionIndex":"0x1d","value":"0x0","v":"0x25","r":"0x8863a36a60b2fa5a621cb01f1d80c324b519c8cd3bc3db559b47cc5e6777d26d","s":"0x3b5ff01f46e137f33f273ab3eaf3d6afb12959d19f8f01a9119d40fa9beb90ea"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x8f0b09787e0356ce6e2f43a2b5a15245137a0f6066a9fbcdf519e8df37a92aa7","input":"0x","nonce":"0x2aaabd","to":"0x863b65fe3b44db9f60dbace119fb08fdd4d2c62e","transactionIndex":"0x1e","value":"0xde5381edb9bafe8","v":"0x25","r":"0x61f134af94880a42bbfaffd277bbd8c80a6fc978e562dff4ca29e9c8b61968ce","s":"0x8a02c26b769e22e966d35d44acc175eb747f57bb9d3fa2c057e23b1529ba9e9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbe4bc2e52dc8bcb6df1a935ddcbd84958a1de639fbefe1da5ed829f8f4f4486b","input":"0x","nonce":"0x9c60","to":"0x585366a5ad43dc56ccbb54e94c48c6f1d931710a","transactionIndex":"0x1f","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x62674294331a2dfb96a2d7480331f18fe9003869e52f32f0a5b88a0094fbff63","s":"0xbf8f9286718f28e13473f4136b8e8989ca247db1075f6cc633fac869532e754"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfbb1b73c4f0bda4f67dca266ce6ef42f520fbb98","gas":"0x2d9dc","gasPrice":"0xbdfd63e00","hash":"0x3eebb2d806a9ed429d460178c89d72364dd35719d1865942234bdd70bdfb258b","input":"0xa9059cbb0000000000000000000000004c59f430c6ebadaad6ccd25f4b9eeeb8f7a22108000000000000000000000000000000000000000000000000000001029f447f3e","nonce":"0xc1052","to":"0x41e5560054824ea6b0732e656e3ad64e20e94e45","transactionIndex":"0x20","value":"0x0","v":"0x25","r":"0x9e20b3b1429b5672d9f05a859633e1e4facb71e308924277811db2e3ebaefedd","s":"0x24803ead950f32f9863811c17f914ae9e831c48973183c036605d96750ee93a3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0xc94b14d966d087f09dc1bab45d5684d5c1f00167a27042e48391c4b97dbec90a","input":"0x","nonce":"0x2aaabe","to":"0x872bad809a1b1ec9a7dd38ac4d7e9b19920a1faf","transactionIndex":"0x21","value":"0xaf0a678d3ca95b","v":"0x25","r":"0xfad929edfbe500b2be3dffed3b9ebe4d9662bbdd211ae388a1c05a693c0054d8","s":"0x69f5962685c91ce1559d9e350b6322539f6d02dfa824d5253f381fda61f8f663"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x3cfa69cea575486acd281c7517bb9e4c74e6e8179065b5210b8ed06054a1c1a6","input":"0x","nonce":"0x9c61","to":"0x7d1340884d2b767da3e87daa3b59960c4e98b791","transactionIndex":"0x22","value":"0x17aadf094fd1c00","v":"0x25","r":"0xcf01d52255575cd6e8cc9045187c293bb950b56e69d152880fd672f026b71213","s":"0x131fe537936d809d1eebf72caa0019142e348d282bb59394f0a6c531338d95f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfbb1b73c4f0bda4f67dca266ce6ef42f520fbb98","gas":"0x2d9dc","gasPrice":"0xbdfd63e00","hash":"0x8e2cad0763aea7b8a1e9b45b394aa0b62343dab30a230948bfdbe19988da31ad","input":"0xa9059cbb0000000000000000000000006ccecb1bdbf8f464f2b58adb417d5a88d0300f0a000000000000000000000000000000000000000000000000000002388f52ea80","nonce":"0xc1053","to":"0x41e5560054824ea6b0732e656e3ad64e20e94e45","transactionIndex":"0x23","value":"0x0","v":"0x26","r":"0x12ceb52c978e7a7e67f58068def1924fd7a500fcace1c39840f19dfdef82a130","s":"0x3d3e4826ade71f3e9079b31d9b8942c9f4f0cfc09bcc1cd66a9fefa9f2dcc8af"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0xd49fd3809e5349c4425c5712ab9fc2c69c825161ab706c1bf3179f30a4e8c5fb","input":"0x","nonce":"0x2aaabf","to":"0x959cd73ae36c115df8ee9d20f5d3101ff3181466","transactionIndex":"0x24","value":"0x17057457ca587d4","v":"0x26","r":"0x707870910fb23d9091244655fa4d6b317939f9e0011b89097ce4903f25ee6e8b","s":"0x16707cf3e313a11f694b881e1463322b07730b79f3133f74befa570fbc78ce78"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4c56dac6f3503162683ae12d2445155aa1f705bb131c14742424944bede67517","input":"0x","nonce":"0x9c62","to":"0xc567f4a3d18d42fc49a5f8c54eaeaad0cc0713d0","transactionIndex":"0x25","value":"0xbe0d6ff05a3800","v":"0x26","r":"0xccb97060a133d58c8f40b7d2ecdba4447b19246f40a154f6e69b91368526f0f0","s":"0x5a6006fe0a064b9e378ee5f3ad329735f844b73b7a3837643737b9f02136824a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x90a0eba75638bce9ebe5554c4695fa9c25e95f85fa7ccb3ab134dddd24912f06","input":"0x","nonce":"0x2aaac0","to":"0x02eee5b2f34918340694c0aece742dc7f8ee0ff9","transactionIndex":"0x26","value":"0x161d70598349dc5","v":"0x25","r":"0x347bbd3db97596d8b48e281437e4038078582d6380ce2bdbe5621f2b04cd9acf","s":"0x9996abfaa8d6fa128c3801b033e6980306ec0185fcdf603b1ef425117023a54"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x448da8c7d24be59ec445f4df143cae6782fc194b1dbd61d07d1fbce99a525d2d","input":"0x","nonce":"0x9c63","to":"0x4530afe8ae24f91875b74da5fe251170177bcbfb","transactionIndex":"0x27","value":"0xc4a234146d6000","v":"0x26","r":"0x85303903f9f1301a6479d32cb6dea765c2a1bf114e59a50fb5b05e37a5b23631","s":"0x17478bce1de2c9fe6a984aeed9f831343376a145164c92277df4e63bcfcaabc3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0xd3ac8ce2a58d2f93adb88cfcf9241e1a682f71f69e61e0da04f3de056c0f3f28","input":"0x","nonce":"0x2aaac1","to":"0x04bdde4339294d8a521a28dc696f2286f0acd3d2","transactionIndex":"0x28","value":"0x185f9a12e284964","v":"0x26","r":"0x67403bb19a16ff477d30e264e4e4c0b6664c220e43b85c89c1fb0459085c0362","s":"0x617a3affb24dc4d38376c3ad4d33e972b3721e8ceadb779151a81aac031641d3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x96f04f1c4a5d8f81e8f541871ca1661662fee633aad36c65089a42418bc5dc5d","input":"0x","nonce":"0x9c64","to":"0x14fc32d88632e190beb08c1929c928954c06e336","transactionIndex":"0x29","value":"0xc3d6fc66994000","v":"0x26","r":"0xc567df5c64232efae75e1285c43f542ea8834aa9459674391e59dfe258598bb8","s":"0xf47712241b38e13645d6b07f8e8f95d4a2ac79b98052f04841100341a3fad1c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x5723eeeb5059dd1f44a3edba5d51f584e8a75fc99633990f9aaf1e23e2516079","input":"0x","nonce":"0x2aaac2","to":"0x30d82cc8a274716b616e858e8fa9d2e7c0fe111b","transactionIndex":"0x2a","value":"0xaeaa14152dfe1e","v":"0x26","r":"0xae89dca52ea390dbca8a00900a19a3dd1165a02c4585efa702777acdf3f87115","s":"0x448a134ad23ee92f3674b827374977336d0cad450c341e4e3972c6cd67b2ecf7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xa5e865c2964b73e24fede17686ef1df138230decd74fe82e4e39b1a0e0caf4d6","input":"0x","nonce":"0x9c65","to":"0xb29f1c22590d62d3b19eee1e10936263588cbf2b","transactionIndex":"0x2b","value":"0xb83e6e7e3cd000","v":"0x26","r":"0x1f479ee111fb49c8de1095073ab81fb8b048ddcccd272350f8ec4dd00a9ad22e","s":"0x2623cd338a54b042288111c855d915961c5941d1cca6489b46d46d010bc0ca98"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x52cd214a2ee626e53b235b9e87b443ab64c4dec4c45fe50a076d51df2ef6e12a","input":"0x","nonce":"0x2aaac3","to":"0x59ee98400e1456902ab7235d3af1e2fe08ccaf68","transactionIndex":"0x2c","value":"0x160138685419374","v":"0x26","r":"0x5a01a830c72bf2b2942c4f3b1a320117efd63d5e612edf4898db840cba35df0c","s":"0x3a5f1675cf8223cb3d72e226775d2ebe71c76cd0c07607cd747e9ef8edabe43f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x5db68a27e4f85fbf00dca00110b4e276a70472b09a253fa0b7c480def7554b7d","input":"0x","nonce":"0x9c66","to":"0x662978339d457e3c5de9ac99177d237cb577de7b","transactionIndex":"0x2d","value":"0x14c9782ba97f000","v":"0x26","r":"0xb244055b1b5e403b97f5f6e34e63b3726f8e5edfecd657895787157b6141e4fd","s":"0x4fab23be1914b18772b54af940f55c30dc6b944ab7c289381c48f2e1fc3164bc"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x3c00053e6b0cb4c2c82cc08df1de2886c527bceb37400af19d151c779b691ac2","input":"0x","nonce":"0x2aaac4","to":"0x1060044fb45772fdb205a7880bf10d98b3faa010","transactionIndex":"0x2e","value":"0x7203ddf4a7d9e58","v":"0x26","r":"0x61d38abdee2ec7eec8604f90900c110ecfb09c583433539ba09fc9987b6aa31e","s":"0x2d66a3034838de7aff0bda31653ee67698bde27a029a89c60f65ebc22a60739f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x5e8df31f2e5ca74f0b9cc072bcedd6b1aa9c890aae72747b386b35653bde4699","input":"0x","nonce":"0x9c67","to":"0x2b3c2f34d384a84a2db92861ef766d074d5dfe76","transactionIndex":"0x2f","value":"0xe036e48b422c00","v":"0x26","r":"0x147dce0b15914b2a5b9f3d6aceb62efab94a0fc3313bdfafc75456b65a7a850e","s":"0x5ca05bd2af4de11aa5faff07586b28d064365ef30ca9374e2746a68496139cb5"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0xf29b1ee3b69fd803e6a4336e1c2878a369c3b7d26a899502525a3e0a3988b1b1","input":"0x","nonce":"0x2aaac5","to":"0x49c059de3c341674028d3c4bd5438695423d673b","transactionIndex":"0x30","value":"0xae22078638a6d8","v":"0x25","r":"0xc6dc245f7ead2b2ea13a7c521e681733cfaed11e3f96d563cecc7f84689db1b1","s":"0x1a2837bfcc1b8eb5195d795fcdc6804c68058ea0328b42cb1ca3d90f897bc8be"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xefe29afe3225aae766b3698218cdc2ff8334871d2cd3e5a73331e8351a01cc3a","input":"0x","nonce":"0x9c68","to":"0x5ab9c59a3924a89fbeebfb614660ef5cb1dc9b27","transactionIndex":"0x31","value":"0xc510558adcf400","v":"0x25","r":"0x730a25ff42566dbd6acf5493b3dde8dc843b0954bf6474effe8a9ff36cf3a7f7","s":"0x29f4d2b0d9a042604db2082c527c42b0c12379e4b018630878caffedf5f1c8f6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x006a5d2207ef79083b3de8cc384fe4afcd78e28ff9603264eb487553292334d9","input":"0x","nonce":"0x2aaac6","to":"0xd97a422673e9f08c3a48c77fe2d880083745aba7","transactionIndex":"0x32","value":"0xae1e77264ee623","v":"0x26","r":"0x78e26d0ec880a8abbd47750dc27189756cbb45d097201de5524361b5dc1d6d4f","s":"0xe80f78a08e838680f1178a00c49750c6af34c0ff211c6472b22b5e65710b13b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8d2a63ae663da52ae3bbce4b0b193c806d920775cbfe78f1a9e2ce5fea730610","input":"0x","nonce":"0x9c69","to":"0x79e53465796e3ed6e4cfbb6108ed5dff81319a3c","transactionIndex":"0x33","value":"0xc96df5268c8400","v":"0x25","r":"0xbbce5c139e0cdfa424dd768acc1bceda22f89707e186987ea5cd652c541ff63","s":"0x3ae7cf7bcb887a14113e91574067abf179268084a6e4bfc64c97465fc7608532"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x8ec819436b821ad573f6a1fbed2a549cb8352c0035916fbfcb0cbbf007cd651c","input":"0x","nonce":"0x2aaac7","to":"0x1b9e602c4cac19e87b5faa3774414f54e362cc94","transactionIndex":"0x34","value":"0x160eb475460c2ef","v":"0x25","r":"0x9c67f12fd81232cc9b4f35fa39a5869efaf9290426a5b707e43373f2b78e726c","s":"0x14fff7e4a5d2fd57046c32273300541d87b1b8fdb89ca1f6e29083d925e29cdf"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x0f3cda751fbf72166bf419f483ef93fe40d4eceef994330d78bacf0ed1ac217e","input":"0x","nonce":"0x9c6a","to":"0x1c01da024f8674268128229b4486282e3091218e","transactionIndex":"0x35","value":"0xc3d6fc66994000","v":"0x26","r":"0xee64ba98405ef13c1046e20add36f83cffba6ec663217dcbe16cdef00866d781","s":"0x4353e70604753d5df5414b44949e6d5d3b0099ac9e908d3f386761a08dcc7681"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0x7b065e3308d58c1509f1af243bae91e6bf59b3af923b90089da3723d6ec0fd29","input":"0x","nonce":"0x2aaac8","to":"0x24702bcaba2cb34d081740605e57b1c0247fa668","transactionIndex":"0x36","value":"0xaf8e4871185ee8","v":"0x26","r":"0x6a3a5d089e0e69fac6406684950d7f8565ef20128d1bd864a2f885e70c45db67","s":"0x320c30498d3aed57d6549a4d89c96e5f35080177162396178a2c5bd7b465143f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x2a06d6d2c3af82096cb73bf602258342876c73b5072f7861b7f8abadafc28385","input":"0x","nonce":"0x9c6b","to":"0x6860e92acb529568c2c529db2e418ff9d39cb1fd","transactionIndex":"0x37","value":"0xb48cc1d8b16800","v":"0x26","r":"0x740010af0df82d950d2e77c857bf35b394573092008920faf64447a747eedcbc","s":"0x12b85f4e1dda634b7412f9707272036fdda220d5e1ffa867deda3231c1ff4945"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xea674fdde714fd979de3edf0f56aa9716b898ec8","gas":"0xc350","gasPrice":"0x4a817c800","hash":"0xfec8c03831f0a5cd907df0ba7e215ad87762b21771b1fef3ad324ad3825f14bb","input":"0x","nonce":"0x2aaac9","to":"0xba0d3ca997f8a5588dadbb7ce8000ca8ca8f79d7","transactionIndex":"0x38","value":"0x162ee6572dc409a","v":"0x26","r":"0x9f9c5920aa859759ab112d6eaf01c03bd4f8d7229224160d30446217f1ffa66a","s":"0x736877ecd30dbbc288f4f04e0e71da02f2cbf2abb52cc552af92d32fdd07089f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xca44697c858a5c081db2d5d27f0b89f30e8c4eaa92d9405cee3dcf674594753b","input":"0x","nonce":"0x9c6c","to":"0xd80e0dad2034dafbf1e56f9fbd9cf05e6d8f385e","transactionIndex":"0x39","value":"0xbfd66e5a367400","v":"0x26","r":"0x2b93cf57287f3ec365155ed3f511e4a653350e97ee21007de2f64f87821380a","s":"0x326f442a483cdfb9fceeefbefa7433bb2703cac555b583d22551f03b870cfb41"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x63cab64fa2fffa46dceed134e9731d0b54632002fe2b72d661fcb45a924242da","input":"0x","nonce":"0x9c6d","to":"0xb64b2a886be9164531a186a8031606361380c1a7","transactionIndex":"0x3a","value":"0xbfd66e5a367400","v":"0x26","r":"0x101551c907ae74aa1d49a3392d960b4101ce8a4ef7faf18c73cafee1fd81dbb9","s":"0x51d110c6cfd780fbfa6c02dad32bac18b734d7e7b4101efd3f0611d086fde41a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xec7220d4e4722386270508e0714bfdcffcd66475453ec1ced9c122bfe7fbc24c","input":"0x","nonce":"0x9c6e","to":"0x44b889082ca7cffa9f91107110754fe0abd07205","transactionIndex":"0x3b","value":"0xb5d019cc00e800","v":"0x26","r":"0x2e74c71007b9d74d9fa4de92f2c352275c0f8857883736d496388eb8acb2bc34","s":"0x63324a97cc0a246cc4901f9ada5ebf3c4a3c97b3b0697ae07b1370ca717cbea9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x33ded543bcc21f070d6e24f363bbfffffa8b8c39198493fc11252e5df2911e5c","input":"0x","nonce":"0x9c6f","to":"0x9257d8f0bde62f59f2d982ac4cd534e07d9dd345","transactionIndex":"0x3c","value":"0x17c1adfe0b47000","v":"0x25","r":"0xb1105d9b6f6a5382285f9ee15710d63bd484657b6f4563d1c40d83abdb401e12","s":"0x7ba86b5c18900e078abd585a6e63c55d9cfb51d093c4963fba6b5349f0183abd"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x6db11488011ec8e8820654a89b2f5e0e8b07e32973c5e2089f3d5f7065c7d181","input":"0x","nonce":"0x9c70","to":"0x5815bedf684599205589c23760509fa9c38a4703","transactionIndex":"0x3d","value":"0xbfd66e5a367400","v":"0x25","r":"0xadcde1c993ef446a648fe2eb469423418a993aea2315aef7db0040e703aa0e48","s":"0x695fcb0eed75e2fa344b896b0fd5e1ea7e1d2ddab9907a15c1f0d6cd48f29a49"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x30597ea097b887c60351a77a1efc12cd4a4fd2ffb7aa49564644cf43b8d0db9b","input":"0x","nonce":"0x9c71","to":"0x890910ab2c8f838de49a882235c1abb73e79a94b","transactionIndex":"0x3e","value":"0xd10ec777941000","v":"0x26","r":"0xb7669d6396e8abcbced7c20f898446ea3ce66ab6eef939f96dc104881d2ba4a9","s":"0x16a4f13c73e5c0f4885951413d683d59b8f209067e16b4c645814dfc60081ed0"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x3668733911bc9d75cd2ae0f7ec09c7f4f8a5cf979b57d44e718e92a20182358d","input":"0x","nonce":"0x9c72","to":"0x1fbb1f26b26379d9cf4a3fd152df619bc61aba0c","transactionIndex":"0x3f","value":"0xbfd66e5a367400","v":"0x26","r":"0xff3292258ac91736001885ceff8c7ed619af300f91e6abfe4e4386baad893fd1","s":"0x66113adf45b41a6f34cd895b3aef90c8d12be07e763abb0bb23d90c3768fa4b4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x5357798120a3efcd659e5c3b6a075bb927aee3cc1d2bede1441bc46717fffeae","input":"0x","nonce":"0x9c73","to":"0x1e3b979311a69a5e4aaf257d2887b2340b23e5ed","transactionIndex":"0x40","value":"0xbca080a4a2e400","v":"0x26","r":"0xdf1b455d46909ed9ad17a630f0bbe1ccbbaf2c6c67639d2235fb4b5f8516f3de","s":"0x5b848302ed3867c09bf756859d72371154295e0ede66432b2e56adbae7e2c824"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4b02dc4da6b4c08222040ae4f3e1a79c0f8fc12f84134161caf188119c82a775","input":"0x","nonce":"0x9c74","to":"0x52f7aaf6429f28359c594831dd720906e9822aa0","transactionIndex":"0x41","value":"0xed90cd1676b800","v":"0x25","r":"0x3c2ead1b59d5090c0c671de8cfc2e68b1df523666f308eb1bce172b4aaaf8189","s":"0x168dd2296d99af982ee467b214d5fbdfb5d3bda5833e98d4a3c2508938a605bc"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9d2338fb32b44f71d384510790c1523342b00dce554f089cdc1b76c11cbc2ba6","input":"0x","nonce":"0x9c75","to":"0x5ce8433eb2b8411bd505ee4be968751aa8f3748f","transactionIndex":"0x42","value":"0xe7962d1595e000","v":"0x26","r":"0xf71aab079829b5d26ec7e66ac88a068bf212c5f3c21cfb9fc56adb18429787e8","s":"0x28804ef7db35b88adc0b0d5943e7923a92b21e9df575a5214859f94d085dd4b3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xaa8961638349e54a3cfe762e88df9a0c81801083ae67dd8da1594d59c2e7dacc","input":"0x","nonce":"0x9c76","to":"0xbb585a66faf023a157067aa4a5b9d704945686b4","transactionIndex":"0x43","value":"0xbe0d6ff05a3800","v":"0x26","r":"0xc0e6c5572dfab3dabf6ff6773f20dc1d6088390e4a8aabe2673ee582d7aacab1","s":"0x28f11928e0b87f0fbdc189ba1098c0c2a3935c73955658f28c68772534cc1cb6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4ddba646404a84ce65c8566f72c52019faa62e3daee934bba7ed65dba0344d96","input":"0x","nonce":"0x9c77","to":"0xfcf8483d73472d9fec2c3daf98b05618fc5f659d","transactionIndex":"0x44","value":"0xbfd66e5a367400","v":"0x25","r":"0xaeaa50298fe64ddc28ca9e4e3c292bd7f31acccbbaa8e83a90e26f0d3e5aa826","s":"0x136c478f5a0534dc5aba89bbb597f65a64d54747560bd56e00a7d2ec79b88b1f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe3dc71466400ca4406b73dc9d37360752b5399949e758004c5898c6c8dbd19a9","input":"0x","nonce":"0x9c78","to":"0xecfee0a3eee9ba6daa6ac29e9c0cc18ac4302f5f","transactionIndex":"0x45","value":"0xc3d6fc66994000","v":"0x25","r":"0xecc5896a0b0cd9dd48efe7c4d014be27c6598205e89a10b803a0f744fa9e9618","s":"0x6190c56db4119fc23fa85ab9f2932d0634682dc472715fbd07919c4dda06ecff"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8408dd887fd59a64e665de856b9815f4b020a6721210a13be19eb55c5c21eead","input":"0x","nonce":"0x9c79","to":"0xc52478f306bc7f45ca93f26ec27b03e03eac7c45","transactionIndex":"0x46","value":"0x2a7700844a13400","v":"0x26","r":"0xcf05b89ee3b870440ee0dcc5810ba8818e43e5eaf300f0d078af2579871177cd","s":"0x6c2ff2f96d8239a18b581efbb38da45fa648c27a0f8a709f20ca017a0121c43e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x0c45616865e3111fce61c7f9100d8f8a76ed13ca137b9a3c5b44402c8e82ac50","input":"0x","nonce":"0x9c7a","to":"0x9b49fb099165fb5eb966d2999e04bd3f6f175bb0","transactionIndex":"0x47","value":"0x6b7f99b36c8e400","v":"0x25","r":"0x8c9021e0e864d0e874386aa25fa7dfa2316077327f3672dab7e7b5c343af47a1","s":"0x46f028b207e2dc03a5f0116b07e4e721abad7379e8775c861fa1ef5d25d84b1"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x0a39cb1bbdd38b2a94379ee9b22fc14c8f4d3374c49077bab4cc48ba9779a02e","input":"0x","nonce":"0x9c7b","to":"0x25b672142b7e4f0d28cdacaf94caf4f4ea34c09d","transactionIndex":"0x48","value":"0x3b6432fb1c31800","v":"0x26","r":"0xb9be3c1bb492cdb18d6d50899a3adc0ae0f332584eae98e1049cf3b1096fcda9","s":"0x74bf6457ca137554ddfa6fe9a86d0474bfadd0c83890d7feb226cd79c7ae0de3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x95208af935d245e659f146820af71ff0a7fbfb353c4fa32823e8cdf4062e6dd8","input":"0x","nonce":"0x9c7c","to":"0x55aea382d3f06b0591a12a1b0dfcae08d6a5903d","transactionIndex":"0x49","value":"0xb26646c5657000","v":"0x26","r":"0xf622164cc9bd21c57f1417089e45fb64e589f32663db953cd8581f7d51acbe7","s":"0x10633d8c1ed7597f94e3ffef7d2cea86b1961193cc89e00275ed99fa054e15be"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xad3b5298c4aa0a1ac2d3c5d680214626d69887d3127d5883cf306f02604d9127","input":"0x","nonce":"0x9c7d","to":"0x493c979945440205866ed35bd7df2284cc5e8aef","transactionIndex":"0x4a","value":"0xb81dc0e359cc00","v":"0x25","r":"0xa82e22f5756a82153ae1c457cf16f74f49f42d07a585877922462deb4409e394","s":"0x2b7bcc0575ad15ae9c6bd8b61b62548bb9e4a8aab41c67dd95a7fdda4b117934"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7daecd2bbfc31817012c988b1325deb998bebdf643a3aeeeadf302a534227f24","input":"0x","nonce":"0x9c7e","to":"0xfd47827a6bff38abdc3fcacb145ccf60326ffd1b","transactionIndex":"0x4b","value":"0x17c1adfe0b47000","v":"0x25","r":"0xeb65fe7953e57784d2607de0add1aad78fe5365ba4eb6ba323f0d28550095440","s":"0x5c04a96968949b3dbdd1ec1e7bd83b1f186cc96b5ae3a394cea9adf6cd53d507"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xdf5b469afc9a6ca28bc7be614fe46726bdd93b069cdebc856f52deff0e32f8f4","input":"0x","nonce":"0x9c7f","to":"0x416e269cc2bf8f9cf56cd70038c0714bb2fb2223","transactionIndex":"0x4c","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x51efedde07c47ae99371c25ae1474c669cc52b100f0ffda4be4936e2892b9331","s":"0x53f6f325d1da57dad6ad2cdd961fc67dfce395372dd18a7774337138b1e2dd9f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7c9ef0fb0cb6c77a338310f8fa497f2af24dc03bc9e05fd5946c2096603127d9","input":"0x","nonce":"0x9c80","to":"0x6fda165e0d011eaa77f70e24bf515abf4338ea21","transactionIndex":"0x4d","value":"0xb2664919715400","v":"0x26","r":"0x8393b13618da6fa0a79851675d230d53f5e32db404f80ecbd319049cac7ebb7d","s":"0x5ffe6c3a035ac49c22e89b547030c2384896b6cf09a90a7de67114b15ec81f6d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8a6d742d1266639c3ff41ad8424d8e5632ed7cfda5795933f05b9c117868a9d9","input":"0x","nonce":"0x9c81","to":"0x2e4689b51bf43fdaf874f3baaec1b750ad15f45d","transactionIndex":"0x4e","value":"0x27ac67bbdac0400","v":"0x25","r":"0xd22d20eded3b91d1842bed217d4e6dcec8c1b560114963d8b2eee281b4686bb8","s":"0x15e26c42245398df1b6f64927c88081a9e692e18358e9d73b6f3c28da44dca63"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbae0db1e3b4f15e0dbeadeff006bc099f72c33cb642077d1825ec9e3966ec572","input":"0x","nonce":"0x9c82","to":"0xbd822e7b7db725c3bbfe7576e24d3c0354497981","transactionIndex":"0x4f","value":"0x17c1adfe0b47000","v":"0x26","r":"0xc0c0f50432bc3e6d02e68909742a0a344cc4f593b548915d5382f4a0899bd868","s":"0x76a0db87a98089f7d29830934bf1a42f6ba3e4ba558c86768f2e08ef8ba808f0"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x83b64144c19654f052676ba7c78771f7121d933fad2ae0be8e950d5b99e16f73","input":"0x","nonce":"0x9c83","to":"0x2d322adbb9984eb45d07e5c219e325099420183a","transactionIndex":"0x50","value":"0xb20840bd382800","v":"0x26","r":"0x3df40d99f3f47dd3b6d1be21e466f765d7b3f17cc8782b07542c7ceed3408057","s":"0xc0db1dee0f544135878b3fc6767b6034d3f6dccdb113a1195a781f234f91e6d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x2e630e16782c85a6046333c8d046004c60905a1509e56a9a3ef8d2a87ff39340","input":"0x","nonce":"0x9c84","to":"0xdba59dd839fb2d3535802e6d187b72c6476be686","transactionIndex":"0x51","value":"0x1203212e37ba400","v":"0x26","r":"0xd15f940513577217deb4921cd4875b55e5c236135c1dee2a161bd13bf489d4cc","s":"0x53ace39ee25fdf63c746c6084ef0b2e53b8f9a10b364704169453760a0e28124"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x95268414532c05cfed0aef91909f68b4416251cd21999e47857561557a33eb08","input":"0x","nonce":"0x9c85","to":"0x686dfcf430777442606254a0e36f4dad68ac9292","transactionIndex":"0x52","value":"0x360f3a05f77a400","v":"0x25","r":"0xcfdcb39b9d026e5265bf2373fbefc27633c97dd65865b0131ea353d971f78c7d","s":"0x5ee707a379ae0b4e7c2566a7034de41cc9e7fa6879db31498df02763a1217204"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb49e13713a8ccee95c78b26bdbc900872de90608b44789ef721910ec74b31f12","input":"0x","nonce":"0x9c86","to":"0x6c429bc1c51930f2c4b5e02dcf7c01e5fbab1df7","transactionIndex":"0x53","value":"0xb35229ba10b000","v":"0x25","r":"0x8afe1f6dd3247bbd388f02038524ae92c93f8a418e5e02ca6d4a0d1ef29fde49","s":"0x257449addbe86c06d81f31057f2a4c39a954110d90873ca184d4eccbbcd7776b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb109ba1e2415021fe64320fdb1ac4a130cdf38f1f59921f68aade8f47f23db3a","input":"0x","nonce":"0x9c87","to":"0x9b128e46a15545ef9656806155f940e3466308c5","transactionIndex":"0x54","value":"0xbe31ef6ebf4c00","v":"0x25","r":"0x5079c8212f9291b36a1d9052e6c04412925770ec63828dfdc07c7c17a3a90cef","s":"0x38ead0e006a6e4a7a9e4fe58710cb5b08d388d8fb3fda9195ac799c0e2ebdcb3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xcbb6fab536358150be07d80ebb21d2ae0cce0c8315276b837667ff8ba1c42d54","input":"0x","nonce":"0x9c88","to":"0x58cded315eb642a8806a0327a505dd04ab3e5774","transactionIndex":"0x55","value":"0xc4a234146d6000","v":"0x26","r":"0xde626c5dae253d07b126b37b53e43a2280c1de5fe5bab9dcd335f232dd1035be","s":"0x3beae86e6665767e70196ca3442a8119194150d855b7a5d710c22bbb2f45b8db"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe6e296227aff21dcee63357daaee930a262b8d3de5272889774a3ccc78bb09f5","input":"0x","nonce":"0x9c89","to":"0x55e425eb13f8f9d3ee84da9ef721223ce595f427","transactionIndex":"0x56","value":"0xc1a2d5e17dac00","v":"0x25","r":"0xe6fd8d422bd3fb8ce8c3c2f45f1bcd830ace8c3d7d583eb46ca3e2e319e5fa0d","s":"0x42f037accaf12c030392819efebc9614ea8ff3b950f6b35b85aebc1ab8b5dfa8"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbf934a74e1a949bf6630dbae24c0a5e1ac655f798914bbba2b276d756a8703ef","input":"0x","nonce":"0x9c8a","to":"0xa31fb325f638ce6f900d07159d036079ae7a1888","transactionIndex":"0x57","value":"0x17c1adfe0b47000","v":"0x26","r":"0x3afdbe081ebc912730ed377fed0917bf3a7512c6d12591262e02aaef583b15ce","s":"0x2d7f01ccc1a049ec0fc197980acc3dc80accd133edf4b1c2125c9a506a78c412"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4f895c2f7db9d7977aa003a04448ef070b07e558366c9ef7fc4101f4c5d00f63","input":"0x","nonce":"0x9c8b","to":"0xb7cc039691cb2c51b3202dcec8833f7294adfe54","transactionIndex":"0x58","value":"0x15f98da41d1c800","v":"0x26","r":"0x1520c62dbc3689f64d70fee49ab384a7c98663396618135783dceced04949218","s":"0x5b833f924dd7a046400b82ea2954bec85b63ba7574e46c5301024331d5417150"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x89ef049ce110fd411ae6c8c54e2533d319187e595f5f3945122eaba4e9b427a0","input":"0x","nonce":"0x9c8c","to":"0x2687cd65def8af50e18390199f6e97b0ba72dc2d","transactionIndex":"0x59","value":"0xe4101efecde800","v":"0x26","r":"0xd3961e8f1a2e38c8d75418233e314019eebea31fd7a8cec038a9ab5b7255f857","s":"0x3b5747f6c422427c2353309982c9a625d5217d3d76e0990afc4cabf871cf39f9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x1f07df926060bc3230dff2458fc219b979f6477f3776d427da441d46bce8f95e","input":"0x","nonce":"0x9c8d","to":"0xf164395df8e000dd4a491be5111952280b2b223c","transactionIndex":"0x5a","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x38f3e7926c67197215a0cfead5022d8868c8b63d8845ac01970037b28f875f12","s":"0x429a6f8ee5d836ceee89dca0f7b0f320c46f565564ca751e14016ff25808fcda"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe335af1cfe6732b34515d2cabf4a00495620726620352378864ad80734e0570d","input":"0x","nonce":"0x9c8e","to":"0x7610640b90b17452501bf94fd8e8f37bc0adfe62","transactionIndex":"0x5b","value":"0x15e55d178169000","v":"0x25","r":"0x13d75120136fa3d8e604aad3b9de1ce1817508db8018677a982dc4e141e18f6f","s":"0x688869e08f498ba4aa3a701c6c15136e96f09aa2ce93e0b8e932074044ac95c8"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x541eb3a6b744879dc009ecc18f38854f587a48fc6c9d27bd71ecc05808a373a8","input":"0x","nonce":"0x9c8f","to":"0xb1a599d720d092dd00b53994ecbb30cf765dec36","transactionIndex":"0x5c","value":"0xc2542436fd6800","v":"0x25","r":"0x9df012b2523eb9f05084b0b215d65e4491afc3f1e526d77e08b35944b85d2cf","s":"0x644b5c2b1171871ad1f7b9a9a4d4273a08b26897f3cc45b9e325e2be48e73044"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xcb64bf354dc6953f8fefd125c52c6a28df664607d8705b51e412de57d61fc782","input":"0x","nonce":"0x9c90","to":"0x1559563f25677581d36d4e624473cbbf73e15180","transactionIndex":"0x5d","value":"0xc3d6fc66994000","v":"0x26","r":"0xad7031b60b01849774d08381af4a65a3dffde85eac5df96040f008bbc950cb6b","s":"0x44fb8a0a9b3d3f26d548ccaa209ddd3b24d1ba549bf60719e451ba780e0bfb29"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb5f62b55faaea308eb18733a19556e2730ec3e9f18f8ac6be15af6e46899837e","input":"0x","nonce":"0x9c91","to":"0x4d314bab18394b57c359639c876ec5ec6a377fb3","transactionIndex":"0x5e","value":"0x5b414595a77c000","v":"0x25","r":"0xf80db29fc81d5d80c120b363f2321b092de2e48bd2dd254a2de0c6e6b33bfea4","s":"0xf7fdabddeb455a59822caa8242f006b1f39235c8bea947fa1a98f04d15ab37e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb7e77c3bc9ffce6c6657282147030007578f15fb0c2b68bf46946e04f381a22b","input":"0x","nonce":"0x9c92","to":"0xbe007fabe0abc3da8b05e1d6ea261056678b8a2b","transactionIndex":"0x5f","value":"0xd10ec777941000","v":"0x26","r":"0xedd33b9aafdbf64b01298fcb08376dec064cfe65d31ae2707d8ba14774b85bca","s":"0x57caf3b2edff6170a338583b5667f1bc83109bf89b774eb10c5ee19a35fcc81d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9de4222cf2a3f10d6fca1729ac7d1669ca981fa4a3a2da22cbfbd98b394d6707","input":"0x","nonce":"0x9c93","to":"0xc3f8a457c2653306e03141fe75a9877493dd7343","transactionIndex":"0x60","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x918dbce8ebfafe208ff78df2710e99f3c0f2112a60027787a0af8fb495d8454d","s":"0x52fa55ca6bbc1cd081edc5b99f4ff131c6166ef9c1752154b59683ba3235f4de"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xad0546ceafd2779a6749d6e2501eb69e291bafb0830dce469eec76cfea70a773","input":"0x","nonce":"0x9c94","to":"0xb769832eb660e512e07258bcc36a0dcb76efac35","transactionIndex":"0x61","value":"0xbfd66e5a367400","v":"0x26","r":"0xa453717ea557e0951f5ed4d1b1afbb9887cf4a5249782ab9cdca467d88e3b0ec","s":"0x5cb9307f135e689c1c5d6573d7f1eaa7517ebbf398673c5fd853716c7dc0092a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc9bbcc2bbf50c6222cf213528617fb13a490ee05581aa68313d206ea1519b2df","input":"0x","nonce":"0x9c95","to":"0x571eec232518d5bb640abaabb4a0dc90a9923fb6","transactionIndex":"0x62","value":"0x2992f07c93bc400","v":"0x25","r":"0xa04da77da585efc49ff19dfbb002379f48bbdc76fa7b7ca92b49c80eb89b951","s":"0xb1dcad3506d3095da8256d6516aeb0f08d28342e5721f85634ef796627b2a7c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x39ab5625f2e3063484f3b8576f4f3e14c5002341f9c287df9b438e5d8fbd0060","input":"0x","nonce":"0x9c96","to":"0x9d34d6f0b5632fe1a5103eff1b051bcedb4ff55f","transactionIndex":"0x63","value":"0x14c9782ba97f000","v":"0x26","r":"0xe8e7a27dcb4295c3177c50a8516e72285bb27f4700508638060009e22efcafb0","s":"0x39733b86274675763eb8f8ea5015d6f37f220fe3bb86ba088428e9c639c4861c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x160c2bd753685879936c491123d8ed1b6ddfb6c24fac1957a7d2ffc83228ff90","input":"0x","nonce":"0x9c97","to":"0xfe1d3a10df8ec413d60ddbc5f864372785b15a0a","transactionIndex":"0x64","value":"0xc3d6fc66994000","v":"0x25","r":"0xf4d4a11d1ac2380662544373f650b4501357b6938f46a8cc511498e6f9af2fc6","s":"0x5d79de1db6e91984fb9fd0afd231d9c218fca8565746b8bf562c5275c82633e2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbc7636268fa7d2e2dbb0a55f27891941b8175ea87c0ddbe81e6c2af867279ec3","input":"0x","nonce":"0x9c98","to":"0xcd865711992c4ec65c6a160b53c89a7d6ac6ae7f","transactionIndex":"0x65","value":"0x31a272005eb3c00","v":"0x26","r":"0xecd2574b39a2a580e8d3d1471d76001cb926af5e644b6ce99625d5509b534471","s":"0x4e5ecf8ebdc96a1aacefee24f842f45d3a0e26104c75848c3a1a4eba1b1b97e2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x937bbde3396f50bd58b228acdc6075099afe6abf9e4f1cf6ca6bcd70f5768f80","input":"0x","nonce":"0x9c99","to":"0x1a31a3cfd3572351a03e7b28a2c31560a918952a","transactionIndex":"0x66","value":"0xbfd66e5a367400","v":"0x26","r":"0xe26c33c48e8d86df5f1f518bf3d1b68b56c589afc6874836b2968881708b980e","s":"0x1979c6cf8c46b224b55e0018d1fafdf7fe8d8623cdd50b648a83b019ed0806e3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb03550a7b57e201b1a2fd1afb376c66e6bfe3425dd948b89d439849939899e57","input":"0x","nonce":"0x9c9a","to":"0xcc2171d4de600277075fe130e0d36ffefa99b5e7","transactionIndex":"0x67","value":"0xcda1be8c933400","v":"0x26","r":"0x1bfd0376436155b78ebaf2124d9d0acaab4dc2067814529978235d03f8bb5ab2","s":"0x54204e780711ae4c4ae1b7456280fb14973308bbb75e830986430f1f291bc53d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x99e6d8bc383e246a6a7a294135344f16346b53eb9fc228da27aff8118e99d4f6","input":"0x","nonce":"0x9c9b","to":"0x92b264dfe333e5f73122225c41cd73db8cff9337","transactionIndex":"0x68","value":"0x1cbed51d319ac00","v":"0x25","r":"0xbca9dda64074c1b01225e1a1bf5d2f6e54ac94ac9c014fa17987815f9dabf8a5","s":"0x7065c3e426f83910d6e4401c877c071ea94cc3e4393b2fa822d637ac83474348"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x1fe4bee972c52aaec26617e86fcf8758eebbbc98b952a8158247af139ed2b54a","input":"0x","nonce":"0x9c9c","to":"0x4ca85ac8e93bc77355db733f4111bb09c345091a","transactionIndex":"0x69","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x42eda4e90e20cba06037bc795dba4c76da79eb3e2bfdcb1bbc08287925816974","s":"0x5b10aa514cc2c1e3c517804ac123e38662ff1a1a67e47b840df5304c2a2af2f7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4597183867eeb0b2195d6d367dffa37c25aa46aff4436b7ba225bccbb3579c7c","input":"0x","nonce":"0x9c9d","to":"0x2b1f68abe6a29b3edd64ceb21ef29158e52590c0","transactionIndex":"0x6a","value":"0xc1da8171cc3000","v":"0x25","r":"0xb7b87b68455e7cb5eb9db18806aba977f6f939ac144bd569da37395d806900e9","s":"0x502a37059a0e7b96840e290676ce5942ecdb1a1aa25757fdd56d7f66976ebdc3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x0bed04a6609b66a5d8b6bd6ce40ef5325bc4c696aeb37776d4d83ec8e7ecd961","input":"0x","nonce":"0x9c9e","to":"0x7924e5578215cdff5f181b64da2927923af16260","transactionIndex":"0x6b","value":"0x14c9782ba97f000","v":"0x26","r":"0x572a3bc3e383ef8bff21c48de4073a88500771cb36b898fcb89dd84522def105","s":"0x7ad3f5a4329166326649743f6ed25a3b2f2556464e767dcb4c36dd837b059ce7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x73c10857457e3f5c45895d9a77fb438b3421eb7c28c26789e2c9cf8151fb9cac","input":"0x","nonce":"0x9c9f","to":"0x0253cd09335b8df37e1c5473ec99a6d70eec1766","transactionIndex":"0x6c","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x3275a10bbe9666457ea5afc4d59e675b0144f76b98663145addd4bc799105e6e","s":"0x5568e0f28186411f3d70bbc4270ab0ab61c08019948350a918390c0e281e241a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb9eddc2db16ff61f58001642ca2a51c8ec95ebfe9e5b036421370077137e193c","input":"0x","nonce":"0x9ca0","to":"0x7651952fcb8ffdf86aa45ba15cc5b17900e2a43b","transactionIndex":"0x6d","value":"0x126409b8e091000","v":"0x25","r":"0xa5e27911e785f9a70a759a769c71e0dca6cf6bbeae4f57c3bdffedfa1bf07fe9","s":"0x301d8a3cc91d42a32377755b8f59a3cb98f6cac73bf437a125a9532aa8d48769"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xd31e51cf1dd441bb59f560208a1f623c8c46be73b64e05d4a502c24966ae2ecb","input":"0x","nonce":"0x9ca1","to":"0xd4f6efba0e8afac5070e2f212ea2399890c661af","transactionIndex":"0x6e","value":"0xbfd66e5a367400","v":"0x25","r":"0x25a991a9e975affade3700b25c8f643e0f5b2da33c080884489c0e9d08de74c4","s":"0x3059552205c70ae443d68470007e1df0ab8590f3bf8c3176a81ca0450a3d4779"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8dffae13ab7104649563acd462619eff57ed1c6aceff9b89aae020d377502113","input":"0x","nonce":"0x9ca2","to":"0xd0d6468dce409bab6c90ac104e43cbde0683ec0b","transactionIndex":"0x6f","value":"0xc3d6fc66994000","v":"0x26","r":"0x5f96592f3e82ca7b68650642d9db0f1ad1224a26341408e757c298b0360e83fd","s":"0x312df679b162ea5c7eeed6295bdb93362a03ffb3dcfb74c5012f9acfffc2b071"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x909401b08022a35b0e6bd1efd6ef2b9c3e29e29884f4358ec1047998e06462eb","input":"0x","nonce":"0x9ca3","to":"0xba8a931df397f5821766d764dfc1123a12725866","transactionIndex":"0x70","value":"0xb2664919715400","v":"0x25","r":"0x8703033f77bec9be225e4edaf8fd4c0067d1e94b4842f039bd872dda4b4f44a8","s":"0x2895e3d128915a86b138eadb90a669facd73fc1c0050ac7ee7091d212de08ed4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x58c48a7b652c7382cf7193760b16fed729c000668c5692e0d1185b7486c221ad","input":"0x","nonce":"0x9ca4","to":"0x4ee1bcaef4fbefa28184325e6a9c4a57d6c5bc83","transactionIndex":"0x71","value":"0xbca080a4a2e400","v":"0x26","r":"0x1d306f27cd242ed559942c8dc28da48f0d3aaa37e99c3ff64ddb3f58a7107779","s":"0x106bd74d4b7249b7c410e289892066d91e0da4b63581e2e9f93f40471bf8ed2b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8360665b9a5b6abeed5d63621ce4a22578aa3eb27090429b676db106de0297c1","input":"0x","nonce":"0x9ca5","to":"0x5d6290be073fcf27fd1affd5f7703feef07f3d5d","transactionIndex":"0x72","value":"0xbfd66e5a367400","v":"0x25","r":"0x665380256bac009ca3bd65e34d8829b417e9ad73e5b9994c875472b374becc07","s":"0x707772a8bae78c26f22f8375f34d277b43db9a1be118c3772e9cddbe76c2e31"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xba38bc44a84a1be2d3ef1b104d7d24b9531951587d8801f55b7e71f442ab303e","input":"0x","nonce":"0x9ca6","to":"0xae0a808e2a772a4302cd78aea2ddc3ba526c6ad5","transactionIndex":"0x73","value":"0xc3d6fc66994000","v":"0x25","r":"0xba3fd5e06b4460de11a18ed944efe58d89175295594d7130a2dda4c73b5f9f39","s":"0x20531064f3c4b9dfb9d7c2e08dedfd4ec187b0525a1ed5e965d748cdc29eb5cc"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9333f58acf1bc7e92224104f057671998592a1909cd5b3acb9c12ee92b325f0d","input":"0x","nonce":"0x9ca7","to":"0xd482e0fc8213cb979aee9f86dd488da365019e5a","transactionIndex":"0x74","value":"0xbb0aad48175000","v":"0x26","r":"0xeeb8f6c5dee495a379c25a2e6f7be0f4779f48df8a112dda804af184f128260f","s":"0x52f8b6b9a51711a16437ccb853c95dff6099b00c7b30a04661b6ccd24e8f1146"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x34f3c1e3163f4b91dbb9a07eb139a6128e11638c944c688064601c1acd5b0500","input":"0x","nonce":"0x9ca8","to":"0x6254be074cd9a548455bf7b852b4c37b1bfe3833","transactionIndex":"0x75","value":"0xb3d90a82e2a800","v":"0x25","r":"0x3722cdea2984b42025e756114fa881d09cc234b1832bbfe5736f1a7c560b408b","s":"0x282ee2ac69f52de76906ce7586d6d1ae74173dbcd0a24a99f80315d414bfa74f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe50993d20c4c2173666f7407fe8a2d51dd4568f55f938427c7383cb528d7d9e6","input":"0x","nonce":"0x9ca9","to":"0xb6e4350b195042a6e2d614aaf2f55c2a250b5d4a","transactionIndex":"0x76","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x57542388d21ed1659704be1021983b6dd7e8c9969f8a3ae1bfb672088fd26955","s":"0x23163ad0713433796a3590e73d4f1c2975b55684e725506f806a25ee715e1f2d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x6bbf3a2ff8d375155554f2393fa9146a543d0a2d989ee879840c5210d6d8af9e","input":"0x","nonce":"0x9caa","to":"0xcf56e5ed5ae72a3073947495960fa7132e54b3df","transactionIndex":"0x77","value":"0xd3057bf2e29400","v":"0x25","r":"0x6b183cf8cd9beedbeb0a9c6d665f422c3e02fcad2e6034a0e5dcf4efb35110a7","s":"0x135376a5a6bc6d5f902c0111f612546166674dc9ca7872b971f7243d046b5415"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x3d3397dc7750bef59af42dbe05cc0012b6155779f36d18e07bdebbe9503d4886","input":"0x","nonce":"0x9cab","to":"0x8de0498a27ba339552efdd739e9feb820059dce6","transactionIndex":"0x78","value":"0xb63eec35f82c00","v":"0x26","r":"0x58ba1a512c9aba3d9f34b170c2e4e111432c6008a553422484a762d4cf0ebecd","s":"0x12544bf2a888c125beb2a9301163294e3a7c041d3d2b109fcb9e4dc809d68614"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9ab6b22db7be7a6995a8c62b3b00a59ea441f62c56b9b3520a431f3c4555643d","input":"0x","nonce":"0x9cac","to":"0xe0344823f21a2e00f17a0afc808fd4c6e002750d","transactionIndex":"0x79","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x86cee2916626102013acf7ca7de60bac8d37b534664066a4a077454608527efe","s":"0x4012fa62f4bb99242ebca7a0bd15cbd8fcb3eaa3b23f7dc5a79900bb8cef9049"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x527d15925e495fe515e1e3367616577d7ec0744094d35bd9546827c33c0b5530","input":"0x","nonce":"0x9cad","to":"0xd4bba6144da7295055fb1b1d1dbec86da8b4d21c","transactionIndex":"0x7a","value":"0xea7b427a49ac00","v":"0x25","r":"0x67b6019a6422ed9ff8560ba76a46df47a0838139ad2db752682738753b6e84a0","s":"0x2711600ac97e3f070f9de07544b188b5875d14467025e981c02cff15bce86fd4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4d3d5cbc698fce23a2ffa4eda0e151bc0e6ae5d98629c9b4b77234fa22a30a5e","input":"0x","nonce":"0x9cae","to":"0x54801393c02e07ed8c5aad855dfb1cbfc8c9a9ef","transactionIndex":"0x7b","value":"0xc3d6fc66994000","v":"0x26","r":"0x35daeb020d4dfd39721785c2d28591e902e53ae245eed0ecdbc25ac7472528e1","s":"0x28d959e8608cceaf0342bf71bf0332bab49c67ba9b76597a136f79caf1a05492"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xd0851fd1a3aafcd50ead7aa1e6dff1c9d2cd09ff4392264718d6d1b8ec027a26","input":"0x","nonce":"0x9caf","to":"0x2fb6665a77c8c6935aae38cc8cd63c79f4978f23","transactionIndex":"0x7c","value":"0x17c1adfe0b47000","v":"0x25","r":"0x9309016ed84c7aae11d7460539ef350906f7b3fe48a92be2571c9773873c86f2","s":"0x1b6dd64e9dc8701c39a17f705b04c81ff8ab46607bdc84c77c212cb293b74617"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x122580b9bf98702f5d63e9541a54dddfc3baca0e9856e2bdf8dcb0cef8209992","input":"0x","nonce":"0x9cb0","to":"0xe7b54f8793c8bb9b29e492ad6e4f8d6d5f5be164","transactionIndex":"0x7d","value":"0xc3d6fc66994000","v":"0x25","r":"0x2c2e7929bdecadb75e1bf53add116c441a11cdf535566f37a6adbdde5dfce143","s":"0x1c684ac9766eac8558f7064e346433bb80baacecf336938c4136e0558efaef64"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xae661bbfc2e410190d8a8adf3af712457502e0700dda717a2a22e7adf94317ed","input":"0x","nonce":"0x9cb1","to":"0xe05ccc1b7a0313fdcb79ff3eb0305e91d5c487a0","transactionIndex":"0x7e","value":"0x2b480f427177c00","v":"0x25","r":"0x6ebe9fd04f7d8a7ed086be355c8385ce99094cd822a42f4b710926c4a04c84e9","s":"0x38cd85fb70f0a9367ced78bf6eed59128ca755f8ab7d3c9a2766b85c6e81cd8a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x399efc768c069ba010a4dd3d6a522e4e215b4e6b183b82adb73bdda2b6ec52ce","input":"0x","nonce":"0x9cb2","to":"0x349510b999a5fda5db4780e2aaead90d1e5ccc50","transactionIndex":"0x7f","value":"0x2f835bfc168e000","v":"0x25","r":"0xd40338eba03278577e0952a6c4323cc678b0953e6a4a6915263562238279fddf","s":"0x180f76e6f04cf4ef6fd8d5cd76aeb38b190641ad858c28e3431b9844623a4c3a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x03c315f23d10b806d1ee3903c4c0ab5bc2648f81489f37e657134a0ebda47a36","input":"0x","nonce":"0x9cb3","to":"0x47871f0665a2e91aba71c73e13de5b11155c8cbe","transactionIndex":"0x80","value":"0x20aa4f2aaf22800","v":"0x25","r":"0x52d4ed029392502b17d06324a92946efc40933fcbf1a36e72ec7671daf11f35","s":"0x4b3b4a12743f4ada37757d363da59bf27c4eb9d923b36b379dd1dd47c3f13e05"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xaf5ba7e04bb64cb7b2f61d81261c6172507d26b18b42b30f49d43affbd8d23d8","input":"0x","nonce":"0x9cb4","to":"0xce260bc65ff5f6ea95efb3dcd5620f4591f5dec1","transactionIndex":"0x81","value":"0xc1da8171cc3000","v":"0x25","r":"0x8f60322824210fdc76f2c9f2d83b64a650ba21788f256da54dffb1278c1ab1c1","s":"0x298ab7cf17efc560d209a520a600f554d5971fa77238fe255b421ff187948e86"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x660d9a64e3c96e78622f870ec011091c3c7fd9fef664ec5573d65d2c0f89f3ef","input":"0x","nonce":"0x9cb5","to":"0x1749deae94e5fa3c9504ab2849168f335c4fffa7","transactionIndex":"0x82","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xa83f70d6054ac06aadb460225c6465d612bbcbc956d022e17b50bcc730f6012d","s":"0x1a66523c602ebe0a5f5bd0ef9c918155dfc17e32bab33f1182f19194f84de284"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc2cf524f088f4496a5731d36a0b6929759153ca770ebb77ec291d9ff27441d3c","input":"0x","nonce":"0x9cb6","to":"0x3bfe9da8c04e53b387196d30ef1b635ff6264bd0","transactionIndex":"0x83","value":"0xcc4e706bbfa800","v":"0x25","r":"0xf0cc09d24d5fdb64e9cb286e1c3ccbf3f110b3d4e110b192d9de42407d692600","s":"0x240ddd910ab5cc583ee199e92c4713db2f7dc9490b8d6d1b9333085b539d7526"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x744b645bf4ab7711762b1ec7a7eff192203f6944efb392038b4dc9c3a27c14f0","input":"0x","nonce":"0x9cb7","to":"0xf2ede79c5a212432ee3e966386e5a01611c79363","transactionIndex":"0x84","value":"0x2a606995ecbc400","v":"0x25","r":"0xf9e9b8cc015f0a903af9981191feb4b5ffccee7356367c20f4bb0d460cb6ece6","s":"0x1d08280e485e095bcc78f50986630abda306524446b1f43c9b3b4a4e5b4703a7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xd068bf23a5d7c792e8b4ce6eeb5b19a6a12dfe5fa3e47bb264d8ea4c6149afc8","input":"0x","nonce":"0x9cb8","to":"0x84d12193cd5f827ac842f98c9a3ea8b5f01e6542","transactionIndex":"0x85","value":"0xc649ac5b14c400","v":"0x25","r":"0x67f9035e3dc6399c195e29e1cca206d6271e8817b78bc7ce8045e67fc21ba952","s":"0x74e7a387695d3a38eb1b213d5a4cad1e656af4a1a2e08d0cabc246b633f630d6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7f61182b71012789a75dcc019c21907390108a669ce7ba70f95c7174970a1f5a","input":"0x","nonce":"0x9cb9","to":"0xda62fa9c85567310844408d4ed1af18b971b3d61","transactionIndex":"0x86","value":"0x14c9782ba97f000","v":"0x26","r":"0x82524dfc74b9f7ba43e164dee5ef1513e2ca9173e6813c7c26b08f0bb4137f7","s":"0x12ca085a43e5508497f9e8bd8c35307ebbffb3ca267a9cef2edb1c514419993e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x61fed8d0284933c47e3d83dbf9809e94917087ef5db0343773a53d5dcc494b57","input":"0x","nonce":"0x9cba","to":"0xed63003b5b433e274b225e2669815941ec23a320","transactionIndex":"0x87","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x988fbda5dd633ce6c9848077b0defc7da98497328dadde7e836cab18d272fdd4","s":"0xa3d1c80b43be4d933156820e7e0eee5e720ddfbd96bd59f2f54a6fd7f2d2072"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc1ea753339019b98d452c9492e3df47ab9363513095d5f929daad42417b6f825","input":"0x","nonce":"0x9cbb","to":"0xda46a91896cd97e7c94924b8ddc2715554d1ea7c","transactionIndex":"0x88","value":"0xbe199b367fe000","v":"0x25","r":"0x7c5c5f220ebea15518f78e36be2d2a170d5a4a356a42cc562772e601deb14b6","s":"0x59217c69942162de414af122a0de01a1ec00bc9c03246c0530de3e15db91a73"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x999eb2b04f57d490da06645c9acb2f0cbea22ce3ab327f4a9636c61ecda5d598","input":"0x","nonce":"0x9cbc","to":"0x4df8fec170166dafb5600350c9947aa999647934","transactionIndex":"0x89","value":"0xc069c2969eb000","v":"0x25","r":"0xa8a89f485bc8d8f53856cc044d795424e4bfe33d5a5f840b1c3f37ec7ebef4b","s":"0x39e73c09306b3f47a8166abe6164f305ca88999df5b02b7cd0527849beacf002"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc711ac0e81667190af33ef264b76d3c7770699347543ff69cae7d9c6175d74da","input":"0x","nonce":"0x9cbd","to":"0xbd4f0ba5c8584b9ec978745f9a03db5784a08527","transactionIndex":"0x8a","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x6ecd6d48196853285f57103c76cc997c26b3ddfe19e26f3880565459d16ac5ed","s":"0x663e7698cf7d5bafb65cd1f2f8626758fccc1a30c47c9a80ff5cc37b5ae905b1"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xa04a3aaf82b481df7a09c053a6b8de58997f9f273b4990939ba0735c62c773d5","input":"0x","nonce":"0x9cbe","to":"0xead137868089c6354d4bd1339e8320729d68b57b","transactionIndex":"0x8b","value":"0xc3d6fc66994000","v":"0x26","r":"0xc6208f84be9cdff67d2556cc6b410165fbb6882994d7d617dda95254ed020260","s":"0x30da6485220fb714888736a77bacb0ac87137748bfee53a901dd49f88e40dad"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x95608d38b8b00bc8b9426727f856eadcc7e705f784f4d3f48ae29dab2e888527","input":"0x","nonce":"0x9cbf","to":"0x39647d3170161f7d338914206f6d58d13798b505","transactionIndex":"0x8c","value":"0xbfd66e5a367400","v":"0x26","r":"0x1a46be403068d8143d02a852fabdfa3770f18c800a929fa4a01d565b34d28657","s":"0x1b136a2d0eea449ad8c746343c5d1d847d093665064f6613825df371dd4773f5"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x00e2d1d6e88dace86c8437fd412997d09a16ed8e782dd86e857c9fea42e22ab4","input":"0x","nonce":"0x9cc0","to":"0xac48389a295b51028822a6962ac5b426bc452a34","transactionIndex":"0x8d","value":"0xc3d6fc66994000","v":"0x25","r":"0x738ef925228e34f3c3d84f1bf731f406fdf88281f2ee392ab86373327ea3ed1b","s":"0x5600337bf5efaf09ff42730158087f1180c40c36fd2961ed2c94ff45e38725b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x2ffbee15977867989ab6023d600214a10810269c270c1a26c758aa3a152af8b4","input":"0x","nonce":"0x9cc1","to":"0x257a3600c0e58fc720cd34bdf13ea61ad38a743d","transactionIndex":"0x8e","value":"0x3b6432fb1c31800","v":"0x26","r":"0x5f57524a2a89ec1652fca3a9f72bce25904d2acaabf1e660c0da25592cdfe43a","s":"0x12707d6a77a4e179f99904fa282bd5e4a0d535bf98a097c4a2d72b455bbb5de0"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x970db809f14039e6b35adf1519596cfa20f60fcca7637b822b04c8c7d5f8ab94","input":"0x","nonce":"0x9cc2","to":"0xf3dcd496198ebab1411ca134792304f895a19eaa","transactionIndex":"0x8f","value":"0xb5d019cc00e800","v":"0x25","r":"0xb5d199c236bc60b3535928ff849cb95087b56e075e2ee663f4800f01235d542e","s":"0x4460e24e3729d10d797f4be88ebbef94890b386c550f96e043b28653d1d1ec6d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x5e5903f0fc43e2be2bc343d9e89d8f0ba621922dc4b6a4ac82e0704ee11f4905","input":"0x","nonce":"0x9cc3","to":"0xa6572c15100a418abd29ea3217b051954e5b48ce","transactionIndex":"0x90","value":"0xfbd1cd91dc2800","v":"0x25","r":"0x288fbc105e6886f096d102d1dc96f90c32016109007fb7680bff77ffc0400019","s":"0x306024b2a455c39222088d37b3569044b39cc2d600c2738e57eda44851b2a00c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xfc0a931a740e71774ecc63e0b764e4c9d0b4f836d7f00ae4d0ae28017099a55b","input":"0x","nonce":"0x9cc4","to":"0xfbe3eaa8fbe8dbd66fcd449c99511c0a57c591fe","transactionIndex":"0x91","value":"0xb482a4c50b0800","v":"0x26","r":"0xef378b58c91bed44c7ced17f4c36ff225e78105169cf2a82bcfcd16a142a71df","s":"0x26881c33faec172f7fe83fd7cdd026fd12573f88c16a17aa5e78f5e3f6ac0506"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4231bb2bec57016216b4e6025dbf126565119c2d168ede494d90a7f1d382e873","input":"0x","nonce":"0x9cc5","to":"0x9b7990106b63911055a652a2026cce6d972db134","transactionIndex":"0x92","value":"0xc3d6fc66994000","v":"0x26","r":"0x1c5f1ac94cc55c5e563100eac213a43ea80c87d6184da054c3521f6fe923b14c","s":"0x54f018e4f182cacd9ecacbe446b23b928beb6171deb28e0b2c8b8ed1e1a710a3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x726daaafbf1ba791f50cce8752e9bf1a6929fe47e54ac1b93aa79222ac7b1680","input":"0x","nonce":"0x9cc6","to":"0x7bddda613c69dc409d67a5e7f922850b95e027ee","transactionIndex":"0x93","value":"0xfcc510c832b400","v":"0x25","r":"0x8ea0986a9fc06f2855011e7fdbd2d5aed22ed88ce7e7b77a034e7c877cb897f7","s":"0x624cd5c8a3140f96dc40a83f56dddbdf9fe84b9b1d557263ac5f6b251d30ed82"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xd1cc0903afdd4a618d7df56324428a98a9b9582bfb254173ed064e6f649020c3","input":"0x","nonce":"0x9cc7","to":"0xa1fa1c9cc2681f806fe184e4f7283fb3080bee60","transactionIndex":"0x94","value":"0x2e6ad7727d09000","v":"0x25","r":"0x510a0c635065b30bbec14934a7ed8d799a6eff849d9fed17688be1bd78fd4e2c","s":"0x648bbea70e0e648a4179e0d3e889f7a26baae54e85403b60b07a2af6edfab618"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x323d707d53a05ec3db824e36ec43814b3bd35e49a084dbd6a3ce4f777f0a1a56","input":"0x","nonce":"0x9cc8","to":"0x39728384467996ec57dcdef0cf4982c82e751885","transactionIndex":"0x95","value":"0xb48cc1d8b16800","v":"0x26","r":"0x621c9fdfbd1496173119f38f3030d3b5eefda05c302a34ef76e08704ce3416c","s":"0x2c5f7eadf8a68d1d4be38db5ff9ef93cddb81722196a472f1348f28d852424d7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x77e76f2309ef91cd2366417b36a430b6a3c485ce48dc3814fc94bdcd34b029c8","input":"0x","nonce":"0x9cc9","to":"0xd4da32ade44649a93b7b08ef193d981dac0f5750","transactionIndex":"0x96","value":"0xeacc67a0b1d400","v":"0x26","r":"0xc536f136181fed929f24ace9936b185b08a412bfa944d7d86b0810f748baf04c","s":"0x6b06d0a6444b72eeaec10b551d126594ce20c0fd9e7bc9ee13c2385673d9bbaa"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc88bfdf60759e31503e65a53cd66efb3f0fc37720aded93ee2c5c1c45a1119d7","input":"0x","nonce":"0x9cca","to":"0x635611df213c557d53afa326effaa65d4ea0ef04","transactionIndex":"0x97","value":"0xbe0d6ff05a3800","v":"0x26","r":"0xcec675c38b42f9557c97581c8c4488619243e3f8a4f1f644b3dd08b900a9e440","s":"0x1821058a9a2ec71f10ff33189005f8545ed003a1d8b2e0f06cb74afba8ca3b61"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xec991c9d8160e0a0a8c8427559f6dbcf6714b472e7ce296be5d21f9c0f904336","input":"0x","nonce":"0x9ccb","to":"0x11e55784679b3c232c089277bcf10e80f1cbadf0","transactionIndex":"0x98","value":"0xbe0d6ff05a3800","v":"0x26","r":"0xbba064582085abd6bb0c35fd2e1c3160ac095d0ddaa0c44f415049d218f63d8","s":"0x5ede010809dd5e0efe1455814e1bf20b108a445afd3d8d7e8ccdb77af513e8e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x621a8230b5080228901854d87058a2d00475d0d4c40787249c51f325a099ca2a","input":"0x","nonce":"0x9ccc","to":"0x24ad36f1264980e4c0cf4f8f3cae33c22681f5fc","transactionIndex":"0x99","value":"0xe13ab79a2d8c00","v":"0x26","r":"0xa74a4e20cdaa096a62e344062d9178ff63ce2f7788eba0dad142905545f49209","s":"0xe42d9884686d6b933753ca48c3f41f0afd1fa0d46058bed61a80b89f0e38033"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xd7bb8fcb3c156badf23c8cd7e51fc3eecad18e01c9116a3a908b4c3619c8de11","input":"0x","nonce":"0x9ccd","to":"0x3bd4147e2843e22a404b3a7ec4e623dcc1d03e2c","transactionIndex":"0x9a","value":"0xc3d6fc66994000","v":"0x26","r":"0xe39c6c476a58562df02ef7a65c0fe91fa1b160461d58fad3dc3e78773ecb209e","s":"0x3918944fcbb70032d45ce38ae1b8433f21acdfd2d8e8cf254860d82bda5cac6f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8cc27b7bab23324ddcc94badd25686dc5b4fc6a6d5dc5e8621a53928f7694154","input":"0x","nonce":"0x9cce","to":"0x539be33e02a71c2794b473ffd7d93457133bd53c","transactionIndex":"0x9b","value":"0x2b97e1c95848000","v":"0x26","r":"0x4e3192b3f87e0ca5c4a0e88d7586d1f82e14aea5afd34216edf5d237a5e1ddeb","s":"0x44be3d194141d488ac5be4e6ee28f8220b745828a8c295419592e7f4ac9108e3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8b8d69a1253996f85ffc76260e1e24440c0aeb426f8a84261c33a5bd325922df","input":"0x","nonce":"0x9ccf","to":"0x895aec706916932f6ff92f396b822a7b742f8894","transactionIndex":"0x9c","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x73220c3777d9143003dea0505379d52edd2b0def10229d662daed56b524187ae","s":"0x1b4962a36536e15098f5cf2a0de19b6235b0417306a0fb0eedbe449a5d35b776"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbfc34a1bbf3f08c83dbfe4f831ef8748d517a4bf8fd329e726cf42fa579cb4cb","input":"0x","nonce":"0x9cd0","to":"0x08fa1cf2fd7584a60e036d28aa1f15e428ead213","transactionIndex":"0x9d","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x8d382c30c2a6b5d163ccf772aa01c5783ab82f9136130446649eafb4e96ddfc0","s":"0x4ce80fdc02b364a26b565a8dfecb12a7245898a21d66591b77a52d4a688583f6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x0fbc84a0583a6bffd51e6eedf56ca022923632aef0e82b9dd542d7adc5d61791","input":"0x","nonce":"0x9cd1","to":"0x906df0ac2b8e313a423698601881cd7019daf577","transactionIndex":"0x9e","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x31f4781e47379574227e2a230ae927e83a55d773ab4cffcc91c0e5608b2c6bfe","s":"0x174d3e978fba0bd7bb4ebd1f7fad798c402b684995f8ec44b46af843ffa5b5b4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe3314f9fdda1e58fb733b7931387e789b56d36c11855a815286417dab541e1be","input":"0x","nonce":"0x9cd2","to":"0x8e88bb62681f28a830d237fd023f2f2d20e7c04d","transactionIndex":"0x9f","value":"0xd28720c9962000","v":"0x25","r":"0xafe02a7c2b2699acfd9c933be9253453c7abca1dfc380dae2969e7ae45c1f8df","s":"0xa0e16b9d8eb149deaa599d6eeb952fb2a7494ddb7047c93babd582eb23e14ea"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x755a581c3a6846514c20e1ae3edd96a0867a2861256197a37ee5bdeddb5d2e69","input":"0x","nonce":"0x9cd3","to":"0x33269065d17f426432be4bfbb773debb4c96f1c8","transactionIndex":"0xa0","value":"0xc3d6fc66994000","v":"0x26","r":"0x196ab468529ac624cdcdee3722add7003460ff30d8fe7acf46e33d20bceecc06","s":"0x71a4036e59d768bd4d343f9477d3190cbc83b402a5ecd8d416a3b616fedbbea"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9df7142ec0c5ee2ab46290936e726a940e445106d6f05be08a3765178cc93b86","input":"0x","nonce":"0x9cd4","to":"0xf957e85e2418b0cb016f61b94e77ce0acc269c50","transactionIndex":"0xa1","value":"0xbca080a4a2e400","v":"0x25","r":"0xfa8205bf60de23ff80b8633931644e68b824a0785f393d565a5da518a1c2630","s":"0x538220e9adeb78c3fe3c4c96936e660a9fd1d1452e87729809cf254a339526d1"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x1730ed3d9f3a10b52de777e0e122a302399da8c1bd6d5da9d9bfa86ee2f89ca9","input":"0x","nonce":"0x9cd5","to":"0x3cd376f5b979e46f0cb5b68c3902860ae43ce85f","transactionIndex":"0xa2","value":"0xe4101efecde800","v":"0x25","r":"0xea5079eb2952368693662d052a2f8f0ef43a9febcd18a85b33491d3b82c93090","s":"0x7698917e0c0c3e64b15da00503d507a6f8b9dd86806e16e38bb16f6aed4f02ca"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x288d87d47b19ba2e65a60fe5e5c90da5d363209d7969057c198583a15bb305af","input":"0x","nonce":"0x9cd6","to":"0x1b2fd12e8b9abf91d99991dad4d9a306765c0367","transactionIndex":"0xa3","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x82bc034aa4e4ee35e3218f0dae7cc8373ffa45acc115d677ca788b716fe6426f","s":"0x3c8be8b725ea4f35d0afc236baa4b2aad175c41ce7b0093a15fdfd77d379d18c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc7dd7fcb098b82b800dc16b3532196c89c3b0faddabdfdae739da5c7c72f3409","input":"0x","nonce":"0x9cd7","to":"0x94e0323df94c4065979c1a421479d08d8df1fa1e","transactionIndex":"0xa4","value":"0xb5bd33937c3000","v":"0x26","r":"0xcf456d759f0cad5e1fc14c1ee8c3d01a6d406684d59752a549717eaa2e9207b1","s":"0x14efaef72e72e1c8198efd2bb11087af1d3b0f4100cea3fca248dad012a405c2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x2145c573f84f33f6c003ccf1eaf9f84c9c0601b5fd5a5e8382e8c1757828b14d","input":"0x","nonce":"0x9cd8","to":"0x7bf32bd578ba355bf700811e599247e810618ee6","transactionIndex":"0xa5","value":"0xc65e071b07fc00","v":"0x25","r":"0xe1b492da8fad24cfd7d349294477e0fad66921bc91623152fc06409de2fb3cca","s":"0x6b16cd9a28dac6af29e3c143ec361d576cddcc8c71eac0c4481618614c62ad57"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x58611b2610635a8b76df6f7e605ff0702df017348abd2238782a3359cb71dcfc","input":"0x","nonce":"0x9cd9","to":"0x2bd7aea169058a604d456297373eb84f5a34cf04","transactionIndex":"0xa6","value":"0x1db2197d8e18c00","v":"0x26","r":"0x891409b0f022fd802e451215ae2ee96c81dc06229cdbd05ddcb9939da5ecd579","s":"0x68f48717051b13bfb8578afe51c449f31a3bdde25ab3ad83b28d8cfdb4611f6d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x87c02b06f25d71f35374a271bb60fe2f99a79af6508d9e33aeec498ae434f73f","input":"0x","nonce":"0x9cda","to":"0x1655649294a57e5c11172c8ab523eda86e4fd1af","transactionIndex":"0xa7","value":"0xc3d6fc66994000","v":"0x25","r":"0xd80e0eaba17f95a944c5b658477975f19c28ac94fb8b9fa2566f3b38f603474c","s":"0x1580339fe9298d464f58e8c4fea9389f7006ca82b20c930b8909b75279c25e9d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x2b3c06aa279ab80455f276671a61a838fdf05b76a54a8818fa64cf95c4490b36","input":"0x","nonce":"0x9cdb","to":"0xcd3a553544775d8611e698467795f358bd7fe55f","transactionIndex":"0xa8","value":"0x11ba73f98f3ac00","v":"0x25","r":"0x1dbfd861141e35072522bda3c1219194eb01c0e330c89b6022ec730ac93dcc31","s":"0x5af83895a635f553e3b9c82a798a7d53a5a3a86488a26fb27737a33264ac9f24"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x56004a943b7e36b67e741870b1890849c34990d048a0a455e808cb132540d496","input":"0x","nonce":"0x9cdc","to":"0x56fc63ad1fdff5630f17543342af12d0aa15d247","transactionIndex":"0xa9","value":"0x154e819e545b400","v":"0x25","r":"0x377d06a741b2450b091d5128e18b1ae4c760ba5207e8d9efbfc6e774b1cf60b8","s":"0x4f0083b5968aefbb5acefd1b338180f8b1004ff1d6c913d029c66432ec007659"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xa29176cca49a109d53efa403efc3c7b35ba2a1b195484f8c05e69b849b894a8a","input":"0x","nonce":"0x9cdd","to":"0x297c6ab093a5e9c17a19bd83005e309aa6bf90fd","transactionIndex":"0xaa","value":"0xbec3e418240c00","v":"0x25","r":"0x511df109f77d1b9d2c7011b0cc8a8bba940abbdc53890544d45beff6c3a61d06","s":"0x11d11e39276e64593c4275673271f41c650c423cdec21468f6309c7c82fdf886"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x1cd5a9fd79b22626d1df97c78709f4db9abe62157196a34ac537c59280490fe9","input":"0x","nonce":"0x9cde","to":"0x287ff03eb0ab5dab611ee1e8e7808289cf122197","transactionIndex":"0xab","value":"0x17186bc5e484400","v":"0x26","r":"0xfb6464f449ed3133bfab98300f69a9af9c2fcfcf70348f4a3cb804e9ea668abe","s":"0x697002a0151af289d5bc4a5b42dd810d34e456ca4ced24b882ad7ab084785f6b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x00c308e3e62fc3b58f7d7702e3882d6d2aeb809859e4487e8fa997943e7a0bb2","input":"0x","nonce":"0x9cdf","to":"0xa85a7429620085477373ccc651ce6aa411c610e7","transactionIndex":"0xac","value":"0xc3d6fc66994000","v":"0x25","r":"0xbbf0e35e491aee18fbb86d3710083e914cc858268a6af2d16426bf7ccb2fd973","s":"0x38a6b8278e842d9223bb24cbe7d32ac4cef3b83cbe567c9c8ba257bf2a38ddd3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xaa0bc5f611f5828d51be4888401de4557a173262078eb6f11315a917fb6c2ebd","input":"0x","nonce":"0x9ce0","to":"0x3f18f9a66a30cbe9d6c9b02ec54254057eacc43b","transactionIndex":"0xad","value":"0x185af0237b74c00","v":"0x26","r":"0x22f3ba4c1b250346e58fce9f135541005c440aef5636ad99bb831fdc64c59be4","s":"0x60978c1e9808dad73a541cf557de560135634d090ba229bebf0524b28b3ffc7d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xd2a4d7e6549ce0578679a0d11d2d2b5f2c4f64172951362739f9b16903e82621","input":"0x","nonce":"0x9ce1","to":"0xa0fd5398b2102ea03918a547cfc58a1fbf4c2403","transactionIndex":"0xae","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x6055a4f416f5e818f9918bea1eaef0a9fe14a3661149a12daae0f48ee88d0998","s":"0x4b03acb5deb9c71ca143971abd748e935db2b76ad8430c8a3cbb2c757ff8fadc"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb42f3a400da89e618a4c4d35f0ed6153bfad69d3346909bd88ee8171d5be5d4a","input":"0x","nonce":"0x9ce2","to":"0xcefce92fb15f3164268589b706191c8362601e97","transactionIndex":"0xaf","value":"0x1db2197d8e18c00","v":"0x25","r":"0xd22d3e17926de80a691c83667373b97e88753d8507b3f61764b494b624ff0e92","s":"0x2c2406f7bcc907e877d2145b1b29ce4b818d14e97d37d2c6dcf0271b22d26af7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x1e8bcea74e6ce6b5ed1c81c6fc9882e0488b4e82614cddb5fad905544d434fca","input":"0x","nonce":"0x9ce3","to":"0x724ac56002fa96bb4476838cee9c22621d392e11","transactionIndex":"0xb0","value":"0xe10d49b62be000","v":"0x26","r":"0x3dd9c54a927146032bb7d6104b7790467ce1c6441524020ed704acf458d58887","s":"0x4ddf7517d33b421d07605d2939c1e3a0a80a10b46ae21ea0e717f23700376112"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x217fe1e5d79996d4d3c2f384a516f58fbc4aa5618ed37be8d8176e1318e4bd2b","input":"0x","nonce":"0x9ce4","to":"0x211544a96613f246545b0b8308ad688697e02b4b","transactionIndex":"0xb1","value":"0xbe0d6ff05a3800","v":"0x26","r":"0xe7e43e38fd4c5ac224564611b60dc13ff3b6834ca9210954033a778a744e8a35","s":"0x2e29744b11609e3758cf7f0486448b82f89296114af13a39e14a573ea491f769"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x1ee443f2fe6e52c762b7e3d305d77c427ebd30cee71c465da6f199f53e37b5a1","input":"0x","nonce":"0x9ce5","to":"0x3a10cba0ec57be6d905e3ae2a3d446b1e2b6f8c3","transactionIndex":"0xb2","value":"0xc2cdc6fc2ea000","v":"0x25","r":"0x17a76b0755c0b70ed372cf66f081d4ca093069d3f6b0b6b01d8b0e30a2b4e80e","s":"0x3f46b120b112c7a3688d51e4cb8712ed64776d7ffbc2d0ec63fc9d3cd07065e2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4dbf985934569c076b2f6190838817453a990ae27aba71e59a4cef7f7d8de7c9","input":"0x","nonce":"0x9ce6","to":"0x13eedb523e8e5c84afade1a43b8a4e447d417c06","transactionIndex":"0xb3","value":"0xbfd66e5a367400","v":"0x25","r":"0x213e26c9232cf2b74adeafb0e055aa261c66cc014d34d0fce46a581c60788eee","s":"0x21a635177917aeee4653bfb94d44db6b218b75009c62f1ea882fea1fc35af5a4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xec9bb7e6c141a37985ab43588ed88f7969395614615636d03c1b79ed7ebe5e59","input":"0x","nonce":"0x9ce7","to":"0xff509eaf1c3cf5ebfdd485fd46ef3122ab080768","transactionIndex":"0xb4","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xba1b67a6465f30389cfa278c492d906b1a122fc7ac4a861719402a6d32b21ed0","s":"0x3116dae25a6df9bb99297ecb492c10dcf5bc87ebf09cd43892f9974eb645fe59"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x1408a4b31b8be29ad4955109cab7bb2caeed3d07abb7477cc5c2e102aa16dc00","input":"0x","nonce":"0x9ce8","to":"0x9ceb693dbc8d0e83b281dc9f2f0c9fbc80cd2179","transactionIndex":"0xb5","value":"0x10488f2b8489c00","v":"0x26","r":"0x94a445991efb25f3f0f172c75af1ab84cd698302b658c7ac1ad1d92e165072e5","s":"0x5a2ba979d90c2f4d78d39b903c88be1859fb22d2edb1275683dcbb500ff0b9d5"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x860d308e608ed19833ff274428e2a9718afcbb9e599bcf7d5b29846b77f938c3","input":"0x","nonce":"0x9ce9","to":"0x18cd86558de106863e994c35a5c63bad30e23838","transactionIndex":"0xb6","value":"0xb2664919715400","v":"0x25","r":"0x769a58a1d432a1caf7b847257659d5f9e90af72db57035a42c64d268ea98a3c9","s":"0xa88b914c5243ceecae1d96273f5c04b5add4e0688b1f7b355a28e270e0747ab"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe9d76862e1bc46fa061bb8d1598c659038ca0c1c621c17ccad338c989dab12d0","input":"0x","nonce":"0x9cea","to":"0xc13c2d8ba7889fab62d820722b2123a13b26e4c2","transactionIndex":"0xb7","value":"0x17c1adfe0b47000","v":"0x25","r":"0x1d31fcf986b4464ea69ebf1ef99c90aa34f8bdd254cfeb1b6e3f62a55a026ecb","s":"0x19461dc3be2733c3ea1319232d8d2247aafbe43dd8f7e898f235f1c065e6b56e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7f816b4793eccbba701e79f0e1aff842515eac816e9984609bb6beb37a42040c","input":"0x","nonce":"0x9ceb","to":"0x845878661700257c0b2b51028272edcbfdd4d0a2","transactionIndex":"0xb8","value":"0x1524b1cfcc2e400","v":"0x25","r":"0x2ed8c352f733813b45fec2a7f4454294cff0e937e0e79a3cc69c1381bcbda3cc","s":"0x44a26812b96e80f40823db93ab2e595f4e317c324b08c92e8b66f9a9cfccab4d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xaf7d3927d7434976786fcef6700fd0ffab006d66508f52d48e0d771453c6d662","input":"0x","nonce":"0x9cec","to":"0xbace08e3c0c1c2f232d83ac08eb506d4528d879d","transactionIndex":"0xb9","value":"0xc223c19fe34800","v":"0x25","r":"0xa9d9eb89ed7f59e74199d6d2520911a726222e6f9874d52be5bd189d9a199df6","s":"0x3b17a05d1304b7219c3a5c09de56979b03fab9f77e7bab3cfb6c9d6bd770abf1"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x06e15c18ef71316b4fcd19ae69a0bc4a78de770e27b18059901136122a9c4e03","input":"0x","nonce":"0x9ced","to":"0x33bfeb8ae567ce99992a353463819f7fc6735d8b","transactionIndex":"0xba","value":"0xbfd66e5a367400","v":"0x26","r":"0x641c4ce339ba76bf21a3d1a629de3a1162b9ca5ca8564eb1bc38608c2eadc0f8","s":"0x637b595c9180335cd72ceab2a6ee5fd489b6ef201f65906cbfcbd755fa3794d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x1ba46c696e1030964f9824bb8ee284d1ff6254ee5404170b9421195ab141c7b2","input":"0x","nonce":"0x9cee","to":"0x34debcfd3992a938f17b58585ad9f5d73a673fd9","transactionIndex":"0xbb","value":"0xc3d6fc66994000","v":"0x25","r":"0xeadc532404bd692779019e4e2cb6dca4c38ca2075661984595b91b18fdd196c4","s":"0x5689b7383296d9233b98af8f422a67c4ca1a7c2e6d286575e6b889f38829b9a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb9487d0a7f752637586666f40fa99896ccccc2803c47cf003333c09275046113","input":"0x","nonce":"0x9cef","to":"0x4e205689f178a5903422ab4fd6410b435a82b165","transactionIndex":"0xbc","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xcc38dca840bd2912df3667aeccfe3711a98420eecf41ca3c14e61f525f191ce3","s":"0x2d469cbb6a1fc81854cf1d976c1653fdbf3ca79bdcb28b8cdb84611f3874728a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xce0293178b71291de5d02b8124f1c252f4018c1d55768dcbbf193f7d361c53a2","input":"0x","nonce":"0x9cf0","to":"0x03f4c3ac41b38e8d9f349e675d0fb4c509b522db","transactionIndex":"0xbd","value":"0x2992f07c93bc400","v":"0x25","r":"0x9eeec756163c4b7c1e73fdb0b4edb4808d325045f47eec192d5097034ebef0d8","s":"0x73102b81a6f71f09fdb6c1931ff817f2ed984c3a9b1d22f84913606f80bc2ed5"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x2cd24cbb022a9bc0352e4c532d939c48ed3f71d644ee841f816fce064f5c2b70","input":"0x","nonce":"0x9cf1","to":"0x0a57963ddfa8cc90383cef7f06fc6e7ab0b35d22","transactionIndex":"0xbe","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x10f8c7721ca343a0cc32e711538ad4eb3d37ba56fab12be5c1f8894aef67a406","s":"0x28f65ad322f0d0a1d381da1053bac2032a392118ff7f5eb9608eb8c6abbfadda"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc3cc47c5c88b194f48aa0d8bec7d2c9ab31dc4e81e7380fc99942b9af503e6f2","input":"0x","nonce":"0x9cf2","to":"0xc6bd787851fc8eb27e9b0328b570549663877735","transactionIndex":"0xbf","value":"0xbfd66e5a367400","v":"0x26","r":"0x3e9d7f4fe67506178fff36ddc6423fb32c489b874210ec4e28882aabc3f3cc75","s":"0x7b0bb7cea70dcae0f136e052d6608062ba7bf41d83e245f2ef6e722e52b469bf"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7cea8c95bee7eb2189dbb6d4444bbab4784c1494336ded6c8d1e761f9b94d618","input":"0x","nonce":"0x9cf3","to":"0x13726a3c3fde08d00532e221957004ff6d1342d3","transactionIndex":"0xc0","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x8a9f17141816d27034ad606ef936ca7c566bba5301cef78511cee9ef5e428d1b","s":"0x21a40f36f9bdf2c4a57f0dfe12ac4d5fbbf114e0188067e84d905f313a847253"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x089ba7df9163e818675a85e53e4236e543c16994423ad1b64a81f43c37b9005b","input":"0x","nonce":"0x9cf4","to":"0x8773379bb3de3de7fe976122cdbdd801f55e4820","transactionIndex":"0xc1","value":"0x187adf8cd328000","v":"0x26","r":"0xa488b0f12d31783b85845bcfc5b1b4ba5ffcfe736acb1f9d35444d1b3905b1b6","s":"0x6a8086dd57efbdc84bf54322738de8faa9ba607f4042ff7dab2c0e267bfb08dc"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb66b54b00b9fcf3c61267c7d0b1762e403bf6f409a8e7275d84a0994946752da","input":"0x","nonce":"0x9cf5","to":"0x78d53308e6ad14799789d7558ce78c73827fb780","transactionIndex":"0xc2","value":"0xbca080a4a2e400","v":"0x25","r":"0xd59825ea762c091be2f0717d1e049bf4a0b818c657d358ac04991c1680d720d2","s":"0x639e1beef12560bc313b2454615a38d84aca04671f5d41979b363cb18852f0f6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x71641cb80e0f1f39ad689acf6e56a429f1c82d7ce64694f30636cc61c98ec174","input":"0x","nonce":"0x9cf6","to":"0x4d73cb2b71fa1f7e5e63a7ab58967cb92bf4b921","transactionIndex":"0xc3","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xbb686da884114b60ecc2ebb307391098cbb273ee4b92d13c1ef7696a8bce3fea","s":"0x19d16886c84b1fcfc7b81ba07c05a57efb42438c410217268d3f4f12deb1a65e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x78d7967c296e433208d48b24a9f9332a38fd0b18781881b893c6eb2c5dd4a570","input":"0x","nonce":"0x9cf7","to":"0xfc9481332ace0c3a7ec57bf0cd4bd39fa115eceb","transactionIndex":"0xc4","value":"0xb731e73ede1c00","v":"0x25","r":"0xfcaad74772d1a076f8188b4a6157a898e0d85670c71cdd842d151aa281b0a3ee","s":"0x32ff5cd65b7379190f099ae6cf86ec0ee383d3ecef36f661d013928676a7d216"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x29f864a1bedabb6457faa0a7fbfb103dd6dc01b7a0ca0be7b1bbec5f93044f4e","input":"0x","nonce":"0x9cf8","to":"0x654240e37aa1beb5b40a18bb9cc69334b3a56175","transactionIndex":"0xc5","value":"0x15064943c09e800","v":"0x25","r":"0xe3ebe65dc975500e1f4743ceb3ae145b8326e72d5668ac8f0db9b65e0c8e9977","s":"0x6b7f1ecf321444dc3100aa1e3af67f4620853b6d3555cca6d44e5b51a9a3fd6d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb1293a984151655a0dcc5ab9059b8276e3f83eed10c0ccbfe4884c318935f106","input":"0x","nonce":"0x9cf9","to":"0xbe5cd7c23c060cd74f64b91424481bc40bb4db83","transactionIndex":"0xc6","value":"0xd76c7c0a756000","v":"0x26","r":"0x2a1c53b2a71916243828174412e55ba03951286cc82947d8490c6fb2e61babc0","s":"0x39a2e24783ae14e66facf41c5b8e44e529e352712bc9962d1ef71bfbe5475b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc793ec632f476aa4edaebee4f358485d245b0026804811b7f6528b49175691ae","input":"0x","nonce":"0x9cfa","to":"0xb82e0f3c72820861037bd7c3d911a96e6cb25497","transactionIndex":"0xc7","value":"0x17c1adfe0b47000","v":"0x26","r":"0x14aabd73b35d878b51c152c0ce5dd892cb5da4796b63f3ae1d3a9c467142d2b8","s":"0x320772c6ba1256843ede366dc1ce288d20af17f36ad9d908bf8b3ab35a6b1aee"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x12169fb22d1405f853c977bbd994f3baf65aeb9ea4482ac9060161c6a4f0cce7","input":"0x","nonce":"0x9cfb","to":"0x32e700e832d99ae47a00227cb068fb5cf3da5edc","transactionIndex":"0xc8","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xbc7b96700d6e7f4ba17e1528574d87ec8ecb2bde20bcf3714e36ba51fbc1351","s":"0x12ccc6c7288102727ff6a4a054afafc3e77237fc35f8b0598aa05588c9eafe6d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x89ba13a7b91f35ef7b98fa20a5f60fec657ded837b72cd7a69c5ee2cf5250edf","input":"0x","nonce":"0x9cfc","to":"0x776438b8e2e99ae520c68424362fec87cccf0eb4","transactionIndex":"0xc9","value":"0x3573c77b995fc00","v":"0x25","r":"0x1687de92e6a9e03f5a26d7e9adf01d703687ae98723f7616059a2eba1042bf4e","s":"0x4cc82767b8bb816344996892375244c67114845fae15c5a4d314f81278bca8c2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x6471d332d78de2d77d20c68621f01bfbfd402f1f5174d5d23f1f65fe6b8835e9","input":"0x","nonce":"0x9cfd","to":"0x9d11002318a9dc9d1933c86f01bc629d51e6a3ec","transactionIndex":"0xca","value":"0x1db2197d8e18c00","v":"0x26","r":"0xf3665db4603eeec0d6b9c126da18d1d0c4e723635416d496d122b51bea8e5c38","s":"0x665537b02e8c6b542695af06167d86232ac78f5f37e9f303aed334bb81715443"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x27aa3212eb0a239a711f186d8a63a42512a18bd9332d7838f523b95118f99749","input":"0x","nonce":"0x9cfe","to":"0x9246543d9461a606b2840433f7c392b5aef8b285","transactionIndex":"0xcb","value":"0xd6c261b9bf0400","v":"0x26","r":"0x4d1af5be4a0c757b54eb66058c3feed92c2a1a85b1baa62dd4e9ce9dfbaf04b0","s":"0x7e284bad216625aa8ce5ae05b475cfbd3c5863ea51885de4b5c90f290cbbde8a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x787da5b7891661565543d353b5dfa70e5873ff85c7e566192963aa3885084aa8","input":"0x","nonce":"0x9cff","to":"0x808c940bf3acbd75bb3499318b352db2432d614f","transactionIndex":"0xcc","value":"0xbfd66e5a367400","v":"0x25","r":"0x5a2bc1e4a21cd2ad8c7819b3bb1da0b14baf103a217a076d719ed41132f57adf","s":"0x19b6341660bc14bccc747f7737be6ab023bf8a9041402a5051013faf812947ec"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xa887294974aa257f4f9a16b7c13d266d55ae0913c59b40da033c3d853b4ec752","input":"0x","nonce":"0x9d00","to":"0xaf758aaae27a66b03dc018e30b8effba820187f8","transactionIndex":"0xcd","value":"0xd10ec777941000","v":"0x26","r":"0x3efc22d04b40946916b5dc10ff039c45a26eecc4c024a11b2480777cae4af45b","s":"0x5364886cfddbbf40cf8fe12aa986ec4579478dc56f4fe0ca12892fe6f3efc591"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xaa9a352aff5cc1bed522ccdd197a644257d9ee7cdc6e8f61b68126e0819e8ab7","input":"0x","nonce":"0x9d01","to":"0xfbf330ad8f876cdd7b89232cfe4b593722882852","transactionIndex":"0xce","value":"0x2e86359cc169800","v":"0x25","r":"0x835f89cae0dded62ea8c6350d3d3bcf652047b57f13bac1ee26d112b7aa59214","s":"0xc6e496eeb284948bed201735ff3bf63c6499910f3d4ce5b7d6b172dde27af23"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x425b4f3ceab69dcc0d05ddd2604dfa40e78160d8cf839630c3e7919cf954ca1e","input":"0x","nonce":"0x9d02","to":"0xeb7d710b47c38c4992da2c3289ba57a85920ebe3","transactionIndex":"0xcf","value":"0xb35229ba10b000","v":"0x25","r":"0x35b710be13362ded9c96271d2d401cfa8ff606f3553827e8327477fd612e3c7c","s":"0x7b4290776818db42b4411a812ab0eb57aaa0884051369a30357e86112446f267"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xa4e40b677c8f444c3a7b97d23533e43d4a3ae21dde8fad55771d4c7ef5937c5c","input":"0x","nonce":"0x9d03","to":"0x28e8318732b762515981ef37804cd4eb6a5758e0","transactionIndex":"0xd0","value":"0xc3d6fc66994000","v":"0x25","r":"0xc783a1e9e5c08743c5427c6847ed19864e9c5adaf95a3e46912380fc377a8f4b","s":"0x4b83d0068197957b2479c6778f88df8bc6728aaf8175bb5b7221de1d689a9360"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xfbacc6bfb7f44322b7709bf52429cd5dd9d9d69d5c247338b1bbe84f015494cc","input":"0x","nonce":"0x9d04","to":"0xf00d3f4ae5b4214a302e464b3d12f031b127d483","transactionIndex":"0xd1","value":"0xb3d90a82e2a800","v":"0x25","r":"0x3f511bbba6e703af96fdf15b9adec24067f8390faa99917226e705617b0093f7","s":"0x154bb661e8272e7134fbd138b127b1b84cb5db49f1ae2a3f778c307d72bed1e4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8b84a4966d38d776c6b8962a1917bbb4f059729e34b99610e2c7ddf79ca49228","input":"0x","nonce":"0x9d05","to":"0xff8d7b0bff0fb85b52d10e5d7945b73161cce477","transactionIndex":"0xd2","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x3f5a312c1d08ec8dec4f42a512d85edebf264f008965941bbc5353e597feb38e","s":"0x715c0b3fe338250faa707432a7cfbd4e52c9bb2308d8a02bcf74b3041e1b57e6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x3e0fc3160fa5c53b4c1e2e3e46b4290771144d0e990ae89804f60c99f32b3cfa","input":"0x","nonce":"0x9d06","to":"0x255157a27d51fabc579ece5361622eaf8c1813c1","transactionIndex":"0xd3","value":"0xc3d6fc66994000","v":"0x26","r":"0xc713976a750fe379a85211f4f02479a7dd0b225ef43576d566f7533acbdda3ba","s":"0x1cc622ba98693076e2d9a21e141f524eac7fb9a888c7bfa889f058c63fa67c88"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8d53b68772193d037d5975e35fab74223b481d40feeea6861fde738bf4ef2671","input":"0x","nonce":"0x9d07","to":"0xd3e8de3b5a63b284bbed2d5cfe9794e3d5aaf221","transactionIndex":"0xd4","value":"0x2bf31b6d7af7400","v":"0x25","r":"0x6c3f46638dce4a49f9d5c743960bc20d6c3db6209ab199eb63ffac809aa8d860","s":"0x777cb49838ed0c4d553aa1ab1614d56b863422ff77b580c8fdc42612fac7daa9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x92d4a4e90c1ce7e3862c41aa95aea5c3354c7bb10b6a4516ccec5504b05cd033","input":"0x","nonce":"0x9d08","to":"0x9f52e533d0d336b0205cd27513d0368ecd27723a","transactionIndex":"0xd5","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x3bd544c739b57fc40be9937ec9af4a6d89e6e48d357a8280b27bd39a320064d1","s":"0x5624ef908fd74fe087ff1e81ce64d11030dc92644f9cf3f51a791fb13482e5f6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xdac1ca5e90336bd34ac6395fdb8d2838abd22a6d22298adc66d402d54bd81587","input":"0x","nonce":"0x9d09","to":"0xb18ed27b948855cb6b70355d15022c5ae1bedf2a","transactionIndex":"0xd6","value":"0x10488f2b8489c00","v":"0x25","r":"0x1499d499a1d314ad6f96ce73f641db22d1bcc69b992a4fe2db823f58182ff833","s":"0x6eb9b31a603012a831b78f14d5b902d2b9d5bc78f365ad8274415eae0b33955a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xab0c181188235dd287a7039351e65ed31a1c3b6ca3e25265672d1ffce9e26d74","input":"0x","nonce":"0x9d0a","to":"0x6f607c25b954d8ecbcdbbd9963339670f266e394","transactionIndex":"0xd7","value":"0x2c8b2629b4c6000","v":"0x25","r":"0x525127a98bbd7ac6bd66e2ed099fcdbaa6bc31fc232916099823fcaa7867132d","s":"0x2477abe88708caf7091f55ede6b4bb822d77a1e025d051f602157b851d092daf"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x83b03938fa0948f26c1b00a62f399c46155988aee9d6d2f01c10b2c4fd185e5b","input":"0x","nonce":"0x9d0b","to":"0x5a5dcb51cd6ce7b05303ab28429edf8d9d3b062e","transactionIndex":"0xd8","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x60a9c574271e060b3fe30f2c91e16824c27ab6487103aa4844d4b21a9161a6ef","s":"0x2af5c7fada52e0fd32d0c79e0decdd6942deecda5433a12695c99a19957fcf5f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9fc136a17fffb9382333c373b4179a3eb7c331885b86a422c31f5257da22a55a","input":"0x","nonce":"0x9d0c","to":"0x6f153c34ccb387a3c65c456f2bc73d02dcd74aa5","transactionIndex":"0xd9","value":"0xbca080a4a2e400","v":"0x26","r":"0x3dd0047baec92ffff8217aead0db0dedb1eee7269bc576612c753832f9d9f226","s":"0x7face9f9fa7c5cafb479f8779f083a74376a15f23b1d45678c5f96fc242e1765"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbf29e878bf6ac8691125f38d804c2c7ff3f73627a554a83609e3c11423da6903","input":"0x","nonce":"0x9d0d","to":"0x3cc6361ffa45d348a6baf3bba05c4fe0eaf15b07","transactionIndex":"0xda","value":"0x1708302ebdfb000","v":"0x26","r":"0x5df3e470d3dc803d9c85224ce70047fc39a523a9d8e0aa269e9e9849696aa7e4","s":"0x50ac36fc9444ccd19262ada9e5c9f5d41eb67a1962dec1b4a76ecde83f7da264"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x351f9816a5f1a92031a5cdd7ced1a49b4626d215df0306ba9d49d99b9a8dcd9a","input":"0x","nonce":"0x9d0e","to":"0x9de1d52959d35e32a2698975a137f183f9511e3f","transactionIndex":"0xdb","value":"0x14c9782ba97f000","v":"0x26","r":"0x548a968998e3260944e30d7a1176218e72fe8add244019aba026ed26dcccfeb1","s":"0x49697d323bb12ebe772e5f62768b98ced32b127d1270ad5be5da2fb57041d8b6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x5d2499b5492e6de32086142ddcb47f24d1e1e7e46c7088af168eb9f74a9332b4","input":"0x","nonce":"0x9d0f","to":"0xd695c7dbd84e5d58c7ba1f26d20b2593e15a1fec","transactionIndex":"0xdc","value":"0x2f55bf3ca595800","v":"0x26","r":"0x191363910d31ca0643f9d1aae7a3f8c8eb81158022f1e7c73dbb2115c8e00917","s":"0x5991eb14537e7801cfa75e0750fab12c6dacac01540783bd9873116ca9adf9f2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x45483edcdeda70664203bfb599bbd94e27673d6f4c43f4566ce9957c468768bf","input":"0x","nonce":"0x9d10","to":"0x745d85da1aa5d82f151fb90a76723e94e7c4cb48","transactionIndex":"0xdd","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xd66e41d88dec87395300a329068cfc53854af5f9c74295a79604b769f6bf9d00","s":"0x389c13b049434448195df4d4198dab5adce0eb7c54f89b234e21c4002277c05b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x180450baae621a037e6325cda67da0d5299f3bc53ab5fb53cd2063db30ad857d","input":"0x","nonce":"0x9d11","to":"0x1f078100f770dca9bc0de8a2e56281e68d10efc6","transactionIndex":"0xde","value":"0xbfd66e5a367400","v":"0x25","r":"0xc5282a113557bc82f1891870d82e0fdfa866631c59fe0ef8fcf492a81b240a84","s":"0x76499a4831ca6ffa0a522d16f08095a03475ec091361196a0cab29e9a64ddd08"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xddd7bb565dc8195e56ec8678041e817be52defefbf1c61ff8e50aa5d2f4995fb","input":"0x","nonce":"0x9d12","to":"0xb88585c62dea87d736c29f0fd4217f70c07c057f","transactionIndex":"0xdf","value":"0x10a4fa1c3e61800","v":"0x25","r":"0xbe3003f71dc134804a94488bab38476c3c783ef50dfa6825669757e3901656f2","s":"0x186151902e221bba4f3c0e6d83da1bff751dec4520bb4d4e411d3fa71429c984"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xf4a3fefea93abbb34748b07264fd97a86239666f0e42b83327e4bb154af88554","input":"0x","nonce":"0x9d13","to":"0xbe2c3874af4ab4ddb7bf24586fdb6cf13780e453","transactionIndex":"0xe0","value":"0xbe0d6ff05a3800","v":"0x26","r":"0xa6b34a07eb15597019cfd7af199a232a6103ff79ff851cd67ce8379817d56ee8","s":"0x578fe3780418a7c7b5c0abe6ba2916eee7654b11ed204d0df84d5893bd31e417"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xd44ef22db1fb7e12e2da4978630583d0371faf280491ba4ea14aa67c05f2f2d3","input":"0x","nonce":"0x9d14","to":"0xa15c242c4311f878eca821af6ca6b2fe2392991a","transactionIndex":"0xe1","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x62ad380c829c8957a7d67a33afd0cbdcf52236b61e0b319f8c44ed8208901179","s":"0x7e7bcb8ce95f10253eabca57b68bfc94094c23da7a15c16a9c3142a8a571ccbc"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xa5fb48b4ecc1e86facb80d4846465650fe8c27953674f66efeb29edc343a2e96","input":"0x","nonce":"0x9d15","to":"0x1c94dd84c1d0ec757ed568c1676541f039c06a6f","transactionIndex":"0xe2","value":"0xc3d6fc66994000","v":"0x26","r":"0x5ce4bf66e7027de1c39cf920e19fee8f5da51ba6231fa06853a08d8826e2ebf0","s":"0x4d536ff7d2dc81be76ce0b9a2fcbe6e7f0e7e0e92517b94d8edff2c7103a934a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x6949634372f1fe260cbacd3db19bb9f5c61b44bd858a50c3af3dce9fe0ccad46","input":"0x","nonce":"0x9d16","to":"0xb3328cb02b0759d71b1837ede36e5674a77c6da2","transactionIndex":"0xe3","value":"0xd248715f3438c00","v":"0x26","r":"0x6961ab1637e1e2b367c49e9ab0e59f1bb4475acc61feab020b3cc65d470f2b01","s":"0x22531490dd06c32be73504df385bf0b39f17c6b710f04930ce2955b9053aff3e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x5d581a05dc437f6a0d451a6d4f068257abbb2ce0fe1bc98aceb6fcd8e03268ba","input":"0x","nonce":"0x9d17","to":"0x4ad9178b47868752beb5aac9685388cac1f1cb7a","transactionIndex":"0xe4","value":"0xb51ebb2a2df000","v":"0x26","r":"0x360b546750e04cacf502754024ac71be0377edc32c1349b7e7eed2937bb7caaa","s":"0x2fafc99957e967677cc43fc67f8a5fd304a7261a08e67e91f9cec5de4fe28500"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9714470a91c3dc2e7dae6ac9d4152d70272ea323ccde232ea35f9057790c21e4","input":"0x","nonce":"0x9d18","to":"0x8fae8ae3f4431c4d4faba4b4756b45de98759e48","transactionIndex":"0xe5","value":"0x41549e7a9f03400","v":"0x26","r":"0x27208885dbd18638b93026f4c30acf509dd027a5c52d8db1228ac2edd4ab87be","s":"0x4b49789d178fa09a9371e15c18d0aaf1dd172a4af9dcb3364613dd58a863a1cc"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xa2b3186efa03b1b54d00998aa0d8897cf278678f404a8060816b6cd806629e04","input":"0x","nonce":"0x9d19","to":"0xfc6418f560acae4419be48f7f299f0aa2185f525","transactionIndex":"0xe6","value":"0xdc51de47784c00","v":"0x26","r":"0x12cb0e577acb62d2dc1ec52f0ddf0e113a4b6ac6f9fb5f9b410dd6852ff137e4","s":"0x10787f0526a00e60d31de51b6066e5bfdf9aadf8b0575c7f9485c49477fca7f9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x73200aa7eb57161747d1bd9a2d11917b45bd6d79caa5d26b7344f7a7502952ef","input":"0x","nonce":"0x9d1a","to":"0xc4a11e92427a5554364ac7e314670adce6c9422c","transactionIndex":"0xe7","value":"0x17c1adfe0b47000","v":"0x26","r":"0xe1faccd7a599b682df63b68836e6a4f4d45223b8ebf2446e7deeed2d01a6201","s":"0x487e703d6e11239513aba25bbfd20b31cf76871b9437a7c16e2faf35f32f939"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x954b6bfcc16c66d3d3cc657348b4bbcc6d4a06f6f9ba779ce4eb96e483634352","input":"0x","nonce":"0x9d1b","to":"0xc11990d182af08898b244393d729d082c04d1e16","transactionIndex":"0xe8","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x701fc2458fc289813b711df4cf032cc35b121fa830dd09e0d6475ee6ba8123f8","s":"0x1abc1a548024efc3d7827607408b1a001856f5490e7a22039e6903341aec37cc"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x91df18865f6a5df609741a128228d2aebdf09aa37b15f97f641e8d9dd88ba034","input":"0x","nonce":"0x9d1c","to":"0xcc5ffda4eb02a170d7182d0dd4f75f25c564ba11","transactionIndex":"0xe9","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x34c6e45d650823ff297591136710152176d81c093e1990da19a1bc4725b18cb1","s":"0x206c8b68f07b35099132551e9b10509585ff4f702f4d05951e71f709ed2b761"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x163d3003030a7a4d384acfc07e3d32ce388749efb2f6ecae44af7de5e3730894","input":"0x","nonce":"0x9d1d","to":"0xf2355719899495d08429900681a14bca060d9879","transactionIndex":"0xea","value":"0xb78eb0a0ba4400","v":"0x26","r":"0xfae4248749423ab2587efc0bb3091a8507e6910ea118f35a0ff44967f2a4d732","s":"0x4f15fc50959fa68880e1c38bc0d75ac501a1cfab2f8dd3b8856ba71f50efbc3c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8c59316f9c6cdc642313e218d1966894991c706395d7b70e4c8c6f73eb3c06eb","input":"0x","nonce":"0x9d1e","to":"0x0ae5b31bb58974b41961d06a865e8ffc1751a3bf","transactionIndex":"0xeb","value":"0x17c1adfe0b47000","v":"0x25","r":"0xcff9d3c7dbfe980e210d13ce817a6852e844b1a281b1df27a89608e655272724","s":"0x4af41ea19ac9119abf9befae40e384be08144a5dea0bab0a6d7ef94371790bf4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x34d76ba55e293dc71439735338540abc154a6b934fdbc1d9a887aeb8e6b00055","input":"0x","nonce":"0x9d1f","to":"0x9be6e5003ebd8c12fc8453adc0bea7c040907145","transactionIndex":"0xec","value":"0xc3d6fc66994000","v":"0x26","r":"0xc9362d7253138a9f4851835862970bc14af545d5414033b0be3d8df042b2263e","s":"0x3f8a0678a5a528458c63e08a0a9412d656bdb972bba090416fa895aefdde73a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x754bbd76215fcf913198131686c42b14790cf6d231b3299dd7d173bcc2989d49","input":"0x","nonce":"0x9d20","to":"0x493ed6708e1709d51aae0f4635dccfe695e17a42","transactionIndex":"0xed","value":"0x1ee22ef601b6400","v":"0x25","r":"0xec291fdd9183fb067ba1297fab3ee2f44eefddab9a84be982145e01c3b1ac225","s":"0x7dbd0d4dbc7a551ab7daaef7b3dff1b4af48d0f666741740222c2af2d7bd233a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xeabc4b6188418f2dad5b245a39c6ffd8771ed87f9d453d254dff1af9371b4a0b","input":"0x","nonce":"0x9d21","to":"0xe0dd007e4c1858198d5333188d1e51a50fe7fa24","transactionIndex":"0xee","value":"0xbf373008f58000","v":"0x26","r":"0x53d0edbefcbf73c8e024d30293fd1ebbbde41f2e0559fb6505256c89b2d404a0","s":"0x4b70ff90da557741e490c44b5d6187541378d038383b0cadf07ae7b122d538c9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8f07bc246af8f3ddaed18a610cf6c270ef1c2dfe109e822e544946c9135b4b67","input":"0x","nonce":"0x9d22","to":"0x840a86928ecc07417570a52a2fadfa07b92fa249","transactionIndex":"0xef","value":"0xc3d6fc66994000","v":"0x25","r":"0x4898903d6c230f74ba3e9ef279ac0ebf89ec7fee7cee57f484449d0c00934f43","s":"0x5bb1e090a72b44aca5108e59616396d53fafa5a099276340c8714ad151f05095"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x69c58c9688d17f6e3a41bb507d45a8a3e466f8288e3b946ad4efca1d45ebb973","input":"0x","nonce":"0x9d23","to":"0xbdaaec2bd3aaa7d7dc7bbb1632ea8407a0400ac8","transactionIndex":"0xf0","value":"0x2f91cda05a5a800","v":"0x25","r":"0x13b6bc5a8cc3e3f573082bf9c5a116676005af6cfa83b09637fa6d5d49ff69eb","s":"0x30d62a01c5facfafe6aa9ec72420df4ab58960c035efc82cb0d74b4dbe47ffe3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x55c64f655d8dea53f2ce64166fd895407dd9137d1c6ab71b5557521b013762cb","input":"0x","nonce":"0x9d24","to":"0xab7cd1de895d8f6acf3a33dc0cff1dbc5d3cf8f8","transactionIndex":"0xf1","value":"0x243a8fb94ab9400","v":"0x26","r":"0xe571d5ec1a3ad2f7ee2e4921ec990fee738f790b8b9cbaa41ce6199dc271557e","s":"0x4281a9021c3baee73922944a32640e83b563d12f1ad7d7080d0f56226957d613"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7e9e4fc7893cf0a9622eb220c1fda03f6de22989ed09c07b5d4e962280a26fa5","input":"0x","nonce":"0x9d25","to":"0x0ed7fd37ac6d0cd11556a390ef5755cfe7e11ee4","transactionIndex":"0xf2","value":"0xb5ab95a5840c00","v":"0x25","r":"0x8c5ecc5b3eee2219e9abace46b7512f1cfd545342db9bb86055a00ad4d01a513","s":"0x29a5fbe512591d06682e59ef9c6189d3bf8452363d2e4b9bf306dc0d0ef8532e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x01da483cc7dd23a9eb7789d099a98dd7defc20ce93445cc3f9b27a3c45b88567","input":"0x","nonce":"0x9d26","to":"0xf90454bbf19f7a77f6b0af28be2c5f488f494246","transactionIndex":"0xf3","value":"0xb236dafb37b800","v":"0x25","r":"0x296b8e9e002db193de14cbac2dba792ac3e10aac099e516efcb426ce0fffa1a8","s":"0x5cb237747f3d97eb69fd34c77464b048ab8a130d35eff139a69f99ebb3a67bfb"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x11415c478db04180237ee25f0d9f25051d28b253fb036a67191d14c794f0aa7c","input":"0x","nonce":"0x9d27","to":"0x13e36fd42db0af1af5daf99cccdbd5d3abd84c75","transactionIndex":"0xf4","value":"0x3c4843281346c00","v":"0x26","r":"0x4a9805021177372d9e45eb50f1c7215124f767adbe27f6d50239745afbaaf2e0","s":"0xc32497ec2419af80fd422f8b513bcf2aaf694b19e82f5be710015ed47be6cc2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9d0098cc74b6c0fd63e186cd7082f1230532cd8c7139c059b3be9418744e7e24","input":"0x","nonce":"0x9d28","to":"0x1aa676e5951dc81d5d423448eab4be659bff8af9","transactionIndex":"0xf5","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xb66285b17cbf0145ec370a5e9f38c931b77d0c2b9dbc1cb105eae92df68cb3d1","s":"0x516b4cf19aa021d5d4547d8b107eab6a71be2141d0e09735835537e32179fb64"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x38022390b5784a49bb4f7b77abeae78d2a4929be84390600a273b3c60e71427f","input":"0x","nonce":"0x9d29","to":"0x137ae004483aa3930b86d70c61e2704a8ed15f92","transactionIndex":"0xf6","value":"0xc78e1bb3f72400","v":"0x26","r":"0x619b7886c3459782bb7a12d9819792a9830ef4006aff306494f513d25adb63ca","s":"0x20e165e8f873c59618ec2f45177391a3d329987f2f269ae849f6449affd432f2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbdd61eb9abee735e5f27d7183ce30a25996e14eaed40604b316b0612795a6c64","input":"0x","nonce":"0x9d2a","to":"0x91a5d62b126dfaad6c9f84208fa7265e35f654e5","transactionIndex":"0xf7","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x1659de2ebd90e88a745c6b6fed1781f709d14740b44cd08cf2a4b89b38120842","s":"0x4ba65b21017b960635bb239784b26ca9cf9cf619b3ebaea46f549a39f813073e"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x4054d4a66cc62b44cbd482e7a7c9a3d47961ee4c92f01383dd4bd217af49f029","input":"0x","nonce":"0x9d2b","to":"0x653df565ec7fd75e6d11c93d2e418df3059c42a2","transactionIndex":"0xf8","value":"0xbfd66e5a367400","v":"0x25","r":"0xb996499cc7de072f5aa5e00195b371b10600226e422fbcce26a66b19e895b460","s":"0x6774c8b83b1c4e02bdc628ac26536e44551b4c0d16f2c69adcba53094af21361"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x14feb3ec93a5784e8e8ea8086c8b0b14bc8a1ba18c2c020b0faca2a3282f233f","input":"0x","nonce":"0x9d2c","to":"0x48ce0a4b875f12a67491cfff924d6ffa26a15095","transactionIndex":"0xf9","value":"0x10488f2b8489c00","v":"0x26","r":"0x5f1487c5db3f0f6810fd94a2358417e199f37fd8b83b12dc730ec254ad66ecc1","s":"0x6d0b167e2cdc8a783b0c4d344ae2f53c8506918c7507d4b786a1829e1b930b1d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbe8033c700e32c4cc9357f236c22e38b46248bc969c1877b1edd5667caa0275e","input":"0x","nonce":"0x9d2d","to":"0xd4f2d58076871ab57b6bfacefc77d89e25520c7b","transactionIndex":"0xfa","value":"0xbfd66e5a367400","v":"0x25","r":"0xaae12497417754109c27af289a5c076bb921bc128502b05afd3707bcde72315c","s":"0x1bf7d8b4fd7ac51a136b09bfaa77baf90adf1a54c06e74e5958c4afe12f7583a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xa1f25d0f7e6bf6871bd49184e89c6281f69df9533d22f1fd85aa6a91aa86bcc6","input":"0x","nonce":"0x9d2e","to":"0xc09c32d40513584b21c1cf9c281ef0606512c2cc","transactionIndex":"0xfb","value":"0xd10ec777941000","v":"0x25","r":"0x8d560c372f294da15f779d0dac2e381cd73571c65f311d7fb681fd73a1424981","s":"0xcfefef34555e00a3be0a99ae73b599ea5af3af892b68305e3eabb1a5c4cd8cb"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9b8740e51e99cfd9aeac76684a2aadaf8a4becb51680e8acf6e67d7885f6ced2","input":"0x","nonce":"0x9d2f","to":"0x466521aebc4b3d385fe15ad735aaea12112b127a","transactionIndex":"0xfc","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xe773f734e166160eab39e86abe317b09fa87dda79dfbf5d6b1549c50e2efbd80","s":"0x20fa53a197715b410e631c5ea0ce32734e4611733104d5d44bfa42eeb50ad84"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x61aa8b97e2c95243396ad3d8987a9514f3cc34cd35a7f2e5ec60625c446c713c","input":"0x","nonce":"0x9d30","to":"0xdb909d1093c83d34ada5d9627560f467344872d2","transactionIndex":"0xfd","value":"0xb63eec35f82c00","v":"0x26","r":"0x2675c0ab6ab44114434e174fd737ad8ebdca6a6a75bd1e6042af22abc7b77095","s":"0x562f6f3642db195e37855c3d8451c82d2e64b1df0de6bb041faa4563ab3d1711"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xaaf865c9b11a37d154dc3a0d82a8b5751ae480cea7dc78144815014a1d47a131","input":"0x","nonce":"0x9d31","to":"0x890b451b2ff30f1da26e5ff815b8e2903609e78f","transactionIndex":"0xfe","value":"0xbca080a4a2e400","v":"0x26","r":"0x927281130e5da54aeafbaefdefba33888fa696a6ae4011397db46e32556bcffe","s":"0x63537c39427a59de124acce253ec54eb36f7f1350e6a31f4da019391d11b52f4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x04b414f36ce448d4382e29c61acc81cd1ff5397fab63fa3e520d367d0b12c907","input":"0x","nonce":"0x9d32","to":"0xf060b2a6f01a05eee307ae90201afa5b13f6670e","transactionIndex":"0xff","value":"0x1c8203dfd9bd000","v":"0x26","r":"0x4c1b44608814b2c80472721e83e9ca5471b48226e8a697ac530c91f90a64a0c7","s":"0x2efd8eb43d46ad4a08c341d855b2feac58d7571ad609155d79ff8f81f1c5b46d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x84835297b78c0fd5b83761e87046aa80c5c4b25028172a6af2e3c3845fe3a973","input":"0x","nonce":"0x9d33","to":"0xe0e6c781b8cba08bc8407eac0101b668d1fa6f49","transactionIndex":"0x100","value":"0xc495a958603400","v":"0x26","r":"0x981b6223c9d3c319716da3cf057da84acf0fef897f4003d8a362d7bda42247db","s":"0x66be134c4bc432125209b5056ef274b7423bcac7cc398cf60b83aaff7b95469f"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xdac06da3dfb3f3f6a0f9c79038e3d08ee33f525ae9868ca0af5d5a9dbbf39a08","input":"0x","nonce":"0x9d34","to":"0xc70f9ad86ccf27090c331a20c11e09e161badb35","transactionIndex":"0x101","value":"0xb555380c72ac00","v":"0x26","r":"0xdfac45d18340cdbe65b97e769ae1845841e580698feaa730b7357211d222a305","s":"0x2a60cb17e470d16b323026e3f048f0a6de30b2629bbfcbdbae5d264f8e51e019"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x27356a5d6167fbf721b223e0be046c9214449802e55498426acdcd2dc96b69bb","input":"0x","nonce":"0x9d35","to":"0x58d0bf6c45fd77edba9e0ad3e46e69dbe1ab2d15","transactionIndex":"0x102","value":"0xbff52062f95000","v":"0x26","r":"0xed00d8e5d37a76921bc78481e6b0f4a137b4a03b151b3a6bac8962484f077778","s":"0x5c9229481247f3af1cf80f7cf0a8292594e35093e038b5c3afeca3a167d2ec77"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xba4819c207044620e3989e499e61e7c03197864bb8b6e815e3691079763695ac","input":"0x","nonce":"0x9d36","to":"0xeb53460104b5b5ce5add099abb75932da9904af5","transactionIndex":"0x103","value":"0xe07fdf4fb6c6c00","v":"0x26","r":"0x86f96350bea35565fb74884e356f9810c9ce1b75502292ecd311a286a4b7fe2d","s":"0x5d234756ad837a45d9c67b9d85f25eadb0fa0e839746a3abf12660b923e07fc2"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xf366be96a8c24c2c5939135c036e1dabc81b8b22118b68ce18600795069685df","input":"0x","nonce":"0x9d37","to":"0x6a16c0c1fef68d68711cc9b35fd5491e89bb2506","transactionIndex":"0x104","value":"0xc3d6fc66994000","v":"0x26","r":"0xeb1e4254f3d1f1c8acaa79c750c3928f2327fd88cf2c02eeae75b6ca74986cea","s":"0x2e700cf3266f445e9d68b9bae03798d5e052c514c1d4bd08703fabae97ca69d9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x77f3beb8f13797edf0979091a9894abc3a2d37ad00ea6c2283d364b2bbc53749","input":"0x","nonce":"0x9d38","to":"0x3b88c148c85f265d0cc2e1bbd22706440266fcc0","transactionIndex":"0x105","value":"0xc3d6fc66994000","v":"0x26","r":"0xa850344302e0bf95410b8307c6bf967b0abdff41f46d03d78332f56c98e4b61c","s":"0x975632db6f8f95168bfdf0f14b46b02d9841235cbb0bc8c2be6833b6e48700b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe8e77fc19f52a337633d20318dac2583b10094a5d886aa12ba86b40d8c445b99","input":"0x","nonce":"0x9d39","to":"0xb89ed0d7c1bab4562d6c9f62ae46e1ca978ac3d7","transactionIndex":"0x106","value":"0xc160e100a6dc00","v":"0x26","r":"0xe857a3fa7b82349a1e49abb8cbca936d234737a4fd9db5fe43af59054e8cf806","s":"0x4530fc86a8dfefe73a8edd8ade26867b0cf704c56a63902bbdd87f8cf2f633c5"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x39b8dcaf327d4494a5d7e334924f063f5115b8f88d5d3e0fb11120857154cf7b","input":"0x","nonce":"0x9d3a","to":"0xbd630d86d647dd1cf11693c8cf1712431596e757","transactionIndex":"0x107","value":"0x3b6432fb1c31800","v":"0x26","r":"0xc962522d9db8c32ec37d6e1d2542f92999d7c92748a1f79d5d535b1f0ab64e7","s":"0x2d9784082a45fa85b38dbb5bc86b1e695bf3461c3319526f7b00524e77b47180"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8687c226e36b53a0e243b950f842f8063252a8131b8dcf5bced13a3d374460b7","input":"0x","nonce":"0x9d3b","to":"0xe8beb6602e9fa7261fb7217772e74a0e0eff5b32","transactionIndex":"0x108","value":"0x274d60dc4dc9000","v":"0x26","r":"0xcc877996f15ea692f268ec668049d9f1e9e5d4e06d294bdedc0e5dd849c044f6","s":"0x7894390aad202383f3513e0280e368b8806b3c84457fcda33865124905fcd2ce"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xd3c6851ad1b73607f350e94caac79d6bf8164db95e2550a176c17c82d686a436","input":"0x","nonce":"0x9d3c","to":"0x7259671a99d6727afc719b6be335b3d12f23315a","transactionIndex":"0x109","value":"0xb3d90a82e2a800","v":"0x26","r":"0xe80d30a2e0221d11e8c8aeeed9415b61a46b8f75717f520757f0a04a30dcb2a7","s":"0x6e8e19c90a794ddcadfe87c155fa907dd120e78230442a8fdd84a3eaad6b8fb9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe3c463f1d97b6783e4d3bab38371467db884276dc506b28d3f499b7dc8633d0f","input":"0x","nonce":"0x9d3d","to":"0x3fa58fe438957db67fec7d98830733cc20ef78e1","transactionIndex":"0x10a","value":"0xbef19c7da23800","v":"0x25","r":"0x12b6c4b531ea1ed93893813ca4ba83711ef77f0aeb5d50496338d61ed4a8073f","s":"0x615ef3572bccaa7a2b67e2016fe27cd5e92476be30dbdd896421a0e885462987"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xdf8d0be70c7f0c1b363ce33d040f854db2dd283018bca934592bf5a0bcd7d9c7","input":"0x","nonce":"0x9d3e","to":"0xc6484480165ad0be7837d9699879f471598f47fb","transactionIndex":"0x10b","value":"0xb213bd63e20400","v":"0x25","r":"0x88d47a6ff2e2adff1b749dc2d98ecfcccc34a431a12f6e6c8f609afaca81e292","s":"0x2b7b96247e151a7d80e1cb2007328c35640b5e88d248861d0c04aa6d5a77dffa"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x31b9b1f7476fec5dfd5fd18d4215a91c59e8f6347890945f4c8cd0efb7bd68b5","input":"0x","nonce":"0x9d3f","to":"0xdf918af8a6fcea8aca4e41033a83f376822c5af3","transactionIndex":"0x10c","value":"0xc2fe6d18a19400","v":"0x25","r":"0xd4604addbb94448503460ff0817f0f282ca9d6593502a55f4a9b614cb0da1862","s":"0x72822737b98c32e340abc5e1d6ee981b5744bfc10a561b9042c9cd4256ff9923"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xeb29d192acdb57fc681038476f689fab44f12b7c75016085f6c3841bdd5081c8","input":"0x","nonce":"0x9d40","to":"0xb8f4c6ebc5adee28bfddfcfb4b99969a3d4d3f00","transactionIndex":"0x10d","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x6a7142f6a976e021731d4565247432a41c9480eea32b2a92b8379242a5582d47","s":"0x1c919c1ed41b784fd02b03f5c34db4e11d073c741683b25e80271cdd277612e7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x68418311e4bd7dd19b15b5c38344aace68b5eedec39aa24835e76e17ab44e3f9","input":"0x","nonce":"0x9d41","to":"0x171125195a8be9c1bfa055ea4cfd111e5ddcbf24","transactionIndex":"0x10e","value":"0x20278dafea97800","v":"0x25","r":"0xdd1b1aef77828c1775ea8fe40e284d38e61215af17a7f71f275853b212091fa5","s":"0x16cf60f614ffa64806b57a6395392fdcc682ee642b5628fbc5efdf09b9a63af1"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xff5a78bf3cdf7ceefb03d933c01ef9d7422bc3560ac872bc2f7e31ae06d610ef","input":"0x","nonce":"0x9d42","to":"0x778ad400d43bd2f7f41e3ff77093bad2cd91be12","transactionIndex":"0x10f","value":"0xc78e1bb3f72400","v":"0x25","r":"0x6928d8a9aa1c15cc31debd4c39279dbdddc877124acf5e9002e75ea90c581a74","s":"0x6cf6d08af094cae7180a0cde1a328c4b224eb6a8d794380ae01e52823cc548ca"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc5caf93d7b27eb0a5d0a0d48df676b8c8788867566e4e23924746ecfefe05e31","input":"0x","nonce":"0x9d43","to":"0x986c672311415938d7586e79a5f638f2b29a3927","transactionIndex":"0x110","value":"0xba0c3c94ab3000","v":"0x26","r":"0x5732311fa0e31c3b8d3d2247ec44072c3ac4b3058b8f8393d3b397c0a8945742","s":"0x492aba48035675bb962b3b9af6d9e7f41251e68982e7f109a065452d3df106b9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x88bdc57f8dd898de0d50a5c5e15648570f2059b4154123923bf0d1676f4ac029","input":"0x","nonce":"0x9d44","to":"0x4c647225087bfff6da1536b4d3542ebf13cc46ba","transactionIndex":"0x111","value":"0x7cb8d1507a76800","v":"0x26","r":"0x746f8df66a4584f2defc5b791ef251bc4a67472c01d173aed64fe7b4a92517af","s":"0x4a20a791bbd9eaa7ce682068fc770ef5139feaafa37d8a00c4a8a694a87c0953"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe79ceed82b17fc949cfbc6136dc826ce72d5b67e9ce4a922a586697ae4e6873e","input":"0x","nonce":"0x9d45","to":"0x4798994ff85419670aa86bcf026e7c5976833249","transactionIndex":"0x112","value":"0x4fe1a5db4928400","v":"0x26","r":"0x2fdf8b414249d409056a19be0b0b55df2d00a18ce9cfe9a63841bceb9ae0eda2","s":"0x7437f0548a236bb86ee90e3789c7167bd60197065a93da26a006efad3600a0f3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc2af406032f7e8684cf7de96048c604d2fff9e3e326c66be0a8ba7b901510b87","input":"0x","nonce":"0x9d46","to":"0x4bddbd1cbe7aaa14d1461178e2cf4943c12fa20b","transactionIndex":"0x113","value":"0x1db2197d8e18c00","v":"0x25","r":"0x88e7c916c1699248231e7b0b01d6045d64efdf5c3e910337a3f1a395b87d1dc6","s":"0x755e8ca9e5bc9abf2a5b086fbdb37c05e505a118c28e58efdbfd4d1da854da2a"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x19ed7609e5fc47fa26b198bd9b58365c4b6067ad02fcc6547b768fa5080be8ba","input":"0x","nonce":"0x9d47","to":"0x60c977bcf64316c88fdba52391d0dda45b129352","transactionIndex":"0x114","value":"0x22726f849d4d000","v":"0x26","r":"0x81b8c25c00abc5654b307058428192835818c810de464c3bb0ad6db58756951","s":"0x4deb54a63c82641983d8497dd69544755933718cc892165a5bddfd5cfc069dfe"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9c432dca59d47809df0c50b93fafe25c5edac9497617b55629d714dca7373596","input":"0x","nonce":"0x9d48","to":"0x374547eed2c3738f09f591fff7bfa417b9a75901","transactionIndex":"0x115","value":"0xc0aa6cd8dd0800","v":"0x26","r":"0xa987421bfb2d3b853b84891b6f85216d66c22c2b2fca15f39150f912ccecf727","s":"0x7f10ab7897ab16da3797ea41272558d65d7def38be91e4c1003348051f412185"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9c5cd08d5ceb6f4d4c3863046b643f3fafa9bbb351533abc71debf1687c18c0e","input":"0x","nonce":"0x9d49","to":"0x3c068db8f6ef4182e75565f5d37eaa8543177c25","transactionIndex":"0x116","value":"0x11de480c08dc400","v":"0x26","r":"0x3525843199367aaa9044153ae0d85e54e3707cb7698cca38097876b02dcd068a","s":"0x77ddfed3ea1e5f7943b2d89610d2371e2c83dc62c00267f2f94b6c0cbb21d962"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x3d22ae6592e88c4ed761f2b9ff688f1aa41982a1bb370ecedf49019843c94630","input":"0x","nonce":"0x9d4a","to":"0xfb3de54d4a6130598e8ff6a039ef30f0b59082aa","transactionIndex":"0x117","value":"0xf711768607c000","v":"0x26","r":"0x2acfc1043321833c91b0b59efc785cd3f6cbdf19dd3419bc2789cec5212e5ebe","s":"0x62460ef4770c05061fa67960019f181056798b8db278626e22851a7856dc0132"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x689d96eb460e8e30e8ea87d0b98a647c0edfcacd594cc5e6eaa1e062cc77b313","input":"0x","nonce":"0x9d4b","to":"0x5d795994944b3aee38fe866c8fe77b68d4b55f22","transactionIndex":"0x118","value":"0xbfd66e5a367400","v":"0x26","r":"0xb1742bec9a7df83d804cec1d6655ff3a3e921806e0b6e9a97b84138ef0b1d075","s":"0x59c1eda35bccbceb17161743d4f44788f7654f1f92afe15cf03ac4cd66d57ba6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x0ecdf674c318bd00f62fbe5413466ec13175c523cb0cb16c4122df6d4d2c24f7","input":"0x","nonce":"0x9d4c","to":"0xe417e7027b38ba90f4250deb71ee602aea6de5c8","transactionIndex":"0x119","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xaca1fc6427a7e3c699b3669cc6ff3ba6c8f2cfa573f97091424997be5d752cc","s":"0x475a62702cd690b0ef846b65868bbb2d726938ff7c2e6b1aa394c49298535c15"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x3dff56ba42771c98a208c77ddf52d77ca3cb19a47392795f5a109b4ed50aaa20","input":"0x","nonce":"0x9d4d","to":"0xf3bc692f1b8a25495c63a5e21906ed7c16cc976a","transactionIndex":"0x11a","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xef42a333834e0ff5c47ac0a96651e3701d2c5f59e424d7f22f0512ed2ba55127","s":"0x535fa706628decb9b4bf85420b8d25eaf94a67eb0d0749b8746762f61c84ca10"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xcdf47726cea581aa0378b01dc18fecc863fbdcf375ca39c5e2bbffe1bfecadd1","input":"0x","nonce":"0x9d4e","to":"0x88483fbc3eac6a4c27e180394cdfe01780b971d9","transactionIndex":"0x11b","value":"0x3b6432fb1c31800","v":"0x26","r":"0x399b92ff667f02a249af27e3fb783eedf9a8fd48745b6609bd0e81641b88c176","s":"0x7a15dda763017a4d4962e716d4e153fa04d9021955250863828c80a5b4a1f35c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7cd35a6c78cdcb0d0cdf2976c4ccc8bc22675d40f87b5be6e309f05f138deebe","input":"0x","nonce":"0x9d4f","to":"0xf31b2602804d986d6298f06f7850fbb1dee44c07","transactionIndex":"0x11c","value":"0x11d1427e8875400","v":"0x26","r":"0x6182f241240e0a693ae127473d0632b75192ec86f25abcd3093d510de46eb7ac","s":"0x71da8f4e8c4df4c4f2cc6489c3799199e7a4dae6be816b0a99cc9338c1ead5c4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xb0628801347233aad9abf0bd2d4cd745cd636e180c573d99f902d467585cb655","input":"0x","nonce":"0x9d50","to":"0x3007abf58617a21fa38383a8d978cf12824e5083","transactionIndex":"0x11d","value":"0xc3d6fc66994000","v":"0x26","r":"0x968d3a6101bf5c9b4d2696815b70d9c2058f9bc771cdb070a191e067e32388ed","s":"0x1e5188201fde674eda698ac00137288ea1c128b00f55ba120d0a1acb47663a3b"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x9cd75bbf02e58624edc86f7fb73648c527dffd236c6b8fdd6809c60f39c69290","input":"0x","nonce":"0x9d51","to":"0xed2fee621473e633b7ae70b35d5a371745b5d7c0","transactionIndex":"0x11e","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x57381d57fbfd2bd4de4581dbef6e526025be89d3b909397b94ab9101c67b240e","s":"0x72cdb9ee50a130bab459b7b5d3571fbaf65143bb4cb92b13d7523e12828233e3"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8295129f3e7f07933f31954ac3119b79d397f4a1442ba43dd8aece46eafad0bb","input":"0x","nonce":"0x9d52","to":"0x996af40e6f835cfe4f6ef7901e841c638183255c","transactionIndex":"0x11f","value":"0x17c1adfe0b47000","v":"0x25","r":"0x3acf5d97079faa59d7f10eb15cac69d606055e9490be84cce0d3f9e9da21b783","s":"0x1ded8203056f75ce13ab52d94a1d7d199b603ad8560a59841e175e6c47766dd7"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x8c4793c0372011a897f8c4114ce8fcdfd02cb568815fba4245a8612c840d22f3","input":"0x","nonce":"0x9d53","to":"0x6b6a72cc53bf65645cd90378ab7235344f57f3d1","transactionIndex":"0x120","value":"0x14316d94b06e800","v":"0x25","r":"0xe1a5e98c7f70e0d6537fe3ddee2c41a5620dc9f485ba57b1b0da9bc19f257fb0","s":"0x14437aff3705bbd139d76c9ac83a00d02ca5dcc5c1deda0855ce506ccb78cbf4"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x527a3a6945c5800af57396b53c89f99d9bca46d64f6b266aa76e3abb7825bb51","input":"0x","nonce":"0x9d54","to":"0x05b03715ab29e54485ee847b926921905779cd4e","transactionIndex":"0x121","value":"0xe8d3be8f66d400","v":"0x26","r":"0x204f995758024eff4af8904d07489f365563e631b88192ab3b19ed98c9729a3","s":"0x77d1b4ee8746bdbb5a3450e3cb5b559095bb67fab461d3d17334ca2749dd70e6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x20185783ecb691a6e0d0e315fa4af57310596745b2f1dd34f8c05418f8e49e67","input":"0x","nonce":"0x9d55","to":"0x84f26e299f3ffcc72e30bcc17057379b9b059450","transactionIndex":"0x122","value":"0xbe0d6ff05a3800","v":"0x25","r":"0x7aea1f615f63ca364d9add4f75f3260367fcb01d072bbf512895ffcfb4d461dc","s":"0x20107d0a72dda0f2e1e76ea3e2bcc6c9afca31c0c54243b6376e1028279c7a32"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xf67b6ef1fd47f4d11e54fa7e9455da9bceb2546bfd7dc8746d0fc90463e29ba2","input":"0x","nonce":"0x9d56","to":"0x989e5a5f88b26d0d8cdb5d575ae4582010cbd9ff","transactionIndex":"0x123","value":"0xbe0d6ff05a3800","v":"0x26","r":"0x1be2409382789e78f0c8415b49b98c9842b7ffe8984594b821c38eae4d1b404e","s":"0xab5b5427ddf4e70ef1bbc627ed1789209c307f86e6805f53fadb0bc6c617317"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x125fb4f1f64f0e3a26fed148a7ddefa52ef94e328bc85d203e4d9f93835d6334","input":"0x","nonce":"0x9d57","to":"0x1bfbeb992ded2e68e6783110048053279c27aaee","transactionIndex":"0x124","value":"0xcda1be8c933400","v":"0x25","r":"0x6eee9dae37a2eb68c5ad7413f36caf03eee0916190894f399dcb101a608be46a","s":"0x6ed3cb6e041a39b01d5a617f74f83f95f092e4504ba8935f3686ca6f75b97f65"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x206ddd0eb0467b94918220c9194ea76c4fd38cd7f1270ba4055bc062947a09e2","input":"0x","nonce":"0x9d58","to":"0x3094c5a507916ad1d30b32704fcba3c781b3b038","transactionIndex":"0x125","value":"0xbca080a4a2e400","v":"0x25","r":"0x12c952bcaa4a479491966d189ab00e94787004433d1cf3f27e44db1533b4fb89","s":"0x1ec37deb9c3c19ab870e9d8d0a28664ba5cdb24827cb415387024752d32ece86"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc8fcbe49bd48d18f1e643d9c30f7eff5b91580ecf18e3cf51a64dc33efef8945","input":"0x","nonce":"0x9d59","to":"0xc1e6d014845c3e9be49b7c7ff404d57eb70bde55","transactionIndex":"0x126","value":"0xb26646c5657000","v":"0x26","r":"0xfc6a142536a53f2c193415f71b30e70873616851a326ff8603d0e2f94bba5e55","s":"0xb6bb5f256d1ac716eeec46450d0be5bc097c1cea3893942edf19c236eda5404"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x808632c02997d355498cfd0958bc6d6234ed895c5714f8038b8156e77092a1dd","input":"0x","nonce":"0x9d5a","to":"0x6eec88f110b7634b7c454ecf6db811bb4e20d1a6","transactionIndex":"0x127","value":"0x30546aba3df2800","v":"0x25","r":"0x5efc9d9e4413f191250d1fa3649568081b18438d460469f38cdf4c4c64e21395","s":"0x548dcec1f369ed12e15e7853b651a9bf123255d7dff536e9651a0732319dba65"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xe501873db84664299077c024d19d5469c9e133f5e9bd473c9f83e1fcc55be399","input":"0x","nonce":"0x9d5b","to":"0x5e27e82fde06a884b709d688a3b054cfbc5d92f3","transactionIndex":"0x128","value":"0xb497a2803e9800","v":"0x26","r":"0x9df43eb8a4464fbf55658e8a1b11acaba33cbb90b8a000a14bd448f1d004799c","s":"0x52e6890fb71ee8e65f2d5127eac2fe795a204455b29909472343477a216c06d"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x3c4957db98859d5b73191adf0adc2721f7fbb1cdb2b89313b7497f2534539622","input":"0x","nonce":"0x9d5c","to":"0x4b916d1e67a42e29365ca2310da3c5c2b4956bb4","transactionIndex":"0x129","value":"0x1762a743bf0e000","v":"0x25","r":"0x2272d8f5f8367dc892ad8fc4d7faac48ae1803eb1cd36f6eed5fdc6c4a40ac9c","s":"0x7e6d5fae5c321780cfd6ea79dc1a2b84ae259128ae1b2b68df70e567d6acc327"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xbca55fac4feaf192382e16e23b7c208e30c86a232a3e217ce03105ce776d4023","input":"0x","nonce":"0x9d5d","to":"0x007ce001301ee96abaa5dbd73e26c1e7b9a16ef5","transactionIndex":"0x12a","value":"0x1e4a2439c7e7800","v":"0x26","r":"0x8fbd9c517cfc6fa4b4d7ea0557f4f60801fb0ae1d955758d03dfce8a7ea068c4","s":"0x6e69a2ed3fa784cea7f7f82e14ed3bd722061607129432d4bb06334b7b80c4ec"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x00a04953f6f4ff9b130e2092117664aa9b8eaedaa7040b0bad7592bb72baafc3","input":"0x","nonce":"0x9d5e","to":"0x0f845cd3da369321429220e6d6e7c3788414e574","transactionIndex":"0x12b","value":"0xb900a526153800","v":"0x25","r":"0x3c743941f289cff5c55e8c83c42dbca60b45919cbede34f337b671bab93de60e","s":"0x12b65e6314dba335249edba1d6bc88caaeec3cddb739203a9b6c40472f0dbfc9"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7f0d62ed1e4e25dabb2f70af5054792df085ca81c0f6ac64121fa97bbb9e39ee","input":"0x","nonce":"0x9d5f","to":"0x3c5b89b3d97e9e56880e4141e24ead232340e4a3","transactionIndex":"0x12c","value":"0xb5d019cc00e800","v":"0x26","r":"0x63972ff9a057b81f446fb119776e16d055399858b236a6d329e45b3452dca643","s":"0x2626b9cc6f3f156b96f5109544afbd5ec4b8ebb125e2b451c3ffdcded38564c6"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xc1c8699fcd8fd3ba414d9d593a3c1de30ed1c03f18a614c5b1f1e2f63de11b8c","input":"0x","nonce":"0x9d60","to":"0xb6bfb46ed86dc95b9a4ac4f9dc54e5eda66f555c","transactionIndex":"0x12d","value":"0x1db2197d8e18c00","v":"0x25","r":"0xa5c8b14f86f3e193d494437b97cfcc44619ccc2fc5ca6930a83efd20f2497443","s":"0x683705920b7dfae3751b43f068e26aba4332a744f7732f362cfcd25334575540"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0xfc57b690a9eae5e5315cefe3dfd24285dce4a4ed089ab9245acf44d3ddabd446","input":"0x","nonce":"0x9d61","to":"0x5304725b936791740704de8795eec60c8bccc3c6","transactionIndex":"0x12e","value":"0xd1cc30c6e63800","v":"0x26","r":"0xa12ad1d0fc0419d5741a47c63b52e007043e5b18d7fc50212138c50fee9adfc7","s":"0x30685b751f0469cc649b7c3cb8c1a7d9fd92c1bdd0448d16063973a43362245c"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x7dcc1722e10be952d4d7c473965d4c82669a7242ea500b4e55ccbbb23777e19e","input":"0x","nonce":"0x9d62","to":"0x5e708092318a8604d4d353d0f1820e256dfbc618","transactionIndex":"0x12f","value":"0xbe0d6ff05a3800","v":"0x26","r":"0xea10d857e88859602a70352d68ee1222554c472fb6be25ffc21afaac7d645bb","s":"0x1f2ce0b79d3297c8d96089d968f0ae94a7d5485ca9e21270f5316dc6fe5dc081"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x89e7853a2fe1e32daeb2c2b06d4cdb1148587c93c049f63bf45c6e302f498c32","input":"0x","nonce":"0x9d63","to":"0x3992c699ddba35a6c706973c6dedbc92eb99462a","transactionIndex":"0x130","value":"0xbe0d6ff05a3800","v":"0x25","r":"0xeec1bdc4d6689af10104b650081fbc49d70b22502afa77b329f7f2d3f617e148","s":"0x1425e1c182fb4496f44e15ef096f634fbdc003003298c3c5220bffc77a7cc804"},{"blockHash":"0x16f37b728aacdb8491eaf8caa84c090285f204d9f6332931144e2fb7fa9c622b","blockNumber":"0x3f29e9","from":"0xfe92a3cf1843b5ec7ccf27b2ae753fac1289fa9d","gas":"0x15f90","gasPrice":"0xee6b2800","hash":"0x983b78add24766c3f9a35cf0c1a471489e92a897d042d0fb8cb4bea11d760015","input":"0x","nonce":"0x9d64","to":"0x2f19943cc9b0352f0cf60924997a49847eef3699","transactionIndex":"0x131","value":"0x12152a80d452c00","v":"0x26","r":"0x13afc637ad749e2aa15f4756ec96dc14504ba5bbadd3dd1f1163aae862e43d1c","s":"0x56876b68b6f58e4c4347e0125aade9cb493bc845eff0037365e3aef08f90452b"}],"transactionsRoot":"0x83975aaf055a868c2d091539397998b8b2a0eb1b25aec5b7aec46515145cafe8","uncles":[]}} diff --git a/statediff/indexer/ipld/test_data/eth-block-body-json-997522 b/statediff/indexer/ipld/test_data/eth-block-body-json-997522 new file mode 100644 index 000000000000..9c385bef3953 --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-block-body-json-997522 @@ -0,0 +1 @@ +{"jsonrpc":"2.0","result":{"author":"0x4bb96091ee9d802ed039c4d1a5f6216f90f81b01","difficulty":"0xae22b2113ed","extraData":"0xd783010400844765746887676f312e352e31856c696e7578","gasLimit":"0x2fefd8","gasUsed":"0x5208","hash":"0x79851e1adb52a8c5490da2df5d8c060b1cc44a3b6eeaada2e20edba5a8e84523","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x4bb96091ee9d802ed039c4d1a5f6216f90f81b01","mixHash":"0x2565992ba4dbd7ab3bb08d1da34051ae1d90c79bc637a21aa2f51f6380bf5f6a","nonce":"0xf7a14147c2320b2d","number":"0xf3892","parentHash":"0x8ad6d5cbe7ec75ed71d5153dd58f2fd413b17c398ad2a7d9309459ce884e6c9b","receiptsRoot":"0xa73a95d90de29c66220c8b8da825cf34ae969efc7f9a878d8ed893565e4b4676","sealFields":["0xa02565992ba4dbd7ab3bb08d1da34051ae1d90c79bc637a21aa2f51f6380bf5f6a","0x88f7a14147c2320b2d"],"sha3Uncles":"0x08793b633d0b21b980107f3e3277c6693f2f3739e0c676a238cbe24d9ae6e252","size":"0x6c0","stateRoot":"0x11e5ea49ecbee25a9b8f267492a5d296ac09cf6179b43bc334242d052bac5963","timestamp":"0x56bf10c5","totalDifficulty":"0x629a0a89232bcd5b","transactions":[{"blockHash":"0x79851e1adb52a8c5490da2df5d8c060b1cc44a3b6eeaada2e20edba5a8e84523","blockNumber":"0xf3892","condition":null,"creates":null,"from":"0x4bb96091ee9d802ed039c4d1a5f6216f90f81b01","gas":"0x15f90","gasPrice":"0xa","hash":"0xd0fc6b051f16468862c462c672532427efef537ea3737b25b10716949d0e2228","input":"0x","networkId":null,"nonce":"0x7c37","publicKey":"0xa9177f27b99a4ad938359d77e0dca4b64e7ce3722c835d8087d4eecb27c8a54d59e2917e6b31ec12e44b1064d102d35815f9707af9571f15e92d1b6fbcd207e9","r":"0x76933e91718154f18db2e993bc96e82abd9a0fac2bae284875341cbecafa837b","raw":"0xf86a827c370a83015f909404a6c6a293340fc3f2244d097b0cfd84d5317ba58844b1eec616322c1c801ba076933e91718154f18db2e993bc96e82abd9a0fac2bae284875341cbecafa837ba02f165c2c4b5f4b786a95e106c48bccc7e065647af5a1942025b6fbfafeabbbf6","s":"0x2f165c2c4b5f4b786a95e106c48bccc7e065647af5a1942025b6fbfafeabbbf6","standardV":"0x0","to":"0x04a6c6a293340fc3f2244d097b0cfd84d5317ba5","transactionIndex":"0x0","v":"0x1b","value":"0x44b1eec616322c1c"}],"transactionsRoot":"0x7ab22cfcf6db5d1628ac888c25e6bc49aba2faaa200fc880f800f1db1e8bd3cc","uncles":["0x319e0dc9a53711579c4ba88062c927a0045443cca57625903ef471d760506a94","0x0324272e484e509c3c9e9e75ad8b48c7d34556e6b269dd72331033fd5cdc1b2a"]},"id":1} diff --git a/statediff/indexer/ipld/test_data/eth-block-body-json-999998 b/statediff/indexer/ipld/test_data/eth-block-body-json-999998 new file mode 100644 index 000000000000..5e9d4d77bff4 --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-block-body-json-999998 @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":1,"result":{"author":"0xf8b483dba2c3b7176a3da549ad41a48bb3121069","difficulty":"0xb6cb9824e57","extraData":"0xd983010302844765746887676f312e342e328777696e646f7773","gasLimit":"0x2fefd8","gasUsed":"0x3d860","hash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0xf8b483dba2c3b7176a3da549ad41a48bb3121069","mixHash":"0xcaf27314d80cb3e888d32646402d617d8f8379ca23a6b0255e974e407ffdd846","nonce":"0xbc7609306a77d0a2","number":"0xf423e","parentHash":"0xc6fd988b2d086a7b6eee3d25bad453830391014ba268cf6cc5d139741cb51273","receiptsRoot":"0xb0310e47b0cc7d3bb24c65ec21ec0ddf8dcf1672bc9866d6ba67e83d33215568","sealFields":["0xcaf27314d80cb3e888d32646402d617d8f8379ca23a6b0255e974e407ffdd846","0xbc7609306a77d0a2"],"sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":"0x764","stateRoot":"0xee8306f6cebba17153516cb6586de61d6294b49bc5534eb9378acb848907b277","timestamp":"0x56bfb3ed","totalDifficulty":"0x63053e0134c03db1","transactions":[{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0x6b5da959786d801c1bedda58f8a071a40f992f03","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x679c178c832194d3f40afbda60421e8cb12f2c6b879a925d2e60b15a2b4d212e","input":"0x","networkId":null,"nonce":"0x111","publicKey":"0x1acb54447b8e66222a23fe267f75e9c7ff46538e5c7b286ee14bcf7ec587f9656c5eb2163e6e3d7dbffd677de22e50d7e067dff34de403d14f5ead2eaf8368a5","r":"0xd5ad60765e2006490e73bf06f4bc9b382b2ea434eb066b60bc4f577cb056603a","raw":"0xf86e820111850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f64f66ddf683000801ca0d5ad60765e2006490e73bf06f4bc9b382b2ea434eb066b60bc4f577cb056603aa00e8d699411b71b08f550a278b05fb1d36174509758ad7370528ae06cb1965a8f","s":"0xe8d699411b71b08f550a278b05fb1d36174509758ad7370528ae06cb1965a8f","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x0","v":"0x1c","value":"0xf64f66ddf683000"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0x9da7521d2b2281b3cd477b553a5dc18b58674f07","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0xfe3189ab9a3c3aaa97a08e9410b6569f7528e38a4c86077ea20ddf33bd2c7ea5","input":"0x","networkId":null,"nonce":"0x79","publicKey":"0xa150bdb9419cf198e7430552880e8b050a09952ae53d1fd82d70941c6be318f21b98dcf93a974b763948c1621e460ec8cead12080fc2759c2e3e4dc884d2308b","r":"0xb31d8d88bfcf7a3dd705bc78a078c75542ca1a993860a3c95b2af317ee3a4b0d","raw":"0xf86c79850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880ef726f7729a1000801ca0b31d8d88bfcf7a3dd705bc78a078c75542ca1a993860a3c95b2af317ee3a4b0da076d529630cef5d1acf0d649faf281ebcb13768effce3eb02a96f5228ad2f5333","s":"0x76d529630cef5d1acf0d649faf281ebcb13768effce3eb02a96f5228ad2f5333","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x1","v":"0x1c","value":"0xef726f7729a1000"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0x707868ea3bfb73007106cfd30f678fdb94d12173","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0xcb7508e8703535fbc801146fa3c7d04798d71a9a0e3bb97a0a14beb733559672","input":"0x","networkId":null,"nonce":"0x251","publicKey":"0x030ad57f373be3cd858bb949365b1438b4383b94fa1b95af0ab5337719539fded4494868e0a82e6df40cddeb9415d8e45a6506ea77c1909c71dd2ec37316da0a","r":"0xbfc3a164f96f95f04ec50af58645d5cf51eaa2473872af9bf23ceab22560e8d6","raw":"0xf86e820251850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88881fc1efd41e37c800801ba0bfc3a164f96f95f04ec50af58645d5cf51eaa2473872af9bf23ceab22560e8d6a053f43d489fd83f8e2c9acbf2d14695c63838c18f420021771f111750aac8efba","s":"0x53f43d489fd83f8e2c9acbf2d14695c63838c18f420021771f111750aac8efba","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x2","v":"0x1b","value":"0x1fc1efd41e37c800"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0xd614cc8e7d44e6e5d48b9b3efd5ffec36098f403","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0xf333f42badd731da2869ce92d95a255f75ac2a16ed043e6b343ed91d4fdbb579","input":"0x","networkId":null,"nonce":"0x18c","publicKey":"0x34ff9f742cb0c7feaf8109a722d4518fd504abedc4f66e4e6bf8ece0726841c132e5660bbabe5dbe83414cda8ddb5b0aae4a649661747a817cfb79045c22d419","r":"0x32a184bbbe6168a2ebfba1be61d3535d45ce580b130eed8df8f5024be97f5bf8","raw":"0xf86e82018c850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880eeee41c060f2400801ca032a184bbbe6168a2ebfba1be61d3535d45ce580b130eed8df8f5024be97f5bf8a071c020aef32840e0f4f5ea2b095faa4602586a471d33c62563146314c4970a93","s":"0x71c020aef32840e0f4f5ea2b095faa4602586a471d33c62563146314c4970a93","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x3","v":"0x1c","value":"0xeeee41c060f2400"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0x078838304c9ee678209ea0959587da9b6f31ebff","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0xfa50db902c56466492e9f32fd543edaa1554a47b2e288175c262685df0537106","input":"0x","networkId":null,"nonce":"0xf46","publicKey":"0x866ede0bed987e0e8736cc94244640df1124b5b789b780bc012b936c2559cc630102e32c1c454f92626542eca44802f3ee44437a031fa1eaabcbdf323891eb93","r":"0x9a569d066c62c64ec8b93c6d268499a276fe882289f6090e65748911ec81b256","raw":"0xf86e820f46850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880e62a83e59ffa400801ca09a569d066c62c64ec8b93c6d268499a276fe882289f6090e65748911ec81b256a01e7b9216b86d6a5517b88a2aaef666732c51486214948fdecd89b9043a30750c","s":"0x1e7b9216b86d6a5517b88a2aaef666732c51486214948fdecd89b9043a30750c","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x4","v":"0x1c","value":"0xe62a83e59ffa400"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0x460825a3542f4823818184020ba3861da1e26872","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x0b4a6c8459c02f647d8a5c667e292de3e45c5f03558a0e814377e5356ebc6234","input":"0x","networkId":null,"nonce":"0x113","publicKey":"0xee1a6b3dc03e8b5329d99b77c33f64767196ce47236b4c9ee2baa87827a6348488926ae6da54abbf788f5d2602dff65984a60020407e7e8b2da160f32e80a344","r":"0x87842eacb46cc63064a8a8f0932ce3f18c0d27f81a8124d2c3a9f751293b11d0","raw":"0xf86e820113850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880efd50e050f64400801ca087842eacb46cc63064a8a8f0932ce3f18c0d27f81a8124d2c3a9f751293b11d0a04e7678e22ce8ec60a04c36fa5685421a3bf8b9d0ff68280a8f31d6db49629afe","s":"0x4e7678e22ce8ec60a04c36fa5685421a3bf8b9d0ff68280a8f31d6db49629afe","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x5","v":"0x1c","value":"0xefd50e050f64400"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0xa29862fb7f9b37374d0c9062ab52bdd74d1af867","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0xc4ea04477167cc599788100bef3306eca140549e747ba531db579eb2a72b1b11","input":"0x","networkId":null,"nonce":"0x59a","publicKey":"0xa3e333b30947a5a685b47b387a92f65a7c5d7b61f6f3016777f720e83fea9fbe5faf6fcb3296e0cd9da6ec9acf30920d5d67c2c4636a79f940b6e2fbe46c14a7","r":"0x90ddc9473c323eebd5c4a35251cd437e62563c883e8e87b141389fde111c5b24","raw":"0xf86e82059a850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f32a22e7fc0f800801ca090ddc9473c323eebd5c4a35251cd437e62563c883e8e87b141389fde111c5b24a039a1dfc3e2b85c74fce62ed7369ac1a62de13b31f4fb47e5fb02232aeefd83f4","s":"0x39a1dfc3e2b85c74fce62ed7369ac1a62de13b31f4fb47e5fb02232aeefd83f4","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x6","v":"0x1c","value":"0xf32a22e7fc0f800"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0x771dd02681c793eb34eff34528309e3657f843fb","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0xf73f661edcb6e8fc0b48a5bb5292e8b5db8ea911e4664ed1f8af1b2e66f6f585","input":"0x","networkId":null,"nonce":"0x211","publicKey":"0xca6db6e9182a094b5cbfa68741ab7c31450582eb65f1c558798a08b230de63a2f25deedc62d276a5f3eef3526282e28c7efdbbcba8e3ed4dad086c2201f10855","r":"0x7ecfd78b2838d73283f6de62bee1a046830fac75fb5b85ede279dbac097feec6","raw":"0xf86e820211850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d888811a2bd08b7075400801ba07ecfd78b2838d73283f6de62bee1a046830fac75fb5b85ede279dbac097feec6a01cfc1ced8140efc2dc71e217d6693665942ef1424affd7d61c134ed462605922","s":"0x1cfc1ced8140efc2dc71e217d6693665942ef1424affd7d61c134ed462605922","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x7","v":"0x1b","value":"0x11a2bd08b7075400"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0xfbe56e8afb28e097a871b2747800079ad5c29c03","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x9b2569e1b26d29730cf262756a6033834e34345f4a18caa241117747ce8cf746","input":"0x","networkId":null,"nonce":"0x6c","publicKey":"0x7c2ee029ec45aa73444091d1a0c3f830bb7f91797b30a1f53c11a2fbec10f7bb7706a9569350da382cc623c2b65d03b480ae96bc168021da4f0df60146f9e16c","r":"0xb1f3b2754a9189b376bc32d03a1097d4fe0cfaae3e55e45a4249127b9b541399","raw":"0xf86c6c850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f94ad612cf85000801ca0b1f3b2754a9189b376bc32d03a1097d4fe0cfaae3e55e45a4249127b9b541399a025b51f84e621e9193dfb7172dfdea0379bbf8d5d73e25de0e2d0dc50f657e249","s":"0x25b51f84e621e9193dfb7172dfdea0379bbf8d5d73e25de0e2d0dc50f657e249","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x8","v":"0x1c","value":"0xf94ad612cf85000"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0xe6ea7febb65f6fb46dc42dea2f873c67aadb1f72","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x5ac04be22ee89dce8c33f334a41ab05e1cbeca16669003c5ffe2c220f772b097","input":"0x","networkId":null,"nonce":"0x170","publicKey":"0xa6238a7419a3321706c6612d7cc647bce4568ec6ce4a999d081077feac54ec8d1e2627484782a15a4a2c2eca0a71bee25b5a82a7ca74c84b75f89ec2f8bbb5ea","r":"0x3c26e80876f0901d3007a8798f9792d426b6f78079dcd06d91019677850b9356","raw":"0xf86e820170850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d8888115740dac6be2400801ca03c26e80876f0901d3007a8798f9792d426b6f78079dcd06d91019677850b9356a028a644324a777b7beade6b8432d6f95f85112863e08c50bd3e22d1594244014c","s":"0x28a644324a777b7beade6b8432d6f95f85112863e08c50bd3e22d1594244014c","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x9","v":"0x1c","value":"0x115740dac6be2400"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0x2a65aca4d5fc5b5c859090a6c34d164135398226","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0x46a83d066750df27119aa3e314641fb3b3ec6e1afc1e768d3da4ac941a6a0a8d","input":"0x","networkId":null,"nonce":"0x2a11d","publicKey":"0x4c3eb5e19c71d8245eaaaba21ef8f94a70e9250848d10ade086f893a7a33a06d7063590e9e6ca88f918d7704840d903298fe802b6047fa7f6d09603eba690c39","r":"0x85bada12a37f21016e8801d6136cd7793192346a0f29f4fd37782d774378a7df","raw":"0xf8708302a11d850ba43b740083015f90945d65e227f4e7bc798cf62526f4bdd47c82e6a590880eb35d6f4e620c00801ca085bada12a37f21016e8801d6136cd7793192346a0f29f4fd37782d774378a7dfa07e1c78a62e1c16b955dc1b56f657c51fe2dfb739c2c1d11fe4845583706719a8","s":"0x7e1c78a62e1c16b955dc1b56f657c51fe2dfb739c2c1d11fe4845583706719a8","standardV":"0x1","to":"0x5d65e227f4e7bc798cf62526f4bdd47c82e6a590","transactionIndex":"0xa","v":"0x1c","value":"0xeb35d6f4e620c00"},{"blockHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","blockNumber":"0xf423e","condition":null,"creates":null,"from":"0x2a65aca4d5fc5b5c859090a6c34d164135398226","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0x15dd5bba84901824fb3aa75618a92b7cbacb454c53eaa962a2ca8667acb06a78","input":"0x","networkId":null,"nonce":"0x2a11e","publicKey":"0x4c3eb5e19c71d8245eaaaba21ef8f94a70e9250848d10ade086f893a7a33a06d7063590e9e6ca88f918d7704840d903298fe802b6047fa7f6d09603eba690c39","r":"0x1611395215c0ede475af6fd3b647c674d18735851060ccad0e0e7a7c150831c9","raw":"0xf8708302a11e850ba43b740083015f909436fab08874deb6cd0e7f916ddee8957630073d47880eb1fbb47be3f800801ca01611395215c0ede475af6fd3b647c674d18735851060ccad0e0e7a7c150831c9a0333716a13f040cbd8ac43462b9cfa8d602d4a3413825d283705bc3d4b22af8de","s":"0x333716a13f040cbd8ac43462b9cfa8d602d4a3413825d283705bc3d4b22af8de","standardV":"0x1","to":"0x36fab08874deb6cd0e7f916ddee8957630073d47","transactionIndex":"0xb","v":"0x1c","value":"0xeb1fbb47be3f800"}],"transactionsRoot":"0x6414d72a4c223bce7d1309869332b148670eb66af4e3b3ba6d1a55aa0bb3fd4f","uncles":[]}} \ No newline at end of file diff --git a/statediff/indexer/ipld/test_data/eth-block-body-json-999999 b/statediff/indexer/ipld/test_data/eth-block-body-json-999999 new file mode 100644 index 000000000000..de007b641fb5 --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-block-body-json-999999 @@ -0,0 +1 @@ +{"jsonrpc":"2.0","result":{"author":"0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5","difficulty":"0xb6b4beb1e8e","extraData":"0xd783010303844765746887676f312e342e32856c696e7578","gasLimit":"0x2fefd8","gasUsed":"0x38658","hash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5","mixHash":"0x5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0","nonce":"0xf491f46b60fe04b3","number":"0xf423f","parentHash":"0xd33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4","receiptsRoot":"0x7fa0f6ca2a01823208d80801edad37e3e3a003b55c89319b45eb1f97862ad229","sealFields":["0xa05b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0","0x88f491f46b60fe04b3"],"sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":"0x6e8","stateRoot":"0xed98aa4b5b19c82fb35364f08508ae0a6dec665fa57663dca94c5d70554cde10","timestamp":"0x56bfb405","totalDifficulty":"0x6305496c80ab5c3f","transactions":[{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0xc3665b8a9224ba8da9a20322f31d599cafa52c5c","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x22879e0bc9602fef59dc0602f9bc385f12632da5cb4eee4b813a0c27159c4d24","input":"0x","networkId":null,"nonce":"0x1d3","publicKey":"0xc3dbee74f1b2b8dbedc417244b7f5a134c6f7769faf9ffe784b3f0fdda7ca52cf914d3f2b3164c009bf939796b77f047ccb4cc113d3bde5b06555b781e0c7149","r":"0x43531017f1569ec692c0bf1ad710ddb5158b60505ea33fb7a21245738539e2d5","raw":"0xf86e8201d3850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d8888102363ac310a4000801ca043531017f1569ec692c0bf1ad710ddb5158b60505ea33fb7a21245738539e2d5a03856c6a1117ff71e9b769ccb6960674038a3326c3dd84c152fc83ada28145a07","s":"0x3856c6a1117ff71e9b769ccb6960674038a3326c3dd84c152fc83ada28145a07","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x0","v":"0x1c","value":"0x102363ac310a4000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x4ce758b0c8aa655b77c14f16bd0190b5715be75a","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x3c634bf5f09f6b5b5ea377df7abb483f422ae5d4ba389c395f14f833de25d362","input":"0x","networkId":null,"nonce":"0x9","publicKey":"0x75022ee25c702fc6a53853843e00e87877e737f9c631a9d831c11693d7e31877a1b09755ab3a5c112decf57339839364b8b9a3c23ada01761b1e3a044e297316","r":"0x8219a4f30cb8dd7d5e1163ac433f207b599d804b0d74ee54c8694014db647700","raw":"0xf86c09850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880ed350879ce50000801ba08219a4f30cb8dd7d5e1163ac433f207b599d804b0d74ee54c8694014db647700a03db2e806986a746d44d675fdbbd7594bb2856946ba257209abfffdd1628141af","s":"0x3db2e806986a746d44d675fdbbd7594bb2856946ba257209abfffdd1628141af","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x1","v":"0x1b","value":"0xed350879ce50000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x30906581413d556de1a018adbe6cc63c88d58512","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x59feccaad599e776cd6635e68b5e19254cca3b38e49437044f1e1d15d00b0576","input":"0x","networkId":null,"nonce":"0x59","publicKey":"0xccf6be26c1eb1c89d5fe958db0112a46e3ac23a95ac0f709ce84a49ae3f20bcf143909bfe67f685caaf362066e1c7e224899f57678bbcecb7a720175bcbb387d","r":"0x1ca26859a6eed116312010359c2e8351d126f31b078a0e2e19aae0acc98d9488","raw":"0xf86c59850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88882b0ca8b9f5f02000801ba01ca26859a6eed116312010359c2e8351d126f31b078a0e2e19aae0acc98d9488a0172c1a299737440a9063af6547d567ca7d269bfc2a9e81ec1de21aa8bd8e17b1","s":"0x172c1a299737440a9063af6547d567ca7d269bfc2a9e81ec1de21aa8bd8e17b1","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x2","v":"0x1b","value":"0x2b0ca8b9f5f02000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x8bec4e6fb1a28820eb1e8ec2d4eae4842ed2f923","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x98a03afa804e248ada5f26e9118ae927d4d3cb60e78c54938dced1cf25ee3567","input":"0x","networkId":null,"nonce":"0x2","publicKey":"0xbc8c89a85804c7859069c13561dbbd8d1d4739ec7d18514c42b3ffea64529cee522a5e20d93373d0074e94c4c7b6eba51c7d2f18ef7c64c37520342acb233795","r":"0xa5aca100a264a8da4a58bef77c5116a6dde42186ac249623c0edcb30189640a","raw":"0xf86c02850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880fd037ba87693800801ba00a5aca100a264a8da4a58bef77c5116a6dde42186ac249623c0edcb30189640aa0783e9439755023b919897574f94337aaac4a1ddc20217e3ac264a7edf813ffdd","s":"0x783e9439755023b919897574f94337aaac4a1ddc20217e3ac264a7edf813ffdd","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x3","v":"0x1b","value":"0xfd037ba87693800"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x4835a9626b02369546502d2949e16b0fda110b0c","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x18f1e6430334ad548bc36fc317016bc9f7a076d1fa50a89fe4e1d095ed3f9562","input":"0x","networkId":null,"nonce":"0xd9","publicKey":"0x91b3b4fe89d112cfc7308619e8aa7de86f14af3f6b6e4e92becb6e29e98207835bbe1a69109c16b14b0eb7285d2b952a9cde6007932afe95e81eefc183f75314","r":"0xb93c6f8dce800a1ec57d70813c4d35e3ffe25a6f1ae9057cf706636cf34d662","raw":"0xf86d81d9850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d888814bac05c835a5400801ba00b93c6f8dce800a1ec57d70813c4d35e3ffe25a6f1ae9057cf706636cf34d662a06d254a5557b7716ef01dd28aa84cc919f397c0a778f3a109a1ee9df2fc530ec0","s":"0x6d254a5557b7716ef01dd28aa84cc919f397c0a778f3a109a1ee9df2fc530ec0","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x4","v":"0x1b","value":"0x14bac05c835a5400"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x9cc72ebf3daaf12c72e48605e1e67b47c95a1911","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0xb1cada8daf63c45750df1ee79eed5a3cf6240e3cebdb6de3f26bc7cf03217bf4","input":"0x","networkId":null,"nonce":"0x34","publicKey":"0x90dff18c1c01d566e6d8bf0190e3e965f98e7f51ccbbe6040f9a9972e88f4ad19f1547406454fbc9e1ebcf4c5f2f1e2df9b9371028fe0a552ecca5f5f0aa4129","r":"0xe9a25c929c26d1a95232ba75aef419a91b470651eb77614695e16c5ba023e383","raw":"0xf86c34850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f258512af0d4000801ba0e9a25c929c26d1a95232ba75aef419a91b470651eb77614695e16c5ba023e383a0679fb2fc0d0b0f3549967c0894ee7d947f07d238a83ef745bc3ced5143a4af36","s":"0x679fb2fc0d0b0f3549967c0894ee7d947f07d238a83ef745bc3ced5143a4af36","standardV":"0x0","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x5","v":"0x1b","value":"0xf258512af0d4000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x5c51467399bc655f0cc6db88df15946717534633","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x4fa879b491e0779fc035758ec77b93c4e51d528d65b64eb055c015a58deff103","input":"0x","networkId":null,"nonce":"0x6f","publicKey":"0x0b7e2532afc2daa33763002525aa6c7edc25ea97d63baeeb2c6f5094f18dca4a0212b52061f9a9091aad5c4380a6506f9a51ddd2d014e78742bf144a58d6ffa0","r":"0x9e0b8360a36d6d0320aef19bd811431b1a692504549da9f05f9b4d9e329993b9","raw":"0xf86c6f850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88881c54e302456eb400801ca09e0b8360a36d6d0320aef19bd811431b1a692504549da9f05f9b4d9e329993b9a05acff70bd8cf82d9d70b11d4e59dc5d54937475ec394ec846263495f61e5e6ee","s":"0x5acff70bd8cf82d9d70b11d4e59dc5d54937475ec394ec846263495f61e5e6ee","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x6","v":"0x1c","value":"0x1c54e302456eb400"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x055d9d7ec193d1e062c6ec4fa80ef89b5c1258f4","gas":"0x5208","gasPrice":"0xdf8475800","hash":"0x1bea59827ab153b20cee79890d221a80fa6a04e552d667504c592ed314fb6d76","input":"0x","networkId":null,"nonce":"0x46","publicKey":"0xfae19a0ac08d36f0229663d45d0c41ca52c4e295c7af82a1b39515a79025175293400d026e0d41767aac42f8b7e4a6687c5762161457d753f1fc0766614868f9","r":"0xb2803f1bfa237bda762d214f71a4c71a7306f55df2880c77d746024e81ccbaa2","raw":"0xf86c46850df84758008252089432be343b94f860124dc4fee278fdcbd38c102d88880f0447b1edca4000801ca0b2803f1bfa237bda762d214f71a4c71a7306f55df2880c77d746024e81ccbaa2a07aeed35c0cbfbe0ed6552fd55b3f57fdc054eeabd02fc61bf66d9a8843aa593a","s":"0x7aeed35c0cbfbe0ed6552fd55b3f57fdc054eeabd02fc61bf66d9a8843aa593a","standardV":"0x1","to":"0x32be343b94f860124dc4fee278fdcbd38c102d88","transactionIndex":"0x7","v":"0x1c","value":"0xf0447b1edca4000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x8e68c0c9b5275fa684291304af9cafe6ceaf2772","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0x73e87db1108a2aa852f48e088ca1a2771f9b7c18af8d1bd77a3cdcc72a750c56","input":"0x","networkId":null,"nonce":"0x3","publicKey":"0xa5e423dfcbdbba1fdbb785367a88235fa2569061d72b6c715111ac21cbef8fc1db860acdef85f1408c760f34b28a4f07d950ac15c4b85d5e528e50f546a89b6d","r":"0x6dccb1349919662c40455aee04472ae307195580837510ecf2e6fc428876eb03","raw":"0xf86d03850ba43b740083015f909426016a2b5d872adc1b131a4cd9d4b18789d0d9eb88016345785d8a0000801ba06dccb1349919662c40455aee04472ae307195580837510ecf2e6fc428876eb03a03b84ea9c3c6462ac086a1d789a167c2735896a6b5a40e85a6e45da8884fe27de","s":"0x3b84ea9c3c6462ac086a1d789a167c2735896a6b5a40e85a6e45da8884fe27de","standardV":"0x0","to":"0x26016a2b5d872adc1b131a4cd9d4b18789d0d9eb","transactionIndex":"0x8","v":"0x1b","value":"0x16345785d8a0000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x2a65aca4d5fc5b5c859090a6c34d164135398226","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0x337a5e90b73f44ffebea73cb3d97738c524f63e1032b30735e43212cff731aee","input":"0x","networkId":null,"nonce":"0x2a11f","publicKey":"0x4c3eb5e19c71d8245eaaaba21ef8f94a70e9250848d10ade086f893a7a33a06d7063590e9e6ca88f918d7704840d903298fe802b6047fa7f6d09603eba690c39","r":"0xaa8909295ff178639df961126970f44b5d894326eb47cead161f6910799a98b8","raw":"0xf8708302a11f850ba43b740083015f90945275c3371ece4d4a5b1e14cf6dbfc2277d58ef92880e93ea6a35f2e000801ba0aa8909295ff178639df961126970f44b5d894326eb47cead161f6910799a98b8a0254d7742eccaf2f4c44bfe638378dcf42bdde9465f231b89003cc7927de5d46e","s":"0x254d7742eccaf2f4c44bfe638378dcf42bdde9465f231b89003cc7927de5d46e","standardV":"0x0","to":"0x5275c3371ece4d4a5b1e14cf6dbfc2277d58ef92","transactionIndex":"0x9","v":"0x1b","value":"0xe93ea6a35f2e000"},{"blockHash":"0xb4fbadf8ea452b139718e2700dc1135cfc81145031c84b7ab27cd710394f7b38","blockNumber":"0xf423f","condition":null,"creates":null,"from":"0x2a65aca4d5fc5b5c859090a6c34d164135398226","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0xc280ab030e20bc9ef72c87b420d58f598bda753ef80a53136a923848b0c89a5c","input":"0x","networkId":null,"nonce":"0x2a120","publicKey":"0x4c3eb5e19c71d8245eaaaba21ef8f94a70e9250848d10ade086f893a7a33a06d7063590e9e6ca88f918d7704840d903298fe802b6047fa7f6d09603eba690c39","r":"0xcfe3ad31d6612f8d787c45f115cc5b43fb22bcc210b62ae71dc7cbf0a6bea8df","raw":"0xf8708302a120850ba43b740083015f90941c51bf013add0857c5d9cf2f71a7f15ca93d4816880e917c4b10c87400801ca0cfe3ad31d6612f8d787c45f115cc5b43fb22bcc210b62ae71dc7cbf0a6bea8dfa057db8998114fae3c337e99dbd8573d4085691880f4576c6c1f6c5bbfe67d6cf0","s":"0x57db8998114fae3c337e99dbd8573d4085691880f4576c6c1f6c5bbfe67d6cf0","standardV":"0x1","to":"0x1c51bf013add0857c5d9cf2f71a7f15ca93d4816","transactionIndex":"0xa","v":"0x1c","value":"0xe917c4b10c87400"}],"transactionsRoot":"0x447cbd8c48f498a6912b10831cdff59c7fbfcbbe735ca92883d4fa06dcd7ae54","uncles":[]},"id":1} diff --git a/statediff/indexer/ipld/test_data/eth-block-body-rlp-997522 b/statediff/indexer/ipld/test_data/eth-block-body-rlp-997522 new file mode 100644 index 000000000000..ca176613e46a Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-block-body-rlp-997522 differ diff --git a/statediff/indexer/ipld/test_data/eth-block-body-rlp-999999 b/statediff/indexer/ipld/test_data/eth-block-body-rlp-999999 new file mode 100644 index 000000000000..3719c36d3ca6 Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-block-body-rlp-999999 differ diff --git a/statediff/indexer/ipld/test_data/eth-block-header-rlp-999996 b/statediff/indexer/ipld/test_data/eth-block-header-rlp-999996 new file mode 100644 index 000000000000..a573d8f8861a Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-block-header-rlp-999996 differ diff --git a/statediff/indexer/ipld/test_data/eth-block-header-rlp-999997 b/statediff/indexer/ipld/test_data/eth-block-header-rlp-999997 new file mode 100644 index 000000000000..3d3cf65afe54 Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-block-header-rlp-999997 differ diff --git a/statediff/indexer/ipld/test_data/eth-block-header-rlp-999999 b/statediff/indexer/ipld/test_data/eth-block-header-rlp-999999 new file mode 100644 index 000000000000..6b79b7056249 Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-block-header-rlp-999999 differ diff --git a/statediff/indexer/ipld/test_data/eth-state-trie-rlp-0e8b34 b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-0e8b34 new file mode 100644 index 000000000000..c423da569f65 Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-0e8b34 differ diff --git a/statediff/indexer/ipld/test_data/eth-state-trie-rlp-56864f b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-56864f new file mode 100644 index 000000000000..56a4c12323aa --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-56864f @@ -0,0 +1 @@ +FKd?f_YA( "a2cUSyI \ No newline at end of file diff --git a/statediff/indexer/ipld/test_data/eth-state-trie-rlp-6fc2d7 b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-6fc2d7 new file mode 100644 index 000000000000..b127a7f53bf3 --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-6fc2d7 @@ -0,0 +1,5 @@ +[G (oU,rBSs^^)77.Yt5_+9FYzFvڏb{(1Y-ʐ*f&HՂЪKUXv R%IJ/A?֦l@UwFI!-jZ9ݻgͳ.+5/ݽ fbfzW [ =@pNIӥRRVI B'ԍrYᤫW{i`Df pJW䌿ejpEﺇ +)j|Φt /;=HWN)i41?$_B7<0M +#k|_ *(Z_ABdfHLb-:F61ufE&Ǖ{rE\IeqEURhź1վ/,XZ: + iK7Š5.8١MQMw t 5R3ȝnInmVD-"54 +\`4A \ No newline at end of file diff --git a/statediff/indexer/ipld/test_data/eth-state-trie-rlp-727994 b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-727994 new file mode 100644 index 000000000000..16199c03469a Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-727994 differ diff --git a/statediff/indexer/ipld/test_data/eth-state-trie-rlp-c9070d b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-c9070d new file mode 100644 index 000000000000..f4dc3f809977 Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-c9070d differ diff --git a/statediff/indexer/ipld/test_data/eth-state-trie-rlp-d5be90 b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-d5be90 new file mode 100644 index 000000000000..4e840ee13bff Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-d5be90 differ diff --git a/statediff/indexer/ipld/test_data/eth-state-trie-rlp-d7f897 b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-d7f897 new file mode 100644 index 000000000000..140ce35c6ce9 Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-d7f897 differ diff --git a/statediff/indexer/ipld/test_data/eth-state-trie-rlp-eb2f5f b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-eb2f5f new file mode 100644 index 000000000000..86387106fc5c Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-state-trie-rlp-eb2f5f differ diff --git a/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-000dd0 b/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-000dd0 new file mode 100644 index 000000000000..2fbe90bd6756 Binary files /dev/null and b/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-000dd0 differ diff --git a/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-113049 b/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-113049 new file mode 100644 index 000000000000..e7407c417811 --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-113049 @@ -0,0 +1 @@ +JN>b$kg$2͠| d \ No newline at end of file diff --git a/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-9d1860 b/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-9d1860 new file mode 100644 index 000000000000..d39f6324f90c --- /dev/null +++ b/statediff/indexer/ipld/test_data/eth-storage-trie-rlp-9d1860 @@ -0,0 +1 @@ +˚?L=d@. + +package ipld + +import ( + "encoding/json" + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + + "github.com/ethereum/go-ethereum/rlp" +) + +const ( + extension = "extension" + leaf = "leaf" + branch = "branch" +) + +// TrieNode is the general abstraction for +//ethereum IPLD trie nodes. +type TrieNode struct { + // leaf, extension or branch + nodeKind string + + // If leaf or extension: [0] is key, [1] is val. + // If branch: [0] - [16] are children. + elements []interface{} + + // IPLD block information + cid cid.Cid + rawdata []byte +} + +/* + OUTPUT +*/ + +type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error) + +// decodeTrieNode returns a TrieNode object from an IPLD block's +// cid and rawdata. +func decodeTrieNode(c cid.Cid, b []byte, + leafDecoder trieNodeLeafDecoder) (*TrieNode, error) { + var ( + i, decoded, elements []interface{} + nodeKind string + err error + ) + + if err = rlp.DecodeBytes(b, &i); err != nil { + return nil, err + } + + codec := c.Type() + switch len(i) { + case 2: + nodeKind, decoded, err = decodeCompactKey(i) + if err != nil { + return nil, err + } + + if nodeKind == extension { + elements, err = parseTrieNodeExtension(decoded, codec) + if err != nil { + return nil, err + } + } + if nodeKind == leaf { + elements, err = leafDecoder(decoded) + if err != nil { + return nil, err + } + } + if nodeKind != extension && nodeKind != leaf { + return nil, fmt.Errorf("unexpected nodeKind returned from decoder") + } + case 17: + nodeKind = branch + elements, err = parseTrieNodeBranch(i, codec) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown trie node type") + } + + return &TrieNode{ + nodeKind: nodeKind, + elements: elements, + rawdata: b, + cid: c, + }, nil +} + +// decodeCompactKey takes a compact key, and returns its nodeKind and value. +func decodeCompactKey(i []interface{}) (string, []interface{}, error) { + first := i[0].([]byte) + last := i[1].([]byte) + + switch first[0] / 16 { + case '\x00': + return extension, []interface{}{ + nibbleToByte(first)[2:], + last, + }, nil + case '\x01': + return extension, []interface{}{ + nibbleToByte(first)[1:], + last, + }, nil + case '\x02': + return leaf, []interface{}{ + nibbleToByte(first)[2:], + last, + }, nil + case '\x03': + return leaf, []interface{}{ + nibbleToByte(first)[1:], + last, + }, nil + default: + return "", nil, fmt.Errorf("unknown hex prefix") + } +} + +// parseTrieNodeExtension helper improves readability +func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) { + return []interface{}{ + i[0].([]byte), + keccak256ToCid(codec, i[1].([]byte)), + }, nil +} + +// parseTrieNodeBranch helper improves readability +func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) { + var out []interface{} + + for i, vi := range i { + v, ok := vi.([]byte) + // Sometimes this throws "panic: interface conversion: interface {} is []interface {}, not []uint8" + // Figure out why, and if it is okay to continue + if !ok { + return nil, fmt.Errorf("unable to decode branch node entry into []byte at position: %d value: %+v", i, vi) + } + + switch len(v) { + case 0: + out = append(out, nil) + case 32: + out = append(out, keccak256ToCid(codec, v)) + default: + return nil, fmt.Errorf("unrecognized object: %v", v) + } + } + + return out, nil +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) { + switch t.nodeKind { + case extension: + return t.resolveTrieNodeExtension(p) + case leaf: + return t.resolveTrieNodeLeaf(p) + case branch: + return t.resolveTrieNodeBranch(p) + default: + return nil, nil, fmt.Errorf("nodeKind case not implemented") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (t *TrieNode) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + var out []string + + switch t.nodeKind { + case extension: + var val string + for _, e := range t.elements[0].([]byte) { + val += fmt.Sprintf("%x", e) + } + return []string{val} + case branch: + for i, elem := range t.elements { + if _, ok := elem.(cid.Cid); ok { + out = append(out, fmt.Sprintf("%x", i)) + } + } + return out + + default: + return nil + } +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := t.Resolve(p) + if err != nil { + return nil, nil, err + } + + lnk, ok := obj.(*node.Link) + if !ok { + return nil, nil, fmt.Errorf("was not a link") + } + + return lnk, rest, nil +} + +// Copy will go away. It is here to comply with the interface. +func (t *TrieNode) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (t *TrieNode) Links() []*node.Link { + var out []*node.Link + + for _, i := range t.elements { + c, ok := i.(cid.Cid) + if ok { + out = append(out, &node.Link{Cid: c}) + } + } + + return out +} + +// Stat will go away. It is here to comply with the interface. +func (t *TrieNode) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (t *TrieNode) Size() (uint64, error) { + return 0, nil +} + +/* + TrieNode functions +*/ + +// MarshalJSON processes the transaction trie into readable JSON format. +func (t *TrieNode) MarshalJSON() ([]byte, error) { + var out map[string]interface{} + + switch t.nodeKind { + case extension: + fallthrough + case leaf: + var hexPrefix string + for _, e := range t.elements[0].([]byte) { + hexPrefix += fmt.Sprintf("%x", e) + } + + // if we got a byte we need to do this casting otherwise + // it will be marshaled to a base64 encoded value + if _, ok := t.elements[1].([]byte); ok { + var hexVal string + for _, e := range t.elements[1].([]byte) { + hexVal += fmt.Sprintf("%x", e) + } + + t.elements[1] = hexVal + } + + out = map[string]interface{}{ + "type": t.nodeKind, + hexPrefix: t.elements[1], + } + + case branch: + out = map[string]interface{}{ + "type": branch, + "0": t.elements[0], + "1": t.elements[1], + "2": t.elements[2], + "3": t.elements[3], + "4": t.elements[4], + "5": t.elements[5], + "6": t.elements[6], + "7": t.elements[7], + "8": t.elements[8], + "9": t.elements[9], + "a": t.elements[10], + "b": t.elements[11], + "c": t.elements[12], + "d": t.elements[13], + "e": t.elements[14], + "f": t.elements[15], + } + default: + return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind) + } + + return json.Marshal(out) +} + +// nibbleToByte expands the nibbles of a byte slice into their own bytes. +func nibbleToByte(k []byte) []byte { + var out []byte + + for _, b := range k { + out = append(out, b/16) + out = append(out, b%16) + } + + return out +} + +// Resolve reading conveniences +func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) { + nibbles := t.elements[0].([]byte) + idx, rest := shiftFromPath(p, len(nibbles)) + if len(idx) < len(nibbles) { + return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension") + } + + for _, i := range idx { + if getHexIndex(string(i)) == -1 { + return nil, nil, fmt.Errorf("invalid path element") + } + } + + for i, n := range nibbles { + if string(idx[i]) != fmt.Sprintf("%x", n) { + return nil, nil, fmt.Errorf("no such link in this extension") + } + } + + return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil +} + +func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) { + nibbles := t.elements[0].([]byte) + + if len(nibbles) != 0 { + idx, rest := shiftFromPath(p, len(nibbles)) + if len(idx) < len(nibbles) { + return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf") + } + + for _, i := range idx { + if getHexIndex(string(i)) == -1 { + return nil, nil, fmt.Errorf("invalid path element") + } + } + + for i, n := range nibbles { + if string(idx[i]) != fmt.Sprintf("%x", n) { + return nil, nil, fmt.Errorf("no such link in this extension") + } + } + + p = rest + } + + link, ok := t.elements[1].(node.Node) + if !ok { + return nil, nil, fmt.Errorf("leaf children is not an IPLD node") + } + + return link.Resolve(p) +} + +func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) { + idx, rest := shiftFromPath(p, 1) + hidx := getHexIndex(idx) + if hidx == -1 { + return nil, nil, fmt.Errorf("incorrect path") + } + + child := t.elements[hidx] + if child != nil { + return &node.Link{Cid: child.(cid.Cid)}, rest, nil + } + return nil, nil, fmt.Errorf("no such link in this branch") +} + +// shiftFromPath extracts from a given path (as a slice of strings) +// the given number of elements as a single string, returning whatever +// it has not taken. +// +// Examples: +// ["0", "a", "something"] and 1 -> "0" and ["a", "something"] +// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"] +// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"] +func shiftFromPath(p []string, i int) (string, []string) { + var ( + out string + rest []string + ) + + for _, pe := range p { + re := "" + for _, c := range pe { + if len(out) < i { + out += string(c) + } else { + re += string(c) + } + } + + if len(out) == i && re != "" { + rest = append(rest, re) + } + } + + return out, rest +} + +// getHexIndex returns to you the integer 0 - 15 equivalent to your +// string character if applicable, or -1 otherwise. +func getHexIndex(s string) int { + if len(s) != 1 { + return -1 + } + + c := s[0] + switch { + case '0' <= c && c <= '9': + return int(c - '0') + case 'a' <= c && c <= 'f': + return int(c - 'a' + 10) + } + + return -1 +} diff --git a/statediff/indexer/mainnet_data/block_12579670.rlp b/statediff/indexer/mainnet_data/block_12579670.rlp new file mode 100644 index 000000000000..6b4f3f773f27 Binary files /dev/null and b/statediff/indexer/mainnet_data/block_12579670.rlp differ diff --git a/statediff/indexer/mainnet_data/block_12600011.rlp b/statediff/indexer/mainnet_data/block_12600011.rlp new file mode 100644 index 000000000000..96032b0c2152 Binary files /dev/null and b/statediff/indexer/mainnet_data/block_12600011.rlp differ diff --git a/statediff/indexer/mainnet_data/block_12619985.rlp b/statediff/indexer/mainnet_data/block_12619985.rlp new file mode 100644 index 000000000000..0e735313f540 Binary files /dev/null and b/statediff/indexer/mainnet_data/block_12619985.rlp differ diff --git a/statediff/indexer/mainnet_data/block_12625121.rlp b/statediff/indexer/mainnet_data/block_12625121.rlp new file mode 100644 index 000000000000..d031e30ea6ca Binary files /dev/null and b/statediff/indexer/mainnet_data/block_12625121.rlp differ diff --git a/statediff/indexer/mainnet_data/block_12655432.rlp b/statediff/indexer/mainnet_data/block_12655432.rlp new file mode 100644 index 000000000000..fafc6bd88eb6 Binary files /dev/null and b/statediff/indexer/mainnet_data/block_12655432.rlp differ diff --git a/statediff/indexer/mainnet_data/block_12914664.rlp b/statediff/indexer/mainnet_data/block_12914664.rlp new file mode 100644 index 000000000000..b8aaeaa61e65 Binary files /dev/null and b/statediff/indexer/mainnet_data/block_12914664.rlp differ diff --git a/statediff/indexer/mainnet_data/receipts_12579670.rlp b/statediff/indexer/mainnet_data/receipts_12579670.rlp new file mode 100644 index 000000000000..e69de8fd9b61 Binary files /dev/null and b/statediff/indexer/mainnet_data/receipts_12579670.rlp differ diff --git a/statediff/indexer/mainnet_data/receipts_12600011.rlp b/statediff/indexer/mainnet_data/receipts_12600011.rlp new file mode 100644 index 000000000000..ae6d4f0c2d08 Binary files /dev/null and b/statediff/indexer/mainnet_data/receipts_12600011.rlp differ diff --git a/statediff/indexer/mainnet_data/receipts_12619985.rlp b/statediff/indexer/mainnet_data/receipts_12619985.rlp new file mode 100644 index 000000000000..a9ba84bd275d Binary files /dev/null and b/statediff/indexer/mainnet_data/receipts_12619985.rlp differ diff --git a/statediff/indexer/mainnet_data/receipts_12625121.rlp b/statediff/indexer/mainnet_data/receipts_12625121.rlp new file mode 100644 index 000000000000..4d3a8532c2bc Binary files /dev/null and b/statediff/indexer/mainnet_data/receipts_12625121.rlp differ diff --git a/statediff/indexer/mainnet_data/receipts_12655432.rlp b/statediff/indexer/mainnet_data/receipts_12655432.rlp new file mode 100644 index 000000000000..f209f01d884f Binary files /dev/null and b/statediff/indexer/mainnet_data/receipts_12655432.rlp differ diff --git a/statediff/indexer/mainnet_data/receipts_12914664.rlp b/statediff/indexer/mainnet_data/receipts_12914664.rlp new file mode 100644 index 000000000000..3cf8e88951e7 Binary files /dev/null and b/statediff/indexer/mainnet_data/receipts_12914664.rlp differ diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go new file mode 100644 index 000000000000..eaa16c1fc7ff --- /dev/null +++ b/statediff/indexer/mocks/test_data.go @@ -0,0 +1,609 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package mocks + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/test_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" +) + +// Test variables +var ( + // block data + // TODO: Update this to `MainnetChainConfig` when `LondonBlock` is added + TestConfig = params.RopstenChainConfig + BlockNumber = TestConfig.LondonBlock + + // canonical block at London height + // includes 5 transactions: 3 Legacy + 1 EIP-2930 + 1 EIP-1559 + MockHeader = types.Header{ + Time: 0, + Number: new(big.Int).Set(BlockNumber), + Root: common.HexToHash("0x0"), + TxHash: common.HexToHash("0x0"), + ReceiptHash: common.HexToHash("0x0"), + Difficulty: big.NewInt(5000000), + Extra: []byte{}, + BaseFee: big.NewInt(params.InitialBaseFee), + Coinbase: common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476777"), + } + MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts(TestConfig, BlockNumber) + MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, new(trie.Trie)) + MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) + + // non-canonical block at London height + // includes 2nd and 5th transactions from the canonical block + MockNonCanonicalHeader = MockHeader + MockNonCanonicalBlockTransactions = types.Transactions{MockTransactions[1], MockTransactions[4]} + MockNonCanonicalBlockReceipts = createNonCanonicalBlockReceipts(TestConfig, BlockNumber, MockNonCanonicalBlockTransactions) + MockNonCanonicalBlock = types.NewBlock(&MockNonCanonicalHeader, MockNonCanonicalBlockTransactions, nil, MockNonCanonicalBlockReceipts, new(trie.Trie)) + MockNonCanonicalHeaderRlp, _ = rlp.EncodeToBytes(MockNonCanonicalBlock.Header()) + + // non-canonical block at London height + 1 + // includes 3rd and 5th transactions from the canonical block + Block2Number = big.NewInt(BlockNumber.Int64() + 1) + MockNonCanonicalHeader2 = types.Header{ + Time: 0, + Number: new(big.Int).Set(Block2Number), + Root: common.HexToHash("0x0"), + TxHash: common.HexToHash("0x0"), + ReceiptHash: common.HexToHash("0x0"), + Difficulty: big.NewInt(6000000), + Extra: []byte{}, + BaseFee: big.NewInt(params.InitialBaseFee), + Coinbase: common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476777"), + } + MockNonCanonicalBlock2Transactions = types.Transactions{MockTransactions[2], MockTransactions[4]} + MockNonCanonicalBlock2Receipts = createNonCanonicalBlockReceipts(TestConfig, Block2Number, MockNonCanonicalBlock2Transactions) + MockNonCanonicalBlock2 = types.NewBlock(&MockNonCanonicalHeader2, MockNonCanonicalBlock2Transactions, nil, MockNonCanonicalBlock2Receipts, new(trie.Trie)) + MockNonCanonicalHeader2Rlp, _ = rlp.EncodeToBytes(MockNonCanonicalBlock2.Header()) + + Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") + ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) + ContractAddress2 = crypto.CreateAddress(SenderAddr, MockTransactions[3].Nonce()) + MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} + mockTopic11 = common.HexToHash("0x04") + mockTopic12 = common.HexToHash("0x06") + mockTopic21 = common.HexToHash("0x05") + mockTopic22 = common.HexToHash("0x07") + ExpectedPostStatus uint64 = 1 + ExpectedPostState1 = common.Bytes2Hex(common.HexToHash("0x1").Bytes()) + ExpectedPostState2 = common.Bytes2Hex(common.HexToHash("0x2").Bytes()) + ExpectedPostState3 = common.Bytes2Hex(common.HexToHash("0x3").Bytes()) + MockLog1 = &types.Log{ + Address: Address, + Topics: []common.Hash{mockTopic11, mockTopic12}, + Data: []byte{}, + } + MockLog2 = &types.Log{ + Address: AnotherAddress, + Topics: []common.Hash{mockTopic21, mockTopic22}, + Data: []byte{}, + } + MockLog3 = &types.Log{ + Address: Address, + Topics: []common.Hash{mockTopic11, mockTopic22}, + Data: []byte{}, + } + MockLog4 = &types.Log{ + Address: AnotherAddress, + Topics: []common.Hash{mockTopic21, mockTopic12}, + Data: []byte{}, + } + ShortLog1 = &types.Log{ + Address: AnotherAddress, + Topics: []common.Hash{}, + Data: []byte{}, + } + ShortLog2 = &types.Log{ + Address: Address, + Topics: []common.Hash{}, + Data: []byte{}, + } + + // access list entries + AccessListEntry1 = types.AccessTuple{ + Address: Address, + } + AccessListEntry2 = types.AccessTuple{ + Address: AnotherAddress, + StorageKeys: []common.Hash{common.BytesToHash(StorageLeafKey), common.BytesToHash(MockStorageLeafKey)}, + } + AccessListEntry1Model = models.AccessListElementModel{ + BlockNumber: BlockNumber.String(), + Index: 0, + Address: Address.Hex(), + } + AccessListEntry2Model = models.AccessListElementModel{ + BlockNumber: BlockNumber.String(), + Index: 1, + Address: AnotherAddress.Hex(), + StorageKeys: []string{common.BytesToHash(StorageLeafKey).Hex(), common.BytesToHash(MockStorageLeafKey).Hex()}, + } + + // statediff data + storageLocation = common.HexToHash("0") + StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes() + mockStorageLocation = common.HexToHash("1") + MockStorageLeafKey = crypto.Keccak256Hash(mockStorageLocation[:]).Bytes() + StorageValue = common.Hex2Bytes("01") + StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") + StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + StoragePartialPath, + StorageValue, + }) + + nonce1 = uint64(1) + ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea") + ContractLeafKey = test_helpers.AddressToLeafKey(ContractAddress) + ContractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: ContractCodeHash.Bytes(), + Root: common.HexToHash(ContractRoot), + }) + ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45") + ContractLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + ContractPartialPath, + ContractAccount, + }) + + Contract2LeafKey = test_helpers.AddressToLeafKey(ContractAddress2) + storage2Location = common.HexToHash("2") + Storage2LeafKey = crypto.Keccak256Hash(storage2Location[:]).Bytes() + storage3Location = common.HexToHash("3") + Storage3LeafKey = crypto.Keccak256Hash(storage3Location[:]).Bytes() + + nonce0 = uint64(0) + AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + AccountLeafKey = test_helpers.Account2LeafKey + RemovedLeafKey = test_helpers.Account1LeafKey + Account, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: nonce0, + Balance: big.NewInt(1000), + CodeHash: AccountCodeHash.Bytes(), + Root: common.HexToHash(AccountRoot), + }) + AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45") + AccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + AccountPartialPath, + Account, + }) + + StateDiffs = []sdtypes.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: ContractLeafKey, + NodeValue: ContractLeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + LeafKey: StorageLeafKey, + NodeValue: StorageLeafNode, + }, + { + Path: []byte{'\x03'}, + NodeType: sdtypes.Removed, + LeafKey: RemovedLeafKey, + NodeValue: []byte{}, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: AccountLeafKey, + NodeValue: AccountLeafNode, + StorageNodes: []sdtypes.StorageNode{}, + }, + { + Path: []byte{'\x02'}, + NodeType: sdtypes.Removed, + LeafKey: RemovedLeafKey, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x07'}, + NodeType: sdtypes.Removed, + LeafKey: Contract2LeafKey, + NodeValue: []byte{}, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Removed, + LeafKey: Storage2LeafKey, + NodeValue: []byte{}, + }, + { + Path: []byte{'\x0f'}, + NodeType: sdtypes.Removed, + LeafKey: Storage3LeafKey, + NodeValue: []byte{}, + }, + }, + }, + } + + // Mock data for testing watched addresses methods + Contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F" + Contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B" + Contract3Address = "0xcfeB164C328CA13EFd3C77E1980d94975aDfedfc" + Contract4Address = "0x0Edf0c4f393a628DE4828B228C48175b3EA297fc" + Contract1CreatedAt = uint64(1) + Contract2CreatedAt = uint64(2) + Contract3CreatedAt = uint64(3) + Contract4CreatedAt = uint64(4) + + LastFilledAt = uint64(0) + WatchedAt1 = uint64(10) + WatchedAt2 = uint64(15) + WatchedAt3 = uint64(20) +) + +type LegacyData struct { + Config *params.ChainConfig + BlockNumber *big.Int + MockHeader types.Header + MockTransactions types.Transactions + MockReceipts types.Receipts + SenderAddr common.Address + MockBlock *types.Block + MockHeaderRlp []byte + Address []byte + AnotherAddress []byte + ContractAddress common.Address + MockContractByteCode []byte + MockLog1 *types.Log + MockLog2 *types.Log + StorageLeafKey []byte + MockStorageLeafKey []byte + StorageLeafNode []byte + ContractLeafKey []byte + ContractAccount []byte + ContractPartialPath []byte + ContractLeafNode []byte + AccountRoot string + AccountLeafNode []byte + StateDiffs []sdtypes.StateNode +} + +func NewLegacyData(config *params.ChainConfig) *LegacyData { + // Block number before london fork. + blockNumber := config.EIP155Block + + mockHeader := types.Header{ + Time: 0, + Number: new(big.Int).Set(blockNumber), + Root: common.HexToHash("0x0"), + TxHash: common.HexToHash("0x0"), + ReceiptHash: common.HexToHash("0x0"), + Difficulty: big.NewInt(5000000), + Extra: []byte{}, + Coinbase: common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476888"), + } + + mockTransactions, mockReceipts, senderAddr := createLegacyTransactionsAndReceipts(config, blockNumber) + mockBlock := types.NewBlock(&mockHeader, mockTransactions, nil, mockReceipts, new(trie.Trie)) + mockHeaderRlp, _ := rlp.EncodeToBytes(mockBlock.Header()) + contractAddress := crypto.CreateAddress(senderAddr, mockTransactions[2].Nonce()) + + return &LegacyData{ + Config: config, + BlockNumber: blockNumber, + MockHeader: mockHeader, + MockTransactions: mockTransactions, + MockReceipts: mockReceipts, + SenderAddr: senderAddr, + MockBlock: mockBlock, + MockHeaderRlp: mockHeaderRlp, + ContractAddress: contractAddress, + MockContractByteCode: MockContractByteCode, + MockLog1: MockLog1, + MockLog2: MockLog2, + StorageLeafKey: StorageLeafKey, + MockStorageLeafKey: MockStorageLeafKey, + StorageLeafNode: StorageLeafNode, + ContractLeafKey: ContractLeafKey, + ContractAccount: ContractAccount, + ContractPartialPath: ContractPartialPath, + ContractLeafNode: ContractLeafNode, + AccountRoot: AccountRoot, + AccountLeafNode: AccountLeafKey, + StateDiffs: StateDiffs, + } +} + +// createLegacyTransactionsAndReceipts is a helper function to generate signed mock legacy transactions and mock receipts with mock logs +func createLegacyTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big.Int) (types.Transactions, types.Receipts, common.Address) { + // make transactions + trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{}) + trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{}) + trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode) + + transactionSigner := types.MakeSigner(config, blockNumber) + mockCurve := elliptic.P256() + mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) + if err != nil { + log.Crit(err.Error()) + } + signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + + senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx + if err != nil { + log.Crit(err.Error()) + } + + // make receipts + mockReceipt1 := types.NewReceipt(nil, false, 50) + mockReceipt1.Logs = []*types.Log{MockLog1} + mockReceipt1.TxHash = signedTrx1.Hash() + mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) + mockReceipt2.Logs = []*types.Log{MockLog2, ShortLog1} + mockReceipt2.TxHash = signedTrx2.Hash() + mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75) + mockReceipt3.Logs = []*types.Log{} + mockReceipt3.TxHash = signedTrx3.Hash() + + return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, senderAddr +} + +// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs +func createTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big.Int) (types.Transactions, types.Receipts, common.Address) { + // make transactions + trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{}) + trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{}) + trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode) + trx4 := types.NewTx(&types.AccessListTx{ + ChainID: config.ChainID, + Nonce: 0, + GasPrice: big.NewInt(100), + Gas: 50, + To: &AnotherAddress, + Value: big.NewInt(999), + Data: []byte{}, + AccessList: types.AccessList{ + AccessListEntry1, + AccessListEntry2, + }, + }) + trx5 := types.NewTx(&types.DynamicFeeTx{ + ChainID: config.ChainID, + Nonce: 0, + GasTipCap: big.NewInt(100), + GasFeeCap: big.NewInt(100), + Gas: 50, + To: &AnotherAddress, + Value: big.NewInt(1000), + Data: []byte{}, + AccessList: types.AccessList{ + AccessListEntry1, + AccessListEntry2, + }, + }) + + transactionSigner := types.MakeSigner(config, blockNumber) + mockCurve := elliptic.P256() + mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) + if err != nil { + log.Crit(err.Error()) + } + signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx4, err := types.SignTx(trx4, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx5, err := types.SignTx(trx5, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + + senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx + if err != nil { + log.Crit(err.Error()) + } + + // make receipts + mockReceipt1 := types.NewReceipt(nil, false, 50) + mockReceipt1.Logs = []*types.Log{MockLog1} + mockReceipt1.TxHash = signedTrx1.Hash() + mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) + mockReceipt2.Logs = []*types.Log{MockLog2, ShortLog1} + mockReceipt2.TxHash = signedTrx2.Hash() + mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75) + mockReceipt3.Logs = []*types.Log{} + mockReceipt3.TxHash = signedTrx3.Hash() + mockReceipt4 := &types.Receipt{ + Type: types.AccessListTxType, + PostState: common.HexToHash("0x3").Bytes(), + Status: types.ReceiptStatusSuccessful, + CumulativeGasUsed: 175, + Logs: []*types.Log{MockLog3, MockLog4, ShortLog2}, + TxHash: signedTrx4.Hash(), + } + mockReceipt5 := &types.Receipt{ + Type: types.DynamicFeeTxType, + PostState: common.HexToHash("0x3").Bytes(), + Status: types.ReceiptStatusSuccessful, + CumulativeGasUsed: 175, + Logs: []*types.Log{}, + TxHash: signedTrx5.Hash(), + } + + return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4, signedTrx5}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4, mockReceipt5}, senderAddr +} + +// createNonCanonicalBlockReceipts is a helper function to generate mock receipts with mock logs for non-canonical blocks +func createNonCanonicalBlockReceipts(config *params.ChainConfig, blockNumber *big.Int, transactions types.Transactions) types.Receipts { + transactionSigner := types.MakeSigner(config, blockNumber) + mockCurve := elliptic.P256() + mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) + if err != nil { + log.Crit(err.Error()) + } + + signedTrx0, err := types.SignTx(transactions[0], transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + + signedTrx1, err := types.SignTx(transactions[1], transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + + mockReceipt0 := types.NewReceipt(common.HexToHash("0x3").Bytes(), false, 300) + mockReceipt0.Logs = []*types.Log{MockLog1, ShortLog1} + mockReceipt0.TxHash = signedTrx0.Hash() + + mockReceipt1 := &types.Receipt{ + Type: types.DynamicFeeTxType, + PostState: common.HexToHash("0x4").Bytes(), + Status: types.ReceiptStatusSuccessful, + CumulativeGasUsed: 300, + Logs: []*types.Log{}, + TxHash: signedTrx1.Hash(), + } + + return types.Receipts{mockReceipt0, mockReceipt1} +} + +// Helper methods for testing watched addresses methods +func GetInsertWatchedAddressesArgs() []sdtypes.WatchAddressArg { + return []sdtypes.WatchAddressArg{ + { + Address: Contract1Address, + CreatedAt: Contract1CreatedAt, + }, + { + Address: Contract2Address, + CreatedAt: Contract2CreatedAt, + }, + } +} + +func GetInsertAlreadyWatchedAddressesArgs() []sdtypes.WatchAddressArg { + return []sdtypes.WatchAddressArg{ + { + Address: Contract3Address, + CreatedAt: Contract3CreatedAt, + }, + { + Address: Contract2Address, + CreatedAt: Contract2CreatedAt, + }, + } +} + +func GetRemoveWatchedAddressesArgs() []sdtypes.WatchAddressArg { + return []sdtypes.WatchAddressArg{ + { + Address: Contract3Address, + CreatedAt: Contract3CreatedAt, + }, + { + Address: Contract2Address, + CreatedAt: Contract2CreatedAt, + }, + } +} + +func GetRemoveNonWatchedAddressesArgs() []sdtypes.WatchAddressArg { + return []sdtypes.WatchAddressArg{ + { + Address: Contract1Address, + CreatedAt: Contract1CreatedAt, + }, + { + Address: Contract2Address, + CreatedAt: Contract2CreatedAt, + }, + } +} + +func GetSetWatchedAddressesArgs() []sdtypes.WatchAddressArg { + return []sdtypes.WatchAddressArg{ + { + Address: Contract1Address, + CreatedAt: Contract1CreatedAt, + }, + { + Address: Contract2Address, + CreatedAt: Contract2CreatedAt, + }, + { + Address: Contract3Address, + CreatedAt: Contract3CreatedAt, + }, + } +} + +func GetSetAlreadyWatchedAddressesArgs() []sdtypes.WatchAddressArg { + return []sdtypes.WatchAddressArg{ + { + Address: Contract4Address, + CreatedAt: Contract4CreatedAt, + }, + { + Address: Contract2Address, + CreatedAt: Contract2CreatedAt, + }, + { + Address: Contract3Address, + CreatedAt: Contract3CreatedAt, + }, + } +} diff --git a/statediff/indexer/models/batch.go b/statediff/indexer/models/batch.go new file mode 100644 index 000000000000..76858c96f83b --- /dev/null +++ b/statediff/indexer/models/batch.go @@ -0,0 +1,126 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package models + +import "github.com/lib/pq" + +// IPLDBatch holds the arguments for a batch insert of IPLD data +type IPLDBatch struct { + BlockNumbers []string + Keys []string + Values [][]byte +} + +// UncleBatch holds the arguments for a batch insert of uncle data +type UncleBatch struct { + BlockNumbers []string + HeaderID []string + BlockHashes []string + ParentHashes []string + CIDs []string + MhKeys []string + Rewards []string +} + +// TxBatch holds the arguments for a batch insert of tx data +type TxBatch struct { + BlockNumbers []string + HeaderIDs []string + Indexes []int64 + TxHashes []string + CIDs []string + MhKeys []string + Dsts []string + Srcs []string + Datas [][]byte + Types []uint8 +} + +// AccessListBatch holds the arguments for a batch insert of access list data +type AccessListBatch struct { + BlockNumbers []string + Indexes []int64 + TxIDs []string + Addresses []string + StorageKeysSets []pq.StringArray +} + +// ReceiptBatch holds the arguments for a batch insert of receipt data +type ReceiptBatch struct { + BlockNumbers []string + HeaderIDs []string + TxIDs []string + LeafCIDs []string + LeafMhKeys []string + PostStatuses []uint64 + PostStates []string + Contracts []string + ContractHashes []string + LogRoots []string +} + +// LogBatch holds the arguments for a batch insert of log data +type LogBatch struct { + BlockNumbers []string + HeaderIDs []string + LeafCIDs []string + LeafMhKeys []string + ReceiptIDs []string + Addresses []string + Indexes []int64 + Datas [][]byte + Topic0s []string + Topic1s []string + Topic2s []string + Topic3s []string +} + +// StateBatch holds the arguments for a batch insert of state data +type StateBatch struct { + BlockNumbers []string + HeaderIDs []string + Paths [][]byte + StateKeys []string + NodeTypes []int + CIDs []string + MhKeys []string + Diff bool +} + +// AccountBatch holds the arguments for a batch insert of account data +type AccountBatch struct { + BlockNumbers []string + HeaderIDs []string + StatePaths [][]byte + Balances []string + Nonces []uint64 + CodeHashes [][]byte + StorageRoots []string +} + +// StorageBatch holds the arguments for a batch insert of storage data +type StorageBatch struct { + BlockNumbers []string + HeaderIDs []string + StatePaths [][]string + Paths [][]byte + StorageKeys []string + NodeTypes []int + CIDs []string + MhKeys []string + Diff bool +} diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go new file mode 100644 index 000000000000..be44e37c72fd --- /dev/null +++ b/statediff/indexer/models/models.go @@ -0,0 +1,169 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package models + +import "github.com/lib/pq" + +// IPLDModel is the db model for public.blocks +type IPLDModel struct { + BlockNumber string `db:"block_number"` + Key string `db:"key"` + Data []byte `db:"data"` +} + +// HeaderModel is the db model for eth.header_cids +type HeaderModel struct { + BlockNumber string `db:"block_number"` + BlockHash string `db:"block_hash"` + ParentHash string `db:"parent_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + TotalDifficulty string `db:"td"` + NodeID string `db:"node_id"` + Reward string `db:"reward"` + StateRoot string `db:"state_root"` + UncleRoot string `db:"uncle_root"` + TxRoot string `db:"tx_root"` + RctRoot string `db:"receipt_root"` + Bloom []byte `db:"bloom"` + Timestamp uint64 `db:"timestamp"` + TimesValidated int64 `db:"times_validated"` + Coinbase string `db:"coinbase"` +} + +// UncleModel is the db model for eth.uncle_cids +type UncleModel struct { + BlockNumber string `db:"block_number"` + HeaderID string `db:"header_id"` + BlockHash string `db:"block_hash"` + ParentHash string `db:"parent_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Reward string `db:"reward"` +} + +// TxModel is the db model for eth.transaction_cids +type TxModel struct { + BlockNumber string `db:"block_number"` + HeaderID string `db:"header_id"` + Index int64 `db:"index"` + TxHash string `db:"tx_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Dst string `db:"dst"` + Src string `db:"src"` + Data []byte `db:"tx_data"` + Type uint8 `db:"tx_type"` + Value string `db:"value"` +} + +// AccessListElementModel is the db model for eth.access_list_entry +type AccessListElementModel struct { + BlockNumber string `db:"block_number"` + Index int64 `db:"index"` + TxID string `db:"tx_id"` + Address string `db:"address"` + StorageKeys pq.StringArray `db:"storage_keys"` +} + +// ReceiptModel is the db model for eth.receipt_cids +type ReceiptModel struct { + BlockNumber string `db:"block_number"` + HeaderID string `db:"header_id"` + TxID string `db:"tx_id"` + LeafCID string `db:"leaf_cid"` + LeafMhKey string `db:"leaf_mh_key"` + PostStatus uint64 `db:"post_status"` + PostState string `db:"post_state"` + Contract string `db:"contract"` + ContractHash string `db:"contract_hash"` + LogRoot string `db:"log_root"` +} + +// StateNodeModel is the db model for eth.state_cids +type StateNodeModel struct { + BlockNumber string `db:"block_number"` + HeaderID string `db:"header_id"` + Path []byte `db:"state_path"` + StateKey string `db:"state_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StorageNodeModel is the db model for eth.storage_cids +type StorageNodeModel struct { + BlockNumber string `db:"block_number"` + HeaderID string `db:"header_id"` + StatePath []byte `db:"state_path"` + Path []byte `db:"storage_path"` + StorageKey string `db:"storage_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key +type StorageNodeWithStateKeyModel struct { + BlockNumber string `db:"block_number"` + HeaderID string `db:"header_id"` + StatePath []byte `db:"state_path"` + Path []byte `db:"storage_path"` + StateKey string `db:"state_leaf_key"` + StorageKey string `db:"storage_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StateAccountModel is a db model for an eth state account (decoded value of state leaf node) +type StateAccountModel struct { + BlockNumber string `db:"block_number"` + HeaderID string `db:"header_id"` + StatePath []byte `db:"state_path"` + Balance string `db:"balance"` + Nonce uint64 `db:"nonce"` + CodeHash []byte `db:"code_hash"` + StorageRoot string `db:"storage_root"` +} + +// LogsModel is the db model for eth.logs +type LogsModel struct { + BlockNumber string `db:"block_number"` + HeaderID string `db:"header_id"` + ReceiptID string `db:"rct_id"` + LeafCID string `db:"leaf_cid"` + LeafMhKey string `db:"leaf_mh_key"` + Address string `db:"address"` + Index int64 `db:"index"` + Data []byte `db:"log_data"` + Topic0 string `db:"topic0"` + Topic1 string `db:"topic1"` + Topic2 string `db:"topic2"` + Topic3 string `db:"topic3"` +} + +// KnownGaps is the data structure for eth_meta.known_gaps +type KnownGapsModel struct { + StartingBlockNumber string `db:"starting_block_number"` + EndingBlockNumber string `db:"ending_block_number"` + CheckedOut bool `db:"checked_out"` + ProcessingKey int64 `db:"processing_key"` +} diff --git a/statediff/indexer/node/node.go b/statediff/indexer/node/node.go new file mode 100644 index 000000000000..527546efa277 --- /dev/null +++ b/statediff/indexer/node/node.go @@ -0,0 +1,25 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package node + +type Info struct { + GenesisBlock string + NetworkID string + ChainID uint64 + ID string + ClientName string +} diff --git a/statediff/indexer/shared/constants.go b/statediff/indexer/shared/constants.go new file mode 100644 index 000000000000..6d1e298ad990 --- /dev/null +++ b/statediff/indexer/shared/constants.go @@ -0,0 +1,23 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +const ( + RemovedNodeStorageCID = "bagmacgzayxjemamg64rtzet6pwznzrydydsqbnstzkbcoo337lmaixmfurya" + RemovedNodeStateCID = "baglacgzayxjemamg64rtzet6pwznzrydydsqbnstzkbcoo337lmaixmfurya" + RemovedNodeMhKey = "/blocks/DMQMLUSGAGDPOIZ4SJ7H3MW4Y4B4BZIAWZJ4VARHHN57VWAELWC2I4A" +) diff --git a/statediff/indexer/shared/db_kind.go b/statediff/indexer/shared/db_kind.go new file mode 100644 index 000000000000..7e7997f957ed --- /dev/null +++ b/statediff/indexer/shared/db_kind.go @@ -0,0 +1,46 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "fmt" + "strings" +) + +// DBType to explicitly type the kind of DB +type DBType string + +const ( + POSTGRES DBType = "Postgres" + DUMP DBType = "Dump" + FILE DBType = "File" + UNKNOWN DBType = "Unknown" +) + +// ResolveDBType resolves a DBType from a provided string +func ResolveDBType(str string) (DBType, error) { + switch strings.ToLower(str) { + case "postgres", "pg": + return POSTGRES, nil + case "dump", "d": + return DUMP, nil + case "file", "f", "fs": + return FILE, nil + default: + return UNKNOWN, fmt.Errorf("unrecognized db type string: %s", str) + } +} diff --git a/statediff/indexer/shared/functions.go b/statediff/indexer/shared/functions.go new file mode 100644 index 000000000000..8b0acbb545fe --- /dev/null +++ b/statediff/indexer/shared/functions.go @@ -0,0 +1,57 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + "github.com/multiformats/go-multihash" +) + +// HandleZeroAddrPointer will return an empty string for a nil address pointer +func HandleZeroAddrPointer(to *common.Address) string { + if to == nil { + return "" + } + return to.Hex() +} + +// HandleZeroAddr will return an empty string for a 0 value address +func HandleZeroAddr(to common.Address) string { + if to.Hex() == "0x0000000000000000000000000000000000000000" { + return "" + } + return to.Hex() +} + +// MultihashKeyFromCID converts a cid into a blockstore-prefixed multihash db key string +func MultihashKeyFromCID(c cid.Cid) string { + dbKey := dshelp.MultihashToDsKey(c.Hash()) + return blockstore.BlockPrefix.String() + dbKey.String() +} + +// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string +func MultihashKeyFromKeccak256(hash common.Hash) (string, error) { + mh, err := multihash.Encode(hash.Bytes(), multihash.KECCAK_256) + if err != nil { + return "", err + } + dbKey := dshelp.MultihashToDsKey(mh) + return blockstore.BlockPrefix.String() + dbKey.String(), nil +} diff --git a/statediff/indexer/shared/reward.go b/statediff/indexer/shared/reward.go new file mode 100644 index 000000000000..3d5752e25b31 --- /dev/null +++ b/statediff/indexer/shared/reward.go @@ -0,0 +1,76 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/core/types" +) + +func CalcEthBlockReward(header *types.Header, uncles []*types.Header, txs types.Transactions, receipts types.Receipts) *big.Int { + staticBlockReward := staticRewardByBlockNumber(header.Number.Uint64()) + transactionFees := calcEthTransactionFees(txs, receipts) + uncleInclusionRewards := calcEthUncleInclusionRewards(header, uncles) + tmp := transactionFees.Add(transactionFees, uncleInclusionRewards) + return tmp.Add(tmp, staticBlockReward) +} + +func CalcUncleMinerReward(blockNumber, uncleBlockNumber uint64) *big.Int { + staticBlockReward := staticRewardByBlockNumber(blockNumber) + rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8)) + mainBlock := new(big.Int).SetUint64(blockNumber) + uncleBlock := new(big.Int).SetUint64(uncleBlockNumber) + uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8)) + uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock) + return rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock) +} + +func staticRewardByBlockNumber(blockNumber uint64) *big.Int { + staticBlockReward := new(big.Int) + //https://blog.ethereum.org/2017/10/12/byzantium-hf-announcement/ + if blockNumber >= 7280000 { + staticBlockReward.SetString("2000000000000000000", 10) + } else if blockNumber >= 4370000 { + staticBlockReward.SetString("3000000000000000000", 10) + } else { + staticBlockReward.SetString("5000000000000000000", 10) + } + return staticBlockReward +} + +func calcEthTransactionFees(txs types.Transactions, receipts types.Receipts) *big.Int { + transactionFees := new(big.Int) + for i, transaction := range txs { + receipt := receipts[i] + gasPrice := big.NewInt(transaction.GasPrice().Int64()) + gasUsed := big.NewInt(int64(receipt.GasUsed)) + transactionFee := gasPrice.Mul(gasPrice, gasUsed) + transactionFees = transactionFees.Add(transactionFees, transactionFee) + } + return transactionFees +} + +func calcEthUncleInclusionRewards(header *types.Header, uncles []*types.Header) *big.Int { + uncleInclusionRewards := new(big.Int) + for range uncles { + staticBlockReward := staticRewardByBlockNumber(header.Number.Uint64()) + staticBlockReward.Div(staticBlockReward, big.NewInt(32)) + uncleInclusionRewards.Add(uncleInclusionRewards, staticBlockReward) + } + return uncleInclusionRewards +} diff --git a/statediff/indexer/test/test.go b/statediff/indexer/test/test.go new file mode 100644 index 000000000000..dedcd36553ba --- /dev/null +++ b/statediff/indexer/test/test.go @@ -0,0 +1,1274 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test + +import ( + "bytes" + "context" + "sort" + "testing" + + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" +) + +// SetupTestData indexes a single mock block along with it's state nodes +func SetupTestData(t *testing.T, ind interfaces.StateDiffIndexer) { + var tx interfaces.Batch + tx, err = ind.PushBlock( + mockBlock, + mocks.MockReceipts, + mocks.MockBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := tx.Submit(err); err != nil { + t.Fatal(err) + } + }() + for _, node := range mocks.StateDiffs { + err = ind.PushStateNode(tx, node, mockBlock.Hash().String()) + require.NoError(t, err) + } + + if batchTx, ok := tx.(*sql.BatchTx); ok { + require.Equal(t, mocks.BlockNumber.String(), batchTx.BlockNumber) + } else if batchTx, ok := tx.(*file.BatchTx); ok { + require.Equal(t, mocks.BlockNumber.String(), batchTx.BlockNumber) + } +} + +func TestPublishAndIndexHeaderIPLDs(t *testing.T, db sql.Database) { + pgStr := `SELECT cid, cast(td AS TEXT), cast(reward AS TEXT), block_hash, coinbase + FROM eth.header_cids + WHERE block_number = $1` + // check header was properly indexed + type res struct { + CID string + TD string + Reward string + BlockHash string `db:"block_hash"` + Coinbase string `db:"coinbase"` + } + header := new(res) + err = db.QueryRow(context.Background(), pgStr, mocks.BlockNumber.Uint64()).Scan( + &header.CID, + &header.TD, + &header.Reward, + &header.BlockHash, + &header.Coinbase) + if err != nil { + t.Fatal(err) + } + require.Equal(t, headerCID.String(), header.CID) + require.Equal(t, mocks.MockBlock.Difficulty().String(), header.TD) + require.Equal(t, "2000000000000021250", header.Reward) + require.Equal(t, mocks.MockHeader.Coinbase.String(), header.Coinbase) + dc, err := cid.Decode(header.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.MockHeaderRlp, data) +} + +func TestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) { + // check that txs were properly indexed and published + trxs := make([]string, 0) + pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) + WHERE header_cids.block_number = $1` + err = db.Select(context.Background(), &trxs, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, 5, len(trxs)) + expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String())) + expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String())) + expectTrue(t, test_helpers.ListContainsString(trxs, trx3CID.String())) + expectTrue(t, test_helpers.ListContainsString(trxs, trx4CID.String())) + expectTrue(t, test_helpers.ListContainsString(trxs, trx5CID.String())) + + transactions := mocks.MockBlock.Transactions() + type txResult struct { + TxType uint8 `db:"tx_type"` + Value string + } + for _, c := range trxs { + dc, err := cid.Decode(c) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + txTypeAndValueStr := `SELECT tx_type, CAST(value as TEXT) FROM eth.transaction_cids WHERE cid = $1` + switch c { + case trx1CID.String(): + require.Equal(t, tx1, data) + txRes := new(txResult) + err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) + if err != nil { + t.Fatal(err) + } + if txRes.TxType != 0 { + t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType) + } + if txRes.Value != transactions[0].Value().String() { + t.Fatalf("expected tx value %s got %s", transactions[0].Value().String(), txRes.Value) + } + case trx2CID.String(): + require.Equal(t, tx2, data) + txRes := new(txResult) + err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) + if err != nil { + t.Fatal(err) + } + if txRes.TxType != 0 { + t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType) + } + if txRes.Value != transactions[1].Value().String() { + t.Fatalf("expected tx value %s got %s", transactions[1].Value().String(), txRes.Value) + } + case trx3CID.String(): + require.Equal(t, tx3, data) + txRes := new(txResult) + err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) + if err != nil { + t.Fatal(err) + } + if txRes.TxType != 0 { + t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType) + } + if txRes.Value != transactions[2].Value().String() { + t.Fatalf("expected tx value %s got %s", transactions[2].Value().String(), txRes.Value) + } + case trx4CID.String(): + require.Equal(t, tx4, data) + txRes := new(txResult) + err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) + if err != nil { + t.Fatal(err) + } + if txRes.TxType != types.AccessListTxType { + t.Fatalf("expected AccessListTxType (1), got %d", txRes.TxType) + } + if txRes.Value != transactions[3].Value().String() { + t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value) + } + accessListElementModels := make([]models.AccessListElementModel, 0) + pgStr = "SELECT cast(access_list_elements.block_number AS TEXT), access_list_elements.index, access_list_elements.tx_id, " + + "access_list_elements.address, access_list_elements.storage_keys FROM eth.access_list_elements " + + "INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC" + err = db.Select(context.Background(), &accessListElementModels, pgStr, c) + if err != nil { + t.Fatal(err) + } + if len(accessListElementModels) != 2 { + t.Fatalf("expected two access list entries, got %d", len(accessListElementModels)) + } + model1 := models.AccessListElementModel{ + BlockNumber: mocks.BlockNumber.String(), + Index: accessListElementModels[0].Index, + Address: accessListElementModels[0].Address, + } + model2 := models.AccessListElementModel{ + BlockNumber: mocks.BlockNumber.String(), + Index: accessListElementModels[1].Index, + Address: accessListElementModels[1].Address, + StorageKeys: accessListElementModels[1].StorageKeys, + } + require.Equal(t, mocks.AccessListEntry1Model, model1) + require.Equal(t, mocks.AccessListEntry2Model, model2) + case trx5CID.String(): + require.Equal(t, tx5, data) + txRes := new(txResult) + err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) + if err != nil { + t.Fatal(err) + } + if txRes.TxType != types.DynamicFeeTxType { + t.Fatalf("expected DynamicFeeTxType (2), got %d", txRes.TxType) + } + if txRes.Value != transactions[4].Value().String() { + t.Fatalf("expected tx value %s got %s", transactions[4].Value().String(), txRes.Value) + } + } + } +} + +func TestPublishAndIndexLogIPLDs(t *testing.T, db sql.Database) { + rcts := make([]string, 0) + rctsPgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE receipt_cids.tx_id = transaction_cids.tx_hash + AND transaction_cids.header_id = header_cids.block_hash + AND header_cids.block_number = $1 + ORDER BY transaction_cids.index` + logsPgStr := `SELECT log_cids.index, log_cids.address, log_cids.topic0, log_cids.topic1, data FROM eth.log_cids + INNER JOIN eth.receipt_cids ON (log_cids.rct_id = receipt_cids.tx_id) + INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key) + WHERE receipt_cids.leaf_cid = $1 ORDER BY eth.log_cids.index ASC` + err = db.Select(context.Background(), &rcts, rctsPgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + if len(rcts) != len(mocks.MockReceipts) { + t.Fatalf("expected %d receipts, got %d", len(mocks.MockReceipts), len(rcts)) + } + + type logIPLD struct { + Index int `db:"index"` + Address string `db:"address"` + Data []byte `db:"data"` + Topic0 string `db:"topic0"` + Topic1 string `db:"topic1"` + } + for i := range rcts { + results := make([]logIPLD, 0) + err = db.Select(context.Background(), &results, logsPgStr, rcts[i]) + require.NoError(t, err) + + expectedLogs := mocks.MockReceipts[i].Logs + require.Equal(t, len(expectedLogs), len(results)) + + var nodeElements []interface{} + for idx, r := range results { + // Attempt to decode the log leaf node. + err = rlp.DecodeBytes(r.Data, &nodeElements) + require.NoError(t, err) + if len(nodeElements) == 2 { + logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx]) + require.NoError(t, err) + // 2nd element of the leaf node contains the encoded log data. + require.Equal(t, nodeElements[1].([]byte), logRaw) + } else { + logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx]) + require.NoError(t, err) + // raw log was IPLDized + require.Equal(t, r.Data, logRaw) + } + } + } +} + +func TestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) { + // check receipts were properly indexed and published + rcts := make([]string, 0) + pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE receipt_cids.tx_id = transaction_cids.tx_hash + AND transaction_cids.header_id = header_cids.block_hash + AND header_cids.block_number = $1 order by transaction_cids.index` + err = db.Select(context.Background(), &rcts, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, 5, len(rcts)) + expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String())) + expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String())) + expectTrue(t, test_helpers.ListContainsString(rcts, rct3CID.String())) + expectTrue(t, test_helpers.ListContainsString(rcts, rct4CID.String())) + expectTrue(t, test_helpers.ListContainsString(rcts, rct5CID.String())) + + for idx, c := range rcts { + result := make([]models.IPLDModel, 0) + pgStr = `SELECT data + FROM eth.receipt_cids + INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key) + WHERE receipt_cids.leaf_cid = $1` + err = db.Select(context.Background(), &result, pgStr, c) + if err != nil { + t.Fatal(err) + } + + // Decode the receipt leaf node. + var nodeElements []interface{} + err = rlp.DecodeBytes(result[0].Data, &nodeElements) + require.NoError(t, err) + + expectedRct, err := mocks.MockReceipts[idx].MarshalBinary() + require.NoError(t, err) + + require.Equal(t, nodeElements[1].([]byte), expectedRct) + + dc, err := cid.Decode(c) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1` + switch c { + case rct1CID.String(): + require.Equal(t, rctLeaf1, data) + var postStatus uint64 + pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1` + err = db.Get(context.Background(), &postStatus, pgStr, c) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.ExpectedPostStatus, postStatus) + case rct2CID.String(): + require.Equal(t, rctLeaf2, data) + var postState string + err = db.Get(context.Background(), &postState, postStatePgStr, c) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.ExpectedPostState1, postState) + case rct3CID.String(): + require.Equal(t, rctLeaf3, data) + var postState string + err = db.Get(context.Background(), &postState, postStatePgStr, c) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.ExpectedPostState2, postState) + case rct4CID.String(): + require.Equal(t, rctLeaf4, data) + var postState string + err = db.Get(context.Background(), &postState, postStatePgStr, c) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.ExpectedPostState3, postState) + case rct5CID.String(): + require.Equal(t, rctLeaf5, data) + var postState string + err = db.Get(context.Background(), &postState, postStatePgStr, c) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.ExpectedPostState3, postState) + } + } +} + +func TestPublishAndIndexStateIPLDs(t *testing.T, db sql.Database) { + // check that state nodes were properly indexed and published + stateNodes := make([]models.StateNodeModel, 0) + pgStr := `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id + FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) + WHERE header_cids.block_number = $1 AND node_type != 3` + err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, 2, len(stateNodes)) + for _, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + pgStr = `SELECT cast(block_number AS TEXT), header_id, state_path, cast(balance AS TEXT), nonce, code_hash, storage_root from eth.state_accounts WHERE header_id = $1 AND state_path = $2` + var account models.StateAccountModel + err = db.Get(context.Background(), &account, pgStr, stateNode.HeaderID, stateNode.Path) + if err != nil { + t.Fatal(err) + } + if stateNode.CID == state1CID.String() { + require.Equal(t, 2, stateNode.NodeType) + require.Equal(t, common.BytesToHash(mocks.ContractLeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x06'}, stateNode.Path) + require.Equal(t, mocks.ContractLeafNode, data) + require.Equal(t, models.StateAccountModel{ + BlockNumber: mocks.BlockNumber.String(), + HeaderID: account.HeaderID, + StatePath: stateNode.Path, + Balance: "0", + CodeHash: mocks.ContractCodeHash.Bytes(), + StorageRoot: mocks.ContractRoot, + Nonce: 1, + }, account) + } + if stateNode.CID == state2CID.String() { + require.Equal(t, 2, stateNode.NodeType) + require.Equal(t, common.BytesToHash(mocks.AccountLeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x0c'}, stateNode.Path) + require.Equal(t, mocks.AccountLeafNode, data) + require.Equal(t, models.StateAccountModel{ + BlockNumber: mocks.BlockNumber.String(), + HeaderID: account.HeaderID, + StatePath: stateNode.Path, + Balance: "1000", + CodeHash: mocks.AccountCodeHash.Bytes(), + StorageRoot: mocks.AccountRoot, + Nonce: 0, + }, account) + } + } + + // check that Removed state nodes were properly indexed and published + stateNodes = make([]models.StateNodeModel, 0) + pgStr = `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id + FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) + WHERE header_cids.block_number = $1 AND node_type = 3 + ORDER BY state_path` + err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, 2, len(stateNodes)) + for idx, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + if idx == 0 { + require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) + require.Equal(t, common.BytesToHash(mocks.RemovedLeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x02'}, stateNode.Path) + require.Equal(t, []byte{}, data) + } + if idx == 1 { + require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID) + require.Equal(t, common.BytesToHash(mocks.Contract2LeafKey).Hex(), stateNode.StateKey) + require.Equal(t, []byte{'\x07'}, stateNode.Path) + require.Equal(t, []byte{}, data) + } + } +} + +func TestPublishAndIndexStorageIPLDs(t *testing.T, db sql.Database) { + // check that storage nodes were properly indexed + storageNodes := make([]models.StorageNodeWithStateKeyModel, 0) + pgStr := `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path + FROM eth.storage_cids, eth.state_cids, eth.header_cids + WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) + AND state_cids.header_id = header_cids.block_hash + AND header_cids.block_number = $1 + AND storage_cids.node_type != 3 + ORDER BY storage_path` + err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, 1, len(storageNodes)) + require.Equal(t, models.StorageNodeWithStateKeyModel{ + BlockNumber: mocks.BlockNumber.String(), + CID: storageCID.String(), + NodeType: 2, + StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{}, + }, storageNodes[0]) + var data []byte + dc, err := cid.Decode(storageNodes[0].CID) + if err != nil { + t.Fatal(err) + } + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.StorageLeafNode, data) + + // check that Removed storage nodes were properly indexed + storageNodes = make([]models.StorageNodeWithStateKeyModel, 0) + pgStr = `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path + FROM eth.storage_cids, eth.state_cids, eth.header_cids + WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) + AND state_cids.header_id = header_cids.block_hash + AND header_cids.block_number = $1 + AND storage_cids.node_type = 3 + ORDER BY storage_path` + err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, 3, len(storageNodes)) + expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{'\x03'}, + }, + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage2LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0e'}, + }, + { + BlockNumber: mocks.BlockNumber.String(), + CID: shared.RemovedNodeStorageCID, + NodeType: 3, + StorageKey: common.BytesToHash(mocks.Storage3LeafKey).Hex(), + StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), + Path: []byte{'\x0f'}, + }, + } + for idx, storageNode := range storageNodes { + require.Equal(t, expectedStorageNodes[idx], storageNode) + dc, err = cid.Decode(storageNode.CID) + if err != nil { + t.Fatal(err) + } + mhKey = dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, []byte{}, data) + } +} + +// SetupTestDataNonCanonical indexes a mock block and a non-canonical mock block at London height +// and a non-canonical block at London height + 1 +// along with their state nodes +func SetupTestDataNonCanonical(t *testing.T, ind interfaces.StateDiffIndexer) { + // index a canonical block at London height + var tx1 interfaces.Batch + tx1, err = ind.PushBlock( + mockBlock, + mocks.MockReceipts, + mocks.MockBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + for _, node := range mocks.StateDiffs { + err = ind.PushStateNode(tx1, node, mockBlock.Hash().String()) + require.NoError(t, err) + } + + if batchTx, ok := tx1.(*sql.BatchTx); ok { + require.Equal(t, mocks.BlockNumber.String(), batchTx.BlockNumber) + } else if batchTx, ok := tx1.(*file.BatchTx); ok { + require.Equal(t, mocks.BlockNumber.String(), batchTx.BlockNumber) + } + + if err := tx1.Submit(err); err != nil { + t.Fatal(err) + } + + // index a non-canonical block at London height + // has transactions overlapping with that of the canonical block + var tx2 interfaces.Batch + tx2, err = ind.PushBlock( + mockNonCanonicalBlock, + mocks.MockNonCanonicalBlockReceipts, + mockNonCanonicalBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + for _, node := range mocks.StateDiffs { + err = ind.PushStateNode(tx2, node, mockNonCanonicalBlock.Hash().String()) + require.NoError(t, err) + } + + if tx, ok := tx2.(*sql.BatchTx); ok { + require.Equal(t, mocks.BlockNumber.String(), tx.BlockNumber) + } else if tx, ok := tx2.(*sql.BatchTx); ok { + require.Equal(t, mocks.BlockNumber.String(), tx.BlockNumber) + } + + if err := tx2.Submit(err); err != nil { + t.Fatal(err) + } + + // index a non-canonical block at London height + 1 + // has transactions overlapping with that of the canonical block + var tx3 interfaces.Batch + tx3, err = ind.PushBlock( + mockNonCanonicalBlock2, + mocks.MockNonCanonicalBlock2Receipts, + mockNonCanonicalBlock2.Difficulty()) + if err != nil { + t.Fatal(err) + } + for _, node := range mocks.StateDiffs[:2] { + err = ind.PushStateNode(tx3, node, mockNonCanonicalBlock2.Hash().String()) + require.NoError(t, err) + } + + if batchTx, ok := tx3.(*sql.BatchTx); ok { + require.Equal(t, mocks.Block2Number.String(), batchTx.BlockNumber) + } else if batchTx, ok := tx3.(*file.BatchTx); ok { + require.Equal(t, mocks.Block2Number.String(), batchTx.BlockNumber) + } + + if err := tx3.Submit(err); err != nil { + t.Fatal(err) + } +} + +func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) { + // check indexed headers + pgStr := `SELECT CAST(block_number as TEXT), block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT), + tx_root, receipt_root, uncle_root, coinbase + FROM eth.header_cids + ORDER BY block_number` + headerRes := make([]models.HeaderModel, 0) + err = db.Select(context.Background(), &headerRes, pgStr) + if err != nil { + t.Fatal(err) + } + + // expect three blocks to be indexed + // a canonical and a non-canonical block at London height, + // non-canonical block at London height + 1 + expectedRes := []models.HeaderModel{ + { + BlockNumber: mockBlock.Number().String(), + BlockHash: mockBlock.Hash().String(), + CID: headerCID.String(), + TotalDifficulty: mockBlock.Difficulty().String(), + TxRoot: mockBlock.TxHash().String(), + RctRoot: mockBlock.ReceiptHash().String(), + UncleRoot: mockBlock.UncleHash().String(), + Coinbase: mocks.MockHeader.Coinbase.String(), + }, + { + BlockNumber: mockNonCanonicalBlock.Number().String(), + BlockHash: mockNonCanonicalBlock.Hash().String(), + CID: mockNonCanonicalHeaderCID.String(), + TotalDifficulty: mockNonCanonicalBlock.Difficulty().String(), + TxRoot: mockNonCanonicalBlock.TxHash().String(), + RctRoot: mockNonCanonicalBlock.ReceiptHash().String(), + UncleRoot: mockNonCanonicalBlock.UncleHash().String(), + Coinbase: mocks.MockNonCanonicalHeader.Coinbase.String(), + }, + { + BlockNumber: mockNonCanonicalBlock2.Number().String(), + BlockHash: mockNonCanonicalBlock2.Hash().String(), + CID: mockNonCanonicalHeader2CID.String(), + TotalDifficulty: mockNonCanonicalBlock2.Difficulty().String(), + TxRoot: mockNonCanonicalBlock2.TxHash().String(), + RctRoot: mockNonCanonicalBlock2.ReceiptHash().String(), + UncleRoot: mockNonCanonicalBlock2.UncleHash().String(), + Coinbase: mocks.MockNonCanonicalHeader2.Coinbase.String(), + }, + } + expectedRes[0].Reward = shared.CalcEthBlockReward(mockBlock.Header(), mockBlock.Uncles(), mockBlock.Transactions(), mocks.MockReceipts).String() + expectedRes[1].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock.Header(), mockNonCanonicalBlock.Uncles(), mockNonCanonicalBlock.Transactions(), mocks.MockNonCanonicalBlockReceipts).String() + expectedRes[2].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock2.Header(), mockNonCanonicalBlock2.Uncles(), mockNonCanonicalBlock2.Transactions(), mocks.MockNonCanonicalBlock2Receipts).String() + + require.Equal(t, len(expectedRes), len(headerRes)) + require.ElementsMatch(t, + []string{mockBlock.Hash().String(), mockNonCanonicalBlock.Hash().String(), mockNonCanonicalBlock2.Hash().String()}, + []string{headerRes[0].BlockHash, headerRes[1].BlockHash, headerRes[2].BlockHash}, + ) + + if headerRes[0].BlockHash == mockBlock.Hash().String() { + require.Equal(t, expectedRes[0], headerRes[0]) + require.Equal(t, expectedRes[1], headerRes[1]) + require.Equal(t, expectedRes[2], headerRes[2]) + } else { + require.Equal(t, expectedRes[1], headerRes[0]) + require.Equal(t, expectedRes[0], headerRes[1]) + require.Equal(t, expectedRes[2], headerRes[2]) + } + + // check indexed IPLD blocks + headerCIDs := []cid.Cid{headerCID, mockNonCanonicalHeaderCID, mockNonCanonicalHeader2CID} + blockNumbers := []uint64{mocks.BlockNumber.Uint64(), mocks.BlockNumber.Uint64(), mocks.Block2Number.Uint64()} + headerRLPs := [][]byte{mocks.MockHeaderRlp, mocks.MockNonCanonicalHeaderRlp, mocks.MockNonCanonicalHeader2Rlp} + for i := range expectedRes { + var data []byte + prefixedKey := shared.MultihashKeyFromCID(headerCIDs[i]) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, blockNumbers[i]) + if err != nil { + t.Fatal(err) + } + require.Equal(t, headerRLPs[i], data) + } +} + +func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database) { + // check indexed transactions + pgStr := `SELECT CAST(block_number as TEXT), header_id, tx_hash, cid, dst, src, index, + tx_data, tx_type, CAST(value as TEXT) + FROM eth.transaction_cids + ORDER BY block_number, index` + txRes := make([]models.TxModel, 0) + err = db.Select(context.Background(), &txRes, pgStr) + if err != nil { + t.Fatal(err) + } + + // expected transactions in the canonical block + mockBlockTxs := mocks.MockBlock.Transactions() + expectedBlockTxs := []models.TxModel{ + { + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[0].Hash().String(), + CID: trx1CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[0].To()), + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockBlockTxs[0].Data(), + Type: mockBlockTxs[0].Type(), + Value: mockBlockTxs[0].Value().String(), + }, + { + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[1].Hash().String(), + CID: trx2CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[1].To()), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockBlockTxs[1].Data(), + Type: mockBlockTxs[1].Type(), + Value: mockBlockTxs[1].Value().String(), + }, + { + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[2].Hash().String(), + CID: trx3CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[2].To()), + Src: mocks.SenderAddr.String(), + Index: 2, + Data: mockBlockTxs[2].Data(), + Type: mockBlockTxs[2].Type(), + Value: mockBlockTxs[2].Value().String(), + }, + { + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[3].Hash().String(), + CID: trx4CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[3].To()), + Src: mocks.SenderAddr.String(), + Index: 3, + Data: mockBlockTxs[3].Data(), + Type: mockBlockTxs[3].Type(), + Value: mockBlockTxs[3].Value().String(), + }, + { + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[4].Hash().String(), + CID: trx5CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[4].To()), + Src: mocks.SenderAddr.String(), + Index: 4, + Data: mockBlockTxs[4].Data(), + Type: mockBlockTxs[4].Type(), + Value: mockBlockTxs[4].Value().String(), + }, + } + + // expected transactions in the non-canonical block at London height + mockNonCanonicalBlockTxs := mockNonCanonicalBlock.Transactions() + expectedNonCanonicalBlockTxs := []models.TxModel{ + { + BlockNumber: mockNonCanonicalBlock.Number().String(), + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxHash: mockNonCanonicalBlockTxs[0].Hash().String(), + CID: trx2CID.String(), + Dst: mockNonCanonicalBlockTxs[0].To().String(), + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockNonCanonicalBlockTxs[0].Data(), + Type: mockNonCanonicalBlockTxs[0].Type(), + Value: mockNonCanonicalBlockTxs[0].Value().String(), + }, + { + BlockNumber: mockNonCanonicalBlock.Number().String(), + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxHash: mockNonCanonicalBlockTxs[1].Hash().String(), + CID: trx5CID.String(), + Dst: mockNonCanonicalBlockTxs[1].To().String(), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockNonCanonicalBlockTxs[1].Data(), + Type: mockNonCanonicalBlockTxs[1].Type(), + Value: mockNonCanonicalBlockTxs[1].Value().String(), + }, + } + + // expected transactions in the non-canonical block at London height + 1 + mockNonCanonicalBlock2Txs := mockNonCanonicalBlock2.Transactions() + expectedNonCanonicalBlock2Txs := []models.TxModel{ + { + BlockNumber: mockNonCanonicalBlock2.Number().String(), + HeaderID: mockNonCanonicalBlock2.Hash().String(), + TxHash: mockNonCanonicalBlock2Txs[0].Hash().String(), + CID: trx3CID.String(), + Dst: "", + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockNonCanonicalBlock2Txs[0].Data(), + Type: mockNonCanonicalBlock2Txs[0].Type(), + Value: mockNonCanonicalBlock2Txs[0].Value().String(), + }, + { + BlockNumber: mockNonCanonicalBlock2.Number().String(), + HeaderID: mockNonCanonicalBlock2.Hash().String(), + TxHash: mockNonCanonicalBlock2Txs[1].Hash().String(), + CID: trx5CID.String(), + Dst: mockNonCanonicalBlock2Txs[1].To().String(), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockNonCanonicalBlock2Txs[1].Data(), + Type: mockNonCanonicalBlock2Txs[1].Type(), + Value: mockNonCanonicalBlock2Txs[1].Value().String(), + }, + } + + require.Equal(t, len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs)+len(expectedNonCanonicalBlock2Txs), len(txRes)) + + // sort results such that non-canonical block transactions come after canonical block ones + sort.SliceStable(txRes, func(i, j int) bool { + if txRes[i].BlockNumber < txRes[j].BlockNumber { + return true + } else if txRes[i].HeaderID == txRes[j].HeaderID { + return txRes[i].Index < txRes[j].Index + } else if txRes[i].HeaderID == mockBlock.Hash().String() { + return true + } else { + return false + } + }) + + for i, expectedTx := range expectedBlockTxs { + require.Equal(t, expectedTx, txRes[i]) + } + for i, expectedTx := range expectedNonCanonicalBlockTxs { + require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+i]) + } + for i, expectedTx := range expectedNonCanonicalBlock2Txs { + require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs)+i]) + } + + // check indexed IPLD blocks + var data []byte + var prefixedKey string + + txCIDs := []cid.Cid{trx1CID, trx2CID, trx3CID, trx4CID, trx5CID} + txRLPs := [][]byte{tx1, tx2, tx3, tx4, tx5} + for i, txCID := range txCIDs { + prefixedKey = shared.MultihashKeyFromCID(txCID) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, txRLPs[i], data) + } +} + +func TestPublishAndIndexReceiptsNonCanonical(t *testing.T, db sql.Database) { + // check indexed receipts + pgStr := `SELECT CAST(block_number as TEXT), header_id, tx_id, leaf_cid, leaf_mh_key, post_status, post_state, contract, contract_hash, log_root + FROM eth.receipt_cids + ORDER BY block_number` + rctRes := make([]models.ReceiptModel, 0) + err = db.Select(context.Background(), &rctRes, pgStr) + if err != nil { + t.Fatal(err) + } + + // expected receipts in the canonical block + rctCids := []cid.Cid{rct1CID, rct2CID, rct3CID, rct4CID, rct5CID} + expectedBlockRctsMap := make(map[string]models.ReceiptModel, len(mocks.MockReceipts)) + for i, mockBlockRct := range mocks.MockReceipts { + rctModel := createRctModel(mockBlockRct, rctCids[i], mockBlock.Number().String()) + expectedBlockRctsMap[rctCids[i].String()] = rctModel + } + + // expected receipts in the non-canonical block at London height + nonCanonicalBlockRctCids := []cid.Cid{nonCanonicalBlockRct1CID, nonCanonicalBlockRct2CID} + expectedNonCanonicalBlockRctsMap := make(map[string]models.ReceiptModel, len(mocks.MockNonCanonicalBlockReceipts)) + for i, mockNonCanonicalBlockRct := range mocks.MockNonCanonicalBlockReceipts { + rctModel := createRctModel(mockNonCanonicalBlockRct, nonCanonicalBlockRctCids[i], mockNonCanonicalBlock.Number().String()) + expectedNonCanonicalBlockRctsMap[nonCanonicalBlockRctCids[i].String()] = rctModel + } + + // expected receipts in the non-canonical block at London height + 1 + nonCanonicalBlock2RctCids := []cid.Cid{nonCanonicalBlock2Rct1CID, nonCanonicalBlock2Rct2CID} + expectedNonCanonicalBlock2RctsMap := make(map[string]models.ReceiptModel, len(mocks.MockNonCanonicalBlock2Receipts)) + for i, mockNonCanonicalBlock2Rct := range mocks.MockNonCanonicalBlock2Receipts { + rctModel := createRctModel(mockNonCanonicalBlock2Rct, nonCanonicalBlock2RctCids[i], mockNonCanonicalBlock2.Number().String()) + expectedNonCanonicalBlock2RctsMap[nonCanonicalBlock2RctCids[i].String()] = rctModel + } + + require.Equal(t, len(expectedBlockRctsMap)+len(expectedNonCanonicalBlockRctsMap)+len(expectedNonCanonicalBlock2RctsMap), len(rctRes)) + + // sort results such that non-canonical block reciepts come after canonical block ones + sort.SliceStable(rctRes, func(i, j int) bool { + if rctRes[i].BlockNumber < rctRes[j].BlockNumber { + return true + } else if rctRes[i].HeaderID == rctRes[j].HeaderID { + return false + } else if rctRes[i].HeaderID == mockBlock.Hash().String() { + return true + } else { + return false + } + }) + + for i := 0; i < len(expectedBlockRctsMap); i++ { + rct := rctRes[i] + require.Contains(t, expectedBlockRctsMap, rct.LeafCID) + require.Equal(t, expectedBlockRctsMap[rct.LeafCID], rct) + } + + for i := 0; i < len(expectedNonCanonicalBlockRctsMap); i++ { + rct := rctRes[len(expectedBlockRctsMap)+i] + require.Contains(t, expectedNonCanonicalBlockRctsMap, rct.LeafCID) + require.Equal(t, expectedNonCanonicalBlockRctsMap[rct.LeafCID], rct) + } + + for i := 0; i < len(expectedNonCanonicalBlock2RctsMap); i++ { + rct := rctRes[len(expectedBlockRctsMap)+len(expectedNonCanonicalBlockRctsMap)+i] + require.Contains(t, expectedNonCanonicalBlock2RctsMap, rct.LeafCID) + require.Equal(t, expectedNonCanonicalBlock2RctsMap[rct.LeafCID], rct) + } + + // check indexed rct IPLD blocks + var data []byte + var prefixedKey string + + rctRLPs := [][]byte{ + rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5, + nonCanonicalBlockRctLeaf1, nonCanonicalBlockRctLeaf2, + } + for i, rctCid := range append(rctCids, nonCanonicalBlockRctCids...) { + prefixedKey = shared.MultihashKeyFromCID(rctCid) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, rctRLPs[i], data) + } + + nonCanonicalBlock2RctRLPs := [][]byte{nonCanonicalBlock2RctLeaf1, nonCanonicalBlock2RctLeaf2} + for i, rctCid := range nonCanonicalBlock2RctCids { + prefixedKey = shared.MultihashKeyFromCID(rctCid) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.Block2Number.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, nonCanonicalBlock2RctRLPs[i], data) + } +} + +func TestPublishAndIndexLogsNonCanonical(t *testing.T, db sql.Database) { + // check indexed logs + pgStr := `SELECT address, log_data, topic0, topic1, topic2, topic3, data + FROM eth.log_cids + INNER JOIN public.blocks ON (log_cids.block_number = blocks.block_number AND log_cids.leaf_mh_key = blocks.key) + WHERE log_cids.block_number = $1 AND header_id = $2 AND rct_id = $3 + ORDER BY log_cids.index ASC` + + type rctWithBlockHash struct { + rct *types.Receipt + blockHash string + blockNumber uint64 + } + mockRcts := make([]rctWithBlockHash, 0) + + // logs in the canonical block + for _, mockBlockRct := range mocks.MockReceipts { + mockRcts = append(mockRcts, rctWithBlockHash{ + mockBlockRct, + mockBlock.Hash().String(), + mockBlock.NumberU64(), + }) + } + + // logs in the non-canonical block at London height + for _, mockBlockRct := range mocks.MockNonCanonicalBlockReceipts { + mockRcts = append(mockRcts, rctWithBlockHash{ + mockBlockRct, + mockNonCanonicalBlock.Hash().String(), + mockNonCanonicalBlock.NumberU64(), + }) + } + + // logs in the non-canonical block at London height + 1 + for _, mockBlockRct := range mocks.MockNonCanonicalBlock2Receipts { + mockRcts = append(mockRcts, rctWithBlockHash{ + mockBlockRct, + mockNonCanonicalBlock2.Hash().String(), + mockNonCanonicalBlock2.NumberU64(), + }) + } + + for _, mockRct := range mockRcts { + type logWithIPLD struct { + models.LogsModel + IPLDData []byte `db:"data"` + } + logRes := make([]logWithIPLD, 0) + err = db.Select(context.Background(), &logRes, pgStr, mockRct.blockNumber, mockRct.blockHash, mockRct.rct.TxHash.String()) + require.NoError(t, err) + require.Equal(t, len(mockRct.rct.Logs), len(logRes)) + + for i, log := range mockRct.rct.Logs { + topicSet := make([]string, 4) + for ti, topic := range log.Topics { + topicSet[ti] = topic.Hex() + } + + expectedLog := models.LogsModel{ + Address: log.Address.String(), + Data: log.Data, + Topic0: topicSet[0], + Topic1: topicSet[1], + Topic2: topicSet[2], + Topic3: topicSet[3], + } + require.Equal(t, expectedLog, logRes[i].LogsModel) + + // check indexed log IPLD block + var nodeElements []interface{} + err = rlp.DecodeBytes(logRes[i].IPLDData, &nodeElements) + require.NoError(t, err) + + if len(nodeElements) == 2 { + logRaw, err := rlp.EncodeToBytes(log) + require.NoError(t, err) + // 2nd element of the leaf node contains the encoded log data. + require.Equal(t, nodeElements[1].([]byte), logRaw) + } else { + logRaw, err := rlp.EncodeToBytes(log) + require.NoError(t, err) + // raw log was IPLDized + require.Equal(t, logRes[i].IPLDData, logRaw) + } + } + } +} + +func TestPublishAndIndexStateNonCanonical(t *testing.T, db sql.Database) { + // check indexed state nodes + pgStr := `SELECT state_path, state_leaf_key, node_type, cid, mh_key, diff + FROM eth.state_cids + WHERE block_number = $1 + AND header_id = $2 + ORDER BY state_path` + + removedNodeCID, _ := cid.Decode(shared.RemovedNodeStateCID) + stateNodeCIDs := []cid.Cid{state1CID, state2CID, removedNodeCID, removedNodeCID} + + // expected state nodes in the canonical and the non-canonical block at London height + expectedStateNodes := make([]models.StateNodeModel, 0) + for i, stateDiff := range mocks.StateDiffs { + expectedStateNodes = append(expectedStateNodes, models.StateNodeModel{ + Path: stateDiff.Path, + StateKey: common.BytesToHash(stateDiff.LeafKey).Hex(), + NodeType: stateDiff.NodeType.Int(), + CID: stateNodeCIDs[i].String(), + MhKey: shared.MultihashKeyFromCID(stateNodeCIDs[i]), + Diff: true, + }) + } + sort.Slice(expectedStateNodes, func(i, j int) bool { + if bytes.Compare(expectedStateNodes[i].Path, expectedStateNodes[j].Path) < 0 { + return true + } else { + return false + } + }) + + // expected state nodes in the non-canonical block at London height + 1 + expectedNonCanonicalBlock2StateNodes := make([]models.StateNodeModel, 0) + for i, stateDiff := range mocks.StateDiffs[:2] { + expectedNonCanonicalBlock2StateNodes = append(expectedNonCanonicalBlock2StateNodes, models.StateNodeModel{ + Path: stateDiff.Path, + StateKey: common.BytesToHash(stateDiff.LeafKey).Hex(), + NodeType: stateDiff.NodeType.Int(), + CID: stateNodeCIDs[i].String(), + MhKey: shared.MultihashKeyFromCID(stateNodeCIDs[i]), + Diff: true, + }) + } + + // check state nodes for canonical block + stateNodes := make([]models.StateNodeModel, 0) + err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64(), mockBlock.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedStateNodes), len(stateNodes)) + + for i, expectedStateNode := range expectedStateNodes { + require.Equal(t, expectedStateNode, stateNodes[i]) + } + + // check state nodes for non-canonical block at London height + stateNodes = make([]models.StateNodeModel, 0) + err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64(), mockNonCanonicalBlock.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedStateNodes), len(stateNodes)) + + for i, expectedStateNode := range expectedStateNodes { + require.Equal(t, expectedStateNode, stateNodes[i]) + } + + // check state nodes for non-canonical block at London height + 1 + stateNodes = make([]models.StateNodeModel, 0) + err = db.Select(context.Background(), &stateNodes, pgStr, mocks.Block2Number.Uint64(), mockNonCanonicalBlock2.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedNonCanonicalBlock2StateNodes), len(stateNodes)) + + for i, expectedStateNode := range expectedNonCanonicalBlock2StateNodes { + require.Equal(t, expectedStateNode, stateNodes[i]) + } +} + +func TestPublishAndIndexStorageNonCanonical(t *testing.T, db sql.Database) { + // check indexed storage nodes + pgStr := `SELECT state_path, storage_path, storage_leaf_key, node_type, cid, mh_key, diff + FROM eth.storage_cids + WHERE block_number = $1 + AND header_id = $2 + ORDER BY state_path, storage_path` + + removedNodeCID, _ := cid.Decode(shared.RemovedNodeStorageCID) + storageNodeCIDs := []cid.Cid{storageCID, removedNodeCID, removedNodeCID, removedNodeCID} + + // expected storage nodes in the canonical and the non-canonical block at London height + expectedStorageNodes := make([]models.StorageNodeModel, 0) + storageNodeIndex := 0 + for _, stateDiff := range mocks.StateDiffs { + for _, storageNode := range stateDiff.StorageNodes { + expectedStorageNodes = append(expectedStorageNodes, models.StorageNodeModel{ + StatePath: stateDiff.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).Hex(), + NodeType: storageNode.NodeType.Int(), + CID: storageNodeCIDs[storageNodeIndex].String(), + MhKey: shared.MultihashKeyFromCID(storageNodeCIDs[storageNodeIndex]), + Diff: true, + }) + storageNodeIndex++ + } + } + sort.Slice(expectedStorageNodes, func(i, j int) bool { + if bytes.Compare(expectedStorageNodes[i].Path, expectedStorageNodes[j].Path) < 0 { + return true + } else { + return false + } + }) + + // expected storage nodes in the non-canonical block at London height + 1 + expectedNonCanonicalBlock2StorageNodes := make([]models.StorageNodeModel, 0) + storageNodeIndex = 0 + for _, stateDiff := range mocks.StateDiffs[:2] { + for _, storageNode := range stateDiff.StorageNodes { + expectedNonCanonicalBlock2StorageNodes = append(expectedNonCanonicalBlock2StorageNodes, models.StorageNodeModel{ + StatePath: stateDiff.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).Hex(), + NodeType: storageNode.NodeType.Int(), + CID: storageNodeCIDs[storageNodeIndex].String(), + MhKey: shared.MultihashKeyFromCID(storageNodeCIDs[storageNodeIndex]), + Diff: true, + }) + storageNodeIndex++ + } + } + + // check storage nodes for canonical block + storageNodes := make([]models.StorageNodeModel, 0) + err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64(), mockBlock.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedStorageNodes), len(storageNodes)) + + for i, expectedStorageNode := range expectedStorageNodes { + require.Equal(t, expectedStorageNode, storageNodes[i]) + } + + // check storage nodes for non-canonical block at London height + storageNodes = make([]models.StorageNodeModel, 0) + err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64(), mockNonCanonicalBlock.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedStorageNodes), len(storageNodes)) + + for i, expectedStorageNode := range expectedStorageNodes { + require.Equal(t, expectedStorageNode, storageNodes[i]) + } + + // check storage nodes for non-canonical block at London height + 1 + storageNodes = make([]models.StorageNodeModel, 0) + err = db.Select(context.Background(), &storageNodes, pgStr, mockNonCanonicalBlock2.NumberU64(), mockNonCanonicalBlock2.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedNonCanonicalBlock2StorageNodes), len(storageNodes)) + + for i, expectedStorageNode := range expectedNonCanonicalBlock2StorageNodes { + require.Equal(t, expectedStorageNode, storageNodes[i]) + } +} diff --git a/statediff/indexer/test/test_init.go b/statediff/indexer/test/test_init.go new file mode 100644 index 000000000000..283d3e0b0ddb --- /dev/null +++ b/statediff/indexer/test/test_init.go @@ -0,0 +1,248 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test + +import ( + "bytes" + "fmt" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" +) + +var ( + err error + ipfsPgGet = `SELECT data FROM public.blocks + WHERE key = $1 AND block_number = $2` + watchedAddressesPgGet = `SELECT * + FROM eth_meta.watched_addresses` + tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte + nonCanonicalBlockRct1, nonCanonicalBlockRct2 []byte + nonCanonicalBlock2Rct1, nonCanonicalBlock2Rct2 []byte + mockBlock, mockNonCanonicalBlock, mockNonCanonicalBlock2 *types.Block + headerCID, mockNonCanonicalHeaderCID, mockNonCanonicalHeader2CID cid.Cid + trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid + rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid + nonCanonicalBlockRct1CID, nonCanonicalBlockRct2CID cid.Cid + nonCanonicalBlock2Rct1CID, nonCanonicalBlock2Rct2CID cid.Cid + rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte + nonCanonicalBlockRctLeaf1, nonCanonicalBlockRctLeaf2 []byte + nonCanonicalBlock2RctLeaf1, nonCanonicalBlock2RctLeaf2 []byte + state1CID, state2CID, storageCID cid.Cid +) + +func init() { + if os.Getenv("MODE") != "statediff" { + fmt.Println("Skipping statediff test") + os.Exit(0) + } + + // canonical block at LondonBlock height + mockBlock = mocks.MockBlock + txs, rcts := mocks.MockBlock.Transactions(), mocks.MockReceipts + + // non-canonical block at LondonBlock height + mockNonCanonicalBlock = mocks.MockNonCanonicalBlock + nonCanonicalBlockRcts := mocks.MockNonCanonicalBlockReceipts + + // non-canonical block at LondonBlock height + 1 + mockNonCanonicalBlock2 = mocks.MockNonCanonicalBlock2 + nonCanonicalBlock2Rcts := mocks.MockNonCanonicalBlock2Receipts + + // encode mock receipts + buf := new(bytes.Buffer) + txs.EncodeIndex(0, buf) + tx1 = make([]byte, buf.Len()) + copy(tx1, buf.Bytes()) + buf.Reset() + + txs.EncodeIndex(1, buf) + tx2 = make([]byte, buf.Len()) + copy(tx2, buf.Bytes()) + buf.Reset() + + txs.EncodeIndex(2, buf) + tx3 = make([]byte, buf.Len()) + copy(tx3, buf.Bytes()) + buf.Reset() + + txs.EncodeIndex(3, buf) + tx4 = make([]byte, buf.Len()) + copy(tx4, buf.Bytes()) + buf.Reset() + + txs.EncodeIndex(4, buf) + tx5 = make([]byte, buf.Len()) + copy(tx5, buf.Bytes()) + buf.Reset() + + rcts.EncodeIndex(0, buf) + rct1 = make([]byte, buf.Len()) + copy(rct1, buf.Bytes()) + buf.Reset() + + rcts.EncodeIndex(1, buf) + rct2 = make([]byte, buf.Len()) + copy(rct2, buf.Bytes()) + buf.Reset() + + rcts.EncodeIndex(2, buf) + rct3 = make([]byte, buf.Len()) + copy(rct3, buf.Bytes()) + buf.Reset() + + rcts.EncodeIndex(3, buf) + rct4 = make([]byte, buf.Len()) + copy(rct4, buf.Bytes()) + buf.Reset() + + rcts.EncodeIndex(4, buf) + rct5 = make([]byte, buf.Len()) + copy(rct5, buf.Bytes()) + buf.Reset() + + // encode mock receipts for non-canonical blocks + nonCanonicalBlockRcts.EncodeIndex(0, buf) + nonCanonicalBlockRct1 = make([]byte, buf.Len()) + copy(nonCanonicalBlockRct1, buf.Bytes()) + buf.Reset() + + nonCanonicalBlockRcts.EncodeIndex(1, buf) + nonCanonicalBlockRct2 = make([]byte, buf.Len()) + copy(nonCanonicalBlockRct2, buf.Bytes()) + buf.Reset() + + nonCanonicalBlock2Rcts.EncodeIndex(0, buf) + nonCanonicalBlock2Rct1 = make([]byte, buf.Len()) + copy(nonCanonicalBlock2Rct1, buf.Bytes()) + buf.Reset() + + nonCanonicalBlock2Rcts.EncodeIndex(1, buf) + nonCanonicalBlock2Rct2 = make([]byte, buf.Len()) + copy(nonCanonicalBlock2Rct2, buf.Bytes()) + buf.Reset() + + headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockHeaderRlp, multihash.KECCAK_256) + mockNonCanonicalHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeaderRlp, multihash.KECCAK_256) + mockNonCanonicalHeader2CID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeader2Rlp, multihash.KECCAK_256) + trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx1, multihash.KECCAK_256) + trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx2, multihash.KECCAK_256) + trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx3, multihash.KECCAK_256) + trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx4, multihash.KECCAK_256) + trx5CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx5, multihash.KECCAK_256) + state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.ContractLeafNode, multihash.KECCAK_256) + state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256) + storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256) + + // create raw receipts + rawRctLeafNodes, rctleafNodeCids := createRctTrie([][]byte{rct1, rct2, rct3, rct4, rct5}) + + rct1CID = rctleafNodeCids[0] + rct2CID = rctleafNodeCids[1] + rct3CID = rctleafNodeCids[2] + rct4CID = rctleafNodeCids[3] + rct5CID = rctleafNodeCids[4] + + rctLeaf1 = rawRctLeafNodes[0] + rctLeaf2 = rawRctLeafNodes[1] + rctLeaf3 = rawRctLeafNodes[2] + rctLeaf4 = rawRctLeafNodes[3] + rctLeaf5 = rawRctLeafNodes[4] + + // create raw receipts for non-canonical blocks + nonCanonicalBlockRawRctLeafNodes, nonCanonicalBlockRctLeafNodeCids := createRctTrie([][]byte{nonCanonicalBlockRct1, nonCanonicalBlockRct2}) + + nonCanonicalBlockRct1CID = nonCanonicalBlockRctLeafNodeCids[0] + nonCanonicalBlockRct2CID = nonCanonicalBlockRctLeafNodeCids[1] + + nonCanonicalBlockRctLeaf1 = nonCanonicalBlockRawRctLeafNodes[0] + nonCanonicalBlockRctLeaf2 = nonCanonicalBlockRawRctLeafNodes[1] + + nonCanonicalBlock2RawRctLeafNodes, nonCanonicalBlock2RctLeafNodeCids := createRctTrie([][]byte{nonCanonicalBlockRct1, nonCanonicalBlockRct2}) + + nonCanonicalBlock2Rct1CID = nonCanonicalBlock2RctLeafNodeCids[0] + nonCanonicalBlock2Rct2CID = nonCanonicalBlock2RctLeafNodeCids[1] + + nonCanonicalBlock2RctLeaf1 = nonCanonicalBlock2RawRctLeafNodes[0] + nonCanonicalBlock2RctLeaf2 = nonCanonicalBlock2RawRctLeafNodes[1] +} + +// createRctTrie creates a receipt trie from the given raw receipts +// returns receipt leaf nodes and their CIDs +func createRctTrie(rcts [][]byte) ([][]byte, []cid.Cid) { + receiptTrie := ipld.NewRctTrie() + + for i, rct := range rcts { + receiptTrie.Add(i, rct) + } + rctLeafNodes, keys, _ := receiptTrie.GetLeafNodes() + + rctleafNodeCids := make([]cid.Cid, len(rctLeafNodes)) + orderedRctLeafNodes := make([][]byte, len(rctLeafNodes)) + for i, rln := range rctLeafNodes { + var idx uint + + r := bytes.NewReader(keys[i].TrieKey) + rlp.Decode(r, &idx) + rctleafNodeCids[idx] = rln.Cid() + orderedRctLeafNodes[idx] = rln.RawData() + } + + return orderedRctLeafNodes, rctleafNodeCids +} + +// createRctModel creates a models.ReceiptModel object from a given ethereum receipt +func createRctModel(rct *types.Receipt, cid cid.Cid, blockNumber string) models.ReceiptModel { + rctModel := models.ReceiptModel{ + BlockNumber: blockNumber, + HeaderID: rct.BlockHash.String(), + TxID: rct.TxHash.String(), + LeafCID: cid.String(), + LeafMhKey: shared.MultihashKeyFromCID(cid), + LogRoot: rct.LogRoot.String(), + } + + contract := shared.HandleZeroAddr(rct.ContractAddress) + rctModel.Contract = contract + if contract != "" { + rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + } + + if len(rct.PostState) == 0 { + rctModel.PostStatus = rct.Status + } else { + rctModel.PostState = common.Bytes2Hex(rct.PostState) + } + + return rctModel +} + +func expectTrue(t *testing.T, value bool) { + if !value { + t.Fatalf("Assertion failed") + } +} diff --git a/statediff/indexer/test/test_legacy.go b/statediff/indexer/test/test_legacy.go new file mode 100644 index 000000000000..5838fea7147a --- /dev/null +++ b/statediff/indexer/test/test_legacy.go @@ -0,0 +1,96 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test + +import ( + "context" + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +var ( + LegacyConfig = params.MainnetChainConfig + legacyData = mocks.NewLegacyData(LegacyConfig) + mockLegacyBlock *types.Block + legacyHeaderCID cid.Cid +) + +func SetupLegacyTestData(t *testing.T, ind interfaces.StateDiffIndexer) { + mockLegacyBlock = legacyData.MockBlock + legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256) + + var tx interfaces.Batch + tx, err = ind.PushBlock( + mockLegacyBlock, + legacyData.MockReceipts, + legacyData.MockBlock.Difficulty()) + require.NoError(t, err) + + defer func() { + if err := tx.Submit(err); err != nil { + t.Fatal(err) + } + }() + for _, node := range legacyData.StateDiffs { + err = ind.PushStateNode(tx, node, mockLegacyBlock.Hash().String()) + require.NoError(t, err) + } + + if batchTx, ok := tx.(*sql.BatchTx); ok { + require.Equal(t, legacyData.BlockNumber.String(), batchTx.BlockNumber) + } else if batchTx, ok := tx.(*file.BatchTx); ok { + require.Equal(t, legacyData.BlockNumber.String(), batchTx.BlockNumber) + } +} + +func TestLegacyIndexer(t *testing.T, db sql.Database) { + pgStr := `SELECT cid, cast(td AS TEXT), cast(reward AS TEXT), block_hash, coinbase + FROM eth.header_cids + WHERE block_number = $1` + // check header was properly indexed + type res struct { + CID string + TD string + Reward string + BlockHash string `db:"block_hash"` + Coinbase string `db:"coinbase"` + } + header := new(res) + err = db.QueryRow(context.Background(), pgStr, legacyData.BlockNumber.Uint64()).Scan( + &header.CID, + &header.TD, + &header.Reward, + &header.BlockHash, + &header.Coinbase) + require.NoError(t, err) + + require.Equal(t, legacyHeaderCID.String(), header.CID) + require.Equal(t, legacyData.MockBlock.Difficulty().String(), header.TD) + require.Equal(t, "5000000000000011250", header.Reward) + require.Equal(t, legacyData.MockHeader.Coinbase.String(), header.Coinbase) + require.Nil(t, legacyData.MockHeader.BaseFee) +} diff --git a/statediff/indexer/test/test_mainnet.go b/statediff/indexer/test/test_mainnet.go new file mode 100644 index 000000000000..24f74eb96d2d --- /dev/null +++ b/statediff/indexer/test/test_mainnet.go @@ -0,0 +1,53 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test + +import ( + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/stretchr/testify/require" +) + +func TestBlock(t *testing.T, ind interfaces.StateDiffIndexer, testBlock *types.Block, testReceipts types.Receipts) { + var tx interfaces.Batch + tx, err = ind.PushBlock( + testBlock, + testReceipts, + testBlock.Difficulty()) + require.NoError(t, err) + + defer func() { + if err := tx.Submit(err); err != nil { + t.Fatal(err) + } + }() + for _, node := range mocks.StateDiffs { + err = ind.PushStateNode(tx, node, testBlock.Hash().String()) + require.NoError(t, err) + } + + if batchTx, ok := tx.(*sql.BatchTx); ok { + require.Equal(t, testBlock.Number().String(), batchTx.BlockNumber) + } else if batchTx, ok := tx.(*file.BatchTx); ok { + require.Equal(t, testBlock.Number().String(), batchTx.BlockNumber) + } +} diff --git a/statediff/indexer/test/test_watched_addresses.go b/statediff/indexer/test/test_watched_addresses.go new file mode 100644 index 000000000000..02949e927bde --- /dev/null +++ b/statediff/indexer/test/test_watched_addresses.go @@ -0,0 +1,258 @@ +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test + +import ( + "context" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/stretchr/testify/require" +) + +type res struct { + Address string `db:"address"` + CreatedAt uint64 `db:"created_at"` + WatchedAt uint64 `db:"watched_at"` + LastFilledAt uint64 `db:"last_filled_at"` +} + +func TestLoadEmptyWatchedAddresses(t *testing.T, ind interfaces.StateDiffIndexer) { + expectedData := []common.Address{} + + rows, err := ind.LoadWatchedAddresses() + require.NoError(t, err) + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestInsertWatchedAddresses(t *testing.T, db sql.Database) { + expectedData := []res{ + { + Address: mocks.Contract1Address, + CreatedAt: mocks.Contract1CreatedAt, + WatchedAt: mocks.WatchedAt1, + LastFilledAt: mocks.LastFilledAt, + }, + { + Address: mocks.Contract2Address, + CreatedAt: mocks.Contract2CreatedAt, + WatchedAt: mocks.WatchedAt1, + LastFilledAt: mocks.LastFilledAt, + }, + } + + rows := []res{} + err = db.Select(context.Background(), &rows, watchedAddressesPgGet) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestInsertAlreadyWatchedAddresses(t *testing.T, db sql.Database) { + expectedData := []res{ + { + Address: mocks.Contract1Address, + CreatedAt: mocks.Contract1CreatedAt, + WatchedAt: mocks.WatchedAt1, + LastFilledAt: mocks.LastFilledAt, + }, + { + Address: mocks.Contract2Address, + CreatedAt: mocks.Contract2CreatedAt, + WatchedAt: mocks.WatchedAt1, + LastFilledAt: mocks.LastFilledAt, + }, + { + Address: mocks.Contract3Address, + CreatedAt: mocks.Contract3CreatedAt, + WatchedAt: mocks.WatchedAt2, + LastFilledAt: mocks.LastFilledAt, + }, + } + + rows := []res{} + err = db.Select(context.Background(), &rows, watchedAddressesPgGet) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestRemoveWatchedAddresses(t *testing.T, db sql.Database) { + expectedData := []res{ + { + Address: mocks.Contract1Address, + CreatedAt: mocks.Contract1CreatedAt, + WatchedAt: mocks.WatchedAt1, + LastFilledAt: mocks.LastFilledAt, + }, + } + + rows := []res{} + err = db.Select(context.Background(), &rows, watchedAddressesPgGet) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestRemoveNonWatchedAddresses(t *testing.T, db sql.Database) { + expectedData := []res{} + + rows := []res{} + err = db.Select(context.Background(), &rows, watchedAddressesPgGet) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestSetWatchedAddresses(t *testing.T, db sql.Database) { + expectedData := []res{ + { + Address: mocks.Contract1Address, + CreatedAt: mocks.Contract1CreatedAt, + WatchedAt: mocks.WatchedAt2, + LastFilledAt: mocks.LastFilledAt, + }, + { + Address: mocks.Contract2Address, + CreatedAt: mocks.Contract2CreatedAt, + WatchedAt: mocks.WatchedAt2, + LastFilledAt: mocks.LastFilledAt, + }, + { + Address: mocks.Contract3Address, + CreatedAt: mocks.Contract3CreatedAt, + WatchedAt: mocks.WatchedAt2, + LastFilledAt: mocks.LastFilledAt, + }, + } + + rows := []res{} + err = db.Select(context.Background(), &rows, watchedAddressesPgGet) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestSetAlreadyWatchedAddresses(t *testing.T, db sql.Database) { + expectedData := []res{ + { + Address: mocks.Contract4Address, + CreatedAt: mocks.Contract4CreatedAt, + WatchedAt: mocks.WatchedAt3, + LastFilledAt: mocks.LastFilledAt, + }, + { + Address: mocks.Contract2Address, + CreatedAt: mocks.Contract2CreatedAt, + WatchedAt: mocks.WatchedAt3, + LastFilledAt: mocks.LastFilledAt, + }, + { + Address: mocks.Contract3Address, + CreatedAt: mocks.Contract3CreatedAt, + WatchedAt: mocks.WatchedAt3, + LastFilledAt: mocks.LastFilledAt, + }, + } + + rows := []res{} + err = db.Select(context.Background(), &rows, watchedAddressesPgGet) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestLoadWatchedAddresses(t *testing.T, ind interfaces.StateDiffIndexer) { + expectedData := []common.Address{ + common.HexToAddress(mocks.Contract4Address), + common.HexToAddress(mocks.Contract2Address), + common.HexToAddress(mocks.Contract3Address), + } + + rows, err := ind.LoadWatchedAddresses() + require.NoError(t, err) + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestClearWatchedAddresses(t *testing.T, db sql.Database) { + expectedData := []res{} + rows := []res{} + err = db.Select(context.Background(), &rows, watchedAddressesPgGet) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} + +func TestClearEmptyWatchedAddresses(t *testing.T, db sql.Database) { + expectedData := []res{} + rows := []res{} + err = db.Select(context.Background(), &rows, watchedAddressesPgGet) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, len(expectedData), len(rows)) + for idx, row := range rows { + require.Equal(t, expectedData[idx], row) + } +} diff --git a/statediff/indexer/test_helpers/mainnet_test_helpers.go b/statediff/indexer/test_helpers/mainnet_test_helpers.go new file mode 100644 index 000000000000..faedee5b845f --- /dev/null +++ b/statediff/indexer/test_helpers/mainnet_test_helpers.go @@ -0,0 +1,248 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test_helpers + +import ( + "context" + "errors" + "fmt" + "math/big" + "os" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rlp" +) + +const ( + defaultBlockFilePath = "../../../mainnet_data/block" + defaultReceiptsFilePath = "../../../mainnet_data/receipts" +) + +const ( + TEST_RAW_URL = "TEST_RAW_URL" + TEST_BLOCK_NUMBER = "TEST_BLOCK_NUMBER" +) + +// ProblemBlocks list of known problem blocks, with funky edge cases +var ProblemBlocks = []int64{ + 12600011, + 12619985, + 12625121, + 12655432, + 12579670, + 12914664, +} + +// TestConfig holds configuration params for mainnet tests +type TestConfig struct { + RawURL string + BlockNumber *big.Int + LocalCache bool +} + +// DefaultTestConfig is the default TestConfig +var DefaultTestConfig = TestConfig{ + RawURL: "http://127.0.0.1:8545", + BlockNumber: big.NewInt(12914664), + LocalCache: true, +} + +func GetTestConfig() TestConfig { + conf := DefaultTestConfig + rawURL := os.Getenv(TEST_RAW_URL) + if rawURL == "" { + fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+ + "then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL) + } else { + conf.RawURL = rawURL + } + return conf +} + +// TestBlockAndReceiptsFromEnv retrieves the block and receipts using env variables to override default config block number +func TestBlockAndReceiptsFromEnv(conf TestConfig) (*types.Block, types.Receipts, error) { + blockNumberStr := os.Getenv(TEST_BLOCK_NUMBER) + blockNumber, ok := new(big.Int).SetString(blockNumberStr, 10) + if !ok { + fmt.Printf("Warning: no blockNumber configured for statediffing mainnet tests, using default (%d)\r\n", + DefaultTestConfig.BlockNumber) + } else { + conf.BlockNumber = blockNumber + } + return TestBlockAndReceipts(conf) +} + +// TestBlockAndReceipts retrieves the block and receipts for the provided test config +// It first tries to load files from the local system before setting up and using an ethclient.Client to pull the data +func TestBlockAndReceipts(conf TestConfig) (*types.Block, types.Receipts, error) { + var cli *ethclient.Client + var err error + var block *types.Block + var receipts types.Receipts + blockFilePath := fmt.Sprintf("%s_%s.rlp", defaultBlockFilePath, conf.BlockNumber.String()) + if _, err = os.Stat(blockFilePath); !errors.Is(err, os.ErrNotExist) { + fmt.Printf("local file (%s) found for block %s\n", blockFilePath, conf.BlockNumber.String()) + block, err = LoadBlockRLP(blockFilePath) + if err != nil { + fmt.Printf("loading local file (%s) failed (%s), dialing remote client at %s\n", blockFilePath, err.Error(), conf.RawURL) + cli, err = ethclient.Dial(conf.RawURL) + if err != nil { + return nil, nil, err + } + block, err = FetchBlock(cli, conf.BlockNumber) + if err != nil { + return nil, nil, err + } + if conf.LocalCache { + if err := WriteBlockRLP(blockFilePath, block); err != nil { + return nil, nil, err + } + } + } + } else { + fmt.Printf("no local file found for block %s, dialing remote client at %s\n", conf.BlockNumber.String(), conf.RawURL) + cli, err = ethclient.Dial(conf.RawURL) + if err != nil { + return nil, nil, err + } + block, err = FetchBlock(cli, conf.BlockNumber) + if err != nil { + return nil, nil, err + } + if conf.LocalCache { + if err := WriteBlockRLP(blockFilePath, block); err != nil { + return nil, nil, err + } + } + } + receiptsFilePath := fmt.Sprintf("%s_%s.rlp", defaultReceiptsFilePath, conf.BlockNumber.String()) + if _, err = os.Stat(receiptsFilePath); !errors.Is(err, os.ErrNotExist) { + fmt.Printf("local file (%s) found for block %s receipts\n", receiptsFilePath, conf.BlockNumber.String()) + receipts, err = LoadReceiptsEncoding(receiptsFilePath, len(block.Transactions())) + if err != nil { + fmt.Printf("loading local file (%s) failed (%s), dialing remote client at %s\n", receiptsFilePath, err.Error(), conf.RawURL) + if cli == nil { + cli, err = ethclient.Dial(conf.RawURL) + if err != nil { + return nil, nil, err + } + } + receipts, err = FetchReceipts(cli, block) + if err != nil { + return nil, nil, err + } + if conf.LocalCache { + if err := WriteReceiptsEncoding(receiptsFilePath, block.Number(), receipts); err != nil { + return nil, nil, err + } + } + } + } else { + fmt.Printf("no local file found for block %s receipts, dialing remote client at %s\n", conf.BlockNumber.String(), conf.RawURL) + if cli == nil { + cli, err = ethclient.Dial(conf.RawURL) + if err != nil { + return nil, nil, err + } + } + receipts, err = FetchReceipts(cli, block) + if err != nil { + return nil, nil, err + } + if conf.LocalCache { + if err := WriteReceiptsEncoding(receiptsFilePath, block.Number(), receipts); err != nil { + return nil, nil, err + } + } + } + return block, receipts, nil +} + +// FetchBlock fetches the block at the provided height using the ethclient.Client +func FetchBlock(cli *ethclient.Client, blockNumber *big.Int) (*types.Block, error) { + return cli.BlockByNumber(context.Background(), blockNumber) +} + +// FetchReceipts fetches the receipts for the provided block using the ethclient.Client +func FetchReceipts(cli *ethclient.Client, block *types.Block) (types.Receipts, error) { + receipts := make(types.Receipts, len(block.Transactions())) + for i, tx := range block.Transactions() { + rct, err := cli.TransactionReceipt(context.Background(), tx.Hash()) + if err != nil { + return nil, err + } + receipts[i] = rct + } + return receipts, nil +} + +// WriteBlockRLP writes out the RLP encoding of the block to the provided filePath +func WriteBlockRLP(filePath string, block *types.Block) error { + if filePath == "" { + filePath = fmt.Sprintf("%s_%s.rlp", defaultBlockFilePath, block.Number().String()) + } + if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot create file, file (%s) already exists", filePath) + } + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("unable to create file (%s), err: %v", filePath, err) + } + fmt.Printf("writing block rlp to file at %s\r\n", filePath) + if err := block.EncodeRLP(file); err != nil { + return err + } + return file.Close() +} + +// LoadBlockRLP loads block from the rlp at filePath +func LoadBlockRLP(filePath string) (*types.Block, error) { + blockBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + block := new(types.Block) + return block, rlp.DecodeBytes(blockBytes, block) +} + +// LoadReceiptsEncoding loads receipts from the encoding at filePath +func LoadReceiptsEncoding(filePath string, cap int) (types.Receipts, error) { + rctsBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + receipts := new(types.Receipts) + return *receipts, rlp.DecodeBytes(rctsBytes, receipts) +} + +// WriteReceiptsEncoding writes out the consensus encoding of the receipts to the provided io.WriteCloser +func WriteReceiptsEncoding(filePath string, blockNumber *big.Int, receipts types.Receipts) error { + if filePath == "" { + filePath = fmt.Sprintf("%s_%s.rlp", defaultReceiptsFilePath, blockNumber.String()) + } + if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot create file, file (%s) already exists", filePath) + } + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("unable to create file (%s), err: %v", filePath, err) + } + defer file.Close() + fmt.Printf("writing receipts rlp to file at %s\r\n", filePath) + return rlp.Encode(file, receipts) +} diff --git a/statediff/indexer/test_helpers/test_helpers.go b/statediff/indexer/test_helpers/test_helpers.go new file mode 100644 index 000000000000..1b5335b7809a --- /dev/null +++ b/statediff/indexer/test_helpers/test_helpers.go @@ -0,0 +1,131 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test_helpers + +import ( + "bufio" + "context" + "os" + "testing" + + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" +) + +// ListContainsString used to check if a list of strings contains a particular string +func ListContainsString(sss []string, s string) bool { + for _, str := range sss { + if s == str { + return true + } + } + return false +} + +// DedupFile removes duplicates from the given file +func DedupFile(filePath string) error { + f, err := os.OpenFile(filePath, os.O_CREATE|os.O_RDONLY, os.ModePerm) + if err != nil { + return err + } + + stmts := make(map[string]struct{}, 0) + sc := bufio.NewScanner(f) + + for sc.Scan() { + s := sc.Text() + stmts[s] = struct{}{} + } + if err != nil { + return err + } + + f.Close() + + f, err = os.Create(filePath) + if err != nil { + return err + } + defer f.Close() + + for stmt := range stmts { + f.Write([]byte(stmt + "\n")) + } + + return nil +} + +// TearDownDB is used to tear down the watcher dbs after tests +func TearDownDB(t *testing.T, db sql.Database) { + ctx := context.Background() + tx, err := db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = tx.Exec(ctx, `DELETE FROM eth.header_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth.uncle_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth.transaction_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth.receipt_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth.state_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth.storage_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth.state_accounts`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth.access_list_elements`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth.log_cids`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM blocks`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM nodes`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `DELETE FROM eth_meta.watched_addresses`) + if err != nil { + t.Fatal(err) + } + err = tx.Commit(ctx) + if err != nil { + t.Fatal(err) + } +} diff --git a/statediff/known_gaps.go b/statediff/known_gaps.go new file mode 100644 index 000000000000..922eb6100a6e --- /dev/null +++ b/statediff/known_gaps.go @@ -0,0 +1,273 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "context" + "fmt" + "io/ioutil" + "math/big" + "os" + "strings" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/models" +) + +var ( + knownGapsInsert = "INSERT INTO eth_meta.known_gaps (starting_block_number, ending_block_number, checked_out, processing_key) " + + "VALUES ('%s', '%s', %t, %d) " + + "ON CONFLICT (starting_block_number) DO UPDATE SET (ending_block_number, processing_key) = ('%s', %d) " + + "WHERE eth_meta.known_gaps.ending_block_number <= '%s';\n" + dbQueryString = "SELECT MAX(block_number) FROM eth.header_cids" + defaultWriteFilePath = "./known_gaps.sql" +) + +type KnownGapsState struct { + // Should we check for gaps by looking at the DB and comparing the latest block with head + checkForGaps bool + // Arbitrary processingKey that can be used down the line to differentiate different geth nodes. + processingKey int64 + // This number indicates the expected difference between blocks. + // Currently, this is 1 since the geth node processes each block. But down the road this can be used in + // Tandom with the processingKey to differentiate block processing logic. + expectedDifference *big.Int + // Indicates if Geth is in an error state + // This is used to indicate the right time to upserts + errorState bool + // This array keeps track of errorBlocks as they occur. + // When the errorState is false again, we can process these blocks. + // Do we need a list, can we have /KnownStartErrorBlock and knownEndErrorBlock ints instead? + knownErrorBlocks []*big.Int + // The filepath to write SQL statements if we can't connect to the DB. + writeFilePath string + // DB object to use for reading and writing to the DB + db sql.Database + //Do we have entries in the local sql file that need to be written to the DB + sqlFileWaitingForWrite bool + // Metrics object used to track metrics. + statediffMetrics statediffMetricsHandles +} + +// Create a new KnownGapsState struct, currently unused. +func NewKnownGapsState(checkForGaps bool, processingKey int64, expectedDifference *big.Int, errorState bool, writeFilePath string, db sql.Database, statediffMetrics statediffMetricsHandles) *KnownGapsState { + return &KnownGapsState{ + checkForGaps: checkForGaps, + processingKey: processingKey, + expectedDifference: expectedDifference, + errorState: errorState, + writeFilePath: writeFilePath, + db: db, + statediffMetrics: statediffMetrics, + } +} + +func minMax(array []*big.Int) (*big.Int, *big.Int) { + var max *big.Int = array[0] + var min *big.Int = array[0] + for _, value := range array { + if max.Cmp(value) == -1 { + max = value + } + if min.Cmp(value) == 1 { + min = value + } + } + return min, max +} + +// This function actually performs the write of the known gaps. It will try to do the following, it only goes to the next step if a failure occurs. +// 1. Write to the DB directly. +// 2. Write to sql file locally. +// 3. Write to prometheus directly. +// 4. Logs and error. +func (kg *KnownGapsState) pushKnownGaps(startingBlockNumber *big.Int, endingBlockNumber *big.Int, checkedOut bool, processingKey int64) error { + if startingBlockNumber.Cmp(endingBlockNumber) == 1 { + return fmt.Errorf("Starting Block %d, is greater than ending block %d", startingBlockNumber, endingBlockNumber) + } + knownGap := models.KnownGapsModel{ + StartingBlockNumber: startingBlockNumber.String(), + EndingBlockNumber: endingBlockNumber.String(), + CheckedOut: checkedOut, + ProcessingKey: processingKey, + } + + log.Info("Updating Metrics for the start and end block") + kg.statediffMetrics.knownGapStart.Update(startingBlockNumber.Int64()) + kg.statediffMetrics.knownGapEnd.Update(endingBlockNumber.Int64()) + + var writeErr error + log.Info("Writing known gaps to the DB") + if kg.db != nil { + dbErr := kg.upsertKnownGaps(knownGap) + if dbErr != nil { + log.Warn("Error writing knownGaps to DB, writing them to file instead") + writeErr = kg.upsertKnownGapsFile(knownGap) + } + } else { + writeErr = kg.upsertKnownGapsFile(knownGap) + } + if writeErr != nil { + log.Error("Unsuccessful when writing to a file", "Error", writeErr) + log.Error("Updating Metrics for the start and end error block") + log.Error("Unable to write the following Gaps to DB or File", "startBlock", startingBlockNumber, "endBlock", endingBlockNumber) + kg.statediffMetrics.knownGapErrorStart.Update(startingBlockNumber.Int64()) + kg.statediffMetrics.knownGapErrorEnd.Update(endingBlockNumber.Int64()) + } + return nil +} + +// This is a simple wrapper function to write gaps from a knownErrorBlocks array. +func (kg *KnownGapsState) captureErrorBlocks(knownErrorBlocks []*big.Int) { + startErrorBlock, endErrorBlock := minMax(knownErrorBlocks) + + log.Warn("The following Gaps were found", "knownErrorBlocks", knownErrorBlocks) + log.Warn("Updating known Gaps table", "startErrorBlock", startErrorBlock, "endErrorBlock", endErrorBlock, "processingKey", kg.processingKey) + kg.pushKnownGaps(startErrorBlock, endErrorBlock, false, kg.processingKey) +} + +// Users provide the latestBlockInDb and the latestBlockOnChain +// as well as the expected difference. This function does some simple math. +// The expected difference for the time being is going to be 1, but as we run +// More geth nodes, the expected difference might fluctuate. +func isGap(latestBlockInDb *big.Int, latestBlockOnChain *big.Int, expectedDifference *big.Int) bool { + latestBlock := big.NewInt(0) + if latestBlock.Sub(latestBlockOnChain, expectedDifference).Cmp(latestBlockInDb) != 0 { + log.Warn("We found a gap", "latestBlockInDb", latestBlockInDb, "latestBlockOnChain", latestBlockOnChain, "expectedDifference", expectedDifference) + return true + } + return false +} + +// This function will check for Gaps and update the DB if gaps are found. +// The processingKey will currently be set to 0, but as we start to leverage horizontal scaling +// It might be a useful parameter to update depending on the geth node. +// TODO: +// REmove the return value +// Write to file if err in writing to DB +func (kg *KnownGapsState) findAndUpdateGaps(latestBlockOnChain *big.Int, expectedDifference *big.Int, processingKey int64) error { + // Make this global + latestBlockInDb, err := kg.queryDbToBigInt(dbQueryString) + if err != nil { + return err + } + + gapExists := isGap(latestBlockInDb, latestBlockOnChain, expectedDifference) + if gapExists { + startBlock := big.NewInt(0) + endBlock := big.NewInt(0) + startBlock.Add(latestBlockInDb, expectedDifference) + endBlock.Sub(latestBlockOnChain, expectedDifference) + + log.Warn("Found Gaps starting at", "startBlock", startBlock, "endingBlock", endBlock) + err := kg.pushKnownGaps(startBlock, endBlock, false, processingKey) + if err != nil { + log.Error("We were unable to write the following gap to the DB", "start Block", startBlock, "endBlock", endBlock, "error", err) + return err + } + } + + return nil +} + +// Upserts known gaps to the DB. +func (kg *KnownGapsState) upsertKnownGaps(knownGaps models.KnownGapsModel) error { + _, err := kg.db.Exec(context.Background(), kg.db.InsertKnownGapsStm(), + knownGaps.StartingBlockNumber, knownGaps.EndingBlockNumber, knownGaps.CheckedOut, knownGaps.ProcessingKey) + if err != nil { + return fmt.Errorf("error upserting known_gaps entry: %v", err) + } + log.Info("Successfully Wrote gaps to the DB", "startBlock", knownGaps.StartingBlockNumber, "endBlock", knownGaps.EndingBlockNumber) + return nil +} + +// Write upsert statement into a local file. +func (kg *KnownGapsState) upsertKnownGapsFile(knownGaps models.KnownGapsModel) error { + insertStmt := []byte(fmt.Sprintf(knownGapsInsert, knownGaps.StartingBlockNumber, knownGaps.EndingBlockNumber, knownGaps.CheckedOut, knownGaps.ProcessingKey, + knownGaps.EndingBlockNumber, knownGaps.ProcessingKey, knownGaps.EndingBlockNumber)) + log.Info("Trying to write file") + if kg.writeFilePath == "" { + kg.writeFilePath = defaultWriteFilePath + } + f, err := os.OpenFile(kg.writeFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Info("Unable to open a file for writing") + return err + } + defer f.Close() + + if _, err = f.Write(insertStmt); err != nil { + log.Info("Unable to open write insert statement to file") + return err + } + log.Info("Wrote the gaps to a local SQL file") + kg.sqlFileWaitingForWrite = true + return nil +} + +func (kg *KnownGapsState) writeSqlFileStmtToDb() error { + log.Info("Writing the local SQL file for KnownGaps to the DB") + file, err := ioutil.ReadFile(kg.writeFilePath) + + if err != nil { + log.Error("Unable to open local SQL File for writing") + return err + } + + requests := strings.Split(string(file), ";") + + for _, request := range requests { + _, err := kg.db.Exec(context.Background(), request) + if err != nil { + log.Error("Unable to run insert statement from file to the DB") + return err + } + } + if err := os.Truncate(kg.writeFilePath, 0); err != nil { + log.Info("Failed to empty knownGaps file after inserting statements to the DB", "error", err) + } + kg.sqlFileWaitingForWrite = false + return nil +} + +// This is a simple wrapper function which will run QueryRow on the DB +func (kg *KnownGapsState) queryDb(queryString string) (string, error) { + var ret string + err := kg.db.QueryRow(context.Background(), queryString).Scan(&ret) + if err != nil { + log.Error(fmt.Sprint("Can't properly query the DB for query: ", queryString)) + return "", err + } + return ret, nil +} + +// This function is a simple wrapper which will call QueryDb but the return value will be +// a big int instead of a string +func (kg *KnownGapsState) queryDbToBigInt(queryString string) (*big.Int, error) { + ret := new(big.Int) + res, err := kg.queryDb(queryString) + if err != nil { + return ret, err + } + ret, ok := ret.SetString(res, 10) + if !ok { + log.Error(fmt.Sprint("Can't turn the res ", res, "into a bigInt")) + return ret, fmt.Errorf("Can't turn %s into a bigInt", res) + } + return ret, nil +} diff --git a/statediff/known_gaps_test.go b/statediff/known_gaps_test.go new file mode 100644 index 000000000000..258bb887ab74 --- /dev/null +++ b/statediff/known_gaps_test.go @@ -0,0 +1,207 @@ +package statediff + +import ( + "context" + "fmt" + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" + "github.com/stretchr/testify/require" +) + +var ( + knownGapsFilePath = "./known_gaps.sql" +) + +type gapValues struct { + knownErrorBlocksStart int64 + knownErrorBlocksEnd int64 + expectedDif int64 + processingKey int64 +} + +// Add clean db +// Test for failures when they are expected, when we go from smaller block to larger block +// We should no longer see the smaller block in DB +func TestKnownGaps(t *testing.T) { + tests := []gapValues{ + // Known Gaps + {knownErrorBlocksStart: 115, knownErrorBlocksEnd: 120, expectedDif: 1, processingKey: 1}, + /// Same tests as above with a new expected DIF + {knownErrorBlocksStart: 1150, knownErrorBlocksEnd: 1200, expectedDif: 2, processingKey: 2}, + // Test update when block number is larger!! + {knownErrorBlocksStart: 1150, knownErrorBlocksEnd: 1204, expectedDif: 2, processingKey: 2}, + // Update when processing key is different! + {knownErrorBlocksStart: 1150, knownErrorBlocksEnd: 1204, expectedDif: 2, processingKey: 10}, + } + + testWriteToDb(t, tests, true) + testWriteToFile(t, tests, true) + testFindAndUpdateGaps(t, true) +} + +// test writing blocks to the DB +func testWriteToDb(t *testing.T, tests []gapValues, wipeDbBeforeStart bool) { + t.Log("Starting Write to DB test") + db := setupDb(t) + + // Clear Table first, this is needed because we updated an entry to have a larger endblock number + // so we can't find the original start and endblock pair. + if wipeDbBeforeStart { + t.Log("Cleaning up eth_meta.known_gaps table") + db.Exec(context.Background(), "DELETE FROM eth_meta.known_gaps") + } + + for _, tc := range tests { + // Create an array with knownGaps based on user inputs + knownGaps := KnownGapsState{ + processingKey: tc.processingKey, + expectedDifference: big.NewInt(tc.expectedDif), + db: db, + statediffMetrics: RegisterStatediffMetrics(metrics.DefaultRegistry), + } + service := &Service{ + KnownGaps: knownGaps, + } + knownErrorBlocks := (make([]*big.Int, 0)) + knownErrorBlocks = createKnownErrorBlocks(knownErrorBlocks, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) + service.KnownGaps.knownErrorBlocks = knownErrorBlocks + // Upsert + testCaptureErrorBlocks(t, service) + // Validate that the upsert was done correctly. + validateUpsert(t, service, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) + } + tearDown(t, db) +} + +// test writing blocks to file and then inserting them to DB +func testWriteToFile(t *testing.T, tests []gapValues, wipeDbBeforeStart bool) { + t.Log("Starting write to file test") + db := setupDb(t) + // Clear Table first, this is needed because we updated an entry to have a larger endblock number + // so we can't find the original start and endblock pair. + if wipeDbBeforeStart { + t.Log("Cleaning up eth_meta.known_gaps table") + db.Exec(context.Background(), "DELETE FROM eth_meta.known_gaps") + } + if _, err := os.Stat(knownGapsFilePath); err == nil { + err := os.Remove(knownGapsFilePath) + if err != nil { + t.Fatal("Can't delete local file") + } + } + tearDown(t, db) + for _, tc := range tests { + knownGaps := KnownGapsState{ + processingKey: tc.processingKey, + expectedDifference: big.NewInt(tc.expectedDif), + writeFilePath: knownGapsFilePath, + statediffMetrics: RegisterStatediffMetrics(metrics.DefaultRegistry), + db: nil, // Only set to nil to be verbose that we can't use it + } + service := &Service{ + KnownGaps: knownGaps, + } + knownErrorBlocks := (make([]*big.Int, 0)) + knownErrorBlocks = createKnownErrorBlocks(knownErrorBlocks, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) + service.KnownGaps.knownErrorBlocks = knownErrorBlocks + + testCaptureErrorBlocks(t, service) + newDb := setupDb(t) + service.KnownGaps.db = newDb + if service.KnownGaps.sqlFileWaitingForWrite { + writeErr := service.KnownGaps.writeSqlFileStmtToDb() + require.NoError(t, writeErr) + } + + // Validate that the upsert was done correctly. + validateUpsert(t, service, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) + tearDown(t, newDb) + } +} + +// Find a gap, if no gaps exist, it will create an arbitrary one +func testFindAndUpdateGaps(t *testing.T, wipeDbBeforeStart bool) { + db := setupDb(t) + + if wipeDbBeforeStart { + db.Exec(context.Background(), "DELETE FROM eth_meta.known_gaps") + } + knownGaps := KnownGapsState{ + processingKey: 1, + expectedDifference: big.NewInt(1), + db: db, + statediffMetrics: RegisterStatediffMetrics(metrics.DefaultRegistry), + } + service := &Service{ + KnownGaps: knownGaps, + } + + latestBlockInDb, err := service.KnownGaps.queryDbToBigInt("SELECT MAX(block_number) FROM eth.header_cids") + if err != nil { + t.Skip("Can't find a block in the eth.header_cids table.. Please put one there") + } + + // Add the gapDifference for testing purposes + gapDifference := big.NewInt(10) // Set a difference between latestBlock in DB and on Chain + expectedDifference := big.NewInt(1) // Set what the expected difference between latestBlock in DB and on Chain should be + + latestBlockOnChain := big.NewInt(0) + latestBlockOnChain.Add(latestBlockInDb, gapDifference) + + t.Log("The latest block on the chain is: ", latestBlockOnChain) + t.Log("The latest block on the DB is: ", latestBlockInDb) + + gapUpsertErr := service.KnownGaps.findAndUpdateGaps(latestBlockOnChain, expectedDifference, 0) + require.NoError(t, gapUpsertErr) + + startBlock := big.NewInt(0) + endBlock := big.NewInt(0) + + startBlock.Add(latestBlockInDb, expectedDifference) + endBlock.Sub(latestBlockOnChain, expectedDifference) + validateUpsert(t, service, startBlock.Int64(), endBlock.Int64()) +} + +// test capturing missed blocks +func testCaptureErrorBlocks(t *testing.T, service *Service) { + service.KnownGaps.captureErrorBlocks(service.KnownGaps.knownErrorBlocks) +} + +// Helper function to create an array of gaps given a start and end block +func createKnownErrorBlocks(knownErrorBlocks []*big.Int, knownErrorBlocksStart int64, knownErrorBlocksEnd int64) []*big.Int { + for i := knownErrorBlocksStart; i <= knownErrorBlocksEnd; i++ { + knownErrorBlocks = append(knownErrorBlocks, big.NewInt(i)) + } + return knownErrorBlocks +} + +// Make sure the upsert was performed correctly +func validateUpsert(t *testing.T, service *Service, startingBlock int64, endingBlock int64) { + t.Logf("Starting to query blocks: %d - %d", startingBlock, endingBlock) + queryString := fmt.Sprintf("SELECT starting_block_number from eth_meta.known_gaps WHERE starting_block_number = %d AND ending_block_number = %d", startingBlock, endingBlock) + + _, queryErr := service.KnownGaps.queryDb(queryString) // Figure out the string. + t.Logf("Updated Known Gaps table starting from, %d, and ending at, %d", startingBlock, endingBlock) + require.NoError(t, queryErr) +} + +// Create a DB object to use +func setupDb(t *testing.T) sql.Database { + db, err := postgres.SetupSQLXDB() + if err != nil { + t.Error("Can't create a DB connection....") + t.Fatal(err) + } + return db +} + +// Teardown the DB +func tearDown(t *testing.T, db sql.Database) { + t.Log("Starting tearDown") + db.Close() +} diff --git a/statediff/mainnet_tests/block0_rlp b/statediff/mainnet_tests/block0_rlp new file mode 100644 index 000000000000..eb912911d159 Binary files /dev/null and b/statediff/mainnet_tests/block0_rlp differ diff --git a/statediff/mainnet_tests/block1_rlp b/statediff/mainnet_tests/block1_rlp new file mode 100644 index 000000000000..d286fafde746 Binary files /dev/null and b/statediff/mainnet_tests/block1_rlp differ diff --git a/statediff/mainnet_tests/block2_rlp b/statediff/mainnet_tests/block2_rlp new file mode 100644 index 000000000000..6a8b8160099b Binary files /dev/null and b/statediff/mainnet_tests/block2_rlp differ diff --git a/statediff/mainnet_tests/block3_rlp b/statediff/mainnet_tests/block3_rlp new file mode 100644 index 000000000000..86f90a83a6e5 Binary files /dev/null and b/statediff/mainnet_tests/block3_rlp differ diff --git a/statediff/mainnet_tests/builder_test.go b/statediff/mainnet_tests/builder_test.go new file mode 100644 index 000000000000..c487304f9e6e --- /dev/null +++ b/statediff/mainnet_tests/builder_test.go @@ -0,0 +1,689 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "math/big" + "os" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/test_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var ( + db ethdb.Database + genesisBlock, block0, block1, block2, block3 *types.Block + block1CoinbaseAddr, block2CoinbaseAddr, block3CoinbaseAddr common.Address + block1CoinbaseHash, block2CoinbaseHash, block3CoinbaseHash common.Hash + builder statediff.Builder + emptyStorage = make([]sdtypes.StorageNode, 0) + + // block 1 data + block1CoinbaseAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(5000000000000000000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + block1CoinbaseLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("38251692195afc818c92b485fcb8a4691af89cbe5a2ab557b83a4261be2a9a"), + block1CoinbaseAccount, + }) + block1CoinbaseLeafNodeHash = crypto.Keccak256(block1CoinbaseLeafNode) + block1x040bBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("cc947d5ebb80600bad471f12c6ad5e4981e3525ecf8a2d982cc032536ae8b66d"), + common.Hex2Bytes("e80e52462e635a834e90e86ccf7673a6430384aac17004d626f4db831f0624bc"), + common.Hex2Bytes("59a8f11f60cb0a8488831f242da02944a26fd269d0608a44b8b873ded9e59e1b"), + common.Hex2Bytes("1ffb51e987e3cbd2e1dc1a64508d2e2b265477e21698b0d10fdf137f35027f40"), + []byte{}, + common.Hex2Bytes("ce5077f49a13ff8199d0e77715fdd7bfd6364774effcd5499bd93cba54b3c644"), + common.Hex2Bytes("f5146783c048e66ce1a776ae990b4255e5fba458ece77fcb83ff6e91d6637a88"), + common.Hex2Bytes("6a0558b6c38852e985cf01c2156517c1c6a1e64c787a953c347825f050b236c6"), + common.Hex2Bytes("56b6e93958b99aaae158cc2329e71a1865ba6f39c67b096922c5cf3ed86b0ae5"), + []byte{}, + common.Hex2Bytes("50d317a89a3405367d66668902f2c9f273a8d0d7d5d790dc516bca142f4a84af"), + common.Hex2Bytes("c72ca72750fdc1af3e6da5c7c5d82c54e4582f15b488a8aa1674058a99825dae"), + common.Hex2Bytes("e1a489df7b18cde818da6d38e235b026c2e61bcd3d34880b3ed0d67e0e4f0159"), + common.Hex2Bytes("b58d5062f2609fd2d68f00d14ab33fef2b373853877cf40bf64729e85b8fdc54"), + block1CoinbaseLeafNodeHash, + []byte{}, + []byte{}, + }) + block1x040bBranchNodeHash = crypto.Keccak256(block1x040bBranchNode) + block1x04BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("a9317a59365ca09cefcd384018696590afffc432e35a97e8f85aa48907bf3247"), + common.Hex2Bytes("e0bc229254ce7a6a736c3953e570ab18b4a7f5f2a9aa3c3057b5f17d250a1cad"), + common.Hex2Bytes("a2484ec8884dbe0cf24ece99d67df0d1fe78992d67cc777636a817cb2ef205aa"), + common.Hex2Bytes("12b78d4078c607747f06bb88bd08f839eaae0e3ac6854e5f65867d4f78abb84e"), + common.Hex2Bytes("359a51862df5462e4cd302f69cb338512f21eb37ce0791b9a562e72ec48b7dbf"), + common.Hex2Bytes("13f8d617b6a734da9235b6ac80bdd7aeaff6120c39aa223638d88f22d4ba4007"), + common.Hex2Bytes("02055c6400e0ec3440a8bb8fdfd7d6b6c57b7bf83e37d7e4e983d416fdd8314e"), + common.Hex2Bytes("4b1cca9eb3e47e805e7f4c80671a9fcd589fd6ddbe1790c3f3e177e8ede01b9e"), + common.Hex2Bytes("70c3815efb23b986018089e009a38e6238b8850b3efd33831913ca6fa9240249"), + common.Hex2Bytes("7084699d2e72a193fd75bb6108ae797b4661696eba2d631d521fc94acc7b3247"), + common.Hex2Bytes("b2b3cd9f1e46eb583a6185d9a96b4e80125e3d75e6191fdcf684892ef52935cb"), + block1x040bBranchNodeHash, + common.Hex2Bytes("34d9ff0fee6c929424e52268dedbc596d10786e909c5a68d6466c2aba17387ce"), + common.Hex2Bytes("7484d5e44b6ee6b10000708c37e035b42b818475620f9316beffc46531d1eebf"), + common.Hex2Bytes("30c8a283adccf2742272563cd3d6710c89ba21eac0118bf5310cfb231bcca77f"), + common.Hex2Bytes("4bae8558d2385b8d3bc6e6ede20bdbc5dbb0b5384c316ba8985682f88d2e506d"), + []byte{}, + }) + block1x04BranchNodeHash = crypto.Keccak256(block1x04BranchNode) + block1RootBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("90dcaf88c40c7bbc95a912cbdde67c175767b31173df9ee4b0d733bfdd511c43"), + common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), + common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), + common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"), + block1x04BranchNodeHash, + common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"), + common.Hex2Bytes("e823850f50bf72baae9d1733a36a444ab65d0a6faaba404f0583ce0ca4dad92d"), + common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"), + common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"), + common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"), + common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"), + common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"), + common.Hex2Bytes("6fc2d754e304c48ce6a517753c62b1a9c1d5925b89707486d7fc08919e0a94ec"), + common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"), + common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"), + common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"), + []byte{}, + }) + + // block 2 data + block2CoinbaseAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(5000000000000000000), + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + block2CoinbaseLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("20679cbcf198c1741a6f4e4473845659a30caa8b26f8d37a0be2e2bc0d8892"), + block2CoinbaseAccount, + }) + block2CoinbaseLeafNodeHash = crypto.Keccak256(block2CoinbaseLeafNode) + block2MovedPremineBalance, _ = new(big.Int).SetString("4000000000000000000000", 10) + block2MovedPremineAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: block2MovedPremineBalance, + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + block2MovedPremineLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("20f2e24db7943eab4415f99e109698863b0fecca1cf9ffc500f38cefbbe29e"), + block2MovedPremineAccount, + }) + block2MovedPremineLeafNodeHash = crypto.Keccak256(block2MovedPremineLeafNode) + block2x00080dBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + block2MovedPremineLeafNodeHash, + []byte{}, + []byte{}, + []byte{}, + block2CoinbaseLeafNodeHash, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block2x00080dBranchNodeHash = crypto.Keccak256(block2x00080dBranchNode) + block2x0008BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("def97a26f824fc3911cf7f8c41dfc9bc93cc36ae2248de22ecae01d6950b2dc9"), + common.Hex2Bytes("234a575e2c5badab8de0f6515b6723195323a0562fbe1316255888637043f1c1"), + common.Hex2Bytes("29659740af1c23306ee8f8294c71a5632ace8c80b1eb61cfdf7022f47ff52305"), + common.Hex2Bytes("cf2681d23bb666d89dec8123bce9e626240a7e2ce7a1e8316b1ee88181c9471c"), + common.Hex2Bytes("18d8de6967fe34b9fd411c74fecc45f8a737961791e70d8ece967bb07cf4d4dc"), + common.Hex2Bytes("7cad60c7cbca8c79c2db5a8fc1baa9381484d43d6c37dfb97718c3a109d47dfc"), + common.Hex2Bytes("2138f5a9062b750b6320e5fac5b134da90a9edbda06ef3e1ae64fb1366ca998c"), + common.Hex2Bytes("532826502a9661fcae7c0f5d2a4c8cb287dfc521e828349543c5a461a9d591ed"), + common.Hex2Bytes("30543537413dd086d4b1560f46b90e8da0f43de5584a138ab036d74e84657523"), + common.Hex2Bytes("c98042928af640bfa1142aca895cd76e146332dce94ddad3426e74ed519ca1e0"), + common.Hex2Bytes("43de3e62cc3148193899d018dff813c04c5b636ce95bd7e828416204292d9ff9"), + []byte{}, + common.Hex2Bytes("78d533b9182bb42f6c16e9ebd5734f0d280179ba1c9b6316c2c1df73f7dd8a54"), + block2x00080dBranchNodeHash, + common.Hex2Bytes("934b736b57a892aaa15a03c7e37746bb096313727135f9841cb64c263785cf81"), + common.Hex2Bytes("38ce97150e90dfd7258901a0ddee72d8e30760a3d0419dbb80135c66588739a2"), + []byte{}, + }) + block2x0008BranchNodeHash = crypto.Keccak256(block2x0008BranchNode) + block2x00BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("e45a9e85cab1b6eb18b30df2c6acc448bbac6a30d81646823b31223e16e5063e"), + common.Hex2Bytes("33bd7171d556b981f6849064eb09412b24fedc0812127db936067043f53db1b9"), + common.Hex2Bytes("ca56945f074da4f15587404593faf3a50d17ea0e21a418ad6ec99bdf4bf3f914"), + common.Hex2Bytes("da23e9004f782df128eea1adff77952dc85f91b7f7ca4893aac5f21d24c3a1c9"), + common.Hex2Bytes("ba5ec61fa780ee02af19db99677c37560fc4f0df5c278d9dfa2837f30f72bc6b"), + common.Hex2Bytes("8310ad91625c2e3429a74066b7e2e0c958325e4e7fa3ec486b73b7c8300cfef7"), + common.Hex2Bytes("732e5c103bf4d5adfef83773026809d9405539b67e93293a02342e83ad2fb766"), + common.Hex2Bytes("30d14ff0c2aab57d1fbaf498ab14519b4e9d94f149a3dc15f0eec5adf8df25e1"), + block2x0008BranchNodeHash, + common.Hex2Bytes("5a43bd92e55aa78df60e70b6b53b6366c4080fd6a5bdd7b533b46aff4a75f6f2"), + common.Hex2Bytes("a0c410aa59efe416b1213166fab680ce330bd46c3ebf877ff14609ee6a383600"), + common.Hex2Bytes("2f41e918786e557293068b1eda9b3f9f86ed4e65a6a5363ee3262109f6e08b17"), + common.Hex2Bytes("01f42a40f02f6f24bb97b09c4d3934e8b03be7cfbb902acc1c8fd67a7a5abace"), + common.Hex2Bytes("0acbdce2787a6ea177209bd13bfc9d0779d7e2b5249e0211a2974164e14312f5"), + common.Hex2Bytes("dadbe113e4132e0c0c3cd4867e0a2044d0e5a3d44b350677ed42fc9244d004d4"), + common.Hex2Bytes("aa7441fefc17d76aedfcaf692fe71014b94c1547b6d129562b34fc5995ca0d1a"), + []byte{}, + }) + block2x00BranchNodeHash = crypto.Keccak256(block2x00BranchNode) + block2RootBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + block2x00BranchNodeHash, + common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), + common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), + common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"), + block1x04BranchNodeHash, + common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"), + common.Hex2Bytes("e823850f50bf72baae9d1733a36a444ab65d0a6faaba404f0583ce0ca4dad92d"), + common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"), + common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"), + common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"), + common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"), + common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"), + common.Hex2Bytes("6fc2d754e304c48ce6a517753c62b1a9c1d5925b89707486d7fc08919e0a94ec"), + common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"), + common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"), + common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"), + []byte{}, + }) + + // block3 data + // path 060e0f + blcok3CoinbaseBalance, _ = new(big.Int).SetString("5156250000000000000", 10) + block3CoinbaseAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: blcok3CoinbaseBalance, + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + block3CoinbaseLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3a174f00e64521a535f35e67c1aa241951c791639b2f3d060f49c5d9fa8b9e"), + block3CoinbaseAccount, + }) + block3CoinbaseLeafNodeHash = crypto.Keccak256(block3CoinbaseLeafNode) + // path 0c0e050703 + block3MovedPremineBalance1, _ = new(big.Int).SetString("3750000000000000000", 10) + block3MovedPremineAccount1, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: block3MovedPremineBalance1, + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + block3MovedPremineLeafNode1, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190"), // ce573ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190 + block3MovedPremineAccount1, + }) + block3MovedPremineLeafNodeHash1 = crypto.Keccak256(block3MovedPremineLeafNode1) + // path 0c0e050708 + block3MovedPremineBalance2, _ = new(big.Int).SetString("1999944000000000000000", 10) + block3MovedPremineAccount2, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: 0, + Balance: block3MovedPremineBalance2, + CodeHash: test_helpers.NullCodeHash.Bytes(), + Root: test_helpers.EmptyContractRoot, + }) + block3MovedPremineLeafNode2, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("33bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012"), // ce5783bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012 + block3MovedPremineAccount2, + }) + block3MovedPremineLeafNodeHash2 = crypto.Keccak256(block3MovedPremineLeafNode2) + + block3x0c0e0507BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + []byte{}, + []byte{}, + []byte{}, + block3MovedPremineLeafNodeHash1, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + block3MovedPremineLeafNodeHash2, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block3x0c0e0507BranchNodeHash = crypto.Keccak256(block3x0c0e0507BranchNode) + + block3x0c0e05BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("452e3beb503b1d87ae7c672b98a8e3fd043a671405502562ae1043dc97151a50"), + []byte{}, + common.Hex2Bytes("2f5bb16f77086f67ce8c4258cb9061cb299e597b2ad4ad6d7ccc474d6d88e85e"), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + block3x0c0e0507BranchNodeHash, + []byte{}, + common.Hex2Bytes("44623e5a9319f83870db0ea4611a25fca1e1da3eeea2be4a091dfc15ab45689e"), + common.Hex2Bytes("b41e047a97f44fa4cb8146467b88c8f4705811029d9e170abb0aba7d0af9f0da"), + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + []byte{}, + }) + block3x0c0e05BranchNodeHash = crypto.Keccak256(block3x0c0e05BranchNode) + + block3x060eBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("94d77c7c30b88829c9989948b206cda5e532b38b49534261c517aebf4a3e6fdb"), + common.Hex2Bytes("a5cf57a50da8204964e834a12a53f9bed7afc9b700a4a81b440122d60c7603a7"), + []byte{}, + common.Hex2Bytes("3730ec0571f34b6c3b178dc26ccb31a3f50c29da9b1921e41b9477ddab41b0fe"), + []byte{}, + common.Hex2Bytes("543952bb9566c2018cf8d7b90d6a7903cdfff3d79ac36189be5322de42fc3fc0"), + []byte{}, + common.Hex2Bytes("c4a49b66f0bcc08531e50cdea5577a281d111fa542eaefd9a9aead8febb0735e"), + common.Hex2Bytes("362ad58916c71463b98c079649fc486c5f082c4f548bd4ab501515f0c5641cb4"), + common.Hex2Bytes("36aae109f6f55f0bd05eb05bb365af2332dfe5f06d3d17903e88534c319eb709"), + common.Hex2Bytes("430dcfc5cc49a6b490dd54138920e8f94e427239c2bccc14705cfd4ff6cc4383"), + common.Hex2Bytes("73ed77563dfed2fdb38900b474db88b2270f449167e0d877fda9e2229f119fe8"), + common.Hex2Bytes("5dfe06013f2a41f1779194ceb07769d019f518b2a694a82fa1661e60fd973eaa"), + common.Hex2Bytes("80bdfd85fbb6b45850bad6e34136aaa1b04711e47469fa2f0d19eca52089efb5"), + []byte{}, + block3CoinbaseLeafNodeHash, + []byte{}, + }) + block3x060eBranchNodeHash = crypto.Keccak256(block3x060eBranchNode) + + block3x0c0eBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("70647f11b2b995d718f9e8aceb44c8839e0055641930d216fa6090280a9d63d5"), + common.Hex2Bytes("fdfb17cd2fba2a14219981cb7886a1977cd85dbef5c767c562f4a5f547febff0"), + common.Hex2Bytes("ff87313253ec6f860142b7bf62efb4cb07ea668c57aa90cbe9ef22b72fee15c7"), + common.Hex2Bytes("3a77b3c26a54ad37bdf4e19c1bce93493ec0f79d9ad90190b70bc840b54918e1"), + common.Hex2Bytes("af1b3b14324561b68f2e24dbcc28673ab35ce3fd0230fe2bc86b3d1931745195"), + block3x0c0e05BranchNodeHash, + common.Hex2Bytes("647dcbfe6aabcd9d219ff40422af4326bfc1ec66703195a78eb48618ddef248d"), + common.Hex2Bytes("2d2bf06159cc8928283c3419a03f08ea34c493a9d002a0ec76d5c429508ccaf4"), + common.Hex2Bytes("d7147251b3f48f25e1e4c6d8f83a00b1eca66e99a4ea0d238942ce72d0ba6414"), + common.Hex2Bytes("cb859370869967594fb29f4e2904413310146733d7fcbd11407f3e47626e0e34"), + common.Hex2Bytes("b93ab9b0bd83963860fbe0b7d543879cfde756ea1618d2a40d85483058cc5a26"), + common.Hex2Bytes("45aee096499d209931457ce251c5c7e5543f22524f67785ff8f0f3f02588b0ed"), + []byte{}, + common.Hex2Bytes("aa2ae9379797c5066bba646108074ae8677e82c923d584b6d1c1268ca3708c5c"), + common.Hex2Bytes("e6eb055f0d8e194c083471479a3de87fa0f90c0f4aaa518416ec1e469ec32e3a"), + common.Hex2Bytes("0cc9c50fc7eba162fb17f2e04e3599c13abbf210d9781864d0edec401ecaebba"), + []byte{}, + }) + block3x0c0eBranchNodeHash = crypto.Keccak256(block3x0c0eBranchNode) + + block3x06BranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("68f7ff8c074d6e4cccd55b5b1c2116a6dd7047d4332090e6db8839362991b0ae"), + common.Hex2Bytes("c446eb4377c750701374c56e50759e6ba68b7adf4d543e718c8b28a99ae3b6ad"), + common.Hex2Bytes("ef2c49ec64cb65eae0d99684e74c8af2bd0206c9a0214d9d3eddf0881dd8412a"), + common.Hex2Bytes("7096c4cc7e8125f0b142d8644ad681f8a8142e210c806f33f3f7004f0e9d6002"), + common.Hex2Bytes("bc9a8ae647b234cd6607b6b0245e3b3d5ec4f7ea006e7eda1f92d02f0ea91116"), + common.Hex2Bytes("a87720deb92ff2f899e809befab9970a61c86148c4fa09d04b77505ee4a5bda5"), + common.Hex2Bytes("2460e5b6ded7c0001de29c15db124614432fef6486370cc9970f63b0d95fd5e2"), + common.Hex2Bytes("ed1c447d4a32bc31e9e32259dc63da10df91231e786332e3df122b301b1f8fc3"), + common.Hex2Bytes("0d27dfc201d995c2323b792860dbca087da7cc56d1698c39b7c4b9277729c5ca"), + common.Hex2Bytes("f6d2be168d9c17643c9ea80c29322b364604cdfd36eef40123d83fad364e43fa"), + common.Hex2Bytes("004bf1c30a5730f464de1a0ba4ac5b5618df66d6106073d08742166e33a7eeb5"), + common.Hex2Bytes("7298d019a57a1b04ac31ed874d654ba0d3c249704c5d9efa1d08959fc89e0779"), + common.Hex2Bytes("fb3d50b7af6f839e371ff8ebd0322e94e6b6fb7888416737f88cf55bcf5859ec"), + common.Hex2Bytes("4e7a2618fa1fc560a73c24839657adf7e48d600ecfb12333678115936597a913"), + block3x060eBranchNodeHash, + common.Hex2Bytes("1909706c5db040f54c19f4050659ad484982145b02474653917de379f15ebb36"), + []byte{}, + }) + block3x06BranchNodeHash = crypto.Keccak256(block3x06BranchNode) + + block3x0cBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("dae48f5b47930c28bb116fbd55e52cd47242c71bf55373b55eb2805ee2e4a929"), + common.Hex2Bytes("0f1f37f337ec800e2e5974e2e7355f10f1a4832b39b846d916c3597a460e0676"), + common.Hex2Bytes("da8f627bb8fbeead17b318e0a8e4f528db310f591bb6ab2deda4a9f7ca902ab5"), + common.Hex2Bytes("971c662648d58295d0d0aa4b8055588da0037619951217c22052802549d94a2f"), + common.Hex2Bytes("ccc701efe4b3413fd6a61a6c9f40e955af774649a8d9fd212d046a5a39ddbb67"), + common.Hex2Bytes("d607cdb32e2bd635ee7f2f9e07bc94ddbd09b10ec0901b66628e15667aec570b"), + common.Hex2Bytes("5b89203dc940e6fa70ec19ad4e01d01849d3a5baa0a8f9c0525256ed490b159f"), + common.Hex2Bytes("b84227d48df68aecc772939a59afa9e1a4ab578f7b698bdb1289e29b6044668e"), + common.Hex2Bytes("fd1c992070b94ace57e48cbf6511a16aa770c645f9f5efba87bbe59d0a042913"), + common.Hex2Bytes("e16a7ccea6748ae90de92f8aef3b3dc248a557b9ac4e296934313f24f7fced5f"), + common.Hex2Bytes("42373cf4a00630d94de90d0a23b8f38ced6b0f7cb818b8925fee8f0c2a28a25a"), + common.Hex2Bytes("5f89d2161c1741ff428864f7889866484cef622de5023a46e795dfdec336319f"), + common.Hex2Bytes("7597a017664526c8c795ce1da27b8b72455c49657113e0455552dbc068c5ba31"), + common.Hex2Bytes("d5be9089012fda2c585a1b961e988ea5efcd3a06988e150a8682091f694b37c5"), + block3x0c0eBranchNodeHash, + common.Hex2Bytes("49bf6e8df0acafd0eff86defeeb305568e44d52d2235cf340ae15c6034e2b241"), + []byte{}, + }) + block3x0cBranchNodeHash = crypto.Keccak256(block3x0cBranchNode) + + block3RootBranchNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("f646da473c426e79f1c796b00d4873f47de1dbe1c9d19d63993a05eeb8b4041d"), + common.Hex2Bytes("babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bd"), + common.Hex2Bytes("473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021"), + common.Hex2Bytes("bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0"), + common.Hex2Bytes("d9cff5d5f2418afd16a4da5c221fdc8bd47520c5927922f69a68177b64da6ac0"), + common.Hex2Bytes("a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7"), + block3x06BranchNodeHash, + common.Hex2Bytes("f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721"), + common.Hex2Bytes("7117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681"), + common.Hex2Bytes("69eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472"), + common.Hex2Bytes("203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8e"), + common.Hex2Bytes("9287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208"), + block3x0cBranchNodeHash, + common.Hex2Bytes("7b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475"), + common.Hex2Bytes("51f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0"), + common.Hex2Bytes("89d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb"), + []byte{}, + }) +) + +func init() { + if os.Getenv("MODE") != "statediff" { + fmt.Println("Skipping statediff test") + os.Exit(0) + } + db = rawdb.NewMemoryDatabase() + genesisBlock = core.DefaultGenesisBlock().MustCommit(db) + genBy, err := rlp.EncodeToBytes(genesisBlock) + if err != nil { + log.Fatal(err) + } + var block0RLP []byte + block0, block0RLP, err = loadBlockFromRLPFile("./block0_rlp") + if err != nil { + log.Fatal(err) + } + if !bytes.Equal(genBy, block0RLP) { + log.Fatal("mainnet genesis blocks do not match") + } + block1, _, err = loadBlockFromRLPFile("./block1_rlp") + if err != nil { + log.Fatal(err) + } + block1CoinbaseAddr = block1.Coinbase() + block1CoinbaseHash = crypto.Keccak256Hash(block1CoinbaseAddr.Bytes()) + block2, _, err = loadBlockFromRLPFile("./block2_rlp") + if err != nil { + log.Fatal(err) + } + block2CoinbaseAddr = block2.Coinbase() + block2CoinbaseHash = crypto.Keccak256Hash(block2CoinbaseAddr.Bytes()) + block3, _, err = loadBlockFromRLPFile("./block3_rlp") + if err != nil { + log.Fatal(err) + } + block3CoinbaseAddr = block3.Coinbase() + block3CoinbaseHash = crypto.Keccak256Hash(block3CoinbaseAddr.Bytes()) +} + +func loadBlockFromRLPFile(filename string) (*types.Block, []byte, error) { + f, err := os.Open(filename) + if err != nil { + return nil, nil, err + } + defer f.Close() + blockRLP, err := ioutil.ReadAll(f) + if err != nil { + return nil, nil, err + } + block := new(types.Block) + return block, blockRLP, rlp.DecodeBytes(blockRLP, block) +} + +func TestBuilderOnMainnetBlocks(t *testing.T) { + chain, _ := core.NewBlockChain(db, nil, params.MainnetChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + _, err := chain.InsertChain([]*types.Block{block1, block2, block3}) + if err != nil { + t.Error(err) + } + params := statediff.Params{ + IntermediateStateNodes: true, + } + builder = statediff.NewBuilder(chain.StateCache()) + + var tests = []struct { + name string + startingArguments statediff.Args + expected *sdtypes.StateObject + }{ + // note that block0 (genesis) has over 1000 nodes due to the pre-allocation for the crowd-sale + // it is not feasible to write a unit test of that size at this time + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + statediff.Args{ + OldStateRoot: block0.Root(), + NewStateRoot: block1.Root(), + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + }, + &sdtypes.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block1RootBranchNode, + }, + { + Path: []byte{'\x04'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block1x04BranchNode, + }, + { + Path: []byte{'\x04', '\x0b'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block1x040bBranchNode, + }, + { + Path: []byte{'\x04', '\x0b', '\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: block1CoinbaseHash.Bytes(), + NodeValue: block1CoinbaseLeafNode, + StorageNodes: emptyStorage, + }, + }, + }, + }, + { + "testBlock2", + // 1000 transferred from testBankAddress to account1Addr + // 1000 transferred from account1Addr to account2Addr + // account1addr creates a new contract + statediff.Args{ + OldStateRoot: block1.Root(), + NewStateRoot: block2.Root(), + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + }, + &sdtypes.StateObject{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block2RootBranchNode, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block2x00BranchNode, + }, + { + Path: []byte{'\x00', '\x08'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block2x0008BranchNode, + }, + { + Path: []byte{'\x00', '\x08', '\x0d'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block2x00080dBranchNode, + }, + // this new leaf at x00 x08 x0d x00 was "created" when a premine account (leaf) was moved from path x00 x08 x0d + // this occurred because of the creation of the new coinbase receiving account (leaf) at x00 x08 x0d x04 + // which necessitates we create a branch at x00 x08 x0d (as shown in the below UpdateAccounts) + { + Path: []byte{'\x00', '\x08', '\x0d', '\x00'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: common.HexToHash("08d0f2e24db7943eab4415f99e109698863b0fecca1cf9ffc500f38cefbbe29e").Bytes(), + NodeValue: block2MovedPremineLeafNode, + }, + { + Path: []byte{'\x00', '\x08', '\x0d', '\x04'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: block2CoinbaseHash.Bytes(), + NodeValue: block2CoinbaseLeafNode, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + statediff.Args{ + OldStateRoot: block2.Root(), + NewStateRoot: block3.Root(), + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + }, + &sdtypes.StateObject{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3RootBranchNode, + }, + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x06BranchNode, + }, + { + Path: []byte{'\x06', '\x0e'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x060eBranchNode, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x0cBranchNode, + }, + { + Path: []byte{'\x0c', '\x0e'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x0c0eBranchNode, + }, + { + Path: []byte{'\x0c', '\x0e', '\x05'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x0c0e05BranchNode, + }, + { + Path: []byte{'\x0c', '\x0e', '\x05', '\x07'}, + NodeType: sdtypes.Branch, + StorageNodes: emptyStorage, + NodeValue: block3x0c0e0507BranchNode, + }, + { // How was this account created??? + Path: []byte{'\x0c', '\x0e', '\x05', '\x07', '\x03'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: common.HexToHash("ce573ced93917e658d10e2d9009470dad72b63c898d173721194a12f2ae5e190").Bytes(), + NodeValue: block3MovedPremineLeafNode1, + }, + { // This account (leaf) used to be at 0c 0e 05 07, likely moves because of the new account above + Path: []byte{'\x0c', '\x0e', '\x05', '\x07', '\x08'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: common.HexToHash("ce5783bc1e69eedf90f402e11f6862da14ed8e50156635a04d6393bbae154012").Bytes(), + NodeValue: block3MovedPremineLeafNode2, + }, + { // this is the new account created due to the coinbase mining a block, it's creation shouldn't affect 0x 0e 05 07 + Path: []byte{'\x06', '\x0e', '\x0f'}, + NodeType: sdtypes.Leaf, + StorageNodes: emptyStorage, + LeafKey: block3CoinbaseHash.Bytes(), + NodeValue: block3CoinbaseLeafNode, + }, + }, + }, + }, + } + + for _, test := range tests { + diff, err := builder.BuildStateDiffObject(test.startingArguments, params) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(&test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff: %+v\nexpected state diff: %+v", diff, test.expected) + } + } +} diff --git a/statediff/metrics.go b/statediff/metrics.go new file mode 100644 index 000000000000..e67499b94328 --- /dev/null +++ b/statediff/metrics.go @@ -0,0 +1,86 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package statediff + +import ( + "strings" + + "github.com/ethereum/go-ethereum/metrics" +) + +const ( + namespace = "statediff" +) + +// Build a fully qualified metric name +func metricName(subsystem, name string) string { + if name == "" { + return "" + } + parts := []string{namespace, name} + if subsystem != "" { + parts = []string{namespace, subsystem, name} + } + // Prometheus uses _ but geth metrics uses / and replaces + return strings.Join(parts, "/") +} + +type statediffMetricsHandles struct { + // Height of latest synced by core.BlockChain + // FIXME + lastSyncHeight metrics.Gauge + // Height of the latest block received from chainEvent channel + lastEventHeight metrics.Gauge + // Height of latest state diff + lastStatediffHeight metrics.Gauge + // Current length of chainEvent channels + serviceLoopChannelLen metrics.Gauge + writeLoopChannelLen metrics.Gauge + // The start block of the known gap + knownGapStart metrics.Gauge + // The end block of the known gap + knownGapEnd metrics.Gauge + // A known gaps start block which had an error being written to the DB + knownGapErrorStart metrics.Gauge + // A known gaps end block which had an error being written to the DB + knownGapErrorEnd metrics.Gauge +} + +func RegisterStatediffMetrics(reg metrics.Registry) statediffMetricsHandles { + ctx := statediffMetricsHandles{ + lastSyncHeight: metrics.NewGauge(), + lastEventHeight: metrics.NewGauge(), + lastStatediffHeight: metrics.NewGauge(), + serviceLoopChannelLen: metrics.NewGauge(), + writeLoopChannelLen: metrics.NewGauge(), + knownGapStart: metrics.NewGauge(), + knownGapEnd: metrics.NewGauge(), + knownGapErrorStart: metrics.NewGauge(), + knownGapErrorEnd: metrics.NewGauge(), + } + subsys := "service" + reg.Register(metricName(subsys, "last_sync_height"), ctx.lastSyncHeight) + reg.Register(metricName(subsys, "last_event_height"), ctx.lastEventHeight) + reg.Register(metricName(subsys, "last_statediff_height"), ctx.lastStatediffHeight) + reg.Register(metricName(subsys, "service_loop_channel_len"), ctx.serviceLoopChannelLen) + reg.Register(metricName(subsys, "write_loop_channel_len"), ctx.writeLoopChannelLen) + reg.Register(metricName(subsys, "known_gaps_start"), ctx.knownGapStart) + reg.Register(metricName(subsys, "known_gaps_end"), ctx.knownGapEnd) + reg.Register(metricName(subsys, "known_gaps_error_start"), ctx.knownGapErrorStart) + reg.Register(metricName(subsys, "known_gaps_error_end"), ctx.knownGapErrorEnd) + return ctx +} diff --git a/statediff/payload.go b/statediff/payload.go new file mode 100644 index 000000000000..233141278f5c --- /dev/null +++ b/statediff/payload.go @@ -0,0 +1,57 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package statediff + +import ( + "encoding/json" + "math/big" +) + +// Payload packages the data to send to statediff subscriptions +type Payload struct { + BlockRlp []byte `json:"blockRlp"` + TotalDifficulty *big.Int `json:"totalDifficulty"` + ReceiptsRlp []byte `json:"receiptsRlp"` + StateObjectRlp []byte `json:"stateObjectRlp" gencodec:"required"` + + encoded []byte + err error +} + +func (sd *Payload) ensureEncoded() { + if sd.encoded == nil && sd.err == nil { + sd.encoded, sd.err = json.Marshal(sd) + } +} + +// Length to implement Encoder interface for Payload +func (sd *Payload) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Encode to implement Encoder interface for Payload +func (sd *Payload) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err +} + +// Subscription struct holds our subscription channels +type Subscription struct { + PayloadChan chan<- Payload + QuitChan chan<- bool +} diff --git a/statediff/service.go b/statediff/service.go new file mode 100644 index 000000000000..b732476476bb --- /dev/null +++ b/statediff/service.go @@ -0,0 +1,1033 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "bytes" + "fmt" + "math/big" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + ind "github.com/ethereum/go-ethereum/statediff/indexer" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + types2 "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" + "github.com/thoas/go-funk" +) + +const ( + chainEventChanSize = 20000 + genesisBlockNumber = 0 + defaultRetryLimit = 3 // default retry limit once deadlock is detected. + deadlockDetected = "deadlock detected" // 40P01 https://www.postgresql.org/docs/current/errcodes-appendix.html + typeAssertionFailed = "type assertion failed" + unexpectedOperation = "unexpected operation" +) + +var writeLoopParams = ParamsWithMutex{ + Params: Params{ + IntermediateStateNodes: true, + IntermediateStorageNodes: true, + IncludeBlock: true, + IncludeReceipts: true, + IncludeTD: true, + IncludeCode: true, + }, +} + +var statediffMetrics = RegisterStatediffMetrics(metrics.DefaultRegistry) + +type blockChain interface { + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + CurrentBlock() *types.Block + GetBlockByHash(hash common.Hash) *types.Block + GetBlockByNumber(number uint64) *types.Block + GetReceiptsByHash(hash common.Hash) types.Receipts + GetTd(hash common.Hash, number uint64) *big.Int + UnlockTrie(root common.Hash) + StateCache() state.Database +} + +// IService is the state-diffing service interface +type IService interface { + // Lifecycle Start() and Stop() methods + node.Lifecycle + // APIs method for getting API(s) for this service + APIs() []rpc.API + // Loop is the main event loop for processing state diffs + Loop(chainEventCh chan core.ChainEvent) + // Subscribe method to subscribe to receive state diff processing output` + Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool, params Params) + // Unsubscribe method to unsubscribe from state diff processing + Unsubscribe(id rpc.ID) error + // StateDiffAt method to get state diff object at specific block + StateDiffAt(blockNumber uint64, params Params) (*Payload, error) + // StateDiffFor method to get state diff object at specific block + StateDiffFor(blockHash common.Hash, params Params) (*Payload, error) + // StateTrieAt method to get state trie object at specific block + StateTrieAt(blockNumber uint64, params Params) (*Payload, error) + // StreamCodeAndCodeHash method to stream out all code and codehash pairs + StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- types2.CodeAndCodeHash, quitChan chan<- bool) + // WriteStateDiffAt method to write state diff object directly to DB + WriteStateDiffAt(blockNumber uint64, params Params) error + // WriteStateDiffFor method to write state diff object directly to DB + WriteStateDiffFor(blockHash common.Hash, params Params) error + // WriteLoop event loop for progressively processing and writing diffs directly to DB + WriteLoop(chainEventCh chan core.ChainEvent) + // Method to change the addresses being watched in write loop params + WatchAddress(operation types2.OperationType, args []types2.WatchAddressArg) error +} + +// Service is the underlying struct for the state diffing service +type Service struct { + // Used to sync access to the Subscriptions + sync.Mutex + // Used to build the state diff objects + Builder Builder + // Used to subscribe to chain events (blocks) + BlockChain blockChain + // Used to signal shutdown of the service + QuitChan chan bool + // A mapping of rpc.IDs to their subscription channels, mapped to their subscription type (hash of the Params rlp) + Subscriptions map[common.Hash]map[rpc.ID]Subscription + // A mapping of subscription params rlp hash to the corresponding subscription params + SubscriptionTypes map[common.Hash]Params + // Cache the last block so that we can avoid having to lookup the next block's parent + BlockCache BlockCache + // The publicBackendAPI which provides useful information about the current state + BackendAPI ethapi.Backend + // Should the statediff service wait for geth to sync to head? + WaitForSync bool + // Used to signal if we should check for KnownGaps + KnownGaps KnownGapsState + // Whether or not we have any subscribers; only if we do, do we processes state diffs + subscribers int32 + // Interface for publishing statediffs as PG-IPLD objects + indexer interfaces.StateDiffIndexer + // Whether to enable writing state diffs directly to track blockchain head. + enableWriteLoop bool + // Size of the worker pool + numWorkers uint + // Number of retry for aborted transactions due to deadlock. + maxRetry uint +} + +// BlockCache caches the last block for safe access from different service loops +type BlockCache struct { + sync.Mutex + blocks map[common.Hash]*types.Block + maxSize uint +} + +func NewBlockCache(max uint) BlockCache { + return BlockCache{ + blocks: make(map[common.Hash]*types.Block), + maxSize: max, + } +} + +// New creates a new statediff.Service +// func New(stack *node.Node, ethServ *eth.Ethereum, dbParams *DBParams, enableWriteLoop bool) error { +func New(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params Config, backend ethapi.Backend) error { + blockChain := ethServ.BlockChain() + var indexer interfaces.StateDiffIndexer + var db sql.Database + var err error + quitCh := make(chan bool) + indexerConfigAvailable := params.IndexerConfig != nil + if indexerConfigAvailable { + info := nodeinfo.Info{ + GenesisBlock: blockChain.Genesis().Hash().Hex(), + NetworkID: strconv.FormatUint(cfg.NetworkId, 10), + ChainID: blockChain.Config().ChainID.Uint64(), + ID: params.ID, + ClientName: params.ClientName, + } + var err error + db, indexer, err = ind.NewStateDiffIndexer(params.Context, blockChain.Config(), info, params.IndexerConfig) + if err != nil { + return err + } + indexer.ReportDBMetrics(10*time.Second, quitCh) + } + + workers := params.NumWorkers + if workers == 0 { + workers = 1 + } + // If we ever have multiple processingKeys we can update them here + // along with the expectedDifference + knownGaps := &KnownGapsState{ + processingKey: 0, + expectedDifference: big.NewInt(1), + errorState: false, + writeFilePath: params.KnownGapsFilePath, + db: db, + statediffMetrics: statediffMetrics, + sqlFileWaitingForWrite: false, + } + if indexerConfigAvailable { + if params.IndexerConfig.Type() == shared.POSTGRES { + knownGaps.checkForGaps = true + } else { + log.Info("We are not going to check for gaps on start up since we are not connected to Postgres!") + knownGaps.checkForGaps = false + } + } + sds := &Service{ + Mutex: sync.Mutex{}, + BlockChain: blockChain, + Builder: NewBuilder(blockChain.StateCache()), + QuitChan: quitCh, + Subscriptions: make(map[common.Hash]map[rpc.ID]Subscription), + SubscriptionTypes: make(map[common.Hash]Params), + BlockCache: NewBlockCache(workers), + BackendAPI: backend, + WaitForSync: params.WaitForSync, + KnownGaps: *knownGaps, + indexer: indexer, + enableWriteLoop: params.EnableWriteLoop, + numWorkers: workers, + maxRetry: defaultRetryLimit, + } + stack.RegisterLifecycle(sds) + stack.RegisterAPIs(sds.APIs()) + + if indexerConfigAvailable { + err = loadWatchedAddresses(indexer) + if err != nil { + return err + } + } + + return nil +} + +// Protocols exports the services p2p protocols, this service has none +func (sds *Service) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +// APIs returns the RPC descriptors the statediff.Service offers +func (sds *Service) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: APIName, + Version: APIVersion, + Service: NewPublicStateDiffAPI(sds), + Public: true, + }, + } +} + +// Return the parent block of currentBlock, using the cached block if available; +// and cache the passed block +func (lbc *BlockCache) getParentBlock(currentBlock *types.Block, bc blockChain) *types.Block { + lbc.Lock() + parentHash := currentBlock.ParentHash() + var parentBlock *types.Block + if block, ok := lbc.blocks[parentHash]; ok { + parentBlock = block + if len(lbc.blocks) > int(lbc.maxSize) { + delete(lbc.blocks, parentHash) + } + } else { + parentBlock = bc.GetBlockByHash(parentHash) + } + lbc.blocks[currentBlock.Hash()] = currentBlock + lbc.Unlock() + return parentBlock +} + +type workerParams struct { + chainEventCh <-chan core.ChainEvent + wg *sync.WaitGroup + id uint +} + +func (sds *Service) WriteLoop(chainEventCh chan core.ChainEvent) { + chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) + defer chainEventSub.Unsubscribe() + errCh := chainEventSub.Err() + var wg sync.WaitGroup + // Process metrics for chain events, then forward to workers + chainEventFwd := make(chan core.ChainEvent, chainEventChanSize) + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case chainEvent := <-chainEventCh: + statediffMetrics.lastEventHeight.Update(int64(chainEvent.Block.Number().Uint64())) + statediffMetrics.writeLoopChannelLen.Update(int64(len(chainEventCh))) + chainEventFwd <- chainEvent + case err := <-errCh: + log.Error("Error from chain event subscription", "error", err) + close(sds.QuitChan) + log.Info("Quitting the statediffing writing loop") + if err := sds.indexer.Close(); err != nil { + log.Error("Error closing indexer", "err", err) + } + return + case <-sds.QuitChan: + log.Info("Quitting the statediffing writing loop") + if err := sds.indexer.Close(); err != nil { + log.Error("Error closing indexer", "err", err) + } + return + } + } + }() + wg.Add(int(sds.numWorkers)) + for worker := uint(0); worker < sds.numWorkers; worker++ { + params := workerParams{chainEventCh: chainEventFwd, wg: &wg, id: worker} + go sds.writeLoopWorker(params) + } + wg.Wait() +} + +func (sds *Service) writeGenesisStateDiff(currBlock *types.Block, workerId uint) { + // For genesis block we need to return the entire state trie hence we diff it with an empty trie. + log.Info("Writing state diff", "block height", genesisBlockNumber, "worker", workerId) + writeLoopParams.RLock() + err := sds.writeStateDiffWithRetry(currBlock, common.Hash{}, writeLoopParams.Params) + writeLoopParams.RUnlock() + if err != nil { + log.Error("statediff.Service.WriteLoop: processing error", "block height", + genesisBlockNumber, "error", err.Error(), "worker", workerId) + return + } + statediffMetrics.lastStatediffHeight.Update(genesisBlockNumber) +} + +func (sds *Service) writeLoopWorker(params workerParams) { + defer params.wg.Done() + for { + select { + //Notify chain event channel of events + case chainEvent := <-params.chainEventCh: + log.Debug("WriteLoop(): chain event received", "event", chainEvent) + currentBlock := chainEvent.Block + parentBlock := sds.BlockCache.getParentBlock(currentBlock, sds.BlockChain) + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", "block height", currentBlock.Number()) + continue + } + + // chainEvent streams block from block 1, but we also need to include data from the genesis block. + if parentBlock.Number().Uint64() == genesisBlockNumber { + sds.writeGenesisStateDiff(parentBlock, params.id) + } + + // If for any reason we need to check for gaps, + // Check and update the gaps table. + if sds.KnownGaps.checkForGaps && !sds.KnownGaps.errorState { + log.Info("Checking for Gaps at", "current block", currentBlock.Number()) + go sds.KnownGaps.findAndUpdateGaps(currentBlock.Number(), sds.KnownGaps.expectedDifference, sds.KnownGaps.processingKey) + sds.KnownGaps.checkForGaps = false + } + + log.Info("Writing state diff", "block height", currentBlock.Number().Uint64(), "worker", params.id) + writeLoopParams.RLock() + err := sds.writeStateDiffWithRetry(currentBlock, parentBlock.Root(), writeLoopParams.Params) + writeLoopParams.RUnlock() + if err != nil { + log.Error("statediff.Service.WriteLoop: processing error", "block height", currentBlock.Number().Uint64(), "error", err.Error(), "worker", params.id) + sds.KnownGaps.errorState = true + log.Warn("Updating the following block to knownErrorBlocks to be inserted into knownGaps table", "blockNumber", currentBlock.Number()) + sds.KnownGaps.knownErrorBlocks = append(sds.KnownGaps.knownErrorBlocks, currentBlock.Number()) + // Write object to startdiff + continue + } + sds.KnownGaps.errorState = false + if sds.KnownGaps.knownErrorBlocks != nil { + // We must pass in parameters by VALUE not reference. + // If we pass them in my reference, the references can change before the computation is complete! + staticKnownErrorBlocks := make([]*big.Int, len(sds.KnownGaps.knownErrorBlocks)) + copy(staticKnownErrorBlocks, sds.KnownGaps.knownErrorBlocks) + sds.KnownGaps.knownErrorBlocks = nil + go sds.KnownGaps.captureErrorBlocks(staticKnownErrorBlocks) + } + + if sds.KnownGaps.sqlFileWaitingForWrite { + log.Info("There are entries in the SQL file for knownGaps that should be written") + err := sds.KnownGaps.writeSqlFileStmtToDb() + if err != nil { + log.Error("Unable to write KnownGap sql file to DB") + } + } + + // TODO: how to handle with concurrent workers + statediffMetrics.lastStatediffHeight.Update(int64(currentBlock.Number().Uint64())) + case <-sds.QuitChan: + log.Info("Quitting the statediff writing process", "worker", params.id) + return + } + } +} + +// Loop is the main processing method +func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { + log.Info("Starting statediff listening loop") + chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) + defer chainEventSub.Unsubscribe() + errCh := chainEventSub.Err() + for { + select { + //Notify chain event channel of events + case chainEvent := <-chainEventCh: + statediffMetrics.serviceLoopChannelLen.Update(int64(len(chainEventCh))) + log.Debug("Loop(): chain event received", "event", chainEvent) + // if we don't have any subscribers, do not process a statediff + if atomic.LoadInt32(&sds.subscribers) == 0 { + log.Debug("Currently no subscribers to the statediffing service; processing is halted") + continue + } + currentBlock := chainEvent.Block + parentBlock := sds.BlockCache.getParentBlock(currentBlock, sds.BlockChain) + + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", "block height", currentBlock.Number()) + continue + } + + // chainEvent streams block from block 1, but we also need to include data from the genesis block. + if parentBlock.Number().Uint64() == genesisBlockNumber { + // For genesis block we need to return the entire state trie hence we diff it with an empty trie. + sds.streamStateDiff(parentBlock, common.Hash{}) + } + + sds.streamStateDiff(currentBlock, parentBlock.Root()) + case err := <-errCh: + log.Error("Error from chain event subscription", "error", err) + close(sds.QuitChan) + log.Info("Quitting the statediffing listening loop") + sds.close() + return + case <-sds.QuitChan: + log.Info("Quitting the statediffing listening loop") + sds.close() + return + } + } +} + +// streamStateDiff method builds the state diff payload for each subscription according to their subscription type and sends them the result +func (sds *Service) streamStateDiff(currentBlock *types.Block, parentRoot common.Hash) { + sds.Lock() + for ty, subs := range sds.Subscriptions { + params, ok := sds.SubscriptionTypes[ty] + if !ok { + log.Error("no parameter set associated with this subscription", "subscription type", ty.Hex()) + sds.closeType(ty) + continue + } + // create payload for this subscription type + payload, err := sds.processStateDiff(currentBlock, parentRoot, params) + if err != nil { + log.Error("statediff processing error", "block height", currentBlock.Number().Uint64(), "parameters", params, "error", err.Error()) + continue + } + for id, sub := range subs { + select { + case sub.PayloadChan <- *payload: + log.Debug("sending statediff payload at head", "height", currentBlock.Number(), "subscription id", id) + default: + log.Info("unable to send statediff payload; channel has no receiver", "subscription id", id) + } + } + } + sds.Unlock() +} + +// StateDiffAt returns a state diff object payload at the specific blockheight +// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data +func (sds *Service) StateDiffAt(blockNumber uint64, params Params) (*Payload, error) { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info("sending state diff", "block height", blockNumber) + + // use watched addresses from statediffing write loop if not provided + if params.WatchedAddresses == nil && writeLoopParams.WatchedAddresses != nil { + writeLoopParams.RLock() + params.WatchedAddresses = make([]common.Address, len(writeLoopParams.WatchedAddresses)) + copy(params.WatchedAddresses, writeLoopParams.WatchedAddresses) + writeLoopParams.RUnlock() + } + // compute leaf paths of watched addresses in the params + params.ComputeWatchedAddressesLeafPaths() + + if blockNumber == 0 { + return sds.processStateDiff(currentBlock, common.Hash{}, params) + } + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + return sds.processStateDiff(currentBlock, parentBlock.Root(), params) +} + +// StateDiffFor returns a state diff object payload for the specific blockhash +// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data +func (sds *Service) StateDiffFor(blockHash common.Hash, params Params) (*Payload, error) { + currentBlock := sds.BlockChain.GetBlockByHash(blockHash) + log.Info("sending state diff", "block hash", blockHash) + + // use watched addresses from statediffing write loop if not provided + if params.WatchedAddresses == nil && writeLoopParams.WatchedAddresses != nil { + writeLoopParams.RLock() + params.WatchedAddresses = make([]common.Address, len(writeLoopParams.WatchedAddresses)) + copy(params.WatchedAddresses, writeLoopParams.WatchedAddresses) + writeLoopParams.RUnlock() + } + // compute leaf paths of watched addresses in the params + params.ComputeWatchedAddressesLeafPaths() + + if currentBlock.NumberU64() == 0 { + return sds.processStateDiff(currentBlock, common.Hash{}, params) + } + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + return sds.processStateDiff(currentBlock, parentBlock.Root(), params) +} + +// processStateDiff method builds the state diff payload from the current block, parent state root, and provided params +func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params Params) (*Payload, error) { + stateDiff, err := sds.Builder.BuildStateDiffObject(Args{ + NewStateRoot: currentBlock.Root(), + OldStateRoot: parentRoot, + BlockHash: currentBlock.Hash(), + BlockNumber: currentBlock.Number(), + }, params) + // allow dereferencing of parent, keep current locked as it should be the next parent + sds.BlockChain.UnlockTrie(parentRoot) + if err != nil { + return nil, err + } + stateDiffRlp, err := rlp.EncodeToBytes(&stateDiff) + if err != nil { + return nil, err + } + log.Info("state diff size", "at block height", currentBlock.Number().Uint64(), "rlp byte size", len(stateDiffRlp)) + return sds.newPayload(stateDiffRlp, currentBlock, params) +} + +func (sds *Service) newPayload(stateObject []byte, block *types.Block, params Params) (*Payload, error) { + payload := &Payload{ + StateObjectRlp: stateObject, + } + if params.IncludeBlock { + blockBuff := new(bytes.Buffer) + if err := block.EncodeRLP(blockBuff); err != nil { + return nil, err + } + payload.BlockRlp = blockBuff.Bytes() + } + if params.IncludeTD { + payload.TotalDifficulty = sds.BlockChain.GetTd(block.Hash(), block.NumberU64()) + } + if params.IncludeReceipts { + receiptBuff := new(bytes.Buffer) + receipts := sds.BlockChain.GetReceiptsByHash(block.Hash()) + if err := rlp.Encode(receiptBuff, receipts); err != nil { + return nil, err + } + payload.ReceiptsRlp = receiptBuff.Bytes() + } + return payload, nil +} + +// StateTrieAt returns a state trie object payload at the specified blockheight +// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data +func (sds *Service) StateTrieAt(blockNumber uint64, params Params) (*Payload, error) { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info("sending state trie", "block height", blockNumber) + + // compute leaf paths of watched addresses in the params + params.ComputeWatchedAddressesLeafPaths() + + return sds.processStateTrie(currentBlock, params) +} + +func (sds *Service) processStateTrie(block *types.Block, params Params) (*Payload, error) { + stateNodes, err := sds.Builder.BuildStateTrieObject(block) + if err != nil { + return nil, err + } + stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes) + if err != nil { + return nil, err + } + log.Info("state trie size", "at block height", block.Number().Uint64(), "rlp byte size", len(stateTrieRlp)) + return sds.newPayload(stateTrieRlp, block, params) +} + +// Subscribe is used by the API to subscribe to the service loop +func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool, params Params) { + log.Info("Subscribing to the statediff service") + if atomic.CompareAndSwapInt32(&sds.subscribers, 0, 1) { + log.Info("State diffing subscription received; beginning statediff processing") + } + + // compute leaf paths of watched addresses in the params + params.ComputeWatchedAddressesLeafPaths() + + // Subscription type is defined as the hash of the rlp-serialized subscription params + by, err := rlp.EncodeToBytes(¶ms) + if err != nil { + log.Error("State diffing params need to be rlp-serializable") + return + } + subscriptionType := crypto.Keccak256Hash(by) + // Add subscriber + sds.Lock() + if sds.Subscriptions[subscriptionType] == nil { + sds.Subscriptions[subscriptionType] = make(map[rpc.ID]Subscription) + } + sds.Subscriptions[subscriptionType][id] = Subscription{ + PayloadChan: sub, + QuitChan: quitChan, + } + sds.SubscriptionTypes[subscriptionType] = params + sds.Unlock() +} + +// Unsubscribe is used to unsubscribe from the service loop +func (sds *Service) Unsubscribe(id rpc.ID) error { + log.Info("Unsubscribing from the statediff service", "subscription id", id) + sds.Lock() + for ty := range sds.Subscriptions { + delete(sds.Subscriptions[ty], id) + if len(sds.Subscriptions[ty]) == 0 { + // If we removed the last subscription of this type, remove the subscription type outright + delete(sds.Subscriptions, ty) + delete(sds.SubscriptionTypes, ty) + } + } + if len(sds.Subscriptions) == 0 { + if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) { + log.Info("No more subscriptions; halting statediff processing") + } + } + sds.Unlock() + return nil +} + +// This function will check the status of geth syncing. +// It will return false if geth has finished syncing. +// It will return a true Geth is still syncing. +func (sds *Service) GetSyncStatus(pubEthAPI *ethapi.EthereumAPI) (bool, error) { + syncStatus, err := pubEthAPI.Syncing() + if err != nil { + return true, err + } + + if syncStatus != false { + return true, err + } + return false, err +} + +// This function calls GetSyncStatus to check if we have caught up to head. +// It will keep looking and checking if we have caught up to head. +// It will only complete if we catch up to head, otherwise it will keep looping forever. +func (sds *Service) WaitingForSync() error { + log.Info("We are going to wait for geth to sync to head!") + + // Has the geth node synced to head? + Synced := false + pubEthAPI := ethapi.NewEthereumAPI(sds.BackendAPI) + for !Synced { + syncStatus, err := sds.GetSyncStatus(pubEthAPI) + if err != nil { + return err + } + if !syncStatus { + log.Info("Geth has caught up to the head of the chain") + Synced = true + } else { + time.Sleep(1 * time.Second) + } + } + return nil +} + +// Start is used to begin the service +func (sds *Service) Start() error { + log.Info("Starting statediff service") + + if sds.WaitForSync { + log.Info("Statediff service will wait until geth has caught up to the head of the chain.") + err := sds.WaitingForSync() + if err != nil { + return err + } + log.Info("Continuing with startdiff start process") + } + chainEventCh := make(chan core.ChainEvent, chainEventChanSize) + go sds.Loop(chainEventCh) + + if sds.enableWriteLoop { + log.Info("Starting statediff DB write loop", "params", writeLoopParams.Params) + chainEventCh := make(chan core.ChainEvent, chainEventChanSize) + go sds.WriteLoop(chainEventCh) + } + + return nil +} + +// Stop is used to close down the service +func (sds *Service) Stop() error { + log.Info("Stopping statediff service") + close(sds.QuitChan) + return nil +} + +// close is used to close all listening subscriptions +func (sds *Service) close() { + sds.Lock() + for ty, subs := range sds.Subscriptions { + for id, sub := range subs { + select { + case sub.QuitChan <- true: + log.Info("closing subscription", "id", id) + default: + log.Info("unable to close subscription; channel has no receiver", "subscription id", id) + } + delete(sds.Subscriptions[ty], id) + } + delete(sds.Subscriptions, ty) + delete(sds.SubscriptionTypes, ty) + } + sds.Unlock() +} + +// closeType is used to close all subscriptions of given type +// closeType needs to be called with subscription access locked +func (sds *Service) closeType(subType common.Hash) { + subs := sds.Subscriptions[subType] + for id, sub := range subs { + sendNonBlockingQuit(id, sub) + } + delete(sds.Subscriptions, subType) + delete(sds.SubscriptionTypes, subType) +} + +func sendNonBlockingQuit(id rpc.ID, sub Subscription) { + select { + case sub.QuitChan <- true: + log.Info("closing subscription", "id", id) + default: + log.Info("unable to close subscription; channel has no receiver", "subscription id", id) + } +} + +// StreamCodeAndCodeHash subscription method for extracting all the codehash=>code mappings that exist in the trie at the provided height +func (sds *Service) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- types2.CodeAndCodeHash, quitChan chan<- bool) { + current := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info("sending code and codehash", "block height", blockNumber) + currentTrie, err := sds.BlockChain.StateCache().OpenTrie(current.Root()) + if err != nil { + log.Error("error creating trie for block", "block height", current.Number(), "err", err) + close(quitChan) + return + } + it := currentTrie.NodeIterator([]byte{}) + leafIt := trie.NewIterator(it) + go func() { + defer close(quitChan) + for leafIt.Next() { + select { + case <-sds.QuitChan: + return + default: + } + account := new(types.StateAccount) + if err := rlp.DecodeBytes(leafIt.Value, account); err != nil { + log.Error("error decoding state account", "err", err) + return + } + codeHash := common.BytesToHash(account.CodeHash) + code, err := sds.BlockChain.StateCache().ContractCode(common.Hash{}, codeHash) + if err != nil { + log.Error("error collecting contract code", "err", err) + return + } + outChan <- types2.CodeAndCodeHash{ + Hash: codeHash, + Code: code, + } + } + }() +} + +// WriteStateDiffAt writes a state diff at the specific blockheight directly to the database +// This operation cannot be performed back past the point of db pruning; it requires an archival node +// for historical data +func (sds *Service) WriteStateDiffAt(blockNumber uint64, params Params) error { + log.Info("writing state diff at", "block height", blockNumber) + + // use watched addresses from statediffing write loop if not provided + if params.WatchedAddresses == nil && writeLoopParams.WatchedAddresses != nil { + writeLoopParams.RLock() + params.WatchedAddresses = make([]common.Address, len(writeLoopParams.WatchedAddresses)) + copy(params.WatchedAddresses, writeLoopParams.WatchedAddresses) + writeLoopParams.RUnlock() + } + // compute leaf paths of watched addresses in the params + params.ComputeWatchedAddressesLeafPaths() + + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + parentRoot := common.Hash{} + if blockNumber != 0 { + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + parentRoot = parentBlock.Root() + } + return sds.writeStateDiffWithRetry(currentBlock, parentRoot, params) +} + +// WriteStateDiffFor writes a state diff for the specific blockhash directly to the database +// This operation cannot be performed back past the point of db pruning; it requires an archival node +// for historical data +func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params Params) error { + log.Info("writing state diff for", "block hash", blockHash) + + // use watched addresses from statediffing write loop if not provided + if params.WatchedAddresses == nil && writeLoopParams.WatchedAddresses != nil { + writeLoopParams.RLock() + params.WatchedAddresses = make([]common.Address, len(writeLoopParams.WatchedAddresses)) + copy(params.WatchedAddresses, writeLoopParams.WatchedAddresses) + writeLoopParams.RUnlock() + } + // compute leaf paths of watched addresses in the params + params.ComputeWatchedAddressesLeafPaths() + + currentBlock := sds.BlockChain.GetBlockByHash(blockHash) + parentRoot := common.Hash{} + if currentBlock.NumberU64() != 0 { + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + parentRoot = parentBlock.Root() + } + return sds.writeStateDiffWithRetry(currentBlock, parentRoot, params) +} + +// Writes a state diff from the current block, parent state root, and provided params +func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params Params) error { + // log.Info("Writing state diff", "block height", block.Number().Uint64()) + var totalDifficulty *big.Int + var receipts types.Receipts + var err error + var tx interfaces.Batch + if params.IncludeTD { + totalDifficulty = sds.BlockChain.GetTd(block.Hash(), block.NumberU64()) + } + if params.IncludeReceipts { + receipts = sds.BlockChain.GetReceiptsByHash(block.Hash()) + } + tx, err = sds.indexer.PushBlock(block, receipts, totalDifficulty) + if err != nil { + return err + } + // defer handling of commit/rollback for any return case + defer func() { + if err := tx.Submit(err); err != nil { + log.Error("batch transaction submission failed", "err", err) + } + }() + output := func(node types2.StateNode) error { + return sds.indexer.PushStateNode(tx, node, block.Hash().String()) + } + codeOutput := func(c types2.CodeAndCodeHash) error { + return sds.indexer.PushCodeAndCodeHash(tx, c) + } + err = sds.Builder.WriteStateDiffObject(types2.StateRoots{ + NewStateRoot: block.Root(), + OldStateRoot: parentRoot, + }, params, output, codeOutput) + + // allow dereferencing of parent, keep current locked as it should be the next parent + sds.BlockChain.UnlockTrie(parentRoot) + if err != nil { + return err + } + return nil +} + +// Wrapper function on writeStateDiff to retry when the deadlock is detected. +func (sds *Service) writeStateDiffWithRetry(block *types.Block, parentRoot common.Hash, params Params) error { + var err error + for i := uint(0); i < sds.maxRetry; i++ { + err = sds.writeStateDiff(block, parentRoot, params) + if err != nil && strings.Contains(err.Error(), deadlockDetected) { + // Retry only when the deadlock is detected. + if i != sds.maxRetry { + log.Info("dead lock detected while writing statediff", "err", err, "retry number", i) + } + continue + } + break + } + return err +} + +// Performs one of following operations on the watched addresses in writeLoopParams and the db: +// add | remove | set | clear +func (sds *Service) WatchAddress(operation types2.OperationType, args []types2.WatchAddressArg) error { + // lock writeLoopParams for a write + writeLoopParams.Lock() + defer writeLoopParams.Unlock() + + // get the current block number + currentBlockNumber := sds.BlockChain.CurrentBlock().Number() + + switch operation { + case types2.Add: + // filter out args having an already watched address with a warning + filteredArgs, ok := funk.Filter(args, func(arg types2.WatchAddressArg) bool { + if funk.Contains(writeLoopParams.WatchedAddresses, common.HexToAddress(arg.Address)) { + log.Warn("Address already being watched", "address", arg.Address) + return false + } + return true + }).([]types2.WatchAddressArg) + if !ok { + return fmt.Errorf("add: filtered args %s", typeAssertionFailed) + } + + // get addresses from the filtered args + filteredAddresses, err := MapWatchAddressArgsToAddresses(filteredArgs) + if err != nil { + return fmt.Errorf("add: filtered addresses %s", err.Error()) + } + + // update the db + if sds.indexer != nil { + err = sds.indexer.InsertWatchedAddresses(filteredArgs, currentBlockNumber) + if err != nil { + return err + } + } + + // update in-memory params + writeLoopParams.WatchedAddresses = append(writeLoopParams.WatchedAddresses, filteredAddresses...) + writeLoopParams.ComputeWatchedAddressesLeafPaths() + case types2.Remove: + // get addresses from args + argAddresses, err := MapWatchAddressArgsToAddresses(args) + if err != nil { + return fmt.Errorf("remove: mapped addresses %s", err.Error()) + } + + // remove the provided addresses from currently watched addresses + addresses, ok := funk.Subtract(writeLoopParams.WatchedAddresses, argAddresses).([]common.Address) + if !ok { + return fmt.Errorf("remove: filtered addresses %s", typeAssertionFailed) + } + + // update the db + if sds.indexer != nil { + err = sds.indexer.RemoveWatchedAddresses(args) + if err != nil { + return err + } + } + + // update in-memory params + writeLoopParams.WatchedAddresses = addresses + writeLoopParams.ComputeWatchedAddressesLeafPaths() + case types2.Set: + // get addresses from args + argAddresses, err := MapWatchAddressArgsToAddresses(args) + if err != nil { + return fmt.Errorf("set: mapped addresses %s", err.Error()) + } + + // update the db + if sds.indexer != nil { + err = sds.indexer.SetWatchedAddresses(args, currentBlockNumber) + if err != nil { + return err + } + } + + // update in-memory params + writeLoopParams.WatchedAddresses = argAddresses + writeLoopParams.ComputeWatchedAddressesLeafPaths() + case types2.Clear: + // update the db + if sds.indexer != nil { + err := sds.indexer.ClearWatchedAddresses() + if err != nil { + return err + } + } + + // update in-memory params + writeLoopParams.WatchedAddresses = []common.Address{} + writeLoopParams.ComputeWatchedAddressesLeafPaths() + + default: + return fmt.Errorf("%s %s", unexpectedOperation, operation) + } + + return nil +} + +// loadWatchedAddresses loads watched addresses to in-memory write loop params +func loadWatchedAddresses(indexer interfaces.StateDiffIndexer) error { + watchedAddresses, err := indexer.LoadWatchedAddresses() + if err != nil { + return err + } + + writeLoopParams.Lock() + defer writeLoopParams.Unlock() + + writeLoopParams.WatchedAddresses = watchedAddresses + writeLoopParams.ComputeWatchedAddressesLeafPaths() + + return nil +} + +// MapWatchAddressArgsToAddresses maps []WatchAddressArg to corresponding []common.Address +func MapWatchAddressArgsToAddresses(args []types2.WatchAddressArg) ([]common.Address, error) { + addresses, ok := funk.Map(args, func(arg types2.WatchAddressArg) common.Address { + return common.HexToAddress(arg.Address) + }).([]common.Address) + if !ok { + return nil, fmt.Errorf(typeAssertionFailed) + } + + return addresses, nil +} diff --git a/statediff/service_test.go b/statediff/service_test.go new file mode 100644 index 000000000000..1df06860810d --- /dev/null +++ b/statediff/service_test.go @@ -0,0 +1,439 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "math/big" + "math/rand" + "reflect" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + statediff "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/test_helpers/mocks" + types2 "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" +) + +func TestServiceLoop(t *testing.T) { + testErrorInChainEventLoop(t) + testErrorInBlockLoop(t) +} + +var ( + eventsChannel = make(chan core.ChainEvent, 1) + + parentRoot1 = common.HexToHash("0x01") + parentRoot2 = common.HexToHash("0x02") + parentHeader1 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot1} + parentHeader2 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot2} + + parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil, new(trie.Trie)) + parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil, new(trie.Trie)) + + parentHash1 = parentBlock1.Hash() + parentHash2 = parentBlock2.Hash() + + testRoot1 = common.HexToHash("0x03") + testRoot2 = common.HexToHash("0x04") + testRoot3 = common.HexToHash("0x04") + header1 = types.Header{ParentHash: parentHash1, Root: testRoot1, Number: big.NewInt(1)} + header2 = types.Header{ParentHash: parentHash2, Root: testRoot2, Number: big.NewInt(2)} + header3 = types.Header{ParentHash: common.HexToHash("parent hash"), Root: testRoot3, Number: big.NewInt(3)} + + testBlock1 = types.NewBlock(&header1, nil, nil, nil, new(trie.Trie)) + testBlock2 = types.NewBlock(&header2, nil, nil, nil, new(trie.Trie)) + testBlock3 = types.NewBlock(&header3, nil, nil, nil, new(trie.Trie)) + + receiptRoot1 = common.HexToHash("0x05") + receiptRoot2 = common.HexToHash("0x06") + receiptRoot3 = common.HexToHash("0x07") + testReceipts1 = []*types.Receipt{types.NewReceipt(receiptRoot1.Bytes(), false, 1000), types.NewReceipt(receiptRoot2.Bytes(), false, 2000)} + testReceipts2 = []*types.Receipt{types.NewReceipt(receiptRoot3.Bytes(), false, 3000)} + + event1 = core.ChainEvent{Block: testBlock1} + event2 = core.ChainEvent{Block: testBlock2} + event3 = core.ChainEvent{Block: testBlock3} + + defaultParams = statediff.Params{ + IncludeBlock: true, + IncludeReceipts: true, + IncludeTD: true, + WatchedAddresses: []common.Address{}, + } +) + +func testErrorInChainEventLoop(t *testing.T) { + //the first chain event causes and error (in blockchain mock) + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + serviceQuit := make(chan bool) + service := statediff.Service{ + Mutex: sync.Mutex{}, + Builder: &builder, + BlockChain: &blockChain, + QuitChan: serviceQuit, + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + BlockCache: statediff.NewBlockCache(1), + } + payloadChan := make(chan statediff.Payload, 2) + quitChan := make(chan bool) + service.Subscribe(rpc.NewID(), payloadChan, quitChan, defaultParams) + testRoot2 = common.HexToHash("0xTestRoot2") + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockMapping[parentBlock2.Hash()] = parentBlock2 + blockChain.SetBlocksForHashes(blockMapping) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) + blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1) + blockChain.SetReceiptsForHash(testBlock2.Hash(), testReceipts2) + + payloads := make([]statediff.Payload, 0, 2) + wg := new(sync.WaitGroup) + wg.Add(1) + go func() { + for i := 0; i < 2; i++ { + select { + case payload := <-payloadChan: + payloads = append(payloads, payload) + case <-quitChan: + } + } + wg.Done() + }() + service.Loop(eventsChannel) + wg.Wait() + if len(payloads) != 2 { + t.Error("Test failure:", t.Name()) + t.Logf("Actual number of payloads does not equal expected.\nactual: %+v\nexpected: 3", len(payloads)) + } + + testReceipts1Rlp, err := rlp.EncodeToBytes(&testReceipts1) + if err != nil { + t.Error(err) + } + testReceipts2Rlp, err := rlp.EncodeToBytes(&testReceipts2) + if err != nil { + t.Error(err) + } + expectedReceiptsRlp := [][]byte{testReceipts1Rlp, testReceipts2Rlp, nil} + for i, payload := range payloads { + if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual receipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i]) + } + } + + defaultParams.ComputeWatchedAddressesLeafPaths() + if !reflect.DeepEqual(builder.Params, defaultParams) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) + } + if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock2.Hash().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual blockhash does not equal expected.\nactual:%x\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock2.Hash().Bytes()) + } + if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual root does not equal expected.\nactual:%x\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) + } + if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual root does not equal expected.\nactual:%x\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) + } + //look up the parent block from its hash + expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()} + if !reflect.DeepEqual(blockChain.HashesLookedUp, expectedHashes) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual looked up parent hashes does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.HashesLookedUp, expectedHashes) + } +} + +func testErrorInBlockLoop(t *testing.T) { + //second block's parent block can't be found + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + service := statediff.Service{ + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + BlockCache: statediff.NewBlockCache(1), + } + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + service.Subscribe(rpc.NewID(), payloadChan, quitChan, defaultParams) + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockChain.SetBlocksForHashes(blockMapping) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) + // Need to have listeners on the channels or the subscription will be closed and the processing halted + go func() { + select { + case <-payloadChan: + case <-quitChan: + } + }() + service.Loop(eventsChannel) + + defaultParams.ComputeWatchedAddressesLeafPaths() + if !reflect.DeepEqual(builder.Params, defaultParams) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) + } + if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) + } + if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual old state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) + } + if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual new state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) + } +} + +func TestGetStateDiffAt(t *testing.T) { + testErrorInStateDiffAt(t) +} + +func testErrorInStateDiffAt(t *testing.T) { + mockStateDiff := types2.StateObject{ + BlockNumber: testBlock1.Number(), + BlockHash: testBlock1.Hash(), + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(&mockStateDiff) + if err != nil { + t.Error(err) + } + expectedReceiptsRlp, err := rlp.EncodeToBytes(&testReceipts1) + if err != nil { + t.Error(err) + } + expectedBlockRlp, err := rlp.EncodeToBytes(testBlock1) + if err != nil { + t.Error(err) + } + expectedStateDiffPayload := statediff.Payload{ + StateObjectRlp: expectedStateDiffRlp, + ReceiptsRlp: expectedReceiptsRlp, + BlockRlp: expectedBlockRlp, + } + expectedStateDiffPayloadRlp, err := rlp.EncodeToBytes(&expectedStateDiffPayload) + if err != nil { + t.Error(err) + } + builder := mocks.Builder{} + builder.SetStateDiffToBuild(mockStateDiff) + blockChain := mocks.BlockChain{} + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockChain.SetBlocksForHashes(blockMapping) + blockChain.SetBlockForNumber(testBlock1, testBlock1.NumberU64()) + blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1) + service := statediff.Service{ + Mutex: sync.Mutex{}, + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + BlockCache: statediff.NewBlockCache(1), + } + stateDiffPayload, err := service.StateDiffAt(testBlock1.NumberU64(), defaultParams) + if err != nil { + t.Error(err) + } + stateDiffPayloadRlp, err := rlp.EncodeToBytes(stateDiffPayload) + if err != nil { + t.Error(err) + } + + defaultParams.ComputeWatchedAddressesLeafPaths() + if !reflect.DeepEqual(builder.Params, defaultParams) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual params does not equal expected.\nactual:%+v\nexpected: %+v", builder.Params, defaultParams) + } + if !bytes.Equal(builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.BlockHash.Bytes(), testBlock1.Hash().Bytes()) + } + if !bytes.Equal(builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual old state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) + } + if !bytes.Equal(builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual new state root does not equal expected.\nactual:%+v\nexpected: %x", builder.Args.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) + } + if !bytes.Equal(expectedStateDiffPayloadRlp, stateDiffPayloadRlp) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual state diff payload does not equal expected.\nactual:%+v\nexpected: %+v", expectedStateDiffPayload, stateDiffPayload) + } +} + +func TestWaitForSync(t *testing.T) { + testWaitForSync(t) + testGetSyncStatus(t) +} + +// This function will create a backend and service object which includes a generic Backend +func createServiceWithMockBackend(curBlock uint64, highestBlock uint64) (*mocks.Backend, *statediff.Service) { + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + backend := mocks.Backend{ + StartingBlock: 1, + CurrBlock: curBlock, + HighestBlock: highestBlock, + SyncedAccounts: 5, + SyncedAccountBytes: 5, + SyncedBytecodes: 5, + SyncedBytecodeBytes: 5, + SyncedStorage: 5, + SyncedStorageBytes: 5, + HealedTrienodes: 5, + HealedTrienodeBytes: 5, + HealedBytecodes: 5, + HealedBytecodeBytes: 5, + HealingTrienodes: 5, + HealingBytecode: 5, + } + + service := &statediff.Service{ + Mutex: sync.Mutex{}, + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + BlockCache: statediff.NewBlockCache(1), + BackendAPI: &backend, + WaitForSync: true, + } + return &backend, service +} + +// This function will test to make sure that the state diff waits +// until the blockchain has caught up to head! +func testWaitForSync(t *testing.T) { + t.Log("Starting Sync") + _, service := createServiceWithMockBackend(10, 10) + err := service.WaitingForSync() + if err != nil { + t.Fatal("Sync Failed") + } + t.Log("Sync Complete") +} + +// This test will run the WaitForSync() at the start of the execusion +// It will then incrementally increase the currentBlock to match the highestBlock +// At each interval it will run the GetSyncStatus to ensure that the return value is not false. +// It will also check to make sure that the WaitForSync() function has not completed! +func testGetSyncStatus(t *testing.T) { + t.Log("Starting Get Sync Status Test") + var highestBlock uint64 = 5 + // Create a backend and a service + // the backend is lagging behind the sync. + backend, service := createServiceWithMockBackend(0, highestBlock) + + checkSyncComplete := make(chan int, 1) + + go func() { + // Start the sync function which will wait for the sync + // Once the sync is complete add a value to the checkSyncComplet channel + t.Log("Starting Sync") + err := service.WaitingForSync() + if err != nil { + t.Error("Sync Failed") + checkSyncComplete <- 1 + } + t.Log("We have finally synced!") + checkSyncComplete <- 0 + }() + + tables := []struct { + currentBlock uint64 + highestBlock uint64 + }{ + {1, highestBlock}, + {2, highestBlock}, + {3, highestBlock}, + {4, highestBlock}, + {5, highestBlock}, + } + + time.Sleep(2 * time.Second) + for _, table := range tables { + // Iterate over each block + // Once the highest block reaches the current block the sync should complete + + // Update the backend current block value + t.Log("Updating Current Block to: ", table.currentBlock) + backend.CurrBlock = table.currentBlock + pubEthAPI := ethapi.NewEthereumAPI(service.BackendAPI) + syncStatus, err := service.GetSyncStatus(pubEthAPI) + + if err != nil { + t.Fatal("Sync Failed") + } + + time.Sleep(2 * time.Second) + + // Make sure if syncStatus is false that WaitForSync has completed! + if !syncStatus && len(checkSyncComplete) == 0 { + t.Error("Sync is complete but WaitForSync is not") + } + + if syncStatus && len(checkSyncComplete) == 1 { + t.Error("Sync is not complete but WaitForSync is") + } + + // Make sure sync hasn't completed and that the checkSyncComplete channel is empty + if syncStatus && len(checkSyncComplete) == 0 { + continue + } + + // This code will only be run if the sync is complete and the WaitForSync function is complete + + // If syncstatus is complete, make sure that the blocks match + if !syncStatus && table.currentBlock != table.highestBlock { + t.Errorf("syncStatus indicated sync was complete even when current block, %d, and highest block %d aren't equal", + table.currentBlock, table.highestBlock) + } + + // Make sure that WaitForSync completed once the current block caught up to head! + checkSyncCompleteVal := <-checkSyncComplete + if checkSyncCompleteVal != 0 { + t.Errorf("syncStatus indicated sync was complete but the checkSyncComplete has a value of %d", + checkSyncCompleteVal) + } else { + t.Log("Test Passed!") + } + } +} diff --git a/statediff/test_helpers/constant.go b/statediff/test_helpers/constant.go new file mode 100644 index 000000000000..ba591ebb4999 --- /dev/null +++ b/statediff/test_helpers/constant.go @@ -0,0 +1,33 @@ +// VulcanizeDB +// Copyright © 2021 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test_helpers + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/params" +) + +var ( + BalanceChange1000 = int64(1000) + BalanceChange10000 = int64(10000) + BalanceChange1Ether = int64(params.Ether) + Block1Account1Balance = big.NewInt(BalanceChange10000) + Block2Account2Balance = big.NewInt(21000000000000) + GasFees = int64(params.GWei) * int64(params.TxGas) + ContractGasLimit = uint64(1000000) +) diff --git a/statediff/test_helpers/helpers.go b/statediff/test_helpers/helpers.go new file mode 100644 index 000000000000..b053d9a1fcc9 --- /dev/null +++ b/statediff/test_helpers/helpers.go @@ -0,0 +1,137 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package test_helpers + +import ( + "github.com/ethereum/go-ethereum/ethdb" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { + g := core.Genesis{ + Alloc: core.GenesisAlloc{addr: {Balance: balance}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + return g.MustCommit(db) +} + +// MakeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. +func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) { + config := params.TestChainConfig + blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen) + chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + return blocks, chain +} + +func TestSelfDestructChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // Block 1 is mined by Account1Addr + // Account1Addr creates a new contract + block.SetCoinbase(TestBankAddress) + tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(params.GWei), ContractCode), signer, TestBankKey) + ContractAddr = crypto.CreateAddress(TestBankAddress, 0) + block.AddTx(tx) + case 1: + // Block 2 is mined by Account1Addr + // Account1Addr self-destructs the contract + block.SetCoinbase(TestBankAddress) + data := common.Hex2Bytes("43D726D6") + tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.GWei), data), signer, TestBankKey) + block.AddTx(tx) + } +} + +func TestChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(BalanceChange10000), params.TxGas, big.NewInt(params.GWei), nil), signer, TestBankKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // Account1Addr passes it on to account #2. + // Account1Addr creates a test contract. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(BalanceChange1Ether), params.TxGas, big.NewInt(params.GWei), nil), signer, TestBankKey) + nonce := block.TxNonce(Account1Addr) + tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(BalanceChange1000), params.TxGas, big.NewInt(params.GWei), nil), signer, Account1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), ContractGasLimit, big.NewInt(params.GWei), ContractCode), signer, Account1Key) + ContractAddr = crypto.CreateAddress(Account1Addr, nonce) + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 2: + // Block 3 has a single tx from the bankAccount to the contract, that transfers no value + // Block 3 is mined by Account2Addr + block.SetCoinbase(Account2Addr) + //put function: c16431b9 + //close function: 43d726d6 + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), params.TxGasContractCreation, big.NewInt(params.GWei), data), signer, TestBankKey) + block.AddTx(tx) + case 3: + // Block 4 has three txs from bankAccount to the contract, that transfer no value + // Two set the two original slot positions to 0 and one sets another position to a new value + // Block 4 is mined by Account2Addr + block.SetCoinbase(Account2Addr) + data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000") + data3 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000009") + + nonce := block.TxNonce(TestBankAddress) + tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data1), signer, TestBankKey) + nonce++ + tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data2), signer, TestBankKey) + nonce++ + tx3, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data3), signer, TestBankKey) + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 4: + // Block 5 has one tx from bankAccount to the contract, that transfers no value + // It sets the one storage value to zero and the other to new value. + // Block 5 is mined by Account1Addr + block.SetCoinbase(Account1Addr) + data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000") + data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + nonce := block.TxNonce(TestBankAddress) + tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data1), signer, TestBankKey) + nonce++ + tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data2), signer, TestBankKey) + block.AddTx(tx1) + block.AddTx(tx2) + case 5: + // Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value + // Block 6 is mined by Account2Addr + block.SetCoinbase(Account2Addr) + data := common.Hex2Bytes("43D726D6") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(Account1Addr), ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data), signer, Account1Key) + block.AddTx(tx) + } +} diff --git a/statediff/test_helpers/mocks/backend.go b/statediff/test_helpers/mocks/backend.go new file mode 100644 index 000000000000..d3f6dc302ed0 --- /dev/null +++ b/statediff/test_helpers/mocks/backend.go @@ -0,0 +1,257 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "context" + "math/big" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +// Builder is a mock state diff builder +type Backend struct { + StartingBlock uint64 + CurrBlock uint64 + HighestBlock uint64 + SyncedAccounts uint64 + SyncedAccountBytes uint64 + SyncedBytecodes uint64 + SyncedBytecodeBytes uint64 + SyncedStorage uint64 + SyncedStorageBytes uint64 + HealedTrienodes uint64 + HealedTrienodeBytes uint64 + HealedBytecodes uint64 + HealedBytecodeBytes uint64 + HealingTrienodes uint64 + HealingBytecode uint64 +} + +// General Ethereum API +func (backend *Backend) SyncProgress() ethereum.SyncProgress { + l := ethereum.SyncProgress{ + StartingBlock: backend.StartingBlock, + CurrentBlock: backend.CurrBlock, + HighestBlock: backend.HighestBlock, + SyncedAccounts: backend.SyncedAccounts, + SyncedAccountBytes: backend.SyncedAccountBytes, + SyncedBytecodes: backend.SyncedBytecodes, + SyncedBytecodeBytes: backend.SyncedBytecodeBytes, + SyncedStorage: backend.SyncedStorage, + SyncedStorageBytes: backend.SyncedStorageBytes, + HealedTrienodes: backend.HealedTrienodes, + HealedTrienodeBytes: backend.HealedTrienodeBytes, + HealedBytecodes: backend.HealedBytecodes, + HealedBytecodeBytes: backend.HealedBytecodeBytes, + HealingTrienodes: backend.HealingTrienodes, + HealingBytecode: backend.HealingBytecode, + } + return l +} + +func (backend *Backend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) ChainDb() ethdb.Database { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) AccountManager() *accounts.Manager { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) ExtRPCEnabled() bool { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) RPCGasCap() uint64 { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) RPCEVMTimeout() time.Duration { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) RPCTxFeeCap() float64 { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) UnprotectedAllowed() bool { + panic("not implemented") // TODO: Implement +} + +// Blockchain API +func (backend *Backend) SetHead(number uint64) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) CurrentHeader() *types.Header { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) CurrentBlock() *types.Block { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetTd(ctx context.Context, hash common.Hash) *big.Int { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +// Transaction pool API +func (backend *Backend) SendTx(ctx context.Context, signedTx *types.Transaction) error { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetPoolTransactions() (types.Transactions, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetPoolTransaction(txHash common.Hash) *types.Transaction { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) Stats() (pending int, queued int) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeNewTxsEvent(_ chan<- core.NewTxsEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +// Filter API +func (backend *Backend) BloomStatus() (uint64, uint64) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + panic("not implemented") +} + +func (backend *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) ChainConfig() *params.ChainConfig { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) Engine() consensus.Engine { + panic("not implemented") // TODO: Implement +} + +func (backend *Backend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return nil, nil +} diff --git a/statediff/test_helpers/mocks/blockchain.go b/statediff/test_helpers/mocks/blockchain.go new file mode 100644 index 000000000000..0c6ff94246a5 --- /dev/null +++ b/statediff/test_helpers/mocks/blockchain.go @@ -0,0 +1,157 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "errors" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/core/state" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" +) + +// BlockChain is a mock blockchain for testing +type BlockChain struct { + HashesLookedUp []common.Hash + blocksToReturnByHash map[common.Hash]*types.Block + blocksToReturnByNumber map[uint64]*types.Block + ChainEvents []core.ChainEvent + Receipts map[common.Hash]types.Receipts + TDByHash map[common.Hash]*big.Int + TDByNum map[uint64]*big.Int + currentBlock *types.Block +} + +// SetBlocksForHashes mock method +func (bc *BlockChain) SetBlocksForHashes(blocks map[common.Hash]*types.Block) { + if bc.blocksToReturnByHash == nil { + bc.blocksToReturnByHash = make(map[common.Hash]*types.Block) + } + bc.blocksToReturnByHash = blocks +} + +// GetBlockByHash mock method +func (bc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { + bc.HashesLookedUp = append(bc.HashesLookedUp, hash) + + var block *types.Block + if len(bc.blocksToReturnByHash) > 0 { + block = bc.blocksToReturnByHash[hash] + } + + return block +} + +// SetChainEvents mock method +func (bc *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) { + bc.ChainEvents = chainEvents +} + +// SubscribeChainEvent mock method +func (bc *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { + subErr := errors.New("subscription error") + + var eventCounter int + subscription := event.NewSubscription(func(quit <-chan struct{}) error { + for _, chainEvent := range bc.ChainEvents { + if eventCounter > 1 { + time.Sleep(250 * time.Millisecond) + return subErr + } + select { + case ch <- chainEvent: + case <-quit: + return nil + } + eventCounter++ + } + return nil + }) + + return subscription +} + +// SetReceiptsForHash test method +func (bc *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) { + if bc.Receipts == nil { + bc.Receipts = make(map[common.Hash]types.Receipts) + } + bc.Receipts[hash] = receipts +} + +// GetReceiptsByHash mock method +func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { + return bc.Receipts[hash] +} + +// SetBlockForNumber test method +func (bc *BlockChain) SetBlockForNumber(block *types.Block, number uint64) { + if bc.blocksToReturnByNumber == nil { + bc.blocksToReturnByNumber = make(map[uint64]*types.Block) + } + bc.blocksToReturnByNumber[number] = block +} + +// GetBlockByNumber mock method +func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { + return bc.blocksToReturnByNumber[number] +} + +// GetTd mock method +func (bc *BlockChain) GetTd(hash common.Hash, blockNum uint64) *big.Int { + if td, ok := bc.TDByHash[hash]; ok { + return td + } + + if td, ok := bc.TDByNum[blockNum]; ok { + return td + } + return nil +} + +// SetCurrentBlock test method +func (bc *BlockChain) SetCurrentBlock(block *types.Block) { + bc.currentBlock = block +} + +// CurrentBlock mock method +func (bc *BlockChain) CurrentBlock() *types.Block { + return bc.currentBlock +} + +func (bc *BlockChain) SetTd(hash common.Hash, blockNum uint64, td *big.Int) { + if bc.TDByHash == nil { + bc.TDByHash = make(map[common.Hash]*big.Int) + } + bc.TDByHash[hash] = td + + if bc.TDByNum == nil { + bc.TDByNum = make(map[uint64]*big.Int) + } + bc.TDByNum[blockNum] = td +} + +func (bc *BlockChain) UnlockTrie(root common.Hash) {} + +func (bc *BlockChain) StateCache() state.Database { + return nil +} diff --git a/statediff/test_helpers/mocks/builder.go b/statediff/test_helpers/mocks/builder.go new file mode 100644 index 000000000000..e2452301a606 --- /dev/null +++ b/statediff/test_helpers/mocks/builder.go @@ -0,0 +1,67 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +// Builder is a mock state diff builder +type Builder struct { + Args statediff.Args + Params statediff.Params + StateRoots sdtypes.StateRoots + stateDiff sdtypes.StateObject + block *types.Block + stateTrie sdtypes.StateObject + builderError error +} + +// BuildStateDiffObject mock method +func (builder *Builder) BuildStateDiffObject(args statediff.Args, params statediff.Params) (sdtypes.StateObject, error) { + builder.Args = args + builder.Params = params + + return builder.stateDiff, builder.builderError +} + +// BuildStateDiffObject mock method +func (builder *Builder) WriteStateDiffObject(args sdtypes.StateRoots, params statediff.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error { + builder.StateRoots = args + builder.Params = params + + return builder.builderError +} + +// BuildStateTrieObject mock method +func (builder *Builder) BuildStateTrieObject(block *types.Block) (sdtypes.StateObject, error) { + builder.block = block + + return builder.stateTrie, builder.builderError +} + +// SetStateDiffToBuild mock method +func (builder *Builder) SetStateDiffToBuild(stateDiff sdtypes.StateObject) { + builder.stateDiff = stateDiff +} + +// SetBuilderError mock method +func (builder *Builder) SetBuilderError(err error) { + builder.builderError = err +} diff --git a/statediff/test_helpers/mocks/indexer.go b/statediff/test_helpers/mocks/indexer.go new file mode 100644 index 000000000000..92005a8b4c4d --- /dev/null +++ b/statediff/test_helpers/mocks/indexer.go @@ -0,0 +1,70 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var _ interfaces.StateDiffIndexer = &StateDiffIndexer{} + +// StateDiffIndexer is a mock state diff indexer +type StateDiffIndexer struct{} + +func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) { + return nil, nil +} + +func (sdi *StateDiffIndexer) PushStateNode(tx interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { + return nil +} + +func (sdi *StateDiffIndexer) PushCodeAndCodeHash(tx interfaces.Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error { + return nil +} + +func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) {} + +func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) { + return nil, nil +} + +func (sdi *StateDiffIndexer) InsertWatchedAddresses(addresses []sdtypes.WatchAddressArg, currentBlock *big.Int) error { + return nil +} + +func (sdi *StateDiffIndexer) RemoveWatchedAddresses(addresses []sdtypes.WatchAddressArg) error { + return nil +} + +func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error { + return nil +} + +func (sdi *StateDiffIndexer) ClearWatchedAddresses() error { + return nil +} + +func (sdi *StateDiffIndexer) Close() error { + return nil +} diff --git a/statediff/test_helpers/mocks/service.go b/statediff/test_helpers/mocks/service.go new file mode 100644 index 000000000000..ab183fddd0f4 --- /dev/null +++ b/statediff/test_helpers/mocks/service.go @@ -0,0 +1,438 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "bytes" + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/thoas/go-funk" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var ( + typeAssertionFailed = "type assertion failed" + unexpectedOperation = "unexpected operation" +) + +// MockStateDiffService is a mock state diff service +type MockStateDiffService struct { + sync.Mutex + Builder statediff.Builder + BlockChain *BlockChain + ReturnProtocol []p2p.Protocol + ReturnAPIs []rpc.API + BlockChan chan *types.Block + ParentBlockChan chan *types.Block + QuitChan chan bool + Subscriptions map[common.Hash]map[rpc.ID]statediff.Subscription + SubscriptionTypes map[common.Hash]statediff.Params + Indexer interfaces.StateDiffIndexer + writeLoopParams statediff.ParamsWithMutex +} + +// Protocols mock method +func (sds *MockStateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +// APIs mock method +func (sds *MockStateDiffService) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: statediff.APIName, + Version: statediff.APIVersion, + Service: statediff.NewPublicStateDiffAPI(sds), + Public: true, + }, + } +} + +// Loop mock method +func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { + //loop through chain events until no more + for { + select { + case block := <-sds.BlockChan: + currentBlock := block + parentBlock := <-sds.ParentBlockChan + parentHash := parentBlock.Hash() + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + continue + } + sds.streamStateDiff(currentBlock, parentBlock.Root()) + case <-sds.QuitChan: + log.Debug("Quitting the statediff block channel") + sds.close() + return + } + } +} + +// streamStateDiff method builds the state diff payload for each subscription according to their subscription type and sends them the result +func (sds *MockStateDiffService) streamStateDiff(currentBlock *types.Block, parentRoot common.Hash) { + sds.Lock() + for ty, subs := range sds.Subscriptions { + params, ok := sds.SubscriptionTypes[ty] + if !ok { + log.Error(fmt.Sprintf("subscriptions type %s do not have a parameter set associated with them", ty.Hex())) + sds.closeType(ty) + continue + } + // create payload for this subscription type + payload, err := sds.processStateDiff(currentBlock, parentRoot, params) + if err != nil { + log.Error(fmt.Sprintf("statediff processing error for subscriptions with parameters: %+v", params)) + sds.closeType(ty) + continue + } + for id, sub := range subs { + select { + case sub.PayloadChan <- *payload: + log.Debug(fmt.Sprintf("sending statediff payload to subscription %s", id)) + default: + log.Info(fmt.Sprintf("unable to send statediff payload to subscription %s; channel has no receiver", id)) + } + } + } + sds.Unlock() +} + +// StateDiffAt mock method +func (sds *MockStateDiffService) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info(fmt.Sprintf("sending state diff at %d", blockNumber)) + if blockNumber == 0 { + return sds.processStateDiff(currentBlock, common.Hash{}, params) + } + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + return sds.processStateDiff(currentBlock, parentBlock.Root(), params) +} + +// StateDiffFor mock method +func (sds *MockStateDiffService) StateDiffFor(blockHash common.Hash, params statediff.Params) (*statediff.Payload, error) { + // TODO: something useful here + return nil, nil +} + +// processStateDiff method builds the state diff payload from the current block, parent state root, and provided params +func (sds *MockStateDiffService) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) { + stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{ + NewStateRoot: currentBlock.Root(), + OldStateRoot: parentRoot, + BlockHash: currentBlock.Hash(), + BlockNumber: currentBlock.Number(), + }, params) + if err != nil { + return nil, err + } + stateDiffRlp, err := rlp.EncodeToBytes(&stateDiff) + if err != nil { + return nil, err + } + return sds.newPayload(stateDiffRlp, currentBlock, params) +} + +func (sds *MockStateDiffService) newPayload(stateObject []byte, block *types.Block, params statediff.Params) (*statediff.Payload, error) { + payload := &statediff.Payload{ + StateObjectRlp: stateObject, + } + if params.IncludeBlock { + blockBuff := new(bytes.Buffer) + if err := block.EncodeRLP(blockBuff); err != nil { + return nil, err + } + payload.BlockRlp = blockBuff.Bytes() + } + if params.IncludeTD { + payload.TotalDifficulty = sds.BlockChain.GetTd(block.Hash(), block.NumberU64()) + } + if params.IncludeReceipts { + receiptBuff := new(bytes.Buffer) + receipts := sds.BlockChain.GetReceiptsByHash(block.Hash()) + if err := rlp.Encode(receiptBuff, receipts); err != nil { + return nil, err + } + payload.ReceiptsRlp = receiptBuff.Bytes() + } + return payload, nil +} + +// WriteStateDiffAt mock method +func (sds *MockStateDiffService) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error { + // TODO: something useful here + return nil +} + +// WriteStateDiffFor mock method +func (sds *MockStateDiffService) WriteStateDiffFor(blockHash common.Hash, params statediff.Params) error { + // TODO: something useful here + return nil +} + +// Loop mock method +func (sds *MockStateDiffService) WriteLoop(chan core.ChainEvent) { + //loop through chain events until no more + for { + select { + case block := <-sds.BlockChan: + currentBlock := block + parentBlock := <-sds.ParentBlockChan + parentHash := parentBlock.Hash() + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + continue + } + // TODO: + // sds.writeStateDiff(currentBlock, parentBlock.Root(), statediff.Params{}) + case <-sds.QuitChan: + log.Debug("Quitting the statediff block channel") + sds.close() + return + } + } +} + +// StateTrieAt mock method +func (sds *MockStateDiffService) StateTrieAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info(fmt.Sprintf("sending state trie at %d", blockNumber)) + return sds.stateTrieAt(currentBlock, params) +} + +func (sds *MockStateDiffService) stateTrieAt(block *types.Block, params statediff.Params) (*statediff.Payload, error) { + stateNodes, err := sds.Builder.BuildStateTrieObject(block) + if err != nil { + return nil, err + } + stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes) + if err != nil { + return nil, err + } + return sds.newPayload(stateTrieRlp, block, params) +} + +// Subscribe is used by the API to subscribe to the service loop +func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool, params statediff.Params) { + // Subscription type is defined as the hash of the rlp-serialized subscription params + by, err := rlp.EncodeToBytes(¶ms) + if err != nil { + return + } + subscriptionType := crypto.Keccak256Hash(by) + // Add subscriber + sds.Lock() + if sds.Subscriptions[subscriptionType] == nil { + sds.Subscriptions[subscriptionType] = make(map[rpc.ID]statediff.Subscription) + } + sds.Subscriptions[subscriptionType][id] = statediff.Subscription{ + PayloadChan: sub, + QuitChan: quitChan, + } + sds.SubscriptionTypes[subscriptionType] = params + sds.Unlock() +} + +// Unsubscribe is used to unsubscribe from the service loop +func (sds *MockStateDiffService) Unsubscribe(id rpc.ID) error { + sds.Lock() + for ty := range sds.Subscriptions { + delete(sds.Subscriptions[ty], id) + if len(sds.Subscriptions[ty]) == 0 { + // If we removed the last subscription of this type, remove the subscription type outright + delete(sds.Subscriptions, ty) + delete(sds.SubscriptionTypes, ty) + } + } + sds.Unlock() + return nil +} + +// close is used to close all listening subscriptions +func (sds *MockStateDiffService) close() { + sds.Lock() + for ty, subs := range sds.Subscriptions { + for id, sub := range subs { + select { + case sub.QuitChan <- true: + log.Info(fmt.Sprintf("closing subscription %s", id)) + default: + log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id)) + } + delete(sds.Subscriptions[ty], id) + } + delete(sds.Subscriptions, ty) + delete(sds.SubscriptionTypes, ty) + } + sds.Unlock() +} + +// Start mock method +func (sds *MockStateDiffService) Start() error { + log.Info("Starting mock statediff service") + if sds.ParentBlockChan == nil || sds.BlockChan == nil { + return errors.New("MockStateDiffingService needs to be configured with a MockParentBlockChan and MockBlockChan") + } + chainEventCh := make(chan core.ChainEvent, 10) + go sds.Loop(chainEventCh) + + return nil +} + +// Stop mock method +func (sds *MockStateDiffService) Stop() error { + log.Info("Stopping mock statediff service") + close(sds.QuitChan) + return nil +} + +// closeType is used to close all subscriptions of given type +// closeType needs to be called with subscription access locked +func (sds *MockStateDiffService) closeType(subType common.Hash) { + subs := sds.Subscriptions[subType] + for id, sub := range subs { + sendNonBlockingQuit(id, sub) + } + delete(sds.Subscriptions, subType) + delete(sds.SubscriptionTypes, subType) +} + +func (sds *MockStateDiffService) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- sdtypes.CodeAndCodeHash, quitChan chan<- bool) { + panic("implement me") +} + +func sendNonBlockingQuit(id rpc.ID, sub statediff.Subscription) { + select { + case sub.QuitChan <- true: + log.Info(fmt.Sprintf("closing subscription %s", id)) + default: + log.Info("unable to close subscription %s; channel has no receiver", id) + } +} + +// Performs one of following operations on the watched addresses in writeLoopParams and the db: +// add | remove | set | clear +func (sds *MockStateDiffService) WatchAddress(operation sdtypes.OperationType, args []sdtypes.WatchAddressArg) error { + // lock writeLoopParams for a write + sds.writeLoopParams.Lock() + defer sds.writeLoopParams.Unlock() + + // get the current block number + currentBlockNumber := sds.BlockChain.CurrentBlock().Number() + + switch operation { + case sdtypes.Add: + // filter out args having an already watched address with a warning + filteredArgs, ok := funk.Filter(args, func(arg sdtypes.WatchAddressArg) bool { + if funk.Contains(sds.writeLoopParams.WatchedAddresses, common.HexToAddress(arg.Address)) { + log.Warn("Address already being watched", "address", arg.Address) + return false + } + return true + }).([]sdtypes.WatchAddressArg) + if !ok { + return fmt.Errorf("add: filtered args %s", typeAssertionFailed) + } + + // get addresses from the filtered args + filteredAddresses, err := statediff.MapWatchAddressArgsToAddresses(filteredArgs) + if err != nil { + return fmt.Errorf("add: filtered addresses %s", err.Error()) + } + + // update the db + err = sds.Indexer.InsertWatchedAddresses(filteredArgs, currentBlockNumber) + if err != nil { + return err + } + + // update in-memory params + sds.writeLoopParams.WatchedAddresses = append(sds.writeLoopParams.WatchedAddresses, filteredAddresses...) + sds.writeLoopParams.ComputeWatchedAddressesLeafPaths() + case sdtypes.Remove: + // get addresses from args + argAddresses, err := statediff.MapWatchAddressArgsToAddresses(args) + if err != nil { + return fmt.Errorf("remove: mapped addresses %s", err.Error()) + } + + // remove the provided addresses from currently watched addresses + addresses, ok := funk.Subtract(sds.writeLoopParams.WatchedAddresses, argAddresses).([]common.Address) + if !ok { + return fmt.Errorf("remove: filtered addresses %s", typeAssertionFailed) + } + + // update the db + err = sds.Indexer.RemoveWatchedAddresses(args) + if err != nil { + return err + } + + // update in-memory params + sds.writeLoopParams.WatchedAddresses = addresses + sds.writeLoopParams.ComputeWatchedAddressesLeafPaths() + case sdtypes.Set: + // get addresses from args + argAddresses, err := statediff.MapWatchAddressArgsToAddresses(args) + if err != nil { + return fmt.Errorf("set: mapped addresses %s", err.Error()) + } + + // update the db + err = sds.Indexer.SetWatchedAddresses(args, currentBlockNumber) + if err != nil { + return err + } + + // update in-memory params + sds.writeLoopParams.WatchedAddresses = argAddresses + sds.writeLoopParams.ComputeWatchedAddressesLeafPaths() + case sdtypes.Clear: + // update the db + err := sds.Indexer.ClearWatchedAddresses() + if err != nil { + return err + } + + // update in-memory params + sds.writeLoopParams.WatchedAddresses = []common.Address{} + sds.writeLoopParams.ComputeWatchedAddressesLeafPaths() + + default: + return fmt.Errorf("%s %s", unexpectedOperation, operation) + } + + return nil +} diff --git a/statediff/test_helpers/mocks/service_test.go b/statediff/test_helpers/mocks/service_test.go new file mode 100644 index 000000000000..7761374334fc --- /dev/null +++ b/statediff/test_helpers/mocks/service_test.go @@ -0,0 +1,540 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "bytes" + "fmt" + "math/big" + "os" + "reflect" + "sort" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/test_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +var ( + emptyStorage = make([]sdtypes.StorageNode, 0) + block0, block1 *types.Block + minerLeafKey = test_helpers.AddressToLeafKey(common.HexToAddress("0x0")) + account1, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: uint64(0), + Balance: big.NewInt(10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), + }) + account1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"), + account1, + }) + minerAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: uint64(0), + Balance: big.NewInt(2000002625000000000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), + }) + minerAccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"), + minerAccount, + }) + bankAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ + Nonce: uint64(1), + Balance: big.NewInt(1999978999999990000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), + }) + bankAccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{ + common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), + bankAccount, + }) + mockTotalDifficulty = big.NewInt(1337) + parameters = statediff.Params{ + IntermediateStateNodes: false, + IncludeTD: true, + IncludeBlock: true, + IncludeReceipts: true, + } +) + +func init() { + if os.Getenv("MODE") != "statediff" { + fmt.Println("Skipping statediff test") + os.Exit(0) + } +} + +func TestAPI(t *testing.T) { + testSubscriptionAPI(t) + testHTTPAPI(t) + testWatchAddressAPI(t) +} + +func testSubscriptionAPI(t *testing.T) { + blocks, chain := test_helpers.MakeChain(1, test_helpers.Genesis, test_helpers.TestChainGen) + defer chain.Stop() + block0 = test_helpers.Genesis + block1 = blocks[0] + expectedBlockRlp, _ := rlp.EncodeToBytes(block1) + mockReceipt := &types.Receipt{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + } + expectedReceiptBytes, _ := rlp.EncodeToBytes(&types.Receipts{mockReceipt}) + expectedStateDiff := sdtypes.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountLeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountLeafNode, + StorageNodes: emptyStorage, + }, + }, + } + expectedStateDiffBytes, _ := rlp.EncodeToBytes(&expectedStateDiff) + + blockChan := make(chan *types.Block) + parentBlockChain := make(chan *types.Block) + serviceQuitChan := make(chan bool) + mockBlockChain := &BlockChain{} + mockBlockChain.SetReceiptsForHash(block1.Hash(), types.Receipts{mockReceipt}) + mockBlockChain.SetTd(block1.Hash(), block1.NumberU64(), mockTotalDifficulty) + mockService := MockStateDiffService{ + Mutex: sync.Mutex{}, + Builder: statediff.NewBuilder(chain.StateCache()), + BlockChan: blockChan, + BlockChain: mockBlockChain, + ParentBlockChan: parentBlockChain, + QuitChan: serviceQuitChan, + Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), + SubscriptionTypes: make(map[common.Hash]statediff.Params), + } + + mockService.Start() + id := rpc.NewID() + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + wg := new(sync.WaitGroup) + wg.Add(1) + go func() { + defer wg.Done() + sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] }) + select { + case payload := <-payloadChan: + if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { + t.Errorf("payload does not have expected block\r\nactual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp) + } + sort.Slice(payload.StateObjectRlp, func(i, j int) bool { return payload.StateObjectRlp[i] < payload.StateObjectRlp[j] }) + if !bytes.Equal(payload.StateObjectRlp, expectedStateDiffBytes) { + t.Errorf("payload does not have expected state diff\r\nactual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateObjectRlp, expectedStateDiffBytes) + } + if !bytes.Equal(expectedReceiptBytes, payload.ReceiptsRlp) { + t.Errorf("payload does not have expected receipts\r\nactual receipt rlp: %v\r\nexpected receipt rlp: %v", payload.ReceiptsRlp, expectedReceiptBytes) + } + if !bytes.Equal(payload.TotalDifficulty.Bytes(), mockTotalDifficulty.Bytes()) { + t.Errorf("payload does not have expected total difficulty\r\nactual td: %d\r\nexpected td: %d", payload.TotalDifficulty.Int64(), mockTotalDifficulty.Int64()) + } + case <-quitChan: + t.Errorf("channel quit before delivering payload") + } + }() + time.Sleep(1 * time.Second) + mockService.Subscribe(id, payloadChan, quitChan, parameters) + blockChan <- block1 + parentBlockChain <- block0 + wg.Wait() +} + +func testHTTPAPI(t *testing.T) { + blocks, chain := test_helpers.MakeChain(1, test_helpers.Genesis, test_helpers.TestChainGen) + defer chain.Stop() + block0 = test_helpers.Genesis + block1 = blocks[0] + expectedBlockRlp, _ := rlp.EncodeToBytes(block1) + mockReceipt := &types.Receipt{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + } + expectedReceiptBytes, _ := rlp.EncodeToBytes(&types.Receipts{mockReceipt}) + expectedStateDiff := sdtypes.StateObject{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + Nodes: []sdtypes.StateNode{ + { + Path: []byte{'\x05'}, + NodeType: sdtypes.Leaf, + LeafKey: minerLeafKey, + NodeValue: minerAccountLeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x0e'}, + NodeType: sdtypes.Leaf, + LeafKey: test_helpers.Account1LeafKey, + NodeValue: account1LeafNode, + StorageNodes: emptyStorage, + }, + { + Path: []byte{'\x00'}, + NodeType: sdtypes.Leaf, + LeafKey: test_helpers.BankLeafKey, + NodeValue: bankAccountLeafNode, + StorageNodes: emptyStorage, + }, + }, + } + expectedStateDiffBytes, _ := rlp.EncodeToBytes(&expectedStateDiff) + mockBlockChain := &BlockChain{} + mockBlockChain.SetBlocksForHashes(map[common.Hash]*types.Block{ + block0.Hash(): block0, + block1.Hash(): block1, + }) + mockBlockChain.SetBlockForNumber(block1, block1.Number().Uint64()) + mockBlockChain.SetReceiptsForHash(block1.Hash(), types.Receipts{mockReceipt}) + mockBlockChain.SetTd(block1.Hash(), block1.NumberU64(), big.NewInt(1337)) + mockService := MockStateDiffService{ + Mutex: sync.Mutex{}, + Builder: statediff.NewBuilder(chain.StateCache()), + BlockChain: mockBlockChain, + } + payload, err := mockService.StateDiffAt(block1.Number().Uint64(), parameters) + if err != nil { + t.Error(err) + } + sort.Slice(payload.StateObjectRlp, func(i, j int) bool { return payload.StateObjectRlp[i] < payload.StateObjectRlp[j] }) + sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] }) + if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { + t.Errorf("payload does not have expected block\r\nactual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp) + } + if !bytes.Equal(payload.StateObjectRlp, expectedStateDiffBytes) { + t.Errorf("payload does not have expected state diff\r\nactual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateObjectRlp, expectedStateDiffBytes) + } + if !bytes.Equal(expectedReceiptBytes, payload.ReceiptsRlp) { + t.Errorf("payload does not have expected receipts\r\nactual receipt rlp: %v\r\nexpected receipt rlp: %v", payload.ReceiptsRlp, expectedReceiptBytes) + } + if !bytes.Equal(payload.TotalDifficulty.Bytes(), mockTotalDifficulty.Bytes()) { + t.Errorf("paylaod does not have the expected total difficulty\r\nactual td: %d\r\nexpected td: %d", payload.TotalDifficulty.Int64(), mockTotalDifficulty.Int64()) + } +} + +func testWatchAddressAPI(t *testing.T) { + blocks, chain := test_helpers.MakeChain(6, test_helpers.Genesis, test_helpers.TestChainGen) + defer chain.Stop() + block6 := blocks[5] + + mockBlockChain := &BlockChain{} + mockBlockChain.SetCurrentBlock(block6) + mockIndexer := StateDiffIndexer{} + mockService := MockStateDiffService{ + BlockChain: mockBlockChain, + Indexer: &mockIndexer, + } + + // test data + var ( + contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F" + contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B" + contract3Address = "0xcfeB164C328CA13EFd3C77E1980d94975aDfedfc" + contract4Address = "0x0Edf0c4f393a628DE4828B228C48175b3EA297fc" + contract1CreatedAt = uint64(1) + contract2CreatedAt = uint64(2) + contract3CreatedAt = uint64(3) + contract4CreatedAt = uint64(4) + + args1 = []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + startingParams1 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + expectedParams1 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract1Address), + common.HexToAddress(contract2Address), + }, + } + + args2 = []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + startingParams2 = expectedParams1 + expectedParams2 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract1Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + }, + } + + args3 = []sdtypes.WatchAddressArg{ + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + startingParams3 = expectedParams2 + expectedParams3 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract1Address), + }, + } + + args4 = []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + } + startingParams4 = expectedParams3 + expectedParams4 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + + args5 = []sdtypes.WatchAddressArg{ + { + Address: contract1Address, + CreatedAt: contract1CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + startingParams5 = expectedParams4 + expectedParams5 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract1Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + }, + } + + args6 = []sdtypes.WatchAddressArg{ + { + Address: contract4Address, + CreatedAt: contract4CreatedAt, + }, + { + Address: contract2Address, + CreatedAt: contract2CreatedAt, + }, + { + Address: contract3Address, + CreatedAt: contract3CreatedAt, + }, + } + startingParams6 = expectedParams5 + expectedParams6 = statediff.Params{ + WatchedAddresses: []common.Address{ + common.HexToAddress(contract4Address), + common.HexToAddress(contract2Address), + common.HexToAddress(contract3Address), + }, + } + + args7 = []sdtypes.WatchAddressArg{} + startingParams7 = expectedParams6 + expectedParams7 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + + args8 = []sdtypes.WatchAddressArg{} + startingParams8 = expectedParams6 + expectedParams8 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + + args9 = []sdtypes.WatchAddressArg{} + startingParams9 = expectedParams8 + expectedParams9 = statediff.Params{ + WatchedAddresses: []common.Address{}, + } + ) + + tests := []struct { + name string + operation sdtypes.OperationType + args []sdtypes.WatchAddressArg + startingParams statediff.Params + expectedParams statediff.Params + expectedErr error + }{ + { + "testAddAddresses", + sdtypes.Add, + args1, + startingParams1, + expectedParams1, + nil, + }, + { + "testAddAddressesSomeWatched", + sdtypes.Add, + args2, + startingParams2, + expectedParams2, + nil, + }, + { + "testRemoveAddresses", + sdtypes.Remove, + args3, + startingParams3, + expectedParams3, + nil, + }, + { + "testRemoveAddressesSomeWatched", + sdtypes.Remove, + args4, + startingParams4, + expectedParams4, + nil, + }, + { + "testSetAddresses", + sdtypes.Set, + args5, + startingParams5, + expectedParams5, + nil, + }, + { + "testSetAddressesSomeWatched", + sdtypes.Set, + args6, + startingParams6, + expectedParams6, + nil, + }, + { + "testSetAddressesEmtpyArgs", + sdtypes.Set, + args7, + startingParams7, + expectedParams7, + nil, + }, + { + "testClearAddresses", + sdtypes.Clear, + args8, + startingParams8, + expectedParams8, + nil, + }, + { + "testClearAddressesEmpty", + sdtypes.Clear, + args9, + startingParams9, + expectedParams9, + nil, + }, + + // invalid args + { + "testInvalidOperation", + "WrongOp", + args9, + startingParams9, + statediff.Params{}, + fmt.Errorf("%s WrongOp", unexpectedOperation), + }, + } + + for _, test := range tests { + // set indexing params + mockService.writeLoopParams = statediff.ParamsWithMutex{ + Params: test.startingParams, + } + mockService.writeLoopParams.ComputeWatchedAddressesLeafPaths() + + // make the API call to change watched addresses + err := mockService.WatchAddress(test.operation, test.args) + if test.expectedErr != nil { + if err.Error() != test.expectedErr.Error() { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual err: %+v\nexpected err: %+v", err, test.expectedErr) + } + + continue + } + if err != nil { + t.Error(err) + } + + // check updated indexing params + test.expectedParams.ComputeWatchedAddressesLeafPaths() + updatedParams := mockService.writeLoopParams.Params + if !reflect.DeepEqual(updatedParams, test.expectedParams) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual params: %+v\nexpected params: %+v", updatedParams, test.expectedParams) + } + } +} diff --git a/statediff/test_helpers/test_data.go b/statediff/test_helpers/test_data.go new file mode 100644 index 000000000000..e5a06a2a156f --- /dev/null +++ b/statediff/test_helpers/test_data.go @@ -0,0 +1,73 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package test_helpers + +import ( + "math/big" + "math/rand" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +// AddressToLeafKey hashes an returns an address +func AddressToLeafKey(address common.Address) []byte { + return crypto.Keccak256(address[:]) +} + +// AddressToEncodedPath hashes an address and appends the even-number leaf flag to it +func AddressToEncodedPath(address common.Address) []byte { + addrHash := crypto.Keccak256(address[:]) + decodedPath := append(EvenLeafFlag, addrHash...) + return decodedPath +} + +// Test variables +var ( + EvenLeafFlag = []byte{byte(2) << 4} + BlockNumber = big.NewInt(rand.Int63()) + BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" + NullCodeHash = crypto.Keccak256Hash([]byte{}) + StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes() + StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() + StorageValue = common.Hex2Bytes("0x03") + NullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") + + Testdb = rawdb.NewMemoryDatabase() + TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + BankLeafKey = AddressToLeafKey(TestBankAddress) + TestBankFunds = big.NewInt(params.Ether * 2) + Genesis = GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds) + + Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + Account1LeafKey = AddressToLeafKey(Account1Addr) + Account2LeafKey = AddressToLeafKey(Account2Addr) + ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506040518060200160405280600160ff16815250600190600161007492919061007a565b506100e4565b82606481019282156100ae579160200282015b828111156100ad578251829060ff1690559160200191906001019061008d565b5b5090506100bb91906100bf565b5090565b6100e191905b808211156100dd5760008160009055506001016100c5565b5090565b90565b6101ca806100f36000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032") + ByteCodeAfterDeployment = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032") + CodeHash = common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127") + ContractAddr common.Address + + EmptyRootNode, _ = rlp.EncodeToBytes(&[]byte{}) + EmptyContractRoot = crypto.Keccak256Hash(EmptyRootNode) +) diff --git a/statediff/trie_helpers/helpers.go b/statediff/trie_helpers/helpers.go new file mode 100644 index 000000000000..087cfe419abc --- /dev/null +++ b/statediff/trie_helpers/helpers.go @@ -0,0 +1,123 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package trie_helpers + +import ( + "fmt" + "sort" + "strings" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" +) + +// CheckKeyType checks what type of key we have +func CheckKeyType(elements []interface{}) (types.NodeType, error) { + if len(elements) > 2 { + return types.Branch, nil + } + if len(elements) < 2 { + return types.Unknown, fmt.Errorf("node cannot be less than two elements in length") + } + switch elements[0].([]byte)[0] / 16 { + case '\x00': + return types.Extension, nil + case '\x01': + return types.Extension, nil + case '\x02': + return types.Leaf, nil + case '\x03': + return types.Leaf, nil + default: + return types.Unknown, fmt.Errorf("unknown hex prefix") + } +} + +// ResolveNode return the state diff node pointed by the iterator. +func ResolveNode(it trie.NodeIterator, trieDB *trie.Database) (types.StateNode, []interface{}, error) { + nodePath := make([]byte, len(it.Path())) + copy(nodePath, it.Path()) + node, err := trieDB.Node(it.Hash()) + if err != nil { + return types.StateNode{}, nil, err + } + var nodeElements []interface{} + if err = rlp.DecodeBytes(node, &nodeElements); err != nil { + return types.StateNode{}, nil, err + } + ty, err := CheckKeyType(nodeElements) + if err != nil { + return types.StateNode{}, nil, err + } + return types.StateNode{ + NodeType: ty, + Path: nodePath, + NodeValue: node, + }, nodeElements, nil +} + +// SortKeys sorts the keys in the account map +func SortKeys(data types.AccountMap) []string { + keys := make([]string, 0, len(data)) + for key := range data { + keys = append(keys, key) + } + sort.Strings(keys) + + return keys +} + +// FindIntersection finds the set of strings from both arrays that are equivalent +// a and b must first be sorted +// this is used to find which keys have been both "deleted" and "created" i.e. they were updated +func FindIntersection(a, b []string) []string { + lenA := len(a) + lenB := len(b) + iOfA, iOfB := 0, 0 + updates := make([]string, 0) + if iOfA >= lenA || iOfB >= lenB { + return updates + } + for { + switch strings.Compare(a[iOfA], b[iOfB]) { + // -1 when a[iOfA] < b[iOfB] + case -1: + iOfA++ + if iOfA >= lenA { + return updates + } + // 0 when a[iOfA] == b[iOfB] + case 0: + updates = append(updates, a[iOfA]) + iOfA++ + iOfB++ + if iOfA >= lenA || iOfB >= lenB { + return updates + } + // 1 when a[iOfA] > b[iOfB] + case 1: + iOfB++ + if iOfB >= lenB { + return updates + } + } + } +} diff --git a/statediff/types/types.go b/statediff/types/types.go new file mode 100644 index 000000000000..0a29adaf892b --- /dev/null +++ b/statediff/types/types.go @@ -0,0 +1,120 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// StateRoots holds the state roots required for generating a state diff +type StateRoots struct { + OldStateRoot, NewStateRoot common.Hash +} + +// StateObject is the final output structure from the builder +type StateObject struct { + BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Nodes []StateNode `json:"nodes" gencodec:"required"` + CodeAndCodeHashes []CodeAndCodeHash `json:"codeMapping"` +} + +// AccountMap is a mapping of hex encoded path => account wrapper +type AccountMap map[string]AccountWrapper + +// AccountWrapper is used to temporary associate the unpacked node with its raw values +type AccountWrapper struct { + Account *types.StateAccount + NodeType NodeType + Path []byte + NodeValue []byte + LeafKey []byte +} + +// NodeType for explicitly setting type of node +type NodeType string + +const ( + Unknown NodeType = "Unknown" + Branch NodeType = "Branch" + Extension NodeType = "Extension" + Leaf NodeType = "Leaf" + Removed NodeType = "Removed" // used to represent paths which have been emptied +) + +func (n NodeType) Int() int { + switch n { + case Branch: + return 0 + case Extension: + return 1 + case Leaf: + return 2 + case Removed: + return 3 + default: + return -1 + } +} + +// StateNode holds the data for a single state diff node +type StateNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + StorageNodes []StorageNode `json:"storage"` + LeafKey []byte `json:"leafKey"` +} + +// StorageNode holds the data for a single storage diff node +type StorageNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + LeafKey []byte `json:"leafKey"` +} + +// CodeAndCodeHash struct for holding codehash => code mappings +// we can't use an actual map because they are not rlp serializable +type CodeAndCodeHash struct { + Hash common.Hash `json:"codeHash"` + Code []byte `json:"code"` +} + +type StateNodeSink func(StateNode) error +type StorageNodeSink func(StorageNode) error +type CodeSink func(CodeAndCodeHash) error + +// OperationType for type of WatchAddress operation +type OperationType string + +const ( + Add OperationType = "add" + Remove OperationType = "remove" + Set OperationType = "set" + Clear OperationType = "clear" +) + +// WatchAddressArg is a arg type for WatchAddress API +type WatchAddressArg struct { + // Address represents common.Address + Address string + CreatedAt uint64 +} diff --git a/trie/encoding.go b/trie/encoding.go index 8ee0022ef3a0..ace45700cde0 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -34,6 +34,11 @@ package trie // in the case of an odd number. All remaining nibbles (now an even number) fit properly // into the remaining bytes. Compact encoding is used for nodes stored on disk. +// HexToCompact converts a hex path to the compact encoded format +func HexToCompact(hex []byte) []byte { + return hexToCompact(hex) +} + func hexToCompact(hex []byte) []byte { terminator := byte(0) if hasTerm(hex) { @@ -80,6 +85,11 @@ func hexToCompactInPlace(hex []byte) int { return binLen } +// CompactToHex converts a compact encoded path to hex format +func CompactToHex(compact []byte) []byte { + return compactToHex(compact) +} + func compactToHex(compact []byte) []byte { if len(compact) == 0 { return compact @@ -105,9 +115,9 @@ func keybytesToHex(str []byte) []byte { return nibbles } -// hexToKeybytes turns hex nibbles into key bytes. +// hexToKeyBytes turns hex nibbles into key bytes. // This can only be used for keys of even length. -func hexToKeybytes(hex []byte) []byte { +func hexToKeyBytes(hex []byte) []byte { if hasTerm(hex) { hex = hex[:len(hex)-1] } diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 16393313f743..7ade0a095dcb 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -71,8 +71,8 @@ func TestHexKeybytes(t *testing.T) { if h := keybytesToHex(test.key); !bytes.Equal(h, test.hexOut) { t.Errorf("keybytesToHex(%x) -> %x, want %x", test.key, h, test.hexOut) } - if k := hexToKeybytes(test.hexIn); !bytes.Equal(k, test.key) { - t.Errorf("hexToKeybytes(%x) -> %x, want %x", test.hexIn, k, test.key) + if k := hexToKeyBytes(test.hexIn); !bytes.Equal(k, test.key) { + t.Errorf("hexToKeyBytes(%x) -> %x, want %x", test.hexIn, k, test.key) } } } @@ -135,6 +135,6 @@ func BenchmarkKeybytesToHex(b *testing.B) { func BenchmarkHexToKeybytes(b *testing.B) { testBytes := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16} for i := 0; i < b.N; i++ { - hexToKeybytes(testBytes) + hexToKeyBytes(testBytes) } } diff --git a/trie/iterator.go b/trie/iterator.go index 1e76625c6213..d2c8b6a785ff 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -190,7 +190,7 @@ func (it *nodeIterator) Leaf() bool { func (it *nodeIterator) LeafKey() []byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - return hexToKeybytes(it.path) + return hexToKeyBytes(it.path) } } panic("not at leaf") diff --git a/trie/sync.go b/trie/sync.go index 303fcbfa22e2..f4fdf1432db0 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -70,7 +70,7 @@ func NewSyncPath(path []byte) SyncPath { if len(path) < 64 { return SyncPath{hexToCompact(path)} } - return SyncPath{hexToKeybytes(path[:64]), hexToCompact(path[64:])} + return SyncPath{hexToKeyBytes(path[:64]), hexToCompact(path[64:])} } // nodeRequest represents a scheduled or already in-flight trie node retrieval request. @@ -417,10 +417,10 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { if node, ok := (child.node).(valueNode); ok { var paths [][]byte if len(child.path) == 2*common.HashLength { - paths = append(paths, hexToKeybytes(child.path)) + paths = append(paths, hexToKeyBytes(child.path)) } else if len(child.path) == 4*common.HashLength { - paths = append(paths, hexToKeybytes(child.path[:2*common.HashLength])) - paths = append(paths, hexToKeybytes(child.path[2*common.HashLength:])) + paths = append(paths, hexToKeyBytes(child.path[:2*common.HashLength])) + paths = append(paths, hexToKeyBytes(child.path[2*common.HashLength:])) } if err := req.callback(paths, child.path, node, req.hash, req.path); err != nil { return nil, err