Rosetta API implementation

.github/workflows/test.yml

@@ -188,6 +188,23 @@ jobs:
           name: "${{ github.sha }}-${{ matrix.part }}-race-output"
           path: ./${{ matrix.part }}-race-output.txt
 
+  test-rosetta:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - uses: technote-space/get-diff-action@v4
+        id: git_diff
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - name: test rosetta
+        run: |
+          make test-rosetta
+        # if: env.GIT_DIFF
+
   race-detector-report:
     runs-on: ubuntu-latest
     needs: [test-race, install-tparse]

Makefile

@@ -316,6 +316,11 @@ test-cover:
 	@export VERSION=$(VERSION); bash -x contrib/test_cover.sh
 .PHONY: test-cover
 
+test-rosetta:
+	docker build -t rosetta-ci:latest -f contrib/rosetta/node/Dockerfile .
+	docker-compose -f contrib/rosetta/docker-compose.yaml up --abort-on-container-exit --exit-code-from test_rosetta --build
+.PHONY: test-rosetta
+
 benchmark:
 	@go test -mod=readonly -bench=. $(PACKAGES_NOSIMULATION)
 .PHONY: benchmark
@@ -470,3 +475,15 @@ localnet-stop:
 	docker-compose down
 
 .PHONY: localnet-start localnet-stop
+
+###############################################################################
+###                                rosetta                                  ###
+###############################################################################
+# builds rosetta test data dir
+rosetta-data:
+	-docker container rm data_dir_build
+	docker build -t rosetta-ci:latest -f contrib/rosetta/node/Dockerfile .
+	docker run --name data_dir_build -t rosetta-ci:latest sh /rosetta/data.sh
+	docker cp data_dir_build:/tmp/data.tar.gz "$(CURDIR)/contrib/rosetta/node/data.tar.gz"
+	docker container rm data_dir_build
+.PHONY: rosetta-data

contrib/githooks/pre-commit

@@ -25,7 +25,7 @@ f_check_cmds
 if [[ $STAGED_GO_FILES != "" ]]; then
   f_echo_stderr "[pre-commit] fmt'ing staged files..."
   for file in $STAGED_GO_FILES; do
-    if [[ $file =~ vendor/ ]] || [[ $file =~ client/lcd/statik/ ]] || [[ $file =~ tests/mocks/ ]] || [[ $file =~ \.pb\.go ]]; then
+    if [[ $file =~ vendor/ ]] || [[ $file =~ client/docs/statik/ ]] || [[ $file =~ tests/mocks/ ]] || [[ $file =~ \.pb\.go ]]; then
       continue
     fi
 

contrib/rosetta/README.md

@@ -0,0 +1,24 @@
+# rosetta
+
+This directory contains the files required to run the rosetta CI. It builds `simapp` based on the current codebase.
+
+## docker-compose.yaml
+
+Builds:
+- cosmos-sdk simapp node, with prefixed data directory, keys etc. This is required to test historical balances.
+- faucet is required so we can test construction API, it was literally impossible to put there a deterministic address to request funds for
+- rosetta is the rosetta node used by rosetta-cli to interact with the cosmos-sdk app
+- test_rosetta runs the rosetta-cli test against construction API and data API
+
+## configuration
+
+Contains the required files to set up rosetta cli and make it work against its workflows
+
+## node
+
+Contains the files for a deterministic network, with fixed keys and some actions on there, to test parsing of msgs and historical balances.
+
+## Notes
+
+- Keyring password is 12345678
+- data.sh creates node data, it's required in case consensus breaking changes are made to quickly recreate replicable node data for rosetta

contrib/rosetta/configuration/bootstrap.json

@@ -0,0 +1,12 @@
+[
+  {
+    "account_identifier": {
+      "address":"cosmos1hdmjfmqmf8ck4pv4evu0s3up0ucm0yjjqfl87e"
+    },
+    "currency":{
+      "symbol":"stake",
+      "decimals":0
+    },
+    "value": "999900000000"
+  }
+]

contrib/rosetta/configuration/data.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+
+set -e
+
+wait_simd() {
+  timeout 30 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 9090
+}
+# this script is used to recreate the data dir
+echo clearing /root/.simapp
+rm -rf /root/.simapp
+echo initting new chain
+# init config files
+simd init simd --chain-id testing
+
+# create accounts
+simd keys add fd --keyring-backend=test
+
+addr=$(simd keys show fd -a --keyring-backend=test)
+
+# give the accounts some money
+simd add-genesis-account "$addr" 1000000000000stake --keyring-backend=test
+
+# save configs for the daemon
+simd gentx fd --chain-id testing --keyring-backend=test
+
+# input genTx to the genesis file
+simd collect-gentxs
+# verify genesis file is fine
+simd validate-genesis
+echo changing network settings
+sed -i 's/127.0.0.1/0.0.0.0/g' /root/.simapp/config/config.toml
+
+# start simd
+echo starting simd...
+simd start --pruning=nothing &
+pid=$!
+echo simd started with PID $pid
+
+echo awaiting for simd to be ready
+wait_simd
+echo simd is ready
+sleep 10
+
+
+# send transaction to deterministic address
+echo sending transaction with addr $addr
+simd tx bank send "$addr" cosmos1wjmt63j4fv9nqda92nsrp2jp2vsukcke4va3pt 100stake --yes --keyring-backend=test --broadcast-mode=block --chain-id=testing
+
+sleep 10
+
+echo stopping simd...
+kill -9 $pid
+
+echo zipping data dir and saving to /tmp/data.tar.gz
+
+tar -czvf /tmp/data.tar.gz /root/.simapp
+
+echo new address for bootstrap.json "$addr"

contrib/rosetta/configuration/faucet.py

@@ -0,0 +1,25 @@
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import subprocess
+
+import os
+
+
+class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
+
+    def do_POST(self):
+        try:
+            content_len = int(self.headers.get('Content-Length'))
+            addr = self.rfile.read(content_len).decode("utf-8")
+            print("sending funds to " + addr)
+            subprocess.call(['sh', './send_funds.sh', addr])
+            self.send_response(200)
+            self.end_headers()
+        except Exception as e:
+            print("failed " + str(e))
+            os._exit(1)
+
+
+if __name__ == "__main__":
+    print("starting faucet server...")
+    httpd = HTTPServer(('0.0.0.0', 8000), SimpleHTTPRequestHandler)
+    httpd.serve_forever()

contrib/rosetta/configuration/rosetta.json

@@ -0,0 +1,51 @@
+{
+  "network": {
+    "blockchain": "app",
+    "network": "network"
+  },
+  "online_url": "http://rosetta:8080",
+  "data_directory": "",
+  "http_timeout": 300,
+  "max_retries": 5,
+  "retry_elapsed_time": 0,
+  "max_online_connections": 0,
+  "max_sync_concurrency": 0,
+  "tip_delay": 60,
+  "log_configuration": true,
+  "construction": {
+    "offline_url": "http://rosetta:8080",
+    "max_offline_connections": 0,
+    "stale_depth": 0,
+    "broadcast_limit": 0,
+    "ignore_broadcast_failures": false,
+    "clear_broadcasts": false,
+    "broadcast_behind_tip": false,
+    "block_broadcast_limit": 0,
+    "rebroadcast_all": false,
+    "constructor_dsl_file": "transfer.ros",
+    "end_conditions": {
+      "create_account": 1,
+      "transfer": 3
+    }
+  },
+  "data": {
+    "active_reconciliation_concurrency": 0,
+    "inactive_reconciliation_concurrency": 0,
+    "inactive_reconciliation_frequency": 0,
+    "log_blocks": false,
+    "log_transactions": false,
+    "log_balance_changes": false,
+    "log_reconciliations": false,
+    "ignore_reconciliation_error": false,
+    "exempt_accounts": "",
+    "bootstrap_balances": "bootstrap.json",
+    "interesting_accounts": "",
+    "reconciliation_disabled": false,
+    "inactive_discrepency_search_disabled": false,
+    "balance_tracking_disabled": false,
+    "coin_tracking_disabled": false,
+    "end_conditions": {
+      "tip": true
+    }
+  }
+}

contrib/rosetta/configuration/run_tests.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -e
+
+addr="abcd"
+
+send_tx() {
+  echo '12345678' | simd tx bank send $addr "$1" "$2"
+}
+
+detect_account() {
+  line=$1
+}
+
+wait_for_rosetta() {
+  timeout 30 sh -c 'until nc -z $0 $1; do sleep 1; done' rosetta 8080
+}
+
+echo "waiting for rosetta instance to be up"
+wait_for_rosetta
+
+echo "checking data API"
+rosetta-cli check:data --configuration-file ./config/rosetta.json
+
+echo "checking construction API"
+rosetta-cli check:construction --configuration-file ./config/rosetta.json
+
+echo "checking staking API"
+rosetta-cli check:construction --configuration-file ./config/staking.json

contrib/rosetta/configuration/send_funds.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -e
+addr=$(simd keys show fd -a --keyring-backend=test)
+echo "12345678" | simd tx bank send "$addr" "$1" 100stake --chain-id="testing" --node tcp://cosmos:26657 --yes --keyring-backend=test

contrib/rosetta/configuration/staking.json

@@ -0,0 +1,30 @@
+{
+  "network": {
+    "blockchain": "app",
+    "network": "network"
+  },
+  "online_url": "http://rosetta:8080",
+  "data_directory": "",
+  "http_timeout": 300,
+  "max_retries": 5,
+  "retry_elapsed_time": 0,
+  "max_online_connections": 0,
+  "max_sync_concurrency": 0,
+  "tip_delay": 60,
+  "log_configuration": true,
+  "construction": {
+    "offline_url": "http://rosetta:8080",
+    "max_offline_connections": 0,
+    "stale_depth": 0,
+    "broadcast_limit": 0,
+    "ignore_broadcast_failures": false,
+    "clear_broadcasts": false,
+    "broadcast_behind_tip": false,
+    "block_broadcast_limit": 0,
+    "rebroadcast_all": false,
+    "constructor_dsl_file": "staking.ros",
+    "end_conditions": {
+      "staking": 3
+    }
+  }
+}