diff --git a/go.mod b/go.mod index 4db9cbf4..4b14c6a6 100644 --- a/go.mod +++ b/go.mod @@ -15,5 +15,8 @@ require ( go.uber.org/zap v1.21.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect - google.golang.org/protobuf v1.27.1 // indirect ) + +require github.com/btcsuite/btcd/btcutil v1.1.1 + +replace github.com/coinbase/rosetta-sdk-go => github.com/Concordium/rosetta-sdk-go v0.7.11-0.20220706095914-7942ab456d4f diff --git a/go.sum b/go.sum index 8797de37..446ceb36 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Concordium/rosetta-sdk-go v0.7.11-0.20220706095914-7942ab456d4f h1:Z7x7orxWLS2ThSZHKRWUFuGCCXtPpczvvdAhE4vgdlk= +github.com/Concordium/rosetta-sdk-go v0.7.11-0.20220706095914-7942ab456d4f/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -41,6 +43,8 @@ github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/anaskhan96/base58check v0.0.0-20181220122047-b05365d494c4 h1:FUDNaUiPOxrVtUmsRSdx7hrvCKXpfQafPpPU0Yh27os= +github.com/anaskhan96/base58check v0.0.0-20181220122047-b05365d494c4/go.mod h1:glPG1rmt/bD3wEXWanFIuoPjC4MG+JEN+i7YhwEYA/Y= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -62,9 +66,15 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401/go.mod h1:Sv4JPQ3/M+teHz9Bo5jBpkNcP0x6r7rdihlNL/7tTAs= +github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= +github.com/btcsuite/btcd/btcec/v2 v2.1.1/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= +github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A= +github.com/btcsuite/btcd/btcutil v1.1.1 h1:hDcDaXiP0uEzR8Biqo2weECKqEw0uHDZ9ixIWevVQqY= +github.com/btcsuite/btcd/btcutil v1.1.1/go.mod h1:nbKlBMNm9FGsdvKvu0essceubPiAcI57pYBNnsLAa34= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -97,8 +107,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 h1:cHRCEzEk+4xerPtuJgO5a71qp7jIwGDKCKq+pte2VHA= -github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= @@ -708,9 +716,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/tester/data.go b/pkg/tester/data.go index e74e9bdc..74fe927a 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -329,6 +329,11 @@ func InitializeData( statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), statefulsyncer.WithSeenConcurrency(int64(config.SeenBlockWorkers)), + statefulsyncer.WithExtraSyncerOpts( + syncer.WithCustomHelper(func(h syncer.Helper) syncer.Helper { + return newConcordiumHelper(h) + }), + ), } if config.Data.PruningFrequency != nil { statefulSyncerOptions = append( diff --git a/pkg/tester/data_concordium.go b/pkg/tester/data_concordium.go new file mode 100644 index 00000000..4213b050 --- /dev/null +++ b/pkg/tester/data_concordium.go @@ -0,0 +1,83 @@ +package tester + +import ( + "context" + "sync" + + "github.com/btcsuite/btcd/btcutil/base58" + "github.com/coinbase/rosetta-sdk-go/syncer" + "github.com/coinbase/rosetta-sdk-go/types" +) + +// concordiumHelper is a wrapper around a sync.Helper that monkey patches fetched blocks +// by rewriting all account addresses into a "normalized" form (the zero'th alias). +// This is necessary for DataTester to do correct accounting. +type concordiumHelper struct { + // helper is the wrapped syncer.Helper instance. + helper syncer.Helper + // cache is a map from the zero-alias to the canonical address (assumed by being the first alias observed - probably the account's creation). + // We must use the canonical/non-aliased address instead of just the zero-alias to support networks such as mainnet + // which started on a protocol version that didn't support aliases. + cache map[string]string + cacheLock sync.RWMutex +} + +func newConcordiumHelper(h syncer.Helper) *concordiumHelper { + return &concordiumHelper{ + helper: h, + cache: make(map[string]string), + } +} + +func (h *concordiumHelper) NetworkStatus(ctx context.Context, network *types.NetworkIdentifier) (*types.NetworkStatusResponse, error) { + return h.helper.NetworkStatus(ctx, network) +} + +func (h *concordiumHelper) Block(ctx context.Context, network *types.NetworkIdentifier, block *types.PartialBlockIdentifier) (*types.Block, error) { + b, err := h.helper.Block(ctx, network, block) + if err == nil { + // Patch all account references in fetched block. + for _, tx := range b.Transactions { + for _, op := range tx.Operations { + h.normalizeAddress(op.Account) + } + } + } + return b, err +} + +func (h *concordiumHelper) normalizeAddress(account *types.AccountIdentifier) { + if account != nil { + a0 := addressToAliasZero(account.Address) + h.cacheLock.RLock() + a, ok := h.cache[a0] + h.cacheLock.RUnlock() + if ok { + account.Address = a + } else { + // NOTE: Due to parallelism, it isn't guaranteed that we see the blocks in strict sequential order. + // But the only case we have a problem is when using a non-canonical alias when querying blocks from a protocol version that doesn't support aliases; + // before the protocol update we never see any aliases. + // So this can only fail if an account was created just before the protocol update took effect and was then referenced with an alias just afterwards (i.e. with in the window of parallelism). + // Then we could get unlucky and process the latter block first, then fail when querying the balance before the update afterwards. + // That situation seems very unlikely. + h.cacheLock.Lock() + h.cache[a0] = account.Address // first time we see the address; assume that it's the canonical variant. + h.cacheLock.Unlock() + } + } +} + +func addressToAliasZero(addr string) string { + addrBytes, versionByte, err := base58.CheckDecode(addr) + if err != nil || len(addrBytes) != 32 { + // Non-Base58Check address is likely virtual (see https://github.com/Concordium/concordium-rosetta/#identifiers); + // return unchanged. + return addr + } + // Zero out last 3 bytes and re-encode. + addrBytes[29] = 0 + addrBytes[30] = 0 + addrBytes[31] = 0 + return base58.CheckEncode(addrBytes, versionByte) +}