Addressing issue coinbase#41 - Allow max concurrency limits configura…

…tion on the indexer
smeyerhot · Dec 24, 2020 · f8f01dc · f8f01dc
1 parent e719cfc
commit f8f01dc
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 16 deletions.
diff --git a/Makefile b/Makefile
@@ -34,16 +34,16 @@ build-release:
 	docker save rosetta-bitcoin:$(version) | gzip > rosetta-bitcoin-$(version).tar.gz;
 
 run-mainnet-online:
-	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC"="64" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
+	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC"="256" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
 
 run-mainnet-offline:
-	docker run -d --rm -e "MAXSYNC"="64" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+	docker run -d --rm -e "MAXSYNC"="256" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 
 run-testnet-online:
-	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC"="64" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
+	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC"="256" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
 
 run-testnet-offline:
-	docker run -d --rm -e "MAXSYNC"="64" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+	docker run -d --rm -e "MAXSYNC"="256" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 
 train:
 	./zstd-train.sh $(network) transaction $(data-directory)

diff --git a/README.md b/README.md
@@ -70,25 +70,25 @@ at port `8080`.
 
 #### Mainnet:Online
 ```text
-docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC"="64" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
+docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC"="256" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-mainnet-online`._
 
 #### Mainnet:Offline
 ```text
-docker run -d --rm -e "MAXSYNC"="64" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+docker run -d --rm -e "MAXSYNC"="256" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-mainnet-offline`._
 
 #### Testnet:Online
 ```text
-docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC"="64" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
+docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC"="256" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-testnet-online`._
 
 #### Testnet:Offline
 ```text
-docker run -d --rm -e "MAXSYNC"="64" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+docker run -d --rm -e "MAXSYNC"="256" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-testnet-offline`._
 

diff --git a/configuration/configuration.go b/configuration/configuration.go
@@ -142,7 +142,7 @@ func LoadConfiguration(baseDirectory string) (*Configuration, error) {
 	maxSyncValue := os.Getenv(MaxSyncConcurrency)
 	switch maxSyncValue {
 	case "":
-		config.MaxSyncConcurrency = syncer.DefaultConcurrency
+		config.MaxSyncConcurrency = syncer.DefaultMaxConcurrency
 	case "0":
 		return nil, errors.New("syncer concurrency must be greater than zero")
 	default:

diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go
@@ -48,14 +48,14 @@ func TestLoadConfiguration(t *testing.T) {
 		"port not set": {
 			Mode:    string(Online),
 			Network: Mainnet,
-			MaxSyncConcurrency: "64",
+			MaxSyncConcurrency: "256",
 			err:     errors.New("PORT must be populated"),
 		},
 		"all set (mainnet)": {
 			Mode:    string(Online),
 			Network: Mainnet,
 			Port:    "1000",
-			MaxSyncConcurrency: "64",
+			MaxSyncConcurrency: "256",
 			cfg: &Configuration{
 				Mode: Online,
 				Network: &types.NetworkIdentifier{
@@ -66,7 +66,7 @@ func TestLoadConfiguration(t *testing.T) {
 				Currency:               bitcoin.MainnetCurrency,
 				GenesisBlockIdentifier: bitcoin.MainnetGenesisBlockIdentifier,
 				Port:                   1000,
-				MaxSyncConcurrency:     64,
+				MaxSyncConcurrency:     256,
 				RPCPort:                mainnetRPCPort,
 				ConfigPath:             mainnetConfigPath,
 				Pruning: &PruningConfiguration{
@@ -86,7 +86,7 @@ func TestLoadConfiguration(t *testing.T) {
 			Mode:    string(Online),
 			Network: Testnet,
 			Port:    "1000",
-			MaxSyncConcurrency: "64",
+			MaxSyncConcurrency: "256",
 			cfg: &Configuration{
 				Mode: Online,
 				Network: &types.NetworkIdentifier{
@@ -97,7 +97,7 @@ func TestLoadConfiguration(t *testing.T) {
 				Currency:               bitcoin.TestnetCurrency,
 				GenesisBlockIdentifier: bitcoin.TestnetGenesisBlockIdentifier,
 				Port:                   1000,
-				MaxSyncConcurrency:     64,
+				MaxSyncConcurrency:     256,
 				RPCPort:                testnetRPCPort,
 				ConfigPath:             testnetConfigPath,
 				Pruning: &PruningConfiguration{
@@ -118,7 +118,6 @@ func TestLoadConfiguration(t *testing.T) {
 			Network: Testnet,
 			Port:    "1000",
 			MaxSyncConcurrency: "",
-
 			cfg: &Configuration{
 				Mode: Online,
 				Network: &types.NetworkIdentifier{
@@ -129,7 +128,7 @@ func TestLoadConfiguration(t *testing.T) {
 				Currency:               bitcoin.TestnetCurrency,
 				GenesisBlockIdentifier: bitcoin.TestnetGenesisBlockIdentifier,
 				Port:                   1000,
-				MaxSyncConcurrency:     64,
+				MaxSyncConcurrency:     256,
 				RPCPort:                testnetRPCPort,
 				ConfigPath:             testnetConfigPath,
 				Pruning: &PruningConfiguration{