From f4a05f0d1db3f2fb65e35cae5141e25c67f369b1 Mon Sep 17 00:00:00 2001 From: Trevor Porter Date: Tue, 17 Dec 2019 17:49:25 -0800 Subject: [PATCH] Upload bootnode info & use in docs instead of static nodes (#2289) --- .../src/cmds/deploy/initial/testnet.ts | 10 +- packages/celotool/src/lib/geth.ts | 11 +- packages/celotool/src/lib/helm_deploy.ts | 3 +- packages/celotool/src/lib/testnet-utils.ts | 108 +++++++++++++----- packages/celotool/src/lib/vm-testnet-utils.ts | 19 +-- .../getting-started/running-a-full-node.md | 6 +- .../getting-started/running-a-validator.md | 12 +- packages/terraform-modules/testnet/outputs.tf | 4 + 8 files changed, 109 insertions(+), 64 deletions(-) diff --git a/packages/celotool/src/cmds/deploy/initial/testnet.ts b/packages/celotool/src/cmds/deploy/initial/testnet.ts index b924d701dea..88a0f903755 100644 --- a/packages/celotool/src/cmds/deploy/initial/testnet.ts +++ b/packages/celotool/src/cmds/deploy/initial/testnet.ts @@ -1,11 +1,7 @@ import { createClusterIfNotExists, setupCluster, switchToClusterFromEnv } from 'src/lib/cluster' import { failIfVmBased } from 'src/lib/env-utils' import { createStaticIPs, installHelmChart, pollForBootnodeLoadBalancer } from 'src/lib/helm_deploy' -import { - uploadEnvFileToGoogleStorage, - uploadGenesisBlockToGoogleStorage, - uploadStaticNodesToGoogleStorage, -} from 'src/lib/testnet-utils' +import { uploadTestnetInfoToGoogleStorage } from 'src/lib/testnet-utils' import yargs from 'yargs' import { InitialArgv } from '../../deploy/initial' @@ -39,7 +35,5 @@ export const handler = async (argv: TestnetInitialArgv) => { // When using an external bootnode, we have to await the bootnode's LB to be up first await pollForBootnodeLoadBalancer(argv.celoEnv) - await uploadGenesisBlockToGoogleStorage(argv.celoEnv) - await uploadStaticNodesToGoogleStorage(argv.celoEnv) - await uploadEnvFileToGoogleStorage(argv.celoEnv) + await uploadTestnetInfoToGoogleStorage(argv.celoEnv) } diff --git a/packages/celotool/src/lib/geth.ts b/packages/celotool/src/lib/geth.ts index 64789d23e03..f26f5b3fda9 100644 --- a/packages/celotool/src/lib/geth.ts +++ b/packages/celotool/src/lib/geth.ts @@ -103,12 +103,21 @@ const getExternalEnodeAddresses = async (namespace: string) => { } export const getBootnodeEnode = async (namespace: string) => { - const ip = await retrieveIPAddress(`${namespace}-bootnode`) + const ip = await retrieveBootnodeIPAddress(namespace) const privateKey = generatePrivateKey(fetchEnv(envVar.MNEMONIC), AccountType.BOOTNODE, 0) const nodeId = privateKeyToPublicKey(privateKey) return [getEnodeAddress(nodeId, ip, DISCOVERY_PORT)] } +const retrieveBootnodeIPAddress = async (namespace: string) => { + if (isVmBased()) { + const outputs = await getTestnetOutputs(namespace) + return outputs.bootnode_ip_address.value + } else { + return retrieveIPAddress(`${namespace}-bootnode`) + } +} + const retrieveTxNodeAddresses = async (namespace: string, txNodesNum: number) => { if (isVmBased()) { const outputs = await getTestnetOutputs(namespace) diff --git a/packages/celotool/src/lib/helm_deploy.ts b/packages/celotool/src/lib/helm_deploy.ts index c02f28744f5..c5bad47d8cf 100644 --- a/packages/celotool/src/lib/helm_deploy.ts +++ b/packages/celotool/src/lib/helm_deploy.ts @@ -427,7 +427,8 @@ export async function pollForBootnodeLoadBalancer(celoEnv: string) { await sleep(LOAD_BALANCER_POLL_INTERVAL) } - await sleep(1000 * 60 * 5) + console.info('Sleeping 1 minute...') + await sleep(1000 * 60) // 1 minute console.info(`\nReset all pods now that the bootnode load balancer has provisioned`) await execCmdWithExitOnFailure(`kubectl delete pod -n ${celoEnv} --selector=component=validators`) diff --git a/packages/celotool/src/lib/testnet-utils.ts b/packages/celotool/src/lib/testnet-utils.ts index 36eb97ccb30..8eed139db3d 100644 --- a/packages/celotool/src/lib/testnet-utils.ts +++ b/packages/celotool/src/lib/testnet-utils.ts @@ -8,7 +8,7 @@ import { getGenesisGoogleStorageUrl } from './endpoints' import { getEnvFile } from './env-utils' import { ensureAuthenticatedGcloudAccount } from './gcloud_utils' import { generateGenesisFromEnv } from './generate_utils' -import { getEnodesWithExternalIPAddresses } from './geth' +import { getBootnodeEnode, getEnodesWithExternalIPAddresses } from './geth' import { execCmdWithExitOnFailure } from './utils' const genesisBlocksBucketName = GenesisBlocksGoogleStorageBucketName @@ -16,15 +16,22 @@ const staticNodesBucketName = StaticNodeUtils.getStaticNodesGoogleStorageBucketN // Someone has taken env_files and I don't even has permission to modify it :/ // See files in this bucket using `$ gsutil ls gs://env_config_files` const envBucketName = 'env_config_files' +const bootnodesBucketName = 'env_bootnodes' + +// uploads genesis block, static nodes, env file, and bootnode to GCS +export async function uploadTestnetInfoToGoogleStorage(networkName: string) { + await uploadGenesisBlockToGoogleStorage(networkName) + await uploadStaticNodesToGoogleStorage(networkName) + await uploadBootnodeToGoogleStorage(networkName) + await uploadEnvFileToGoogleStorage(networkName) +} export async function uploadGenesisBlockToGoogleStorage(networkName: string) { console.info(`\nUploading genesis block for ${networkName} to Google cloud storage`) const genesisBlockJsonData = generateGenesisFromEnv() console.debug(`Genesis block is ${genesisBlockJsonData} \n`) - const localTmpFilePath = `/tmp/${networkName}_genesis-block` - fs.writeFileSync(localTmpFilePath, genesisBlockJsonData) - await uploadFileToGoogleStorage( - localTmpFilePath, + await uploadDataToGoogleStorage( + genesisBlockJsonData, genesisBlocksBucketName, networkName, true, @@ -41,31 +48,15 @@ export async function getGenesisBlockFromGoogleStorage(networkName: string) { export async function uploadStaticNodesToGoogleStorage(networkName: string) { console.info(`\nUploading static nodes for ${networkName} to Google cloud storage...`) // Get node json file - let nodesJsonData: string | null = null - const numAttempts = 100 - for (let i = 1; i <= numAttempts; i++) { - try { - nodesJsonData = JSON.stringify(await getEnodesWithExternalIPAddresses(networkName)) - break - } catch (error) { - const sleepTimeBasisInMs = 1000 - const sleepTimeInMs = sleepTimeBasisInMs * Math.pow(2, i) - console.warn( - `${new Date().toLocaleTimeString()} Failed to get static nodes information, attempt: ${i}/${numAttempts}, ` + - `retry after sleeping for ${sleepTimeInMs} milli-seconds`, - error - ) - await sleep(sleepTimeInMs) - } - } + const nodesJsonData: string[] | null = await retryCmd(() => + getEnodesWithExternalIPAddresses(networkName) + ) if (nodesJsonData === null) { throw new Error('Fail to get static nodes information') } console.debug('Static nodes are ' + nodesJsonData + '\n') - const localTmpFilePath = `/tmp/${networkName}_static-nodes` - fs.writeFileSync(localTmpFilePath, nodesJsonData) - await uploadFileToGoogleStorage( - localTmpFilePath, + await uploadDataToGoogleStorage( + nodesJsonData, staticNodesBucketName, networkName, true, @@ -73,6 +64,23 @@ export async function uploadStaticNodesToGoogleStorage(networkName: string) { ) } +export async function uploadBootnodeToGoogleStorage(networkName: string) { + console.info(`\nUploading bootnode for ${networkName} to Google Cloud Storage...`) + const [bootnodeEnode] = await retryCmd(() => getBootnodeEnode(networkName)) + if (!bootnodeEnode) { + throw new Error('Failed to get bootnode enode') + } + // for now there is always only one bootnodde + console.info('Bootnode enode:', bootnodeEnode) + await uploadDataToGoogleStorage( + bootnodeEnode, + bootnodesBucketName, + networkName, + true, // make it public + 'text/plain' + ) +} + export async function uploadEnvFileToGoogleStorage(networkName: string) { const envFileName = getEnvFile(networkName) const userInfo = `${await getGoogleCloudUserInfo()}` @@ -90,10 +98,8 @@ export async function uploadEnvFileToGoogleStorage(networkName: string) { `# Last modified on on ${Date()}\n` + `# Base commit: "https://github.com/${repo}/commit/${commitHash}"\n` const fullData = metaData + '\n' + envFileData - const localTmpFilePath = `/tmp/${networkName}_env-file` - fs.writeFileSync(localTmpFilePath, fullData) - await uploadFileToGoogleStorage( - localTmpFilePath, + await uploadDataToGoogleStorage( + fullData, envBucketName, networkName, false /* keep file private */, @@ -101,6 +107,29 @@ export async function uploadEnvFileToGoogleStorage(networkName: string) { ) } +async function retryCmd( + cmd: () => Promise, + numAttempts: number = 100, + maxTimeoutMs: number = 15000 +) { + for (let i = 1; i <= numAttempts; i++) { + try { + const result = await cmd() + return result + } catch (error) { + const sleepTimeBasisInMs = 1000 + const sleepTimeInMs = Math.min(sleepTimeBasisInMs * Math.pow(2, i), maxTimeoutMs) + console.warn( + `${new Date().toLocaleTimeString()} Retry attempt: ${i}/${numAttempts}, ` + + `retry after sleeping for ${sleepTimeInMs} milli-seconds`, + error + ) + await sleep(sleepTimeInMs) + } + } + return null +} + async function getGoogleCloudUserInfo(): Promise { const cmd = 'gcloud config get-value account' const stdout = (await execCmdWithExitOnFailure(cmd))[0] @@ -123,6 +152,25 @@ async function getCommitHash(): Promise { return stdout.split(' ')[1].trim() } +// Writes data to a temporary file & uploads it to GCS +export function uploadDataToGoogleStorage( + data: any, + googleStorageBucketName: string, + googleStorageFileName: string, + makeFileWorldReadable: boolean, + contentType: string +) { + const localTmpFilePath = `/tmp/${googleStorageBucketName}-${googleStorageFileName}` + fs.writeFileSync(localTmpFilePath, data) + return uploadFileToGoogleStorage( + localTmpFilePath, + googleStorageBucketName, + googleStorageFileName, + makeFileWorldReadable, + contentType + ) +} + // TODO(yerdua): make this communicate or handle auth issues reasonably. Ideally, // it should catch an auth error and tell the user to login with `gcloud auth login`. // So, if you run into an error that says something about being unauthorized, diff --git a/packages/celotool/src/lib/vm-testnet-utils.ts b/packages/celotool/src/lib/vm-testnet-utils.ts index 156c068f44a..91ccf962abf 100644 --- a/packages/celotool/src/lib/vm-testnet-utils.ts +++ b/packages/celotool/src/lib/vm-testnet-utils.ts @@ -1,4 +1,3 @@ -import { writeFileSync } from 'fs' import sleep from 'sleep-promise' import { confirmAction, envVar, fetchEnv, fetchEnvOrFallback } from './env-utils' import { @@ -20,12 +19,7 @@ import { TerraformVars, untaintTerraformModuleResource, } from './terraform' -import { - uploadEnvFileToGoogleStorage, - uploadFileToGoogleStorage, - uploadGenesisBlockToGoogleStorage, - uploadStaticNodesToGoogleStorage, -} from './testnet-utils' +import { uploadDataToGoogleStorage, uploadTestnetInfoToGoogleStorage } from './testnet-utils' import { execCmd } from './utils' // Keys = gcloud project name @@ -125,10 +119,7 @@ export async function deploy( await generateAndUploadSecrets(celoEnv) } }) - - await uploadGenesisBlockToGoogleStorage(celoEnv) - await uploadStaticNodesToGoogleStorage(celoEnv) - await uploadEnvFileToGoogleStorage(celoEnv) + await uploadTestnetInfoToGoogleStorage(celoEnv) } async function deployModule( @@ -340,11 +331,9 @@ export async function generateAndUploadSecrets(celoEnv: string) { } function uploadSecrets(celoEnv: string, secrets: string, resourceName: string) { - const localTmpFilePath = `/tmp/${celoEnv}-${resourceName}-secrets` - writeFileSync(localTmpFilePath, secrets) const cloudStorageFileName = `${secretsBasePath(celoEnv)}/.env.${resourceName}` - return uploadFileToGoogleStorage( - localTmpFilePath, + return uploadDataToGoogleStorage( + secrets, secretsBucketName(), cloudStorageFileName, false, diff --git a/packages/docs/getting-started/running-a-full-node.md b/packages/docs/getting-started/running-a-full-node.md index 03a5df4b04f..3f43e6ba6b8 100644 --- a/packages/docs/getting-started/running-a-full-node.md +++ b/packages/docs/getting-started/running-a-full-node.md @@ -74,10 +74,10 @@ The genesis block is the first block in the chain, and is specific to each netwo docker run -v $PWD:/root/.celo --rm $CELO_IMAGE init /celo/genesis.json ``` -In order to allow the node to sync with the network, give it the address of existing nodes in the network: +In order to allow the node to sync with the network, get the enode URLs of the bootnodes: ```bash -docker run -v $PWD:/root/.celo --rm --entrypoint cp $CELO_IMAGE /celo/static-nodes.json /root/.celo/ +export BOOTNODE_ENODES=`docker run --rm --entrypoint cat $CELO_IMAGE /celo/bootnodes` ``` ## Start the node @@ -85,7 +85,7 @@ docker run -v $PWD:/root/.celo --rm --entrypoint cp $CELO_IMAGE /celo/static-nod This command specifies the settings needed to run the node, and gets it started. ```bash -docker run --name celo-fullnode -d --restart always -p 127.0.0.1:8545:8545 -p 127.0.0.1:8546:8546 -p 30303:30303 -p 30303:30303/udp -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --rpc --rpcaddr 0.0.0.0 --rpcapi eth,net,web3,debug,admin,personal --lightserv 90 --lightpeers 1000 --maxpeers 1100 --etherbase $CELO_ACCOUNT_ADDRESS +docker run --name celo-fullnode -d --restart always -p 127.0.0.1:8545:8545 -p 127.0.0.1:8546:8546 -p 30303:30303 -p 30303:30303/udp -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --rpc --rpcaddr 0.0.0.0 --rpcapi eth,net,web3,debug,admin,personal --lightserv 90 --lightpeers 1000 --maxpeers 1100 --etherbase $CELO_ACCOUNT_ADDRESS --bootnodes $BOOTNODE_ENODES ``` You'll start seeing some output. There may be some errors or warnings that are ignorable. After a few minutes, you should see lines that look like this. This means your node has synced with the network and is receiving blocks. diff --git a/packages/docs/getting-started/running-a-validator.md b/packages/docs/getting-started/running-a-validator.md index 04e839fddc8..182c16784a0 100644 --- a/packages/docs/getting-started/running-a-validator.md +++ b/packages/docs/getting-started/running-a-validator.md @@ -245,7 +245,7 @@ We'll get back to this machine later, but for now, let's give it a proxy. ### Deploy a proxy -To avoid exposing the validator to the public internet, we are deploying a proxy node which is responsible to communicate with the network. On our Proxy machine, we'll setup the node as per usual now: +To avoid exposing the validator to the public internet, we are deploying a proxy node which is responsible to communicate with the network. On our Proxy machine, we'll set up the node and get the bootnode enode URLs to use for discovering other nodes. ```bash # On the proxy machine @@ -254,7 +254,7 @@ export CELO_IMAGE=us.gcr.io/celo-testnet/celo-node:baklava mkdir celo-proxy-node cd celo-proxy-node docker run -v $PWD:/root/.celo --rm -it $CELO_IMAGE init /celo/genesis.json -docker run -v $PWD:/root/.celo --rm -it --entrypoint cp $CELO_IMAGE /celo/static-nodes.json /root/.celo/ +export BOOTNODE_ENODES=`docker run --rm --entrypoint cat $CELO_IMAGE /celo/bootnodes` ``` You can then run the proxy with the following command. Be sure to replace `` with the name you'd like to use for your Validator account. @@ -264,7 +264,7 @@ You can then run the proxy with the following command. Be sure to replace ` -docker run --name celo-proxy -it --restart always -p 30303:30303 -p 30303:30303/udp -p 30503:30503 -p 30503:30503/udp -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --proxy.proxy --proxy.proxiedvalidatoraddress $CELO_VALIDATOR_SIGNER_ADDRESS --proxy.internalendpoint :30503 --etherbase $CELO_VALIDATOR_SIGNER_ADDRESS --ethstats=-proxy@baklava-ethstats.celo-testnet.org +docker run --name celo-proxy -it --restart always -p 30303:30303 -p 30303:30303/udp -p 30503:30503 -p 30503:30503/udp -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --proxy.proxy --proxy.proxiedvalidatoraddress $CELO_VALIDATOR_SIGNER_ADDRESS --proxy.internalendpoint :30503 --etherbase $CELO_VALIDATOR_SIGNER_ADDRESS --bootnodes $BOOTNODE_ENODES --ethstats=-proxy@baklava-ethstats.celo-testnet.org ``` {% hint style="info" %} @@ -277,7 +277,7 @@ Once the proxy is running, we will need to retrieve its enode and IP address so ```bash # On the proxy machine, retrieve the proxy enode -echo $(docker exec celo-proxy geth --exec "admin.nodeInfo['enode'].split('//')[1].split('@')[0]" attach | tr -d '"') +docker exec celo-proxy geth --exec "admin.nodeInfo['enode'].split('//')[1].split('@')[0]" attach | tr -d '"' ``` Now we need to set the proxy enode and proxy IP address in environment variables on the validator machine. @@ -532,7 +532,7 @@ export CELO_VALIDATOR_ADDRESS= mkdir celo-attestations-node cd celo-attestations-node docker run -v $PWD:/root/.celo --rm -it $CELO_IMAGE init /celo/genesis.json -docker run -v $PWD:/root/.celo --rm -it --entrypoint cp $CELO_IMAGE /celo/static-nodes.json /root/.celo/ +export BOOTNODE_ENODES=`docker run --rm --entrypoint cat $CELO_IMAGE /celo/bootnodes` docker run -v $PWD:/root/.celo --rm -it $CELO_IMAGE account new export CELO_ATTESTATION_SIGNER_ADDRESS= ``` @@ -558,7 +558,7 @@ You can now run the node for the attestation service in the background. In the b ```bash # On the Attestation machine echo > .password -docker run --name celo-attestations -it --restart always -p 8545:8545 -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --rpc --rpcaddr 0.0.0.0 --rpcapi eth,net,web3,debug,admin --unlock $CELO_ATTESTATION_SIGNER_ADDRESS --password /root/.celo/.password +docker run --name celo-attestations -it --restart always -p 8545:8545 -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --rpc --rpcaddr 0.0.0.0 --rpcapi eth,net,web3,debug,admin --unlock $CELO_ATTESTATION_SIGNER_ADDRESS --password /root/.celo/.password --bootnodes $BOOTNODE_ENODES ``` Next we will set up the Attestation Service itself. First, specify the following environment variables: diff --git a/packages/terraform-modules/testnet/outputs.tf b/packages/terraform-modules/testnet/outputs.tf index 552d37c2320..d2d163423c6 100644 --- a/packages/terraform-modules/testnet/outputs.tf +++ b/packages/terraform-modules/testnet/outputs.tf @@ -1,3 +1,7 @@ +output bootnode_ip_address { + value = module.bootnode.ip_address +} + output tx_node_internal_ip_addresses { value = module.tx_node.internal_ip_addresses }