Upload bootnode info & use in docs instead of static nodes (#2289)

packages/celotool/src/cmds/deploy/initial/testnet.ts

@@ -1,11 +1,7 @@
 import { createClusterIfNotExists, setupCluster, switchToClusterFromEnv } from 'src/lib/cluster'
 import { failIfVmBased } from 'src/lib/env-utils'
 import { createStaticIPs, installHelmChart, pollForBootnodeLoadBalancer } from 'src/lib/helm_deploy'
-import {
-  uploadEnvFileToGoogleStorage,
-  uploadGenesisBlockToGoogleStorage,
-  uploadStaticNodesToGoogleStorage,
-} from 'src/lib/testnet-utils'
+import { uploadTestnetInfoToGoogleStorage } from 'src/lib/testnet-utils'
 import yargs from 'yargs'
 import { InitialArgv } from '../../deploy/initial'
 
@@ -39,7 +35,5 @@ export const handler = async (argv: TestnetInitialArgv) => {
   // When using an external bootnode, we have to await the bootnode's LB to be up first
   await pollForBootnodeLoadBalancer(argv.celoEnv)
 
-  await uploadGenesisBlockToGoogleStorage(argv.celoEnv)
-  await uploadStaticNodesToGoogleStorage(argv.celoEnv)
-  await uploadEnvFileToGoogleStorage(argv.celoEnv)
+  await uploadTestnetInfoToGoogleStorage(argv.celoEnv)
 }

packages/celotool/src/lib/geth.ts

@@ -103,12 +103,21 @@ const getExternalEnodeAddresses = async (namespace: string) => {
 }
 
 export const getBootnodeEnode = async (namespace: string) => {
-  const ip = await retrieveIPAddress(`${namespace}-bootnode`)
+  const ip = await retrieveBootnodeIPAddress(namespace)
   const privateKey = generatePrivateKey(fetchEnv(envVar.MNEMONIC), AccountType.BOOTNODE, 0)
   const nodeId = privateKeyToPublicKey(privateKey)
   return [getEnodeAddress(nodeId, ip, DISCOVERY_PORT)]
 }
 
+const retrieveBootnodeIPAddress = async (namespace: string) => {
+  if (isVmBased()) {
+    const outputs = await getTestnetOutputs(namespace)
+    return outputs.bootnode_ip_address.value
+  } else {
+    return retrieveIPAddress(`${namespace}-bootnode`)
+  }
+}
+
 const retrieveTxNodeAddresses = async (namespace: string, txNodesNum: number) => {
   if (isVmBased()) {
     const outputs = await getTestnetOutputs(namespace)

packages/celotool/src/lib/helm_deploy.ts

@@ -427,7 +427,8 @@ export async function pollForBootnodeLoadBalancer(celoEnv: string) {
     await sleep(LOAD_BALANCER_POLL_INTERVAL)
   }
 
-  await sleep(1000 * 60 * 5)
+  console.info('Sleeping 1 minute...')
+  await sleep(1000 * 60) // 1 minute
 
   console.info(`\nReset all pods now that the bootnode load balancer has provisioned`)
   await execCmdWithExitOnFailure(`kubectl delete pod -n ${celoEnv} --selector=component=validators`)

packages/celotool/src/lib/testnet-utils.ts

@@ -8,23 +8,30 @@ import { getGenesisGoogleStorageUrl } from './endpoints'
 import { getEnvFile } from './env-utils'
 import { ensureAuthenticatedGcloudAccount } from './gcloud_utils'
 import { generateGenesisFromEnv } from './generate_utils'
-import { getEnodesWithExternalIPAddresses } from './geth'
+import { getBootnodeEnode, getEnodesWithExternalIPAddresses } from './geth'
 import { execCmdWithExitOnFailure } from './utils'
 
 const genesisBlocksBucketName = GenesisBlocksGoogleStorageBucketName
 const staticNodesBucketName = StaticNodeUtils.getStaticNodesGoogleStorageBucketName()
 // Someone has taken env_files and I don't even has permission to modify it :/
 // See files in this bucket using `$ gsutil ls gs://env_config_files`
 const envBucketName = 'env_config_files'
+const bootnodesBucketName = 'env_bootnodes'
+
+// uploads genesis block, static nodes, env file, and bootnode to GCS
+export async function uploadTestnetInfoToGoogleStorage(networkName: string) {
+  await uploadGenesisBlockToGoogleStorage(networkName)
+  await uploadStaticNodesToGoogleStorage(networkName)
+  await uploadBootnodeToGoogleStorage(networkName)
+  await uploadEnvFileToGoogleStorage(networkName)
+}
 
 export async function uploadGenesisBlockToGoogleStorage(networkName: string) {
   console.info(`\nUploading genesis block for ${networkName} to Google cloud storage`)
   const genesisBlockJsonData = generateGenesisFromEnv()
   console.debug(`Genesis block is ${genesisBlockJsonData} \n`)
-  const localTmpFilePath = `/tmp/${networkName}_genesis-block`
-  fs.writeFileSync(localTmpFilePath, genesisBlockJsonData)
-  await uploadFileToGoogleStorage(
-    localTmpFilePath,
+  await uploadDataToGoogleStorage(
+    genesisBlockJsonData,
     genesisBlocksBucketName,
     networkName,
     true,
@@ -41,38 +48,39 @@ export async function getGenesisBlockFromGoogleStorage(networkName: string) {
 export async function uploadStaticNodesToGoogleStorage(networkName: string) {
   console.info(`\nUploading static nodes for ${networkName} to Google cloud storage...`)
   // Get node json file
-  let nodesJsonData: string | null = null
-  const numAttempts = 100
-  for (let i = 1; i <= numAttempts; i++) {
-    try {
-      nodesJsonData = JSON.stringify(await getEnodesWithExternalIPAddresses(networkName))
-      break
-    } catch (error) {
-      const sleepTimeBasisInMs = 1000
-      const sleepTimeInMs = sleepTimeBasisInMs * Math.pow(2, i)
-      console.warn(
-        `${new Date().toLocaleTimeString()} Failed to get static nodes information, attempt: ${i}/${numAttempts}, ` +
-          `retry after sleeping for ${sleepTimeInMs} milli-seconds`,
-        error
-      )
-      await sleep(sleepTimeInMs)
-    }
-  }
+  const nodesJsonData: string[] | null = await retryCmd(() =>
+    getEnodesWithExternalIPAddresses(networkName)
+  )
   if (nodesJsonData === null) {
     throw new Error('Fail to get static nodes information')
   }
   console.debug('Static nodes are ' + nodesJsonData + '\n')
-  const localTmpFilePath = `/tmp/${networkName}_static-nodes`
-  fs.writeFileSync(localTmpFilePath, nodesJsonData)
-  await uploadFileToGoogleStorage(
-    localTmpFilePath,
+  await uploadDataToGoogleStorage(
+    nodesJsonData,
     staticNodesBucketName,
     networkName,
     true,
     'application/json'
   )
 }
 
+export async function uploadBootnodeToGoogleStorage(networkName: string) {
+  console.info(`\nUploading bootnode for ${networkName} to Google Cloud Storage...`)
+  const [bootnodeEnode] = await retryCmd(() => getBootnodeEnode(networkName))
+  if (!bootnodeEnode) {
+    throw new Error('Failed to get bootnode enode')
+  }
+  // for now there is always only one bootnodde
+  console.info('Bootnode enode:', bootnodeEnode)
+  await uploadDataToGoogleStorage(
+    bootnodeEnode,
+    bootnodesBucketName,
+    networkName,
+    true, // make it public
+    'text/plain'
+  )
+}
+
 export async function uploadEnvFileToGoogleStorage(networkName: string) {
   const envFileName = getEnvFile(networkName)
   const userInfo = `${await getGoogleCloudUserInfo()}`
@@ -90,17 +98,38 @@ export async function uploadEnvFileToGoogleStorage(networkName: string) {
     `# Last modified on on ${Date()}\n` +
     `# Base commit: "https://github.com/${repo}/commit/${commitHash}"\n`
   const fullData = metaData + '\n' + envFileData
-  const localTmpFilePath = `/tmp/${networkName}_env-file`
-  fs.writeFileSync(localTmpFilePath, fullData)
-  await uploadFileToGoogleStorage(
-    localTmpFilePath,
+  await uploadDataToGoogleStorage(
+    fullData,
     envBucketName,
     networkName,
     false /* keep file private */,
     'text/plain'
   )
 }
 
+async function retryCmd(
+  cmd: () => Promise<any>,
+  numAttempts: number = 100,
+  maxTimeoutMs: number = 15000
+) {
+  for (let i = 1; i <= numAttempts; i++) {
+    try {
+      const result = await cmd()
+      return result
+    } catch (error) {
+      const sleepTimeBasisInMs = 1000
+      const sleepTimeInMs = Math.min(sleepTimeBasisInMs * Math.pow(2, i), maxTimeoutMs)
+      console.warn(
+        `${new Date().toLocaleTimeString()} Retry attempt: ${i}/${numAttempts}, ` +
+          `retry after sleeping for ${sleepTimeInMs} milli-seconds`,
+        error
+      )
+      await sleep(sleepTimeInMs)
+    }
+  }
+  return null
+}
+
 async function getGoogleCloudUserInfo(): Promise<string> {
   const cmd = 'gcloud config get-value account'
   const stdout = (await execCmdWithExitOnFailure(cmd))[0]
@@ -123,6 +152,25 @@ async function getCommitHash(): Promise<string> {
   return stdout.split(' ')[1].trim()
 }
 
+// Writes data to a temporary file & uploads it to GCS
+export function uploadDataToGoogleStorage(
+  data: any,
+  googleStorageBucketName: string,
+  googleStorageFileName: string,
+  makeFileWorldReadable: boolean,
+  contentType: string
+) {
+  const localTmpFilePath = `/tmp/${googleStorageBucketName}-${googleStorageFileName}`
+  fs.writeFileSync(localTmpFilePath, data)
+  return uploadFileToGoogleStorage(
+    localTmpFilePath,
+    googleStorageBucketName,
+    googleStorageFileName,
+    makeFileWorldReadable,
+    contentType
+  )
+}
+
 // TODO(yerdua): make this communicate or handle auth issues reasonably. Ideally,
 //   it should catch an auth error and tell the user to login with `gcloud auth login`.
 //   So, if you run into an error that says something about being unauthorized,

packages/celotool/src/lib/vm-testnet-utils.ts

@@ -1,4 +1,3 @@
-import { writeFileSync } from 'fs'
 import sleep from 'sleep-promise'
 import { confirmAction, envVar, fetchEnv, fetchEnvOrFallback } from './env-utils'
 import {
@@ -20,12 +19,7 @@ import {
   TerraformVars,
   untaintTerraformModuleResource,
 } from './terraform'
-import {
-  uploadEnvFileToGoogleStorage,
-  uploadFileToGoogleStorage,
-  uploadGenesisBlockToGoogleStorage,
-  uploadStaticNodesToGoogleStorage,
-} from './testnet-utils'
+import { uploadDataToGoogleStorage, uploadTestnetInfoToGoogleStorage } from './testnet-utils'
 import { execCmd } from './utils'
 
 // Keys = gcloud project name
@@ -125,10 +119,7 @@ export async function deploy(
       await generateAndUploadSecrets(celoEnv)
     }
   })
-
-  await uploadGenesisBlockToGoogleStorage(celoEnv)
-  await uploadStaticNodesToGoogleStorage(celoEnv)
-  await uploadEnvFileToGoogleStorage(celoEnv)
+  await uploadTestnetInfoToGoogleStorage(celoEnv)
 }
 
 async function deployModule(
@@ -340,11 +331,9 @@ export async function generateAndUploadSecrets(celoEnv: string) {
 }
 
 function uploadSecrets(celoEnv: string, secrets: string, resourceName: string) {
-  const localTmpFilePath = `/tmp/${celoEnv}-${resourceName}-secrets`
-  writeFileSync(localTmpFilePath, secrets)
   const cloudStorageFileName = `${secretsBasePath(celoEnv)}/.env.${resourceName}`
-  return uploadFileToGoogleStorage(
-    localTmpFilePath,
+  return uploadDataToGoogleStorage(
+    secrets,
     secretsBucketName(),
     cloudStorageFileName,
     false,

packages/docs/getting-started/running-a-full-node.md

@@ -74,18 +74,18 @@ The genesis block is the first block in the chain, and is specific to each netwo
 docker run -v $PWD:/root/.celo --rm $CELO_IMAGE init /celo/genesis.json
 ```
 
-In order to allow the node to sync with the network, give it the address of existing nodes in the network:
+In order to allow the node to sync with the network, get the enode URLs of the bootnodes:
 
 ```bash
-docker run -v $PWD:/root/.celo --rm --entrypoint cp $CELO_IMAGE /celo/static-nodes.json /root/.celo/
+export BOOTNODE_ENODES=`docker run --rm --entrypoint cat $CELO_IMAGE /celo/bootnodes`
 ```
 
 ## Start the node
 
 This command specifies the settings needed to run the node, and gets it started.
 
 ```bash
-docker run --name celo-fullnode -d --restart always -p 127.0.0.1:8545:8545 -p 127.0.0.1:8546:8546 -p 30303:30303 -p 30303:30303/udp -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --rpc --rpcaddr 0.0.0.0 --rpcapi eth,net,web3,debug,admin,personal --lightserv 90 --lightpeers 1000 --maxpeers 1100 --etherbase $CELO_ACCOUNT_ADDRESS
+docker run --name celo-fullnode -d --restart always -p 127.0.0.1:8545:8545 -p 127.0.0.1:8546:8546 -p 30303:30303 -p 30303:30303/udp -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --rpc --rpcaddr 0.0.0.0 --rpcapi eth,net,web3,debug,admin,personal --lightserv 90 --lightpeers 1000 --maxpeers 1100 --etherbase $CELO_ACCOUNT_ADDRESS --bootnodes $BOOTNODE_ENODES
 ```
 
 You'll start seeing some output. There may be some errors or warnings that are ignorable. After a few minutes, you should see lines that look like this. This means your node has synced with the network and is receiving blocks.

packages/docs/getting-started/running-a-validator.md

@@ -245,7 +245,7 @@ We'll get back to this machine later, but for now, let's give it a proxy.
 
 ### Deploy a proxy
 
-To avoid exposing the validator to the public internet, we are deploying a proxy node which is responsible to communicate with the network. On our Proxy machine, we'll setup the node as per usual now:
+To avoid exposing the validator to the public internet, we are deploying a proxy node which is responsible to communicate with the network. On our Proxy machine, we'll set up the node and get the bootnode enode URLs to use for discovering other nodes.
 
 ```bash
 # On the proxy machine
@@ -254,7 +254,7 @@ export CELO_IMAGE=us.gcr.io/celo-testnet/celo-node:baklava
 mkdir celo-proxy-node
 cd celo-proxy-node
 docker run -v $PWD:/root/.celo --rm -it $CELO_IMAGE init /celo/genesis.json
-docker run -v $PWD:/root/.celo --rm -it --entrypoint cp $CELO_IMAGE /celo/static-nodes.json /root/.celo/
+export BOOTNODE_ENODES=`docker run --rm --entrypoint cat $CELO_IMAGE /celo/bootnodes`
 ```
 
 You can then run the proxy with the following command. Be sure to replace `<YOUR-VALIDATOR-NAME>` with the name you'd like to use for your Validator account.
@@ -264,7 +264,7 @@ You can then run the proxy with the following command. Be sure to replace `<YOUR
 # Note that you'll have to export CELO_VALIDATOR_SIGNER_ADDRESS and $NETWORK_ID on this machine
 export NETWORK_ID=121119
 export CELO_VALIDATOR_SIGNER_ADDRESS=<YOUR-VALIDATOR-SIGNER-ADDRESS>
-docker run --name celo-proxy -it --restart always -p 30303:30303 -p 30303:30303/udp -p 30503:30503 -p 30503:30503/udp -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --proxy.proxy --proxy.proxiedvalidatoraddress $CELO_VALIDATOR_SIGNER_ADDRESS --proxy.internalendpoint :30503 --etherbase $CELO_VALIDATOR_SIGNER_ADDRESS --ethstats=<YOUR-VALIDATOR-NAME>-proxy@baklava-ethstats.celo-testnet.org
+docker run --name celo-proxy -it --restart always -p 30303:30303 -p 30303:30303/udp -p 30503:30503 -p 30503:30503/udp -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --proxy.proxy --proxy.proxiedvalidatoraddress $CELO_VALIDATOR_SIGNER_ADDRESS --proxy.internalendpoint :30503 --etherbase $CELO_VALIDATOR_SIGNER_ADDRESS --bootnodes $BOOTNODE_ENODES --ethstats=<YOUR-VALIDATOR-NAME>-proxy@baklava-ethstats.celo-testnet.org
 ```
 
 {% hint style="info" %}
@@ -277,7 +277,7 @@ Once the proxy is running, we will need to retrieve its enode and IP address so
 
 ```bash
 # On the proxy machine, retrieve the proxy enode
-echo $(docker exec celo-proxy geth --exec "admin.nodeInfo['enode'].split('//')[1].split('@')[0]" attach | tr -d '"')
+docker exec celo-proxy geth --exec "admin.nodeInfo['enode'].split('//')[1].split('@')[0]" attach | tr -d '"'
 ```
 
 Now we need to set the proxy enode and proxy IP address in environment variables on the validator machine.
@@ -532,7 +532,7 @@ export CELO_VALIDATOR_ADDRESS=<CELO_VALIDATOR_ADDRESS>
 mkdir celo-attestations-node
 cd celo-attestations-node
 docker run -v $PWD:/root/.celo --rm -it $CELO_IMAGE init /celo/genesis.json
-docker run -v $PWD:/root/.celo --rm -it --entrypoint cp $CELO_IMAGE /celo/static-nodes.json /root/.celo/
+export BOOTNODE_ENODES=`docker run --rm --entrypoint cat $CELO_IMAGE /celo/bootnodes`
 docker run -v $PWD:/root/.celo --rm -it $CELO_IMAGE account new
 export CELO_ATTESTATION_SIGNER_ADDRESS=<YOUR-ATTESTATION-SIGNER-ADDRESS>
 ```
@@ -558,7 +558,7 @@ You can now run the node for the attestation service in the background. In the b
 ```bash
 # On the Attestation machine
 echo <ATTESTATION-SIGNER-PASSWORD> > .password
-docker run --name celo-attestations -it --restart always -p 8545:8545 -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --rpc --rpcaddr 0.0.0.0 --rpcapi eth,net,web3,debug,admin --unlock $CELO_ATTESTATION_SIGNER_ADDRESS --password /root/.celo/.password
+docker run --name celo-attestations -it --restart always -p 8545:8545 -v $PWD:/root/.celo $CELO_IMAGE --verbosity 3 --networkid $NETWORK_ID --syncmode full --rpc --rpcaddr 0.0.0.0 --rpcapi eth,net,web3,debug,admin --unlock $CELO_ATTESTATION_SIGNER_ADDRESS --password /root/.celo/.password --bootnodes $BOOTNODE_ENODES
 ```
 
 Next we will set up the Attestation Service itself. First, specify the following environment variables:

packages/terraform-modules/testnet/outputs.tf

@@ -1,3 +1,7 @@
+output bootnode_ip_address {
+  value = module.bootnode.ip_address
+}
+
 output tx_node_internal_ip_addresses {
   value = module.tx_node.internal_ip_addresses
 }