From 6ef3735c18931f2e0de9e77e67fc852fd84a3c86 Mon Sep 17 00:00:00 2001 From: Asa Oines Date: Tue, 12 Nov 2019 12:17:09 -0800 Subject: [PATCH] Remove integration sync test --- .circleci/config.yml | 17 --- .../celotool/ci_test_sync_with_network.sh | 62 ---------- .../celotool/geth_tests/network_sync_test.sh | 51 -------- .../e2e-tests/verify_ultralight_geth_logs.ts | 109 ------------------ 4 files changed, 239 deletions(-) delete mode 100755 packages/celotool/ci_test_sync_with_network.sh delete mode 100755 packages/celotool/geth_tests/network_sync_test.sh delete mode 100644 packages/celotool/src/e2e-tests/verify_ultralight_geth_logs.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 97a9a7acd55..d208c4db5b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -583,23 +583,6 @@ jobs: cd packages/celotool ./ci_test_sync.sh checkout master - end-to-end-geth-integration-sync-test: - <<: *e2e-defaults - steps: - - attach_workspace: - at: ~/app - - run: - name: Check if the test should run - command: | - FILES_TO_CHECK="${PWD}/packages/celotool,${PWD}/packages/protocol,${PWD}/.circleci/config.yml" - ./scripts/ci_check_if_test_should_run_v2.sh ${FILES_TO_CHECK} - - run: - name: Run test - command: | - set -e - cd packages/celotool - ./ci_test_sync_with_network.sh checkout master - end-to-end-geth-attestations-test: <<: *e2e-defaults resource_class: medium+ diff --git a/packages/celotool/ci_test_sync_with_network.sh b/packages/celotool/ci_test_sync_with_network.sh deleted file mode 100755 index 99645d25d71..00000000000 --- a/packages/celotool/ci_test_sync_with_network.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# This test starts a local node which tries to sync with remotely running nodes and -# verifies that the sync works. - -# For testing a particular commit hash of Geth repo (usually, on Circle CI) -# Usage: ci_test_sync_with_network.sh checkout -# For testing the local Geth dir (usually, for manual testing) -# Usage: ci_test_sync_with_network.sh local - -if [ "${1}" == "checkout" ]; then - export GETH_DIR="/tmp/geth" - # Test master by default. - COMMIT_HASH_TO_TEST=${2:-"master"} - echo "Checking out geth at commit hash ${COMMIT_HASH_TO_TEST}..." - # Shallow clone up to depth of 20. If the COMMIT_HASH_TO_TEST is not within the last 20 hashes then - # this test will fail. This will force someone to keep updating the COMMIT_HASH_TO_TEST we are - # testing. Clone up to 20 takes about 4 seconds on my machine and a full clone is - # about 60 seconds as of May 20, 2019. The difference will only grow over time. - git clone --depth 20 https://github.com/celo-org/celo-blockchain.git ${GETH_DIR} && cd ${GETH_DIR} && git checkout ${COMMIT_HASH_TO_TEST} && cd - -elif [ "${1}" == "local" ]; then - export GETH_DIR="${2}" - echo "Testing using local geth dir ${GETH_DIR}..." -fi - -# For now, the script assumes that it runs from a sub-dir of sub-dir of monorepo directory. -CELO_MONOREPO_DIR="${PWD}/../.." -# Assume that the logs are in /tmp/geth_stdout -GETH_LOG_FILE=/tmp/geth_stdout - -# usage: test_ultralight_sync -test_ultralight_sync () { - NETWORK_NAME=$1 - echo "Testing ultralight sync with '${NETWORK_NAME}' network" - # Run the sync in ultralight mode - geth_tests/network_sync_test.sh ${NETWORK_NAME} ultralight - # Verify what happened by reading the logs. - ${CELO_MONOREPO_DIR}/node_modules/.bin/mocha -r ts-node/register ${CELO_MONOREPO_DIR}/packages/celotool/src/e2e-tests/verify_ultralight_geth_logs.ts --network "${NETWORK_NAME}" --gethlogfile ${GETH_LOG_FILE} -} - -# Some code in celotool requires this file to contain the MNEMONOIC. -# The value of MNEMONOIC does not matter. -if [[ ! -e ${CELO_MONOREPO_DIR}/.env.mnemonic ]]; then - echo "MNEMONOIC=anything random" > ${CELO_MONOREPO_DIR}/.env.mnemonic -fi - -# Test syncing -export NETWORK_NAME="integration" -# Add an extra echo at the end to dump a new line, this makes the results a bit more readable. -geth_tests/network_sync_test.sh ${NETWORK_NAME} full && echo -# This is broken, I am not sure why, therefore, commented for now. -# geth_tests/network_sync_test.sh ${NETWORK_NAME} fast && echo -geth_tests/network_sync_test.sh ${NETWORK_NAME} light && echo -test_ultralight_sync ${NETWORK_NAME} && echo - -export NETWORK_NAME="alfajoresstaging" -geth_tests/network_sync_test.sh ${NETWORK_NAME} full && echo -# This is broken, I am not sure why, therefore, commented for now. -# geth_tests/network_sync_test.sh ${NETWORK_NAME} fast && echo -geth_tests/network_sync_test.sh ${NETWORK_NAME} light && echo -test_ultralight_sync ${NETWORK_NAME} && echo diff --git a/packages/celotool/geth_tests/network_sync_test.sh b/packages/celotool/geth_tests/network_sync_test.sh deleted file mode 100755 index 2997b623f55..00000000000 --- a/packages/celotool/geth_tests/network_sync_test.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Usage: geth_tests/integration_network_sync_test.sh [network name] [sync mode] -# Default to testing the integration network -NETWORK_NAME=${1:-"integration"} -# Default to testing the full sync mode -SYNCMODE=${2:-"full"} - -echo "This test will start a local node in '${SYNCMODE}' sync mode which will connect to network '${NETWORK_NAME}' and verify that syncing works" - -echo "Setting constants..." -# For now, the script assumes that it runs from a sub-dir of sub-dir of monorepo directory. -CELO_MONOREPO_DIR="${PWD}/../.." - -DATA_DIR="/tmp/tmp1" -GENESIS_FILE_PATH="/tmp/genesis_ibft.json" - -GETH_BINARY="${GETH_DIR}/build/bin/geth --datadir ${DATA_DIR}" -CELOTOOLJS="${CELO_MONOREPO_DIR}/packages/celotool/bin/celotooljs.sh" - -curl "https://www.googleapis.com/storage/v1/b/genesis_blocks/o/${NETWORK_NAME}?alt=media" --output ${GENESIS_FILE_PATH} - -${CELOTOOLJS} geth build --geth-dir ${GETH_DIR} - -rm -rf ${DATA_DIR} -${GETH_BINARY} init ${GENESIS_FILE_PATH} 1>/dev/null 2>/dev/null -curl "https://www.googleapis.com/storage/v1/b/static_nodes/o/${NETWORK_NAME}?alt=media" --output ${DATA_DIR}/static-nodes.json - -echo "Running geth in the background..." -LOG_FILE="/tmp/geth_stdout" -# Run geth in the background -${CELOTOOLJS} geth run \ - --geth-dir ${GETH_DIR} \ - --data-dir ${DATA_DIR} \ - --sync-mode ${SYNCMODE} 1>${LOG_FILE} 2>/tmp/geth_stderr & -# let it sync -sleep 20 -latestBlock=$(${GETH_BINARY} attach -exec eth.blockNumber) -echo "Latest block number is ${latestBlock}" - -pkill -9 geth - -if [ "$latestBlock" -eq "0" ]; then - echo "Sync is not working with network '${NETWORK_NAME}' in mode '${SYNCMODE}', see logs in ${LOG_FILE}" - if test ${CI}; then - echo "Running on CI, dumping logs from ${LOG_FILE}..." - cat ${LOG_FILE} - fi - exit 1 -fi diff --git a/packages/celotool/src/e2e-tests/verify_ultralight_geth_logs.ts b/packages/celotool/src/e2e-tests/verify_ultralight_geth_logs.ts deleted file mode 100644 index 0d79f7b608b..00000000000 --- a/packages/celotool/src/e2e-tests/verify_ultralight_geth_logs.ts +++ /dev/null @@ -1,109 +0,0 @@ -import GenesisBlockUtils from '@celo/walletkit/lib/src/genesis-block-utils' -import { equal, notEqual } from 'assert' -import * as fs from 'fs' - -// These tests read logs from a client which was running in Ultralight sync mode and verifies that -// only epoch headers are fetched till the height block and all headers are fetched afrerwards. -describe('Ultralight client', () => { - let epoch: number - - before(async () => { - const genesis = JSON.parse(await GenesisBlockUtils.getGenesisBlockAsync(argv.network)) - if (genesis.config.istanbul.epoch) { - epoch = Number(genesis.config.istanbul.epoch) - } else { - throw Error('epoch not found in genesis block') - } - }) - - beforeEach(function(this: any) { - this.timeout(0) - }) - - const argv = require('minimist')(process.argv.slice(2)) - const logfile = argv.gethlogfile - - let origin: number = -1 - let height: number = 0 - const insertedHeaderNumbers: number[] = [] - - console.debug('Reading logs from ' + logfile) - const fileContents = fs.readFileSync(logfile, 'utf8') - - // Fetch origin - const originInfo = fileContents.match('After the check origin is \\d+') - if (originInfo === null) { - throw Error('Origin is null') - } - const arr1 = originInfo[0].split(' ') - origin = parseInt(arr1[arr1.length - 1], 10) - console.debug('origin is ' + origin) - - // Fetch height - const heightInfo = fileContents.match('height is \\d+') - if (heightInfo === null) { - throw Error('Height is null') - } - const arr2 = heightInfo[0].split(' ') - height = parseInt(arr2[arr2.length - 1], 10) - console.debug('Height is ' + height) - - // Fetch all inserted headers - const insertedHeadersInfo = fileContents.match( - new RegExp('Inserted new header.*?number=\\d+', 'g') - ) - if (insertedHeadersInfo === null) { - throw Error('insertedHeadersInfo is null') - } - insertedHeadersInfo.forEach((insertedHeader) => { - const arr3 = insertedHeader.split('=') - const headerNumber = parseInt(arr3[arr3.length - 1], 10) - console.debug('Inserted header is ' + headerNumber) - insertedHeaderNumbers.push(headerNumber) - }) - - it('sync must start from 0', () => { - equal(origin, 0, 'Start header is not zero, it is ' + origin) - }) - - it('latest known header must be non-zero', () => { - notEqual(height, 0, 'Latest known header is zero') - }) - - it('height header must be fetched', () => { - let heightHeaderFetched: boolean = false - for (const headerNumber of insertedHeaderNumbers) { - if (headerNumber === height) { - heightHeaderFetched = true - break - } - } - equal(heightHeaderFetched, true, 'height header ' + height + ' not fetched') - }) - - it('must only download epoch blocks till height', () => { - for (const headerNumber of insertedHeaderNumbers) { - if (headerNumber < height) { - equal(headerNumber % epoch, 0, 'Non-epoch header below height fetched') - } - } - }) - - it('must fetch all headers after height', () => { - for ( - let i = insertedHeaderNumbers.length - 1; - i >= 0 && insertedHeaderNumbers[i] > height; - i++ - ) { - equal( - insertedHeaderNumbers[i] - insertedHeaderNumbers[i - 1], - 1, - 'Header(s) between ' + - insertedHeaderNumbers[i] + - ' and ' + - insertedHeaderNumbers[i - 1] + - ' are missing' - ) - } - }) -})