diff --git a/.cicd/build.sh b/.cicd/build.sh new file mode 100755 index 00000000000..4acfd81ead7 --- /dev/null +++ b/.cicd/build.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -eo pipefail +. ./.cicd/helpers/general.sh +mkdir -p $BUILD_DIR +CMAKE_EXTRAS="-DBUILD_MONGO_DB_PLUGIN=true" +if [[ $(uname) == 'Darwin' ]]; then + # You can't use chained commands in execute + [[ $TRAVIS == true ]] && export PINNED=false && ccache -s && CMAKE_EXTRAS="-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" && ./$CICD_DIR/platforms/macos-10.14.sh + ( [[ ! $PINNED == false || $UNPINNED == true ]] ) && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make" + cd $BUILD_DIR + cmake $CMAKE_EXTRAS .. + make -j$JOBS +else # Linux + ARGS=${ARGS:-"--rm --init -v $(pwd):$MOUNTED_DIR"} + . $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/$IMAGE_TAG.dockerfile + PRE_COMMANDS="cd $MOUNTED_DIR/build" + # PRE_COMMANDS: Executed pre-cmake + # CMAKE_EXTRAS: Executed within and right before the cmake path (cmake CMAKE_EXTRAS ..) + [[ ! $IMAGE_TAG =~ 'unpinned' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$MOUNTED_DIR/.cicd/helpers/clang.make -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + if [[ $IMAGE_TAG == 'amazon_linux-2' ]]; then + PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH" + elif [[ $IMAGE_TAG == 'centos-7.6' ]]; then + PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH" + elif [[ $IMAGE_TAG == 'ubuntu-16.04' ]]; then + PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH" + elif [[ $IMAGE_TAG == 'ubuntu-18.04' ]]; then + PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH" + elif [[ $IMAGE_TAG == 'amazon_linux-2-unpinned' ]]; then + PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH" + CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" + elif [[ $IMAGE_TAG == 'centos-7.6-unpinned' ]]; then + PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && export PATH=/usr/lib64/ccache:\\\$PATH" + elif [[ $IMAGE_TAG == 'ubuntu-18.04-unpinned' ]]; then + PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH" + CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" + fi + BUILD_COMMANDS="cmake $CMAKE_EXTRAS -DCMAKE_BUILD_TYPE='Release' .. && make -j$JOBS" + # Docker Commands + if [[ $BUILDKITE == true ]]; then + # Generate Base Images + $CICD_DIR/generate-base-images.sh + [[ $ENABLE_INSTALL == true ]] && COMMANDS="cp -r $MOUNTED_DIR /root/eosio && cd /root/eosio/build &&" + COMMANDS="$COMMANDS $BUILD_COMMANDS" + [[ $ENABLE_INSTALL == true ]] && COMMANDS="$COMMANDS && make install" + elif [[ $TRAVIS == true ]]; then + ARGS="$ARGS -v /usr/lib/ccache -v $HOME/.ccache:/opt/.ccache -e JOBS -e TRAVIS -e CCACHE_DIR=/opt/.ccache" + COMMANDS="ccache -s && $BUILD_COMMANDS" + fi + COMMANDS="$PRE_COMMANDS && $COMMANDS" + echo "$ docker run $ARGS $(buildkite-intrinsics) $FULL_TAG bash -c \"$COMMANDS\"" + eval docker run $ARGS $(buildkite-intrinsics) $FULL_TAG bash -c \"$COMMANDS\" +fi \ No newline at end of file diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh new file mode 100755 index 00000000000..005e0f78907 --- /dev/null +++ b/.cicd/generate-base-images.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -eo pipefail +. ./.cicd/helpers/general.sh +. $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/${IMAGE_TAG}.dockerfile +# look for Docker image +echo "+++ :mag_right: Looking for $FULL_TAG" +ORG_REPO=$(echo $FULL_TAG | cut -d: -f1) +TAG=$(echo $FULL_TAG | cut -d: -f2) +EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG") +# build, if neccessary +if [[ $EXISTS =~ '404 page not found' || $EXISTS =~ 'manifest unknown' ]]; then # if we cannot pull the image, we build and push it first + docker build -t $FULL_TAG -f $CICD_DIR/platforms/${IMAGE_TAG}.dockerfile . + docker push $FULL_TAG +else + echo "$FULL_TAG already exists." +fi diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh new file mode 100755 index 00000000000..c085b8487bd --- /dev/null +++ b/.cicd/generate-pipeline.sh @@ -0,0 +1,366 @@ +#!/bin/bash +set -eo pipefail +. ./.cicd/helpers/general.sh +export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'} +export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'} +# Use files in platforms dir as source of truth for what platforms we need to generate steps for +export PLATFORMS_JSON_ARRAY='[]' +for FILE in $(ls $CICD_DIR/platforms); do + # Ability to skip mac or linux by not even creating the json block + ( [[ $SKIP_MAC == true ]] && [[ $FILE =~ 'macos' ]] ) && continue + ( [[ $SKIP_LINUX == true ]] && [[ ! $FILE =~ 'macos' ]] ) && continue + # Prevent using both platform files (only use unpinned or pinned) + if [[ $PINNED == false || $UNPINNED == true ]] && [[ ! $FILE =~ 'macos' ]]; then + export SKIP_CONTRACT_BUILDER=${SKIP_CONTRACT_BUILDER:-true} + export SKIP_PACKAGE_BUILDER=${SKIP_PACKAGE_BUILDER:-true} + [[ ! $FILE =~ 'unpinned' ]] && continue + else + [[ $FILE =~ 'unpinned' ]] && continue + fi + export FILE_NAME=$(echo $FILE | awk '{split($0,a,/\.(d|s)/); print a[1] }') + export PLATFORM_NAME=$(echo $FILE_NAME | cut -d- -f1 | sed 's/os/OS/g') + export PLATFORM_NAME_UPCASE=$(echo $PLATFORM_NAME | tr a-z A-Z) + export VERSION_MAJOR=$(echo $FILE_NAME | cut -d- -f2 | cut -d. -f1) + [[ $(echo $FILE_NAME | cut -d- -f2) =~ '.' ]] && export VERSION_MINOR="_$(echo $FILE_NAME | cut -d- -f2 | cut -d. -f2)" || export VERSION_MINOR='' + export VERSION_FULL=$(echo $FILE_NAME | cut -d- -f2) + OLDIFS=$IFS + IFS="_" + set $PLATFORM_NAME + IFS=$OLDIFS + export PLATFORM_NAME_FULL="$(capitalize $1)$( [[ ! -z $2 ]] && echo "_$(capitalize $2)" || true ) $VERSION_FULL" + [[ $FILE_NAME =~ 'amazon' ]] && export ICON=':aws:' + [[ $FILE_NAME =~ 'ubuntu' ]] && export ICON=':ubuntu:' + [[ $FILE_NAME =~ 'centos' ]] && export ICON=':centos:' + [[ $FILE_NAME =~ 'macos' ]] && export ICON=':darwin:' + . $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/$FILE # returns HASHED_IMAGE_TAG, etc + export PLATFORMS_JSON_ARRAY=$(echo $PLATFORMS_JSON_ARRAY | jq -c '. += [{ + "FILE_NAME": env.FILE_NAME, + "PLATFORM_NAME": env.PLATFORM_NAME, + "PLATFORM_NAME_UPCASE": env.PLATFORM_NAME_UPCASE, + "VERSION_MAJOR": env.VERSION_MAJOR, + "VERSION_MINOR": env.VERSION_MINOR, + "VERSION_FULL": env.VERSION_FULL, + "PLATFORM_NAME_FULL": env.PLATFORM_NAME_FULL, + "DOCKERHUB_FULL_TAG": env.FULL_TAG, + "HASHED_IMAGE_TAG": env.HASHED_IMAGE_TAG, + "ICON": env.ICON + }]') +done +oIFS="$IFS" +IFS=$'' +nIFS=$IFS # Needed to fix array splitting (\n won't work) +################### +# Anka Ensure Tag # +echo $PLATFORMS_JSON_ARRAY | jq -cr ".[]" | while read -r PLATFORM_JSON; do + if [[ $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) =~ 'macos' ]]; then + cat < + { + if (lineNumber >= begin && ((regex && key.test(line)) || (!regex && line.includes(key)))) + { + found = true; + return true; // c-style break + } + lineNumber += 1; + return false; // for the linter, plz delete when linter is fixed + }); + return (found) ? lineNumber : -1; +} + +// given a buildkite job, return a sanitized log file +async function getLog(job) +{ + if (debug) console.log(`getLog(${job.raw_log_url})`); // DEBUG + const logText = await download(job.raw_log_url + buildkiteAccessToken); + // returns log lowercase, with single spaces and '\n' only, and only ascii-printable characters + return sanitize(logText); // made this a separate function for unit testing purposes +} + +// given a Buildkite environment, return the operating system used +function getOS(environment) +{ + if (debug) console.log(`getOS(${environment.BUILDKITE_LABEL})`); // DEBUG + if (isNullOrEmpty(environment) || isNullOrEmpty(environment.BUILDKITE_LABEL)) + { + console.log('ERROR: getOS() called with empty environment.BUILDKITE_LABEL!'); + console.log(JSON.stringify(environment)); + return null; + } + const label = environment.BUILDKITE_LABEL.toLowerCase(); + if ((/aws(?!.*[23])/.test(label) || /amazon(?!.*[23])/.test(label))) + return 'Amazon Linux 1'; + if (/aws.*2/.test(label) || /amazon.*2/.test(label)) + return 'Amazon Linux 2'; + if (/centos(?!.*[89])/.test(label)) + return 'CentOS 7'; + if (/fedora(?!.*2[89])/.test(label) && /fedora(?!.*3\d)/.test(label)) + return 'Fedora 27'; + if (/high.*sierra/.test(label)) + return 'High Sierra'; + if (/mojave/.test(label)) + return 'Mojave'; + if (/ubuntu.*16.*04/.test(label) || /ubuntu.*16(?!.*10)/.test(label)) + return 'Ubuntu 16.04'; + if (/ubuntu.*18.*04/.test(label) || /ubuntu.*18(?!.*10)/.test(label)) + return 'Ubuntu 18.04'; + if (/docker/.test(label)) + return 'Docker'; + return 'Unknown'; +} + +// given a Buildkite job, return the test-results.xml file as JSON +async function getXML(job) +{ + if (debug) console.log('getXML()'); // DEBUG + const xmlFilename = 'test-results.xml'; + const artifacts = await download(job.artifacts_url + buildkiteAccessToken); + const testResultsArtifact = JSON.parse(artifacts).filter(artifact => artifact.filename === xmlFilename); + if (isNullOrEmpty(testResultsArtifact)) + { + console.log(`WARNING: No ${xmlFilename} found for "${job.name}"! Link: ${job.web_url}`); + return null; + } + const urlBuildkite = testResultsArtifact[0].download_url; + const rawXML = await download(urlBuildkite + buildkiteAccessToken); + const xmlOptions = + { + attrNameProcessors: [function lower(name) { return name.toLowerCase(); }], + explicitArray: false, // do not put single strings in single-element arrays + mergeAttrs: true, // make attributes children of their node + normalizeTags: true, // convert all tag names to lowercase + }; + let xmlError, xmlTestResults; + await XML.parseString(rawXML, xmlOptions, (err, result) => {xmlTestResults = result; xmlError = err;}); + if (isNullOrEmpty(xmlError)) + return xmlTestResults; + console.log(`WARNING: Failed to parse xml for "${job.name}" job! Link: ${job.web_url}`); + console.log(JSON.stringify(xmlError)); + return null; +} + +// test if variable is empty +function isNullOrEmpty(str) +{ + return (str === null || str === undefined || str.length === 0 || /^\s*$/.test(str)); +} + +// return array of test results from a buildkite job log +function parseLog(logText) +{ + if (debug) console.log('parseLog()'); // DEBUG + const lines = logText.split('\n'); + const resultLines = lines.filter(line => /test\s+#\d+/.test(line)); // 'grep' for the test result lines + // parse the strings and make test records + return resultLines.map((line) => + { + const y = line.trim().split(/test\s+#\d+/).pop(); // remove everything before the test declaration + const parts = y.split(/\s+/).slice(1, -1); // split the line and remove the test number and time unit + const testName = parts[0]; + const testTime = parts[(parts.length - 1)]; + const rawResult = parts.slice(1, -1).join(); + let testResult; + if (rawResult.includes('failed')) + testResult = 'Failed'; + else if (rawResult.includes('passed')) + testResult = 'Passed'; + else + testResult = 'Exception'; + return { testName, testResult, testTime }; // create a test record + }); +} + +// return array of test results from an xUnit-formatted JSON object +function parseXunit(xUnit) +{ + if (debug) console.log('parseXunit()'); // DEBUG + if (isNullOrEmpty(xUnit)) + { + console.log('WARNING: xUnit is empty!'); + return null; + } + return xUnit.site.testing.test.map((test) => + { + const testName = test.name; + const testTime = test.results.namedmeasurement.filter(x => /execution\s+time/.test(x.name.toLowerCase()))[0].value; + let testResult; + if (test.status.includes('failed')) + testResult = 'Failed'; + else if (test.status.includes('passed')) + testResult = 'Passed'; + else + testResult = 'Exception'; + return { testName, testResult, testTime }; + }); +} + +// returns text lowercase, with single spaces and '\n' only, and only ascii-printable characters +function sanitize(text) +{ + if (debug) console.log(`sanitize(text) where text.length = ${text.length} bytes`); // DEBUG + const chunkSize = 131072; // process text in 128 kB chunks + if (text.length > chunkSize) + return sanitize(text.slice(0, chunkSize)).concat(sanitize(text.slice(chunkSize))); + return text + .replace(/(?!\n)\r(?!\n)/g, '\n').replace(/\r/g, '') // convert all line endings to '\n' + .replace(/[^\S\n]+/g, ' ') // convert all whitespace to ' ' + .replace(/[^ -~\n]+/g, '') // remove non-printable characters + .toLowerCase(); +} + +// input is array of whole lines containing "test #" and ("failed" or "exception") +function testDiagnostics(test, logText) +{ + if (debug) + { + console.log(`testDiagnostics(test, logText) where logText.length = ${logText.length} bytes and test is`); // DEBUG + console.log(JSON.stringify(test)); + } + // get basic information + const testResultLine = new RegExp(`test\\s+#\\d+.*${test.testName}`, 'g'); // regex defining "test #" line + const startIndex = getLineNumber(logText, testResultLine); + const output = { errorMsg: null, lineNumber: startIndex + 1, stackTrace: null }; // default output + // filter tests + if (test.testResult.toLowerCase() === 'passed') + return output; + output.errorMsg = 'test diangostics are not enabled for this pipeline'; + if (!pipelineWhitelist.includes(test.pipeline)) + return output; + // diagnostics + if (debug) console.log('Running diagnostics...'); // DEBUG + output.errorMsg = 'uncategorized'; + const testLog = logText.split(testResultLine)[1].split(/test\s*#/)[0].split('\n'); // get log output from this test only, as array of lines + let errorLine = testLog[0]; // first line, from "test ## name" to '\n' exclusive + if (/\.+ *\** *not run\s+0+\.0+ sec$/.test(errorLine)) // not run + output.errorMsg = 'test not run'; + else if (/\.+ *\** *time *out\s+\d+\.\d+ sec$/.test(errorLine)) // timeout + output.errorMsg = 'test timeout'; + else if (/exception/.test(errorLine)) // test exception + output.errorMsg = errorLine.split('exception')[1].replace(/[: \d.]/g, '').replace(/sec$/, ''); // isolate the error message after exception + else if (/fc::.*exception/.test(testLog.filter(line => !isNullOrEmpty(line))[1])) // fc exception + { + [, errorLine] = testLog.filter(line => !isNullOrEmpty(line)); // get first line + output.errorMsg = `fc::${errorLine.split('::')[1].replace(/['",]/g, '').split(' ')[0]}`; // isolate fx exception body + } + else if (testLog.join('\n').includes('ctest:')) // ctest exception + { + [errorLine] = testLog.filter(line => line.includes('ctest:')); + output.errorMsg = `ctest:${errorLine.split('ctest:')[1]}`; + } + else if (!isNullOrEmpty(testLog.filter(line => /boost.+exception/.test(line)))) // boost exception + { + [errorLine] = testLog.filter(line => /boost.+exception/.test(line)); + output.errorMsg = `boost: ${errorLine.replace(/[()]/g, '').split(/: (.+)/)[1]}`; // capturing parenthesis, split only at first ' :' + output.stackTrace = testLog.filter(line => /thread-\d+/.test(line))[0].split('thread-')[1].replace(/^\d+/, '').trim().replace(/[[]\d+m$/, ''); // get the bottom of the stack trace + } + else if (/unit[-_. ]+test/.test(test.testName) || /plugin[-_. ]+test/.test(test.testName)) // unit test, application exception + { + if (!isNullOrEmpty(testLog.filter(line => line.includes('exception: ')))) + { + [errorLine] = testLog.filter(line => line.includes('exception: ')); + [, output.errorMsg] = errorLine.replace(/[()]/g, '').split(/: (.+)/); // capturing parenthesis, split only at first ' :' + output.stackTrace = testLog.filter(line => /thread-\d+/.test(line))[0].split('thread-')[1].replace(/^\d+/, '').trim().replace(/[[]\d+m$/, ''); // get the bottom of the stack trace + } + // else uncategorized unit test + } + // else integration test, add cross-referencing code here (or uncategorized) + if (errorLine !== testLog[0]) // get real line number from log file + output.lineNumber = getLineNumber(logText, errorLine, startIndex) + 1; + return output; +} + +// return test metrics given a buildkite job or build +async function testMetrics(buildkiteObject) +{ + if (!isNullOrEmpty(buildkiteObject.type)) // input is a Buildkite job object + { + const job = buildkiteObject; + console.log(`Processing test metrics for "${job.name}"${(inBuildkite) ? '' : ` at ${job.web_url}`}...`); + if (isNullOrEmpty(job.exit_status)) + { + console.log(`${(inBuildkite) ? '+++ :warning: ' : ''}WARNING: "${job.name}" was skipped!`); + return null; + } + // get test results + const logText = await getLog(job); + let testResults; + let xUnit; + try + { + xUnit = await getXML(job); + testResults = parseXunit(xUnit); + } + catch (error) + { + console.log(`XML processing failed for "${job.name}"! Link: ${job.web_url}`); + console.log(JSON.stringify(error)); + testResults = null; + } + finally + { + if (isNullOrEmpty(testResults)) + testResults = parseLog(logText); + } + // get test metrics + const env = await getEnvironment(job); + env.BUILDKITE_REPO = env.BUILDKITE_REPO.replace(new RegExp('^git@github.com:(EOSIO/)?'), '').replace(new RegExp('.git$'), ''); + const metrics = []; + const os = getOS(env); + testResults.forEach((result) => + { + // add test properties + const test = + { + ...result, // add testName, testResult, testTime + agentName: env.BUILDKITE_AGENT_NAME, + agentRole: env.BUILDKITE_AGENT_META_DATA_QUEUE || env.BUILDKITE_AGENT_META_DATA_ROLE, + branch: env.BUILDKITE_BRANCH, + buildNumber: env.BUILDKITE_BUILD_NUMBER, + commit: env.BUILDKITE_COMMIT, + job: env.BUILDKITE_LABEL, + os, + pipeline: env.BUILDKITE_PIPELINE_SLUG, + repo: env.BUILDKITE_REPO, + testTime: parseFloat(result.testTime), + url: job.web_url, + }; + metrics.push({ ...test, ...testDiagnostics(test, logText) }); + }); + return metrics; + } + else if (!isNullOrEmpty(buildkiteObject.number)) // input is a Buildkite build object + { + const build = buildkiteObject; + console.log(`Processing test metrics for ${build.pipeline.slug} build ${build.number}${(inBuildkite) ? '' : ` at ${build.web_url}`}...`); + let metrics = [], promises = []; + // process test metrics + build.jobs.filter(job => job.type === 'script' && /test/.test(job.name.toLowerCase()) && ! /test metrics/.test(job.name.toLowerCase())).forEach((job) => + { + promises.push( + testMetrics(job) + .then((moreMetrics) => { + if (!isNullOrEmpty(moreMetrics)) + metrics = metrics.concat(moreMetrics); + else + console.log(`${(inBuildkite) ? '+++ :warning: ' : ''}WARNING: "${job.name}" metrics are empty!\nmetrics = ${JSON.stringify(moreMetrics)}`); + }).catch((error) => { + console.log(`${(inBuildkite) ? '+++ :no_entry: ' : ''}ERROR: Failed to process test metrics for "${job.name}"! Link: ${job.web_url}`); + console.log(JSON.stringify(error)); + errorCount++; + }) + ); + }); + await Promise.all(promises); + return metrics; + } + else // something else + { + console.log(`${(inBuildkite) ? '+++ :no_entry: ' : ''}ERROR: Buildkite object not recognized or not a test step!`); + console.log(JSON.stringify({buildkiteObject})); + return null; + } +} + +/* main */ +async function main() +{ + if (debug) console.log(`$ ${process.argv.join(' ')}`); + let build, metrics = null; + console.log(`${(inBuildkite) ? '+++ :evergreen_tree: ' : ''}Getting information from enviroment...`); + const buildNumber = process.env.BUILDKITE_BUILD_NUMBER || process.argv[2]; + const pipeline = process.env.BUILDKITE_PIPELINE_SLUG || process.argv[3]; + if (debug) + { + console.log(`BUILDKITE=${process.env.BUILDKITE}`); + console.log(`BUILDKITE_BUILD_NUMBER=${process.env.BUILDKITE_BUILD_NUMBER}`); + console.log(`BUILDKITE_PIPELINE_SLUG=${process.env.BUILDKITE_PIPELINE_SLUG}`); + console.log(' State:') + console.log(`inBuildkite = "${inBuildkite}"`); + console.log(`buildNumber = "${buildNumber}"`); + console.log(`pipeline = "${pipeline}"`); + } + if (isNullOrEmpty(buildNumber) || isNullOrEmpty(pipeline) || isNullOrEmpty(process.env.BUILDKITE_API_KEY)) + { + console.log(`${(inBuildkite) ? '+++ :no_entry: ' : ''}ERROR: Missing required inputs!`); + if (isNullOrEmpty(process.env.BUILDKITE_API_KEY)) console.log('- Buildkite API key, as BUILDKITE_API_KEY environment variable'); + if (isNullOrEmpty(buildNumber)) console.log('- Build Number, as BUILDKITE_BUILD_NUMBER or argument 1'); + if (isNullOrEmpty(pipeline)) console.log('- Pipeline Slug, as BUILDKITE_PIPELINE_SLUG or argument 2'); + errorCount = -1; + } + else + { + console.log(`${(inBuildkite) ? '+++ :bar_chart: ' : ''}Processing test metrics...`); + build = await getBuild(pipeline, buildNumber); + metrics = await testMetrics(build); + console.log('Done processing test metrics.'); + } + console.log(`${(inBuildkite) ? '+++ :pencil: ' : ''}Writing to file...`); + fs.writeFileSync(outputFile, JSON.stringify({ metrics })); + console.log(`Saved metrics to "${outputFile}" in "${process.cwd()}".`); + if (inBuildkite) + { + console.log('+++ :arrow_up: Uploading artifact...'); + execSync(`buildkite-agent artifact upload ${outputFile}`); + } + if (errorCount === 0) + console.log(`${(inBuildkite) ? '+++ :white_check_mark: ' : ''}Done!`); + else + { + console.log(`${(inBuildkite) ? '+++ :warning: ' : ''}Finished with errors.`); + console.log(`Please send automation a link to this job${(isNullOrEmpty(build)) ? '.' : `: ${build.web_url}`}`); + console.log('@kj4ezj or @zreyn on Telegram'); + } + return (inBuildkite) ? process.exit(EXIT_SUCCESS) : process.exit(errorCount); +}; + +main(); \ No newline at end of file diff --git a/.cicd/metrics/test-metrics.tar.gz b/.cicd/metrics/test-metrics.tar.gz new file mode 100644 index 00000000000..2381787ca06 Binary files /dev/null and b/.cicd/metrics/test-metrics.tar.gz differ diff --git a/.cicd/package.sh b/.cicd/package.sh new file mode 100755 index 00000000000..79a8dae0088 --- /dev/null +++ b/.cicd/package.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -eo pipefail +. ./.cicd/helpers/general.sh +mkdir -p $BUILD_DIR +if [[ $(uname) == 'Darwin' ]]; then + bash -c "cd build/packages && chmod 755 ./*.sh && ./generate_package.sh brew" + ARTIFACT='*.rb;*.tar.gz' + cd build/packages + [[ -d x86_64 ]] && cd 'x86_64' # backwards-compatibility with release/1.6.x + buildkite-agent artifact upload "./$ARTIFACT" --agent-access-token $BUILDKITE_AGENT_ACCESS_TOKEN + for A in $(echo $ARTIFACT | tr ';' ' '); do + if [[ $(ls $A | grep -c '') == 0 ]]; then + echo "+++ :no_entry: ERROR: Expected artifact \"$A\" not found!" + pwd + ls -la + exit 1 + fi + done +else # Linux + ARGS=${ARGS:-"--rm --init -v $(pwd):$MOUNTED_DIR"} + . $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/$IMAGE_TAG.dockerfile + PRE_COMMANDS="cd $MOUNTED_DIR/build/packages && chmod 755 ./*.sh" + if [[ "$IMAGE_TAG" =~ "ubuntu" ]]; then + ARTIFACT='*.deb' + PACKAGE_TYPE='deb' + PACKAGE_COMMANDS="./generate_package.sh $PACKAGE_TYPE" + elif [[ "$IMAGE_TAG" =~ "centos" ]]; then + ARTIFACT='*.rpm' + PACKAGE_TYPE='rpm' + PACKAGE_COMMANDS="mkdir -p ~/rpmbuild/BUILD && mkdir -p ~/rpmbuild/BUILDROOT && mkdir -p ~/rpmbuild/RPMS && mkdir -p ~/rpmbuild/SOURCES && mkdir -p ~/rpmbuild/SPECS && mkdir -p ~/rpmbuild/SRPMS && yum install -y rpm-build && ./generate_package.sh $PACKAGE_TYPE" + fi + COMMANDS="$PRE_COMMANDS && $PACKAGE_COMMANDS" + echo "docker run $ARGS $(buildkite-intrinsics) $FULL_TAG bash -c \"$COMMANDS\"" + eval docker run $ARGS $(buildkite-intrinsics) $FULL_TAG bash -c \"$COMMANDS\" + cd build/packages + [[ -d x86_64 ]] && cd 'x86_64' # backwards-compatibility with release/1.6.x + buildkite-agent artifact upload "./$ARTIFACT" --agent-access-token $BUILDKITE_AGENT_ACCESS_TOKEN + for A in $(echo $ARTIFACT | tr ';' ' '); do + if [[ $(ls $A | grep -c '') == 0 ]]; then + echo "+++ :no_entry: ERROR: Expected artifact \"$A\" not found!" + pwd + ls -la + exit 1 + fi + done +fi \ No newline at end of file diff --git a/.cicd/pipeline.yml b/.cicd/pipeline.yml new file mode 100644 index 00000000000..50cf1a713d7 --- /dev/null +++ b/.cicd/pipeline.yml @@ -0,0 +1,8 @@ +steps: + - wait + + - label: ":pipeline: Generate Pipeline Steps" + command: "./.cicd/generate-pipeline.sh | buildkite-agent pipeline upload" + agents: + queue: "automation-basic-builder-fleet" + timeout: ${TIMEOUT:-10} \ No newline at end of file diff --git a/.cicd/platforms/amazon_linux-2-unpinned.dockerfile b/.cicd/platforms/amazon_linux-2-unpinned.dockerfile new file mode 100644 index 00000000000..4c6d1e3db36 --- /dev/null +++ b/.cicd/platforms/amazon_linux-2-unpinned.dockerfile @@ -0,0 +1,67 @@ +FROM amazonlinux:2.0.20190508 +ENV VERSION 1 +# install dependencies. +RUN yum update -y && \ + yum install -y which git sudo procps-ng util-linux autoconf automake \ + libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ + libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ + graphviz clang patch +# build cmake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + tar -xzf cmake-3.13.2.tar.gz && \ + cd cmake-3.13.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 +# build llvm +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm && \ + cd llvm && \ + mkdir build && \ + cd build && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 && \ + tar -xjf boost_1_70_0.tar.bz2 && \ + cd boost_1_70_0 && \ + ./bootstrap.sh --prefix=/usr/local && \ + ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_70_0.tar.bz2 /boost_1_70_0 +# build mongodb +RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ + tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && \ + rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz +# build mongodb c driver +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ + tar -xzf mongo-c-driver-1.13.0.tar.gz && \ + cd mongo-c-driver-1.13.0 && \ + mkdir -p build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 +# build mongodb cxx driver +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ + tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ + cd mongo-cxx-driver-r3.4.0 && \ + sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ + sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ + cd build && \ + cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 +# add mongodb to path +ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin +# install ccache +RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm && \ + yum install -y ccache-3.3.4-1.el7.x86_64.rpm \ No newline at end of file diff --git a/.cicd/platforms/amazon_linux-2.dockerfile b/.cicd/platforms/amazon_linux-2.dockerfile new file mode 100644 index 00000000000..3f930fcb4de --- /dev/null +++ b/.cicd/platforms/amazon_linux-2.dockerfile @@ -0,0 +1,84 @@ +FROM amazonlinux:2.0.20190508 +ENV VERSION 1 +# install dependencies. +RUN yum update -y && \ + yum install -y which git sudo procps-ng util-linux autoconf automake \ + libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ + libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ + graphviz patch gcc gcc-c++ +# build cmake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + tar -xzf cmake-3.13.2.tar.gz && \ + cd cmake-3.13.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 +# build clang8 +RUN git clone --single-branch --branch release_80 https://git.llvm.org/git/llvm.git clang8 && cd clang8 && git checkout 18e41dc && \ + cd tools && git clone --single-branch --branch release_80 https://git.llvm.org/git/lld.git && cd lld && git checkout d60a035 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/polly.git && cd polly && git checkout 1bc06e5 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang.git clang && cd clang && git checkout a03da8b && \ + cd tools && mkdir extra && cd extra && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && git checkout 6b34834 && \ + cd /clang8/projects && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxx.git && cd libcxx && git checkout 1853712 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxxabi.git && cd libcxxabi && git checkout d7338a4 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libunwind.git && cd libunwind && git checkout 57f6739 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/compiler-rt.git && cd compiler-rt && git checkout 5bc7979 && \ + mkdir /clang8/build && cd /clang8/build && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. && \ + make -j $(nproc) && \ + make install && \ + cd / && \ + rm -rf /clang8 +COPY ./.cicd/helpers/clang.make /tmp/clang.cmake +# build llvm +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm && \ + cd llvm && \ + mkdir build && \ + cd build && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 && \ + tar -xjf boost_1_70_0.tar.bz2 && \ + cd boost_1_70_0 && \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ + ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1' linkflags='-stdlib=libc++' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_70_0.tar.bz2 /boost_1_70_0 +# build mongodb +RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ + tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && \ + rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz +# build mongodb c driver +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ + tar -xzf mongo-c-driver-1.13.0.tar.gz && \ + cd mongo-c-driver-1.13.0 && \ + mkdir -p build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 +# build mongodb cxx driver +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ + tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ + cd mongo-cxx-driver-r3.4.0 && \ + sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ + sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ + cd build && \ + cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 +# add mongodb to path +ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin +# install ccache +RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm && \ + yum install -y ccache-3.3.4-1.el7.x86_64.rpm \ No newline at end of file diff --git a/.cicd/platforms/centos-7.6-unpinned.dockerfile b/.cicd/platforms/centos-7.6-unpinned.dockerfile new file mode 100644 index 00000000000..cb6bdbb369c --- /dev/null +++ b/.cicd/platforms/centos-7.6-unpinned.dockerfile @@ -0,0 +1,82 @@ +FROM centos:7.6.1810 +ENV VERSION 1 +# install dependencies. +RUN yum update -y && \ + yum --enablerepo=extras install -y centos-release-scl && \ + yum --enablerepo=extras install -y devtoolset-8 && \ + yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ + graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ + python python-devel rh-python36 gettext-devel file libusbx-devel \ + libcurl-devel patch +# build cmake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + source /opt/rh/devtoolset-8/enable && \ + source /opt/rh/rh-python36/enable && \ + tar -xzf cmake-3.13.2.tar.gz && \ + cd cmake-3.13.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 +# build llvm +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm && \ + source /opt/rh/devtoolset-8/enable && \ + source /opt/rh/rh-python36/enable && \ + cd llvm && \ + mkdir build && \ + cd build && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 && \ + source /opt/rh/devtoolset-8/enable && \ + source /opt/rh/rh-python36/enable && \ + tar -xjf boost_1_70_0.tar.bz2 && \ + cd boost_1_70_0 && \ + ./bootstrap.sh --prefix=/usr/local && \ + ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_70_0.tar.bz2 /boost_1_70_0 +# build mongodb +RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ + tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && \ + rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz +# build mongodb c driver +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ + source /opt/rh/devtoolset-8/enable && \ + source /opt/rh/rh-python36/enable && \ + tar -xzf mongo-c-driver-1.13.0.tar.gz && \ + cd mongo-c-driver-1.13.0 && \ + mkdir -p build && \ + cd build && \ + cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 +# build mongodb cxx driver +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ + source /opt/rh/devtoolset-8/enable && \ + source /opt/rh/rh-python36/enable && \ + tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ + cd mongo-cxx-driver-r3.4.0 && \ + sed -i 's/\"maxAwaitTimeMS\", ount/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ + sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ + cd build && \ + cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 +# add mongodb to path +ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin +# install ccache +RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm && \ + yum install -y ccache-3.3.4-1.el7.x86_64.rpm +# fix ccache for centos +RUN cd /usr/lib64/ccache && ln -s ../../bin/ccache c++ +ENV CCACHE_PATH="/opt/rh/devtoolset-8/root/usr/bin" \ No newline at end of file diff --git a/.cicd/platforms/centos-7.6.dockerfile b/.cicd/platforms/centos-7.6.dockerfile new file mode 100644 index 00000000000..9dc0fc25265 --- /dev/null +++ b/.cicd/platforms/centos-7.6.dockerfile @@ -0,0 +1,95 @@ +FROM centos:7.6.1810 +ENV VERSION 1 +# install dependencies. +RUN yum update -y && \ + yum --enablerepo=extras install -y centos-release-scl && \ + yum --enablerepo=extras install -y devtoolset-8 && \ + yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ + graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ + python python-devel rh-python36 gettext-devel file libusbx-devel \ + libcurl-devel patch +# build cmake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + source /opt/rh/devtoolset-8/enable && \ + source /opt/rh/rh-python36/enable && \ + tar -xzf cmake-3.13.2.tar.gz && \ + cd cmake-3.13.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 +COPY ./scripts/clang-devtoolset8-support.patch /tmp/clang-devtoolset8-support.patch +# build clang8 +RUN git clone --single-branch --branch release_80 https://git.llvm.org/git/llvm.git clang8 && cd clang8 && git checkout 18e41dc && \ + cd tools && git clone --single-branch --branch release_80 https://git.llvm.org/git/lld.git && cd lld && git checkout d60a035 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/polly.git && cd polly && git checkout 1bc06e5 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang.git clang && cd clang && git checkout a03da8b && \ + patch -p2 < /tmp/clang-devtoolset8-support.patch && \ + cd tools && mkdir extra && cd extra && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && git checkout 6b34834 && \ + cd /clang8/projects && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxx.git && cd libcxx && git checkout 1853712 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxxabi.git && cd libcxxabi && git checkout d7338a4 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libunwind.git && cd libunwind && git checkout 57f6739 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/compiler-rt.git && cd compiler-rt && git checkout 5bc7979 && \ + mkdir /clang8/build && cd /clang8/build && \ + source /opt/rh/devtoolset-8/enable && \ + source /opt/rh/rh-python36/enable && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. && \ + make -j $(nproc) && \ + make install && \ + cd / && \ + rm -rf /clang8 +COPY ./.cicd/helpers/clang.make /tmp/clang.cmake +# build llvm +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm && \ + cd llvm && \ + mkdir build && \ + cd build && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 && \ + tar -xjf boost_1_70_0.tar.bz2 && \ + cd boost_1_70_0 && \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ + ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1' linkflags='-stdlib=libc++' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_70_0.tar.bz2 /boost_1_70_0 +# build mongodb +RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ + tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && \ + rm -rf mongodb-linux-x86_64-amazon-3.6.3.tgz +# build mongodb c driver +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ + tar -xzf mongo-c-driver-1.13.0.tar.gz && \ + cd mongo-c-driver-1.13.0 && \ + mkdir -p build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 +# build mongodb cxx driver +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ + tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ + cd mongo-cxx-driver-r3.4.0 && \ + sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ + sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ + cd build && \ + cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 +# add mongodb to path +ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin +# install ccache +RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm && \ + yum install -y ccache-3.3.4-1.el7.x86_64.rpm +# fix ccache for centos +RUN cd /usr/lib64/ccache && ln -s ../../bin/ccache c++ +ENV CCACHE_PATH="/opt/rh/devtoolset-8/root/usr/bin" \ No newline at end of file diff --git a/.cicd/platforms/macos-10.14.sh b/.cicd/platforms/macos-10.14.sh new file mode 100755 index 00000000000..0be904d60c0 --- /dev/null +++ b/.cicd/platforms/macos-10.14.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -eo pipefail +VERSION=1 +brew update +brew install git boost@1.70 cmake python@2 python libtool libusb graphviz automake wget gmp llvm@4 pkgconfig doxygen openssl || true +# install clang from source +if [[ ! $PINNED == false || $UNPINNED == true ]]; then + git clone --single-branch --branch release_80 https://git.llvm.org/git/llvm.git clang8 + cd clang8 + git checkout 18e41dc + cd tools + git clone --single-branch --branch release_80 https://git.llvm.org/git/lld.git + cd lld + git checkout d60a035 + cd ../ + git clone --single-branch --branch release_80 https://git.llvm.org/git/polly.git + cd polly + git checkout 1bc06e5 + cd ../ + git clone --single-branch --branch release_80 https://git.llvm.org/git/clang.git clang + cd clang + git checkout a03da8b + cd tools + mkdir extra + cd extra + git clone --single-branch --branch release_80 https://git.llvm.org/git/clang-tools-extra.git + cd clang-tools-extra + git checkout 6b34834 + cd ../../../../../projects/ + git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxx.git + cd libcxx + git checkout 1853712 + cd ../ + git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxxabi.git + cd libcxxabi + git checkout d7338a4 + cd ../ + git clone --single-branch --branch release_80 https://git.llvm.org/git/libunwind.git + cd libunwind + git checkout 57f6739 + cd ../ + git clone --single-branch --branch release_80 https://git.llvm.org/git/compiler-rt.git + cd compiler-rt + git checkout 5bc7979 + mkdir ../../build + cd ../../build + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. + make -j $(getconf _NPROCESSORS_ONLN) + sudo make install + cd ../.. + rm -rf clang8 +fi +# install mongoDB +cd ~ +curl -OL https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz +tar -xzf mongodb-osx-ssl-x86_64-3.6.3.tgz +rm -f mongodb-osx-ssl-x86_64-3.6.3.tgz +ln -s ~/mongodb-osx-x86_64-3.6.3 ~/mongodb +# install mongo-c-driver from source +cd /tmp +curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz +tar -xzf mongo-c-driver-1.13.0.tar.gz +cd mongo-c-driver-1.13.0 +mkdir -p cmake-build +cd cmake-build +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX='/usr/local' -DENABLE_BSON=ON -DENABLE_SSL=DARWIN -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF .. +make -j $(getconf _NPROCESSORS_ONLN) +sudo make install +cd ../.. +rm mongo-c-driver-1.13.0.tar.gz +# install mongo-cxx-driver from source +cd /tmp +curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz +tar -xzf mongo-cxx-driver-r3.4.0.tar.gz +cd mongo-cxx-driver-r3.4.0/build +cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX='/usr/local' .. +make -j $(getconf _NPROCESSORS_ONLN) VERBOSE=1 +sudo make install +cd ../.. +rm -f mongo-cxx-driver-r3.4.0.tar.gz \ No newline at end of file diff --git a/.cicd/platforms/ubuntu-16.04.dockerfile b/.cicd/platforms/ubuntu-16.04.dockerfile new file mode 100644 index 00000000000..6bfbe1b1769 --- /dev/null +++ b/.cicd/platforms/ubuntu-16.04.dockerfile @@ -0,0 +1,90 @@ +FROM ubuntu:16.04 +ENV VERSION 1 +# install dependencies. +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git automake \ + libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ + python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ + sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https +# build cmake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + tar -xzf cmake-3.13.2.tar.gz && \ + cd cmake-3.13.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 +# build clang +RUN git clone --single-branch --branch release_80 https://git.llvm.org/git/llvm.git clang8 && cd clang8 && git checkout 18e41dc && \ + cd tools && git clone --single-branch --branch release_80 https://git.llvm.org/git/lld.git && cd lld && git checkout d60a035 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/polly.git && cd polly && git checkout 1bc06e5 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang.git clang && cd clang && git checkout a03da8b && \ + cd tools && mkdir extra && cd extra && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && git checkout 6b34834 && \ + cd /clang8/projects && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxx.git && cd libcxx && git checkout 1853712 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxxabi.git && cd libcxxabi && git checkout d7338a4 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libunwind.git && cd libunwind && git checkout 57f6739 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/compiler-rt.git && cd compiler-rt && git checkout 5bc7979 && \ + mkdir /clang8/build && cd /clang8/build && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. && \ + make -j $(nproc) && \ + make install && \ + cd / && \ + rm -rf /clang8 +COPY ./.cicd/helpers/clang.make /tmp/clang.cmake +# build llvm +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm && \ + cd llvm && \ + mkdir build && \ + cd build && \ + cmake -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 && \ + tar -xjf boost_1_70_0.tar.bz2 && \ + cd boost_1_70_0 && \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ + ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1' linkflags='-stdlib=libc++' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_70_0.tar.bz2 /boost_1_70_0 +# build mongodb +RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz && \ + tar -xzf mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz && \ + rm -f mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz +# build mongodb c driver +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ + tar -xzf mongo-c-driver-1.13.0.tar.gz && \ + cd mongo-c-driver-1.13.0 && \ + mkdir -p build && \ + cd build && \ + cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 +# build mongodb cxx driver +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ + tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ + cd mongo-cxx-driver-r3.4.0 && \ + sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ + sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ + cd build && \ + cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 +# add mongodb to path +ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1604-3.6.3/bin +# install ccache +RUN curl -LO https://github.com/ccache/ccache/releases/download/v3.4.1/ccache-3.4.1.tar.gz && \ + tar -xzf ccache-3.4.1.tar.gz && \ + cd ccache-3.4.1 && \ + ./configure && \ + make && \ + make install && \ + cd / && \ + rm -rf ccache-3.4.1.tar.gz /ccache-3.4.1 \ No newline at end of file diff --git a/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile b/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile new file mode 100644 index 00000000000..fe682400ea9 --- /dev/null +++ b/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile @@ -0,0 +1,55 @@ +FROM ubuntu:18.04 +ENV VERSION 1 +# install dependencies. +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ + bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ + autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ + autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ + libcurl4-gnutls-dev pkg-config patch llvm-4.0 clang ccache +# build cmake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + tar -xzf cmake-3.13.2.tar.gz && \ + cd cmake-3.13.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 && \ + tar -xjf boost_1_70_0.tar.bz2 && \ + cd boost_1_70_0 && \ + ./bootstrap.sh --prefix=/usr/local && \ + ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_70_0.tar.bz2 /boost_1_70_0 +# build mongodb +RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ + tar -xzf mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ + rm -f mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz +# build mongodb c driver +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ + tar -xzf mongo-c-driver-1.13.0.tar.gz && \ + cd mongo-c-driver-1.13.0 && \ + mkdir -p build && \ + cd build && \ + cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 +# build mongodb cxx driver +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ + tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ + cd mongo-cxx-driver-r3.4.0 && \ + sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ + sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ + cd build && \ + cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 +# add mongodb to path +ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1804-4.1.1/bin \ No newline at end of file diff --git a/.cicd/platforms/ubuntu-18.04.dockerfile b/.cicd/platforms/ubuntu-18.04.dockerfile new file mode 100644 index 00000000000..02544dfa051 --- /dev/null +++ b/.cicd/platforms/ubuntu-18.04.dockerfile @@ -0,0 +1,82 @@ +FROM ubuntu:18.04 +ENV VERSION 1 +# install dependencies. +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ + bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ + autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ + autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ + libcurl4-gnutls-dev pkg-config patch ccache +# build cmake. +RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + tar -xzf cmake-3.13.2.tar.gz && \ + cd cmake-3.13.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 +# build clang8 +RUN git clone --single-branch --branch release_80 https://git.llvm.org/git/llvm.git clang8 && cd clang8 && git checkout 18e41dc && \ + cd tools && git clone --single-branch --branch release_80 https://git.llvm.org/git/lld.git && cd lld && git checkout d60a035 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/polly.git && cd polly && git checkout 1bc06e5 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang.git clang && cd clang && git checkout a03da8b && \ + cd tools && mkdir extra && cd extra && git clone --single-branch --branch release_80 https://git.llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && git checkout 6b34834 && \ + cd /clang8/projects && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxx.git && cd libcxx && git checkout 1853712 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libcxxabi.git && cd libcxxabi && git checkout d7338a4 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/libunwind.git && cd libunwind && git checkout 57f6739 && \ + cd ../ && git clone --single-branch --branch release_80 https://git.llvm.org/git/compiler-rt.git && cd compiler-rt && git checkout 5bc7979 && \ + mkdir /clang8/build && cd /clang8/build && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. && \ + make -j $(nproc) && \ + make install && \ + cd / && \ + rm -rf /clang8 +COPY ./.cicd/helpers/clang.make /tmp/clang.cmake +# build llvm +RUN git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git llvm && \ + cd llvm && \ + mkdir build && \ + cd build && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.bz2 && \ + tar -xjf boost_1_70_0.tar.bz2 && \ + cd boost_1_70_0 && \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ + ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1' linkflags='-stdlib=libc++' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_70_0.tar.bz2 /boost_1_70_0 +# build mongodb +RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ + tar -xzf mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ + rm -f mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz +# build mongodb c driver +RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ + tar -xzf mongo-c-driver-1.13.0.tar.gz && \ + cd mongo-c-driver-1.13.0 && \ + mkdir -p build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 +# build mongodb cxx driver +RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ + tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ + cd mongo-cxx-driver-r3.4.0 && \ + sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ + sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ + cd build && \ + cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 +# add mongodb to path +ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1804-4.1.1/bin \ No newline at end of file diff --git a/.cicd/submodule-regression-check.sh b/.cicd/submodule-regression-check.sh new file mode 100755 index 00000000000..d0dc7bf117d --- /dev/null +++ b/.cicd/submodule-regression-check.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -eo pipefail +declare -A PR_MAP +declare -A BASE_MAP +# Support Travis and BK +if ${TRAVIS:-false}; then + BASE_BRANCH=$TRAVIS_BRANCH + CURRENT_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH} # We default to TRAVIS_BRANCH if it's not a PR so it passes on non PR runs +else + BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-$BUILDKITE_BRANCH} + CURRENT_BRANCH=$BUILDKITE_BRANCH +fi +echo "getting submodule info for $CURRENT_BRANCH" +while read -r a b; do + PR_MAP[$a]=$b +done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') +echo "getting submodule info for $BASE_BRANCH" +git checkout $BASE_BRANCH &> /dev/null +git submodule update --init &> /dev/null +while read -r a b; do + BASE_MAP[$a]=$b +done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') +for k in "${!BASE_MAP[@]}"; do + base_ts=${BASE_MAP[$k]} + pr_ts=${PR_MAP[$k]} + echo "submodule $k" + echo " timestamp on $CURRENT_BRANCH: $pr_ts" + echo " timestamp on $BASE_BRANCH: $base_ts" + if (( $pr_ts < $base_ts)); then + echo "$k is older on $CURRENT_BRANCH than $BASE_BRANCH; investigating..." + if for c in `git log $CURRENT_BRANCH ^$BASE_BRANCH --pretty=format:"%H"`; do git show --pretty="" --name-only $c; done | grep -q "^$k$"; then + echo "ERROR: $k has regressed" + exit 1 + else + echo "$k was not in the diff; no regression detected" + fi + fi +done \ No newline at end of file diff --git a/.cicd/test.sh b/.cicd/test.sh new file mode 100755 index 00000000000..a0f3dd73361 --- /dev/null +++ b/.cicd/test.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -eo pipefail +# variables +. ./.cicd/helpers/general.sh +# tests +if [[ $(uname) == 'Darwin' ]]; then # macOS + export PATH=$PATH:~/mongodb/bin + ./"$@" + EXIT_STATUS=$? +else # Linux + . $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/$IMAGE_TAG.dockerfile + echo "$ docker run --rm --init -v $(pwd):$MOUNTED_DIR $(buildkite-intrinsics) $FULL_TAG bash -c \"$MOUNTED_DIR/$@\"" + eval docker run --rm --init -v $(pwd):$MOUNTED_DIR $(buildkite-intrinsics) $FULL_TAG bash -c \"$MOUNTED_DIR/$@\" + EXIT_STATUS=$? +fi +# buildkite +if [[ "$BUILDKITE" == 'true' ]]; then + cd build + # upload artifacts + echo '+++ :arrow_up: Uploading Artifacts' + echo 'Compressing core dumps...' + [[ $((`ls -1 core.* 2>/dev/null | wc -l`)) != 0 ]] && tar czf core.tar.gz core.* || : # collect core dumps + echo 'Exporting xUnit XML' + mv -f ./Testing/$(ls ./Testing/ | grep '2' | tail -n 1)/Test.xml test-results.xml + echo 'Uploading artifacts' + [[ -f config.ini ]] && buildkite-agent artifact upload config.ini + [[ -f core.tar.gz ]] && buildkite-agent artifact upload core.tar.gz + [[ -f genesis.json ]] && buildkite-agent artifact upload genesis.json + [[ -f mongod.log ]] && buildkite-agent artifact upload mongod.log + buildkite-agent artifact upload test-results.xml + echo 'Done uploading artifacts.' +fi +# re-throw +if [[ "$EXIT_STATUS" != 0 ]]; then + echo "Failing due to non-zero exit status from ctest: $EXIT_STATUS" + exit $EXIT_STATUS +fi \ No newline at end of file diff --git a/.gitignore b/.gitignore index 46a19161c91..6dd6c1ca492 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.dot *.abi.hpp *.cmake +!.cicd !CMakeModules/*.cmake *.ninja \#* diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000..110f2756374 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,54 @@ +language: cpp +git: + depth: false # prevent git describe failure when executing the appbase version.cmake.in +cache: + ccache: true + directories: + - $HOME/Library/Caches/Homebrew +if: fork = true OR type = api OR type = cron +matrix: + include: + - os: linux + dist: xenial + services: docker + env: + - IMAGE_TAG='ubuntu-18.04' + - os: linux + dist: xenial + services: docker + env: + - IMAGE_TAG='ubuntu-16.04' + - os: linux + dist: xenial + services: docker + env: + - IMAGE_TAG='amazon_linux-2' + - os: linux + dist: xenial + services: docker + env: + - IMAGE_TAG='centos-7.6' + - os: osx + osx_image: xcode10.2 + addons: + homebrew: + update: true + packages: + - graphviz + - libtool + - gmp + - llvm@4 + - pkgconfig + - python + - python@2 + - doxygen + - libusb + - openssl + - boost@1.70 + - ccache + env: + - PATH="/usr/local/opt/ccache/libexec:$PATH" +script: "ccache --max-size=1G && ./.cicd/build.sh && ./.cicd/test.sh scripts/parallel-test.sh && ./.cicd/test.sh scripts/serial-test.sh" +notifications: + webhooks: + secure: gmqODqoFAil2cR7v++ibqRNECBOSD/VJX+2qPa7XptkVWmVMzbII5CNgBQAscjFsp9arHPMXCCzkBi847PCSiHdsnYFQ4T273FLRWr3cDbLjfmR+BJ7dGKvQnlpSi2Ze2TtAPJyRl+iv+cxDj7cWE5zw2c4xbgh1a/cNO+/ayUfFkyMEIfVWRsHkdkra4gOLywou0XRLHr4CX1V60uU7uuqATnIMMi7gQYwiKKtZqjkbf8wcBvZirDhjQ6lDPN5tnZo6L4QHmqjtzNJg/UrD4h+zES53dLVI4uxlXRAwwpw+mJOFA3QE/3FT+bMQjLCffUz4gZaWcdgebPYzrwSWUbJoFdWAOwcTqivQY0FIQzcz/r6uGWcwWTavzkPEbg68BVM2BZId/0110J6feeTkpJ3MPV+UsIoGTvbg50vi/I06icftuZ/cLqDj3+Emifm7Jlr1sRTSdqtYAJj/2ImUfsb46cwgjAVhFOTvc+KuPgJQgvOXV7bZkxEr5qDWo8Al2sV8BWb83j1rMlZ4LfERokImDVqxu2kkcunchzvhtYFTesSpmwegVpwceCtOtO0rEUgATnfTEHzk2rm8nuz4UtidsQnluUKqmKD0QCqHXFfn+3ZRJsDqr+iCYdxv1BAeAVc9q1L7bgrKDMGiJgkxuhZ2v3J2SflWLvjZjFDduuc= diff --git a/scripts/long-running-test.sh b/scripts/long-running-test.sh new file mode 100755 index 00000000000..1b2b86db8ae --- /dev/null +++ b/scripts/long-running-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -eo pipefail +# variables +echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" +GIT_ROOT="$(dirname $BASH_SOURCE[0])/.." +[[ -z "$TEST" ]] && export TEST=$1 +if [[ "$(uname)" == 'Linux' ]]; then + . /etc/os-release + if [[ "$ID" == 'centos' ]]; then + [[ -f /opt/rh/rh-python36/enable ]] && source /opt/rh/rh-python36/enable + fi +fi +cd $GIT_ROOT/build +# mongoDB +if [[ ! -z "$(pgrep mongod)" ]]; then + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Killing old MongoDB" + $(pgrep mongod | xargs kill -9) || : +fi +echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Starting new MongoDB" +[[ ! -d ~/data/mongodb && ! -d mongodata ]] && mkdir mongodata +echo "$ mongod --fork --logpath $(pwd)/mongod.log $([[ -d ~/data/mongodb ]] && echo '--dbpath ~/data/mongodb' || echo "--dbpath $(pwd)/mongodata") $([[ -f ~/etc/mongod.conf ]] && echo '-f ~/etc/mongod.conf')" +eval mongod --fork --logpath $(pwd)/mongod.log $([[ -d ~/data/mongodb ]] && echo '--dbpath ~/data/mongodb' || echo "--dbpath $(pwd)/mongodata") $([[ -f ~/etc/mongod.conf ]] && echo '-f ~/etc/mongod.conf') +# tests +if [[ -z "$TEST" ]]; then # run all serial tests + # count tests + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running Long-Running Tests" + TEST_COUNT=$(ctest -N -L long_running_tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') + if [[ $TEST_COUNT > 0 ]]; then + echo "$TEST_COUNT tests found." + # run tests + set +e # defer ctest error handling to end + echo '$ ctest -L long_running_tests --output-on-failure -T Test' + ctest -L long_running_tests --output-on-failure -T Test + EXIT_STATUS=$? + echo 'Done running long-running tests.' + else + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests registered with ctest! Exiting..." + EXIT_STATUS='1' + fi +else # run specific serial test + # ensure test exists + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running $TEST" + TEST_COUNT=$(ctest -N -R ^$TEST$ | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') + if [[ $TEST_COUNT > 0 ]]; then + echo "$TEST found." + # run tests + set +e # defer ctest error handling to end + echo "$ ctest -R ^$TEST$ --output-on-failure -T Test" + ctest -R ^$TEST$ --output-on-failure -T Test + EXIT_STATUS=$? + echo "Done running $TEST." + else + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests matching \"$TEST\" registered with ctest! Exiting..." + EXIT_STATUS='1' + fi +fi +if [[ ! -z "$(pgrep mongod)" ]]; then + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Killing MongoDB" + $(pgrep mongod | xargs kill -9) || : +fi +exit $EXIT_STATUS \ No newline at end of file diff --git a/scripts/parallel-test.sh b/scripts/parallel-test.sh new file mode 100755 index 00000000000..9c333b5babd --- /dev/null +++ b/scripts/parallel-test.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -eo pipefail +# variables +echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" +[[ -z "$CPUS" ]] && export CPUS=$(getconf _NPROCESSORS_ONLN) +GIT_ROOT="$(dirname $BASH_SOURCE[0])/.." +if [[ "$(uname)" == 'Linux' ]]; then + . /etc/os-release + if [[ "$ID" == 'centos' ]]; then + [[ -f /opt/rh/rh-python36/enable ]] && source /opt/rh/rh-python36/enable + fi +fi +cd $GIT_ROOT/build +# count tests +echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running Parallelizable Tests" +TEST_COUNT=$(ctest -N -LE _tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') +if [[ $TEST_COUNT > 0 ]]; then + echo "$TEST_COUNT tests found." +else + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests registered with ctest! Exiting..." + exit 1 +fi +# run tests +set +e # defer ctest error handling to end +echo "$ ctest -j $CPUS -LE _tests --output-on-failure -T Test" +ctest -j $CPUS -LE _tests --output-on-failure -T Test +EXIT_STATUS=$? +echo 'Done running parallelizable tests.' +exit $EXIT_STATUS \ No newline at end of file diff --git a/scripts/serial-test.sh b/scripts/serial-test.sh new file mode 100755 index 00000000000..84c8729009d --- /dev/null +++ b/scripts/serial-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -eo pipefail +# variables +echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" +GIT_ROOT="$(dirname $BASH_SOURCE[0])/.." +[[ -z "$TEST" ]] && export TEST=$1 +if [[ "$(uname)" == 'Linux' ]]; then + . /etc/os-release + if [[ "$ID" == 'centos' ]]; then + [[ -f /opt/rh/rh-python36/enable ]] && source /opt/rh/rh-python36/enable + fi +fi +cd $GIT_ROOT/build +# mongoDB +if [[ ! -z "$(pgrep mongod)" ]]; then + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Killing old MongoDB" + $(pgrep mongod | xargs kill -9) || : +fi +echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Starting new MongoDB" +[[ ! -d ~/data/mongodb && ! -d mongodata ]] && mkdir mongodata +echo "$ mongod --fork --logpath $(pwd)/mongod.log $([[ -d ~/data/mongodb ]] && echo '--dbpath ~/data/mongodb' || echo "--dbpath $(pwd)/mongodata") $(if [[ -f ~/etc/mongod.conf ]]; then echo '-f ~/etc/mongod.conf'; elif [[ -f /usr/local/etc/mongod.conf ]]; then echo '-f /usr/local/etc/mongod.conf'; fi)" +eval mongod --fork --logpath $(pwd)/mongod.log $([[ -d ~/data/mongodb ]] && echo '--dbpath ~/data/mongodb' || echo "--dbpath $(pwd)/mongodata") $(if [[ -f ~/etc/mongod.conf ]]; then echo '-f ~/etc/mongod.conf'; elif [[ -f /usr/local/etc/mongod.conf ]]; then echo '-f /usr/local/etc/mongod.conf'; fi) +# tests +if [[ -z "$TEST" ]]; then # run all serial tests + # count tests + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running Non-Parallelizable Tests" + TEST_COUNT=$(ctest -N -L nonparallelizable_tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') + if [[ $TEST_COUNT > 0 ]]; then + echo "$TEST_COUNT tests found." + # run tests + set +e # defer ctest error handling to end + echo '$ ctest -L nonparallelizable_tests --output-on-failure -T Test' + ctest -L nonparallelizable_tests --output-on-failure -T Test + EXIT_STATUS=$? + echo 'Done running non-parallelizable tests.' + else + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests registered with ctest! Exiting..." + EXIT_STATUS='1' + fi +else # run specific serial test + # ensure test exists + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running $TEST" + TEST_COUNT=$(ctest -N -R ^$TEST$ | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') + if [[ $TEST_COUNT > 0 ]]; then + echo "$TEST found." + # run tests + set +e # defer ctest error handling to end + echo "$ ctest -R ^$TEST$ --output-on-failure -T Test" + ctest -R ^$TEST$ --output-on-failure -T Test + EXIT_STATUS=$? + echo "Done running $TEST." + else + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests matching \"$TEST\" registered with ctest! Exiting..." + EXIT_STATUS='1' + fi +fi +if [[ ! -z "$(pgrep mongod)" ]]; then + echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Killing MongoDB" + $(pgrep mongod | xargs kill -9) || : +fi +exit $EXIT_STATUS \ No newline at end of file