diff --git a/.circleci/build.sh b/.circleci/build.sh deleted file mode 100755 index 786650b627..0000000000 --- a/.circleci/build.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash - -# This script setup an working environemnt for running glow tests and gtest driver. -# By default, we run with the enabled CPU backend and disabled OpenCL backend. -set -ex - -# Add support for https apt sources. -wget http://security.ubuntu.com/ubuntu/pool/main/a/apt/apt-transport-https_1.2.32ubuntu0.2_amd64.deb -echo "93475e4cc5e7a86de63fea0316f3f2cd8b791cf4d6ea50a6d63f5bd8e1da5726 apt-transport-https_1.2.32ubuntu0.2_amd64.deb" | sha256sum -c -sudo dpkg -i apt-transport-https_1.2.32ubuntu0.2_amd64.deb -rm apt-transport-https_1.2.32ubuntu0.2_amd64.deb - -export MAX_JOBS=8 -if [ "${CIRCLE_JOB}" != "COVERAGE" ]; then - if hash sccache 2>/dev/null; then - SCCACHE_BIN_DIR="/tmp/sccache" - mkdir -p "$SCCACHE_BIN_DIR" - for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do - ( - echo "#!/bin/sh" - echo "exec $(which sccache) $(which $compiler) \"\$@\"" - ) > "$SCCACHE_BIN_DIR/$compiler" - chmod +x "$SCCACHE_BIN_DIR/$compiler" - done - export PATH="$SCCACHE_BIN_DIR:$PATH" - fi -fi - -install_pocl() { - sudo apt-get install -y ocl-icd-opencl-dev clinfo libhwloc-dev opencl-headers - - git clone https://github.com/pocl/pocl.git - cd pocl && git checkout 4efafa82c087b5e846a9f8083d46b3cdac2f698b && cd ../ - mkdir build_pocl - cd build_pocl - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/clang++-8 -DCMAKE_C_COMPILER=/usr/bin/clang-8 -DENABLE_ICD=ON ../pocl - make -j`nproc` - sudo make install - - sudo mkdir -p /etc/OpenCL/vendors/ - sudo cp /usr/local/etc/OpenCL/vendors/pocl.icd /etc/OpenCL/vendors/ - - clinfo - cd ../ -} - -install_fmt() { - git clone https://github.com/fmtlib/fmt --branch 7.1.3 - pushd fmt - mkdir build - cd build - cmake .. - make -j`nproc` - sudo make install - popd -} - -upgrade_python() { - echo "Removing old python..."; - sudo apt-get remove --purge -y python3.6 python3-pip libpython3-dev - sudo apt-get autoremove -y - - echo "Installing dependencies for new python..." - sudo apt-get update - sudo apt-get install -y libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev - - echo "Installing new python..." - mkdir python-src - pushd python-src - wget https://www.python.org/ftp/python/3.9.0/Python-3.9.0.tgz - tar xvf Python-3.9.0.tgz - cd Python-3.9.0 - ./configure --enable-shared - sudo make altinstall - popd - - echo "Adjusting system to recognize new python..." - sudo touch /etc/ld.so.conf.d/glowDevLibs.conf - echo "/usr/local/lib/" | sudo tee -a /etc/ld.so.conf.d/glowDevLibs.conf - sudo ldconfig - sudo rm /usr/local/bin/pip - sudo ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip - - echo "Installing virtualenv..." - sudo pip3.9 install virtualenv -} - -GLOW_DEPS="libpng-dev libgoogle-glog-dev libboost-all-dev libdouble-conversion-dev libgflags-dev libjemalloc-dev libpthread-stubs0-dev libevent-dev libssl-dev" - -if [ "${CIRCLE_JOB}" == "CHECK_CLANG_AND_PEP8_FORMAT" ]; then - sudo apt-get update - upgrade_python -else - # Install Glow dependencies - sudo apt-get update - - # Redirect clang - sudo ln -s /usr/bin/clang-8 /usr/bin/clang - sudo ln -s /usr/bin/clang++-8 /usr/bin/clang++ - sudo ln -s /usr/bin/llvm-symbolizer-8 /usr/bin/llvm-symbolizer - sudo ln -s /usr/bin/llvm-config-8 /usr/bin/llvm-config-8.0 - - sudo apt-get install -y ${GLOW_DEPS} - install_fmt -fi - -# Since we are using llvm-7 in these two branches, we cannot use pip install cmake -if [ "${CIRCLE_JOB}" != "PYTORCH" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ]; then - sudo pip install cmake==3.17.3 -else - sudo apt-get install cmake -fi - -# Install ninja, (newest version of) autopep8 through pip -sudo pip install ninja -hash cmake ninja - -# Build glow -GLOW_DIR=$PWD -cd ${GLOW_DIR} -mkdir build && cd build - -CMAKE_ARGS=() - -CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-Werror") -CMAKE_ARGS+=("-DGLOW_WITH_CPU=ON") -CMAKE_ARGS+=("-DGLOW_WITH_HABANA=OFF") - -if [[ "${CIRCLE_JOB}" == "ASAN" ]]; then - CMAKE_ARGS+=("-DGLOW_USE_SANITIZER='Address;Undefined'") - CMAKE_ARGS+=("-DGLOW_WITH_OPENCL=OFF") - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") -elif [[ "$CIRCLE_JOB" == "COVERAGE" ]]; then - sudo apt-get install wget - sudo apt-get install -y lcov - sudo pip install awscli --upgrade - ../utils/install_protobuf.sh - CC=gcc-5 CXX=g++-5 cmake -G Ninja \ - -DCMAKE_BUILD_TYPE=Debug -DGLOW_WITH_OPENCL=OFF -DGLOW_WITH_CPU=ON \ - -DLLVM_DIR=/usr/lib/llvm-7/cmake \ - -DGLOW_USE_COVERAGE=ON \ - ../ -elif [[ "$CIRCLE_JOB" == "CHECK_CLANG_AND_PEP8_FORMAT" ]]; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main" - sudo apt-get update - sudo apt-get install -y clang-format-11 - cd /tmp - python3.9 -m virtualenv venv - source venv/bin/activate - pip install black==22.3.0 - cd ${GLOW_DIR} -elif [[ "$CIRCLE_JOB" == "OPENCL" ]]; then - install_pocl - CMAKE_ARGS+=("-DGLOW_WITH_OPENCL=ON") -elif [[ "$CIRCLE_JOB" == "FEATURE_COMPILATION" ]]; then - CMAKE_ARGS+=("-DGLOW_USE_PNG_IF_REQUIRED=OFF") -elif [[ "$CIRCLE_JOB" == "32B_DIM_T" ]]; then - install_pocl - CMAKE_ARGS+=("-DTENSOR_DIMS_32_BITS=ON -DGLOW_WITH_OPENCL=ON") -else - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") - if [[ "${CIRCLE_JOB}" == "SHARED" ]]; then - CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=ON") - fi -fi - -if [ "${CIRCLE_JOB}" != "COVERAGE" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ] && [ "${CIRCLE_JOB}" != "PYTORCH" ]; then - cmake -GNinja ${CMAKE_ARGS[*]} ../ - ninja -fi - -# Report sccache hit/miss stats -if hash sccache 2>/dev/null; then - sccache --show-stats -fi diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 284deb1426..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,114 +0,0 @@ -version: 2 - -# NOTE: We only perform the merge in build step and not in test step, because -# all source files will be shared from build to test -merge_with_master: &merge_with_master - name: Merge Onto Master - command: | - if [[ "${CIRCLE_BRANCH}" != "master" ]]; then - git config --global user.email "circleci.ossci@gmail.com" - git config --global user.name "CircleCI" - - git config remote.origin.url "$CIRCLE_REPOSITORY_URL" - git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master - git fetch --tags --progress "$CIRCLE_REPOSITORY_URL" +refs/heads/master:refs/remotes/origin/master --depth=50 --quiet - - export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/master` - echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET} - export GIT_COMMIT=${CIRCLE_SHA1} - echo "GIT_COMMIT: " ${GIT_COMMIT} - - git checkout -f ${GIT_COMMIT} - git reset --hard ${GIT_COMMIT} - git merge --no-edit --no-ff ${GIT_MERGE_TARGET} - fi - -update_submodule: &update_submodule - name: Update Submodule - command: | - git submodule sync - git submodule update --recursive --init - -linux_default: &linux_default - resource_class: large - machine: - image: ubuntu-2004:202010-01 - steps: - - run: - name: Install OpenSSH 8.1p1 - command: | - sudo apt-get update; - mkdir ~/tempdownload; - cd ~/tempdownload; - wget https://cdn.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-8.1p1.tar.gz; - tar zxvf openssh-8.1p1.tar.gz; - cd openssh-8.1p1 && ./configure && make && sudo make install; - - checkout - - run: - <<: *merge_with_master - - run: - <<: *update_submodule - - run: - name: Launch Docker Container - no_output_timeout: "1h" - command: | - set -e - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_FOR_ECR_READ_WRITE} - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY_FOR_ECR_READ_WRITE} - pip install awscli==1.16.35 -qqq - eval $(aws ecr get-login --region us-east-1 --no-include-email) - - docker pull ${DOCKER_IMAGE} - - sudo pkill -SIGHUP dockerd - WORKDIR=/var/lib/jenkins/workspace - pid=$(docker run -t -d -w $WORKDIR ${DOCKER_IMAGE}) - docker cp /home/circleci/project/. "$pid:$WORKDIR" - docker exec -u jenkins ${pid} sudo chown -R jenkins ${WORKDIR} - echo ${pid} > .docker_pid - - run: - name: Build - no_output_timeout: "1h" - command: | - docker exec -e CIRCLE_JOB=${CIRCLE_JOB} -e SCCACHE_BUCKET=${SCCACHE_BUCKET} -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET} -u jenkins $(cat .docker_pid) .circleci/build.sh - - run: - name: Test - no_output_timeout: "1h" - command: | - docker exec -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -e CIRCLE_BRANCH=${CIRCLE_BRANCH} -e CIRCLE_JOB=${CIRCLE_JOB} -u jenkins $(cat .docker_pid) .circleci/test.sh - -jobs: - DEBUG: - environment: - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang8-ubuntu16.04:283" - <<: *linux_default - FEATURE_COMPILATION: - environment: - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang8-ubuntu16.04:283" - <<: *linux_default - OPENCL: - environment: - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang8-ubuntu16.04:283" - <<: *linux_default - ASAN: - environment: - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang8-ubuntu16.04:283" - <<: *linux_default - 32B_DIM_T: - environment: - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang8-ubuntu16.04:283" - <<: *linux_default - CHECK_CLANG_AND_PEP8_FORMAT: - environment: - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:287" - <<: *linux_default -workflows: - version: 2 - build: - jobs: - - DEBUG - - FEATURE_COMPILATION - - OPENCL - - ASAN - - 32B_DIM_T - - CHECK_CLANG_AND_PEP8_FORMAT diff --git a/.circleci/test.sh b/.circleci/test.sh deleted file mode 100755 index df19d028a4..0000000000 --- a/.circleci/test.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -# This script runs all tests in glow, including onnxifi gtests - -set -euxo pipefail - -export GLOW_SRC=$PWD -export GLOW_BUILD_DIR=${GLOW_SRC}/build -export LOADER=${GLOW_BUILD_DIR}/bin/image-classifier -export LSAN_OPTIONS="suppressions=$GLOW_SRC/.circleci/lsan_suppressions.txt" -export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -export IMAGES_DIR=${GLOW_SRC}/tests/images/ - -# Pass in which tests to run (one of {test, test_unopt}). -run_unit_tests() { - CTEST_PARALLEL_LEVEL=4 GLOG_minloglevel=3 ninja "${1}" || ( cat Testing/Temporary/LastTest.log && exit 1 ) -} - -run_and_check_lenet_mnist_bundle() { - for q in "" "quantized_" - do - cd "${GLOW_BUILD_DIR}/bundles/${q}lenet_mnist/" - rm -f raw_results.txt - for f in ${IMAGES_DIR}/mnist/* - do - # Assume that there is only one file with this format (prepended with Quantized or not) - ./*LeNetMnistBundle ${f} | grep "Result: " >> raw_results.txt - done - diff raw_results.txt "${GLOW_SRC}/.ci/lenet_mnist_expected_output.txt" - cd - - done -} - -run_and_check_resnet50_bundle() { - for q in "" "quantized_" - do - cd "${GLOW_BUILD_DIR}/bundles/${q}resnet50/" - rm -f raw_results.txt - for f in ${IMAGES_DIR}/imagenet/* - do - # Assume that there is only one file with this format (prepended with Quantized or not) - ./*ResNet50Bundle ${f} | grep "Result: " >> raw_results.txt - done - diff raw_results.txt "${GLOW_SRC}/.ci/resnet50_expected_output.txt" - cd - - done -} - -run_and_check_bundle_instrument() { - cd "${GLOW_BUILD_DIR}/bundles/bundle_instrument/" - # Compare console output. - ./BundleInstrument ${IMAGES_DIR}/mnist/0_1009.png >> raw_results.txt - diff raw_results.txt "${GLOW_SRC}/.ci/bundle_instrument_expected_output.txt" - # Compare binary dumps between instrument-debug and instrument-ir. - for file in ./instrument-debug-data/*.bin - do - file_name=$(basename $file) - diff ./instrument-debug-data/${file_name} ./instrument-ir-data/${file_name} - done - cd - -} - -run_and_check_bundle_with_multiple_entries() { - cd "${GLOW_BUILD_DIR}/bundles/bundle_with_multiple_entries/" - # Compare console output. - ./bundle_with_multiple_entries >> raw_results.txt - diff raw_results.txt "${GLOW_SRC}/.ci/bundle_with_multiple_entries_expected_output.txt" - cd - -} - -run_and_check_bundle_with_extra_objects() { - cd "${GLOW_BUILD_DIR}/bundles/bundle_with_extra_objects/" - # Compare console output. - ./BundleWithExtraObjects >> raw_results.txt - diff raw_results.txt "${GLOW_SRC}/.ci/bundle_with_extra_objects_expected_output.txt" - cd - -} - -run_and_check_bundle_tflite_custom() { - cd "${GLOW_BUILD_DIR}/bundles/bundle_tflite_custom/" - # Compare console output. - ./BundleTFLiteCustom >> raw_results.txt - diff raw_results.txt "${GLOW_SRC}/.ci/bundle_tflite_custom_expected_output.txt" - cd - -} - -# Run unit tests and bundle tests. -cd "${GLOW_BUILD_DIR}" -case ${CIRCLE_JOB} in - ASAN) - run_unit_tests check - ;; - OPENCL) - run_unit_tests check - ;; - FEATURE_COMPILATION) - # FEATURE_COMPILATION is a compilation only CI job, thus tests - # are not requited. - ;; - DEBUG) - run_unit_tests check - run_unit_tests test_unopt - ;; - SHARED) - # No tests with shared libs; it's similar to DEBUG. - ;; - 32B_DIM_T) - # A lot of 32b dim_t issues are not revealed at build time, thus - # run the unit test suite also. - run_unit_tests check - ;; - COVERAGE) - cd "${GLOW_SRC}" - cd build - ../.circleci/run_coverage.sh - ;; - CHECK_CLANG_AND_PEP8_FORMAT) - cd "${GLOW_SRC}" - sudo ln -s /usr/bin/clang-format-11 /usr/bin/clang-format - source /tmp/venv/bin/activate - ./utils/format.sh check - ;; - *) - echo "Error, '${CIRCLE_JOB}' not valid mode; Please, check .circleci/test.sh for list of supported tests." - exit 1 - ;; -esac