Skip to content

Replace hocload.sh with std::filesystem utils #9655

Replace hocload.sh with std::filesystem utils

Replace hocload.sh with std::filesystem utils #9655

Workflow file for this run

name: NEURON CI
concurrency:
# Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }}
cancel-in-progress: true
on:
merge_group:
push:
branches:
# If nothing else, this is important for the ccache logic below...
- master
- release/**
pull_request:
branches:
- master
- release/**
# TODO : https://github.com/neuronsimulator/nrn/issues/1063
# paths-ignore:
# - '**.md'
# - '**.rst'
# - 'docs/**'
jobs:
ci:
runs-on: ${{ matrix.os }}
name: ${{ matrix.os }} - ${{ matrix.config.build_mode }} (${{ matrix.config.cmake_option }}${{ matrix.config.config_options }}${{ matrix.config.matrix_eval }}${{ matrix.config.sanitizer }})
timeout-minutes: 75
env:
INSTALL_DIR: install
SDK_ROOT: $(xcrun --sdk macosx --show-sdk-path)
SKIP_WHEELHOUSE_REPAIR: true
BUILD_TYPE: Release
DESIRED_CMAKE_VERSION: 3.17
DYNAMIC_PYTHON_CMAKE_VERSION: 3.18
PY_MIN_VERSION: ${{ matrix.config.python_min_version || '3.9' }}
PY_MAX_VERSION: ${{ matrix.config.python_max_version || '3.13' }}
MUSIC_INSTALL_DIR: /opt/MUSIC
# hash of commit containing mpi4py 4 fix
MUSIC_VERSION: '13f312338dcccebfe74d391b1b24f1b6d816ac6c'
strategy:
matrix:
os: [macOS-13, ubuntu-20.04]
config:
- { matrix_eval : "CC=gcc-9 CXX=g++-9", build_mode: "setuptools"}
- { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", music: ON}
- { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", python_dynamic: ON}
- { matrix_eval : "CC=gcc-9 CXX=g++-9" , build_mode: "cmake", cmake_option: "-DNRN_ENABLE_CORENEURON=ON"}
- { matrix_eval : "CC=gcc-9 CXX=g++-9", build_mode: "cmake", cmake_option: "-DNRN_ENABLE_MPI=OFF -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_CORENEURON=ON"}
- { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", cmake_option: "-DNRN_ENABLE_PYTHON=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_CORENEURON=ON"}
include:
- os: ubuntu-22.04
config:
build_mode: cmake
cmake_option: -DNRN_ENABLE_CORENEURON=ON
-DNRN_ENABLE_INTERVIEWS=OFF -DNMODL_SANITIZERS=undefined
flag_warnings: ON
sanitizer: undefined
- os: ubuntu-22.04
config:
build_mode: cmake
# TODO: CoreNEURON is only LeakSanitizer-clean if we disable MPI
cmake_option: -DNRN_ENABLE_CORENEURON=ON
-DNRN_ENABLE_INTERVIEWS=OFF -DNMODL_SANITIZERS=address
# TODO: address-leak is the dream, but there are many problems,
# including external ones from the MPI implementations
sanitizer: address
- os: ubuntu-24.04
config:
build_mode: cmake
# Cannot use a non-instrumented OpenMP with TSan, and we don't
# have a TSan-instrumented OpenMP runtime available.
# TODO: debug RX3D + TSan
cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_MPI=OFF
-DCORENRN_ENABLE_OPENMP=OFF -DNRN_ENABLE_RX3D=OFF
sanitizer: thread
- os: macOS-13
config:
build_mode: cmake
# TODO: investigate rxd test timeouts in this build and re-enable them
cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_INTERVIEWS=OFF
-DNRN_ENABLE_RX3D=OFF -DNMODL_SANITIZERS=address
sanitizer: address
- os: macOS-14
config:
build_mode: cmake
# TODO: investigate rxd test timeouts in this build and re-enable them
cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_INTERVIEWS=OFF
-DNRN_ENABLE_RX3D=OFF -DNMODL_SANITIZERS=address
sanitizer: thread
fail-fast: false
steps:
- name: Fix kernel mmap rnd bits
# Asan in llvm 14 provided in ubuntu 22.04 is incompatible with
# high-entropy ASLR in much newer kernels that GitHub runners are
# using leading to random crashes: https://reviews.llvm.org/D148280
run: sudo sysctl vm.mmap_rnd_bits=28
if: matrix.os == 'ubuntu-22.04'
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version : ${{(matrix.config.python_dynamic || matrix.config.build_mode == 'setuptools') && env.DYNAMIC_PYTHON_CMAKE_VERSION || env.DESIRED_CMAKE_VERSION}}
- name: Install homebrew packages
if: startsWith(matrix.os, 'macOS')
run: |
# Unlink and re-link to prevent errors when GitHub macOS runner images
# install Python outside of brew; See actions/setup-python#577 and BlueBrain/libsonata/pull/317
brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
brew install ccache coreutils doxygen flex bison mpich ninja xz autoconf automake libtool
# We use both for dynamic mpi in nrn
brew unlink mpich
brew install openmpi
brew install --cask xquartz
if [[ "${{matrix.os}}" == "macOS-14" ]]; then
brew install cmake
echo "$(brew --prefix)/opt/cmake/bin" >> $GITHUB_PATH
fi
echo "$(brew --prefix)/opt/flex/bin:$(brew --prefix)/opt/bison/bin" >> $GITHUB_PATH
# Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
if [[ "${{matrix.os}}" == "macOS-13" ]]; then
echo CMAKE_BUILD_PARALLEL_LEVEL=4 >> $GITHUB_ENV
echo CTEST_PARALLEL_LEVEL=4 >> $GITHUB_ENV
else
echo CMAKE_BUILD_PARALLEL_LEVEL=3 >> $GITHUB_ENV
echo CTEST_PARALLEL_LEVEL=3 >> $GITHUB_ENV
fi
echo CI_OS_NAME=osx >> $GITHUB_ENV
shell: bash
- name: Install apt packages
if: startsWith(matrix.os, 'ubuntu')
run: |
sudo apt-get install build-essential ccache libopenmpi-dev \
libmpich-dev libx11-dev libxcomposite-dev mpich ninja-build \
openmpi-bin flex libfl-dev bison libreadline-dev
# The sanitizer builds use ubuntu 22.04
if [[ "${{matrix.os}}" == "ubuntu-20.04" ]]; then
sudo apt-get install g++-7 g++-8
fi
# Core https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
echo CMAKE_BUILD_PARALLEL_LEVEL=4 >> $GITHUB_ENV
echo CTEST_PARALLEL_LEVEL=4 >> $GITHUB_ENV
echo CI_OS_NAME=linux >> $GITHUB_ENV
shell: bash
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Clone nmodl
working-directory: ${{runner.workspace}}/nrn
run: |
git submodule update --init --recursive --force --depth 1 -- external/nmodl
- name: Set up Python@${{ env.PY_MIN_VERSION }}
if: ${{matrix.config.python_dynamic == 'ON'}}
uses: actions/setup-python@v5
with:
python-version: ${{ env.PY_MIN_VERSION }}
- name: Install Python@${{ env.PY_MIN_VERSION }} dependencies
if: ${{ matrix.config.python_dynamic == 'ON' }}
working-directory: ${{runner.workspace}}/nrn
run: |
python -m pip install --upgrade -r external/nmodl/requirements.txt
python -m pip install --upgrade -r ci_requirements.txt
python -m pip install --upgrade pip -r nrn_requirements.txt
- name: Set up Python@${{ env.PY_MAX_VERSION }}
uses: actions/setup-python@v5
with:
python-version: ${{ env.PY_MAX_VERSION }}
- name: Install Python@${{ env.PY_MAX_VERSION }} dependencies
working-directory: ${{runner.workspace}}/nrn
run: |
python -m pip install --upgrade -r external/nmodl/requirements.txt
python -m pip install --upgrade -r ci_requirements.txt
python -m pip install --upgrade pip -r nrn_requirements.txt
- name: Install a new automake
# A automake >= 1.16.5 is needed for python 3.12 because it generates a python script
# called py-compile and the original one is not supporting this version of python
# Once ubuntu got a newer version of automake we can remove this part.
if: matrix.config.music == 'ON' && startsWith(matrix.os, 'ubuntu')
run: |
curl -L -o automake.tar.xz https://ftpmirror.gnu.org/gnu/automake/automake-1.16.5.tar.xz
tar -xf automake.tar.xz
cd automake-1.16.5/
./configure --prefix=/usr/
make -j
sudo make -j install
automake --version
working-directory: ${{runner.temp}}
- name: Setup MUSIC@${{ env.MUSIC_VERSION }}
if: matrix.config.music == 'ON'
run: |
python3 -m venv music-venv
source music-venv/bin/activate
python3 -m pip install mpi4py cython numpy setuptools
sudo mkdir -p $MUSIC_INSTALL_DIR
sudo chown -R $USER $MUSIC_INSTALL_DIR
curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/${MUSIC_VERSION}.zip
unzip MUSIC.zip && mv MUSIC-* MUSIC && cd MUSIC
./autogen.sh
# on some systems MPI library detection fails, provide exact flags/compilers
./configure --with-python-sys-prefix --prefix=$MUSIC_INSTALL_DIR --disable-anysource MPI_CXXFLAGS="-g -O3" MPI_CFLAGS="-g -O3" MPI_LDFLAGS=" " CC=mpicc CXX=mpicxx
make -j install
deactivate
working-directory: ${{runner.temp}}
- name: Register gcc problem matcher
if: ${{matrix.config.flag_warnings == 'ON'}}
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Register sanitizer problem matcher
if: ${{matrix.config.sanitizer}}
run: echo "::add-matcher::.github/problem-matchers/${{matrix.config.sanitizer}}.json"
- name: Hash config dictionary
run: |
cat << EOF > matrix.json
${{toJSON(matrix.config)}}
EOF
echo matrix.config JSON:
cat matrix.json
echo -----
- name: Restore compiler cache
uses: actions/cache/restore@v4
id: restore-compiler-cache
with:
path: ${{runner.workspace}}/ccache
key: ${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-${{github.sha}}
restore-keys: |
${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-
${{matrix.os}}-${{hashfiles('matrix.json')}}-
- name: Build and Test
id: build-test
shell: bash
working-directory: ${{runner.workspace}}/nrn
run: |
# OS related
if [ "$RUNNER_OS" == "Linux" ]; then
export ${MATRIX_EVAL};
export SHELL="/bin/bash"
else
export CXX=${CXX:-g++};
export CC=${CC:-gcc};
fi
if [ "$RUNNER_OS" == "macOS" ]; then
# TODO - this is a workaround that was implemented for Azure being reported as getting stuck.
# However it does not get stuck: neuron module not found and script goes to interpreter, seeming stuck.
# This needs to be addressed and SKIP_EMBEDED_PYTHON_TEST logic removed everywhere.
export SKIP_EMBEDED_PYTHON_TEST="true"
# long TMPDIR path on MacOS can results into runtime failures with OpenMPI
# Set shorter path as discussed in https://github.com/open-mpi/ompi/issues/8510
export TMPDIR=/tmp/$GITHUB_JOB
mkdir -p $TMPDIR
fi
# Python setup
export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR/lib/python/
# Python setup
export PYTHON_MIN=$(command -v $PYTHON_MIN_NAME);
export PYTHON_MAX=$(command -v $PYTHON_MAX_NAME);
export PYTHON=$PYTHON_MAX
if [ "$RUNNER_OS" == "macOS" ]; then
# Python is not installed as a framework, so we need to writ 'backend: TkAgg' to `matplotlibrc`.
# Since we are in a virtual environment, we cannot use `$HOME/matplotlibrc`
# The following solution is generic and relies on `matplotlib.__file__` to know where to append backend setup.
$PYTHON -c "import os,matplotlib; f =open(os.path.join(os.path.dirname(matplotlib.__file__), 'mpl-data/matplotlibrc'),'a'); f.write('backend: TkAgg');f.close();"
fi;
# Some logging
echo $LANG
echo $LC_ALL
python3 -c 'import os,sys; os.set_blocking(sys.stdout.fileno(), True)'
cmake --version
# different builds with CMake
if [[ "$BUILD_MODE" == "cmake" ]]; then
cmake_args=(-G Ninja)
# Sanitizer-specific setup
if [[ -n "${{matrix.config.sanitizer}}" ]]; then
if [ "$RUNNER_OS" == "Linux" ]; then
if [[ "${{matrix.config.sanitizer}}" == "thread" ]]; then
# GitHub/ubuntu-22.04 + clang-14 seems to have problems with TSan.
# Vanilla 22.04 + clang-16 from apt.llvm.org seemed to work.
# Use gcc-12 instead, as GitHub/ubuntu-22.04 already has it.
CC=$(command -v gcc-12)
CXX=$(command -v g++-12)
else
CC=$(command -v clang-14)
CXX=$(command -v clang++-14)
symbolizer_path="$(readlink -f "$(command -v llvm-symbolizer-14)")"
cmake_args+=(-DLLVM_SYMBOLIZER_PATH="${symbolizer_path}")
fi
fi
cmake_args+=(-DCMAKE_BUILD_TYPE=Custom \
-DCMAKE_C_FLAGS="-O1 -g" \
-DCMAKE_CXX_FLAGS="-O1 -g" \
-DNRN_SANITIZERS=$(echo ${{matrix.config.sanitizer}} | sed -e 's/-/,/g'))
fi
cmake_args+=(-DCMAKE_C_COMPILER="${CC}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER="${CXX}" \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-DNRN_ENABLE_TESTS=ON \
-DNRN_ENABLE_PERFORMANCE_TESTS=OFF \
${{matrix.config.cmake_option}})
if [[ "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then
cmake_args+=(-DNRN_ENABLE_PYTHON=ON \
-DNRN_ENABLE_PYTHON_DYNAMIC=ON \
-DNRN_PYTHON_DYNAMIC="${PYTHON_MIN};${PYTHON_MAX}" \
-DNRN_ENABLE_CORENEURON=ON)
else
cmake_args+=(-DPYTHON_EXECUTABLE="${PYTHON}")
fi
if [[ "$NRN_ENABLE_MUSIC" == "ON" ]]; then
cmake_args+=(-DNRN_ENABLE_MUSIC=ON \
-DCMAKE_PREFIX_PATH=${MUSIC_INSTALL_DIR} \
-DMUSIC_ROOT=${MUSIC_INSTALL_DIR})
fi
# Enable more warnings in the builds whose compiler warnings we
# highlight in the GitHub UI
if [[ "${{matrix.config.flag_warnings}}" == "ON" ]]; then
cmake_args+=(-DNRN_EXTRA_CXX_FLAGS="-Wall \
-Wno-char-subscripts \
-Wno-unknown-pragmas \
-Wno-unused-variable \
-Wno-unused-function \
-Wno-unused-but-set-variable \
-Wno-reorder \
-Wno-sign-compare" \
-DNRN_EXTRA_MECH_CXX_FLAGS="-Wno-sometimes-uninitialized \
-Wno-missing-braces")
fi
mkdir build && cd build
echo "Building with: ${cmake_args[@]}"
cmake .. "${cmake_args[@]}"
if ccache --version | grep -E '^ccache version 4\.(4|4\.1)$'
then
echo "------- Disable ccache direct mode -------"
# https://github.com/ccache/ccache/issues/935
export CCACHE_NODIRECT=1
fi
ccache -z
# Older versions don't support -v (verbose)
ccache -vs 2>/dev/null || ccache -s
cmake --build . --parallel
ccache -vs 2>/dev/null || ccache -s
if [ "$RUNNER_OS" == "macOS" ]
then
mkdir -p src/nrnpython
echo $'[install]\nprefix='>src/nrnpython/setup.cfg;
fi
if [[ "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then
echo "--RUNNING BASIC TESTS FROM BUILD DIR--"
for python in "${PYTHON_MIN}" "${PYTHON_MAX}"
do
echo "Using ${python}"
NEURONHOME="${PWD}/share/nrn" \
PYTHONPATH="${PWD}/lib/python" \
PATH="${PWD}/bin" \
LD_LIBRARY_PATH="${PWD}/lib:${LD_LIBRARY_PATH}" \
DYLD_LIBRARY_PATH="${PWD}/lib:${DYLD_LIBRARY_PATH}" \
"${python}" -c "from neuron import h; import neuron; neuron.test()"
done
fi
ctest --output-on-failure
cmake --build . --target install
export PATH="${INSTALL_DIR}/bin:${PATH}"
if [[ -f "${INSTALL_DIR}/bin/nrn-enable-sanitizer" ]]; then
echo --- bin/nrn-enable-sanitizer ---
cat bin/nrn-enable-sanitizer
echo ---
nrn_enable_sanitizer=${INSTALL_DIR}/bin/nrn-enable-sanitizer
nrn_enable_sanitizer_preload_python="${nrn_enable_sanitizer} --preload python"
else
echo nrn-enable-sanitizer not found, not using it
fi
elif [[ "$BUILD_MODE" == "setuptools" ]]; then
./packaging/python/build_wheels.bash CI;
fi;
if [[ -z "${nrn_enable_sanitizer_preload_python}" ]]; then
nrn_enable_sanitizer_preload_python="${PYTHON}"
fi
# basic test for cmake when python is not disabled
if [[ "$BUILD_MODE" == "cmake" && ! "${cmake_args[*]}" =~ "NRN_ENABLE_PYTHON=OFF" ]]; then
${nrn_enable_sanitizer_preload_python} --version && ${nrn_enable_sanitizer_preload_python} -c 'import neuron; neuron.test()'
fi;
# test neurondemo with cmake
if [[ "$BUILD_MODE" != "setuptools" ]]; then
${nrn_enable_sanitizer} neurondemo -nogui -c 'demo(4)' -c 'run()' -c 'quit()'
fi;
# with cmake dynamic check python_min and python_max together
if [[ "$BUILD_MODE" == "cmake" && "$NRN_ENABLE_PYTHON_DYNAMIC" == "ON" ]]; then
${nrn_enable_sanitizer_preload_python} -c 'import neuron; neuron.test()'
$PYTHON_MIN -c 'import neuron; neuron.test()'
fi;
# run rxd tests manually if rxd is enabled *and CoreNEURON is
# disabled -- otherwise hh-related tests fail
if [[ "$BUILD_MODE" == "cmake" \
&& ! "${cmake_args[*]}" =~ "NRN_ENABLE_RX3D=OFF" \
&& ! "${cmake_args[*]}" =~ "NRN_ENABLE_CORENEURON=ON" ]]; then
${nrn_enable_sanitizer_preload_python} ../share/lib/python/neuron/rxdtests/run_all.py
fi;
if [ "$BUILD_MODE" == "setuptools" ]; then
neuron_wheel=wheelhouse/NEURON*.whl;
# test with virtual environment
./packaging/python/test_wheels.sh $PYTHON $neuron_wheel
# test with global installation
./packaging/python/test_wheels.sh $PYTHON $neuron_wheel false
fi;
env:
BUILD_MODE: ${{ matrix.config.build_mode }}
CCACHE_BASEDIR: ${{runner.workspace}}/nrn
CCACHE_DIR: ${{runner.workspace}}/ccache
NRN_ENABLE_PYTHON_DYNAMIC : ${{ matrix.config.python_dynamic }}
NRN_ENABLE_MUSIC: ${{ matrix.config.music }}
PYTHON_MIN_NAME: "python${{ env.PY_MIN_VERSION }}"
PYTHON_MAX_NAME: "python${{ env.PY_MAX_VERSION }}"
INSTALL_DIR : ${{ runner.workspace }}/install
MATRIX_EVAL: ${{ matrix.config.matrix_eval }}
- name: Save compiler cache
uses: actions/cache/save@v4
if: always() && steps.restore-compiler-cache.outputs.cache-hit != 'true'
with:
path: ${{runner.workspace}}/ccache
key: |
${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-
${{matrix.os}}-${{hashfiles('matrix.json')}}-
# This step will set up an SSH connection on tmate.io for live debugging.
# To enable it, you have to:
# * add 'live-debug-ci' to your PR title
# * push something to your PR branch (note that just re-running the pipeline disregards the title update)
- name: live debug session on failure (manual steps required, check `.github/neuron-ci.yml`)
if: failure() && contains(github.event.pull_request.title, 'live-debug-ci')
uses: mxschmitt/action-tmate@v3
# see https://github.com/orgs/community/discussions/26822
final:
name: Final CI
needs: [ci]
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
- name: Check ci matrix all done
if: >-
${{
contains(needs.*.result, 'failure')
|| contains(needs.*.result, 'cancelled')
|| contains(needs.*.result, 'skipped')
}}
run: exit 1