Merge branch 'develop' into cug20

.coveragerc

@@ -0,0 +1,26 @@
+# .coveragerc to control coverage.py
+[run]
+branch = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = coverage_html_report

.github/workflows/unit_tests.yml

@@ -39,7 +39,7 @@ jobs:
             python: 3.8
       fail-fast: false
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
 
     - name: set up Python
       uses: actions/setup-python@v1
@@ -49,6 +49,14 @@ jobs:
 
     - name: install OS & Python packages
       run: |
+        # use apt-spy2 to select closest apt mirror,
+        # which helps avoid connectivity issues in Azure;
+        # see https://github.com/actions/virtual-environments/issues/675
+        sudo gem install apt-spy2
+        sudo apt-spy2 check
+        sudo apt-spy2 fix --commit
+        # after selecting a specific mirror, we need to run 'apt-get update'
+        sudo apt-get update
         # for modules tool
         sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
@@ -153,19 +161,15 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53"
+          EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
-          # test bootstrap script (only compatible with Python 2 for now)
-          if [[ ${{matrix.python}} =~ '2.' ]]; then
-              export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
-              python easybuild/scripts/bootstrap_eb.py $PREFIX
-              # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
-              unset PYTHONPATH
-              # simple sanity check on bootstrapped EasyBuild module (skip when testing with Python 3, for now)
-              module use $PREFIX/modules/all
-              module load EasyBuild
-              eb --version
-          else
-              echo "Testing of bootstrap script skipped when testing with Python ${{matrix.python}}"
-          fi
+          # test bootstrap script
+          export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
+          python easybuild/scripts/bootstrap_eb.py $PREFIX
+          # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
+          unset PYTHONPATH
+          # simple sanity check on bootstrapped EasyBuild module
+          module use $PREFIX/modules/all
+          module load EasyBuild
+          eb --version

.gitignore

@@ -1,3 +1,4 @@
+.idea
 .pydevproject
 .project
 LICENSE_HEADER

.travis.yml

@@ -38,6 +38,8 @@ before_install:
     - pip --version
     - pip install --upgrade pip
     - pip --version
+    # coveralls doesn't support Python 2.6 anymore, so don't try to install it when testing with Python 2.6
+    - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then pip install coveralls; fi    
     - pip install -r requirements.txt
     # git config is required to make actual git commits (cfr. tests for GitRepository)
     - git config --global user.name "Travis CI"
@@ -55,14 +57,18 @@ script:
     - if [ ! -z $MOD_INIT ] && [ ! -z $LMOD_VERSION ]; then alias ml=foobar; fi
     # set up environment for modules tool (if $MOD_INIT is defined)
     - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; type module; fi
-    # install GitHub token
+    # install GitHub token;
+    # unset $GITHUB_TOKEN environment variable after installing token,
+    # to avoid that it is included in environment dump that is included in EasyBuild debug logs,
+    # which causes test_from_pr_token_log to fail...
     - if [ ! -z $GITHUB_TOKEN ]; then
         if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ];
             then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
             else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
         fi;
         python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
-      fi
+      fi;
+      unset GITHUB_TOKEN;
     - if [ ! -z $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi
     - if [ ! -z $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi
     # create 'source distribution' tarball, like we do when publishing a release to PyPI
@@ -88,8 +94,10 @@ script:
     # create file owned by root but writable by anyone (used by test_copy_file)
     - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
     - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
-    # run test suite
-    - python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
+    # run coverage on all travis builds except for Python 2.6
+    - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coverage run -m test.framework.suite 2>&1 | tee test_framework_suite.log; coverage report -m --ignore-errors; fi  
+    # invoke the regression test for Python 2.6 the original way without coverage
+    - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log; fi
     # try and make sure output of running tests is clean (no printed messages/warnings)
     - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|lib/python2.6/site-packages|requires Lmod as modules tool"
     # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
@@ -100,12 +108,15 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53"
+    - EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
-    # test bootstrap script (skip when testing with Python 3 for now, since latest EasyBuild release is not compatible with Python 3 yet)
-    - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap; fi
+    # test bootstrap script
+    - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap
     # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
     - unset PYTHONPATH
-    # simply sanity check on bootstrapped EasyBuild module (skip when testing with Python 3, for now)
-    - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; fi
-    - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then module load EasyBuild; eb --version; fi
+    # simply sanity check on bootstrapped EasyBuild module
+    - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all
+    - module load EasyBuild; eb --version
+after_success:
+  - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coveralls; fi  
+

RELEASE_NOTES

@@ -3,6 +3,15 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.1.2 (March 16th 2020)
+------------------------
+
+bugfix release
+
+- fix gitdb dependency on Python 2.6 in test configuration (#3212)
+- fix broken test for --review-pr by using different PR to test with (#3226)
+- censor authorization part of headers before logging ReST API request (#3248)
+
 v4.1.1 (January 16th 2020)
 --------------------------
 

contrib/hooks/README.rst

@@ -0,0 +1,15 @@
+Example implementations of EasyBuild hooks
+=================================
+
+.. image:: https://easybuilders.github.io/easybuild/images/easybuild_logo_small.png
+   :align: center
+
+EasyBuild website: https://easybuilders.github.io/easybuild/
+docs: https://easybuild.readthedocs.io
+
+This directory contain examples of implementations of EasyBuild hooks
+used at various sites, along with a couple of small examples with
+explanations.
+
+See https://easybuild.readthedocs.io/en/latest/Hooks.html for
+documentation on hooks in EasyBuild.

contrib/hooks/add_delete_configopt.py

@@ -0,0 +1,32 @@
+# Small example of how to add/delete a configure option.
+#
+# Author: Åke Sandgren, HPC2N
+
+# We need to be able to distinguish between versions of OpenMPI
+from distutils.version import LooseVersion
+
+
+def pre_configure_hook(self, *args, **kwargs):
+    # Check that we're dealing with the correct easyconfig file
+    if self.name == 'OpenMPI':
+        extra_opts = ""
+        # Enable using pmi from slurm
+        extra_opts += "--with-pmi=/lap/slurm "
+
+        # And enable munge for OpenMPI versions that knows about it
+        if LooseVersion(self.version) >= LooseVersion('2'):
+            extra_opts += "--with-munge "
+
+        # Now add the options
+        self.log.info("[pre-configure hook] Adding %s" % extra_opts)
+        self.cfg.update('configopts', extra_opts)
+
+        # Now we delete some options
+        # For newer versions of OpenMPI we can re-enable ucx, i.e. delete the --without-ucx flag
+        if LooseVersion(self.version) >= LooseVersion('2.1'):
+            self.log.info("[pre-configure hook] Re-enabling ucx")
+            self.cfg['configopts'] = self.cfg['configopts'].replace('--without-ucx', ' ')
+
+        # And we can remove the --disable-dlopen option from the easyconfig file
+        self.log.info("[pre-configure hook] Re-enabling dlopen")
+        self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ')