From e51e4e4fc8f6390c8891a2e76290d2d35f405876 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 10 Feb 2020 20:06:32 +0100
Subject: [PATCH 01/18] implement support creating/dumping/loading index of
 files in path + leverage this in search_file function

---
 easybuild/tools/filetools.py | 90 +++++++++++++++++++++++++++++-------
 test/framework/filetools.py  | 45 ++++++++++++++++++
 2 files changed, 119 insertions(+), 16 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index e414ed68a7..d8b3f0e773 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -109,6 +109,7 @@
     r'~': "_tilde_",
 }
 
+PATH_INDEX_FILENAME = '.eb-path-index'
 
 CHECKSUM_TYPE_MD5 = 'md5'
 CHECKSUM_TYPE_SHA256 = 'sha256'
@@ -589,6 +590,62 @@ def download_file(filename, url, path, forced=False):
         return None
 
 
+def create_index(path, ignore_dirs=None):
+    """
+    Create index for files in specified path.
+    """
+    if ignore_dirs is None:
+        ignore_dirs = []
+
+    index = set()
+
+    for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
+        for filename in filenames:
+            # use relative paths in index
+            index.add(os.path.join(dirpath[len(path)+1:], filename))
+
+        # do not consider (certain) hidden directories
+        # note: we still need to consider e.g., .local !
+        # replace list elements using [:], so os.walk doesn't process deleted directories
+        # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
+        dirnames[:] = [d for d in dirnames if d not in ignore_dirs]
+
+    return index
+
+
+def dump_index(path):
+    """
+    Create index for files in specified path, and dump it to file (alphabetically sorted).
+    """
+
+    index_fp = os.path.join(path, PATH_INDEX_FILENAME)
+    index_contents = create_index(path)
+
+    write_file(index_fp, '\n'.join(sorted(index_contents)))
+
+
+def load_index(path, ignore_dirs=None):
+    """
+    Load index for specified path, and return contents (or None if no index exists).
+    """
+    if ignore_dirs is None:
+        ignore_dirs = []
+
+    index_fp = os.path.join(path, PATH_INDEX_FILENAME)
+
+    index, res = None, set()
+
+    if os.path.exists(index_fp):
+        index = read_file(index_fp).splitlines()
+
+        for path in index:
+            path_dirs = path.split(os.path.sep)[:-1]
+            if not any(d in path_dirs for d in ignore_dirs):
+                res.add(path)
+
+    return res
+
+
 def find_easyconfigs(path, ignore_dirs=None):
     """
     Find .eb easyconfig files in path
@@ -654,22 +711,23 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
         if not terse:
             print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent)
 
-        for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
-            for filename in filenames:
-                if query.search(filename):
-                    if not path_hits:
-                        var = "CFGS%d" % var_index
-                        var_index += 1
-                    if filename_only:
-                        path_hits.append(filename)
-                    else:
-                        path_hits.append(os.path.join(dirpath, filename))
-
-            # do not consider (certain) hidden directories
-            # note: we still need to consider e.g., .local !
-            # replace list elements using [:], so os.walk doesn't process deleted directories
-            # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
-            dirnames[:] = [d for d in dirnames if d not in ignore_dirs]
+        path_index = load_index(path, ignore_dirs=ignore_dirs)
+        if path_index:
+            _log.info("Cache found for %s, so using it...", path)
+        else:
+            _log.info("No index found for %s, creating one...", path)
+            path_index = create_index(path, ignore_dirs=ignore_dirs)
+
+        for filepath in path_index:
+            filename = os.path.basename(filepath)
+            if query.search(filename):
+                if not path_hits:
+                    var = "CFGS%d" % var_index
+                    var_index += 1
+                if filename_only:
+                    path_hits.append(filename)
+                else:
+                    path_hits.append(os.path.join(path, filepath))
 
         path_hits = sorted(path_hits)
 
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 640176ee33..51dfb22f0e 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1652,6 +1652,51 @@ def test_remove(self):
 
         ft.adjust_permissions(self.test_prefix, stat.S_IWUSR, add=True)
 
+    def test_index_functions(self):
+        """Test *_index functions."""
+
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+
+        # first test create_index function
+        index = ft.create_index(test_ecs)
+        self.assertEqual(len(index), 79)
+
+        expected = [
+            os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
+            os.path.join('t', 'toy', 'toy-0.0.eb'),
+            os.path.join('s', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb'),
+        ]
+        for fn in expected:
+            self.assertTrue(fn in index)
+
+        for fp in index:
+            self.assertTrue(fp.endswith('.eb'))
+
+        # set up some files to create actual index file for
+        ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g'))
+
+        # test dump_index function
+        ft.dump_index(self.test_prefix)
+
+        index_fp = os.path.join(self.test_prefix, '.eb-path-index')
+        self.assertTrue(os.path.exists(index_fp))
+
+        index_txt = ft.read_file(index_fp)
+        expected = [
+            os.path.join('g', 'gzip', 'gzip-1.4.eb'),
+            os.path.join('g', 'GCC', 'GCC-7.3.0-2.30.eb'),
+            os.path.join('g', 'gompic', 'gompic-2018a.eb'),
+        ]
+        for fn in expected:
+            regex = re.compile('^%s$' % fn, re.M)
+            self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt))
+
+        # test load_index function
+        index = ft.load_index(self.test_prefix)
+        self.assertEqual(len(index), 24)
+        for fn in expected:
+            self.assertTrue(fn in index)
+
     def test_search_file(self):
         """Test search_file function."""
         test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')

From ce29a3877dde2abceb766613630e5e9782478b99 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 10 Feb 2020 20:08:07 +0100
Subject: [PATCH 02/18] use path index in robot_find_easyconfig, if available
 (and cache it)

---
 easybuild/framework/easyconfig/easyconfig.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index b3e8af1cb8..97e32a02a4 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -61,7 +61,8 @@
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN
 from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme
 from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name
-from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file
+from easybuild.tools.filetools import create_index, find_backup_name_candidate, find_easyconfigs, load_index
+from easybuild.tools.filetools import read_file, write_file
 from easybuild.tools.hooks import PARSE, load_hooks, run_hook
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
@@ -102,6 +103,7 @@
 
 _easyconfig_files_cache = {}
 _easyconfigs_cache = {}
+_path_indexes = {}
 
 
 def handle_deprecated_or_replaced_easyconfig_parameters(ec_method):
@@ -1890,10 +1892,19 @@ def robot_find_easyconfig(name, version):
 
     res = None
     for path in paths:
+        if path in _path_indexes:
+            path_index = _path_indexes[path]
+            _log.info("Found loaded index for %s", path)
+        else:
+            path_index = load_index(path)
+            if path_index:
+                _path_indexes[path] = path_index
+                _log.info("Loaded index for %s", path)
+
         easyconfigs_paths = create_paths(path, name, version)
         for easyconfig_path in easyconfigs_paths:
             _log.debug("Checking easyconfig path %s" % easyconfig_path)
-            if os.path.isfile(easyconfig_path):
+            if easyconfig_path in path_index or os.path.isfile(easyconfig_path):
                 _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path))
                 _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path)
                 res = _easyconfig_files_cache[key]

From a4f3d673315c2bd937e1d948a32bd21f69c37ecd Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 12 Feb 2020 08:23:07 +0100
Subject: [PATCH 03/18] make create_index check whether specified path is an
 existing directory, load_index return None if there is no index & dump_index
 return path to index file

---
 easybuild/tools/filetools.py | 15 +++++++++++----
 test/framework/filetools.py  | 17 +++++++++++++----
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index d8b3f0e773..af73f2bafe 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -599,6 +599,11 @@ def create_index(path, ignore_dirs=None):
 
     index = set()
 
+    if not os.path.exists(path):
+        raise EasyBuildError("Specified path does not exist: %s", path)
+    elif not os.path.isdir(path):
+        raise EasyBuildError("Specified path is not a directory: %s", path)
+
     for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
         for filename in filenames:
             # use relative paths in index
@@ -623,6 +628,8 @@ def dump_index(path):
 
     write_file(index_fp, '\n'.join(sorted(index_contents)))
 
+    return index_fp
+
 
 def load_index(path, ignore_dirs=None):
     """
@@ -643,7 +650,7 @@ def load_index(path, ignore_dirs=None):
             if not any(d in path_dirs for d in ignore_dirs):
                 res.add(path)
 
-    return res
+    return res or None
 
 
 def find_easyconfigs(path, ignore_dirs=None):
@@ -712,11 +719,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
             print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent)
 
         path_index = load_index(path, ignore_dirs=ignore_dirs)
-        if path_index:
-            _log.info("Cache found for %s, so using it...", path)
-        else:
+        if path_index is None:
             _log.info("No index found for %s, creating one...", path)
             path_index = create_index(path, ignore_dirs=ignore_dirs)
+        else:
+            _log.info("Index found for %s, so using it...", path)
 
         for filepath in path_index:
             filename = os.path.basename(filepath)
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 51dfb22f0e..7e1ad84138 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1657,7 +1657,17 @@ def test_index_functions(self):
 
         test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
 
-        # first test create_index function
+        # create_index checks whether specified path is an existing directory
+        doesnotexist = os.path.join(self.test_prefix, 'doesnotexist')
+        self.assertErrorRegex(EasyBuildError, "Specified path does not exist", ft.create_index, doesnotexist)
+
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+        self.assertErrorRegex(EasyBuildError, "Specified path is not a directory", ft.create_index, toy_ec)
+
+        # load_index just returns None if there is no index in specified directory
+        self.assertEqual(ft.load_index(self.test_prefix), None)
+
+        # create index for test easyconfigs
         index = ft.create_index(test_ecs)
         self.assertEqual(len(index), 79)
 
@@ -1676,10 +1686,9 @@ def test_index_functions(self):
         ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g'))
 
         # test dump_index function
-        ft.dump_index(self.test_prefix)
-
-        index_fp = os.path.join(self.test_prefix, '.eb-path-index')
+        index_fp = ft.dump_index(self.test_prefix)
         self.assertTrue(os.path.exists(index_fp))
+        self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp)))
 
         index_txt = ft.read_file(index_fp)
         expected = [

From 5d3b2637351e756e7b2d2969a18e65be261d0002 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 12 Feb 2020 08:24:36 +0100
Subject: [PATCH 04/18] create index for path if no index is available in
 robot_find_easyconfig, and cache it

---
 easybuild/framework/easyconfig/easyconfig.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 97e32a02a4..2aaee7f28d 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -1897,10 +1897,14 @@ def robot_find_easyconfig(name, version):
             _log.info("Found loaded index for %s", path)
         else:
             path_index = load_index(path)
-            if path_index:
-                _path_indexes[path] = path_index
+            if path_index is None:
+                _log.info("No index found for %s, so creating it...", path)
+                path_index = create_index(path)
+            else:
                 _log.info("Loaded index for %s", path)
 
+            _path_indexes[path] = path_index
+
         easyconfigs_paths = create_paths(path, name, version)
         for easyconfig_path in easyconfigs_paths:
             _log.debug("Checking easyconfig path %s" % easyconfig_path)

From 078f09930e81aaf8f5d3d659494992532baffb6a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 12 Feb 2020 11:38:42 +0100
Subject: [PATCH 05/18] add support for --create-index

---
 easybuild/main.py          | 10 +++++++++-
 easybuild/tools/options.py |  1 +
 test/framework/options.py  | 19 +++++++++++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index 69c47a7293..7eeb8d9d84 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -56,7 +56,8 @@
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software
-from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, read_file, write_file
+from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, dump_index, load_index
+from easybuild.tools.filetools import read_file, write_file
 from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig
 from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr
 from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr
@@ -255,9 +256,16 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     elif options.list_software:
         print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed'))
 
+    elif options.create_index:
+        print_msg("Creating index for %s..." % options.create_index, prefix=False)
+        index_fp = dump_index(options.create_index)
+        index = load_index(options.create_index)
+        print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False)
+
     # non-verbose cleanup after handling GitHub integration stuff or printing terse info
     early_stop_options = [
         options.check_github,
+        options.create_index,
         options.install_github_token,
         options.list_installed_software,
         options.list_software,
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index d149ee3d79..efe6e644ce 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -681,6 +681,7 @@ def easyconfig_options(self):
         descr = ("Options for Easyconfigs", "Options that affect all specified easyconfig files.")
 
         opts = OrderedDict({
+            'create-index': ("Create index for files in specified directory", None, 'store', None),
             'fix-deprecated-easyconfigs': ("Fix use of deprecated functionality in specified easyconfig files.",
                                            None, 'store_true', False),
             'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)",
diff --git a/test/framework/options.py b/test/framework/options.py
index bcb3dcbe09..e3540827fa 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -4676,6 +4676,25 @@ def test_cuda_compute_capabilities(self):
         regex = re.compile(r"^cuda-compute-capabilities\s*\(C\)\s*=\s*3\.5, 6\.2, 7\.0$", re.M)
         self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt))
 
+    def test_create_index(self):
+        """Test --create-index option."""
+        test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+        remove_dir(self.test_prefix)
+        copy_dir(test_ecs, self.test_prefix)
+
+        args = ['--create-index', self.test_prefix]
+        stdout, stderr = self._run_mock_eb(args, raise_error=True)
+
+        self.assertEqual(stderr, '')
+
+        patterns = [
+            r"^Creating index for %s\.\.\.$",
+            r"^Index created at %s/\.eb-path-index \([0-9]+ files\)$",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern % self.test_prefix, re.M)
+            self.assertTrue(regex.search(stdout), "Pattern %s matches in: %s" % (regex.pattern, stdout))
+
 
 def suite():
     """ returns all the testcases in this module """

From 7cd4e9dfe5345201dd9260b4e83add5875a6d9bf Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 21 Feb 2020 09:10:01 +0100
Subject: [PATCH 06/18] add configuration option to specify maximum age of
 index file

---
 easybuild/tools/config.py  | 4 ++++
 easybuild/tools/options.py | 8 +++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index ab98bcad6d..51ff946f21 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -78,6 +78,7 @@
 CONT_TYPES = [CONT_TYPE_DOCKER, CONT_TYPE_SINGULARITY]
 DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY
 
+DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60  # 1 week (in seconds)
 DEFAULT_JOB_BACKEND = 'GC3Pie'
 DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
 DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5
@@ -270,6 +271,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     DEFAULT_CONT_TYPE: [
         'container_type',
     ],
+    DEFAULT_INDEX_MAX_AGE: [
+        'index_max_age',
+    ],
     DEFAULT_MAX_FAIL_RATIO_PERMS: [
         'max_fail_ratio_adjust_permissions',
     ],
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index efe6e644ce..22d7299f7f 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -59,9 +59,9 @@
 from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError
 from easybuild.tools.build_log import init_logging, log_start, print_warning, raise_easybuilderror
 from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE
-from easybuild.tools.config import DEFAULT_ALLOW_LOADED_MODULES, DEFAULT_FORCE_DOWNLOAD, DEFAULT_JOB_BACKEND
-from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS, DEFAULT_MNS
-from easybuild.tools.config import DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES
+from easybuild.tools.config import DEFAULT_ALLOW_LOADED_MODULES, DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE
+from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
+from easybuild.tools.config import DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES
 from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE
 from easybuild.tools.config import DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY, EBROOT_ENV_VAR_ACTIONS, ERROR
 from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR
@@ -684,6 +684,8 @@ def easyconfig_options(self):
             'create-index': ("Create index for files in specified directory", None, 'store', None),
             'fix-deprecated-easyconfigs': ("Fix use of deprecated functionality in specified easyconfig files.",
                                            None, 'store_true', False),
+            'index-max-age': ("Maximum age for index before it is considered stale (in seconds)",
+                              int, 'store', DEFAULT_INDEX_MAX_AGE),
             'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)",
                                  'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES),
             'local-var-naming-check': ("Mode to use when checking whether local variables follow the recommended "

From ecd10ddc38c439eb475aac061cda09f88843e0e5 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 21 Feb 2020 09:10:24 +0100
Subject: [PATCH 07/18] add support to dump_index for specifying maximum age of
 index + make load_index check whether index is still valid based on age

---
 easybuild/main.py            |  2 +-
 easybuild/tools/filetools.py | 64 +++++++++++++++++++++++++++++-------
 test/framework/filetools.py  | 47 ++++++++++++++++++++++++--
 3 files changed, 98 insertions(+), 15 deletions(-)

diff --git a/easybuild/main.py b/easybuild/main.py
index 7eeb8d9d84..123b39064e 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -258,7 +258,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
 
     elif options.create_index:
         print_msg("Creating index for %s..." % options.create_index, prefix=False)
-        index_fp = dump_index(options.create_index)
+        index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age)
         index = load_index(options.create_index)
         print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False)
 
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index af73f2bafe..32d6dfbe53 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -56,7 +56,7 @@
 from easybuild.base import fancylogger
 from easybuild.tools import run
 # import build_log must stay, to use of EasyBuildLog
-from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg
+from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.py2vs3 import std_urllib, string_type
 from easybuild.tools.utilities import nub
@@ -618,15 +618,29 @@ def create_index(path, ignore_dirs=None):
     return index
 
 
-def dump_index(path):
+def dump_index(path, max_age_sec=None):
     """
     Create index for files in specified path, and dump it to file (alphabetically sorted).
     """
+    if max_age_sec is None:
+        max_age_sec = build_option('index_max_age')
 
     index_fp = os.path.join(path, PATH_INDEX_FILENAME)
     index_contents = create_index(path)
 
-    write_file(index_fp, '\n'.join(sorted(index_contents)))
+    curr_ts = datetime.datetime.now()
+    if max_age_sec == 0:
+        end_ts = datetime.datetime.max
+    else:
+        end_ts = curr_ts + datetime.timedelta(0, max_age_sec)
+
+    lines = [
+        "# created at: %s" % str(curr_ts),
+        "# valid until: %s" % str(end_ts),
+    ]
+    lines.extend(sorted(index_contents))
+
+    write_file(index_fp, '\n'.join(lines), always_overwrite=False)
 
     return index_fp
 
@@ -639,18 +653,46 @@ def load_index(path, ignore_dirs=None):
         ignore_dirs = []
 
     index_fp = os.path.join(path, PATH_INDEX_FILENAME)
-
-    index, res = None, set()
+    index = set()
 
     if os.path.exists(index_fp):
-        index = read_file(index_fp).splitlines()
+        lines = read_file(index_fp).splitlines()
+
+        valid_ts_regex = re.compile("^# valid until: (.*)", re.M)
+        valid_ts = None
 
-        for path in index:
-            path_dirs = path.split(os.path.sep)[:-1]
-            if not any(d in path_dirs for d in ignore_dirs):
-                res.add(path)
+        for line in lines:
 
-    return res or None
+            # extract "valid until" timestamp, so we can check whether index is still valid
+            if valid_ts is None:
+                res = valid_ts_regex.match(line)
+            else:
+                res = None
+
+            if res:
+                valid_ts = res.group(1)
+                try:
+                    valid_ts = datetime.datetime.strptime(valid_ts, '%Y-%m-%d %H:%M:%S.%f')
+                except ValueError as err:
+                    raise EasyBuildError("Failed to parse timestamp '%s' for index at %s: %s", valid_ts, path, err)
+
+            elif line.startswith('#'):
+                _log.info("Ignoring unknown header line '%s' in index for %s", line, path)
+
+            else:
+                # filter out files that are in an ignored directory
+                path_dirs = line.split(os.path.sep)[:-1]
+                if not any(d in path_dirs for d in ignore_dirs):
+                    index.add(line)
+
+        # check whether index is still valid
+        if valid_ts:
+            curr_ts = datetime.datetime.now()
+            if curr_ts > valid_ts:
+                print_warning("Index for %s is no longer valid (too old), so ignoring it...", path)
+                index = None
+
+    return index or None
 
 
 def find_easyconfigs(path, ignore_dirs=None):
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 7e1ad84138..5cd56d5e51 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -38,6 +38,7 @@
 import stat
 import sys
 import tempfile
+import time
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 from unittest import TextTestRunner
 
@@ -1690,13 +1691,18 @@ def test_index_functions(self):
         self.assertTrue(os.path.exists(index_fp))
         self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp)))
 
-        index_txt = ft.read_file(index_fp)
+        datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+"
+        expected_header = [
+            "# created at: " + datestamp_pattern,
+            "# valid until: " + datestamp_pattern,
+        ]
         expected = [
             os.path.join('g', 'gzip', 'gzip-1.4.eb'),
             os.path.join('g', 'GCC', 'GCC-7.3.0-2.30.eb'),
             os.path.join('g', 'gompic', 'gompic-2018a.eb'),
         ]
-        for fn in expected:
+        index_txt = ft.read_file(index_fp)
+        for fn in expected_header + expected:
             regex = re.compile('^%s$' % fn, re.M)
             self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt))
 
@@ -1704,7 +1710,42 @@ def test_index_functions(self):
         index = ft.load_index(self.test_prefix)
         self.assertEqual(len(index), 24)
         for fn in expected:
-            self.assertTrue(fn in index)
+            self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index)))
+
+        # dump_index will not overwrite existing index without force
+        error_pattern = "File exists, not overwriting it without --force"
+        self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, self.test_prefix)
+
+        ft.remove_file(index_fp)
+
+        # test creating index file that's infinitely valid
+        index_fp = ft.dump_index(self.test_prefix, max_age_sec=0)
+        index_txt = ft.read_file(index_fp)
+        expected_header[1] = "# valid until: 9999-12-31 23:59:59\.9+"
+        for fn in expected_header + expected:
+            regex = re.compile('^%s$' % fn, re.M)
+            self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt))
+        index = ft.load_index(self.test_prefix)
+        self.assertEqual(len(index), 24)
+        for fn in expected:
+            self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index)))
+
+        ft.remove_file(index_fp)
+
+        # test creating index file that's only valid for a (very) short amount of time
+        index_fp = ft.dump_index(self.test_prefix, max_age_sec=1)
+        time.sleep(3)
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        index = ft.load_index(self.test_prefix)
+        stderr = self.get_stderr()
+        stdout = self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        self.assertTrue(index is None)
+        self.assertFalse(stdout)
+        regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix)
+        self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr))
 
     def test_search_file(self):
         """Test search_file function."""

From b103a9e688360f1f2b83cab6e1048eca11d619fa Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 21 Feb 2020 10:09:20 +0100
Subject: [PATCH 08/18] take into account non-existing paths in
 robot_search_easyconfig while creating/loading index

---
 easybuild/framework/easyconfig/easyconfig.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 2aaee7f28d..6acb84ece8 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -1895,7 +1895,7 @@ def robot_find_easyconfig(name, version):
         if path in _path_indexes:
             path_index = _path_indexes[path]
             _log.info("Found loaded index for %s", path)
-        else:
+        elif os.path.exists(path):
             path_index = load_index(path)
             if path_index is None:
                 _log.info("No index found for %s, so creating it...", path)
@@ -1904,6 +1904,8 @@ def robot_find_easyconfig(name, version):
                 _log.info("Loaded index for %s", path)
 
             _path_indexes[path] = path_index
+        else:
+            path_index = []
 
         easyconfigs_paths = create_paths(path, name, version)
         for easyconfig_path in easyconfigs_paths:

From 88e676bcc414588b9420ac8ffaa4ad8e7ca2cd3f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 21 Feb 2020 10:36:22 +0100
Subject: [PATCH 09/18] extend test for --create-index

---
 test/framework/options.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/test/framework/options.py b/test/framework/options.py
index e3540827fa..52103c0000 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -4695,6 +4695,32 @@ def test_create_index(self):
             regex = re.compile(pattern % self.test_prefix, re.M)
             self.assertTrue(regex.search(stdout), "Pattern %s matches in: %s" % (regex.pattern, stdout))
 
+        # check contents of index
+        index_fp = os.path.join(self.test_prefix, '.eb-path-index')
+        index_txt = read_file(index_fp)
+
+        datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+"
+        patterns = [
+            r"^# created at: " + datestamp_pattern + '$',
+            r"^# valid until: " + datestamp_pattern + '$',
+            r"^g/GCC/GCC-7.3.0-2.30.eb",
+            r"^t/toy/toy-0\.0\.eb",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt))
+
+        # existing index is not overwritten without --force
+        error_pattern = "File exists, not overwriting it without --force: .*/.eb-path-index"
+        self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True)
+
+        # also test creating index that's infinitely valid
+        args.extend(['--index-max-ag=0', '--force'])
+        self._run_mock_eb(args, raise_error=True)
+        index_txt = read_file(index_fp)
+        regex = re.compile(r"^# valid until: 9999-12-31 23:59:59", re.M)
+        self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt))
+
 
 def suite():
     """ returns all the testcases in this module """

From 0efe3e3737d3ad919d6e58f187d03ac887370069 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 21 Feb 2020 10:40:38 +0100
Subject: [PATCH 10/18] print message when valid index is being used

---
 easybuild/tools/filetools.py |  2 ++
 test/framework/filetools.py  | 23 +++++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 32d6dfbe53..9d58e39330 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -691,6 +691,8 @@ def load_index(path, ignore_dirs=None):
             if curr_ts > valid_ts:
                 print_warning("Index for %s is no longer valid (too old), so ignoring it...", path)
                 index = None
+            else:
+                print_msg("found valid index for %s, so using it...", path)
 
     return index or None
 
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 5cd56d5e51..b3237a492a 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1707,7 +1707,18 @@ def test_index_functions(self):
             self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt))
 
         # test load_index function
+        self.mock_stderr(True)
+        self.mock_stdout(True)
         index = ft.load_index(self.test_prefix)
+        stderr = self.get_stderr()
+        stdout = self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+
+        self.assertFalse(stderr)
+        regex = re.compile("^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
+        self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
+
         self.assertEqual(len(index), 24)
         for fn in expected:
             self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index)))
@@ -1725,7 +1736,19 @@ def test_index_functions(self):
         for fn in expected_header + expected:
             regex = re.compile('^%s$' % fn, re.M)
             self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt))
+
+        self.mock_stderr(True)
+        self.mock_stdout(True)
         index = ft.load_index(self.test_prefix)
+        stderr = self.get_stderr()
+        stdout = self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+
+        self.assertFalse(stderr)
+        regex = re.compile("^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
+        self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
+
         self.assertEqual(len(index), 24)
         for fn in expected:
             self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index)))

From 91a279b242c784969312e106ff47a8204e08690a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 21 Feb 2020 10:44:33 +0100
Subject: [PATCH 11/18] take into account non-existing paths in search_file

---
 easybuild/tools/filetools.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 9d58e39330..b0cd86c569 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -764,8 +764,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
 
         path_index = load_index(path, ignore_dirs=ignore_dirs)
         if path_index is None:
-            _log.info("No index found for %s, creating one...", path)
-            path_index = create_index(path, ignore_dirs=ignore_dirs)
+            if os.path.exists(path):
+                _log.info("No index found for %s, creating one...", path)
+                path_index = create_index(path, ignore_dirs=ignore_dirs)
+            else:
+                path_index = []
         else:
             _log.info("Index found for %s, so using it...", path)
 

From 78b64976d095db46c75c2c7b2953c5b866e37f3d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 29 Feb 2020 16:33:04 +0100
Subject: [PATCH 12/18] appease the Hound

---
 easybuild/tools/filetools.py | 2 +-
 test/framework/filetools.py  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 4da98b6ddb..4645e37041 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -614,7 +614,7 @@ def create_index(path, ignore_dirs=None):
     for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
         for filename in filenames:
             # use relative paths in index
-            index.add(os.path.join(dirpath[len(path)+1:], filename))
+            index.add(os.path.join(dirpath[len(path) + 1:], filename))
 
         # do not consider (certain) hidden directories
         # note: we still need to consider e.g., .local !
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index c049d96b52..f4a10d4771 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1738,7 +1738,7 @@ def test_index_functions(self):
         self.mock_stdout(False)
 
         self.assertFalse(stderr)
-        regex = re.compile("^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
+        regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
         self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
 
         self.assertEqual(len(index), 24)
@@ -1754,7 +1754,7 @@ def test_index_functions(self):
         # test creating index file that's infinitely valid
         index_fp = ft.dump_index(self.test_prefix, max_age_sec=0)
         index_txt = ft.read_file(index_fp)
-        expected_header[1] = "# valid until: 9999-12-31 23:59:59\.9+"
+        expected_header[1] = r"# valid until: 9999-12-31 23:59:59\.9+"
         for fn in expected_header + expected:
             regex = re.compile('^%s$' % fn, re.M)
             self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt))
@@ -1768,7 +1768,7 @@ def test_index_functions(self):
         self.mock_stdout(False)
 
         self.assertFalse(stderr)
-        regex = re.compile("^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
+        regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
         self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))
 
         self.assertEqual(len(index), 24)

From 6c1507567225c09ae0e3c43430d52db43b345805 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 29 Feb 2020 16:58:39 +0100
Subject: [PATCH 13/18] add support for ignoring search index via
 --ignore-index

---
 easybuild/framework/easyconfig/easyconfig.py |  6 ++-
 easybuild/tools/config.py                    |  1 +
 easybuild/tools/filetools.py                 |  2 +-
 easybuild/tools/options.py                   |  1 +
 test/framework/options.py                    | 43 ++++++++++++++++++++
 5 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index babdda2676..f5bebaee5b 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -1901,7 +1901,11 @@ def robot_find_easyconfig(name, version):
 
     res = None
     for path in paths:
-        if path in _path_indexes:
+
+        if build_option('ignore_index'):
+            _log.info("Ignoring index for %s...", path)
+            path_index = []
+        elif path in _path_indexes:
             path_index = _path_indexes[path]
             _log.info("Found loaded index for %s", path)
         elif os.path.exists(path):
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 51ff946f21..1c16245e21 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -226,6 +226,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'group_writable_installdir',
         'hidden',
         'ignore_checksums',
+        'ignore_index',
         'install_latest_eb_release',
         'lib64_fallback_sanity_check',
         'logtostdout',
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 4645e37041..d651ab2b6b 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -770,7 +770,7 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
             print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent)
 
         path_index = load_index(path, ignore_dirs=ignore_dirs)
-        if path_index is None:
+        if path_index is None or build_option('ignore_index'):
             if os.path.exists(path):
                 _log.info("No index found for %s, creating one...", path)
                 path_index = create_index(path, ignore_dirs=ignore_dirs)
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 22d7299f7f..1540bb18c5 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -684,6 +684,7 @@ def easyconfig_options(self):
             'create-index': ("Create index for files in specified directory", None, 'store', None),
             'fix-deprecated-easyconfigs': ("Fix use of deprecated functionality in specified easyconfig files.",
                                            None, 'store_true', False),
+            'ignore-index': ("Ignore index when searching for files", None, 'store_true', False),
             'index-max-age': ("Maximum age for index before it is considered stale (in seconds)",
                               int, 'store', DEFAULT_INDEX_MAX_AGE),
             'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)",
diff --git a/test/framework/options.py b/test/framework/options.py
index 52103c0000..3b9fc440fc 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -776,6 +776,49 @@ def test_search(self):
                 args = [opt, pattern, '--robot', test_easyconfigs_dir]
                 self.assertErrorRegex(EasyBuildError, "Invalid search query", self.eb_main, args, raise_error=True)
 
+    def test_ignore_index(self):
+        """
+        Test use of --ignore-index.
+        """
+
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
+        toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+        copy_file(toy_ec, self.test_prefix)
+
+        # install index that list more files than are actually available,
+        # so we can check whether it's used
+        index_txt = '\n'.join([
+            'toy-0.0.eb',
+            'toy-1.2.3.eb',
+            'toy-4.5.6.eb',
+        ])
+        write_file(os.path.join(self.test_prefix, '.eb-path-index'), index_txt)
+
+        args = [
+            '--search=toy',
+            '--robot-paths=%s' % self.test_prefix,
+        ]
+        self.mock_stdout(True)
+        self.eb_main(args, testing=False, raise_error=True)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+
+        for toy_ec_fn in ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb']:
+            regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M)
+            self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
+        args.append('--ignore-index')
+        self.mock_stdout(True)
+        self.eb_main(args, testing=False, raise_error=True)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+
+        regex = re.compile(re.escape(os.path.join(self.test_prefix, 'toy-0.0.eb')), re.M)
+        self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+        for toy_ec_fn in ['toy-1.2.3.eb', 'toy-4.5.6.eb']:
+            regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M)
+            self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout))
+
     def test_search_archived(self):
         "Test searching for archived easyconfigs"
         args = ['--search-filename=^intel']

From 6bb21315a0e2f7e1918428c3a47f722817e40fb6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 8 Mar 2020 10:35:11 +0100
Subject: [PATCH 14/18] fix determining relative paths in create_index

---
 easybuild/tools/filetools.py |  6 +++---
 test/framework/filetools.py  | 30 ++++++++++++++++--------------
 test/framework/options.py    | 10 ++++------
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index d651ab2b6b..a45382a2a7 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -611,15 +611,15 @@ def create_index(path, ignore_dirs=None):
     elif not os.path.isdir(path):
         raise EasyBuildError("Specified path is not a directory: %s", path)
 
-    for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
+    for (dirpath, dirnames, filenames) in os.walk(path, topdown=True, followlinks=True):
         for filename in filenames:
             # use relative paths in index
-            index.add(os.path.join(dirpath[len(path) + 1:], filename))
+            index.add(os.path.join(os.path.relpath(dirpath, path), filename))
 
         # do not consider (certain) hidden directories
         # note: we still need to consider e.g., .local !
         # replace list elements using [:], so os.walk doesn't process deleted directories
-        # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
+        # see https://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
         dirnames[:] = [d for d in dirnames if d not in ignore_dirs]
 
     return index
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index f4a10d4771..cd76667397 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1690,20 +1690,22 @@ def test_index_functions(self):
         # load_index just returns None if there is no index in specified directory
         self.assertEqual(ft.load_index(self.test_prefix), None)
 
-        # create index for test easyconfigs
-        index = ft.create_index(test_ecs)
-        self.assertEqual(len(index), 79)
-
-        expected = [
-            os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
-            os.path.join('t', 'toy', 'toy-0.0.eb'),
-            os.path.join('s', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb'),
-        ]
-        for fn in expected:
-            self.assertTrue(fn in index)
-
-        for fp in index:
-            self.assertTrue(fp.endswith('.eb'))
+        # create index for test easyconfigs;
+        # test with specified path with and without trailing '/'s
+        for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
+            index = ft.create_index(path)
+            self.assertEqual(len(index), 79)
+
+            expected = [
+                os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
+                os.path.join('t', 'toy', 'toy-0.0.eb'),
+                os.path.join('s', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb'),
+            ]
+            for fn in expected:
+                self.assertTrue(fn in index)
+
+            for fp in index:
+                self.assertTrue(fp.endswith('.eb'))
 
         # set up some files to create actual index file for
         ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g'))
diff --git a/test/framework/options.py b/test/framework/options.py
index 3b9fc440fc..a7f1d9bdad 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -785,13 +785,11 @@ def test_ignore_index(self):
         toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb')
         copy_file(toy_ec, self.test_prefix)
 
+        toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb']
+
         # install index that list more files than are actually available,
         # so we can check whether it's used
-        index_txt = '\n'.join([
-            'toy-0.0.eb',
-            'toy-1.2.3.eb',
-            'toy-4.5.6.eb',
-        ])
+        index_txt = '\n'.join(toy_ec_list)
         write_file(os.path.join(self.test_prefix, '.eb-path-index'), index_txt)
 
         args = [
@@ -803,7 +801,7 @@ def test_ignore_index(self):
         stdout = self.get_stdout()
         self.mock_stdout(False)
 
-        for toy_ec_fn in ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb']:
+        for toy_ec_fn in toy_ec_list:
             regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M)
             self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
 

From 4dc4554f637d3c4f32fdff0a067a4ef4ffce3863 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sun, 8 Mar 2020 13:38:16 +0100
Subject: [PATCH 15/18] avoid that relative paths start with './' in
 create_index

---
 easybuild/tools/filetools.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index a45382a2a7..922a8ce5d6 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -614,7 +614,11 @@ def create_index(path, ignore_dirs=None):
     for (dirpath, dirnames, filenames) in os.walk(path, topdown=True, followlinks=True):
         for filename in filenames:
             # use relative paths in index
-            index.add(os.path.join(os.path.relpath(dirpath, path), filename))
+            rel_dirpath = os.path.relpath(dirpath, path)
+            # avoid that relative paths start with './'
+            if rel_dirpath == '.':
+                rel_dirpath = ''
+            index.add(os.path.join(rel_dirpath, filename))
 
         # do not consider (certain) hidden directories
         # note: we still need to consider e.g., .local !

From 72c6b3c011f8dafd4f5f376a1ce3d013dcafbb98 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Apr 2020 10:31:57 +0200
Subject: [PATCH 16/18] fix minor typo in test_create_index

---
 test/framework/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/options.py b/test/framework/options.py
index 9740759008..8f681b0cab 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -5016,7 +5016,7 @@ def test_create_index(self):
         self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True)
 
         # also test creating index that's infinitely valid
-        args.extend(['--index-max-ag=0', '--force'])
+        args.extend(['--index-max-age=0', '--force'])
         self._run_mock_eb(args, raise_error=True)
         index_txt = read_file(index_fp)
         regex = re.compile(r"^# valid until: 9999-12-31 23:59:59", re.M)

From f64ae2c95690d8bf7c5174be664d2eee3612195a Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Apr 2020 10:34:44 +0200
Subject: [PATCH 17/18] fix typo in import + appease the Hound

---
 easybuild/framework/easyconfig/easyconfig.py | 4 ++--
 easybuild/tools/filetools.py                 | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 3e8d6b1487..a47c3618d0 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -62,8 +62,8 @@
 from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
 from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme
-from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name
-from easybuild.tools.filetools import create_index, find_backup_name_candidate, find_easyconfigs, load_index
+from easybuild.tools.filetools import copy_file, create_index, decode_class_name, encode_class_name
+from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, load_index
 from easybuild.tools.filetools import read_file, write_file
 from easybuild.tools.hooks import PARSE, load_hooks, run_hook
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 41b59892a7..bc8ed98652 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -59,7 +59,7 @@
 from easybuild.tools import run
 # import build_log must stay, to use of EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
-from easybuild.tools.config import ,GENERIC_EASYBLOCK_PKG build_option
+from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, build_option
 from easybuild.tools.py2vs3 import std_urllib, string_type
 from easybuild.tools.utilities import nub, remove_unwanted_chars
 

From b31bfa84af621f627ad5c71c9203f4ee958a2cee Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Apr 2020 10:59:00 +0200
Subject: [PATCH 18/18] take into account --ignore-index in load_index + check
 for it in tests

---
 easybuild/tools/filetools.py | 5 ++++-
 test/framework/filetools.py  | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index bc8ed98652..8f357d9c6b 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -684,7 +684,10 @@ def load_index(path, ignore_dirs=None):
     index_fp = os.path.join(path, PATH_INDEX_FILENAME)
     index = set()
 
-    if os.path.exists(index_fp):
+    if build_option('ignore_index'):
+        _log.info("Ignoring index for %s...", path)
+
+    elif os.path.exists(index_fp):
         lines = read_file(index_fp).splitlines()
 
         valid_ts_regex = re.compile("^# valid until: (.*)", re.M)
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index d3e1519b4e..f03d126e8f 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1794,6 +1794,10 @@ def test_index_functions(self):
         regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix)
         self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr))
 
+        # check whether load_index takes into account --ignore-index
+        init_config(build_options={'ignore_index': True})
+        self.assertEqual(ft.load_index(self.test_prefix), None)
+
     def test_search_file(self):
         """Test search_file function."""
         test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')