Merge pull request #3189 from migueldiascosta/new_pr_easyblocks_frame…

…work

add support for targeting easyblocks and framework in --new-pr (REVIEW)

easybuild/framework/easyconfig/easyconfig.py

@@ -45,6 +45,7 @@
 import re
 from distutils.version import LooseVersion
 
+import easybuild.tools.filetools as filetools
 from easybuild.base import fancylogger
 from easybuild.framework.easyconfig import MANDATORY
 from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER
@@ -58,9 +59,10 @@
 from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
 from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
-from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN
+from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
+from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
 from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme
-from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name
+from easybuild.tools.filetools import copy_file, decode_class_name, encode_class_name
 from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file
 from easybuild.tools.hooks import PARSE, load_hooks, run_hook
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
@@ -1696,8 +1698,8 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
 
 def is_generic_easyblock(easyblock):
     """Return whether specified easyblock name is a generic easyblock or not."""
-
-    return easyblock and not easyblock.startswith(EASYBLOCK_CLASS_PREFIX)
+    _log.deprecated("is_generic_easyblock function was moved to easybuild.tools.filetools", '5.0')
+    return filetools.is_generic_easyblock(easyblock)
 
 
 def get_module_path(name, generic=None, decode=True):
@@ -1712,7 +1714,7 @@ def get_module_path(name, generic=None, decode=True):
         return None
 
     if generic is None:
-        generic = is_generic_easyblock(name)
+        generic = filetools.is_generic_easyblock(name)
 
     # example: 'EB_VSC_minus_tools' should result in 'vsc_tools'
     if decode:
@@ -1721,7 +1723,7 @@ def get_module_path(name, generic=None, decode=True):
 
     modpath = ['easybuild', 'easyblocks']
     if generic:
-        modpath.append('generic')
+        modpath.append(GENERIC_EASYBLOCK_PKG)
 
     return '.'.join(modpath + [module_name])
 

easybuild/framework/easyconfig/tools.py

@@ -604,17 +604,21 @@ def dump_env_script(easyconfigs):
 
 def categorize_files_by_type(paths):
     """
-    Splits list of filepaths into a 3 separate lists: easyconfigs, files to delete and patch files
+    Splits list of filepaths into a 4 separate lists: easyconfigs, files to delete, patch files and
+    files with extension .py
     """
     res = {
         'easyconfigs': [],
         'files_to_delete': [],
         'patch_files': [],
+        'py_files': [],
     }
 
     for path in paths:
         if path.startswith(':'):
             res['files_to_delete'].append(path[1:])
+        elif path.endswith('.py'):
+            res['py_files'].append(path)
         # file must exist in order to check whether it's a patch file
         elif os.path.isfile(path) and is_patch_file(path):
             res['patch_files'].append(path)

easybuild/tools/config.py

@@ -112,6 +112,9 @@
 FORCE_DOWNLOAD_CHOICES = [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES]
 DEFAULT_FORCE_DOWNLOAD = FORCE_DOWNLOAD_SOURCES
 
+# package name for generic easyblocks
+GENERIC_EASYBLOCK_PKG = 'generic'
+
 # general module class
 GENERAL_CLASS = 'all'
 

easybuild/tools/filetools.py

@@ -43,6 +43,8 @@
 import fileinput
 import glob
 import hashlib
+import imp
+import inspect
 import os
 import re
 import shutil
@@ -57,9 +59,9 @@
 from easybuild.tools import run
 # import build_log must stay, to use of EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg
-from easybuild.tools.config import build_option
+from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, build_option
 from easybuild.tools.py2vs3 import std_urllib, string_type
-from easybuild.tools.utilities import nub
+from easybuild.tools.utilities import nub, remove_unwanted_chars
 
 try:
     import requests
@@ -2011,3 +2013,94 @@ def install_fake_vsc():
     sys.path.insert(0, fake_vsc_path)
 
     return fake_vsc_path
+
+
+def get_easyblock_class_name(path):
+    """Make sure file is an easyblock and get easyblock class name"""
+    fn = os.path.basename(path).split('.')[0]
+    mod = imp.load_source(fn, path)
+    clsmembers = inspect.getmembers(mod, inspect.isclass)
+    for cn, co in clsmembers:
+        if co.__module__ == mod.__name__:
+            ancestors = inspect.getmro(co)
+            if any(a.__name__ == 'EasyBlock' for a in ancestors):
+                return cn
+    return None
+
+
+def is_generic_easyblock(easyblock):
+    """Return whether specified easyblock name is a generic easyblock or not."""
+
+    return easyblock and not easyblock.startswith(EASYBLOCK_CLASS_PREFIX)
+
+
+def copy_easyblocks(paths, target_dir):
+    """ Find right location for easyblock file and copy it there"""
+    file_info = {
+        'eb_names': [],
+        'paths_in_repo': [],
+        'new': [],
+    }
+
+    subdir = os.path.join('easybuild', 'easyblocks')
+    if os.path.exists(os.path.join(target_dir, subdir)):
+        for path in paths:
+            cn = get_easyblock_class_name(path)
+            if not cn:
+                raise EasyBuildError("Could not determine easyblock class from file %s" % path)
+
+            eb_name = remove_unwanted_chars(decode_class_name(cn).replace('-', '_')).lower()
+
+            if is_generic_easyblock(cn):
+                pkgdir = GENERIC_EASYBLOCK_PKG
+            else:
+                pkgdir = eb_name[0]
+
+            target_path = os.path.join(subdir, pkgdir, eb_name + '.py')
+
+            full_target_path = os.path.join(target_dir, target_path)
+            file_info['eb_names'].append(eb_name)
+            file_info['paths_in_repo'].append(full_target_path)
+            file_info['new'].append(not os.path.exists(full_target_path))
+            copy_file(path, full_target_path, force_in_dry_run=True)
+
+    else:
+        raise EasyBuildError("Could not find %s subdir in %s", subdir, target_dir)
+
+    return file_info
+
+
+def copy_framework_files(paths, target_dir):
+    """ Find right location for framework file and copy it there"""
+    file_info = {
+        'paths_in_repo': [],
+        'new': [],
+    }
+
+    paths = [os.path.abspath(path) for path in paths]
+
+    framework_topdir = 'easybuild-framework'
+
+    for path in paths:
+        target_path = None
+        dirnames = os.path.dirname(path).split(os.path.sep)
+
+        if framework_topdir in dirnames:
+            # construct subdirectory by grabbing last entry in dirnames until we hit 'easybuild-framework' dir
+            subdirs = []
+            while(dirnames[-1] != framework_topdir):
+                subdirs.insert(0, dirnames.pop())
+
+            parent_dir = os.path.join(*subdirs) if subdirs else ''
+            target_path = os.path.join(target_dir, parent_dir, os.path.basename(path))
+        else:
+            raise EasyBuildError("Specified path '%s' does not include a '%s' directory!", path, framework_topdir)
+
+        if target_path:
+            file_info['paths_in_repo'].append(target_path)
+            file_info['new'].append(not os.path.exists(target_path))
+            copy_file(path, target_path)
+        else:
+            raise EasyBuildError("Couldn't find parent folder of updated file: %s", path)
+
+    return file_info