Merge pull request #4 from boegel/generic-easyblock-docs-issue636

sync with develop & resolve conflict

easybuild/framework/easyblock.py

@@ -59,7 +59,7 @@
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, print_error, print_msg
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
-from easybuild.tools.config import install_path, log_path, read_only_installdir, source_paths
+from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env
 from easybuild.tools.filetools import DEFAULT_CHECKSUM
 from easybuild.tools.filetools import adjust_permissions, apply_patch, convert_name, download_file, encode_class_name
@@ -68,9 +68,10 @@
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator
-from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
+from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import get_software_root, modules_tool
+from easybuild.tools.package.utilities import package
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
 from easybuild.tools.systemtools import det_parallelism, use_group
@@ -86,6 +87,7 @@
 MODULE_STEP = 'module'
 PACKAGE_STEP = 'package'
 PATCH_STEP = 'patch'
+PERMISSIONS_STEP = 'permissions'
 POSTPROC_STEP = 'postproc'
 PREPARE_STEP = 'prepare'
 READY_STEP = 'ready'
@@ -1490,15 +1492,34 @@ def extensions_step(self, fetch=False):
         self.clean_up_fake_module(fake_mod_data)
 
     def package_step(self):
-        """Package software (e.g. into an RPM)."""
-        pass
+        """Package installed software (e.g., into an RPM), if requested, using selected package tool."""
+
+        if build_option('package'):
+
+            pkgtype = build_option('package_type')
+            pkgdir_dest = os.path.abspath(package_path())
+            opt_force = build_option('force')
+
+            self.log.info("Generating %s package in %s", pkgtype, pkgdir_dest)
+            pkgdir_src = package(self)
+
+            mkdir(pkgdir_dest)
+
+            for src_file in glob.glob(os.path.join(pkgdir_src, "*.%s" % pkgtype)):
+                dest_file = os.path.join(pkgdir_dest, os.path.basename(src_file))
+                if os.path.exists(dest_file) and not opt_force:
+                    raise EasyBuildError("Unable to copy package %s to %s (already exists).", src_file, dest_file)
+                else:
+                    self.log.info("Copied package %s to %s", src_file, pkgdir_dest)
+                    shutil.copy(src_file, pkgdir_dest)
+
+        else:
+            self.log.info("Skipping package step (not enabled)")
 
     def post_install_step(self):
         """
         Do some postprocessing
         - run post install commands if any were specified
-        - set file permissions ....
-        Installing user must be member of the group that it is changed to
         """
         if self.cfg['postinstallcmds'] is not None:
             # make sure we have a list of commands
@@ -1509,27 +1530,6 @@ def post_install_step(self):
                     raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd)
                 run_cmd(cmd, simple=True, log_ok=True, log_all=True)
 
-        if self.group is not None:
-            # remove permissions for others, and set group ID
-            try:
-                perms = stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH
-                adjust_permissions(self.installdir, perms, add=False, recursive=True, group_id=self.group[1],
-                                   relative=True, ignore_errors=True)
-            except EasyBuildError, err:
-                raise EasyBuildError("Unable to change group permissions of file(s): %s", err)
-            self.log.info("Successfully made software only available for group %s (gid %s)" % self.group)
-
-        if read_only_installdir():
-            # remove write permissions for everyone
-            perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH
-            adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True)
-            self.log.info("Successfully removed write permissions recursively for *EVERYONE* on install dir.")
-        else:
-            # remove write permissions for group and other to protect installation
-            perms = stat.S_IWGRP | stat.S_IWOTH
-            adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True)
-            self.log.info("Successfully removed write permissions recursively for group/other on install dir.")
-
     def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False):
         """
         Do a sanity check on the installation
@@ -1715,6 +1715,39 @@ def make_module_step(self, fake=False):
 
         return modpath
 
+    def permissions_step(self):
+        """
+        Finalize installation procedure: adjust permissions as configured, change group ownership (if requested).
+        Installing user must be member of the group that it is changed to.
+        """
+        if self.group is not None:
+            # remove permissions for others, and set group ID
+            try:
+                perms = stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH
+                adjust_permissions(self.installdir, perms, add=False, recursive=True, group_id=self.group[1],
+                                   relative=True, ignore_errors=True)
+            except EasyBuildError, err:
+                raise EasyBuildError("Unable to change group permissions of file(s): %s", err)
+            self.log.info("Successfully made software only available for group %s (gid %s)" % self.group)
+
+        if build_option('read_only_installdir'):
+            # remove write permissions for everyone
+            perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH
+            adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True)
+            self.log.info("Successfully removed write permissions recursively for *EVERYONE* on install dir.")
+
+        elif build_option('group_writable_installdir'):
+            # enable write permissions for group
+            perms = stat.S_IWGRP
+            adjust_permissions(self.installdir, perms, add=True, recursive=True, relative=True, ignore_errors=True)
+            self.log.info("Successfully enabled write permissions recursively for group on install dir.")
+
+        else:
+            # remove write permissions for group and other
+            perms = stat.S_IWGRP | stat.S_IWOTH
+            adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True)
+            self.log.info("Successfully removed write permissions recursively for group/other on install dir.")
+
     def test_cases_step(self):
         """
         Run provided test cases.
@@ -1863,11 +1896,12 @@ def prepare_step_spec(initial):
         # part 3: post-iteration part
         steps_part3 = [
             (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step()], False),
-            (PACKAGE_STEP, 'packaging', [lambda x: x.package_step()], True),
             (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step()], True),
             (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step()], False),
             (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step()], False),
             (MODULE_STEP, 'creating module', [lambda x: x.make_module_step()], False),
+            (PERMISSIONS_STEP, 'permissions', [lambda x: x.permissions_step()], False),
+            (PACKAGE_STEP, 'packaging', [lambda x: x.package_step()], False),
         ]
 
         # full list of steps, included iterated steps
@@ -1981,6 +2015,9 @@ def build_and_install_one(ecdict, init_env):
                 new_log_dir = os.path.dirname(app.logfile)
         else:
             new_log_dir = os.path.join(app.installdir, config.log_path())
+            if build_option('read_only_installdir'):
+                # temporarily re-enable write permissions for copying log/easyconfig to install dir
+                adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=False)
 
             # collect build stats
             _log.info("Collecting build stats...")
@@ -2022,6 +2059,10 @@ def build_and_install_one(ecdict, init_env):
         except (IOError, OSError), err:
             print_error("Failed to copy easyconfig %s to %s: %s" % (spec, newspec, err))
 
+        if build_option('read_only_installdir'):
+            # take away user write permissions (again)
+            adjust_permissions(new_log_dir, stat.S_IWUSR, add=False, recursive=False)
+
     # build failed
     else:
         success = False

easybuild/framework/easyconfig/easyconfig.py

@@ -422,9 +422,9 @@ def dependencies(self):
 
         # if filter-deps option is provided we "clean" the list of dependencies for
         # each processed easyconfig to remove the unwanted dependencies
+        self.log.debug("Dependencies BEFORE filtering: %s" % deps)
         filter_deps = build_option('filter_deps')
         if filter_deps:
-            self.log.debug("Dependencies BEFORE filtering: %s" % deps)
             filtered_deps = []
             for dep in deps:
                 if dep['name'] not in filter_deps:

easybuild/main.py

@@ -37,6 +37,7 @@
 """
 import copy
 import os
+import stat
 import sys
 import tempfile
 import traceback
@@ -53,9 +54,10 @@
 from easybuild.framework.easyconfig.tools import get_paths_for, parse_easyconfigs, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
 from easybuild.tools.config import get_repository, get_repositorypath
-from easybuild.tools.filetools import cleanup, write_file
+from easybuild.tools.filetools import adjust_permissions, cleanup, write_file
 from easybuild.tools.options import process_software_build_specs
 from easybuild.tools.robot import det_robot_path, dry_run, resolve_dependencies, search_easyconfigs
+from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_module_list, session_state
@@ -129,7 +131,14 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
         test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state)
         if 'log_file' in ec_res:
             test_report_fp = "%s_test_report.md" % '.'.join(ec_res['log_file'].split('.')[:-1])
-            write_file(test_report_fp, test_report_txt)
+            parent_dir = os.path.dirname(test_report_fp)
+            # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used
+            if os.stat(parent_dir).st_mode & 0200:
+                write_file(test_report_fp, test_report_txt)
+            else:
+                adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False)
+                write_file(test_report_fp, test_report_txt)
+                adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
 
         if not ec_res['success'] and exit_on_failure:
             if 'traceback' in ec_res:
@@ -210,6 +219,12 @@ def main(testing_data=(None, None, None)):
     config.init(options, config_options_dict)
     config.init_build_options(build_options=build_options, cmdline_options=options)
 
+    # check whether packaging is supported when it's being used
+    if options.package:
+        check_pkg_support()
+    else:
+        _log.debug("Packaging not enabled, so not checking for packaging support.")
+
     # update session state
     eb_config = eb_go.generate_cmd_line(add_default=True)
     modlist = session_module_list(testing=testing)  # build options must be initialized first before 'module list' works

easybuild/tools/config.py

@@ -52,6 +52,10 @@
 _log = fancylogger.getLogger('config', fname=False)
 
 
+PKG_TOOL_FPM = 'fpm'
+PKG_TYPE_RPM = 'rpm'
+
+
 DEFAULT_JOB_BACKEND = 'PbsPython'
 DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
 DEFAULT_MNS = 'EasyBuildMNS'
@@ -60,16 +64,20 @@
 DEFAULT_PATH_SUBDIRS = {
     'buildpath': 'build',
     'installpath': '',
+    'packagepath': 'packages',
     'repositorypath': 'ebfiles_repo',
     'sourcepath': 'sources',
     'subdir_modules': 'modules',
     'subdir_software': 'software',
 }
+DEFAULT_PKG_RELEASE = '1'
+DEFAULT_PKG_TOOL = PKG_TOOL_FPM
+DEFAULT_PKG_TYPE = PKG_TYPE_RPM
+DEFAULT_PNS = 'EasyBuildPNS'
 DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild")
 DEFAULT_REPOSITORY = 'FileRepository'
 DEFAULT_STRICT = run.WARN
 
-
 # utility function for obtaining default paths
 def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     """Create full path, avoid '/' at the end."""
@@ -115,8 +123,11 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'debug',
         'experimental',
         'force',
+        'group_writable_installdir',
         'hidden',
         'module_only',
+        'package',
+        'read_only_installdir',
         'robot',
         'sequential',
         'set_gid_bit',
@@ -131,6 +142,15 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     DEFAULT_STRICT: [
         'strict',
     ],
+    DEFAULT_PKG_RELEASE: [
+        'package_release',
+    ],
+    DEFAULT_PKG_TOOL: [
+        'package_tool',
+    ],
+    DEFAULT_PKG_TYPE: [
+        'package_type',
+    ],
 }
 # build option that do not have a perfectly matching command line option
 BUILD_OPTIONS_OTHER = {
@@ -203,6 +223,8 @@ class ConfigurationVariables(FrozenDictKnownKeys):
         'module_naming_scheme',
         'module_syntax',
         'modules_tool',
+        'packagepath',
+        'package_naming_scheme',
         'prefix',
         'repository',
         'repositorypath',
@@ -371,6 +393,20 @@ def get_repositorypath():
     return ConfigurationVariables()['repositorypath']
 
 
+def get_package_naming_scheme():
+    """
+    Return the package naming scheme
+    """
+    return ConfigurationVariables()['package_naming_scheme']
+
+
+def package_path():
+    """
+    Return the path where built packages are copied to
+    """
+    return ConfigurationVariables()['packagepath']
+
+
 def get_modules_tool():
     """
     Return modules tool (EnvironmentModulesC, Lmod, ...)
@@ -464,16 +500,6 @@ def get_log_filename(name, version, add_salt=False):
     return filepath
 
 
-def read_only_installdir():
-    """
-    Return whether installation dir should be fully read-only after installation.
-    """
-    # FIXME (see issue #123): add a config option to set this, should be True by default (?)
-    # this also needs to be checked when --force is used;
-    # install dir will have to (temporarily) be made writeable again for owner in that case
-    return False
-
-
 def module_classes():
     """
     Return list of module classes specified in config file.