Improve config variable expansion in sandboxing.

law.cfg.example

@@ -68,7 +68,8 @@
 
 ; --- Environment variable expansion ---------------------------------------------------------------
 ; Configuration values that contain environment variables are automatically expanded by most getters
-; of the law configuration parser, such as "get_expanded", "get_expanded_int", etc.
+; of the law configuration parser, such as "get_expanded", "get_expanded_int", etc. If desired, you
+; can avoid this expanion by backslash-escaping characters such as "$" or "~" to "\$" or "\~".
 
 ; --- The "None" value -----------------------------------------------------------------------------
 ; Getters of the law configuration parser such as "get_default", get_expanded", "get_expanded_int",
@@ -1042,7 +1043,8 @@
 ; Note:
 ; Here you can define environment variables via key-value pairs that are accessible in a bash
 ; sandbox. When an option has no value, i.e., when only a key is given, the value of the variable of
-; the host environment is used.
+; the host environment is used. Also note that, unless escaped with a blackslach, "$" and "~" are
+; expanded with variables of the outer task environment.
 ; The environment variables defined in this section are applied to all bash sandboxes. To configure
 ; variables per bash initialization file, create a section "bash_sandbox_env_<init_file>" with the
 ; desired values.
@@ -1086,7 +1088,8 @@
 ; Note:
 ; Here you can define environment variables via key-value pairs that are accessible in a venv
 ; sandbox. When an option has no value, i.e., when only a key is given, the value of the variable of
-; the host environment is used.
+; the host environment is used. Also note that, unless escaped with a blackslach, "$" and "~" are
+; expanded with variables of the outer task environment.
 ; The environment variables defined in this section are applied to all venv sandboxes. To configure
 ; variables per venv directory, create a section "venv_sandbox_env_<venv_dir>" with the desired
 ; values.
@@ -1159,7 +1162,8 @@
 ; Note:
 ; Here you can define environment variables via key-value pairs that are accessible in a docker
 ; sandbox. When an option has no value, i.e., when only a key is given, the value of the variable of
-; the host environment is used.
+; the host environment is used. Also note that, unless escaped with a blackslach, "$" and "~" are
+; expanded with variables of the outer task environment.
 ; The environment variables defined in this section are applied to all docker sandboxes. To
 ; configure variables per image, create a section "docker_sandbox_env_<image_name>" with the desired
 ; values.
@@ -1254,7 +1258,8 @@
 ; Note:
 ; Here you can define environment variables via key-value pairs that are accessible in a singularity
 ; sandbox. When an option has no value, i.e., when only a key is given, the value of the variable of
-; the host environment is used.
+; the host environment is used. Also note that, unless escaped with a blackslach, "$" and "~" are
+; expanded with variables of the outer task environment.
 ; The environment variables defined in this section are applied to all singularity sandboxes. To
 ; configure variables per image, create a section "singularity_sandbox_env_<image_name>" with the
 ; desired values.
@@ -1315,7 +1320,8 @@
 ; Note:
 ; Here you can define environment variables via key-value pairs that are accessible in a cmssw
 ; sandbox. When an option has no value, i.e., when only a key is given, the value of the variable of
-; the host environment is used.
+; the host environment is used. Also note that, unless escaped with a blackslach, "$" and "~" are
+; expanded with variables of the outer task environment.
 ; The environment variables defined in this section are applied to all cmssw sandboxes. To
 ; configure variables per cmssw version, create a section "cmssw_sandbox_env_<cmssw_version>" with
 ; the desired values.

law/config.py

@@ -176,6 +176,22 @@ def _parse_option_ref(cls, value, default_section=None):
             return None
         return (m.group("section") or default_section, m.group("option"))
 
+    @classmethod
+    def _expand_path(cls, path, expand_vars=True, expand_user=True):
+        if expand_vars:
+            ph = "__law_tilde__"
+            path = path.replace(r"\~", ph)
+            path = os.path.expanduser(path)
+            path = path.replace(ph, "~")
+
+        if expand_user:
+            ph = "__law_dollar__"
+            path = path.replace(r"\$", ph)
+            path = os.path.expandvars(path)
+            path = path.replace(ph, "$")
+
+        return path
+
     def __init__(self, config_file="", skip_defaults=False, skip_fallbacks=False,
             skip_includes=False, skip_env_sync=False, skip_luigi_sync=False):
         ConfigParser.__init__(self, allow_no_value=True)
@@ -260,23 +276,23 @@ def _convert_to_boolean(self, value):
 
         if value.lower() not in self._boolean_states:
             raise ValueError("Not a boolean: {}".format(value))
+
         return self._boolean_states[value.lower()]
 
     def _get_type_converter(self, type, value):
         if type in (str, "str", "s"):
             return str
         if type in (int, "int", "i"):
             return str_to_int
-        elif type in (float, "float", "f"):
+        if type in (float, "float", "f"):
             return float
-        elif type in (bool, "bool", "boolean", "b"):
+        if type in (bool, "bool", "boolean", "b"):
             if isinstance(value, six.string_types):
                 return self._convert_to_boolean
-            else:
-                return bool
-        else:
-            raise ValueError("unknown 'type' argument ({}), must be 'str', 'int', 'float', or "
-                "'bool'".format(type))
+            return bool
+
+        raise ValueError("unknown 'type' argument ({}), must be 'str', 'int', 'float', or "
+            "'bool'".format(type))
 
     def optionxform(self, option):
         """"""
@@ -292,10 +308,7 @@ def options(self, section, prefix=None, expand_vars=True, expand_user=True):
         for option in ConfigParser.options(self, section):
             if prefix and not option.startswith(prefix):
                 continue
-            if expand_vars:
-                option = os.path.expandvars(option)
-            if expand_user:
-                option = os.path.expanduser(option)
+            option = self._expand_path(option, expand_vars=expand_vars, expand_user=expand_user)
             options.append(option)
         return options
 
@@ -412,10 +425,7 @@ def get_default(self, section, option, default=no_value, type=None, expand_vars=
         # (which should always be the case, but subclasses might overwrite get())
         if isinstance(value, six.string_types):
             # expand
-            if expand_vars:
-                value = os.path.expandvars(value)
-            if expand_user:
-                value = os.path.expanduser(value)
+            value = self._expand_path(value, expand_vars=expand_vars, expand_user=expand_user)
 
             # resolve references
             if dereference:

law/sandbox/base.py

@@ -285,8 +285,8 @@ def _get_volumes(self):
         # extend by volumes from the config file
         cfg = Config.instance()
         section = self.get_config_section(postfix="volumes")
-        for hdir, cdir in cfg.items(section, expand_vars=False, expand_user=False):
-            volumes[os.path.expandvars(os.path.expanduser(hdir))] = cdir
+        for hdir, cdir in cfg.items(section):
+            volumes[hdir] = cdir
 
         # extend by volumes defined on task level
         if self.task:
@@ -314,8 +314,8 @@ def _build_setup_cmds(self, env):
         # commands that are used to setup the env and actual run commands
         setup_cmds = []
 
-        for tpl in six.iteritems(env):
-            setup_cmds.append("export {}=\"{}\"".format(*tpl))
+        for key, value in env.items():
+            setup_cmds.append("export {}=\"{}\"".format(key, value))
 
         if self.task:
             setup_cmds.extend(self.task.sandbox_setup_cmds())
@@ -500,7 +500,7 @@ def print_banner(msg, color):
 
         # log the command
         if cmd:
-            self.task.logger.debug("sandbox command:\n{}".format(cmd))
+            logger.debug("sandbox command:\n{}".format(cmd))
         sys.stdout.flush()
 
         try:

law/task/base.py

@@ -358,9 +358,6 @@ class Register(BaseRegister):
     def __call__(cls, *args, **kwargs):
         inst = super(Register, cls).__call__(*args, **kwargs)
 
-        # manually set the root task once
-        root_task(inst)
-
         # check for interactive parameters
         for param in inst.interactive_params:
             value = getattr(inst, param)