Add 'law luigid' sub cli.

.github/workflows/lint_and_test.yml

@@ -97,7 +97,7 @@ jobs:
         run: ./tests/docker.sh riga/law ./tests/coverage.sh
 
       - name: Upload report 🔝
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.xml

docs/api/cli/index.rst

@@ -12,3 +12,5 @@ law.cli
    completion
    config
    software
+   quickstart
+   luigid

docs/api/cli/luigid.rst

@@ -0,0 +1,5 @@
+law.cli.luigid
+==============
+
+.. automodule:: law.cli.luigid
+   :members:

docs/api/cli/quickstart.rst

@@ -0,0 +1,5 @@
+law.cli.quickstart
+==================
+
+.. automodule:: law.cli.quickstart
+   :members:

law.cfg.example

@@ -853,9 +853,17 @@
 ; existing, the values above are used. The only exception is ``htcondor_job_file_dir_cleanup`` whose
 ; default value is ``False``.
 
+; htcondor_job_grouping_submit
+; Desciption: Whether to use job grouping (cluster submission in HTCondor nomenclature) or not. If
+; not, the standard batched submission is used and settings such as "htcondor_chunk_size_submit" and
+; "htcondor_merge_job_files" are considered.
+; Type: boolean
+; Default: True
+
 ; htcondor_chunk_size_submit
 ; Description: Number of jobs that can be submitted in parallel inside a single call to
-; "law.htcondor.HTCondorJobManager.submit", i.e., in a single ``condor_submit`` command.
+; "law.htcondor.HTCondorJobManager.submit", i.e., in a single "condor_submit" command. Ignored when
+; job grouping is enabled in "htcondor_job_grouping_submit".
 ; Type: ``int``
 ; Default: ``25``
 
@@ -874,8 +882,9 @@
 
 ; htcondor_merge_job_files
 ; Description: A boolean flag that decides whether multiple job description files should be merged
-; into a single file before submission. When ``False``, the ``htcondor_chunk_size_submit`` option is
-; not considered either.
+; into a single file before submission. Ignored when job grouping is enabled in
+; "htcondor_job_grouping_submit". When "False", the "htcondor_chunk_size_submit" option is not
+; considered either.
 ; Type: ``bool``
 ; Default: ``True``
 

law/cli/cli.py

@@ -13,7 +13,7 @@
 import law
 
 
-progs = ["run", "index", "config", "software", "completion", "location", "quickstart"]
+progs = ["run", "index", "config", "software", "completion", "location", "quickstart", "luigid"]
 
 
 def run(argv: list[str] | None = None) -> int:
@@ -52,13 +52,14 @@ def run(argv: list[str] | None = None) -> int:
     # argv that is passed to the prog execution when set
     prog_argv = None
 
-    # parse args and dispatch execution, with "run" being a special case
+    # parse args and dispatch execution, considering some special cases
     prog: str | None = argv[0] if argv else None
     if prog == "run":
-        # only pass the prog and the task family to the parser
-        # and let luigi's parsing handle the rest downstream during execute
         args = parser.parse_args(argv[:2])
         prog_argv = ["law run"] + argv
+    elif prog == "luigid":
+        args = parser.parse_args(argv[:1])
+        prog_argv = ["law luigid"] + argv
     else:
         args = parser.parse_args(argv)
 

law/cli/completion.sh

@@ -26,7 +26,7 @@ _law_complete() {
     local index_file="${LAW_INDEX_FILE:-${law_home}/index}"
 
     # common parameters
-    local common_params="run index config software completion location quickstart --help --version"
+    local common_params="run index config software completion location quickstart luigid --help --version"
 
     # the current word
     local cur="${COMP_WORDS[COMP_CWORD]}"
@@ -204,6 +204,14 @@ _law_complete() {
         COMPREPLY=( $( compgen -W "$( echo ${words} )" -P "--" -- "${inp}" ) )
         return "0"
 
+    # complete the "luigid" subcommand
+    elif [ "${sub_cmd}" = "luigid" ]; then
+        local words="help background pidfile logdir state-path address unix-socket port"
+        local inp="${cur##-}"
+        inp="${inp##-}"
+        COMPREPLY=( $( compgen -W "$( echo ${words} )" -P "--" -- "${inp}" ) )
+        return "0"
+
     fi
 }
 

law/cli/luigid.py

@@ -0,0 +1,47 @@
+# coding: utf-8
+
+"""
+"law luigid" cli subprogram.
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from law.config import Config
+
+
+def setup_parser(sub_parsers: argparse._SubParsersAction) -> None:
+    """
+    Sets up the command line parser for the *luigid* subprogram and adds it to *sub_parsers*.
+    This is identical to running ``luigid`` directly from the command line, but this also loads the
+    law config prior to starting the scheduler process and optionally syncrhonizes it with luigi's
+    config.
+    """
+    parser = sub_parsers.add_parser(
+        "luigid",
+        prog="law luigid",
+        description="Starts the 'luigid' central scheduler.",
+        add_help=False,
+    )
+
+    parser.add_argument(
+        "arguments",
+        nargs="*",
+        help="luigid arguments",
+    )
+
+
+def execute(args: argparse.Namespace, argv: list[str]) -> int:
+    """
+    Initializes the law config object and executes the *luigid* subprogram with parsed commandline
+    *args*.
+    """
+    # initialize the config
+    Config.instance()
+
+    # forward to luigid
+    from luigi.cmdline import luigid  # type: ignore[import-untyped]
+    luigid(argv[2:])
+
+    return 0

law/cli/run.py

@@ -11,8 +11,6 @@
 import pathlib
 import argparse
 
-from luigi.cmdline import luigi_run  # type: ignore[import-untyped]
-
 from law.config import Config
 from law.task.base import Task
 from law.util import abort
@@ -89,6 +87,7 @@ def execute(args: argparse.Namespace, argv: list[str]) -> int:
         return abort(f"task '{args.task_family}' not found")
 
     # run luigi
+    from luigi.cmdline import luigi_run  # type: ignore[import-untyped]
     sys.argv[0] += " run"
     success = luigi_run([task_family] + argv[3:])
 

law/target/collection.py

@@ -179,7 +179,7 @@ def exists(self, **kwargs) -> bool:
 
         # simple counting with early stopping criteria for both success and fail cases
         n = 0
-        for i, targets in enumerate(self.iter_existing(**kwargs)):
+        for i, _ in enumerate(self.iter_existing(**kwargs)):
             n += 1
 
             # check for early success