Merge branch 'master' into centralised-reg-parse

.github/workflows/bash.yml

@@ -59,7 +59,7 @@ jobs:
         run: |
           docker cp bash:/root/cylc-run .
 
-      - name: Upload
+      - name: Upload artifact
         if: failure()
         uses: actions/upload-artifact@v2
         with:

.github/workflows/test_fast.yml

@@ -21,8 +21,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with:
-          fetch-depth: 2  # required by codecov
 
       - name: Configure Python
         uses: actions/setup-python@v2
@@ -49,17 +47,17 @@ jobs:
 
       - name: Install
         run: |
-          pip install ."[all]"
+          pip install -e ."[all]"
 
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
-      - name: style
+      - name: Style
         run: |
           flake8
           etc/bin/shellchecker
 
-      - name: typing
+      - name: Typing
         run: mypy
 
       - name: Doctests
@@ -74,7 +72,13 @@ jobs:
         run: |
           pytest --cov --cov-append -n 5 tests/integration
 
-      - name: Coverage
+      - name: Coverage report
         run: |
+          coverage xml
           coverage report
-          bash <(curl -s https://codecov.io/bash)
+
+      - name: Codecov upload
+        uses: codecov/codecov-action@v2
+        with:
+          name: '"${{ github.workflow }} ${{ matrix.os }} py-${{ matrix.python-version }}"'
+          fail_ci_if_error: true

.github/workflows/test_functional.yml

@@ -221,15 +221,15 @@ jobs:
           find "${TMPDIR:-/tmp}/${USER}/cylctb-"* -type f \
               -exec echo '====== {} ======' \; -exec cat '{}' \;
 
-      - name: Set Upload Name
+      - name: Set artifact upload name
         if: failure() && steps.test.outcome == 'failure'
         id: uploadname
         run: |
           # artifact name cannot contain '/' characters
           CID="$(sed 's|/|-|g' <<< "${{ matrix.name || matrix.chunk }}")"
           echo "::set-output name=uploadname::$CID"
 
-      - name: Upload
+      - name: Upload artifact
         if: failure() && steps.test.outcome == 'failure'
         uses: actions/upload-artifact@v2
         with:
@@ -262,8 +262,14 @@ jobs:
         run: |
           etc/bin/swarm kill
 
-      - name: Coverage
+      - name: Combine coverage & report
         run: |
           coverage combine -a
+          coverage xml
           coverage report
-          bash <(curl -s https://codecov.io/bash)
+
+      - name: Codecov upload
+        uses: codecov/codecov-action@v2
+        with:
+          name: '"${{ github.workflow }} ${{ matrix.name }} ${{ matrix.chunk }}"'
+          fail_ci_if_error: true

CHANGES.md

@@ -63,6 +63,9 @@ erroneous use of both `expr => bar` and `expr => !bar` in the same graph.
 infers the latest numbered run of the workflow for most commands (e.g. you can
 run `cylc pause foo` instead of having to type out `foo/run3`).
 
+[#4346](https://github.com/cylc/cylc-flow/pull/4346) -
+Use natural sort order for the `cylc scan --sort` option.
+
 ### Fixes
 
 [#4310](https://github.com/cylc/cylc-flow/pull/4310 -

cylc/flow/loggingutil.py

@@ -58,16 +58,21 @@ class CylcLogFormatter(logging.Formatter):
     #       deamonise script (url, pid) are not wrapped
     MAX_WIDTH = 999
 
-    def __init__(self, timestamp=True, color=False, max_width=None):
+    def __init__(
+        self, timestamp=True, color=False, max_width=None, dev_info=False
+    ):
         self.timestamp = None
         self.color = None
         self.max_width = self.MAX_WIDTH
         self.wrapper = None
         self.configure(timestamp, color, max_width)
-        # You may find adding %(filename)s %(lineno)d are useful when debugging
+        prefix = '%(asctime)s %(levelname)-2s - '
+        if dev_info is True:
+            prefix += '[%(module)s:%(lineno)d] - '
+
         logging.Formatter.__init__(
             self,
-            '%(asctime)s %(levelname)-2s - %(message)s',
+            prefix + '%(message)s',
             '%Y-%m-%dT%H:%M:%S%Z')
 
     def configure(self, timestamp=None, color=None, max_width=None):

cylc/flow/option_parsers.py

@@ -406,7 +406,9 @@ def parse_args(self, api_args, remove_opts=None):
             LOG.removeHandler(LOG.handlers[0])
         errhandler = logging.StreamHandler(sys.stderr)
         errhandler.setFormatter(CylcLogFormatter(
-            timestamp=options.log_timestamp))
+            timestamp=options.log_timestamp,
+            dev_info=bool(options.verbosity > 2)
+        ))
         LOG.addHandler(errhandler)
 
         return (options, args)

cylc/flow/scripts/report_timings.py

@@ -129,7 +129,6 @@ def main(parser: COP, options: 'Values', workflow: str) -> None:
 
     run_db = _get_dao(workflow)
     row_buf = format_rows(*run_db.select_task_times())
-
     with smart_open(options.output_filename) as output:
         if options.show_raw:
             output.write(row_buf.getvalue())
@@ -218,7 +217,7 @@ def write_summary(self, buf=None):
         if buf is None:
             buf = sys.stdout
         self.write_summary_header(buf)
-        for group, df in self.by_host_and_runner:
+        for group, df in self.by_host_and_job_runner:
             self.write_group_header(buf, group)
             df_reshape = self._reshape_timings(df)
             df_describe = df.groupby(level='name').describe()

cylc/flow/scripts/scan.py

@@ -72,6 +72,7 @@
 )
 from cylc.flow.print_tree import get_tree
 from cylc.flow.terminal import cli_function
+from cylc.flow.util import natural_sort_key
 from cylc.flow.workflow_files import ContactFileFields as Cont
 
 
@@ -356,7 +357,7 @@ def sort_function(flow):
         state = 2
     else:
         state = 3
-    return (state, flow['name'])
+    return (state, *natural_sort_key(flow['name']))
 
 
 async def _sorted(pipe, formatter, opts, write):

cylc/flow/util.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from contextlib import suppress
+from functools import partial
+import re
+from typing import List, Any
+
+
+_NAT_SORT_SPLIT = re.compile(r'([\d\.]+)')
+
+
+def natural_sort_key(key: str, fcns=(int, str)) -> List[Any]:
+    """Returns a key suitable for sorting.
+
+    Splits the key into sortable chunks to preserve numerical order.
+
+    Examples:
+        >>> natural_sort_key('a1b2c3')
+        ['a', 1, 'b', 2, 'c', 3]
+        >>> natural_sort_key('a123b')
+        ['a', 123, 'b']
+        >>> natural_sort_key('a1.23b', fcns=(float, str))
+        ['a', 1.23, 'b']
+        >>> natural_sort_key('a.b')
+        ['a', '.', 'b']
+
+    """
+    ret = []
+    for item in _NAT_SORT_SPLIT.split(key):
+        for fcn in fcns:
+            with suppress(TypeError, ValueError):
+                ret.append(fcn(item))
+                break
+    if ret[-1] == '':
+        ret.pop(-1)
+    return ret
+
+
+def natural_sort(items: List[str], fcns=(int, str)) -> None:
+    """Sorts a list preserving numerical order.
+
+    Note this is an in-place sort.
+
+    Examples:
+        >>> lst = ['a10', 'a1', 'a2']
+        >>> natural_sort(lst)
+        >>> lst
+        ['a1', 'a2', 'a10']
+
+        >>> lst = ['a1', '1a']
+        >>> natural_sort(lst)
+        >>> lst
+        ['1a', 'a1']
+
+    """
+    items.sort(key=partial(natural_sort_key, fcns=fcns))

tests/functional/logging/04-dev_mode.t

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test log dev-mode.
+
+. "$(dirname "$0")/test_header"
+set_test_number 5
+init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__'
+[scheduling]
+    cycling mode = integer
+    initial cycle point = 1
+    final cycle point = 1
+    [[graph]]
+        P1 = t1
+[runtime]
+    [[t1]]
+        script = true
+__FLOW_CONFIG__
+
+run_ok "${TEST_NAME_BASE}-validate-plain" \
+    cylc validate "${WORKFLOW_NAME}"
+
+run_ok "${TEST_NAME_BASE}-validate-vvv" \
+    cylc validate -vvv "${WORKFLOW_NAME}"
+grep_ok " DEBUG - \[config:.*\]" "${TEST_NAME_BASE}-validate-vvv.stderr"
+
+
+run_ok "${TEST_NAME_BASE}-validate-vvv--no-timestamp" \
+    cylc validate -vvv --no-timestamp "${WORKFLOW_NAME}"
+grep_ok "^DEBUG - \[config:.*\]" "${TEST_NAME_BASE}-validate-vvv--no-timestamp.stderr"
+
+purge
+exit

tests/unit/test_loggingutil.py

@@ -17,11 +17,14 @@
 import logging
 import tempfile
 import unittest
+import pytest
 
+from pytest import param
 from unittest import mock
 
 from cylc.flow import LOG
-from cylc.flow.loggingutil import TimestampRotatingFileHandler
+from cylc.flow.loggingutil import (
+    TimestampRotatingFileHandler, CylcLogFormatter)
 
 
 class TestLoggingutil(unittest.TestCase):
@@ -82,3 +85,27 @@ def test_value_error_raises_system_exit(
 
 if __name__ == '__main__':
     unittest.main()
+
+
+@pytest.mark.parametrize(
+    'dev_info, expect',
+    [
+        param(
+            True,
+                (
+                    '%(asctime)s %(levelname)-2s - [%(module)s:%(lineno)d] - '
+                    '%(message)s'
+                ),
+            id='dev_info=True'
+        ),
+        param(
+            False,
+            '%(asctime)s %(levelname)-2s - %(message)s',
+            id='dev_info=False'
+        )
+    ]
+)
+def test_CylcLogFormatter__init__dev_info(dev_info, expect):
+    """dev_info switch changes the logging format string."""
+    formatter = CylcLogFormatter(dev_info=dev_info)
+    assert formatter._fmt == expect