Command ls works fine

TODO

@@ -13,6 +13,7 @@ Write custom generator splitter
 Write export of task
 
 Config should raise SyntaxError with filename, lineno, offset, text
+If not able to get these info, set to "unknown"
 
 Action stdout and stderr, create file in Popen
 
@@ -21,16 +22,18 @@ Accept both single file and re pattern
 
 backend_data in manager.config should be completed
 
+Use a navigator list file instead of 00-xx, 01-xxx, 02-yyy
+
 ConfigManager should deal with multiple config files from extensions
 
 logging color
 
 Test on ubuntu to check the shell scripts about space in dir name, especially running on sh
 
+Configure TZ in jsubrc
+
 
 
 Badge for Travis, Test Coverage, Code Climate
 
 Write full tests
-
-Write documentation on jsubpy.github.io (http://idratherbewriting.com/documentation-theme-jekyll/)

jsub/__init__.py

@@ -43,9 +43,9 @@ def rename(self, task_id, new_task_name):
         op = self.__load_operation()
         return op.rename(task_id, new_task_name)
 
-    def list(self, task_id=None):
+    def ls(self, task_id=None):
         op = self.__load_operation()
-        return op.list(task_id)
+        return op.ls(task_id)
 
     def show(self, task_id):
         op = self.__load_operation()

jsub/command/__init__.py

@@ -1,13 +1,12 @@
-#!/usr/bin/env python
-
 import click
 
 from jsub import version as jsub_version
 
-from jsub.command.create  import Create
-from jsub.command.submit  import Submit
-from jsub.command.run     import Run
-from jsub.command.show    import Show
+from jsub.command.create import Create
+from jsub.command.submit import Submit
+from jsub.command.run    import Run
+from jsub.command.ls     import Ls
+from jsub.command.show   import Show
 
 
 @click.group()
@@ -23,10 +22,11 @@ def version():
 
 
 @cli.command()
-@click.argument('task_profile')
+@click.argument('task_profile', type=click.Path(exists=True))
 @click.pass_context
 def create(ctx, task_profile):
-    cmd = Create(jsubrc=ctx.obj['jsubrc'], task_profile_file=task_profile)
+    task_profile_file = click.format_filename(task_profile)
+    cmd = Create(jsubrc=ctx.obj['jsubrc'], task_profile_file=task_profile_file)
     cmd.execute()
 
 
@@ -41,10 +41,11 @@ def submit(ctx, dry_run, task_id):
 
 @cli.command()
 @click.option('--dry-run', is_flag=True, help='Create necessary files without final submission')
-@click.argument('task_profile')
+@click.argument('task_profile', type=click.Path(exists=True))
 @click.pass_context
 def run(ctx, dry_run, task_profile):
-    cmd = Run(jsubrc=ctx.obj['jsubrc'], task_profile_file=task_profile, dry_run=dry_run)
+    task_profile_file = click.format_filename(task_profile)
+    cmd = Run(jsubrc=ctx.obj['jsubrc'], task_profile_file=task_profile_file, dry_run=dry_run)
     cmd.execute()
 
 
@@ -56,6 +57,14 @@ def show(ctx, task_id):
     cmd.execute()
 
 
+@cli.command()
+@click.argument('task_ids', type=int, nargs=-1)
+@click.pass_context
+def ls(ctx, task_ids):
+    cmd = Ls(jsubrc=ctx.obj['jsubrc'], task_ids=list(task_ids))
+    cmd.execute()
+
+
 def main():
     cli(obj={})
 

jsub/command/ls.py

@@ -0,0 +1,89 @@
+import datetime
+
+import click
+
+from jsub import Jsub
+
+
+COLUMN_TITLE = {
+    'task_id':    'Task ID',
+    'name':       'Name',
+    'app':        'Application',
+    'backend':    'Backend',
+    'status':     'Status',
+    'created_at': 'Creation Time (UTC)',
+}
+
+
+def _convert_table_data(tasks_data, columns):
+    table_data = []
+    for task_data in tasks_data:
+        line_data = []
+        for col in columns:
+            if col not in task_data:
+                line_data.append('N/A')
+                continue
+
+            if col in ['app', 'backend']:
+                if 'type' in task_data[col]:
+                    line_data.append(task_data[col]['type'])
+                else:
+                    line_data.append(str(task_data[col]))
+            elif col in ['created_at']:
+                created_time = datetime.datetime.strptime(task_data[col], '%Y-%m-%dT%H:%M:%S.%f')
+                time_str = datetime.datetime.strftime(created_time, '%Y-%m-%d %H:%M:%S')
+                line_data.append(time_str)
+            else:
+                line_data.append(str(task_data[col]))
+        table_data.append(line_data)
+    return table_data
+
+def _print_header(columns, widths):
+    for col, w in zip(columns, widths):
+        fmt = '%%-%ds' % w
+        click.echo(fmt % COLUMN_TITLE[col], nl=False)
+        click.echo(' ', nl=False)
+    click.echo('')
+
+def _print_hr(widths):
+    for w in widths:
+        click.echo('-'*w, nl=False)
+        click.echo(' ', nl=False)
+    click.echo('')
+
+def _print_data(table_data, widths):
+    for line_data in table_data:
+        for data, w in zip(line_data, widths):
+            fmt = '%%-%ds' % w
+            click.echo(fmt % data, nl=False)
+            click.echo(' ', nl=False)
+        click.echo('')
+
+def _print_table(table_data, columns):
+    widths = []
+    index = 0
+    for col in columns:
+        width = len(COLUMN_TITLE[col])
+        for data in table_data:
+            width = max(width, len(data[index]))
+        widths.append(width+1)
+        index += 1
+
+    _print_header(columns, widths)
+    _print_hr(widths)
+    _print_data(table_data, widths)
+
+
+class Ls(object):
+    def __init__(self, jsubrc, task_ids):
+        self.__jsubrc   = jsubrc
+        self.__task_ids = task_ids if len(task_ids) else None
+
+    def execute(self):
+        j = Jsub(self.__jsubrc)
+        tasks_data = j.ls(self.__task_ids)
+
+        columns = ['task_id', 'name', 'app', 'backend', 'status', 'created_at']
+
+        table_data = _convert_table_data(tasks_data, columns)
+        _print_table(table_data, columns)

jsub/error.py

@@ -1,8 +1,26 @@
 class JsubError(Exception):
     pass
 
+
 class SplitterParamError(JsubError):
     pass
 
 class LauncherNotFoundError(JsubError):
     pass
+
+
+class TaskIdFormatError(JsubError):
+    pass
+
+class TaskNotFoundError(JsubError):
+    pass
+
+
+class RepoError(JsubError):
+    pass
+
+class RepoReadError(RepoError):
+    pass
+
+class RepoWriteError(RepoError):
+    pass

jsub/exts/bootstrap/shell/bootstrap.sh

@@ -26,7 +26,6 @@ mkdir -p "$output_root"
 
 
 log_file="${log_root}/bootstrap.log"
-err_file="${log_root}/bootstrap.err"
 
 logging '================================================================================'
 logging "Current directory: ${main_root}"
@@ -36,7 +35,7 @@ logging "Search for valid navigator..."
 for navigator_dir in "${navigator_root}"/*
 do
     navigator_exe=$(cat "${navigator_dir}/executable")
-    navigator_ok=$(${navigator_dir}/${navigator_exe} --validate 2>/dev/null)
+    navigator_ok=$(${navigator_dir}/${navigator_exe} --validate 2>>$log_file)
     if [ $? = 0 -a "$navigator_ok" = 'JSUB navigator OK' ]; then
         navigator="$navigator_dir"
         break

jsub/exts/repo/file_system.py

@@ -1,25 +1,30 @@
 import os
 import json
+import logging
 import fcntl
 
-from jsub.util import safe_mkdir
+from jsub.util  import safe_mkdir
+from jsub.error import RepoReadError
+from jsub.error import TaskNotFoundError
 
 ID_FILENAME = 'id'
 
 class FileSystem(object):
     def __init__(self, param):
-        self.__repo_dir = os.path.expanduser(param.get('dir', '~/jsub_repo'))
+        self.__repo_dir = os.path.expanduser(param.get('dir', '~/jsub/repo'))
         self.__task_dir = os.path.join(self.__repo_dir, 'task')
-        self.__id_file = os.path.join(self.__repo_dir, ID_FILENAME)
+        self.__id_file  = os.path.join(self.__repo_dir, ID_FILENAME)
+
+        self.__logger = logging.getLogger('JSUB')
 
         self.__create_repo_dir()
 
         self.__json_format = param.get('format', 'compact')
 
     def save_task(self, data):
-        if 'task_id' not in data:
-            data['task_id'] = self.__new_task_id()
-        task_path = os.path.join(self.__task_dir, str(data['task_id']))
+        if 'id' not in data:
+            data['id'] = self.__new_task_id()
+        task_path = os.path.join(self.__task_dir, str(data['id']))
 
         data_str = self.__json_str(data)
         with open(task_path, 'a+') as f:
@@ -28,19 +33,35 @@ def save_task(self, data):
             f.truncate()
             f.write(data_str)
 
-    def all_task_data(self):
+    def find_by_id(self, task_id):
+        return self.task_data(task_id)
+
+    def find_by_ids(self, task_ids):
         all_data = []
-        for task_id in os.listdir(self.__task_dir):
-            all_data.append(self.task_data(task_id))
+        for task_id in task_ids:
+            try:
+                td = self.task_data(task_id)
+                all_data.append(td)
+            except RepoReadError as e:
+                self.__logger.debug(e)
         return all_data
 
+    def all_task_data(self, order='asc'):
+        task_ids = os.listdir(self.__task_dir)
+        task_ids.sort(key=int, reverse=(order=='desc'))
+        return self.find_by_ids(task_ids)
+
     def task_data(self, task_id):
         task_path = os.path.join(self.__task_dir, str(task_id))
         with open(task_path, 'a+') as f:
             fcntl.flock(f, fcntl.LOCK_EX)
             f.seek(0)
             data_str = f.read()
-        return json.loads(data_str)
+
+        try:
+            return json.loads(data_str)
+        except ValueError as e:
+            raise RepoReadError('JSON decode error on task %s: %s' % (task_id, e))
 
     def __create_repo_dir(self):
         safe_mkdir(self.__repo_dir)

jsub/manager/navigator.py

@@ -18,9 +18,9 @@ def scenario_format(self, navigators):
         return formats
 
     def create_navigators(self, navigators, dst_dir):
-        digit_number = len(str(len(navigators)))
+        digit_number = len(str(len(navigators) - 1))
         navigator_format = '%%0%dd-%%s' % max(digit_number, 2)
-        navigator_index = 1
+        navigator_index = 0
         for navigator_type in navigators:
             nav_dir = self.__ext_mgr.ext_dir('navigator', navigator_type)
             nav_param = self.__nav_param(navigator_type)

jsub/operation/__init__.py

@@ -1,5 +1,6 @@
 from jsub.operation.create import Create
 from jsub.operation.submit import Submit
+from jsub.operation.ls     import Ls
 
 class Operation(object):
     def __init__(self, manager):
@@ -13,3 +14,7 @@ def create(self, task_profile):
     def submit(self, task, sub_ids=None, dry_run=False):
         handler = Submit(self.__manager, task, sub_ids, dry_run)
         return handler.handle()
+
+    def ls(self, task_id):
+        handler = Ls(self.__manager, task_id)
+        return handler.handle()

jsub/operation/ls.py

@@ -0,0 +1,36 @@
+from jsub.error import TaskIdFormatError
+
+def _convert_tasks_data(tasks):
+    tasks_data = []
+    for task in tasks:
+        tasks_data.append(task.data)
+    return tasks_data
+
+class Ls(object):
+    def __init__(self, manager, task_id=None):
+        self.__manager = manager
+        self.__task_id = task_id
+
+        self.__initialize_manager()
+
+    def __initialize_manager(self):
+        self.__config_mgr   = self.__manager.load_config_manager()
+
+
+    def handle(self):
+        if self.__task_id is None:
+            return self.__all_tasks()
+
+        if isinstance(self.__task_id, (int, list)):
+            return self.__find_tasks(self.__task_id)
+
+        raise TaskIdFormatError('Unknown task ID format: %s' % self.__task_id)
+
+
+    def __all_tasks(self):
+        task_pool = self.__manager.load_task_pool()
+        return _convert_tasks_data(task_pool.all())
+
+    def __find_tasks(self, task_ids):
+        task_pool = self.__manager.load_task_pool()
+        return _convert_tasks_data(task_pool.find(task_ids))

jsub/operation/submit.py

@@ -11,7 +11,7 @@ def __init__(self, manager, task, sub_ids=None, dry_run=False):
         self.__sub_ids = sub_ids
         self.__dry_run = dry_run
 
-        self.__logger  = logging.getLogger('JSUB')
+        self.__logger = logging.getLogger('JSUB')
 
         if self.__sub_ids is None:
             self.__sub_ids = list(range(len(self.__task.data['jobvar'])))

jsub/support/jsubrc.example

@@ -1,5 +1,8 @@
 log_level: DEBUG
+time_zone: utc
+
 extensions: []
+
 max_jobs: 100000
 
 backend: dirac