#283 added possibility to exclude files from server_file parameters

samples/configs/parameterized.json

@@ -205,6 +205,10 @@
       "file_dir": "/var/log",
       "file_extensions": [
         "log"
+      ],
+      "excluded_files": [
+        "auth*",
+        "/var/**/user*"
       ]
     },
     {
@@ -223,12 +227,18 @@
         "json",
         ".log",
         "TXT"
+      ],
+      "excluded_files": [
+        ".git",
+        "**/tests",
+        "**/processes/**/*.log"
       ]
     },
     {
       "name": "Editable list",
       "param": "--editable_list",
       "type": "editable_list",
+      "required": true,
       "values": [
         "Value A",
         "Value B",

src/config/script/list_values.py

@@ -4,6 +4,7 @@
 
 from model.model_helper import is_empty, fill_parameter_values, InvalidFileException, list_files
 from utils import process_utils
+from utils.file_utils import FileMatcher
 
 LOGGER = logging.getLogger('list_values')
 
@@ -104,11 +105,18 @@ def get_values(self, parameter_values):
 
 class FilesProvider(ValuesProvider):
 
-    def __init__(self, file_dir, file_type=None, file_extensions=None) -> None:
+    def __init__(self,
+                 file_dir,
+                 file_type=None,
+                 file_extensions=None,
+                 excluded_files_matcher: FileMatcher = None) -> None:
         self._file_dir = file_dir
 
         try:
-            self._values = list_files(file_dir, file_type, file_extensions)
+            self._values = list_files(file_dir,
+                                      file_type=file_type,
+                                      file_extensions=file_extensions,
+                                      excluded_files_matcher=excluded_files_matcher)
         except InvalidFileException as e:
             LOGGER.warning('Failed to list files for ' + file_dir + ': ' + str(e))
             self._values = []

src/model/model_helper.py

@@ -1,11 +1,13 @@
 import logging
 import os
+import pathlib
 import re
 from datetime import datetime
 
 import utils.env_utils as env_utils
 from config.constants import FILE_TYPE_DIR, FILE_TYPE_FILE
 from utils import date_utils
+from utils.file_utils import FileMatcher
 from utils.string_utils import is_blank
 
 ENV_VAR_PREFIX = '$$'
@@ -223,30 +225,36 @@ def normalize_extension(extension):
     return re.sub('^\.', '', extension).lower()
 
 
-def list_files(dir, file_type=None, file_extensions=None):
+def list_files(dir, *, file_type=None, file_extensions=None, excluded_files_matcher: FileMatcher = None):
     if not os.path.exists(dir) or not os.path.isdir(dir):
         raise InvalidFileException(dir, 'Directory not found')
 
     result = []
 
+    if excluded_files_matcher and excluded_files_matcher.has_match(pathlib.Path(dir)):
+        return result
+
     if not is_empty(file_extensions):
         file_type = FILE_TYPE_FILE
 
     sorted_files = sorted(os.listdir(dir), key=lambda s: s.casefold())
     for file in sorted_files:
-        file_path = os.path.join(dir, file)
+        file_path = pathlib.Path(dir, file)
 
         if file_type:
-            if file_type == FILE_TYPE_DIR and not os.path.isdir(file_path):
+            if file_type == FILE_TYPE_DIR and not file_path.is_dir():
                 continue
-            elif file_type == FILE_TYPE_FILE and not os.path.isfile(file_path):
+            elif file_type == FILE_TYPE_FILE and not file_path.is_file():
                 continue
 
-        if file_extensions and not os.path.isdir(file_path):
-            _, extension = os.path.splitext(file_path)
+        if file_extensions and file_path.is_file():
+            extension = file_path.suffix
             if normalize_extension(extension) not in file_extensions:
                 continue
 
+        if excluded_files_matcher and excluded_files_matcher.has_match(file_path):
+            continue
+
         result.append(file)
 
     return result

src/model/parameter_config.py

@@ -12,6 +12,7 @@
     normalize_extension, read_bool_from_config, InvalidValueException
 from react.properties import ObservableDict, observable_fields
 from utils import file_utils, string_utils, process_utils
+from utils.file_utils import FileMatcher
 from utils.string_utils import strip
 
 LOGGER = logging.getLogger('script_server.parameter_config')
@@ -84,6 +85,7 @@ def _reload(self):
         self.file_extensions = _resolve_file_extensions(config, 'file_extensions')
         self.file_type = _resolve_parameter_file_type(config, 'file_type', self.file_extensions)
         self.file_recursive = read_bool_from_config('file_recursive', config, default=False)
+        self.excluded_files_matcher = _resolve_excluded_files(config, 'excluded_files', self._list_files_dir)
 
         self.type = self._read_type(config)
 
@@ -171,7 +173,8 @@ def _create_values_provider(self, values_config, type, constant):
             return NoneValuesProvider()
 
         if self._is_plain_server_file():
-            return FilesProvider(self._list_files_dir, self.file_type, self.file_extensions)
+            return FilesProvider(self._list_files_dir, self.file_type, self.file_extensions,
+                                 self.excluded_files_matcher)
 
         if (type != 'list') and (type != PARAM_TYPE_MULTISELECT) and (type != PARAM_TYPE_EDITABLE_LIST):
             return NoneValuesProvider()
@@ -353,11 +356,16 @@ def list_files(self, path):
         result = []
 
         if is_empty(self.file_type) or self.file_type == FILE_TYPE_FILE:
-            files = model_helper.list_files(full_path, FILE_TYPE_FILE, self.file_extensions)
+            files = model_helper.list_files(full_path,
+                                            file_type=FILE_TYPE_FILE,
+                                            file_extensions=self.file_extensions,
+                                            excluded_files_matcher=self.excluded_files_matcher)
             for file in files:
                 result.append({'name': file, 'type': FILE_TYPE_FILE, 'readable': True})
 
-        dirs = model_helper.list_files(full_path, FILE_TYPE_DIR)
+        dirs = model_helper.list_files(full_path,
+                                       file_type=FILE_TYPE_DIR,
+                                       excluded_files_matcher=self.excluded_files_matcher)
         for dir in dirs:
             dir_path = os.path.join(full_path, dir)
 
@@ -383,6 +391,9 @@ def _validate_recursive_path(self, path, intermediate):
 
         full_path = self._build_list_file_path(path)
 
+        if self.excluded_files_matcher.has_match(full_path):
+            return 'Path ' + value_string + ' is excluded'
+
         if not os.path.exists(full_path):
             return 'Path ' + value_string + ' does not exist'
 
@@ -398,7 +409,10 @@ def _validate_recursive_path(self, path, intermediate):
             file = path[-1]
 
             dir_path = self._build_list_file_path(dir)
-            allowed_files = model_helper.list_files(dir_path, self.file_type, self.file_extensions)
+            allowed_files = model_helper.list_files(dir_path,
+                                                    file_type=self.file_type,
+                                                    file_extensions=self.file_extensions,
+                                                    excluded_files_matcher=self.excluded_files_matcher)
             if file not in allowed_files:
                 return 'Path ' + value_string + ' is not allowed'
 
@@ -453,6 +467,15 @@ def _resolve_file_extensions(config, key):
     return [normalize_extension(e) for e in strip(result)]
 
 
+def _resolve_excluded_files(config, key, file_dir):
+    raw_patterns = model_helper.read_list(config, key)
+    if raw_patterns is None:
+        patterns = []
+    else:
+        patterns = [resolve_env_vars(e) for e in strip(raw_patterns)]
+    return FileMatcher(patterns, file_dir)
+
+
 def _resolve_parameter_file_type(config, key, file_extensions):
     if file_extensions:
         return FILE_TYPE_FILE
@@ -472,9 +495,13 @@ def __init__(self, param_name, error_message) -> None:
 
 
 def get_sorted_config(param_config):
-    key_order = ['name', 'required', 'param', 'repeat_param', 'type', 'no_value', 'default', 'constant', 'description', 'secure',
-                 'values', 'min', 'max', 'multiple_arguments', 'same_arg_param', 'separator', 'file_dir', 'file_recursive', 'file_type',
-                 'file_extensions']
+    key_order = ['name', 'required', 'param', 'repeat_param', 'type', 'no_value', 'default', 'constant', 'description',
+                 'secure',
+                 'values', 'min', 'max', 'multiple_arguments', 'same_arg_param', 'separator', 'file_dir',
+                 'file_recursive',
+                 'file_type',
+                 'file_extensions',
+                 'excluded_files']
 
     def get_order(key):
         if key in key_order:

src/tests/model_helper_test.py

@@ -8,6 +8,8 @@
     InvalidFileException, read_bool_from_config, InvalidValueException, InvalidValueTypeException, read_str_from_config
 from tests import test_utils
 from tests.test_utils import create_parameter_model, set_env_value
+from utils import file_utils
+from utils.file_utils import FileMatcher
 
 
 class TestReadList(unittest.TestCase):
@@ -325,6 +327,170 @@ def tearDown(self):
         test_utils.cleanup()
 
 
+class ListFilesWithExclusionsTest(unittest.TestCase):
+
+    def test_plain_relative_path(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files)
+
+        matcher = self.create_matcher(['file2'])
+        files = model_helper.list_files(test_utils.temp_folder, excluded_files_matcher=matcher)
+        self.assertEqual(['file1', 'file3'], files)
+
+    def test_plain_absolute_path(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files)
+
+        excluded_file = os.path.abspath(os.path.join(test_utils.temp_folder, 'file2'))
+        matcher = self.create_matcher([excluded_file])
+        files = model_helper.list_files(test_utils.temp_folder, excluded_files_matcher=matcher)
+        self.assertEqual(['file1', 'file3'], files)
+
+    def test_plain_relative_path_in_subfolder(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files, 'sub')
+
+        subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
+        matcher = self.create_matcher([(os.path.join('sub', 'file2'))])
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual(['file1', 'file3'], files)
+
+    def test_plain_relative_path_is_folder(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files, 'sub')
+
+        subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
+        matcher = self.create_matcher(['sub'])
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual([], files)
+
+    def test_glob_relative_path(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files)
+
+        matcher = self.create_matcher(['*1'])
+        files = model_helper.list_files(test_utils.temp_folder, excluded_files_matcher=matcher)
+        self.assertEqual(['file2', 'file3'], files)
+
+    def test_glob_absolute_path(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files)
+
+        matcher = self.create_matcher([file_utils.normalize_path('*1', test_utils.temp_folder)])
+        files = model_helper.list_files(test_utils.temp_folder, excluded_files_matcher=matcher)
+        self.assertEqual(['file2', 'file3'], files)
+
+    def test_glob_relative_path_with_subfolder(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files, 'sub')
+
+        matcher = self.create_matcher([os.path.join('sub', '*3')])
+        subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual(['file1', 'file2'], files)
+
+    def test_glob_relative_path_is_subfolder(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files, 'sub')
+
+        matcher = self.create_matcher(['*ub'])
+        subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual([], files)
+
+    def test_recursive_glob_relative_path(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files, 'sub')
+
+        matcher = self.create_matcher(['**/file1'])
+        subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual(['file2', 'file3'], files)
+
+    def test_recursive_glob_absolute_path(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files, 'sub')
+
+        matcher = self.create_matcher([file_utils.normalize_path('**/file1', test_utils.temp_folder)])
+        subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual(['file2', 'file3'], files)
+
+    def test_recursive_glob_absolute_path_and_deep_nested(self):
+        created_files = ['file1', 'file2', 'file3']
+        abc_subfolder = os.path.join('a', 'b', 'c')
+        test_utils.create_files(created_files, abc_subfolder)
+
+        matcher = self.create_matcher([file_utils.normalize_path('**/file1', test_utils.temp_folder)])
+        abc_path = os.path.join(test_utils.temp_folder, abc_subfolder)
+        files = model_helper.list_files(abc_path, excluded_files_matcher=matcher)
+        self.assertEqual(['file2', 'file3'], files)
+
+    def test_recursive_glob_absolute_path_and_deep_nested_and_multiple_globs(self):
+        created_files = ['file1', 'file2', 'file3']
+        sub_sub_subfolder = os.path.join('a', 'b', 'c', 'd', 'e')
+        test_utils.create_files(created_files, sub_sub_subfolder)
+
+        matcher = self.create_matcher([file_utils.normalize_path('**/c/**/file1', test_utils.temp_folder)])
+        subfolder_path = os.path.join(test_utils.temp_folder, sub_sub_subfolder)
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual(['file2', 'file3'], files)
+
+    def test_recursive_glob_relative_path_any_match(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files, 'sub')
+
+        matcher = self.create_matcher(['**'])
+        subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual([], files)
+
+    def test_recursive_glob_relative_path_match_any_in_subfolder(self):
+        created_files = ['file1', 'file2', 'file3']
+        subfolder = os.path.join('a', 'b', 'c', 'd', 'e')
+        test_utils.create_files(created_files, subfolder)
+
+        matcher = self.create_matcher(['**/e/**'])
+        subfolder_path = os.path.join(test_utils.temp_folder, subfolder)
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual([], files)
+
+    def test_recursive_glob_relative_different_work_dir(self):
+        created_files = ['file1', 'file2', 'file3']
+        test_utils.create_files(created_files, 'sub')
+
+        matcher = FileMatcher(['**'], test_utils.temp_folder + '2')
+        subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
+        files = model_helper.list_files(subfolder_path, excluded_files_matcher=matcher)
+        self.assertEqual(['file1', 'file2', 'file3'], files)
+
+    def test_multiple_exclusions(self):
+        created_files = ['file1', 'file2', 'file3', 'file4']
+        test_utils.create_files(created_files)
+
+        matcher = self.create_matcher(['*2', 'file1', file_utils.normalize_path('file4', test_utils.temp_folder)])
+        files = model_helper.list_files(test_utils.temp_folder, excluded_files_matcher=matcher)
+        self.assertEqual(['file3'], files)
+
+    def test_multiple_exclusions_when_no_match(self):
+        created_files = ['fileA', 'fileB', 'fileC', 'fileD']
+        test_utils.create_files(created_files)
+
+        matcher = self.create_matcher(['*2', 'file1', file_utils.normalize_path('file4', test_utils.temp_folder)])
+        files = model_helper.list_files(test_utils.temp_folder, excluded_files_matcher=matcher)
+        self.assertEqual(created_files, files)
+
+    @staticmethod
+    def create_matcher(excluded_paths):
+        return FileMatcher(excluded_paths, test_utils.temp_folder)
+
+    def setUp(self):
+        test_utils.setup()
+
+    def tearDown(self):
+        test_utils.cleanup()
+
+
 class TestReadIntFromConfig(unittest.TestCase):
     def test_normal_int_value(self):
         value = model_helper.read_int_from_config('abc', {'abc': 123})