From 1d6f64ef2d2123093f3ee06ec7bdd731287687ff Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Sat, 18 Aug 2018 14:16:20 +0100
Subject: [PATCH 01/16] add empty and not empty assertions

---
 salt/modules/saltcheck.py | 61 ++++++++++++++++++++++++++++++++-------
 1 file changed, 51 insertions(+), 10 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index e4801d6a4e91..95a693d7a412 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -295,7 +295,8 @@ def __init__(self):
                                   assertIn assertNotIn
                                   assertGreater
                                   assertGreaterEqual
-                                  assertLess assertLessEqual'''.split()
+                                  assertLess assertLessEqual
+                                  assertEmpty assertNotEmpty'''.split()
         self.auto_update_master_cache = _get_auto_update_cache_value
         # self.salt_lc = salt.client.Caller(mopts=__opts__)
         self.salt_lc = salt.client.Caller()
@@ -309,13 +310,24 @@ def __is_valid_test(self, test_dict):
         - a test name
         - a valid module and function
         - a valid assertion
-        - an expected return value
+        - an expected return value - if assertion type requires it
+
+        6 points needed for standard test
+        4 points needed for test with assertion not requiring expected return
         '''
+        exp_ret_key = 'expected-return' in test_dict.keys()
         tots = 0  # need total of >= 6 to be a valid test
         m_and_f = test_dict.get('module_and_function', None)
         assertion = test_dict.get('assertion', None)
         expected_return = test_dict.get('expected-return', None)
         log.info("__is_valid_test has test: %s", test_dict)
+        if assertion in ["assertEmpty",
+                         "assertNotEmpty",
+                         "assertTrue",
+                         "assertFalse"]:
+            required_total = 4
+        else:
+            required_total = 6
         if m_and_f:
             tots += 1
             module, function = m_and_f.split('.')
@@ -324,16 +336,17 @@ def __is_valid_test(self, test_dict):
             if _is_valid_function(module, function):
                 tots += 1
             log.info("__is_valid_test has valid m_and_f")
-        if assertion:
+        if assertion in self.assertions_list:
+            tots += 1
+            log.info("__is_valid_test has valid_assertion")
+        if exp_ret_key:
             tots += 1
-            if assertion in self.assertions_list:
-                tots += 1
-                log.info("__is_valid_test has valid_assertion")
         if expected_return:
             tots += 1
             log.info("__is_valid_test has valid_expected_return")
+        # log the test score for debug purposes
         log.info("__is_valid_test score: %s", tots)
-        return tots >= 6
+        return tots >= required_total
 
     def call_salt_command(self,
                           fun,
@@ -369,16 +382,17 @@ def run_test(self, test_dict):
             assertion = test_dict['assertion']
             expected_return = test_dict['expected-return']
             actual_return = self.call_salt_command(mod_and_func, args, kwargs)
-            if assertion != "assertIn":
+            if assertion not in ["assertIn", "assertNotIn", "assertEmpty", "assertNotEmpty",
+                                 "assertTrue", "assertFalse"]:
                 expected_return = self.cast_expected_to_returned_type(expected_return, actual_return)
             if assertion == "assertEqual":
                 value = self.__assert_equal(expected_return, actual_return)
             elif assertion == "assertNotEqual":
                 value = self.__assert_not_equal(expected_return, actual_return)
             elif assertion == "assertTrue":
-                value = self.__assert_true(expected_return)
+                value = self.__assert_true(actual_return)
             elif assertion == "assertFalse":
-                value = self.__assert_false(expected_return)
+                value = self.__assert_false(actual_return)
             elif assertion == "assertIn":
                 value = self.__assert_in(expected_return, actual_return)
             elif assertion == "assertNotIn":
@@ -391,6 +405,10 @@ def run_test(self, test_dict):
                 value = self.__assert_less(expected_return, actual_return)
             elif assertion == "assertLessEqual":
                 value = self.__assert_less_equal(expected_return, actual_return)
+            elif assertion == "assertEmpty":
+                value = self.__assert_empty(actual_return)
+            elif assertion == "assertNotEmpty":
+                value = self.__assert_not_empty(actual_return)
             else:
                 value = "Fail - bas assertion"
         else:
@@ -544,6 +562,29 @@ def __assert_less_equal(expected, returned):
         return result
 
     @staticmethod
+    def __assert_empty(returned):
+        '''
+        Test if a returned value is empty
+        '''
+        result = "Pass"
+        try:
+            assert (not returned), "{0} is not empty".format(returned)
+        except AssertionError as err:
+            result = "Fail: " + six.text_type(err)
+        return result
+
+    @staticmethod
+    def __assert_not_empty(returned):
+        '''
+        Test if a returned value is not empty
+        '''
+        result = "Pass"
+        try:
+            assert (returned), "{0} is empty".format(returned)
+        except AssertionError as err:
+            result = "Fail: " + six.text_type(err)
+        return result
+    @staticmethod
     def get_state_search_path_list():
         '''
         For the state file system, return a list of paths to search for states

From 7d16eb2ca8c733c9a82bcc72f32b600121d54b75 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Sat, 18 Aug 2018 14:54:37 +0100
Subject: [PATCH 02/16] add assertEmpty and assertNotEmpty tests

---
 tests/unit/modules/test_saltcheck.py | 36 ++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/tests/unit/modules/test_saltcheck.py b/tests/unit/modules/test_saltcheck.py
index da0d41c6542d..e241fa2fa728 100644
--- a/tests/unit/modules/test_saltcheck.py
+++ b/tests/unit/modules/test_saltcheck.py
@@ -321,6 +321,42 @@ def test__assert_less_equal3(self):
             mybool = sc_instance._SaltCheck__assert_less_equal(aaa, bbb)
             self.assertEqual(mybool, 'Pass')
 
+    def test__assert_empty(self):
+        '''test'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
+                                             'cp.cache_master': MagicMock(return_value=[True])
+                                             }):
+            sc_instance = saltcheck.SaltCheck()
+            mybool = sc_instance._SaltCheck__assert_empty("")
+            self.assertEqual(mybool, 'Pass')
+
+    def test__assert_empty_fail(self):
+        '''test'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
+                                             'cp.cache_master': MagicMock(return_value=[True])
+                                             }):
+            sc_instance = saltcheck.SaltCheck()
+            mybool = sc_instance._SaltCheck__assert_empty("data")
+            self.assertNotEqual(mybool, 'Pass')
+
+    def test__assert__not_empty(self):
+        '''test'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
+                                             'cp.cache_master': MagicMock(return_value=[True])
+                                             }):
+            sc_instance = saltcheck.SaltCheck()
+            mybool = sc_instance._SaltCheck__assert_not_empty("data")
+            self.assertEqual(mybool, 'Pass')
+
+    def test__assert__not_empty_fail(self):
+        '''test'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
+                                             'cp.cache_master': MagicMock(return_value=[True])
+                                             }):
+            sc_instance = saltcheck.SaltCheck()
+            mybool = sc_instance._SaltCheck__assert_not_empty("")
+            self.assertNotEqual(mybool, 'Pass')
+
     def test_run_test_1(self):
         '''test'''
         with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),

From c569cbec6ba8651f0c3bff1de2d84dc48daeccfa Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Sun, 19 Aug 2018 07:39:04 +0100
Subject: [PATCH 03/16] do not require expected-return key. Wording updates

---
 salt/modules/saltcheck.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 95a693d7a412..b91dca064690 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -380,7 +380,7 @@ def run_test(self, test_dict):
             args = test_dict.get('args', None)
             kwargs = test_dict.get('kwargs', None)
             assertion = test_dict['assertion']
-            expected_return = test_dict['expected-return']
+            expected_return = test_dict.get('expected-return', None)
             actual_return = self.call_salt_command(mod_and_func, args, kwargs)
             if assertion not in ["assertIn", "assertNotIn", "assertEmpty", "assertNotEmpty",
                                  "assertTrue", "assertFalse"]:
@@ -410,7 +410,7 @@ def run_test(self, test_dict):
             elif assertion == "assertNotEmpty":
                 value = self.__assert_not_empty(actual_return)
             else:
-                value = "Fail - bas assertion"
+                value = "Fail - bad assertion"
         else:
             return "Fail - invalid test"
         return value
@@ -580,10 +580,11 @@ def __assert_not_empty(returned):
         '''
         result = "Pass"
         try:
-            assert (returned), "{0} is empty".format(returned)
+            assert (returned), "value is empty".format(returned)
         except AssertionError as err:
             result = "Fail: " + six.text_type(err)
         return result
+
     @staticmethod
     def get_state_search_path_list():
         '''

From e69de861923af6ca33950e25346961bb926fce1c Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Sun, 19 Aug 2018 08:08:16 +0100
Subject: [PATCH 04/16] documentation update and rename most functions as
 private in an attempt to cleanup web documentation display

---
 salt/modules/saltcheck.py | 73 ++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 48 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index b91dca064690..526a67a43f30 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -21,7 +21,7 @@
 
 Example file system layout:
 
-.. code-block: txt
+.. code-block:: txt
 
     /srv/salt/apache/
         init.sls
@@ -42,6 +42,10 @@
       assertion: assertEqual
       expected-return:  'hello'
 
+Support assertions:
+assertEqual assertNotEqual assertTrue assertFalse assertIn assertNotIn assertGreater
+assertGreaterEqual assertLess assertLessEqual assertEmpty assertNotEmpty
+
 '''
 
 # Import Python libs
@@ -126,14 +130,14 @@ def run_state_tests(state):
         salt '*' saltcheck.run_state_tests postfix
     '''
     scheck = SaltCheck()
-    paths = scheck.get_state_search_path_list()
+    paths = scheck._get_state_search_path_list()
     stl = StateTestLoader(search_paths=paths)
     results = {}
     sls_list = _get_state_sls(state)
     for state_name in sls_list:
-        mypath = stl.convert_sls_to_path(state_name)
-        stl.add_test_files_for_sls(mypath)
-        stl.load_test_suite()
+        mypath = stl._convert_sls_to_path(state_name)
+        stl._add_test_files_for_sls(mypath)
+        stl._load_test_suite()
         results_dict = {}
         for key, value in stl.test_dict.items():
             result = scheck.run_test(value)
@@ -171,7 +175,7 @@ def run_highstate_tests():
         salt '*' saltcheck.run_highstate_tests
     '''
     scheck = SaltCheck()
-    paths = scheck.get_state_search_path_list()
+    paths = scheck._get_state_search_path_list()
     stl = StateTestLoader(search_paths=paths)
     results = {}
     sls_list = _get_top_states()
@@ -183,9 +187,9 @@ def run_highstate_tests():
                 all_states.append(state)
 
     for state_name in all_states:
-        mypath = stl.convert_sls_to_path(state_name)
-        stl.add_test_files_for_sls(mypath)
-        stl.load_test_suite()
+        mypath = stl._convert_sls_to_path(state_name)
+        stl._add_test_files_for_sls(mypath)
+        stl._load_test_suite()
         results_dict = {}
         for key, value in stl.test_dict.items():
             result = scheck.run_test(value)
@@ -348,7 +352,7 @@ def __is_valid_test(self, test_dict):
         log.info("__is_valid_test score: %s", tots)
         return tots >= required_total
 
-    def call_salt_command(self,
+    def _call_salt_command(self,
                           fun,
                           args,
                           kwargs):
@@ -381,10 +385,10 @@ def run_test(self, test_dict):
             kwargs = test_dict.get('kwargs', None)
             assertion = test_dict['assertion']
             expected_return = test_dict.get('expected-return', None)
-            actual_return = self.call_salt_command(mod_and_func, args, kwargs)
+            actual_return = self._call_salt_command(mod_and_func, args, kwargs)
             if assertion not in ["assertIn", "assertNotIn", "assertEmpty", "assertNotEmpty",
                                  "assertTrue", "assertFalse"]:
-                expected_return = self.cast_expected_to_returned_type(expected_return, actual_return)
+                expected_return = self._cast_expected_to_returned_type(expected_return, actual_return)
             if assertion == "assertEqual":
                 value = self.__assert_equal(expected_return, actual_return)
             elif assertion == "assertNotEqual":
@@ -416,7 +420,7 @@ def run_test(self, test_dict):
         return value
 
     @staticmethod
-    def cast_expected_to_returned_type(expected, returned):
+    def _cast_expected_to_returned_type(expected, returned):
         '''
         Determine the type of variable returned
         Cast the expected to the type of variable returned
@@ -586,7 +590,7 @@ def __assert_not_empty(returned):
         return result
 
     @staticmethod
-    def get_state_search_path_list():
+    def _get_state_search_path_list():
         '''
         For the state file system, return a list of paths to search for states
         '''
@@ -614,32 +618,16 @@ def __init__(self, search_paths):
         self.test_files = []  # list of file paths
         self.test_dict = {}
 
-    def load_test_suite(self):
+    def _load_test_suite(self):
         '''
         Load tests either from one file, or a set of files
         '''
         self.test_dict = {}
         for myfile in self.test_files:
-            # self.load_file(myfile)
-            self.load_file_salt_rendered(myfile)
+            self._load_file_salt_rendered(myfile)
         self.test_files = []
 
-    def load_file(self, filepath):
-        '''
-        loads in one test file
-        '''
-        try:
-            with __utils__['files.fopen'](filepath, 'r') as myfile:
-                # with salt.utils.files.fopen(filepath, 'r') as myfile:
-                # with open(filepath, 'r') as myfile:
-                contents_yaml = salt.utils.data.decode(salt.utils.yaml.safe_load(myfile))
-                for key, value in contents_yaml.items():
-                    self.test_dict[key] = value
-        except:
-            raise
-        return
-
-    def load_file_salt_rendered(self, filepath):
+    def _load_file_salt_rendered(self, filepath):
         '''
         loads in one test file
         '''
@@ -651,7 +639,7 @@ def load_file_salt_rendered(self, filepath):
             self.test_dict[key] = value
         return
 
-    def gather_files(self, filepath):
+    def _gather_files(self, filepath):
         '''
         Gather files for a test suite
         '''
@@ -669,25 +657,14 @@ def gather_files(self, filepath):
         return
 
     @staticmethod
-    def convert_sls_to_paths(sls_list):
-        '''
-        Converting sls to paths
-        '''
-        new_sls_list = []
-        for sls in sls_list:
-            sls = sls.replace(".", os.sep)
-            new_sls_list.append(sls)
-        return new_sls_list
-
-    @staticmethod
-    def convert_sls_to_path(sls):
+    def _convert_sls_to_path(sls):
         '''
         Converting sls to paths
         '''
         sls = sls.replace(".", os.sep)
         return sls
 
-    def add_test_files_for_sls(self, sls_path):
+    def _add_test_files_for_sls(self, sls_path):
         '''
         Adding test files
         '''
@@ -699,7 +676,7 @@ def add_test_files_for_sls(self, sls_path):
                 # for dirname, subdirlist, filelist in salt.utils.path.os_walk(rootdir, topdown=True):
                 for dirname, subdirlist, dummy in salt.utils.path.os_walk(rootdir, topdown=True):
                     if "saltcheck-tests" in subdirlist:
-                        self.gather_files(dirname)
+                        self._gather_files(dirname)
                         log.info("test_files list: %s", self.test_files)
                         log.info("found subdir match in = %s", dirname)
                     else:

From 67c8b39d5d249f5e73a8b0243d1d47e0651729ef Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Sun, 19 Aug 2018 08:22:19 +0100
Subject: [PATCH 05/16] typo correction

---
 salt/modules/saltcheck.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 526a67a43f30..41a712cb6263 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -42,7 +42,7 @@
       assertion: assertEqual
       expected-return:  'hello'
 
-Support assertions:
+Supported assertions:
 assertEqual assertNotEqual assertTrue assertFalse assertIn assertNotIn assertGreater
 assertGreaterEqual assertLess assertLessEqual assertEmpty assertNotEmpty
 
@@ -289,7 +289,6 @@ class SaltCheck(object):
     '''
 
     def __init__(self):
-        # self.sls_list_top = []
         self.sls_list_state = []
         self.modules = []
         self.results_dict = {}

From 7221ffcf5fc5a6441e20b6e8cd9001a1ca8f7f0a Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Sun, 19 Aug 2018 16:39:29 +0100
Subject: [PATCH 06/16] Sync with upstream  - minor cleanups  - support test
 timing (changes output format) Support saltenv

---
 salt/modules/saltcheck.py | 124 ++++++++++++++++++++++----------------
 1 file changed, 72 insertions(+), 52 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 41a712cb6263..a225a0660f8b 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -75,7 +75,7 @@ def __virtual__():
     return __virtualname__
 
 
-def update_master_cache():
+def update_master_cache(saltenv=None):
     '''
     Updates the master cache onto the minion - transfers all salt-check-tests
     Should be done one time before running tests, and if tests are updated
@@ -87,7 +87,8 @@ def update_master_cache():
 
         salt '*' saltcheck.update_master_cache
     '''
-    __salt__['cp.cache_master']()
+    log.info("Updating files for environment: %s", saltenv)
+    __salt__['cp.cache_master'](saltenv)
     return True
 
 
@@ -116,7 +117,7 @@ def run_test(**kwargs):
         return "Test must be a dictionary"
 
 
-def run_state_tests(state):
+def run_state_tests(state, saltenv=None):
     '''
     Execute all tests for a salt state and return results
     Nested states will also be tested
@@ -129,8 +130,10 @@ def run_state_tests(state):
 
         salt '*' saltcheck.run_state_tests postfix
     '''
-    scheck = SaltCheck()
-    paths = scheck._get_state_search_path_list()
+    if not saltenv:
+        saltenv = __opts__['saltenv']
+    scheck = SaltCheck(saltenv)
+    paths = scheck._get_state_search_path_list(saltenv)
     stl = StateTestLoader(search_paths=paths)
     results = {}
     sls_list = _get_state_sls(state)
@@ -143,28 +146,10 @@ def run_state_tests(state):
             result = scheck.run_test(value)
             results_dict[key] = result
         results[state_name] = results_dict
-    passed = 0
-    failed = 0
-    missing_tests = 0
-    for state in results:
-        if len(results[state].items()) == 0:
-            missing_tests = missing_tests + 1
-        else:
-            for dummy, val in results[state].items():
-                log.info("dummy=%s, val=%s", dummy, val)
-                if val.startswith('Pass'):
-                    passed = passed + 1
-                if val.startswith('Fail'):
-                    failed = failed + 1
-    out_list = []
-    for key, value in results.items():
-        out_list.append({key: value})
-    out_list.sort()
-    out_list.append({"TEST RESULTS": {'Passed': passed, 'Failed': failed, 'Missing Tests': missing_tests}})
-    return out_list
+    return _generate_out_list(results)
 
 
-def run_highstate_tests():
+def run_highstate_tests(saltenv=None):
     '''
     Execute all tests for a salt highstate and return results
 
@@ -174,11 +159,15 @@ def run_highstate_tests():
 
         salt '*' saltcheck.run_highstate_tests
     '''
-    scheck = SaltCheck()
-    paths = scheck._get_state_search_path_list()
+    if not saltenv:
+        saltenv = __opts__['saltenv']
+    if not saltenv:
+        saltenv = 'base'
+    scheck = SaltCheck(saltenv)
+    paths = scheck._get_state_search_path_list(saltenv)
     stl = StateTestLoader(search_paths=paths)
     results = {}
-    sls_list = _get_top_states()
+    sls_list = _get_top_states(saltenv)
     all_states = []
     for top_state in sls_list:
         sls_list = _get_state_sls(top_state)
@@ -195,24 +184,34 @@ def run_highstate_tests():
             result = scheck.run_test(value)
             results_dict[key] = result
         results[state_name] = results_dict
+    return _generate_out_list(results)
+
+
+def _generate_out_list(results):
+    ''' generate test results output list '''
     passed = 0
     failed = 0
+    skipped = 0
     missing_tests = 0
+    total_time = 0.0
     for state in results:
         if len(results[state].items()) == 0:
             missing_tests = missing_tests + 1
         else:
             for dummy, val in results[state].items():
                 log.info("dummy=%s, val=%s", dummy, val)
-                if val.startswith('Pass'):
+                if val['status'].startswith('Pass'):
                     passed = passed + 1
-                if val.startswith('Fail'):
+                if val['status'].startswith('Fail'):
                     failed = failed + 1
+                if val['status'].startswith('Skip'):
+                    skipped = skipped + 1
+                total_time = total_time + float(val['duration'])
     out_list = []
     for key, value in results.items():
         out_list.append({key: value})
     out_list.sort()
-    out_list.append({"TEST RESULTS": {'Passed': passed, 'Failed': failed, 'Missing Tests': missing_tests}})
+    out_list.append({"TEST RESULTS": {'Execution Time': round(total_time, 4), 'Passed': passed, 'Failed': failed, 'Skipped': skipped, 'Missing Tests': missing_tests}})
     return out_list
 
 
@@ -253,14 +252,14 @@ def _is_valid_function(module_name, function):
     return "{0}.{1}".format(module_name, function) in functions
 
 
-def _get_top_states():
+def _get_top_states(saltenv='base'):
     '''
     Equivalent to a salt cli: salt web state.show_top
     '''
     alt_states = []
     try:
         returned = __salt__['state.show_top']()
-        for i in returned['base']:
+        for i in returned[saltenv]:
             alt_states.append(i)
     except Exception:
         raise
@@ -284,15 +283,13 @@ def _get_state_sls(state):
 
 
 class SaltCheck(object):
-    '''
-    This class implements the saltcheck
-    '''
 
-    def __init__(self):
+    def __init__(self, saltenv='base'):
         self.sls_list_state = []
         self.modules = []
         self.results_dict = {}
         self.results_dict_summary = {}
+        self.saltenv = saltenv
         self.assertions_list = '''assertEqual assertNotEqual
                                   assertTrue assertFalse
                                   assertIn assertNotIn
@@ -304,7 +301,7 @@ def __init__(self):
         # self.salt_lc = salt.client.Caller(mopts=__opts__)
         self.salt_lc = salt.client.Caller()
         if self.auto_update_master_cache:
-            update_master_cache()
+            update_master_cache(saltenv)
 
     def __is_valid_test(self, test_dict):
         '''
@@ -318,12 +315,15 @@ def __is_valid_test(self, test_dict):
         6 points needed for standard test
         4 points needed for test with assertion not requiring expected return
         '''
-        exp_ret_key = 'expected-return' in test_dict.keys()
         tots = 0  # need total of >= 6 to be a valid test
+        skip = test_dict.get('skip', False)
         m_and_f = test_dict.get('module_and_function', None)
         assertion = test_dict.get('assertion', None)
-        expected_return = test_dict.get('expected-return', None)
+        exp_ret_key = 'expected-return' in test_dict.keys()
+        exp_ret_val = test_dict.get('expected-return', None)
         log.info("__is_valid_test has test: %s", test_dict)
+        if skip:
+            required_total = 0
         if assertion in ["assertEmpty",
                          "assertNotEmpty",
                          "assertTrue",
@@ -331,6 +331,7 @@ def __is_valid_test(self, test_dict):
             required_total = 4
         else:
             required_total = 6
+
         if m_and_f:
             tots += 1
             module, function = m_and_f.split('.')
@@ -340,15 +341,17 @@ def __is_valid_test(self, test_dict):
                 tots += 1
             log.info("__is_valid_test has valid m_and_f")
         if assertion in self.assertions_list:
-            tots += 1
             log.info("__is_valid_test has valid_assertion")
+            tots += 1
+
         if exp_ret_key:
             tots += 1
-        if expected_return:
+
+        if exp_ret_val is not None:
             tots += 1
-            log.info("__is_valid_test has valid_expected_return")
+
         # log the test score for debug purposes
-        log.info("__is_valid_test score: %s", tots)
+        log.info("__test score: %s and required: %s", tots, required_total)
         return tots >= required_total
 
     def _call_salt_command(self,
@@ -378,10 +381,24 @@ def run_test(self, test_dict):
         '''
         Run a single saltcheck test
         '''
+        start = time.time()
         if self.__is_valid_test(test_dict):
+            skip = test_dict.get('skip', False)
+            if skip:
+                return {'status': 'Skip', 'duration': 0.0}
             mod_and_func = test_dict['module_and_function']
             args = test_dict.get('args', None)
             kwargs = test_dict.get('kwargs', None)
+            pillar_data = test_dict.get('pillar-data', None)
+            if pillar_data:
+                if not kwargs:
+                    kwargs = {}
+                kwargs['pillar'] = pillar_data
+            else:
+                # make sure we clean pillar from previous test
+                if kwargs:
+                    kwargs.pop('pillar')
+
             assertion = test_dict['assertion']
             expected_return = test_dict.get('expected-return', None)
             actual_return = self._call_salt_command(mod_and_func, args, kwargs)
@@ -415,8 +432,13 @@ def run_test(self, test_dict):
             else:
                 value = "Fail - bad assertion"
         else:
-            return "Fail - invalid test"
-        return value
+            value = "Fail - invalid test"
+        end = time.time()
+        result = {}
+        result['status'] = value
+        result['duration'] = round(end - start, 4)
+        return result
+
 
     @staticmethod
     def _cast_expected_to_returned_type(expected, returned):
@@ -589,18 +611,15 @@ def __assert_not_empty(returned):
         return result
 
     @staticmethod
-    def _get_state_search_path_list():
+    def _get_state_search_path_list(saltenv='base'):
         '''
         For the state file system, return a list of paths to search for states
         '''
         # state cache should be updated before running this method
         search_list = []
         cachedir = __opts__.get('cachedir', None)
-        environment = __opts__['saltenv']
-        if environment:
-            path = cachedir + os.sep + "files" + os.sep + environment
-            search_list.append(path)
-        path = cachedir + os.sep + "files" + os.sep + "base"
+        log.info("Searching for files in saltenv: %s", saltenv)
+        path = cachedir + os.sep + "files" + os.sep + saltenv
         search_list.append(path)
         return search_list
 
@@ -645,6 +664,7 @@ def _gather_files(self, filepath):
         self.test_files = []
         log.info("gather_files: %s", time.time())
         filepath = filepath + os.sep + 'saltcheck-tests'
+
         rootdir = filepath
         # for dirname, subdirlist, filelist in salt.utils.path.os_walk(rootdir):
         for dirname, dummy, filelist in salt.utils.path.os_walk(rootdir):

From b92fd1f8cbe79609c640f1ff4782867518b04904 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Sun, 19 Aug 2018 18:36:17 +0100
Subject: [PATCH 07/16] cleanup saltenv and assert output

---
 salt/modules/saltcheck.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index a225a0660f8b..72a1611cccf0 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -75,7 +75,7 @@ def __virtual__():
     return __virtualname__
 
 
-def update_master_cache(saltenv=None):
+def update_master_cache(saltenv='base'):
     '''
     Updates the master cache onto the minion - transfers all salt-check-tests
     Should be done one time before running tests, and if tests are updated
@@ -132,6 +132,8 @@ def run_state_tests(state, saltenv=None):
     '''
     if not saltenv:
         saltenv = __opts__['saltenv']
+    if not saltenv:
+        saltenv = 'base'
     scheck = SaltCheck(saltenv)
     paths = scheck._get_state_search_path_list(saltenv)
     stl = StateTestLoader(search_paths=paths)
@@ -521,7 +523,7 @@ def __assert_in(expected, returned):
         '''
         result = "Pass"
         try:
-            assert (expected in returned), "{0} not False".format(returned)
+            assert (expected in returned), "{0} not found in {1}".format(expected, returned)
         except AssertionError as err:
             result = "Fail: " + six.text_type(err)
         return result
@@ -533,7 +535,7 @@ def __assert_not_in(expected, returned):
         '''
         result = "Pass"
         try:
-            assert (expected not in returned), "{0} not False".format(returned)
+            assert (expected not in returned), "{0} was found in {1}".format(expected, returned)
         except AssertionError as err:
             result = "Fail: " + six.text_type(err)
         return result

From e6062b59b974e61f08cd0e43ed2df2dc562dc869 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Mon, 20 Aug 2018 06:45:43 +0100
Subject: [PATCH 08/16] correct assert output

---
 salt/modules/saltcheck.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 72a1611cccf0..53dc3b941d63 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -607,7 +607,7 @@ def __assert_not_empty(returned):
         '''
         result = "Pass"
         try:
-            assert (returned), "value is empty".format(returned)
+            assert (returned), "value is empty"
         except AssertionError as err:
             result = "Fail: " + six.text_type(err)
         return result

From 9dd0d412155c306ece95d125653096f8d85c7e27 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Mon, 20 Aug 2018 07:01:15 +0100
Subject: [PATCH 09/16] Add second example

---
 salt/modules/saltcheck.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 53dc3b941d63..5f85dc704d0b 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -30,11 +30,11 @@
             pkg_and_mods.tst
             config.tst
 
-Example:
+Example1:
 
 .. code-block:: yaml
 
-    echo-test-hello:
+    echo_test_hello:
       module_and_function: test.echo
       args:
         - "hello"
@@ -42,10 +42,23 @@
       assertion: assertEqual
       expected-return:  'hello'
 
+Example2:
+
+.. code-block:: jinja
+
+    {% for package in ["apache2", "openssh"] %}
+    {# or another example #}
+    {# for package in salt['pillar.get']("packages") #}
+    test_{{ package }}_latest:
+      module_and_function: pkg.upgrade_available
+      args:
+        - {{ package }}
+      assertion: assertFalse
+    {% endfor %}
+
 Supported assertions:
 assertEqual assertNotEqual assertTrue assertFalse assertIn assertNotIn assertGreater
 assertGreaterEqual assertLess assertLessEqual assertEmpty assertNotEmpty
-
 '''
 
 # Import Python libs

From 5542d95d6e3c7580cb4a42c6ed1f47229b8b5ab0 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Mon, 20 Aug 2018 07:03:32 +0100
Subject: [PATCH 10/16] Fix lint

---
 salt/modules/saltcheck.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 5f85dc704d0b..b6016e9ed356 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -454,7 +454,6 @@ def run_test(self, test_dict):
         result['duration'] = round(end - start, 4)
         return result
 
-
     @staticmethod
     def _cast_expected_to_returned_type(expected, returned):
         '''

From f54ada1a5c172ed2b5d2f07d9a38d770a77734b5 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Mon, 20 Aug 2018 14:44:26 +0100
Subject: [PATCH 11/16] correct processing through eventbus

---
 salt/modules/saltcheck.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index b6016e9ed356..1acbae8fc686 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -313,8 +313,9 @@ def __init__(self, saltenv='base'):
                                   assertLess assertLessEqual
                                   assertEmpty assertNotEmpty'''.split()
         self.auto_update_master_cache = _get_auto_update_cache_value
-        # self.salt_lc = salt.client.Caller(mopts=__opts__)
-        self.salt_lc = salt.client.Caller()
+        local_opts = salt.config.minion_config(__opts__['conf_file'])
+        local_opts['file_client'] = 'local'
+        self.salt_lc = salt.client.Caller(mopts=local_opts)
         if self.auto_update_master_cache:
             update_master_cache(saltenv)
 

From 19cafea98fc7416a895ac3a4727136a8ea0c6a11 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Mon, 20 Aug 2018 15:26:50 +0100
Subject: [PATCH 12/16] Correct tests for new __opts__ keys, adjust for
 function renaming, and support the new output format with the status key

---
 tests/unit/modules/test_saltcheck.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/unit/modules/test_saltcheck.py b/tests/unit/modules/test_saltcheck.py
index e241fa2fa728..4099cb94ac90 100644
--- a/tests/unit/modules/test_saltcheck.py
+++ b/tests/unit/modules/test_saltcheck.py
@@ -37,11 +37,12 @@ def setup_loader_modules(self):
         local_opts = salt.config.minion_config(
             os.path.join(syspaths.CONFIG_DIR, 'minion'))
         local_opts['file_client'] = 'local'
+        local_opts['conf_file'] = '/etc/salt/minion'
         patcher = patch('salt.config.minion_config',
                         MagicMock(return_value=local_opts))
         patcher.start()
         self.addCleanup(patcher.stop)
-        return {saltcheck: {}}
+        return {saltcheck: {'__opts__': local_opts}}
 
     def test_call_salt_command(self):
         '''test simple test.echo module'''
@@ -50,7 +51,7 @@ def test_call_salt_command(self):
                                              'cp.cache_master': MagicMock(return_value=[True])
                                              }):
             sc_instance = saltcheck.SaltCheck()
-            returned = sc_instance.call_salt_command(fun="test.echo", args=['hello'], kwargs=None)
+            returned = sc_instance._call_salt_command(fun="test.echo", args=['hello'], kwargs=None)
             self.assertEqual(returned, 'hello')
 
     def test_update_master_cache(self):
@@ -64,7 +65,7 @@ def test_call_salt_command2(self):
                                              'cp.cache_master': MagicMock(return_value=[True])
                                              }):
             sc_instance = saltcheck.SaltCheck()
-            returned = sc_instance.call_salt_command(fun="test.echo", args=['hello'], kwargs=None)
+            returned = sc_instance._call_salt_command(fun="test.echo", args=['hello'], kwargs=None)
             self.assertNotEqual(returned, 'not-hello')
 
     def test__assert_equal1(self):
@@ -368,4 +369,4 @@ def test_run_test_1(self):
                                                 "expected-return": "This works!",
                                                 "args": ["This works!"]
                                                 })
-            self.assertEqual(returned, 'Pass')
+            self.assertEqual(returned['status'], 'Pass')

From 04155fd44521e13daf4f68c999776d052dcc586c Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Wed, 22 Aug 2018 07:37:52 +0100
Subject: [PATCH 13/16] add memoize to module and function checks

---
 salt/modules/saltcheck.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 1acbae8fc686..49b884a9fad3 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -74,6 +74,7 @@
 import salt.utils.yaml
 import salt.client
 import salt.exceptions
+from salt.utils.decorators import memoize
 from salt.ext import six
 
 log = logging.getLogger(__name__)
@@ -226,7 +227,9 @@ def _generate_out_list(results):
     for key, value in results.items():
         out_list.append({key: value})
     out_list.sort()
-    out_list.append({"TEST RESULTS": {'Execution Time': round(total_time, 4), 'Passed': passed, 'Failed': failed, 'Skipped': skipped, 'Missing Tests': missing_tests}})
+    out_list.append({"TEST RESULTS": {'Execution Time': round(total_time, 4),
+                    'Passed': passed, 'Failed': failed, 'Skipped': skipped,
+                    'Missing Tests': missing_tests}})
     return out_list
 
 
@@ -239,7 +242,7 @@ def _render_file(file_path):
     log.info("rendered: %s", rendered)
     return rendered
 
-
+@memoize
 def _is_valid_module(module):
     '''
     Return a list of all modules available on minion
@@ -255,7 +258,7 @@ def _get_auto_update_cache_value():
     __salt__['config.get']('auto_update_master_cache')
     return True
 
-
+@memoize
 def _is_valid_function(module_name, function):
     '''
     Determine if a function is valid for a module

From 147129a24fd439d362505b8a415178ccc2f38fb8 Mon Sep 17 00:00:00 2001
From: Nicole Thomas <nicole@saltstack.com>
Date: Fri, 24 Aug 2018 16:07:34 -0400
Subject: [PATCH 14/16] Lint: Add blank lines

---
 salt/modules/saltcheck.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 49b884a9fad3..2f3cdd94eba1 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -30,7 +30,7 @@
             pkg_and_mods.tst
             config.tst
 
-Example1:
+Example 1:
 
 .. code-block:: yaml
 
@@ -42,7 +42,7 @@
       assertion: assertEqual
       expected-return:  'hello'
 
-Example2:
+Example 2:
 
 .. code-block:: jinja
 
@@ -242,6 +242,7 @@ def _render_file(file_path):
     log.info("rendered: %s", rendered)
     return rendered
 
+
 @memoize
 def _is_valid_module(module):
     '''
@@ -258,6 +259,7 @@ def _get_auto_update_cache_value():
     __salt__['config.get']('auto_update_master_cache')
     return True
 
+
 @memoize
 def _is_valid_function(module_name, function):
     '''

From 6e5be945168068b696bff4e23f9363849cbceb75 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Fri, 24 Aug 2018 22:04:01 +0100
Subject: [PATCH 15/16] Refactor tst file location detection. Now runs tests
 with matching names to states. If previous behavior is desired, then
 check_all=True can be passed to run all tests in the specified
 saltcheck-tests directory.

---
 salt/modules/saltcheck.py | 124 +++++++++++++++++++-------------------
 1 file changed, 62 insertions(+), 62 deletions(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index 49b884a9fad3..e693dff482e5 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -14,7 +14,7 @@
 Tests for a state should be created in files ending in ``*.tst`` and placed in
 the ``saltcheck-tests`` folder.
 
-Multiple tests can be created in a file. Multiple ``*.tst`` files can be
+Multiple tests can be specified in a file. Multiple ``*.tst`` files can be
 created in the ``saltcheck-tests`` folder. Salt rendering is supported in test
 files (e.g. ``yaml + jinja``). The ``id`` of a test works in the same manner as
 in salt state files. They should be unique and descriptive.
@@ -27,9 +27,17 @@
         init.sls
         config.sls
         saltcheck-tests/
-            pkg_and_mods.tst
+            init.tst
             config.tst
 
+Tests can be run for each state, or all apache tests
+
+.. code-block:: bash
+
+    salt '*' saltcheck.run_state_tests apache,apache.config
+    salt '*' saltcheck.run_state_tests apache check_all=True
+    salt '*' saltcheck.run_highstate_tests
+
 Example1:
 
 .. code-block:: yaml
@@ -131,32 +139,32 @@ def run_test(**kwargs):
         return "Test must be a dictionary"
 
 
-def run_state_tests(state, saltenv=None):
+def run_state_tests(state, saltenv=None, check_all=False):
     '''
     Execute all tests for a salt state and return results
     Nested states will also be tested
 
     :param str state: the name of a user defined state
+    :param bool check_all: boolean to run all tests in state/saltcheck-tests directory
 
     CLI Example:
 
     .. code-block:: bash
 
-        salt '*' saltcheck.run_state_tests postfix
+        salt '*' saltcheck.run_state_tests postfix,common
     '''
     if not saltenv:
         saltenv = __opts__['saltenv']
     if not saltenv:
         saltenv = 'base'
     scheck = SaltCheck(saltenv)
-    paths = scheck._get_state_search_path_list(saltenv)
+    paths = scheck.get_state_search_path_list(saltenv)
     stl = StateTestLoader(search_paths=paths)
     results = {}
-    sls_list = _get_state_sls(state)
+    sls_list = state.split(',')
     for state_name in sls_list:
-        mypath = stl._convert_sls_to_path(state_name)
-        stl._add_test_files_for_sls(mypath)
-        stl._load_test_suite()
+        stl.add_test_files_for_sls(state_name, check_all)
+        stl.load_test_suite()
         results_dict = {}
         for key, value in stl.test_dict.items():
             result = scheck.run_test(value)
@@ -180,21 +188,18 @@ def run_highstate_tests(saltenv=None):
     if not saltenv:
         saltenv = 'base'
     scheck = SaltCheck(saltenv)
-    paths = scheck._get_state_search_path_list(saltenv)
+    paths = scheck.get_state_search_path_list(saltenv)
     stl = StateTestLoader(search_paths=paths)
     results = {}
     sls_list = _get_top_states(saltenv)
     all_states = []
-    for top_state in sls_list:
-        sls_list = _get_state_sls(top_state)
-        for state in sls_list:
-            if state not in all_states:
-                all_states.append(state)
+    for state in sls_list:
+        if state not in all_states:
+            all_states.append(state)
 
     for state_name in all_states:
-        mypath = stl._convert_sls_to_path(state_name)
-        stl._add_test_files_for_sls(mypath)
-        stl._load_test_suite()
+        stl.add_test_files_for_sls(state_name)
+        stl.load_test_suite()
         results_dict = {}
         for key, value in stl.test_dict.items():
             result = scheck.run_test(value)
@@ -204,14 +209,16 @@ def run_highstate_tests(saltenv=None):
 
 
 def _generate_out_list(results):
-    ''' generate test results output list '''
+    '''
+    generate test results output list
+    '''
     passed = 0
     failed = 0
     skipped = 0
     missing_tests = 0
     total_time = 0.0
     for state in results:
-        if len(results[state].items()) == 0:
+        if not results[state].items():
             missing_tests = missing_tests + 1
         else:
             for dummy, val in results[state].items():
@@ -227,9 +234,9 @@ def _generate_out_list(results):
     for key, value in results.items():
         out_list.append({key: value})
     out_list.sort()
-    out_list.append({"TEST RESULTS": {'Execution Time': round(total_time, 4),
-                    'Passed': passed, 'Failed': failed, 'Skipped': skipped,
-                    'Missing Tests': missing_tests}})
+    out_list.append({'TEST RESULTS': {'Execution Time': round(total_time, 4),
+                     'Passed': passed, 'Failed': failed, 'Skipped': skipped,
+                     'Missing Tests': missing_tests}})
     return out_list
 
 
@@ -242,6 +249,7 @@ def _render_file(file_path):
     log.info("rendered: %s", rendered)
     return rendered
 
+
 @memoize
 def _is_valid_module(module):
     '''
@@ -258,6 +266,7 @@ def _get_auto_update_cache_value():
     __salt__['config.get']('auto_update_master_cache')
     return True
 
+
 @memoize
 def _is_valid_function(module_name, function):
     '''
@@ -285,22 +294,10 @@ def _get_top_states(saltenv='base'):
     return alt_states
 
 
-def _get_state_sls(state):
+class SaltCheck(object):
     '''
-    Equivalent to a salt cli: salt web state.show_low_sls STATE
+    This class validates and runs the saltchecks
     '''
-    sls_list_state = []
-    try:
-        returned = __salt__['state.show_low_sls'](state)
-        for i in returned:
-            if i['__sls__'] not in sls_list_state:
-                sls_list_state.append(i['__sls__'])
-    except Exception:
-        raise
-    return sls_list_state
-
-
-class SaltCheck(object):
 
     def __init__(self, saltenv='base'):
         self.sls_list_state = []
@@ -374,9 +371,9 @@ def __is_valid_test(self, test_dict):
         return tots >= required_total
 
     def _call_salt_command(self,
-                          fun,
-                          args,
-                          kwargs):
+                           fun,
+                           args,
+                           kwargs):
         '''
         Generic call of salt Caller command
         '''
@@ -629,7 +626,7 @@ def __assert_not_empty(returned):
         return result
 
     @staticmethod
-    def _get_state_search_path_list(saltenv='base'):
+    def get_state_search_path_list(saltenv='base'):
         '''
         For the state file system, return a list of paths to search for states
         '''
@@ -654,7 +651,7 @@ def __init__(self, search_paths):
         self.test_files = []  # list of file paths
         self.test_dict = {}
 
-    def _load_test_suite(self):
+    def load_test_suite(self):
         '''
         Load tests either from one file, or a set of files
         '''
@@ -680,16 +677,13 @@ def _gather_files(self, filepath):
         Gather files for a test suite
         '''
         self.test_files = []
-        log.info("gather_files: %s", time.time())
         filepath = filepath + os.sep + 'saltcheck-tests'
-
-        rootdir = filepath
-        # for dirname, subdirlist, filelist in salt.utils.path.os_walk(rootdir):
-        for dirname, dummy, filelist in salt.utils.path.os_walk(rootdir):
+        for dirname, dummy, filelist in salt.utils.path.os_walk(filepath):
             for fname in filelist:
                 if fname.endswith('.tst'):
                     start_path = dirname + os.sep + fname
                     full_path = os.path.abspath(start_path)
+                    log.info("Found test: %s", full_path)
                     self.test_files.append(full_path)
         return
 
@@ -701,24 +695,30 @@ def _convert_sls_to_path(sls):
         sls = sls.replace(".", os.sep)
         return sls
 
-    def _add_test_files_for_sls(self, sls_path):
+    def add_test_files_for_sls(self, sls_name, check_all=False):
         '''
         Adding test files
         '''
+        sls_split = sls_name.rpartition('.')
         for path in self.search_paths:
-            full_path = path + os.sep + sls_path
-            rootdir = full_path
-            if os.path.isdir(full_path):
-                log.info("searching path= %s", full_path)
-                # for dirname, subdirlist, filelist in salt.utils.path.os_walk(rootdir, topdown=True):
-                for dirname, subdirlist, dummy in salt.utils.path.os_walk(rootdir, topdown=True):
-                    if "saltcheck-tests" in subdirlist:
-                        self._gather_files(dirname)
-                        log.info("test_files list: %s", self.test_files)
-                        log.info("found subdir match in = %s", dirname)
-                    else:
-                        log.info("did not find subdir match in = %s", dirname)
-                    del subdirlist[:]
+            if sls_split[0]:
+                base_path = path + os.sep + self._convert_sls_to_path(sls_split[0])
+            else:
+                base_path = path
+            if os.path.isdir(base_path):
+                log.info("searching path: %s", base_path)
+                if check_all:
+                    # Find and run all tests in the state/saltcheck-tests directory
+                    self._gather_files(base_path + os.sep + sls_split[2])
+                    return
+                init_path = base_path + os.sep + sls_split[2] + os.sep + 'saltcheck-tests' + os.sep + 'init.tst'
+                name_path = base_path + os.sep + 'saltcheck-tests' + os.sep + sls_split[2] + '.tst'
+                if os.path.isfile(init_path):
+                    self.test_files.append(init_path)
+                    log.info("Found test init: %s", init_path)
+                if os.path.isfile(name_path):
+                    self.test_files.append(name_path)
+                    log.info("Found test named: %s", name_path)
             else:
-                log.info("path is not a directory= %s", full_path)
+                log.info("path is not a directory: %s", base_path)
         return

From 81c8cd1dba0085a63af6632c68e2b05ecc1cb8a1 Mon Sep 17 00:00:00 2001
From: Christian McHugh <mchughc@dnb.com>
Date: Fri, 24 Aug 2018 22:21:12 +0100
Subject: [PATCH 16/16] correct lint lines

---
 salt/modules/saltcheck.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py
index e4b2f6d6c9fd..dc1083491e85 100644
--- a/salt/modules/saltcheck.py
+++ b/salt/modules/saltcheck.py
@@ -267,7 +267,6 @@ def _get_auto_update_cache_value():
     return True
 
 
-
 @memoize
 def _is_valid_function(module_name, function):
     '''