Add a --skipvectors argument to the run command (#141)

For automated tests, it is useful to be able to skip some vectors that
are failing on a specific hardware or with a specific decoder.

This adds an argument to skip those vectors and mark then as Not Run.
The generated reports will mark them as Skipped.

Signed-off-by: Detlev Casanova <detlev.casanova@collabora.com>

README.md

@@ -352,8 +352,9 @@ optional arguments:
 
 usage: fluster.py run [-h] [-j JOBS] [-t TIMEOUT] [-ff] [-q]
 [-ts TESTSUITES [TESTSUITES ...]] [-tv TESTVECTORS [TESTVECTORS ...]]
-[-d DECODERS [DECODERS ...]] [-s] [-so SUMMARY_OUTPUT]
-[-f {md,csv,junitxml}] [-k] [-th THRESHOLD] [-tth TIME_THRESHOLD] [-v]
+[-sv SKIPVECTORS [SKIPVECTORS ...]] [-d DECODERS [DECODERS ...]] [-s]
+[-so SUMMARY_OUTPUT] [-f {md,csv,junitxml}] [-k] [-th THRESHOLD]
+[-tth TIME_THRESHOLD] [-v]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -367,6 +368,8 @@ optional arguments:
                         run only the specific test suites
   -tv TESTVECTORS [TESTVECTORS ...], --testvectors TESTVECTORS [TESTVECTORS ...]
                         run only the specific test vectors
+  -sv SKIPVECTORS [SKIPVECTORS ...], --skipvectors SKIPVECTORS [SKIPVECTORS ...]
+                        skip the specific test vectors
   -d DECODERS [DECODERS ...], --decoders DECODERS [DECODERS ...]
                         run only the specific decoders
   -s, --summary         generate a summary in Markdown format for each test

fluster/fluster.py

@@ -50,6 +50,7 @@ def __init__(
         test_suites: List[str],
         decoders: List[str],
         test_vectors: List[str],
+        skip_vectors: List[str],
         failfast: bool = False,
         quiet: bool = False,
         reference: bool = False,
@@ -68,6 +69,7 @@ def __init__(
         self.decoders_names = decoders
         self.decoders: List[Decoder] = []
         self.test_vectors_names = test_vectors
+        self.skip_vectors_names = skip_vectors
         self.failfast = failfast
         self.quiet = quiet
         self.reference = reference
@@ -80,7 +82,11 @@ def __init__(
         self.summary_format = summary_format
 
     def to_test_suite_context(
-        self, decoder: Decoder, results_dir: str, test_vectors: List[str]
+        self,
+        decoder: Decoder,
+        results_dir: str,
+        test_vectors: List[str],
+        skip_vectors: List[str],
     ) -> TestSuiteContext:
         """Create a TestSuite's Context from this"""
         ts_context = TestSuiteContext(
@@ -92,6 +98,7 @@ def to_test_suite_context(
             results_dir=results_dir,
             reference=self.reference,
             test_vectors=test_vectors,
+            skip_vectors=skip_vectors,
             keep_files=self.keep_files,
             verbose=self.verbose,
         )
@@ -227,6 +234,8 @@ def _normalize_context(self, ctx: Context) -> None:
             ctx.decoders_names = [x.lower() for x in ctx.decoders_names]
         if ctx.test_vectors_names:
             ctx.test_vectors_names = [x.lower() for x in ctx.test_vectors_names]
+        if ctx.skip_vectors_names:
+            ctx.skip_vectors_names = [x.lower() for x in ctx.skip_vectors_names]
         ctx.test_suites = self._get_matches(
             ctx.test_suites_names, self.test_suites, "test suite"
         )
@@ -264,7 +273,10 @@ def run_test_suites(self, ctx: Context) -> None:
                     continue
                 test_suite_res = test_suite.run(
                     ctx.to_test_suite_context(
-                        decoder, self.results_dir, ctx.test_vectors_names
+                        decoder,
+                        self.results_dir,
+                        ctx.test_vectors_names,
+                        ctx.skip_vectors_names,
                     )
                 )
 
@@ -362,7 +374,9 @@ def _parse_suite_results(
 
                 for vector in suite_decoder_res[1].test_vectors.values():
                     jcase = junitp.TestCase(vector.name)
-                    if vector.test_result not in [
+                    if vector.test_result == TestVectorResult.NOT_RUN:
+                        jcase.result = [junitp.Skipped()]
+                    elif vector.test_result not in [
                         TestVectorResult.SUCCESS,
                         TestVectorResult.REFERENCE,
                     ]:

fluster/main.py

@@ -214,6 +214,12 @@ def _add_run_cmd(self, subparsers: Any) -> None:
             help="run only the specific test vectors",
             nargs="+",
         )
+        subparser.add_argument(
+            "-sv",
+            "--skipvectors",
+            help="skip the specific test vectors",
+            nargs="+",
+        )
         subparser.add_argument(
             "-d",
             "--decoders",
@@ -343,6 +349,7 @@ def _run_cmd(self, args: Any, fluster: Fluster) -> None:
             timeout=args.timeout,
             decoders=args.decoders,
             test_vectors=args.testvectors,
+            skip_vectors=args.skipvectors,
             failfast=args.failfast,
             quiet=args.quiet,
             summary=args.summary or args.summary_output,
@@ -365,6 +372,7 @@ def _reference_cmd(self, args: Any, fluster: Fluster) -> None:
             test_suites=args.testsuites,
             decoders=args.decoder,
             test_vectors=[],
+            skip_vectors=[],
             quiet=args.quiet,
             verbose=args.verbose,
             reference=True,

fluster/test.py

@@ -35,6 +35,7 @@ def __init__(
         decoder: Decoder,
         test_suite: Any,  # can't use TestSuite type because of circular dependency
         test_vector: TestVector,
+        skip: bool,
         results_dir: str,
         reference: bool,
         timeout: int,
@@ -44,6 +45,7 @@ def __init__(
         self.decoder = decoder
         self.test_suite = test_suite
         self.test_vector = test_vector
+        self.skip = skip
         self.resources_dir = self.test_suite.resources_dir
         self.results_dir = results_dir
         self.reference = reference
@@ -54,6 +56,13 @@ def __init__(
         super().__init__(test_vector.name)
 
     def _test(self) -> None:
+        if self.skip:
+            self.test_suite.test_vectors[
+                self.test_vector.name
+            ].test_result = TestVectorResult.NOT_RUN
+
+            return
+
         output_filepath = os.path.join(self.results_dir, self.test_vector.name + ".out")
 
         input_filepath = os.path.join(

fluster/test_suite.py

@@ -69,6 +69,7 @@ def __init__(
         results_dir: str,
         reference: bool = False,
         test_vectors: Optional[List[str]] = None,
+        skip_vectors: Optional[List[str]] = None,
         failing_test_vectors: Optional[List[str]] = None,
         keep_files: bool = False,
         verbose: bool = False,
@@ -81,6 +82,7 @@ def __init__(
         self.results_dir = results_dir
         self.reference = reference
         self.test_vectors = test_vectors
+        self.skip_vectors = skip_vectors
         self.failing_test_vectors = failing_test_vectors
         self.keep_files = keep_files
         self.verbose = verbose
@@ -403,6 +405,8 @@ def run(self, ctx: Context) -> Optional["TestSuite"]:
         string = f"Running test suite {self.name} with decoder {ctx.decoder.name}\n"
         if ctx.test_vectors:
             string += f'Test vectors {" ".join(ctx.test_vectors)}\n'
+        if ctx.skip_vectors:
+            string += f'Skipping test vectors {" ".join(ctx.skip_vectors)}\n'
         string += f"Using {ctx.jobs} parallel job(s)"
         print(string)
         print("*" * 100 + "\n")
@@ -422,14 +426,19 @@ def generate_tests(self, ctx: Context) -> List[Test]:
         tests = []
         test_vectors_run = {}
         for name, test_vector in self.test_vectors.items():
+            skip = False
             if ctx.test_vectors:
                 if test_vector.name.lower() not in ctx.test_vectors:
                     continue
+            if ctx.skip_vectors:
+                if test_vector.name.lower() in ctx.skip_vectors:
+                    skip = True
             tests.append(
                 Test(
                     ctx.decoder,
                     self,
                     test_vector,
+                    skip,
                     ctx.results_dir,
                     ctx.reference,
                     ctx.timeout,