Test suite improvements (#730)

* Tidy up some comments and pylint warnings.

* Exit if a testdata/ input file isn't found.

This allows the other tests to still run even if you haven't cloned
testdata.

* Print a message when the decode succeeds.

* Put ld-cut's output file in the testout directory.

* Add testdata to .gitignore.

* Convert tests into rules in the Makefile.

You can now say "make check" to run the tests locally.

* Handle unexpected EOF in JsonReader.

Previously it would get stuck in a loop if it encountered EOF in the
middle of a string.

* Handle \/ as a valid escape sequence in JsonReader.

* Add some unit tests for JsonReader.

.github/workflows/tests.yml

@@ -33,71 +33,6 @@ jobs:
       timeout-minutes: 5
       run: make install prefix=/usr DESTDIR=/tmp/staging
 
-    - name: Run testfilter
-      timeout-minutes: 5
-      run: tools/library/filter/testfilter/testfilter
-
-    - name: Run testvbidecoder
-      timeout-minutes: 5
-      run: tools/library/tbc/testvbidecoder/testvbidecoder
-
-    - name: Test ld-cut (NTSC)
-      timeout-minutes: 10
-      run: |
-        scripts/test-decode \
-          --cut-seek 30255 \
-          --cut-length 4 \
-          --expect-frames 4 \
-          --expect-vbi 9151563,15925845,15925845 \
-          testdata/ve-snw-cut.lds
-    
-    - name: Test ld-cut (PAL)
-      timeout-minutes: 10
-      run: |
-        scripts/test-decode \
-          --pal \
-          --cut-seek 760 \
-          --cut-length 4 \
-          --expect-frames 4 \
-          --expect-vbi 9152512,15730528,15730528 \
-          testdata/pal/ggv-mb-1khz.ldf
-
-    - name: Decode NTSC CAV
-      timeout-minutes: 10
-      run: |
-        scripts/test-decode \
-          --decoder mono --decoder ntsc2d --decoder ntsc3d \
-          --expect-frames 29 \
-          --expect-bpsnr 43.3 \
-          --expect-vbi 9151563,15925840,15925840 \
-          --expect-efm-samples 40572 \
-          testdata/ve-snw-cut.lds
-
-    - name: Decode NTSC CLV
-      timeout-minutes: 10
-      run: |
-        scripts/test-decode \
-          --expect-frames 4 \
-          --expect-bpsnr 37.6 \
-          --expect-vbi 9167913,15785241,15785241 \
-          testdata/issues/176/issue176.lds
-
-    - name: Decode PAL CAV
+    - name: Run tests
       timeout-minutes: 10
-      run: |
-        scripts/test-decode --pal \
-          --decoder mono --decoder pal2d --decoder transform2d --decoder transform3d \
-          --expect-frames 4 \
-          --expect-bpsnr 38.4 \
-          --expect-vbi 9151527,16065688,16065688 \
-          --expect-efm-samples 5292 \
-          testdata/pal/jason-testpattern.lds
-
-    - name: Decode PAL CLV
-      timeout-minutes: 10
-      run: |
-        scripts/test-decode --pal --no-efm \
-          --expect-frames 9 \
-          --expect-bpsnr 30.3 \
-          --expect-vbi 0,8449774,8449774 \
-          testdata/pal/kagemusha-leadout-cbar.ldf
+      run: make check

.gitignore

@@ -50,6 +50,7 @@ __pycache__
 
 # ld-decode data files
 *.tbc
+testdata
 testout
 
 # Profiling results

Makefile

@@ -73,6 +73,94 @@ install-tools:
 clean-tools:
 	$(MAKE) -C tools clean
 
+### Tests ###
+
+# To run the test suite, clone the ld-decode-testdata repo as "testdata",
+# and do "make check".
+
+TESTCASES = \
+	check-library-filter \
+	check-library-metadata \
+	check-library-vbidecoder \
+	check-ld-cut-ntsc \
+	check-ld-cut-pal \
+	check-decode-ntsc-cav \
+	check-decode-ntsc-clv \
+	check-decode-pal-cav \
+	check-decode-pal-clv
+
+check: $(TESTCASES)
+.PHONY: check $(TESTCASES)
+
+TESTING = printf '\n\#\# Testing %s\n\n'
+
+check-library-filter:
+	@$(TESTING) "library: filter"
+	@tools/library/filter/testfilter/testfilter
+
+check-library-metadata:
+	@$(TESTING) "library: metadata"
+	@tools/library/tbc/testmetadata/testmetadata
+
+check-library-vbidecoder:
+	@$(TESTING) "library: vbidecoder"
+	@tools/library/tbc/testvbidecoder/testvbidecoder
+
+check-ld-cut-ntsc:
+	@$(TESTING) "ld-cut (NTSC)"
+	@scripts/test-decode \
+		--cut-seek 30255 \
+		--cut-length 4 \
+		--expect-frames 4 \
+		--expect-vbi 9151563,15925845,15925845 \
+		testdata/ve-snw-cut.lds
+
+check-ld-cut-pal:
+	@$(TESTING) "ld-cut (PAL)"
+	@scripts/test-decode \
+		--pal \
+		--cut-seek 760 \
+		--cut-length 4 \
+		--expect-frames 4 \
+		--expect-vbi 9152512,15730528,15730528 \
+		testdata/pal/ggv-mb-1khz.ldf
+
+check-decode-ntsc-cav:
+	@$(TESTING) "full decode (NTSC CAV)"
+	@scripts/test-decode \
+		--decoder mono --decoder ntsc2d --decoder ntsc3d \
+		--expect-frames 29 \
+		--expect-bpsnr 43.3 \
+		--expect-vbi 9151563,15925840,15925840 \
+		--expect-efm-samples 40572 \
+		testdata/ve-snw-cut.lds
+
+check-decode-ntsc-clv:
+	@$(TESTING) "full decode (NTSC CLV)"
+	@scripts/test-decode \
+		--expect-frames 4 \
+		--expect-bpsnr 37.6 \
+		--expect-vbi 9167913,15785241,15785241 \
+		testdata/issues/176/issue176.lds
+
+check-decode-pal-cav:
+	@$(TESTING) "full decode (PAL CAV)"
+	@scripts/test-decode --pal \
+		--decoder mono --decoder pal2d --decoder transform2d --decoder transform3d \
+		--expect-frames 4 \
+		--expect-bpsnr 38.4 \
+		--expect-vbi 9151527,16065688,16065688 \
+		--expect-efm-samples 5292 \
+		testdata/pal/jason-testpattern.lds
+
+check-decode-pal-clv:
+	@$(TESTING) "full decode (PAL CLV)"
+	@scripts/test-decode --pal --no-efm \
+		--expect-frames 9 \
+		--expect-bpsnr 30.3 \
+		--expect-vbi 0,8449774,8449774 \
+		testdata/pal/kagemusha-leadout-cbar.ldf
+
 ### Generated files, not updated automatically ###
 
 tools/library/filter/deemp.h: scripts/filtermaker

scripts/test-decode

@@ -1,7 +1,8 @@
 #!/usr/bin/python3
 #
 # test-decode - run the decoding toolchain against an RF sample
-# Copyright (C) 2019 Adam Sampson
+# Copyright (C) 2019-2022 Adam Sampson
+# Copyright (C) 2022 Chad Page
 #
 # This file is part of ld-decode.
 #
@@ -37,6 +38,8 @@ def die(*args):
     sys.exit(1)
 
 def safe_unlink(filename):
+    """Remove a file if it exists; if not, do nothing."""
+
     try:
         os.unlink(filename)
     except FileNotFoundError:
@@ -65,9 +68,10 @@ def run_command(cmd, **kwopts):
         die(cmd[0], 'failed with exit code', rc)
 
 def run_ld_cut(args):
-    """Run ld-decode."""
+    """Run ld-cut."""
 
-    safe_unlink(args.infile + '.cut.ldf')
+    cutfile = args.output + '.cut.ldf'
+    safe_unlink(cutfile)
 
     cmd = [src_dir + '/ld-cut']
     if args.pal:
@@ -78,15 +82,18 @@ def run_ld_cut(args):
         cmd += ['-s', str(args.cutstart)]
     else:
         die(cmd[0], 'internal error: run_ld_cut() run without seek or start points')
-    
+
     cmd += ['-l', str(args.cutlength)]
-    cmd += [args.infile, args.infile + '.cut.ldf']
+    cmd += [args.infile, cutfile]
 
     # Set PATH so it can invoke helper programs
     env = os.environ.copy()
     env['PATH'] = src_dir + ':' + env['PATH']
     run_command(cmd, env=env)
 
+    # Use the output as the input for the rest of the decoding process
+    args.infile = cutfile
+
 def run_ld_decode(args):
     """Run ld-decode."""
 
@@ -97,10 +104,7 @@ def run_ld_decode(args):
     if args.pal:
         cmd += ['--pal']
 
-    cutfile = args.infile + '.cut.ldf'
-    infile = cutfile if os.path.isfile(cutfile) else args.infile
-
-    cmd += [infile, args.output]
+    cmd += [args.infile, args.output]
 
     # Set PATH so it can invoke helper programs
     env = os.environ.copy()
@@ -137,7 +141,7 @@ def run_ld_process_vbi(args):
         if args.expect_bpsnr > bpsnr:
             die(json_file, 'has median bPSNR', bpsnr, 'dB, expected',
                 args.expect_bpsnr, 'dB')
-           
+
     # Print VBI data (useful for finding --expect_vbi values)
     if args.print_vbi:
         for field in data["fields"]:
@@ -261,7 +265,8 @@ def main():
                        help='get N frames from ld-cut')
     group.add_argument('--decoder', metavar='decoder', action='append',
                        dest='decoders', default=[],
-                       help='use specific ld-chroma-decoder decoder (use more than once to test multiple decoders)')
+                       help='use specific ld-chroma-decoder decoder '
+                            '(use more than once to test multiple decoders)')
     group = parser.add_argument_group("Sanity checks")
     group.add_argument('--expect-frames', metavar='N', type=int,
                        help='expect at least N frames of video output')
@@ -280,6 +285,10 @@ def main():
     if args.decoders == []:
         args.decoders = [None]
 
+    if args.infile.startswith("testdata/") and not os.path.exists(args.infile):
+        print('Input file', args.infile, 'not present, skipping test')
+        return
+
     # Find the top-level source directory
     prog_path = os.path.realpath(sys.argv[0])
     global src_dir
@@ -297,7 +306,8 @@ def main():
     if output_dir != '':
         os.makedirs(output_dir, exist_ok=True)
 
-    safe_unlink(args.infile + '.cut.ldf')
+    # Produce a trimmed input file using ld-cut, if requested
+    orig_infile = args.infile
     if args.cutseek is not None or args.cutstart is not None:
         run_ld_cut(args)
 
@@ -310,6 +320,7 @@ def main():
     for decoder in args.decoders:
         run_ld_chroma_decoder(args, decoder)
 
+    print('\nDecoding', orig_infile, 'completed successfully')
+
 if __name__ == '__main__':
     main()
-

tools/library/tbc/jsonio.cpp

@@ -192,12 +192,12 @@ void JsonReader::discard()
     }
 }
 
-// Get the next input character
+// Get the next input character, returning 0 on EOF or error
 char JsonReader::get()
 {
     char c;
     input.get(c);
-    // XXX check EOF
+    if (!input.good()) return 0;
     ++position;
     return c;
 }
@@ -232,13 +232,17 @@ void JsonReader::readString(std::string &value)
     while (true) {
         c = get();
         switch (c) {
+        case 0:
+            // End of input
+            throwError("end of input in string");
         case '"':
             // End of string
             return;
         case '\\':
             c = get();
             switch (c) {
             case '"':
+            case '/':
             case '\\':
                 value.push_back(c);
                 break;