Use @test_broken

JuliaPlots · Sep 9, 2024 · d556448 · d556448
1 parent e3aebe6
commit d556448
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 8 deletions.
diff --git a/PlotsBase/test/runtests.jl b/PlotsBase/test/runtests.jl
@@ -54,17 +54,17 @@ is_ci() || @eval using Gtk  # see JuliaPlots/VisualRegressionTests.jl/issues/30
 
 ref_name(i) = "ref" * lpad(i, 3, '0')
 
-const blacklist = if VERSION.major == 1 && VERSION.minor ≥ 9
+const broken_examples = if VERSION.major == 1 && VERSION.minor ≥ 9
     [
-        25,
+        25, # FIXME: remove, when StatsPlots supports Plots v2
         30, # FIXME: remove, when StatsPlots supports Plots v2
-        41,
-    ]  # FIXME: github.com/JuliaLang/julia/issues/47261
+        41, # FIXME: https://github.com/JuliaLang/julia/issues/47261
+    ]
 else
     []
 end
 if Sys.isapple()
-    push!(blacklist, 50)  # FIXME: https://github.com/jheinen/GR.jl/issues/550
+    push!(broken_examples, 50)  # FIXME: https://github.com/jheinen/GR.jl/issues/550
 end
 
 for name ∈ (

diff --git a/PlotsBase/test/test_backends.jl b/PlotsBase/test/test_backends.jl
@@ -66,7 +66,7 @@ is_pkgeval() || @testset "Backends" begin
         @test filesize(fn) > 1_000
     end
     Sys.islinux() && for be ∈ TEST_BACKENDS
-        skip = vcat(PlotsBase._backend_skips[be], blacklist)
+        skip = vcat(PlotsBase._backend_skips[be], broken_examples)
         PlotsBase.test_examples(be; skip, callback, disp = is_ci(), strict = true)  # `ci` display for coverage
         closeall()
     end

diff --git a/PlotsBase/test/test_reference.jl b/PlotsBase/test/test_reference.jl
@@ -105,14 +105,19 @@ end
 function image_comparison_facts(
     pkg::Symbol;
     skip = [],          # skip these examples (int index)
+    broken = [],        # known broken examples (int index)
     only = nothing,     # limit to these examples (int index)
     debug = false,      # print debug information ?
     sigma = [1, 1],     # number of pixels to "blur"
     tol = 1e-2,         # acceptable error (percent)
 )
     for i ∈ setdiff(1:length(PlotsBase._examples), skip)
         if only ≡ nothing || i in only
-            @test success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            if i ∈ broken
+                @test_broken success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            else
+                @test success(image_comparison_tests(pkg, i; debug, sigma, tol))
+            end
         end
     end
 end
@@ -141,7 +146,8 @@ end
         image_comparison_facts(
             :gr,
             tol = PLOTS_IMG_TOL,
-            skip = vcat(PlotsBase._backend_skips[:gr], blacklist),
+            skip = vcat(PlotsBase._backend_skips[:gr]),
+            broken = broken_examples,
         )
     end
 end