Fail tests marked with .only if running in CI

modules/core/src/weaver/suites.scala

@@ -1,5 +1,8 @@
 package weaver
 
+import scala.concurrent.duration.FiniteDuration
+
+import cats.data.Chain
 import cats.effect.{ Async, Resource }
 import cats.syntax.all._
 
@@ -92,19 +95,29 @@ abstract class MutableFSuite[F[_]] extends RunnableSuite[F]  {
     def usingRes(run : Res => F[Expectations]) : Unit = apply(run)
   }
 
+  def isCI: Boolean = "true" == System.getenv("CI")
+
   override def spec(args: List[String]) : Stream[F, TestOutcome] =
     synchronized {
       if (!isInitialized) isInitialized = true
-      val testsNotIgnored = testSeq.filterNot(_._1.tags(TestName.Tags.ignore))
-      val testsTaggedOnly = testsNotIgnored.filter(_._1.tags(TestName.Tags.only))
-      val filteredTests = if (testsTaggedOnly.isEmpty) {
+      val testsNotIgnored: Seq[(TestName, Res => F[TestOutcome])] = testSeq.filterNot(_._1.tags(TestName.Tags.ignore))
+      val testsTaggedOnly: Seq[(TestName, Res => F[TestOutcome])]  = testSeq.filter(_._1.tags(TestName.Tags.only))
+      val onlyTestsNotIgnored = testsTaggedOnly.filter(taggedOnly => testsNotIgnored.contains(taggedOnly))
+      val filteredTests = if (onlyTestsNotIgnored.isEmpty) {
         val argsFilter = Filters.filterTests(this.name)(args)
         testsNotIgnored.collect {
           case (name, test) if argsFilter(name) => test
         }
-      } else testsTaggedOnly.map(_._2)
+      } else onlyTestsNotIgnored.map(_._2)
       val parallism = math.max(1, maxParallelism)
-      if (filteredTests.isEmpty) Stream.empty // no need to allocate resources
+
+      if (testsTaggedOnly.nonEmpty && isCI) {
+        val failureOutcomes = testsTaggedOnly
+          .map(_._1)
+          .map(onlyNotOnCiFailure)
+        Stream.emits(failureOutcomes).lift[F](effectCompat.effect)
+      }
+      else if (filteredTests.isEmpty) Stream.empty // no need to allocate resources
       else for {
         resource <- Stream.resource(sharedResource)
         tests = filteredTests.map(_.apply(resource))
@@ -114,6 +127,20 @@ abstract class MutableFSuite[F[_]] extends RunnableSuite[F]  {
       } yield result
     }
 
+  private[this] def onlyNotOnCiFailure(test: TestName): TestOutcome = {
+    val result = Result.Failure(
+      msg = "'Only' tag is not allowed when `isCI=true`",
+      source = None,
+      location = List(test.location)
+    )
+    TestOutcome(
+      name = test.name,
+      duration = FiniteDuration(0, "ns"),
+      result = result,
+      log = Chain.empty
+    )
+  }
+
   private[this] var testSeq = Seq.empty[(TestName, Res => F[TestOutcome])]
 
   def plan: List[TestName] = testSeq.map(_._1).toList

modules/framework/cats/test/src-jvm/junit/JUnitRunnerTests.scala

@@ -37,7 +37,7 @@ object JUnitRunnerTests extends IOSuite {
       expect.same(filteredNotifs, expected) and
         exists(failureMessage)(s =>
           expect(s.contains("oops")) && expect(
-            s.contains("JUnitRunnerTests.scala")))
+            s.contains("Meta.scala")))
     }
   }
 
@@ -54,6 +54,35 @@ object JUnitRunnerTests extends IOSuite {
     }
   }
 
+  test("Tests tagged with only fail when ran on CI") { blocker =>
+    run(blocker, Meta.OnlyFailsOnCi).map { notifications =>
+      def testFailure(name: String, lineNumber: Int) = {
+        val srcPath  = "modules/framework/cats/test/src-jvm/junit/Meta.scala"
+        val msgLine1 = s"- $name 0ms"
+        val msgLine2 =
+          s"  'Only' tag is not allowed when `isCI=true` ($srcPath:$lineNumber)"
+        TestFailure(
+          name = name + "(weaver.junit.Meta$OnlyFailsOnCi$)",
+          message =
+            s"$msgLine1\n$msgLine2\n\n"
+        )
+      }
+      val expected = List(
+        TestSuiteStarted("weaver.junit.Meta$OnlyFailsOnCi$"),
+        TestIgnored("normal test(weaver.junit.Meta$OnlyFailsOnCi$)"),
+        TestIgnored("not only(weaver.junit.Meta$OnlyFailsOnCi$)"),
+        TestStarted("first only test(weaver.junit.Meta$OnlyFailsOnCi$)"),
+        testFailure("first only test", 46),
+        TestFinished("first only test(weaver.junit.Meta$OnlyFailsOnCi$)"),
+        TestStarted("second only test(weaver.junit.Meta$OnlyFailsOnCi$)"),
+        testFailure("second only test", 50),
+        TestFinished("second only test(weaver.junit.Meta$OnlyFailsOnCi$)"),
+        TestSuiteFinished("weaver.junit.Meta$OnlyFailsOnCi$")
+      )
+      expect.same(notifications, expected)
+    }
+  }
+
   test("Only tests tagged with only are ran (unless also tagged ignored)") {
     blocker =>
       run(blocker, Meta.IgnoreAndOnly).map { notifications =>
@@ -85,6 +114,36 @@ object JUnitRunnerTests extends IOSuite {
     }
   }
 
+  test(
+    "Even if all tests are ignored, will fail if a test is tagged with only") {
+    blocker =>
+      run(blocker, Meta.OnlyFailsOnCiEvenIfIgnored).map { notifications =>
+        def testFailure(name: String, lineNumber: Int) = {
+          val srcPath  = "modules/framework/cats/test/src-jvm/junit/Meta.scala"
+          val msgLine1 = s"- $name 0ms"
+          val msgLine2 =
+            s"  'Only' tag is not allowed when `isCI=true` ($srcPath:$lineNumber)"
+          TestFailure(
+            name = name + "(weaver.junit.Meta$OnlyFailsOnCiEvenIfIgnored$)",
+            message =
+              s"$msgLine1\n$msgLine2\n\n"
+          )
+        }
+        val expected = List(
+          TestSuiteStarted("weaver.junit.Meta$OnlyFailsOnCiEvenIfIgnored$"),
+          TestIgnored(
+            "only and ignored(weaver.junit.Meta$OnlyFailsOnCiEvenIfIgnored$)"),
+          TestStarted(
+            "only and ignored(weaver.junit.Meta$OnlyFailsOnCiEvenIfIgnored$)"),
+          testFailure("only and ignored", 110),
+          TestFinished(
+            "only and ignored(weaver.junit.Meta$OnlyFailsOnCiEvenIfIgnored$)"),
+          TestSuiteFinished("weaver.junit.Meta$OnlyFailsOnCiEvenIfIgnored$")
+        )
+        expect.same(notifications, expected)
+      }
+  }
+
   test("Works when suite asks for global resources") {
     blocker =>
       run(blocker, classOf[Meta.Sharing]).map { notifications =>
@@ -143,91 +202,3 @@ object JUnitRunnerTests extends IOSuite {
   }
 
 }
-
-object Meta {
-
-  object MySuite extends SimpleIOSuite {
-
-    override def maxParallelism: Int = 1
-
-    pureTest("success") {
-      success
-    }
-
-    pureTest("failure") {
-      failure("oops")
-    }
-
-    test("ignore") {
-      ignore("just because")
-    }
-
-  }
-
-  object Only extends SimpleIOSuite {
-
-    override def maxParallelism: Int = 1
-
-    pureTest("only".only) {
-      success
-    }
-
-    pureTest("not only") {
-      failure("foo")
-    }
-
-  }
-
-  object Ignore extends SimpleIOSuite {
-
-    override def maxParallelism: Int = 1
-
-    pureTest("not ignored 1") {
-      success
-    }
-
-    pureTest("not ignored 2") {
-      success
-    }
-
-    pureTest("is ignored".ignore) {
-      failure("foo")
-    }
-
-  }
-
-  object IgnoreAndOnly extends SimpleIOSuite {
-
-    override def maxParallelism: Int = 1
-
-    pureTest("only".only) {
-      success
-    }
-
-    pureTest("not tagged") {
-      failure("foo")
-    }
-
-    pureTest("only and ignored".only.ignore) {
-      failure("foo")
-    }
-
-    pureTest("is ignored".ignore) {
-      failure("foo")
-    }
-
-  }
-
-  class Sharing(global: GlobalRead) extends IOSuite {
-
-    type Res = Unit
-    // Just checking the suite does not crash
-    def sharedResource: Resource[IO, Unit] = global.getR[Int]().map(_ => ())
-
-    pureTest("foo") {
-      success
-    }
-
-  }
-
-}

modules/framework/cats/test/src-jvm/junit/Meta.scala

@@ -0,0 +1,128 @@
+package weaver
+package junit
+
+import cats.effect.{ IO, Resource }
+
+object Meta {
+
+  object MySuite extends SimpleIOSuite {
+
+    override def maxParallelism: Int = 1
+
+    pureTest("success") {
+      success
+    }
+
+    pureTest("failure") {
+      failure("oops")
+    }
+
+    test("ignore") {
+      ignore("just because")
+    }
+
+  }
+
+  object Only extends SimpleIOSuite {
+
+    override def maxParallelism: Int = 1
+    override def isCI: Boolean       = false
+
+    pureTest("only".only) {
+      success
+    }
+
+    pureTest("not only") {
+      failure("foo")
+    }
+
+  }
+
+  object OnlyFailsOnCi extends SimpleIOSuite {
+
+    override def maxParallelism: Int = 1
+    override def isCI: Boolean       = true
+
+    pureTest("first only test".only) {
+      success
+    }
+
+    pureTest("second only test".only) {
+      success
+    }
+
+    pureTest("normal test") {
+      success
+    }
+
+    pureTest("not only") {
+      failure("foo")
+    }
+
+  }
+
+  object Ignore extends SimpleIOSuite {
+
+    override def maxParallelism: Int = 1
+
+    pureTest("not ignored 1") {
+      success
+    }
+
+    pureTest("not ignored 2") {
+      success
+    }
+
+    pureTest("is ignored".ignore) {
+      failure("foo")
+    }
+
+  }
+
+  object IgnoreAndOnly extends SimpleIOSuite {
+
+    override def maxParallelism: Int = 1
+    override def isCI: Boolean       = false
+
+    pureTest("only".only) {
+      success
+    }
+
+    pureTest("not tagged") {
+      failure("foo")
+    }
+
+    pureTest("only and ignored".only.ignore) {
+      failure("foo")
+    }
+
+    pureTest("is ignored".ignore) {
+      failure("foo")
+    }
+
+  }
+
+  object OnlyFailsOnCiEvenIfIgnored extends SimpleIOSuite {
+
+    override def maxParallelism: Int = 1
+    override def isCI: Boolean       = true
+
+    pureTest("only and ignored".only.ignore) {
+      failure("foo")
+    }
+
+  }
+
+  class Sharing(global: GlobalRead) extends IOSuite {
+
+    type Res = Unit
+    // Just checking the suite does not crash
+    def sharedResource: Resource[IO, Unit] = global.getR[Int]().map(_ => ())
+
+    pureTest("foo") {
+      success
+    }
+
+  }
+
+}