Skip to content

Commit

Permalink
Add --slience-skips and --force-newlines
Browse files Browse the repository at this point in the history 
--silence-skips makes it easier to declutter test debugging sessions where only a single spec is being run

--force-newlines ensures a newline character appears after every spec.  this may help some CI systems flush their output.
  • Loading branch information
@onsi
onsi committed May 21, 2024
1 parent 42013d6 commit f010b65
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 8 deletions.
4 changes: 4 additions & 0 deletions docs/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2697,6 +2697,8 @@ These mechanisms can all be used in concert. They combine with the following ru
- Programmatic filters always apply and result in a non-zero exit code. Any additional CLI filters only apply to the subset of specs selected by the programmatic filters.
- When multiple CLI filters (`--label-filter`, `--focus-file/--skip-file`, `--focus/--skip`) are provided they are all ANDed together. The spec must satisfy the label filter query **and** any location-based filters **and** any description based filters.

If you have a large test suite and would like to avoid printing out all the `S` skip delimiters you can run with `--silence-skips` to suppress them.

#### Avoiding filtering out all tests

Especially for CI it is useful to fail when all tests were filtered out by accident (either via skip or typo in label filter).
Expand Down Expand Up @@ -3717,6 +3719,8 @@ Here's why:

If running on Github actions: `--github-output` will make the output more readable in the Github actions console.

If your CI system will only flush if a newline character is seen you may want to set `--force-newlines` to ensure that the output is flushed correctly.

### Supporting Custom Suite Configuration

There are contexts where you may want to change some aspects of a suite's behavior based on user-provided configuration. There are two widely adopted means of doing this: environment variables and command-line flags.
Expand Down
10 changes: 9 additions & 1 deletion reporters/default_reporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,11 @@ func (r *DefaultReporter) DidRun(report types.SpecReport) {
v := r.conf.Verbosity()
inParallel := report.RunningInParallel

//should we completely omit this spec?
if report.State.Is(types.SpecStateSkipped) && r.conf.SilenceSkips {
return
}

header := r.specDenoter
if report.LeafNodeType.Is(types.NodeTypesForSuiteLevelNodes) {
header = fmt.Sprintf("[%s]", report.LeafNodeType)
Expand Down Expand Up @@ -278,9 +283,12 @@ func (r *DefaultReporter) DidRun(report types.SpecReport) {
}
}

// If we have no content to show, jsut emit the header and return
// If we have no content to show, just emit the header and return
if !reportHasContent {
r.emit(r.f(highlightColor + header + "{{/}}"))
if r.conf.ForceNewlines {
r.emit("\n")
}
return
}

Expand Down
42 changes: 35 additions & 7 deletions reporters/default_reporter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ func S(options ...interface{}) types.SpecReport {
return report
}

type ConfigFlag uint8
type ConfigFlag uint16

const (
Succinct ConfigFlag = 1 << iota
Expand All @@ -177,6 +177,8 @@ const (
FullTrace
ShowNodeEvents
GithubOutput
SilenceSkips
ForceNewlines

Parallel //used in the WillRun => DidRun specs to capture behavior when running in parallel
)
Expand Down Expand Up @@ -208,6 +210,12 @@ func (cf ConfigFlag) String() string {
if cf.Has(GithubOutput) {
out = append(out, "github-output")
}
if cf.Has(SilenceSkips) {
out = append(out, "silence-skips")
}
if cf.Has(ForceNewlines) {
out = append(out, "force-newlines")
}
return strings.Join(out, "|")
}

Expand All @@ -231,6 +239,8 @@ func C(flags ...ConfigFlag) types.ReporterConfig {
FullTrace: f.Has(FullTrace),
ShowNodeEvents: f.Has(ShowNodeEvents),
GithubOutput: f.Has(GithubOutput),
SilenceSkips: f.Has(SilenceSkips),
ForceNewlines: f.Has(ForceNewlines),
}
}

Expand Down Expand Up @@ -572,7 +582,10 @@ var _ = Describe("DefaultReporter", func() {
S(CLS(cl0, cl1), CTS("A", "B"), "C", cl2),
Case(Succinct, Normal, Succinct|Parallel, Normal|Parallel,
"{{green}}"+DENOTER+"{{/}}"),
Case(Verbose,
Case(Succinct|ForceNewlines, Normal|ForceNewlines, Succinct|Parallel|ForceNewlines, Normal|Parallel|ForceNewlines,
"{{green}}"+DENOTER+"{{/}}",
""),
Case(Verbose, Verbose|ForceNewlines,
DELIMITER,
"{{/}}A {{gray}}B {{/}}{{bold}}C{{/}}",
"{{gray}}cl2.go:80{{/}}",
Expand Down Expand Up @@ -694,7 +707,10 @@ var _ = Describe("DefaultReporter", func() {
),
Case(Succinct, Normal, Succinct|Parallel, Normal|Parallel,
spr("{{green}}%s{{/}}", DENOTER)),
Case(Verbose, VeryVerbose,
Case(Succinct|ForceNewlines, Normal|ForceNewlines, Succinct|Parallel|ForceNewlines, Normal|Parallel|ForceNewlines,
spr("{{green}}%s{{/}}", DENOTER),
""),
Case(Verbose, VeryVerbose, Verbose|ForceNewlines, VeryVerbose|ForceNewlines,
DELIMITER,
"{{/}}{{bold}}A{{/}}",
"{{gray}}cl0.go:12{{/}}",
Expand Down Expand Up @@ -722,7 +738,7 @@ var _ = Describe("DefaultReporter", func() {
S(types.NodeTypeIt, "A", cl0,
RE("my entry", cl1),
),
Case(Succinct, Normal, Succinct|Parallel, Normal|Parallel,
Case(Succinct, Normal, Succinct|Parallel, Normal|Parallel, Succinct|ForceNewlines, Normal|ForceNewlines, Succinct|Parallel|ForceNewlines, Normal|Parallel|ForceNewlines,
DELIMITER,
spr("{{green}}%s [1.000 seconds]{{/}}", DENOTER),
"{{green}}{{bold}}A{{/}}",
Expand All @@ -733,7 +749,7 @@ var _ = Describe("DefaultReporter", func() {
" {{gray}}<< Report Entries{{/}}",
DELIMITER,
""),
Case(Verbose, VeryVerbose,
Case(Verbose, VeryVerbose, Verbose|ForceNewlines, VeryVerbose|ForceNewlines,
DELIMITER,
"{{/}}{{bold}}A{{/}}",
"{{gray}}cl0.go:12{{/}}",
Expand Down Expand Up @@ -800,6 +816,9 @@ var _ = Describe("DefaultReporter", func() {
),
Case(Succinct, Normal, Succinct|Parallel, Normal|Parallel, Succinct|ShowNodeEvents, Normal|ShowNodeEvents,
spr("{{green}}%s{{/}}", DENOTER)),
Case(Succinct|ForceNewlines, Normal|ForceNewlines, Succinct|Parallel|ForceNewlines, Normal|Parallel|ForceNewlines, Succinct|ShowNodeEvents|ForceNewlines, Normal|ShowNodeEvents|ForceNewlines,
spr("{{green}}%s{{/}}", DENOTER),
""),
Case(Verbose, VeryVerbose, //nothing to see here since things are emitted while streaming, which we don't simulate
DELIMITER,
"{{/}}{{bold}}A{{/}}",
Expand Down Expand Up @@ -873,7 +892,10 @@ var _ = Describe("DefaultReporter", func() {
S(types.NodeTypeIt, "A", types.SpecStateSkipped, cl0),
Case(Succinct, Normal, Succinct|Parallel, Normal|Parallel, Verbose, Verbose|Parallel,
"{{cyan}}S{{/}}"),
Case(VeryVerbose,
Case(Succinct|ForceNewlines, Normal|ForceNewlines, Succinct|Parallel|ForceNewlines, Normal|Parallel|ForceNewlines, Verbose|ForceNewlines, Verbose|Parallel|ForceNewlines,
"{{cyan}}S{{/}}",
""),
Case(VeryVerbose, VeryVerbose|ForceNewlines,
"{{cyan}}S [SKIPPED]{{/}}",
"{{cyan}}{{bold}}A{{/}}",
"{{gray}}cl0.go:12{{/}}",
Expand All @@ -886,6 +908,7 @@ var _ = Describe("DefaultReporter", func() {
"{{gray}}cl0.go:12{{/}}",
DELIMITER,
""),
Case(Succinct|SilenceSkips, Normal|SilenceSkips, Succinct|Parallel|SilenceSkips, Normal|Parallel|SilenceSkips, Verbose|SilenceSkips, Verbose|Parallel|SilenceSkips, VeryVerbose|SilenceSkips, VeryVerbose|Parallel|SilenceSkips, ""),
),
Entry("a user-skipped test",
S(types.NodeTypeIt, "A", types.SpecStateSkipped, cl0,
Expand Down Expand Up @@ -931,6 +954,7 @@ var _ = Describe("DefaultReporter", func() {
" {{gray}}<< Timeline{{/}}",
DELIMITER,
""),
Case(Succinct|SilenceSkips, Normal|SilenceSkips, Succinct|Parallel|SilenceSkips, Normal|Parallel|SilenceSkips, Verbose|SilenceSkips, Verbose|Parallel|SilenceSkips, VeryVerbose|SilenceSkips, VeryVerbose|Parallel|SilenceSkips, ""),
),
Entry("a user-skipped test with timeline content",
S(types.NodeTypeIt, "A", types.SpecStateSkipped, cl0,
Expand Down Expand Up @@ -983,12 +1007,16 @@ var _ = Describe("DefaultReporter", func() {
DELIMITER,
"",
),
Case(Succinct|SilenceSkips, Normal|SilenceSkips, Succinct|Parallel|SilenceSkips, Normal|Parallel|SilenceSkips, Verbose|SilenceSkips, Verbose|Parallel|SilenceSkips, VeryVerbose|SilenceSkips, VeryVerbose|Parallel|SilenceSkips, ""),
),
Entry("a pending test",
S(types.NodeTypeIt, "C", types.SpecStatePending, cl2, CTS("A", "B"), CLS(cl0, cl1)),
Case(Succinct, Succinct|Parallel,
"{{yellow}}P{{/}}"),
Case(Normal, Normal|Parallel, Verbose|Parallel,
Case(Succinct|ForceNewlines, Succinct|Parallel|ForceNewlines,
"{{yellow}}P{{/}}",
""),
Case(Normal, Normal|Parallel, Verbose|Parallel, Normal|ForceNewlines, Normal|Parallel|ForceNewlines, Verbose|Parallel|ForceNewlines,
DELIMITER,
"{{yellow}}P [PENDING]{{/}}",
"{{/}}A {{gray}}B {{yellow}}{{bold}}C{{/}}",
Expand Down
6 changes: 6 additions & 0 deletions types/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ type ReporterConfig struct {
FullTrace bool
ShowNodeEvents bool
GithubOutput bool
SilenceSkips bool
ForceNewlines bool

JSONReport string
JUnitReport string
Expand Down Expand Up @@ -337,6 +339,10 @@ var ReporterConfigFlags = GinkgoFlags{
Usage: "If set, default reporter prints node > Enter and < Exit events when specs fail"},
{KeyPath: "R.GithubOutput", Name: "github-output", SectionKey: "output",
Usage: "If set, default reporter prints easier to manage output in Github Actions."},
{KeyPath: "R.SilenceSkips", Name: "silence-skips", SectionKey: "output",
Usage: "If set, default reporter will not print out skipped tests."},
{KeyPath: "R.ForceNewlines", Name: "force-newlines", SectionKey: "output",
Usage: "If set, default reporter will ensure a newline appears after each test."},

{KeyPath: "R.JSONReport", Name: "json-report", UsageArgument: "filename.json", SectionKey: "output",
Usage: "If set, Ginkgo will generate a JSON-formatted test report at the specified location."},
Expand Down

0 comments on commit f010b65

Please sign in to comment.