Skip to content

Commit

Permalink
internal/civisibility: add support for unskippable tests and suites
Browse files Browse the repository at this point in the history
  • Loading branch information
tonyredondo committed Oct 31, 2024
1 parent c16be07 commit 546cfc0
Show file tree
Hide file tree
Showing 6 changed files with 82 additions and 21 deletions.
6 changes: 6 additions & 0 deletions internal/civisibility/constants/test_tags.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,12 @@ const (

// CodeCoverageEnabled indicates that code coverage is enabled
CodeCoverageEnabled = "test.code_coverage.enabled"

// TestUnskippable indicates that the test is unskippable
TestUnskippable = "test.itr.unskippable"

// TestForcedToRun indicates that the test is forced to run because is unskippable
TestForcedToRun = "test.itr.forced_run"
)

// Define valid test status types.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
"github.com/stretchr/testify/assert"
)

//dd:suite.unskippable

// TestGetFieldPointerFrom tests the getFieldPointerFrom function.
func TestGetFieldPointerFrom(t *testing.T) {
// Create a mock struct with a private field
Expand Down
35 changes: 22 additions & 13 deletions internal/civisibility/integrations/gotesting/testcontroller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,10 @@ func runIntelligentTestRunnerTests(m *testing.M) {
Suite: "testing_test.go",
Name: "TestRetryAlwaysFail",
},
{
Suite: "testing_test.go",
Name: "TestNormalPassingAfterRetryAlwaysFail",
},
})
defer server.Close()

Expand Down Expand Up @@ -373,29 +377,34 @@ func runIntelligentTestRunnerTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
itrTest01 := checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 1)
itrTest02 := checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestGetFieldPointerFrom", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestGetInternalTestArray", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestGetInternalBenchmarkArray", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestCommonPrivateFields_AddLevel", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestGetBenchmarkPrivateFields", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02/sub01", 0)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02/sub01/sub03", 0)
itrTest03 := checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/yellow_should_return_color", 0)
checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/banana_should_return_fruit", 0)
checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/duck_should_return_animal", 0)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestSkip", 1)
itrTest04 := checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithPanic", 1)
itrTest05 := checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithPanic", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 1)

// check ITR spans
var itrTests []mocktracer.Span
itrTests = append(itrTests, itrTest01...)
itrTests = append(itrTests, itrTest02...)
itrTests = append(itrTests, itrTest03...)
itrTests = append(itrTests, itrTest04...)
itrTests = append(itrTests, itrTest05...)
checkSpansByTagValue(itrTests, constants.TestStatus, constants.TestStatusSkip, 5)
checkSpansByTagValue(itrTests, constants.TestSkipReason, constants.SkippedByITRReason, 5)
// 5 tests skipped by ITR and 1 normal skipped test
checkSpansByTagValue(finishedSpans, constants.TestStatus, constants.TestStatusSkip, 6)
checkSpansByTagValue(finishedSpans, constants.TestSkipReason, constants.SkippedByITRReason, 5)

// check unskippable tests
// 5 tests from unskippable suite in reflections_test.go and 2 unskippable tests from testing_test.go
checkSpansByTagValue(finishedSpans, constants.TestUnskippable, "true", 7)
checkSpansByTagValue(finishedSpans, constants.TestForcedToRun, "true", 1)

// check spans by type
checkSpansByType(finishedSpans,
Expand Down
19 changes: 12 additions & 7 deletions internal/civisibility/integrations/gotesting/testing.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,13 +217,18 @@ func (ddm *M) executeInternalTest(testInfo *testingTInfo) func(*testing.T) {

// Check if the test needs to be skipped by ITR
if testSkippedByITR {
test.SetTag(constants.TestSkippedByITR, "true")
test.CloseWithFinishTimeAndSkipReason(integrations.ResultStatusSkip, time.Now(), constants.SkippedByITRReason)
session.SetTag(constants.ITRTestsSkipped, "true")
session.SetTag(constants.ITRTestsSkippingCount, numOfTestsSkipped.Add(1))
checkModuleAndSuite(module, suite)
t.Skip(constants.SkippedByITRReason)
return
// check if the test was marked as unskippable
if test.Context().Value(constants.TestUnskippable) != true {
test.SetTag(constants.TestSkippedByITR, "true")
test.CloseWithFinishTimeAndSkipReason(integrations.ResultStatusSkip, time.Now(), constants.SkippedByITRReason)
session.SetTag(constants.ITRTestsSkipped, "true")
session.SetTag(constants.ITRTestsSkippingCount, numOfTestsSkipped.Add(1))
checkModuleAndSuite(module, suite)
t.Skip(constants.SkippedByITRReason)
return
} else {
test.SetTag(constants.TestForcedToRun, "true")
}
}

// Check if the coverage is enabled
Expand Down
2 changes: 2 additions & 0 deletions internal/civisibility/integrations/gotesting/testing_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,12 @@ func TestRetryWithFail(t *testing.T) {
}
}

//dd:test.unskippable
func TestNormalPassingAfterRetryAlwaysFail(t *testing.T) {}

var run int

//dd:test.unskippable
func TestEarlyFlakeDetection(t *testing.T) {
run++
fmt.Printf(" Run: %d", run)
Expand Down
39 changes: 38 additions & 1 deletion internal/civisibility/integrations/manual_api_ddtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,35 @@ func (t *tslvTest) SetTestFunc(fn *runtime.Func) {
// parse the entire file where the function is defined to create an abstract syntax tree (AST)
// if we can't parse the file (source code is not available) we silently bail out
fset := token.NewFileSet()
fileNode, err := parser.ParseFile(fset, absolutePath, nil, parser.AllErrors)
fileNode, err := parser.ParseFile(fset, absolutePath, nil, parser.AllErrors|parser.ParseComments)
if err == nil {

// let's check if the suite was marked as unskippable before
isUnskippable, hasUnskippableValue := t.suite.ctx.Value(constants.TestUnskippable).(bool)
if !hasUnskippableValue {
// check for suite level unskippable comment at the top of the file
for _, commentGroup := range fileNode.Comments {
for _, comment := range commentGroup.List {
if strings.Contains(comment.Text, "//dd:suite.unskippable") {
isUnskippable = true
break
}
}
if isUnskippable {
break
}
}
t.suite.ctx = context.WithValue(t.suite.ctx, constants.TestUnskippable, isUnskippable)
}

// get the function name without the package name
fullName := fn.Name()
firstDot := strings.LastIndex(fullName, ".") + 1
name := fullName[firstDot:]

// variable to store the ending line of the function
var endLine int

// traverse the AST to find the function declaration for the target function
ast.Inspect(fileNode, func(n ast.Node) bool {
// check if the current node is a function declaration
Expand All @@ -172,6 +192,17 @@ func (t *tslvTest) SetTestFunc(fn *runtime.Func) {
if funcDecl.Name.Name == name {
// get the line number of the end of the function body
endLine = fset.Position(funcDecl.Body.End()).Line
// check for comments above the function declaration to look for unskippable tag
// but only if we haven't found a suite level unskippable comment
if !isUnskippable && funcDecl.Doc != nil {
for _, comment := range funcDecl.Doc.List {
if strings.Contains(comment.Text, "//dd:test.unskippable") {
isUnskippable = true
break
}
}
}

// stop further inspection since we have found the target function
return false
}
Expand All @@ -194,6 +225,12 @@ func (t *tslvTest) SetTestFunc(fn *runtime.Func) {
if endLine >= startLine {
t.SetTag(constants.TestSourceEndLine, endLine)
}

// if the function is marked as unskippable, set the appropriate tag
if isUnskippable {
t.SetTag(constants.TestUnskippable, "true")
t.ctx = context.WithValue(t.ctx, constants.TestUnskippable, true)
}
}

// get the codeowner of the function
Expand Down

0 comments on commit 546cfc0

Please sign in to comment.