diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 93b3aa9..181f1f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,7 +45,7 @@ jobs: - name: Run examples run: make examples - name: Archive code coverage results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: allure-results path: ./examples/allure-results \ No newline at end of file diff --git a/README.MD b/README.MD index d378727..f25c33d 100644 --- a/README.MD +++ b/README.MD @@ -209,7 +209,7 @@ func (i *ExampleSuite) BeforeAll(t provider.T) { // Preparing host host, err := url.Parse("https://jsonplaceholder.typicode.com/") if err != nil { - t.Fatalf("could not parse url, error %v", err) + t.Fatalf("could not parse url, error %w", err) } i.host = host diff --git a/assert.go b/assert.go index bc6db8b..8161215 100644 --- a/assert.go +++ b/assert.go @@ -53,7 +53,7 @@ func (it *Test) assertHeaders(t internalT, headers http.Header) []error { return nil } - return executeWithStep(t, "Assert headers", func(t T) []error { + return it.executeWithStep(t, "Assert headers", func(t T) []error { errs := make([]error, 0) // Execute assert only response for _, f := range asserts { @@ -85,7 +85,7 @@ func (it *Test) assertResponse(t internalT, resp *http.Response) []error { return nil } - return executeWithStep(t, "Assert response", func(t T) []error { + return it.executeWithStep(t, "Assert response", func(t T) []error { errs := make([]error, 0) // Execute assert only response for _, f := range asserts { @@ -117,7 +117,7 @@ func (it *Test) assertBody(t internalT, body []byte) []error { return nil } - return executeWithStep(t, "Assert body", func(t T) []error { + return it.executeWithStep(t, "Assert body", func(t T) []error { errs := make([]error, 0) // Execute assert only response for _, f := range asserts { diff --git a/builder.go b/builder.go index a90b4b8..1d0a0fc 100644 --- a/builder.go +++ b/builder.go @@ -103,7 +103,7 @@ func createDefaultTest(m *HTTPTestMaker) *Test { Middleware: createMiddlewareFromTemplate(m.middleware), AllureStep: new(AllureStep), Request: &Request{ - Repeat: new(RequestRepeatPolitic), + Retry: new(RequestRetryPolitic), }, Expect: &Expect{JSONSchema: new(ExpectJSONSchema)}, } diff --git a/builder_request.go b/builder_request.go index 2e89046..67657f4 100644 --- a/builder_request.go +++ b/builder_request.go @@ -8,8 +8,9 @@ import ( // RequestRepeat is a function for set options in request // if response.Code != Expect.Code, than request will repeat Count counts with Delay delay. // Default delay is 1 second. +// Deprecated: use RequestRetry instead func (qt *cute) RequestRepeat(count int) RequestHTTPBuilder { - qt.tests[qt.countTests].Request.Repeat.Count = count + qt.tests[qt.countTests].Request.Retry.Count = count return qt } @@ -17,8 +18,9 @@ func (qt *cute) RequestRepeat(count int) RequestHTTPBuilder { // RequestRepeatDelay set delay for request repeat. // if response.Code != Expect.Code, than request will repeat Count counts with Delay delay. // Default delay is 1 second. +// Deprecated: use RequestRetryDelay instead func (qt *cute) RequestRepeatDelay(delay time.Duration) RequestHTTPBuilder { - qt.tests[qt.countTests].Request.Repeat.Delay = delay + qt.tests[qt.countTests].Request.Retry.Delay = delay return qt } @@ -27,28 +29,84 @@ func (qt *cute) RequestRepeatDelay(delay time.Duration) RequestHTTPBuilder { // if response.Code != Expect.Code, than request will repeat Count counts with Delay delay. // if Optional is true and request is failed, than test step allure will be skipped, and t.Fail() will not execute. // If Broken is true and request is failed, than test step allure will be broken, and t.Fail() will not execute. +// Deprecated: use RequestRetryPolitic instead func (qt *cute) RequestRepeatPolitic(politic *RequestRepeatPolitic) RequestHTTPBuilder { if politic == nil { - panic("politic is nil in RequestRepeatPolitic") + panic("politic is nil in RequestRetryPolitic") } - qt.tests[qt.countTests].Request.Repeat = politic + qt.tests[qt.countTests].Request.Retry = &RequestRetryPolitic{ + Count: politic.Count, + Delay: politic.Delay, + Optional: politic.Optional, + Broken: politic.Broken, + } return qt } // RequestRepeatOptional set option politic for request repeat. // if Optional is true and request is failed, than test step allure will be skipped, and t.Fail() will not execute. +// Deprecated: use RequestRetryOptional instead func (qt *cute) RequestRepeatOptional(option bool) RequestHTTPBuilder { - qt.tests[qt.countTests].Request.Repeat.Optional = option + qt.tests[qt.countTests].Request.Retry.Optional = option return qt } // RequestRepeatBroken set broken politic for request repeat. // If Broken is true and request is failed, than test step allure will be broken, and t.Fail() will not execute. +// Deprecated: use RequestRetryBroken instead func (qt *cute) RequestRepeatBroken(broken bool) RequestHTTPBuilder { - qt.tests[qt.countTests].Request.Repeat.Broken = broken + qt.tests[qt.countTests].Request.Retry.Broken = broken + + return qt +} + +// RequestRetry is a function for set options in request +// if response.Code != Expect.Code, than request will repeat Count counts with Delay delay. +// Default delay is 1 second. +func (qt *cute) RequestRetry(count int) RequestHTTPBuilder { + qt.tests[qt.countTests].Request.Retry.Count = count + + return qt +} + +// RequestRetryDelay set delay for request repeat. +// if response.Code != Expect.Code, than request will repeat Count counts with Delay delay. +// Default delay is 1 second. +func (qt *cute) RequestRetryDelay(delay time.Duration) RequestHTTPBuilder { + qt.tests[qt.countTests].Request.Retry.Delay = delay + + return qt +} + +// RequestRetryPolitic set politic for request repeat. +// if response.Code != Expect.Code, than request will repeat Count counts with Delay delay. +// if Optional is true and request is failed, than test step allure will be skipped, and t.Fail() will not execute. +// If Broken is true and request is failed, than test step allure will be broken, and t.Fail() will not execute. +func (qt *cute) RequestRetryPolitic(politic *RequestRetryPolitic) RequestHTTPBuilder { + if politic == nil { + panic("politic is nil in RequestRetryPolitic") + } + + qt.tests[qt.countTests].Request.Retry = politic + + return qt +} + +// RequestRetryOptional set option politic for request repeat. +// if Optional is true and request is failed, than test step allure will be skipped, and t.Fail() will not execute. +func (qt *cute) RequestRetryOptional(option bool) RequestHTTPBuilder { + qt.tests[qt.countTests].Request.Retry.Optional = option + + return qt +} + +// RequestRetryBroken set broken politic for request repeat. +// If Broken is true and request is failed, than test step allure will be broken, and t.Fail() will not execute. +func (qt *cute) RequestRetryBroken(broken bool) RequestHTTPBuilder { + qt.tests[qt.countTests].Request.Retry.Broken = broken return qt } diff --git a/builder_retry.go b/builder_retry.go new file mode 100644 index 0000000..1865672 --- /dev/null +++ b/builder_retry.go @@ -0,0 +1,29 @@ +package cute + +import "time" + +// Retry is a function for configure test repeat +// if response.Code != Expect.Code or any of asserts are failed/broken than test will repeat counts with delay. +// Default delay is 1 second. +func (qt *cute) Retry(count int) MiddlewareRequest { + if count < 1 { + panic("count must be greater than 0") + } + + qt.tests[qt.countTests].Retry.MaxAttempts = count + + return qt +} + +// RetryDelay set delay for test repeat. +// if response.Code != Expect.Code or any of asserts are failed/broken than test will repeat counts with delay. +// Default delay is 1 second. +func (qt *cute) RetryDelay(delay time.Duration) MiddlewareRequest { + if delay < 0 { + panic("delay must be greater than or equal to 0") + } + + qt.tests[qt.countTests].Retry.Delay = delay + + return qt +} diff --git a/builder_test.go b/builder_test.go index eb57af7..0ee4ccf 100644 --- a/builder_test.go +++ b/builder_test.go @@ -285,8 +285,8 @@ func TestHTTPTestMaker(t *testing.T) { Link(link). Description(desc). CreateStep(stepName). - RequestRepeat(repeatCount). - RequestRepeatDelay(repeatDelay). + RequestRetry(repeatCount). + RequestRetryDelay(repeatDelay). Request(req). ExpectExecuteTimeout(executeTime). ExpectStatus(status). @@ -330,8 +330,8 @@ func TestHTTPTestMaker(t *testing.T) { require.Equal(t, setIssue, resHt.allureLinks.issue) require.Equal(t, setTestCase, resHt.allureLinks.testCase) require.Equal(t, link, resHt.allureLinks.link) - require.Equal(t, repeatCount, resTest.Request.Repeat.Count) - require.Equal(t, repeatDelay, resTest.Request.Repeat.Delay) + require.Equal(t, repeatCount, resTest.Request.Retry.Count) + require.Equal(t, repeatDelay, resTest.Request.Retry.Delay) require.Equal(t, len(assertHeaders), len(resTest.Expect.AssertHeaders)) require.Equal(t, len(assertHeadersT), len(resTest.Expect.AssertHeadersT)) @@ -360,7 +360,7 @@ func TestCreateDefaultTest(t *testing.T) { BeforeT: make([]BeforeExecuteT, 0), }, Request: &Request{ - Repeat: new(RequestRepeatPolitic), + Retry: new(RequestRetryPolitic), }, Expect: &Expect{ JSONSchema: new(ExpectJSONSchema), diff --git a/cute.go b/cute.go index 62fbc25..fa0a3b1 100644 --- a/cute.go +++ b/cute.go @@ -113,27 +113,6 @@ func createAllureT(t *testing.T) *common.Common { return newT } -// executeTestsInsideStep is method for run group of tests inside provider.StepCtx -func (qt *cute) executeTestsInsideStep(ctx context.Context, stepCtx provider.StepCtx) []ResultsHTTPBuilder { - var ( - res = make([]ResultsHTTPBuilder, 0) - ) - - // Cycle for change number of Test - for i := 0; i <= qt.countTests; i++ { - currentTest := qt.tests[i] - - result := currentTest.executeInsideStep(ctx, stepCtx) - - // Remove from base struct all asserts - currentTest.clearFields() - - res = append(res, result) - } - - return res -} - // executeTests is method for run tests // It's could be table tests or usual tests func (qt *cute) executeTests(ctx context.Context, allureProvider allureProvider) []ResultsHTTPBuilder { @@ -153,7 +132,7 @@ func (qt *cute) executeTests(ctx context.Context, allureProvider allureProvider) // Set current test name inT.Title(tableTestName) - res = append(res, qt.executeSingleTest(ctx, inT, currentTest)) + res = append(res, qt.executeInsideAllure(ctx, inT, currentTest)) }) } else { currentTest.Name = allureProvider.Name() @@ -161,14 +140,16 @@ func (qt *cute) executeTests(ctx context.Context, allureProvider allureProvider) // set labels qt.setAllureInformation(allureProvider) - res = append(res, qt.executeSingleTest(ctx, allureProvider, currentTest)) + res = append(res, qt.executeInsideAllure(ctx, allureProvider, currentTest)) } } return res } -func (qt *cute) executeSingleTest(ctx context.Context, allureProvider allureProvider, currentTest *Test) ResultsHTTPBuilder { +// executeInsideAllure is method for run test inside allure +// It's could be table tests or usual tests +func (qt *cute) executeInsideAllure(ctx context.Context, allureProvider allureProvider, currentTest *Test) ResultsHTTPBuilder { resT := currentTest.executeInsideAllure(ctx, allureProvider) // Remove from base struct all asserts @@ -176,3 +157,24 @@ func (qt *cute) executeSingleTest(ctx context.Context, allureProvider allureProv return resT } + +// executeTestsInsideStep is method for run group of tests inside provider.StepCtx +func (qt *cute) executeTestsInsideStep(ctx context.Context, stepCtx provider.StepCtx) []ResultsHTTPBuilder { + var ( + res = make([]ResultsHTTPBuilder, 0) + ) + + // Cycle for change number of Test + for i := 0; i <= qt.countTests; i++ { + currentTest := qt.tests[i] + + result := currentTest.executeInsideStep(ctx, stepCtx) + + // Remove from base struct all asserts + currentTest.clearFields() + + res = append(res, result) + } + + return res +} diff --git a/examples/single_test.go b/examples/single_test.go index 7b623e4..c67bd05 100644 --- a/examples/single_test.go +++ b/examples/single_test.go @@ -30,7 +30,7 @@ func Test_Single_1(t *testing.T) { Description("some_description"). Parallel(). Create(). - RequestRepeat(3). + RequestRetry(3). RequestBuilder( cute.WithURI("https://jsonplaceholder.typicode.com/posts/1/comments"), cute.WithMarshalBody(struct { @@ -96,14 +96,16 @@ func Test_Single_Broken(t *testing.T) { }, ). ExecuteTest(context.Background(), t) + + t.Skip() } func Test_Single_RepeatPolitic_Optional_Success_Test(t *testing.T) { cute.NewTestBuilder(). Title("Test_Single_RepeatPolitic_Optional_Success_Test"). Create(). - RequestRepeat(2). - RequestRepeatOptional(true). + RequestRetry(2). + RequestRetryOptional(true). RequestBuilder( cute.WithURI("https://jsonplaceholder.typicode.com/posts/1/comments"), ). @@ -120,8 +122,8 @@ func Test_Single_RepeatPolitic_Broken_Failed_Test(t *testing.T) { cute.NewTestBuilder(). Title("Test_Single_RepeatPolitic_Broken_Failed_Test"). Create(). - RequestRepeat(2). - RequestRepeatOptional(true). + RequestRetry(2). + RequestRetryOptional(false). RequestBuilder( cute.WithURI("https://jsonplaceholder.typicode.com/posts/1/comments"), ). @@ -173,8 +175,8 @@ func Test_Single_2_AllureRunner(t *testing.T) { Tag("single_test"). Description("some_description"). Create(). - RequestRepeatDelay(3*time.Second). // delay before new try - RequestRepeat(3). // count attempts + RequestRetryDelay(3*time.Second). // delay before new try + RequestRetry(3). // count attempts RequestBuilder( cute.WithURL(u), cute.WithMethod(http.MethodGet), diff --git a/examples/suite/one_step.go b/examples/suite/one_step.go index b241a8c..86bbb47 100644 --- a/examples/suite/one_step.go +++ b/examples/suite/one_step.go @@ -24,43 +24,44 @@ import ( Response: [ - { - "postId": 1, - "id": 1, - "name": "id labore ex et quam laborum", - "email": "Eliseo@gardner.biz", - "body": "laudantium enim quasi est quidem magnam voluptate ipsam eos\ntempora quo necessitatibus\ndolor quam autem quasi\nreiciendis et nam sapiente accusantium" - }, - { - "postId": 1, - "id": 2, - "name": "quo vero reiciendis velit similique earum", - "email": "Jayne_Kuhic@sydney.com", - "body": "est natus enim nihil est dolore omnis voluptatem numquam\net omnis occaecati quod ullam at\nvoluptatem error expedita pariatur\nnihil sint nostrum voluptatem reiciendis et" - }, - { - "postId": 1, - "id": 3, - "name": "odio adipisci rerum aut animi", - "email": "Nikita@garfield.biz", - "body": "quia molestiae reprehenderit quasi aspernatur\naut expedita occaecati aliquam eveniet laudantium\nomnis quibusdam delectus saepe quia accusamus maiores nam est\ncum et ducimus et vero voluptates excepturi deleniti ratione" - }, - { - "postId": 1, - "id": 4, - "name": "alias odio sit", - "email": "Lew@alysha.tv", - "body": "non et atque\noccaecati deserunt quas accusantium unde odit nobis qui voluptatem\nquia voluptas consequuntur itaque dolor\net qui rerum deleniti ut occaecati" - }, - { - "postId": 1, - "id": 5, - "name": "vero eaque aliquid doloribus et culpa", - "email": "Hayden@althea.biz", - "body": "harum non quasi et ratione\ntempore iure ex voluptates in ratione\nharum architecto fugit inventore cupiditate\nvoluptates magni quo et" - } -] + { + "postId": 1, + "id": 1, + "name": "id labore ex et quam laborum", + "email": "Eliseo@gardner.biz", + "body": "laudantium enim quasi est quidem magnam voluptate ipsam eos\ntempora quo necessitatibus\ndolor quam autem quasi\nreiciendis et nam sapiente accusantium" + }, + { + "postId": 1, + "id": 2, + "name": "quo vero reiciendis velit similique earum", + "email": "Jayne_Kuhic@sydney.com", + "body": "est natus enim nihil est dolore omnis voluptatem numquam\net omnis occaecati quod ullam at\nvoluptatem error expedita pariatur\nnihil sint nostrum voluptatem reiciendis et" + }, + { + "postId": 1, + "id": 3, + "name": "odio adipisci rerum aut animi", + "email": "Nikita@garfield.biz", + "body": "quia molestiae reprehenderit quasi aspernatur\naut expedita occaecati aliquam eveniet laudantium\nomnis quibusdam delectus saepe quia accusamus maiores nam est\ncum et ducimus et vero voluptates excepturi deleniti ratione" + }, + { + "postId": 1, + "id": 4, + "name": "alias odio sit", + "email": "Lew@alysha.tv", + "body": "non et atque\noccaecati deserunt quas accusantium unde odit nobis qui voluptatem\nquia voluptas consequuntur itaque dolor\net qui rerum deleniti ut occaecati" + }, + { + "postId": 1, + "id": 5, + "name": "vero eaque aliquid doloribus et culpa", + "email": "Hayden@althea.biz", + "body": "harum non quasi et ratione\ntempore iure ex voluptates in ratione\nharum architecto fugit inventore cupiditate\nvoluptates magni quo et" + } + +] */ func (i *ExampleSuite) Test_OneStep(t provider.T) { var ( diff --git a/examples/table_test/table_test.go b/examples/table_test/table_test.go index 158a0e1..9f61bea 100644 --- a/examples/table_test/table_test.go +++ b/examples/table_test/table_test.go @@ -9,7 +9,9 @@ import ( "net/http" "net/url" "os" + "strconv" "testing" + "time" "github.com/ozontech/cute" "github.com/ozontech/cute/asserts/json" @@ -111,6 +113,133 @@ func Test_One_Execute(t *testing.T) { test.Execute(context.Background(), t) } +func Test_Array_Retry_OptionalFirstTries(t *testing.T) { + tests := []*cute.Test{ + { + Name: "test_1", + + Retry: &cute.Retry{ + MaxAttempts: 10, + Delay: 1 * time.Second, + }, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/Random/201,202"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 201, + }, + }, + { + Name: "test_2", + Retry: &cute.Retry{ + MaxAttempts: 10, + Delay: 1 * time.Second, + }, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/Random/403,404"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 404, + }, + }, + } + + for _, test := range tests { + test.Execute(context.Background(), t) + } +} + +func Test_Array_Retry_OptionalFirstTries_UltimatelyFailing(t *testing.T) { + tests := []*cute.Test{ + { + Name: "test_1", + + Retry: &cute.Retry{ + MaxAttempts: 4, + Delay: 1 * time.Second, + }, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/Random/202,200"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 201, + }, + }, + { + Name: "test_2", + Retry: &cute.Retry{ + MaxAttempts: 3, + Delay: 1 * time.Second, + }, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/Random/403,401"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 404, + }, + }, + } + + for _, test := range tests { + test.Execute(context.Background(), t) + } +} + +func Test_Array_TimeoutRetry(t *testing.T) { + var executeTimeout = 3000 + + tests := []*cute.Test{ + { + Retry: &cute.Retry{ + MaxAttempts: 2, + }, + Name: "test_timeout", + Middleware: &cute.Middleware{ + Before: []cute.BeforeExecute{ + cute.BeforeExecute(func(request *http.Request) error { + query := request.URL.Query() + query.Set("sleep", strconv.Itoa(executeTimeout)) + request.URL.RawQuery = query.Encode() + executeTimeout = executeTimeout - 1000 + return nil + }), + }, + }, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/202?sleep=3000"), + cute.WithBody([]byte("{\"test\":\"abc\"}")), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 202, + ExecuteTime: 3 * time.Second, + }, + }, + } + + for _, test := range tests { + test.Execute(context.Background(), t) + } +} + func Test_Array(t *testing.T) { tests := []*cute.Test{ { @@ -123,7 +252,7 @@ func Test_Array(t *testing.T) { }, }, Expect: &cute.Expect{ - Code: 200, + Code: 201, }, }, { @@ -152,3 +281,205 @@ func Test_Array(t *testing.T) { test.Execute(context.Background(), t) } } + +func Test_Array_All_Parallel(t *testing.T) { + tests := []*cute.Test{ + { + Name: "test_201", + Parallel: true, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/201"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 201, + }, + }, + { + Name: "test_200_delay_5s", + Parallel: true, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/200?sleep=5000"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 200, + }, + }, + { + Name: "test_202_delay_3s", + Parallel: true, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/202?sleep=3000"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 202, + }, + }, + { + Name: "test_203", + Parallel: true, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/203"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 203, + }, + }, + } + + for _, test := range tests { + test.Execute(context.Background(), t) + } +} + +func Test_Array_Some_Parallel(t *testing.T) { + tests := []*cute.Test{ + { + Name: "test_parallel_1", + Parallel: true, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/201?sleep=1000"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 201, + }, + }, + { + Name: "test_parallel_2", + Parallel: true, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/202?sleep=1000"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 202, + }, + }, + { + Name: "test_1_sequential", + Parallel: false, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://jsonplaceholder.typicode.com/posts/1/comments"), + cute.WithMethod(http.MethodPost), + }, + }, + Expect: &cute.Expect{ + Code: 201, + }, + }, + { + Name: "test_2_sequential", + Parallel: false, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://jsonplaceholder.typicode.com/posts/1/comments"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 200, + AssertBody: []cute.AssertBody{ + json.Equal("$[0].email", "Eliseo@gardner.biz"), + json.Present("$[1].name"), + }, + }, + }, + } + + for _, test := range tests { + test.Execute(context.Background(), t) + } +} + +func Test_Array_Retry(t *testing.T) { + tests := []*cute.Test{ + { + Name: "test_1", + Parallel: true, + Retry: &cute.Retry{ + MaxAttempts: 10, + Delay: 1, + }, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/Random/201,202"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 201, + }, + }, + { + Name: "test_2", + Parallel: true, + Retry: &cute.Retry{ + MaxAttempts: 10, + Delay: 1, + }, + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/Random/403,404"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 404, + }, + }, + } + + for _, test := range tests { + test.Execute(context.Background(), t) + } +} + +func Test_Array_Timeout(t *testing.T) { + tests := []*cute.Test{ + { + Name: "test_timeout", + Middleware: nil, + Request: &cute.Request{ + Builders: []cute.RequestBuilder{ + cute.WithURI("https://httpstat.us/202?sleep=3000"), + cute.WithMethod(http.MethodGet), + }, + }, + Expect: &cute.Expect{ + Code: 202, + ExecuteTime: 2 * time.Second, + }, + }, + } + + for _, test := range tests { + test.Execute(context.Background(), t) + } +} diff --git a/examples/two_step_test.go b/examples/two_step_test.go index dbb4daf..8838fea 100644 --- a/examples/two_step_test.go +++ b/examples/two_step_test.go @@ -21,7 +21,7 @@ func Test_TwoSteps_1(t *testing.T) { Title("Test with two requests."). Tags("two_steps"). Parallel(). - CreateStep("Creat entry /posts/1"). + CreateStep("Create entry /posts/1"). // CreateWithStep first step diff --git a/interface.go b/interface.go index 4af22d4..5cac53a 100644 --- a/interface.go +++ b/interface.go @@ -87,11 +87,25 @@ type MiddlewareTable interface { // MiddlewareRequest is function for create requests or add After/Before functions type MiddlewareRequest interface { RequestHTTPBuilder + RetryPolitic BeforeTest AfterTest } +// RetryPolitic is a scope of methods to configure test repeat +type RetryPolitic interface { + // Retry is a function for configure test repeat + // if response.Code != Expect.Code or any of asserts are failed/broken than test will repeat counts with delay. + // Default delay is 1 second. + Retry(count int) MiddlewareRequest + + // RetryDelay set delay for test repeat. + // if response.Code != Expect.Code or any of asserts are failed/broken than test will repeat counts with delay. + // Default delay is 1 second. + RetryDelay(timeout time.Duration) MiddlewareRequest +} + // BeforeTest are functions for processing request before test execution // Same functions: // Before @@ -174,26 +188,36 @@ type RequestParams interface { // RequestRepeat is a function for set options in request // if response.Code != Expect.Code, than request will repeat counts with delay. // Default delay is 1 second. + // Deprecated: use RequestRetry instead RequestRepeat(count int) RequestHTTPBuilder + RequestRetry(count int) RequestHTTPBuilder // RequestRepeatDelay set delay for request repeat. // if response.Code != Expect.Code, than request will repeat counts with delay. // Default delay is 1 second. + // Deprecated: use RequestRetryDelay instead RequestRepeatDelay(delay time.Duration) RequestHTTPBuilder + RequestRetryDelay(delay time.Duration) RequestHTTPBuilder // RequestRepeatPolitic is a politic for repeat request. // if response.Code != Expect.Code, than request will repeat counts with delay. // if Optional is true and request is failed, than test step allure will be skipped, and t.Fail() will not execute. // If Broken is true and request is failed, than test step allure will be broken, and t.Fail() will execute. + // Deprecated: use RequestRetryPolitic instead RequestRepeatPolitic(politic *RequestRepeatPolitic) RequestHTTPBuilder + RequestRetryPolitic(politic *RequestRetryPolitic) RequestHTTPBuilder // RequestRepeatOptional is a option politic for repeat request. // if Optional is true and request is failed, than test step allure will be skipped, and t.Fail() will not execute. + // Deprecated: use RequestRetryOptional instead RequestRepeatOptional(optional bool) RequestHTTPBuilder + RequestRetryOptional(optional bool) RequestHTTPBuilder // RequestRepeatBroken is a broken politic for repeat request. // If Broken is true and request is failed, than test step allure will be broken, and t.Fail() will execute. + // Deprecated: use RequestRetryBroken instead RequestRepeatBroken(broken bool) RequestHTTPBuilder + RequestRetryBroken(broken bool) RequestHTTPBuilder } // ExpectHTTPBuilder is a scope of methods for validate http response diff --git a/jsonschema.go b/jsonschema.go index cb27619..d9a0453 100644 --- a/jsonschema.go +++ b/jsonschema.go @@ -25,7 +25,7 @@ func (it *Test) validateJSONSchema(t internalT, body []byte) []error { return nil } - return executeWithStep(t, "Validate body by JSON schema", func(t T) []error { + return it.executeWithStep(t, "Validate body by JSON schema", func(_ T) []error { return checkJSONSchema(expect, body) }) } diff --git a/jsonschema_test.go b/jsonschema_test.go index 79a7a1a..5e51407 100644 --- a/jsonschema_test.go +++ b/jsonschema_test.go @@ -12,6 +12,8 @@ func TestValidateJSONSchemaEmptySchema(t *testing.T) { tBuilder = createDefaultTest(&HTTPTestMaker{middleware: new(Middleware)}) ) + tBuilder.initEmptyFields() + errs := tBuilder.validateJSONSchema(nil, []byte{}) require.Len(t, errs, 0) } @@ -22,6 +24,8 @@ func TestValidateJSONSchemaFromString(t *testing.T) { tempT = createAllureT(t) ) + tBuilder.initEmptyFields() + body := []byte(` { "firstName": "Boris", @@ -61,6 +65,8 @@ func TestValidateJSONSchemaFromStringWithError(t *testing.T) { tempT = createAllureT(t) ) + tBuilder.initEmptyFields() + body := []byte(` { "firstName": "Boris", @@ -109,6 +115,8 @@ func TestValidateJSONSchemaFromByteWithTwoError(t *testing.T) { tempT = createAllureT(t) ) + tBuilder.initEmptyFields() + body := []byte(` { "firstName": "Boris", diff --git a/logger.go b/logger.go index 5c0e1ab..da1bcf7 100644 --- a/logger.go +++ b/logger.go @@ -31,8 +31,12 @@ func (it *Test) logf(t tlogger, level, format string, args ...interface{}) { if it.Name == "" { name = t.Name() } - - t.Logf("[%s][%s] %v\n", name, level, fmt.Sprintf(format, args...)) + // If we are in a retry context, add some indication in the logs about the current attempt + if it.Retry.MaxAttempts != 1 { + t.Logf("[%s][%s](Attempt #%d) %v\n", name, level, it.Retry.currentCount, fmt.Sprintf(format, args...)) + } else { + t.Logf("[%s][%s] %v\n", name, level, fmt.Sprintf(format, args...)) + } } func (it *Test) errorf(t tlogger, level, format string, args ...interface{}) { @@ -41,6 +45,10 @@ func (it *Test) errorf(t tlogger, level, format string, args ...interface{}) { if it.Name == "" { name = t.Name() } - - t.Errorf("[%s][%s] %v\n", name, level, fmt.Sprintf(format, args...)) + // If we are in a retry context, add some indication in the logs about the current attempt + if it.Retry.MaxAttempts != 1 { + t.Logf("[%s][%s](Attempt #%d) %v\n", name, level, it.Retry.currentCount, fmt.Sprintf(format, args...)) + } else { + t.Logf("[%s][%s] %v\n", name, level, fmt.Sprintf(format, args...)) + } } diff --git a/provider.go b/provider.go index e2bc3f8..b4f9946 100644 --- a/provider.go +++ b/provider.go @@ -27,6 +27,9 @@ type allureProvider interface { } type internalT interface { + Broken() + BrokenNow() + tProvider logProvider stepProvider diff --git a/roundtripper.go b/roundtripper.go index 5a75fd4..bb971c6 100644 --- a/roundtripper.go +++ b/roundtripper.go @@ -2,6 +2,7 @@ package cute import ( "context" + "errors" "fmt" "io" "net/http" @@ -24,23 +25,23 @@ func (it *Test) makeRequest(t internalT, req *http.Request) (*http.Response, []e scope = make([]error, 0) ) - if it.Request.Repeat.Delay != 0 { - delay = it.Request.Repeat.Delay + if it.Request.Retry.Delay != 0 { + delay = it.Request.Retry.Delay } - if it.Request.Repeat.Count != 0 { - countRepeat = it.Request.Repeat.Count + if it.Request.Retry.Count != 0 { + countRepeat = it.Request.Retry.Count } for i := 1; i <= countRepeat; i++ { - executeWithStep(t, createTitle(i, countRepeat, req), func(t T) []error { + it.executeWithStep(t, createTitle(i, countRepeat, req), func(t T) []error { resp, err = it.doRequest(t, req) if err != nil { - if it.Request.Repeat.Broken { + if it.Request.Retry.Broken { err = wrapBrokenError(err) } - if it.Request.Repeat.Optional { + if it.Request.Retry.Optional { err = wrapOptionalError(err) } @@ -73,6 +74,20 @@ func (it *Test) doRequest(t T, baseReq *http.Request) (*http.Response, error) { resp, httpErr := it.httpClient.Do(req) + // if the timeout is triggered, we properly log the timeout error on allure and in traces + if errors.Is(httpErr, context.DeadlineExceeded) { + // Add information (method, host, curl) about request to Allure step + // should be after httpClient.Do and from resp.Request, because in roundTripper request may be changed + if addErr := it.addInformationRequest(t, req); addErr != nil { + // Ignore err return, because it's connected with test logic + it.Error(t, "Could not log information about request. error %v", addErr) + } + + return nil, cuteErrors.NewEmptyAssertError( + "Request timeout", + fmt.Sprintf("expected request to be completed in %v, but was not", it.Expect.ExecuteTime)) + } + // http client has case wheh it return response and error in one time // we have to check this case if resp == nil { @@ -87,14 +102,14 @@ func (it *Test) doRequest(t T, baseReq *http.Request) (*http.Response, error) { // BAD CODE. Need to copy body, because we can't read body again from resp.Request.Body. Problem is io.Reader resp.Request.Body, baseReq.Body, err = utils.DrainBody(baseReq.Body) if err != nil { - it.Error(t, "Could not drain body from baseReq.Body. Error %v", err) + it.Error(t, "Could not drain body from baseReq.Body. error %v", err) // Ignore err return, because it's connected with test logic } // Add information (method, host, curl) about request to Allure step // should be after httpClient.Do and from resp.Request, because in roundTripper request may be changed if addErr := it.addInformationRequest(t, resp.Request); addErr != nil { - it.Error(t, "[ERROR] Could not log information about request. Error %v", addErr) + it.Error(t, "Could not log information about request. error %v", addErr) // Ignore err return, because it's connected with test logic } @@ -105,11 +120,11 @@ func (it *Test) doRequest(t T, baseReq *http.Request) (*http.Response, error) { // Add information (code, body, headers) about response to Allure step if addErr := it.addInformationResponse(t, resp); addErr != nil { // Ignore err return, because it's connected with test logic - it.Error(t, "[ERROR] Could not log information about response. Error %v", addErr) + it.Error(t, "Could not log information about response. error %v", addErr) } if validErr := it.validateResponseCode(resp); validErr != nil { - return nil, validErr + return resp, validErr } return resp, nil diff --git a/step.go b/step.go index 263c208..c8ba0b7 100644 --- a/step.go +++ b/step.go @@ -1,16 +1,22 @@ package cute import ( + "fmt" + "github.com/ozontech/allure-go/pkg/allure" "github.com/ozontech/allure-go/pkg/framework/provider" "github.com/ozontech/cute/errors" ) -func executeWithStep(t internalT, stepName string, execute func(t T) []error) []error { +func (it *Test) executeWithStep(t internalT, stepName string, execute func(t T) []error) []error { var ( errs []error ) + // Add attempt indication in Allure if more than 1 attempt + if it.Retry.MaxAttempts != 1 { + stepName = fmt.Sprintf("[Attempt #%d] %v", it.Retry.currentCount, stepName) + } t.WithNewStep(stepName, func(stepCtx provider.StepCtx) { errs = execute(stepCtx) processStepErrors(stepCtx, errs) diff --git a/test.go b/test.go index 975329a..d29cb46 100644 --- a/test.go +++ b/test.go @@ -30,11 +30,14 @@ var ( // Test is a main struct of test. // You may field Request and Expect for create simple test +// Parallel can be used to control the parallelism of a Test type Test struct { httpClient *http.Client jsonMarshaler JSONMarshaler - Name string + Name string + Parallel bool + Retry *Retry AllureStep *AllureStep Middleware *Middleware @@ -42,19 +45,41 @@ type Test struct { Expect *Expect } +// Retry is a struct to control the retry of a whole single test (not only the request) +// The test will be retried up to MaxAttempts times +// The retries will only be executed if the test is having errors +// If the test is successful at any iteration between attempt 1 and MaxAttempts, the loop will break and return the result as successful +// The status of the test (success or fail) will be based on either the first attempt that is successful, or, if no attempt +// is successful, it will be based on the latest execution +// Delay is the number of seconds to wait before attempting to run the test again. It will only wait if Delay is set. +type Retry struct { + currentCount int + MaxAttempts int + Delay time.Duration +} + // Request is struct with HTTP request. // You may use your *http.Request or create new with help Builders type Request struct { Base *http.Request Builders []RequestBuilder - Repeat *RequestRepeatPolitic + Retry *RequestRetryPolitic } -// RequestRepeatPolitic is struct for repeat politic +// RequestRetryPolitic is struct for repeat politic // if Optional is true and request is failed, than test step allure will be skip, and t.Fail() will not execute. // If Broken is true and request is failed, than test step allure will be broken, and t.Fail() will not execute. // If Optional and Broken is false, than test step will be failed, and t.Fail() will execute. // If response.Code != Expect.Code, than request will repeat Count counts with Delay delay. +type RequestRetryPolitic struct { + Count int + Delay time.Duration + Optional bool + Broken bool +} + +// RequestRepeatPolitic is struct for repeat politic +// Deprecated: use RequestRetryPolitic type RequestRepeatPolitic struct { Count int Delay time.Duration @@ -126,6 +151,9 @@ func (it *Test) Execute(ctx context.Context, t tProvider) ResultsHTTPBuilder { } internalT.Run(it.Name, func(inT provider.T) { + if it.Parallel { + inT.Parallel() + } res = it.executeInsideAllure(ctx, inT) }) @@ -137,7 +165,7 @@ func (it *Test) clearFields() { it.Middleware = new(Middleware) it.Expect = new(Expect) it.Request = new(Request) - it.Request.Repeat = new(RequestRepeatPolitic) + it.Request.Retry = new(RequestRetryPolitic) it.Expect.JSONSchema = new(ExpectJSONSchema) } @@ -162,60 +190,127 @@ func (it *Test) initEmptyFields() { it.Request = new(Request) } - if it.Request.Repeat == nil { - it.Request.Repeat = new(RequestRepeatPolitic) + if it.Request.Retry == nil { + it.Request.Retry = new(RequestRetryPolitic) } if it.Expect.JSONSchema == nil { it.Expect.JSONSchema = new(ExpectJSONSchema) } + + if it.Retry == nil { + it.Retry = &Retry{ + // we set the default value to 1, because we count the first attempt as 1 + MaxAttempts: 1, + currentCount: 1, + } + } + + // We need to set the current count to 1 here, because we count the first attempt as 1 + it.Retry.currentCount = 1 } // executeInsideStep is method for start test with provider.StepCtx // It's test inside the step func (it *Test) executeInsideStep(ctx context.Context, t internalT) ResultsHTTPBuilder { - resp, errs := it.startTest(ctx, t) + // Set empty fields in test + it.initEmptyFields() - resultState := it.processTestErrors(t, errs) + // we don't want to defer the finish message, because it will be logged in processTestErrors + it.Info(t, "Start test") - return newTestResult(t.Name(), resp, resultState, errs) + return it.startRepeatableTest(ctx, t) } func (it *Test) executeInsideAllure(ctx context.Context, allureProvider allureProvider) ResultsHTTPBuilder { - var ( - resp *http.Response - errs []error - name = allureProvider.Name() + "_" + it.Name - ) - - it.Info(allureProvider, "Start test") - // Set empty fields in test it.initEmptyFields() - if it.AllureStep.Name != "" { - // Set name of test for results - name = it.AllureStep.Name + // we don't want to defer the finish message, because it will be logged in processTestErrors + it.Info(allureProvider, "Start test") + if it.AllureStep.Name != "" { // Execute test inside step - resp, errs = it.startTestWithStep(ctx, allureProvider) + return it.startTestInsideStep(ctx, allureProvider) } else { - // Execute Test - resp, errs = it.startTest(ctx, allureProvider) + return it.startRepeatableTest(ctx, allureProvider) + } +} + +// startRepeatableTest is method for start test with repeatable execution +func (it *Test) startRepeatableTest(ctx context.Context, t internalT) ResultsHTTPBuilder { + var ( + resp *http.Response + errs []error + resultState ResultState + ) + + for ; it.Retry.currentCount <= it.Retry.MaxAttempts; it.Retry.currentCount++ { + resp, errs = it.startTest(ctx, t) + + resultState = it.processTestErrors(t, errs) + + // we don't want to keep errors if we will retry test + // we have to return to user only errors from last try + + // if the test is successful, we break the loop + if resultState == ResultStateSuccess { + break + } + + // if we have a delay, we wait before the next attempt + // and we only wait if we are not at the last attempt + if it.Retry.currentCount != it.Retry.MaxAttempts && it.Retry.Delay != 0 { + it.Info(t, "The test had errors, retrying...") + time.Sleep(it.Retry.Delay) + } + } + + switch resultState { + case ResultStateBroken: + t.BrokenNow() + it.Info(t, "Test broken") + case ResultStateFail: + t.Fail() + it.Error(t, "Test failed") + case resultStateFailNow: + t.FailNow() + it.Error(t, "Test failed") + case ResultStateSuccess: + it.Info(t, "Test finished successfully") } - resultState := it.processTestErrors(allureProvider, errs) + return newTestResult(it.Name, resp, resultState, errs) +} + +func (it *Test) startTestInsideStep(ctx context.Context, t internalT) ResultsHTTPBuilder { + var ( + result ResultsHTTPBuilder + ) - return newTestResult(name, resp, resultState, errs) + t.WithNewStep(it.AllureStep.Name, func(stepCtx provider.StepCtx) { + it.Info(t, "Start step %v", it.AllureStep.Name) + defer it.Info(t, "Finish step %v", it.AllureStep.Name) + + result = it.startRepeatableTest(ctx, stepCtx) + + if result.GetResultState() == ResultStateFail { + stepCtx.Fail() + } + }) + + return result } // processTestErrors returns flag, which mean finish test or not. // If test has only optional errors, than test will be success -// If test has broken errors, than test will be broken on allure and executed t.FailNow(). -// If test has require errors, than test will be failed on allure and executed t.FailNow(). +// If test has broken errors, than test will be broken on allure +// If test has require errors, than test will be failed on allure func (it *Test) processTestErrors(t internalT, errs []error) ResultState { if len(errs) == 0 { + it.Info(t, "Test finished successfully") + return ResultStateSuccess } @@ -225,7 +320,7 @@ func (it *Test) processTestErrors(t internalT, errs []error) ResultState { ) for _, err := range errs { - message := fmt.Sprintf("Error %v", err.Error()) + message := fmt.Sprintf("error %v", err.Error()) if tErr, ok := err.(cuteErrors.OptionalError); ok { if tErr.IsOptional() { @@ -273,43 +368,9 @@ func (it *Test) processTestErrors(t internalT, errs []error) ResultState { it.Error(t, "Test finished with %v errors", countNotOptionalErrors) } - switch state { - case ResultStateBroken: - t.FailNow() - it.Info(t, "Test broken") - case ResultStateFail: - t.Fail() - it.Error(t, "Test failed") - case resultStateFailNow: - t.FailNow() - it.Error(t, "Test failed") - case ResultStateSuccess: - it.Info(t, "Test success") - } - return state } -func (it *Test) startTestWithStep(ctx context.Context, t internalT) (*http.Response, []error) { - var ( - resp *http.Response - errs []error - ) - - t.WithNewStep(it.AllureStep.Name, func(stepCtx provider.StepCtx) { - it.Info(t, "Start step %v", it.AllureStep.Name) - defer it.Info(t, "Finish step %v", it.AllureStep.Name) - - resp, errs = it.startTest(ctx, stepCtx) - - if len(errs) != 0 { - stepCtx.Fail() - } - }) - - return resp, errs -} - func (it *Test) startTest(ctx context.Context, t internalT) (*http.Response, []error) { var ( resp *http.Response @@ -365,7 +426,7 @@ func (it *Test) afterTest(t internalT, resp *http.Response, errs []error) []erro return nil } - return executeWithStep(t, "After", func(t T) []error { + return it.executeWithStep(t, "After", func(t T) []error { scope := make([]error, 0) for _, execute := range it.Middleware.After { @@ -389,7 +450,7 @@ func (it *Test) beforeTest(t internalT, req *http.Request) []error { return nil } - return executeWithStep(t, "Before", func(t T) []error { + return it.executeWithStep(t, "Before", func(t T) []error { scope := make([]error, 0) for _, execute := range it.Middleware.Before { @@ -604,12 +665,12 @@ func (it *Test) validateResponse(t internalT, resp *http.Response) []error { saveBody, resp.Body, err = utils.DrainBody(resp.Body) if err != nil { - return append(scope, fmt.Errorf("could not drain response body. error %v", err)) + return append(scope, fmt.Errorf("could not drain response body. error %w", err)) } body, err := utils.GetBody(saveBody) if err != nil { - return append(scope, fmt.Errorf("could not get response body. error %v", err)) + return append(scope, fmt.Errorf("could not get response body. error %w", err)) } // Execute asserts for body diff --git a/test_test.go b/test_test.go index 9e80304..ca34f77 100644 --- a/test_test.go +++ b/test_test.go @@ -157,6 +157,8 @@ func TestValidateResponseWithErrors(t *testing.T) { } ) + ht.initEmptyFields() + errs := ht.validateResponse(temp, resp) require.Len(t, errs, 2)