Skip to content

Commit

Permalink
Merge pull request #441 from ergebnis/fix/execution-order
Browse files Browse the repository at this point in the history 
Fix: Do not run tests in random execution order
  • Loading branch information
@localheinz
localheinz authored Dec 21, 2023
2 parents ceb0f97 + b6376fe commit de16304
Show file tree
Hide file tree
Showing 158 changed files with 82 additions and 234 deletions.
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/Configuration/Defaults/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/Configuration/Defaults/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/Configuration/Defaults/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/Configuration/Defaults/phpunit.xml
Random %seed: %s

............ 12 / 12 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/Configuration/MaximumCount/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/Configuration/MaximumCount/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/Configuration/MaximumCount/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/Configuration/MaximumCount/phpunit.xml
Random %seed: %s

...... 6 / 6 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/Configuration/MaximumDuration/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/Configuration/MaximumDuration/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/Configuration/MaximumDuration/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/Configuration/MaximumDuration/phpunit.xml
Random %seed: %s

............ 12 / 12 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/Bare/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/Bare/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/Bare/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/Bare/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/Combination/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/Combination/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/Combination/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/Combination/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/WithAfterAnnotation/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/WithAfterAnnotation/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/WithTearDown/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/WithTearDown/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/WithAssertPostConditions/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/WithAssertPostConditions/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/WithAssertPostConditions/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/WithAssertPostConditions/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/WithAssertPreConditions/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/WithAssertPreConditions/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/WithAssertPreConditions/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/WithAssertPreConditions/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/WithDataProvider/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/WithDataProvider/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/WithDataProvider/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/WithDataProvider/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/WithSetUp/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/WithSetUp/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/WithSetUp/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/WithSetUp/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/WithSetUpBeforeClass/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/WithSetUpBeforeClass/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/WithSetUpBeforeClass/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/WithSetUpBeforeClass/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/WithTearDown/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/WithTearDown/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/WithTearDown/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/WithTearDown/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestCase/WithTearDownAfterClass/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestCase/WithTearDownAfterClass/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestCase/WithTearDownAfterClass/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestCase/WithTearDownAfterClass/phpunit.xml
Random %seed: %s

... 3 / 3 (100%)

Expand Down
2 changes: 1 addition & 1 deletion .../EndToEnd/Version08/TestMethod/WithMaximumDurationAndSlowThresholdAnnotations/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestMethod/WithMaximumDurationAndSlowThresholdAnnotations/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestMethod/WithMaximumDurationAndSlowThresholdAnnotations/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestMethod/WithMaximumDurationAndSlowThresholdAnnotations/phpunit.xml
Random %seed: %s

.. 2 / 2 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestMethod/WithMaximumDurationAnnotation/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestMethod/WithMaximumDurationAnnotation/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestMethod/WithMaximumDurationAnnotation/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

require_once __DIR__ . '/../../../../../vendor/autoload.php';

Expand All @@ -18,7 +17,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestMethod/WithMaximumDurationAnnotation/phpunit.xml
Random %seed: %s

.... 4 / 4 (100%)

Expand Down
2 changes: 1 addition & 1 deletion test/EndToEnd/Version08/TestMethod/WithRunInSeparateProcessAnnotation/phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
cacheResult="false"
colors="true"
columns="max"
executionOrder="random"
executionOrder="default"
forceCoversAnnotation="true"
stopOnError="false"
stopOnFailure="false"
Expand Down
2 changes: 0 additions & 2 deletions test/EndToEnd/Version08/TestMethod/WithRunInSeparateProcessAnnotation/test.phpt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ declare(strict_types=1);
use PHPUnit\TextUI;

$_SERVER['argv'][] = '--configuration=test/EndToEnd/Version08/TestMethod/WithRunInSeparateProcessAnnotation/phpunit.xml';
$_SERVER['argv'][] = '--random-order-seed=1234567890';

/**
* @see https://github.com/sebastianbergmann/phpunit/blob/8.5.19/src/Framework/TestCase.php#L754C1-L756
Expand All @@ -23,7 +22,6 @@ PHPUnit %s

Runtime: %s
Configuration: %s/EndToEnd/Version08/TestMethod/WithRunInSeparateProcessAnnotation/phpunit.xml
Random %seed: %s

.... 4 / 4 (100%)

Expand Down
Loading

0 comments on commit de16304

Please sign in to comment.