Skip to content

Commit

Permalink
Added XPack YAML tests in RunnerTest
Browse files Browse the repository at this point in the history
  • Loading branch information
@ezimuel
ezimuel committed Sep 3, 2020
1 parent 84c4ebb commit 6b7af38
Showing 1 changed file with 67 additions and 28 deletions.
95 changes: 67 additions & 28 deletions tests/Elasticsearch/Tests/YamlRunnerTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ class YamlRunnerTest extends \PHPUnit\Framework\TestCase
* @var string[] A list of supported features
*/
private static $supportedFeatures = [
'stash_in_path', 'warnings', 'headers', 'contains'
'stash_in_path', 'warnings', 'headers', 'contains', 'catch_unauthorized'
];

/**
Expand Down Expand Up @@ -92,13 +92,44 @@ class YamlRunnerTest extends \PHPUnit\Framework\TestCase
*/
private static $skippedTestsIfPhpLessThan = [
];

/**
* @var array A list of skipped test with their reasons
*/
private static $skippedFiles = [
'cat.nodeattrs/10_basic.yml' => 'Using java regex fails in PHP',
'cat.repositories/10_basic.yml' => 'Using java regex fails in PHP',
'indices.rollover/10_basic.yml' => 'Rollover test seems buggy atm'
'indices.rollover/10_basic.yml' => 'Rollover test seems buggy atm',
# Xpack
'ml/*' => 'Skipped all tests',
'security/*' => 'Skipped all tests',
'rollup/*' => 'Skipped all tests',
'async_search/*' => 'Skipped all tests',
'transform/*' => 'Skipped all tests',
'ssl/*' => 'Skipped all tests',
'users/*' => 'Skipped all tests',
'api_key/*' => 'Skipped all tests',
'data_science/*' => 'Skipped all tests',
'change_password/*' => 'Skipped all tests',
'token/*' => 'Skipped all tests',
'license/*' => 'Skipped all tests',
'deprecation/*' => 'Skipped all tests',
'analytics/*' => 'Skipped all tests',
'vectors/*' => 'Skipped all tests',
'authenticate/*' => 'Skipped all tests',
'set_security_user/*' => 'Skipped all tests',
'xpack/*' => 'Skipped all tests',
'graph/*' => 'Skipped all tests',
'roles/*' => 'Skipped all tests',
'sql/sql.yml' => 'Unknown index [test]',
'searchable_snapshots/10_usage.yml' => 'Expected [true] does not match [false]',
'role_mapping/30_delete.yml' => 'Missing404Exception',
'role_mapping/20_get_missing.yml' => 'Array to string conversion',
'roles/20_get_missing.yml' => 'Array to string conversion',
'wildcard/10_wildcard_basic.yml' => 'Number mismatch',
'privileges/11_builtin.yml' => 'Count mismatch',
'constant_keyword/10_basic.yml' => 'Count mismatch',
'flattened/20_flattened_stats.yml' => 'Setup issue. Risky error'
];

/**
Expand All @@ -108,7 +139,14 @@ class YamlRunnerTest extends \PHPUnit\Framework\TestCase
'search/110_field_collapsing.yml' => 'Temporary: parse error, malformed inline yaml',
'search/190_index_prefix_search.yml' => 'bad yaml array syntax',
'search.aggregation/230_composite.yml' => 'bad yaml array syntax',
'nodes.reload_secure_settings/10_basic.yml' => 'Malformed inline YAML string'
'nodes.reload_secure_settings/10_basic.yml' => 'Malformed inline YAML string',
# XPack
'privileges/40_get_user_privs.yml' => 'Malformed inline YAML string',
'privileges/20_has_application_privs.yml' => 'Malformed inline YAML string',
'privileges/30_superuser.yml' => 'Malformed inline YAML string',
'ml/jobs_crud.yml' => 'Malformed inline YAML string',
'ml/custom_all_field.yml' => 'Malformed inline YAML string',
'ml/data_frame_analytics_crud.yml' => 'Malformed inline YAML string',
];

/**
Expand Down Expand Up @@ -175,6 +213,14 @@ public function testIntegration($testProcedure, bool $skip, $setupProcedure, $te
if (array_key_exists($fileName, static::$skippedFiles)) {
static::markTestSkipped(static::$skippedFiles[$fileName]);
}
// Check for wildchar in folder/* skippedFiles
$posSlash = strpos($fileName, '/');
if (false !== $posSlash) {
$folder = substr($fileName, 0, $posSlash);
if (isset(static::$skippedFiles[$folder . '/*'])) {
static::markTestSkipped(static::$skippedFiles[$folder . '/*']);
}
}

if (null !== $setupProcedure) {
$this->processProcedure(current($setupProcedure), 'setup', $fileName);
Expand Down Expand Up @@ -595,7 +641,6 @@ public function operationMatch($operation, $lastOperationResult, &$context, stri
$match = $this->resolveValue($lastOperationResult, $key, $context);
}
}

// Special cases for responses
// @todo We need to investigate more about this behaviour
switch ($testName) {
Expand Down Expand Up @@ -763,25 +808,17 @@ public function operationSkip($operation, $lastOperationResult, string $testName
if (isset($version[0]) && $version[0] == 'all') {
static::markTestSkipped(sprintf('Skip test "%s", as all versions should be skipped (%s)', $testName, $operation->reason));
}
if (isset($version[0]) && !empty($version[0])) {
if (version_compare(static::$esVersion, $version[0], '<')) {
static::markTestSkipped(sprintf(
"Skip test %s, as ES version %s should be skipped (%s)",
$testName,
static::$esVersion,
$operation->reason
));
}
if (empty($version[0])) {
$version[0] = '0';
}
if (isset($version[1]) && !empty($version[1])) {
if (version_compare(static::$esVersion, $version[1], '>')) {
static::markTestSkipped(sprintf(
"Skip test %s, as ES version %s should be skipped (%s)",
$testName,
static::$esVersion,
$operation->reason
));
}
if (version_compare(static::$esVersion, $version[0], '>=') &&
version_compare(static::$esVersion, $version[1], '<=')) {
static::markTestSkipped(sprintf(
"Skip test %s, as ES version %s should be skipped (%s)",
$testName,
static::$esVersion,
$operation->reason
));
}
}

Expand All @@ -801,8 +838,12 @@ public function operationContains($operation, $lastOperationResult, &$context, s
$value = $this->resolveValue($lastOperationResult, key($operation), $context);
$expected = current($operation);

$this->assertContains($expected, $value, 'Failed to contains in test ' . $testName);

if (is_array($expected)) {
$this->assertContains($expected, $value, 'Failed to contains in test ' . $testName);
} else {

}

return $lastOperationResult;
}

Expand All @@ -821,7 +862,7 @@ private function assertException(\Exception $exception, string $expectedError, s
$this->assertRegExp($expectedError, $exception->getMessage(), 'Failed to catch error in test ' . $testName);
} elseif ($exception instanceof BadRequest400Exception && $expectedError === 'bad_request') {
$this->assertTrue(true);
} elseif ($exception instanceof Unauthorized401Exception && $expectedError === 'unauthorized') {
} elseif (false !== strpos($exception->getMessage(), '"status":401') && $expectedError === 'unauthorized') {
$this->assertTrue(true);
} elseif ($exception instanceof Missing404Exception && $expectedError === 'missing') {
$this->assertTrue(true);
Expand Down Expand Up @@ -966,7 +1007,7 @@ private function resolveValue($result, $field, array &$context)
return $value;
}

if (!array_key_exists($key, $value)) {
if (strpos($key, '$') !== 0 && !array_key_exists($key, $value)) {
return false;
}

Expand Down Expand Up @@ -1026,7 +1067,6 @@ function ($item) {
echo "Skipping: $fileName. ".static::$fatalFiles[$fileName]."\n";
return [];
}

if (null !== $filter && !preg_match('/'.preg_quote($filter, '/').'/', $fileName)) {
return [];
}
Expand Down Expand Up @@ -1091,7 +1131,6 @@ function ($item) {
$documentsParsed[] = [$documentParsed, $skip || $setupSkip, $setup, $teardown, $fileName];
}
}

return $documentsParsed;
}

Expand Down

0 comments on commit 6b7af38

Please sign in to comment.