From 8cebb29dd0c26eb0bd3067fe081ceae04d3ac9b2 Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Thu, 8 Nov 2018 14:40:39 -0600 Subject: [PATCH 01/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../Integrity/DeclarativeDependencyTest.php | 784 ++++++++++++++++++ 1 file changed, 784 insertions(+) create mode 100644 dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php new file mode 100644 index 0000000000000..d1fe1bd57a834 --- /dev/null +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php @@ -0,0 +1,784 @@ +readJsonFile($root . '/composer.json', true); + if (preg_match('/magento\/project-*/', $rootJson['name']) == 1) { + // The Dependency test is skipped for vendor/magento build + self::markTestSkipped( + 'MAGETWO-43654: The build is running from vendor/magento. DependencyTest is skipped.' + ); + } + } + + /** + * Initialise map of dependencies. + * + * @throws \Exception + */ + private function initDeclaredDependencies() + { + $jsonFiles = Files::init()->getComposerFiles(ComponentRegistrar::MODULE, false); + foreach ($jsonFiles as $file) { + $json = new \Magento\Framework\Config\Composer\Package($this->readJsonFile($file)); + $moduleName = $this->convertModuleName($json->get('name')); + $require = array_keys((array)$json->get('require')); + $this->presetDependencies($moduleName, $require, self::TYPE_HARD); + } + } + + /** + * Read data from json file. + * + * @param string $file + * @return mixed + * @throws \Exception + */ + private function readJsonFile(string $file, bool $asArray = false) + { + $decodedJson = json_decode(file_get_contents($file), $asArray); + if (null == $decodedJson) { + throw new \Exception("Invalid Json: $file"); + } + + return $decodedJson; + } + + /** + * @throws \Exception + */ + public function testUndeclaredDependencies() + { + $this->initDeclaredDependencies(); + $invoker = new \Magento\Framework\App\Utility\AggregateInvoker($this); + $invoker( + /** + * Check undeclared modules dependencies for specified file + * + * @param string $fileType + * @param string $file + */ + function ($file) { + $componentRegistrar = new ComponentRegistrar(); + $foundModuleName = ''; + foreach ($componentRegistrar->getPaths(ComponentRegistrar::MODULE) as $moduleName => $moduleDir) { + if (strpos($file, $moduleDir . '/') !== false) { + $foundModuleName = str_replace('_', '\\', $moduleName); + break; + } + } + if (empty($foundModuleName)) { + return; + } + + $dependencies = $this->getDependenciesFromFiles($file); + $dependencies = $this->filterSelfDependency($foundModuleName, $dependencies); + $undeclaredDependency = $this->collectDependencies($foundModuleName, $dependencies); + + $result = []; + foreach ($undeclaredDependency as $name => $modules) { + $modules = array_unique($modules); + $result[] = $this->getErrorMessage($name) . "\n" . implode("\t\n", $modules); + } + if (count($result)) { + $this->fail( + 'Module ' . $moduleName . ' has undeclared dependencies: ' . "\n" . implode("\t\n", $result) + ); + } + }, + $this->prepareFiles(Files::init()->getDbSchemaFiles()) + ); + } + + /** + * Remove self dependencies. + * + * @param string $moduleName + * @param array $dependencies + * @return array + */ + private function filterSelfDependency(string $moduleName, array $dependencies):array + { + foreach ($dependencies as $id => $modules) { + $decodedId = $this->decodeDependencyId($id); + $entityType = $decodedId['entityType']; + if ($entityType === self::SCHEMA_ENTITY_TABLE || $entityType === "column") { + if (array_search($moduleName, $modules) !== false) { + unset($dependencies[$id]); + } + } else { + $dependencies[$id] = $this->filterComplexDependency($moduleName, $modules); + } + } + + return array_filter($dependencies); + } + + /** + * Remove already declared dependencies. + * + * @param string $moduleName + * @param array $modules + * @return array + */ + private function filterComplexDependency(string $moduleName, array $modules): array + { + $resultDependencies = []; + if (!is_array(reset($modules))) { + if (array_search($moduleName, $modules) === false) { + $resultDependencies = $modules; + } + } else { + foreach ($modules as $dependencySet) { + if (array_search($moduleName, $dependencySet) === false) { + $resultDependencies = array_merge( + $resultDependencies, + $dependencySet + ); + } + } + } + + return array_values(array_unique($resultDependencies)); + } + + /** + * Retrieve declarative schema declaration. + * + * @return array + * @throws \Exception + */ + private function getDeclarativeSchema(): array + { + if ($this->dbSchemaDeclaration) { + return $this->dbSchemaDeclaration; + } + + $entityTypes = [self::SCHEMA_ENTITY_COLUMN, self::SCHEMA_ENTITY_CONSTRAINT, self::SCHEMA_ENTITY_INDEX]; + $declaration = []; + foreach (Files::init()->getDbSchemaFiles() as $filePath) { + $filePath = reset($filePath); + preg_match('#app/code/(\w+/\w+)#', $filePath, $result); + $moduleName = str_replace('/', '\\', $result[1]); + $moduleDeclaration = $this->getDbSchemaDeclaration($filePath); + + foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { + if (!isset($tableDeclaration['modules'])) { + $tableDeclaration['modules'] = []; + } + array_push($tableDeclaration['modules'], $moduleName); + $moduleDeclaration = array_replace_recursive( + $moduleDeclaration, + [self::SCHEMA_ENTITY_TABLE => + [ + $tableName => $tableDeclaration, + ] + ] + ); + foreach ($entityTypes as $entityType) { + if (!isset($tableDeclaration[$entityType])) { + continue; + } + $moduleDeclaration = array_replace_recursive( + $moduleDeclaration, + [self::SCHEMA_ENTITY_TABLE => + [ + $tableName => + $this->addModuleAssigment($tableDeclaration, $entityType, $moduleName) + ] + ] + ); + } + } + $declaration = array_merge_recursive($declaration, $moduleDeclaration); + } + $this->dbSchemaDeclaration = $declaration; + + return $this->dbSchemaDeclaration; + } + + /** + * Get declared dependencies. + * + * @param string $tableName + * @param string $entityType + * @param null|string $entityName + * @return array + * @throws \Exception + */ + private function resolveEntityDependencies(string $tableName, string $entityType, ?string $entityName = null): array + { + switch ($entityType) { + case self::SCHEMA_ENTITY_COLUMN: + case self::SCHEMA_ENTITY_CONSTRAINT: + case self::SCHEMA_ENTITY_INDEX: + return $this->getDeclarativeSchema() + [self::SCHEMA_ENTITY_TABLE][$tableName][$entityType][$entityName]['modules']; + break; + case self::SCHEMA_ENTITY_TABLE: + return $this->getDeclarativeSchema()[self::SCHEMA_ENTITY_TABLE][$tableName]['modules']; + break; + default: + return []; + } + } + + /** + * @param string $filePath + * @return array + */ + private function getDbSchemaDeclaration(string $filePath): array + { + $dom = new \DOMDocument(); + $dom->loadXML(file_get_contents($filePath)); + return (new Converter())->convert($dom); + } + + /** + * Add dependency on the current module. + * + * @param array $tableDeclaration + * @param string $entityType + * @param string $moduleName + * @return array + */ + private function addModuleAssigment( + array $tableDeclaration, + string $entityType, + string $moduleName + ): array { + $declarationWithAssigment = []; + foreach ($tableDeclaration[$entityType] as $entityName => $entityDeclaration) { + if (!isset($entityDeclaration['modules'])) { + $entityDeclaration['modules'] = []; + } + if (!$this->isEntityDisabled($entityDeclaration)) { + array_push($entityDeclaration['modules'], $moduleName); + } + + $declarationWithAssigment[$entityType][$entityName] = $entityDeclaration; + } + + return $declarationWithAssigment; + } + + /** + * Retrieve dependencies from files. + * + * @param string $file + * @return string[] + * @throws \Exception + */ + private function getDependenciesFromFiles($file) + { + $moduleDbSchema = $this->getDbSchemaDeclaration($file); + $dependencies = array_merge_recursive( + $this->getDisabledDependencies($moduleDbSchema), + $this->getConstraintDependencies($moduleDbSchema), + $this->getIndexDependencies($moduleDbSchema) + ); + return $dependencies; + } + + /** + * Retrieve dependencies for disabled entities. + * + * @param array $moduleDeclaration + * @return array + * @throws \Exception + */ + private function getDisabledDependencies(array $moduleDeclaration): array + { + $disabledDependencies = []; + $entityTypes = [self::SCHEMA_ENTITY_COLUMN, self::SCHEMA_ENTITY_CONSTRAINT, self::SCHEMA_ENTITY_INDEX]; + foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { + foreach ($entityTypes as $entityType) { + if (!isset($tableDeclaration[$entityType])) { + continue; + } + foreach ($tableDeclaration[$entityType] as $entityName => $entityDeclaration) { + if ($this->isEntityDisabled($entityDeclaration)) { + $dependencyIdentifier = $this->getDependencyId($tableName, $entityType, $entityName); + $disabledDependencies[$dependencyIdentifier] = + $this->resolveEntityDependencies($tableName, $entityType, $entityName); + } + } + } + if ($this->isEntityDisabled($tableDeclaration)) { + $disabledDependencies[$this->getDependencyId($tableName)] = + $this->resolveEntityDependencies($tableName, self::SCHEMA_ENTITY_TABLE); + } + } + + return $disabledDependencies; + } + + /** + * Retrieve dependencies for foreign entities. + * + * @param array $constraintDeclaration + * @return array + * @throws \Exception + */ + private function getFKDependencies(array $constraintDeclaration): array + { + $referenceDependencyIdentifier = + $this->getDependencyId( + $constraintDeclaration['referenceTable'], + self::SCHEMA_ENTITY_CONSTRAINT, + $constraintDeclaration['referenceId'] + ); + $dependencyIdentifier = + $this->getDependencyId( + $constraintDeclaration[self::SCHEMA_ENTITY_TABLE], + self::SCHEMA_ENTITY_CONSTRAINT, + $constraintDeclaration['referenceId'] + ); + + $constraintDependencies = []; + $constraintDependencies[$referenceDependencyIdentifier] = + $this->resolveEntityDependencies( + $constraintDeclaration['referenceTable'], + self::SCHEMA_ENTITY_COLUMN, + $constraintDeclaration['referenceColumn'] + ); + $constraintDependencies[$dependencyIdentifier] = + $this->resolveEntityDependencies( + $constraintDeclaration[self::SCHEMA_ENTITY_TABLE], + self::SCHEMA_ENTITY_COLUMN, + $constraintDeclaration[self::SCHEMA_ENTITY_COLUMN] + ); + + return $constraintDependencies; + } + + /** + * Retrieve dependencies for constraint entities. + * + * @param array $moduleDeclaration + * @return array + * @throws \Exception + */ + private function getConstraintDependencies(array $moduleDeclaration): array + { + $constraintDependencies = []; + foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { + if (empty($tableDeclaration[self::SCHEMA_ENTITY_CONSTRAINT])) { + continue; + } + foreach ($tableDeclaration[self::SCHEMA_ENTITY_CONSTRAINT] as $constraintName => $constraintDeclaration) { + if ($this->isEntityDisabled($constraintDeclaration)) { + continue; + } + $dependencyIdentifier = + $this->getDependencyId($tableName, self::SCHEMA_ENTITY_CONSTRAINT, $constraintName); + switch ($constraintDeclaration['type']) { + case 'foreign': + $constraintDependencies = array_merge( + $constraintDependencies, + $this->getFKDependencies($constraintDeclaration) + ); + break; + case 'primary': + case 'unique': + $constraintDependencies[$dependencyIdentifier] = $this->getComplexDependency( + $tableName, + $constraintDeclaration + ); + } + } + } + return $constraintDependencies; + } + + /** + * Calculate complex dependency. + * + * @param string $tableName + * @param array $entityDeclaration + * @return array + * @throws \Exception + */ + private function getComplexDependency(string $tableName, array $entityDeclaration): array + { + $complexDependency = []; + if (empty($entityDeclaration[self::SCHEMA_ENTITY_COLUMN])) { + return $complexDependency; + } + + if (!is_array($entityDeclaration[self::SCHEMA_ENTITY_COLUMN])) { + $entityDeclaration[self::SCHEMA_ENTITY_COLUMN] = [$entityDeclaration[self::SCHEMA_ENTITY_COLUMN]]; + } + + foreach (array_keys($entityDeclaration[self::SCHEMA_ENTITY_COLUMN]) as $columnName) { + $complexDependency[] = + $this->resolveEntityDependencies($tableName, self::SCHEMA_ENTITY_COLUMN, $columnName); + } + + return array_values($complexDependency); + } + + /** + * Retrieve dependencies for index entities. + * + * @param array $moduleDeclaration + * @return array + * @throws \Exception + */ + private function getIndexDependencies(array $moduleDeclaration): array + { + $indexDependencies = []; + foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { + if (empty($tableDeclaration[self::SCHEMA_ENTITY_INDEX])) { + continue; + } + foreach ($tableDeclaration[self::SCHEMA_ENTITY_INDEX] as $indexName => $indexDeclaration) { + if ($this->isEntityDisabled($indexDeclaration)) { + continue; + } + $dependencyIdentifier = + $this->getDependencyId($tableName, self::SCHEMA_ENTITY_INDEX, $indexName); + $indexDependencies[$dependencyIdentifier] = + $this->getComplexDependency($tableName, $indexDeclaration); + } + } + + return $indexDependencies; + } + + /** + * Check status of the entity declaration. + * + * @param array $entityDeclaration + * @return bool + */ + private function isEntityDisabled(array $entityDeclaration): bool + { + return isset($entityDeclaration['disabled']) && $entityDeclaration['disabled'] == true; + } + + /** + * Retrive dependency id. + * + * @param string $tableName + * @param string $entityType + * @param null|string $entityName + * @return string + */ + private function getDependencyId( + string $tableName, + string $entityType = self::SCHEMA_ENTITY_TABLE, + ?string $entityName = null + ) { + return implode('___', [$tableName, $entityType, $entityName ?: $tableName]); + } + + /** + * Retrieve dependency parameters from dependency id. + * + * @param string $id + * @return array + */ + private function decodeDependencyId(string $id): array + { + $decodedValues = explode('___', $id); + $result = [ + 'tableName' => $decodedValues[0], + 'entityType' => $decodedValues[1], + 'entityName' => $decodedValues[2], + ]; + return $result; + } + + /** + * Retrieve error message for dependency. + * + * @param string $id + * @return string + */ + private function getErrorMessage(string $id): string + { + $decodedId = $this->decodeDependencyId($id); + $entityType = $decodedId['entityType']; + if ($entityType === self::SCHEMA_ENTITY_TABLE) { + $message = sprintf( + 'Table %s has undeclared dependency on one of the next modules.', + $decodedId['tableName'] + ); + } else { + $message = sprintf( + '%s %s from %s table has undeclared dependency on one of the next modules.', + ucfirst($entityType), + $decodedId['entityName'], + $decodedId['tableName'] + ); + } + + return $message; + } + + /** + * Collect module dependencies. + * + * @param string $currentModuleName + * @param array $dependencies + * @return array + */ + private function collectDependencies($currentModuleName, $dependencies = []) + { + if (!count($dependencies)) { + return []; + } + foreach ($dependencies as $dependencyName => $dependency) { + $this->collectDependency($dependencyName, $dependency, $currentModuleName); + } + + return $this->getDeclaredDependencies($currentModuleName, self::TYPE_HARD, self::MAP_TYPE_FOUND); + } + + /** + * Collect a module dependency. + * + * @param string $dependencyName + * @param array $dependency + * @param string $currentModule + */ + private function collectDependency( + string $dependencyName, + array $dependency, + string $currentModule + ) { + $declared = $this->getDeclaredDependencies($currentModule, self::TYPE_HARD, self::MAP_TYPE_DECLARED); + $checkResult = array_intersect($declared, $dependency); + + if (empty($checkResult)) { + $this->addDependencies( + $currentModule, + self::TYPE_HARD, + self::MAP_TYPE_FOUND, + [ + $dependencyName => $dependency, + ] + ); + } + } + + /** + * Convert file list to data provider structure. + * + * @param string[] $files + * @return array + */ + private function prepareFiles(array $files): array + { + $result = []; + foreach ($files as $relativePath => $file) { + $absolutePath = reset($file); + $result[$relativePath] = [$absolutePath]; + } + return $result; + } + + /** + * Add dependencies to dependency list. + * + * @param string $moduleName + * @param array $packageNames + * @param string $type + * + * @return void + * @throws \Exception + */ + private function presetDependencies( + string $moduleName, + array $packageNames, + string $type + ): void { + $packageNames = array_filter($packageNames, function ($packageName) { + return $this->getModuleName($packageName) || + 0 === strpos($packageName, 'magento/') && 'magento/magento-composer-installer' != $packageName; + }); + + foreach ($packageNames as $packageName) { + $this->addDependencies( + $moduleName, + $type, + self::MAP_TYPE_DECLARED, + [$this->convertModuleName($packageName)] + ); + } + } + + /** + * Converts a composer json component name into the Magento Module form. + * + * @param string $jsonName The name of a composer json component or dependency e.g. 'magento/module-theme' + * @return string The corresponding Magento Module e.g. 'Magento\Theme' + * @throws \Exception + */ + private function convertModuleName(string $jsonName): string + { + $moduleName = $this->getModuleName($jsonName); + if ($moduleName) { + return $moduleName; + } + + if ( + strpos($jsonName, 'magento/magento') !== false + || strpos($jsonName, 'magento/framework') !== false + ) { + $moduleName = str_replace('/', "\t", $jsonName); + $moduleName = str_replace('framework-', "Framework\t", $moduleName); + $moduleName = str_replace('-', ' ', $moduleName); + $moduleName = ucwords($moduleName); + $moduleName = str_replace("\t", '\\', $moduleName); + $moduleName = str_replace(' ', '', $moduleName); + } else { + $moduleName = $jsonName; + } + + return $moduleName; + } + + /** + * Returns package name on module name mapping. + * + * @return array + * @throws \Exception + */ + private function getPackageModuleMapping(): array + { + if (!$this->packageModuleMapping) { + $jsonFiles = Files::init()->getComposerFiles(ComponentRegistrar::MODULE, false); + + $packageModuleMapping = []; + foreach ($jsonFiles as $file) { + $moduleXml = simplexml_load_file(dirname($file) . '/etc/module.xml'); + $moduleName = str_replace('_', '\\', (string)$moduleXml->module->attributes()->name); + $composerJson = $this->readJsonFile($file); + $packageName = $composerJson->name; + $packageModuleMapping[$packageName] = $moduleName; + } + + $this->packageModuleMapping = $packageModuleMapping; + } + + return $this->packageModuleMapping; + } + + /** + * Retrive Magento style module name. + * + * @param string $packageName + * @return null|string + * @throws \Exception + */ + private function getModuleName(string $packageName): ?string + { + return $this->getPackageModuleMapping()[$packageName] ?? null; + } + + /** + * Retrieve array of dependency items. + * + * @param $module + * @param $type + * @param $mapType + * @return array + */ + private function getDeclaredDependencies(string $module, string $type, string $mapType) + { + return $this->mapDependencies[$module][$type][$mapType] ?? []; + } + + /** + * Add dependency map items. + * + * @param $module + * @param $type + * @param $mapType + * @param $dependencies + */ + protected function addDependencies(string $module, string $type, string $mapType, array $dependencies) + { + $this->mapDependencies[$module][$type][$mapType] = array_merge_recursive( + $this->getDeclaredDependencies($module, $type, $mapType), + $dependencies + ); + } +} \ No newline at end of file From 58b0c691f54d8b3c52090488809cf9b62efc36a1 Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Fri, 9 Nov 2018 16:32:21 -0600 Subject: [PATCH 02/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../Dependency/DeclarativeSchemaRule.php | 101 --- .../Integrity/DeclarativeDependencyTest.php | 685 +--------------- .../DeclarativeSchemaDependencyProvider.php | 759 ++++++++++++++++++ .../Magento/Test/Integrity/DependencyTest.php | 59 +- 4 files changed, 803 insertions(+), 801 deletions(-) delete mode 100644 dev/tests/static/framework/Magento/TestFramework/Dependency/DeclarativeSchemaRule.php create mode 100644 dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php diff --git a/dev/tests/static/framework/Magento/TestFramework/Dependency/DeclarativeSchemaRule.php b/dev/tests/static/framework/Magento/TestFramework/Dependency/DeclarativeSchemaRule.php deleted file mode 100644 index 72e4fff7e5162..0000000000000 --- a/dev/tests/static/framework/Magento/TestFramework/Dependency/DeclarativeSchemaRule.php +++ /dev/null @@ -1,101 +0,0 @@ -_moduleTableMap = $tables; - } - - /** - * Gets external dependencies information for current module by analyzing db_schema.xml files contents. - * - * @param string $currentModule - * @param string $fileType - * @param string $file - * @param string $contents - * @return array - * @SuppressWarnings(PHPMD.CyclomaticComplexity) - * @SuppressWarnings(PHPMD.NPathComplexity) - */ - public function getDependencyInfo($currentModule, $fileType, $file, &$contents) - { - if ('db_schema' != $fileType || !preg_match('#.*/db_schema\.xml$#', $file)) { - return []; - } - - $dependenciesInfo = []; - $unKnowTables = []; - - $dom = new \DOMDocument(); - $dom->loadXML($contents); - $tables = $dom->getElementsByTagName('table'); - $constraints = $dom->getElementsByTagName('constraint'); - - $tableNames = []; - $foreignKeyTables = []; - $foreignKeyReferenceTables = []; - - /** @var \DOMElement $table */ - foreach ($tables as $table) { - $tableNames[] = $table->getAttribute('name'); - } - - /** @var \DOMElement $constraint */ - foreach ($constraints as $constraint) { - $xsiType = $constraint->getAttribute('xsi:type'); - if (strtolower($xsiType) == 'foreign' && $constraint->getAttribute('disabled') !== '1') { - $foreignKeyTables[] = $constraint->getAttribute('table'); - $foreignKeyReferenceTables[] = $constraint->getAttribute('referenceTable'); - } - } - - $tableNames = array_unique(array_merge($tableNames, $foreignKeyReferenceTables, $foreignKeyTables)); - - /** @var string $table */ - foreach ($tableNames as $table) { - if (!isset($this->_moduleTableMap[$table])) { - $unKnowTables[$file][$table] = $table; - continue; - } - if (strtolower($currentModule) !== strtolower($this->_moduleTableMap[$table])) { - $dependenciesInfo[] = [ - 'module' => $this->_moduleTableMap[$table], - 'type' => RuleInterface::TYPE_HARD, - 'source' => $table, - ]; - } - } - - foreach ($unKnowTables as $tables) { - foreach ($tables as $table) { - $dependenciesInfo[] = ['module' => 'Unknown', 'source' => $table]; - } - } - return $dependenciesInfo; - } -} diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php index d1fe1bd57a834..257da9669d9b5 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php @@ -8,66 +8,19 @@ namespace Magento\Test\Integrity; +use Magento\Test\Integrity\Dependency\DeclarativeSchemaDependencyProvider; use Magento\Framework\App\Utility\Files; use Magento\Framework\Component\ComponentRegistrar; -use Magento\Framework\Setup\Declaration\Schema\Config\Converter; /** * Class DeclarativeDependencyTest - * - * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) */ class DeclarativeDependencyTest extends \PHPUnit\Framework\TestCase { /** - * Types of dependency between modules. + * @var DeclarativeSchemaDependencyProvider */ - const TYPE_HARD = 'hard'; - - /** - * The identifier of dependency for mapping. - */ - const MAP_TYPE_DECLARED = 'declared'; - - /** - * The identifier of dependency for mapping. - */ - const MAP_TYPE_FOUND = 'found'; - - /** - * Declarative name for table entity of the declarative schema. - */ - const SCHEMA_ENTITY_TABLE = 'table'; - - /** - * Declarative name for column entity of the declarative schema. - */ - const SCHEMA_ENTITY_COLUMN = 'column'; - - /** - * Declarative name for constraint entity of the declarative schema. - */ - const SCHEMA_ENTITY_CONSTRAINT = 'constraint'; - - /** - * Declarative name for index entity of the declarative schema. - */ - const SCHEMA_ENTITY_INDEX = 'index'; - - /** - * @var array - */ - private $mapDependencies = []; - - /** - * @var array - */ - private $dbSchemaDeclaration = []; - - /** - * @var array - */ - private $packageModuleMapping = []; + private $dependencyProvider; /** * Sets up data @@ -84,39 +37,7 @@ protected function setUp() 'MAGETWO-43654: The build is running from vendor/magento. DependencyTest is skipped.' ); } - } - - /** - * Initialise map of dependencies. - * - * @throws \Exception - */ - private function initDeclaredDependencies() - { - $jsonFiles = Files::init()->getComposerFiles(ComponentRegistrar::MODULE, false); - foreach ($jsonFiles as $file) { - $json = new \Magento\Framework\Config\Composer\Package($this->readJsonFile($file)); - $moduleName = $this->convertModuleName($json->get('name')); - $require = array_keys((array)$json->get('require')); - $this->presetDependencies($moduleName, $require, self::TYPE_HARD); - } - } - - /** - * Read data from json file. - * - * @param string $file - * @return mixed - * @throws \Exception - */ - private function readJsonFile(string $file, bool $asArray = false) - { - $decodedJson = json_decode(file_get_contents($file), $asArray); - if (null == $decodedJson) { - throw new \Exception("Invalid Json: $file"); - } - - return $decodedJson; + $this->dependencyProvider = new DeclarativeSchemaDependencyProvider(); } /** @@ -124,7 +45,6 @@ private function readJsonFile(string $file, bool $asArray = false) */ public function testUndeclaredDependencies() { - $this->initDeclaredDependencies(); $invoker = new \Magento\Framework\App\Utility\AggregateInvoker($this); $invoker( /** @@ -146,9 +66,7 @@ function ($file) { return; } - $dependencies = $this->getDependenciesFromFiles($file); - $dependencies = $this->filterSelfDependency($foundModuleName, $dependencies); - $undeclaredDependency = $this->collectDependencies($foundModuleName, $dependencies); + $undeclaredDependency = $this->dependencyProvider->getUndeclaredModuleDependencies($foundModuleName); $result = []; foreach ($undeclaredDependency as $name => $modules) { @@ -166,402 +84,18 @@ function ($file) { } /** - * Remove self dependencies. - * - * @param string $moduleName - * @param array $dependencies - * @return array - */ - private function filterSelfDependency(string $moduleName, array $dependencies):array - { - foreach ($dependencies as $id => $modules) { - $decodedId = $this->decodeDependencyId($id); - $entityType = $decodedId['entityType']; - if ($entityType === self::SCHEMA_ENTITY_TABLE || $entityType === "column") { - if (array_search($moduleName, $modules) !== false) { - unset($dependencies[$id]); - } - } else { - $dependencies[$id] = $this->filterComplexDependency($moduleName, $modules); - } - } - - return array_filter($dependencies); - } - - /** - * Remove already declared dependencies. - * - * @param string $moduleName - * @param array $modules - * @return array - */ - private function filterComplexDependency(string $moduleName, array $modules): array - { - $resultDependencies = []; - if (!is_array(reset($modules))) { - if (array_search($moduleName, $modules) === false) { - $resultDependencies = $modules; - } - } else { - foreach ($modules as $dependencySet) { - if (array_search($moduleName, $dependencySet) === false) { - $resultDependencies = array_merge( - $resultDependencies, - $dependencySet - ); - } - } - } - - return array_values(array_unique($resultDependencies)); - } - - /** - * Retrieve declarative schema declaration. - * - * @return array - * @throws \Exception - */ - private function getDeclarativeSchema(): array - { - if ($this->dbSchemaDeclaration) { - return $this->dbSchemaDeclaration; - } - - $entityTypes = [self::SCHEMA_ENTITY_COLUMN, self::SCHEMA_ENTITY_CONSTRAINT, self::SCHEMA_ENTITY_INDEX]; - $declaration = []; - foreach (Files::init()->getDbSchemaFiles() as $filePath) { - $filePath = reset($filePath); - preg_match('#app/code/(\w+/\w+)#', $filePath, $result); - $moduleName = str_replace('/', '\\', $result[1]); - $moduleDeclaration = $this->getDbSchemaDeclaration($filePath); - - foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { - if (!isset($tableDeclaration['modules'])) { - $tableDeclaration['modules'] = []; - } - array_push($tableDeclaration['modules'], $moduleName); - $moduleDeclaration = array_replace_recursive( - $moduleDeclaration, - [self::SCHEMA_ENTITY_TABLE => - [ - $tableName => $tableDeclaration, - ] - ] - ); - foreach ($entityTypes as $entityType) { - if (!isset($tableDeclaration[$entityType])) { - continue; - } - $moduleDeclaration = array_replace_recursive( - $moduleDeclaration, - [self::SCHEMA_ENTITY_TABLE => - [ - $tableName => - $this->addModuleAssigment($tableDeclaration, $entityType, $moduleName) - ] - ] - ); - } - } - $declaration = array_merge_recursive($declaration, $moduleDeclaration); - } - $this->dbSchemaDeclaration = $declaration; - - return $this->dbSchemaDeclaration; - } - - /** - * Get declared dependencies. - * - * @param string $tableName - * @param string $entityType - * @param null|string $entityName - * @return array - * @throws \Exception - */ - private function resolveEntityDependencies(string $tableName, string $entityType, ?string $entityName = null): array - { - switch ($entityType) { - case self::SCHEMA_ENTITY_COLUMN: - case self::SCHEMA_ENTITY_CONSTRAINT: - case self::SCHEMA_ENTITY_INDEX: - return $this->getDeclarativeSchema() - [self::SCHEMA_ENTITY_TABLE][$tableName][$entityType][$entityName]['modules']; - break; - case self::SCHEMA_ENTITY_TABLE: - return $this->getDeclarativeSchema()[self::SCHEMA_ENTITY_TABLE][$tableName]['modules']; - break; - default: - return []; - } - } - - /** - * @param string $filePath - * @return array - */ - private function getDbSchemaDeclaration(string $filePath): array - { - $dom = new \DOMDocument(); - $dom->loadXML(file_get_contents($filePath)); - return (new Converter())->convert($dom); - } - - /** - * Add dependency on the current module. - * - * @param array $tableDeclaration - * @param string $entityType - * @param string $moduleName - * @return array - */ - private function addModuleAssigment( - array $tableDeclaration, - string $entityType, - string $moduleName - ): array { - $declarationWithAssigment = []; - foreach ($tableDeclaration[$entityType] as $entityName => $entityDeclaration) { - if (!isset($entityDeclaration['modules'])) { - $entityDeclaration['modules'] = []; - } - if (!$this->isEntityDisabled($entityDeclaration)) { - array_push($entityDeclaration['modules'], $moduleName); - } - - $declarationWithAssigment[$entityType][$entityName] = $entityDeclaration; - } - - return $declarationWithAssigment; - } - - /** - * Retrieve dependencies from files. - * - * @param string $file - * @return string[] - * @throws \Exception - */ - private function getDependenciesFromFiles($file) - { - $moduleDbSchema = $this->getDbSchemaDeclaration($file); - $dependencies = array_merge_recursive( - $this->getDisabledDependencies($moduleDbSchema), - $this->getConstraintDependencies($moduleDbSchema), - $this->getIndexDependencies($moduleDbSchema) - ); - return $dependencies; - } - - /** - * Retrieve dependencies for disabled entities. - * - * @param array $moduleDeclaration - * @return array - * @throws \Exception - */ - private function getDisabledDependencies(array $moduleDeclaration): array - { - $disabledDependencies = []; - $entityTypes = [self::SCHEMA_ENTITY_COLUMN, self::SCHEMA_ENTITY_CONSTRAINT, self::SCHEMA_ENTITY_INDEX]; - foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { - foreach ($entityTypes as $entityType) { - if (!isset($tableDeclaration[$entityType])) { - continue; - } - foreach ($tableDeclaration[$entityType] as $entityName => $entityDeclaration) { - if ($this->isEntityDisabled($entityDeclaration)) { - $dependencyIdentifier = $this->getDependencyId($tableName, $entityType, $entityName); - $disabledDependencies[$dependencyIdentifier] = - $this->resolveEntityDependencies($tableName, $entityType, $entityName); - } - } - } - if ($this->isEntityDisabled($tableDeclaration)) { - $disabledDependencies[$this->getDependencyId($tableName)] = - $this->resolveEntityDependencies($tableName, self::SCHEMA_ENTITY_TABLE); - } - } - - return $disabledDependencies; - } - - /** - * Retrieve dependencies for foreign entities. - * - * @param array $constraintDeclaration - * @return array - * @throws \Exception - */ - private function getFKDependencies(array $constraintDeclaration): array - { - $referenceDependencyIdentifier = - $this->getDependencyId( - $constraintDeclaration['referenceTable'], - self::SCHEMA_ENTITY_CONSTRAINT, - $constraintDeclaration['referenceId'] - ); - $dependencyIdentifier = - $this->getDependencyId( - $constraintDeclaration[self::SCHEMA_ENTITY_TABLE], - self::SCHEMA_ENTITY_CONSTRAINT, - $constraintDeclaration['referenceId'] - ); - - $constraintDependencies = []; - $constraintDependencies[$referenceDependencyIdentifier] = - $this->resolveEntityDependencies( - $constraintDeclaration['referenceTable'], - self::SCHEMA_ENTITY_COLUMN, - $constraintDeclaration['referenceColumn'] - ); - $constraintDependencies[$dependencyIdentifier] = - $this->resolveEntityDependencies( - $constraintDeclaration[self::SCHEMA_ENTITY_TABLE], - self::SCHEMA_ENTITY_COLUMN, - $constraintDeclaration[self::SCHEMA_ENTITY_COLUMN] - ); - - return $constraintDependencies; - } - - /** - * Retrieve dependencies for constraint entities. - * - * @param array $moduleDeclaration - * @return array - * @throws \Exception - */ - private function getConstraintDependencies(array $moduleDeclaration): array - { - $constraintDependencies = []; - foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { - if (empty($tableDeclaration[self::SCHEMA_ENTITY_CONSTRAINT])) { - continue; - } - foreach ($tableDeclaration[self::SCHEMA_ENTITY_CONSTRAINT] as $constraintName => $constraintDeclaration) { - if ($this->isEntityDisabled($constraintDeclaration)) { - continue; - } - $dependencyIdentifier = - $this->getDependencyId($tableName, self::SCHEMA_ENTITY_CONSTRAINT, $constraintName); - switch ($constraintDeclaration['type']) { - case 'foreign': - $constraintDependencies = array_merge( - $constraintDependencies, - $this->getFKDependencies($constraintDeclaration) - ); - break; - case 'primary': - case 'unique': - $constraintDependencies[$dependencyIdentifier] = $this->getComplexDependency( - $tableName, - $constraintDeclaration - ); - } - } - } - return $constraintDependencies; - } - - /** - * Calculate complex dependency. - * - * @param string $tableName - * @param array $entityDeclaration - * @return array - * @throws \Exception - */ - private function getComplexDependency(string $tableName, array $entityDeclaration): array - { - $complexDependency = []; - if (empty($entityDeclaration[self::SCHEMA_ENTITY_COLUMN])) { - return $complexDependency; - } - - if (!is_array($entityDeclaration[self::SCHEMA_ENTITY_COLUMN])) { - $entityDeclaration[self::SCHEMA_ENTITY_COLUMN] = [$entityDeclaration[self::SCHEMA_ENTITY_COLUMN]]; - } - - foreach (array_keys($entityDeclaration[self::SCHEMA_ENTITY_COLUMN]) as $columnName) { - $complexDependency[] = - $this->resolveEntityDependencies($tableName, self::SCHEMA_ENTITY_COLUMN, $columnName); - } - - return array_values($complexDependency); - } - - /** - * Retrieve dependencies for index entities. + * Convert file list to data provider structure. * - * @param array $moduleDeclaration + * @param string[] $files * @return array - * @throws \Exception */ - private function getIndexDependencies(array $moduleDeclaration): array + private function prepareFiles(array $files): array { - $indexDependencies = []; - foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { - if (empty($tableDeclaration[self::SCHEMA_ENTITY_INDEX])) { - continue; - } - foreach ($tableDeclaration[self::SCHEMA_ENTITY_INDEX] as $indexName => $indexDeclaration) { - if ($this->isEntityDisabled($indexDeclaration)) { - continue; - } - $dependencyIdentifier = - $this->getDependencyId($tableName, self::SCHEMA_ENTITY_INDEX, $indexName); - $indexDependencies[$dependencyIdentifier] = - $this->getComplexDependency($tableName, $indexDeclaration); - } + $result = []; + foreach ($files as $relativePath => $file) { + $absolutePath = reset($file); + $result[$relativePath] = [$absolutePath]; } - - return $indexDependencies; - } - - /** - * Check status of the entity declaration. - * - * @param array $entityDeclaration - * @return bool - */ - private function isEntityDisabled(array $entityDeclaration): bool - { - return isset($entityDeclaration['disabled']) && $entityDeclaration['disabled'] == true; - } - - /** - * Retrive dependency id. - * - * @param string $tableName - * @param string $entityType - * @param null|string $entityName - * @return string - */ - private function getDependencyId( - string $tableName, - string $entityType = self::SCHEMA_ENTITY_TABLE, - ?string $entityName = null - ) { - return implode('___', [$tableName, $entityType, $entityName ?: $tableName]); - } - - /** - * Retrieve dependency parameters from dependency id. - * - * @param string $id - * @return array - */ - private function decodeDependencyId(string $id): array - { - $decodedValues = explode('___', $id); - $result = [ - 'tableName' => $decodedValues[0], - 'entityType' => $decodedValues[1], - 'entityName' => $decodedValues[2], - ]; return $result; } @@ -573,9 +107,9 @@ private function decodeDependencyId(string $id): array */ private function getErrorMessage(string $id): string { - $decodedId = $this->decodeDependencyId($id); + $decodedId = $this->dependencyProvider->decodeDependencyId($id); $entityType = $decodedId['entityType']; - if ($entityType === self::SCHEMA_ENTITY_TABLE) { + if ($entityType === DeclarativeSchemaDependencyProvider::SCHEMA_ENTITY_TABLE) { $message = sprintf( 'Table %s has undeclared dependency on one of the next modules.', $decodedId['tableName'] @@ -593,192 +127,19 @@ private function getErrorMessage(string $id): string } /** - * Collect module dependencies. - * - * @param string $currentModuleName - * @param array $dependencies - * @return array - */ - private function collectDependencies($currentModuleName, $dependencies = []) - { - if (!count($dependencies)) { - return []; - } - foreach ($dependencies as $dependencyName => $dependency) { - $this->collectDependency($dependencyName, $dependency, $currentModuleName); - } - - return $this->getDeclaredDependencies($currentModuleName, self::TYPE_HARD, self::MAP_TYPE_FOUND); - } - - /** - * Collect a module dependency. - * - * @param string $dependencyName - * @param array $dependency - * @param string $currentModule - */ - private function collectDependency( - string $dependencyName, - array $dependency, - string $currentModule - ) { - $declared = $this->getDeclaredDependencies($currentModule, self::TYPE_HARD, self::MAP_TYPE_DECLARED); - $checkResult = array_intersect($declared, $dependency); - - if (empty($checkResult)) { - $this->addDependencies( - $currentModule, - self::TYPE_HARD, - self::MAP_TYPE_FOUND, - [ - $dependencyName => $dependency, - ] - ); - } - } - - /** - * Convert file list to data provider structure. - * - * @param string[] $files - * @return array - */ - private function prepareFiles(array $files): array - { - $result = []; - foreach ($files as $relativePath => $file) { - $absolutePath = reset($file); - $result[$relativePath] = [$absolutePath]; - } - return $result; - } - - /** - * Add dependencies to dependency list. - * - * @param string $moduleName - * @param array $packageNames - * @param string $type - * - * @return void - * @throws \Exception - */ - private function presetDependencies( - string $moduleName, - array $packageNames, - string $type - ): void { - $packageNames = array_filter($packageNames, function ($packageName) { - return $this->getModuleName($packageName) || - 0 === strpos($packageName, 'magento/') && 'magento/magento-composer-installer' != $packageName; - }); - - foreach ($packageNames as $packageName) { - $this->addDependencies( - $moduleName, - $type, - self::MAP_TYPE_DECLARED, - [$this->convertModuleName($packageName)] - ); - } - } - - /** - * Converts a composer json component name into the Magento Module form. - * - * @param string $jsonName The name of a composer json component or dependency e.g. 'magento/module-theme' - * @return string The corresponding Magento Module e.g. 'Magento\Theme' - * @throws \Exception - */ - private function convertModuleName(string $jsonName): string - { - $moduleName = $this->getModuleName($jsonName); - if ($moduleName) { - return $moduleName; - } - - if ( - strpos($jsonName, 'magento/magento') !== false - || strpos($jsonName, 'magento/framework') !== false - ) { - $moduleName = str_replace('/', "\t", $jsonName); - $moduleName = str_replace('framework-', "Framework\t", $moduleName); - $moduleName = str_replace('-', ' ', $moduleName); - $moduleName = ucwords($moduleName); - $moduleName = str_replace("\t", '\\', $moduleName); - $moduleName = str_replace(' ', '', $moduleName); - } else { - $moduleName = $jsonName; - } - - return $moduleName; - } - - /** - * Returns package name on module name mapping. + * Read data from json file. * - * @return array + * @param string $file + * @return mixed * @throws \Exception */ - private function getPackageModuleMapping(): array + private function readJsonFile(string $file, bool $asArray = false) { - if (!$this->packageModuleMapping) { - $jsonFiles = Files::init()->getComposerFiles(ComponentRegistrar::MODULE, false); - - $packageModuleMapping = []; - foreach ($jsonFiles as $file) { - $moduleXml = simplexml_load_file(dirname($file) . '/etc/module.xml'); - $moduleName = str_replace('_', '\\', (string)$moduleXml->module->attributes()->name); - $composerJson = $this->readJsonFile($file); - $packageName = $composerJson->name; - $packageModuleMapping[$packageName] = $moduleName; - } - - $this->packageModuleMapping = $packageModuleMapping; + $decodedJson = json_decode(file_get_contents($file), $asArray); + if (null == $decodedJson) { + throw new \Exception("Invalid Json: $file"); } - return $this->packageModuleMapping; - } - - /** - * Retrive Magento style module name. - * - * @param string $packageName - * @return null|string - * @throws \Exception - */ - private function getModuleName(string $packageName): ?string - { - return $this->getPackageModuleMapping()[$packageName] ?? null; - } - - /** - * Retrieve array of dependency items. - * - * @param $module - * @param $type - * @param $mapType - * @return array - */ - private function getDeclaredDependencies(string $module, string $type, string $mapType) - { - return $this->mapDependencies[$module][$type][$mapType] ?? []; - } - - /** - * Add dependency map items. - * - * @param $module - * @param $type - * @param $mapType - * @param $dependencies - */ - protected function addDependencies(string $module, string $type, string $mapType, array $dependencies) - { - $this->mapDependencies[$module][$type][$mapType] = array_merge_recursive( - $this->getDeclaredDependencies($module, $type, $mapType), - $dependencies - ); + return $decodedJson; } -} \ No newline at end of file +} diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php b/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php new file mode 100644 index 0000000000000..b9050721c6910 --- /dev/null +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php @@ -0,0 +1,759 @@ +initDeclaredDependencies(); + $dependencies = $this->getDependenciesFromFiles($this->getSchemaFileNameByModuleName($moduleName)); + $dependencies = $this->filterSelfDependency($moduleName, $dependencies); + $declared = $this->getDeclaredDependencies($moduleName, self::TYPE_HARD, self::MAP_TYPE_DECLARED); + + $existingDeclared = []; + foreach ($dependencies as $dependency) { + $checkResult = array_intersect($declared, $dependency); + if ($checkResult) { + $existingDeclared = array_merge(array_values($checkResult)); + } + } + + return array_unique($existingDeclared); + } + + /** + * Provide undeclared dependencies between modules based on the declarative schema configuration. + * + * [ + * $dependencyId => [$module1, $module2, $module3 ...], + * ... + * ] + * + * @param string $moduleName + * @return array + * @throws \Exception + */ + public function getUndeclaredModuleDependencies(string $moduleName): array + { + $this->initDeclaredDependencies(); + $dependencies = $this->getDependenciesFromFiles($this->getSchemaFileNameByModuleName($moduleName)); + $dependencies = $this->filterSelfDependency($moduleName, $dependencies); + return $this->collectDependencies($moduleName, $dependencies); + } + + /** + * Provide schema file name by module name. + * + * @param string $module + * @return string + * @throws \Exception + */ + private function getSchemaFileNameByModuleName(string $module): string + { + if (empty($this->moduleSchemaFileMapping)) { + $componentRegistrar = new ComponentRegistrar(); + foreach (array_values(Files::init()->getDbSchemaFiles()) as $filePath) { + $filePath = reset($filePath); + foreach ($componentRegistrar->getPaths(ComponentRegistrar::MODULE) as $moduleName => $moduleDir) { + if (strpos($filePath, $moduleDir . '/') !== false) { + $foundModuleName = str_replace('_', '\\', $moduleName); + $this->moduleSchemaFileMapping[$foundModuleName] = $filePath; + break; + } + } + } + } + + return $this->moduleSchemaFileMapping[$module] ?? ''; + } + + /** + * Initialise map of dependencies. + * + * @throws \Exception + */ + private function initDeclaredDependencies() + { + if (empty($this->mapDependencies)) { + $jsonFiles = Files::init()->getComposerFiles(ComponentRegistrar::MODULE, false); + foreach ($jsonFiles as $file) { + $json = new \Magento\Framework\Config\Composer\Package($this->readJsonFile($file)); + $moduleName = $this->convertModuleName($json->get('name')); + $require = array_keys((array)$json->get('require')); + $this->presetDependencies($moduleName, $require, self::TYPE_HARD); + } + } + } + + /** + * Read data from json file. + * + * @param string $file + * @return mixed + * @throws \Exception + */ + private function readJsonFile(string $file, bool $asArray = false) + { + $decodedJson = json_decode(file_get_contents($file), $asArray); + if (null == $decodedJson) { + throw new \Exception("Invalid Json: $file"); + } + + return $decodedJson; + } + + /** + * Remove self dependencies. + * + * @param string $moduleName + * @param array $dependencies + * @return array + */ + private function filterSelfDependency(string $moduleName, array $dependencies):array + { + foreach ($dependencies as $id => $modules) { + $decodedId = $this->decodeDependencyId($id); + $entityType = $decodedId['entityType']; + if ($entityType === self::SCHEMA_ENTITY_TABLE || $entityType === "column") { + if (array_search($moduleName, $modules) !== false) { + unset($dependencies[$id]); + } + } else { + $dependencies[$id] = $this->filterComplexDependency($moduleName, $modules); + } + } + + return array_filter($dependencies); + } + + /** + * Remove already declared dependencies. + * + * @param string $moduleName + * @param array $modules + * @return array + */ + private function filterComplexDependency(string $moduleName, array $modules): array + { + $resultDependencies = []; + if (!is_array(reset($modules))) { + if (array_search($moduleName, $modules) === false) { + $resultDependencies = $modules; + } + } else { + foreach ($modules as $dependencySet) { + if (array_search($moduleName, $dependencySet) === false) { + $resultDependencies = array_merge( + $resultDependencies, + $dependencySet + ); + } + } + } + + return array_values(array_unique($resultDependencies)); + } + + /** + * Retrieve declarative schema declaration. + * + * @return array + * @throws \Exception + */ + private function getDeclarativeSchema(): array + { + if ($this->dbSchemaDeclaration) { + return $this->dbSchemaDeclaration; + } + + $entityTypes = [self::SCHEMA_ENTITY_COLUMN, self::SCHEMA_ENTITY_CONSTRAINT, self::SCHEMA_ENTITY_INDEX]; + $declaration = []; + foreach (Files::init()->getDbSchemaFiles() as $filePath) { + $filePath = reset($filePath); + preg_match('#app/code/(\w+/\w+)#', $filePath, $result); + $moduleName = str_replace('/', '\\', $result[1]); + $moduleDeclaration = $this->getDbSchemaDeclaration($filePath); + + foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { + if (!isset($tableDeclaration['modules'])) { + $tableDeclaration['modules'] = []; + } + array_push($tableDeclaration['modules'], $moduleName); + $moduleDeclaration = array_replace_recursive( + $moduleDeclaration, + [self::SCHEMA_ENTITY_TABLE => + [ + $tableName => $tableDeclaration, + ] + ] + ); + foreach ($entityTypes as $entityType) { + if (!isset($tableDeclaration[$entityType])) { + continue; + } + $moduleDeclaration = array_replace_recursive( + $moduleDeclaration, + [self::SCHEMA_ENTITY_TABLE => + [ + $tableName => + $this->addModuleAssigment($tableDeclaration, $entityType, $moduleName) + ] + ] + ); + } + } + $declaration = array_merge_recursive($declaration, $moduleDeclaration); + } + $this->dbSchemaDeclaration = $declaration; + + return $this->dbSchemaDeclaration; + } + + /** + * Get declared dependencies. + * + * @param string $tableName + * @param string $entityType + * @param null|string $entityName + * @return array + * @throws \Exception + */ + private function resolveEntityDependencies(string $tableName, string $entityType, ?string $entityName = null): array + { + switch ($entityType) { + case self::SCHEMA_ENTITY_COLUMN: + case self::SCHEMA_ENTITY_CONSTRAINT: + case self::SCHEMA_ENTITY_INDEX: + return $this->getDeclarativeSchema() + [self::SCHEMA_ENTITY_TABLE][$tableName][$entityType][$entityName]['modules']; + break; + case self::SCHEMA_ENTITY_TABLE: + return $this->getDeclarativeSchema()[self::SCHEMA_ENTITY_TABLE][$tableName]['modules']; + break; + default: + return []; + } + } + + /** + * @param string $filePath + * @return array + */ + private function getDbSchemaDeclaration(string $filePath): array + { + $dom = new \DOMDocument(); + $dom->loadXML(file_get_contents($filePath)); + return (new Converter())->convert($dom); + } + + /** + * Add dependency on the current module. + * + * @param array $tableDeclaration + * @param string $entityType + * @param string $moduleName + * @return array + */ + private function addModuleAssigment( + array $tableDeclaration, + string $entityType, + string $moduleName + ): array { + $declarationWithAssigment = []; + foreach ($tableDeclaration[$entityType] as $entityName => $entityDeclaration) { + if (!isset($entityDeclaration['modules'])) { + $entityDeclaration['modules'] = []; + } + if (!$this->isEntityDisabled($entityDeclaration)) { + array_push($entityDeclaration['modules'], $moduleName); + } + + $declarationWithAssigment[$entityType][$entityName] = $entityDeclaration; + } + + return $declarationWithAssigment; + } + + /** + * Retrieve dependencies from files. + * + * @param string $file + * @return string[] + * @throws \Exception + */ + private function getDependenciesFromFiles($file) + { + if (!$file) { + return []; + } + + $moduleDbSchema = $this->getDbSchemaDeclaration($file); + $dependencies = array_merge_recursive( + $this->getDisabledDependencies($moduleDbSchema), + $this->getConstraintDependencies($moduleDbSchema), + $this->getIndexDependencies($moduleDbSchema) + ); + return $dependencies; + } + + /** + * Retrieve dependencies for disabled entities. + * + * @param array $moduleDeclaration + * @return array + * @throws \Exception + */ + private function getDisabledDependencies(array $moduleDeclaration): array + { + $disabledDependencies = []; + $entityTypes = [self::SCHEMA_ENTITY_COLUMN, self::SCHEMA_ENTITY_CONSTRAINT, self::SCHEMA_ENTITY_INDEX]; + foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { + foreach ($entityTypes as $entityType) { + if (!isset($tableDeclaration[$entityType])) { + continue; + } + foreach ($tableDeclaration[$entityType] as $entityName => $entityDeclaration) { + if ($this->isEntityDisabled($entityDeclaration)) { + $dependencyIdentifier = $this->getDependencyId($tableName, $entityType, $entityName); + $disabledDependencies[$dependencyIdentifier] = + $this->resolveEntityDependencies($tableName, $entityType, $entityName); + } + } + } + if ($this->isEntityDisabled($tableDeclaration)) { + $disabledDependencies[$this->getDependencyId($tableName)] = + $this->resolveEntityDependencies($tableName, self::SCHEMA_ENTITY_TABLE); + } + } + + return $disabledDependencies; + } + + /** + * Retrieve dependencies for foreign entities. + * + * @param array $constraintDeclaration + * @return array + * @throws \Exception + */ + private function getFKDependencies(array $constraintDeclaration): array + { + $referenceDependencyIdentifier = + $this->getDependencyId( + $constraintDeclaration['referenceTable'], + self::SCHEMA_ENTITY_CONSTRAINT, + $constraintDeclaration['referenceId'] + ); + $dependencyIdentifier = + $this->getDependencyId( + $constraintDeclaration[self::SCHEMA_ENTITY_TABLE], + self::SCHEMA_ENTITY_CONSTRAINT, + $constraintDeclaration['referenceId'] + ); + + $constraintDependencies = []; + $constraintDependencies[$referenceDependencyIdentifier] = + $this->resolveEntityDependencies( + $constraintDeclaration['referenceTable'], + self::SCHEMA_ENTITY_COLUMN, + $constraintDeclaration['referenceColumn'] + ); + $constraintDependencies[$dependencyIdentifier] = + $this->resolveEntityDependencies( + $constraintDeclaration[self::SCHEMA_ENTITY_TABLE], + self::SCHEMA_ENTITY_COLUMN, + $constraintDeclaration[self::SCHEMA_ENTITY_COLUMN] + ); + + return $constraintDependencies; + } + + /** + * Retrieve dependencies for constraint entities. + * + * @param array $moduleDeclaration + * @return array + * @throws \Exception + */ + private function getConstraintDependencies(array $moduleDeclaration): array + { + $constraintDependencies = []; + foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { + if (empty($tableDeclaration[self::SCHEMA_ENTITY_CONSTRAINT])) { + continue; + } + foreach ($tableDeclaration[self::SCHEMA_ENTITY_CONSTRAINT] as $constraintName => $constraintDeclaration) { + if ($this->isEntityDisabled($constraintDeclaration)) { + continue; + } + $dependencyIdentifier = + $this->getDependencyId($tableName, self::SCHEMA_ENTITY_CONSTRAINT, $constraintName); + switch ($constraintDeclaration['type']) { + case 'foreign': + $constraintDependencies = array_merge( + $constraintDependencies, + $this->getFKDependencies($constraintDeclaration) + ); + break; + case 'primary': + case 'unique': + $constraintDependencies[$dependencyIdentifier] = $this->getComplexDependency( + $tableName, + $constraintDeclaration + ); + } + } + } + return $constraintDependencies; + } + + /** + * Calculate complex dependency. + * + * @param string $tableName + * @param array $entityDeclaration + * @return array + * @throws \Exception + */ + private function getComplexDependency(string $tableName, array $entityDeclaration): array + { + $complexDependency = []; + if (empty($entityDeclaration[self::SCHEMA_ENTITY_COLUMN])) { + return $complexDependency; + } + + if (!is_array($entityDeclaration[self::SCHEMA_ENTITY_COLUMN])) { + $entityDeclaration[self::SCHEMA_ENTITY_COLUMN] = [$entityDeclaration[self::SCHEMA_ENTITY_COLUMN]]; + } + + foreach (array_keys($entityDeclaration[self::SCHEMA_ENTITY_COLUMN]) as $columnName) { + $complexDependency[] = + $this->resolveEntityDependencies($tableName, self::SCHEMA_ENTITY_COLUMN, $columnName); + } + + return array_values($complexDependency); + } + + /** + * Retrieve dependencies for index entities. + * + * @param array $moduleDeclaration + * @return array + * @throws \Exception + */ + private function getIndexDependencies(array $moduleDeclaration): array + { + $indexDependencies = []; + foreach ($moduleDeclaration[self::SCHEMA_ENTITY_TABLE] as $tableName => $tableDeclaration) { + if (empty($tableDeclaration[self::SCHEMA_ENTITY_INDEX])) { + continue; + } + foreach ($tableDeclaration[self::SCHEMA_ENTITY_INDEX] as $indexName => $indexDeclaration) { + if ($this->isEntityDisabled($indexDeclaration)) { + continue; + } + $dependencyIdentifier = + $this->getDependencyId($tableName, self::SCHEMA_ENTITY_INDEX, $indexName); + $indexDependencies[$dependencyIdentifier] = + $this->getComplexDependency($tableName, $indexDeclaration); + } + } + + return $indexDependencies; + } + + /** + * Check status of the entity declaration. + * + * @param array $entityDeclaration + * @return bool + */ + private function isEntityDisabled(array $entityDeclaration): bool + { + return isset($entityDeclaration['disabled']) && $entityDeclaration['disabled'] == true; + } + + /** + * Retrieve dependency id. + * + * @param string $tableName + * @param string $entityType + * @param null|string $entityName + * @return string + */ + private function getDependencyId( + string $tableName, + string $entityType = self::SCHEMA_ENTITY_TABLE, + ?string $entityName = null + ) { + return implode('___', [$tableName, $entityType, $entityName ?: $tableName]); + } + + /** + * Retrieve dependency parameters from dependency id. + * + * @param string $id + * @return array + */ + public static function decodeDependencyId(string $id): array + { + $decodedValues = explode('___', $id); + $result = [ + 'tableName' => $decodedValues[0], + 'entityType' => $decodedValues[1], + 'entityName' => $decodedValues[2], + ]; + return $result; + } + + /** + * Collect module dependencies. + * + * @param string $currentModuleName + * @param array $dependencies + * @return array + */ + private function collectDependencies($currentModuleName, $dependencies = []) + { + if (!count($dependencies)) { + return []; + } + foreach ($dependencies as $dependencyName => $dependency) { + $this->collectDependency($dependencyName, $dependency, $currentModuleName); + } + + return $this->getDeclaredDependencies($currentModuleName, self::TYPE_HARD, self::MAP_TYPE_FOUND); + } + + /** + * Collect a module dependency. + * + * @param string $dependencyName + * @param array $dependency + * @param string $currentModule + */ + private function collectDependency( + string $dependencyName, + array $dependency, + string $currentModule + ) { + $declared = $this->getDeclaredDependencies($currentModule, self::TYPE_HARD, self::MAP_TYPE_DECLARED); + $checkResult = array_intersect($declared, $dependency); + + if (empty($checkResult)) { + $this->addDependencies( + $currentModule, + self::TYPE_HARD, + self::MAP_TYPE_FOUND, + [ + $dependencyName => $dependency, + ] + ); + } + } + + /** + * Add dependencies to dependency list. + * + * @param string $moduleName + * @param array $packageNames + * @param string $type + * + * @return void + * @throws \Exception + */ + private function presetDependencies( + string $moduleName, + array $packageNames, + string $type + ): void { + $packageNames = array_filter($packageNames, function ($packageName) { + return $this->getModuleName($packageName) || + 0 === strpos($packageName, 'magento/') && 'magento/magento-composer-installer' != $packageName; + }); + + foreach ($packageNames as $packageName) { + $this->addDependencies( + $moduleName, + $type, + self::MAP_TYPE_DECLARED, + [$this->convertModuleName($packageName)] + ); + } + } + + /** + * Returns package name on module name mapping. + * + * @return array + * @throws \Exception + */ + private function getPackageModuleMapping(): array + { + if (!$this->packageModuleMapping) { + $jsonFiles = Files::init()->getComposerFiles(ComponentRegistrar::MODULE, false); + + $packageModuleMapping = []; + foreach ($jsonFiles as $file) { + $moduleXml = simplexml_load_file(dirname($file) . '/etc/module.xml'); + $moduleName = str_replace('_', '\\', (string)$moduleXml->module->attributes()->name); + $composerJson = $this->readJsonFile($file); + $packageName = $composerJson->name; + $packageModuleMapping[$packageName] = $moduleName; + } + + $this->packageModuleMapping = $packageModuleMapping; + } + + return $this->packageModuleMapping; + } + + /** + * Retrieve Magento style module name. + * + * @param string $packageName + * @return null|string + * @throws \Exception + */ + private function getModuleName(string $packageName): ?string + { + return $this->getPackageModuleMapping()[$packageName] ?? null; + } + + /** + * Retrieve array of dependency items. + * + * @param $module + * @param $type + * @param $mapType + * @return array + */ + private function getDeclaredDependencies(string $module, string $type, string $mapType) + { + return $this->mapDependencies[$module][$type][$mapType] ?? []; + } + + /** + * Add dependency map items. + * + * @param $module + * @param $type + * @param $mapType + * @param $dependencies + */ + protected function addDependencies(string $module, string $type, string $mapType, array $dependencies) + { + $this->mapDependencies[$module][$type][$mapType] = array_merge_recursive( + $this->getDeclaredDependencies($module, $type, $mapType), + $dependencies + ); + } + + /** + * Converts a composer json component name into the Magento Module form. + * + * @param string $jsonName The name of a composer json component or dependency e.g. 'magento/module-theme' + * @return string The corresponding Magento Module e.g. 'Magento\Theme' + * @throws \Exception + */ + private function convertModuleName(string $jsonName): string + { + $moduleName = $this->getModuleName($jsonName); + if ($moduleName) { + return $moduleName; + } + + if (strpos($jsonName, 'magento/magento') !== false + || strpos($jsonName, 'magento/framework') !== false + ) { + $moduleName = str_replace('/', "\t", $jsonName); + $moduleName = str_replace('framework-', "Framework\t", $moduleName); + $moduleName = str_replace('-', ' ', $moduleName); + $moduleName = ucwords($moduleName); + $moduleName = str_replace("\t", '\\', $moduleName); + $moduleName = str_replace(' ', '', $moduleName); + } else { + $moduleName = $jsonName; + } + + return $moduleName; + } +} \ No newline at end of file diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php index a4113abed8030..83dd58ce45bdd 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php @@ -10,8 +10,8 @@ use Magento\Framework\App\Utility\Files; use Magento\Framework\Component\ComponentRegistrar; +use Magento\Test\Integrity\Dependency\DeclarativeSchemaDependencyProvider; use Magento\TestFramework\Dependency\DbRule; -use Magento\TestFramework\Dependency\DeclarativeSchemaRule; use Magento\TestFramework\Dependency\DiRule; use Magento\TestFramework\Dependency\LayoutRule; use Magento\TestFramework\Dependency\PhpRule; @@ -25,19 +25,28 @@ class DependencyTest extends \PHPUnit\Framework\TestCase { /** - * Types of dependencies between modules + * Soft dependency between modules */ const TYPE_SOFT = 'soft'; + /** + * Hard dependency between modules + */ const TYPE_HARD = 'hard'; /** - * Types of dependencies map arrays + * The identifier of dependency for mapping. */ const MAP_TYPE_DECLARED = 'declared'; + /** + * The identifier of dependency for mapping. + */ const MAP_TYPE_FOUND = 'found'; + /** + * The identifier of dependency for mapping. + */ const MAP_TYPE_REDUNDANT = 'redundant'; /** @@ -56,17 +65,6 @@ class DependencyTest extends \PHPUnit\Framework\TestCase */ protected static $_listConfigXml = []; - /** - * List of db_schema.xml files by modules - * - * Format: array( - * '{Module_Name}' => '{Filename}' - * ) - * - * @var array - */ - protected static $_listDbSchemaXml = []; - /** * List of routes.xml files by modules * @@ -174,7 +172,6 @@ public static function setUpBeforeClass() self::$_namespaces = implode('|', Files::init()->getNamespaces()); self::_prepareListConfigXml(); - self::_prepareListDbSchemaXml(); self::_prepareListRoutesXml(); self::_prepareMapRouters(); @@ -230,7 +227,6 @@ protected static function _initRules() $dbRuleTables = array_merge($dbRuleTables, @include $fileName); } self::$_rulesInstances = [ - new DeclarativeSchemaRule($dbRuleTables), new PhpRule(self::$_mapRouters, self::$_mapLayoutBlocks), new DbRule($dbRuleTables), new LayoutRule( @@ -261,7 +257,6 @@ protected function _getCleanedFileContents($fileType, $file) break; case 'layout': case 'config': - case 'db_schema': //Removing xml comments $contents = preg_replace('~\~s', '', $contents); break; @@ -285,6 +280,9 @@ function ($matches) use ($contents, &$contentsWithoutHtml) { return $contents; } + /** + * @inheritdoc + */ public function testUndeclared() { $invoker = new \Magento\Framework\App\Utility\AggregateInvoker($this); @@ -416,17 +414,21 @@ private function collectDependency($dependency, $currentModule, &$undeclared) /** * Collect redundant dependencies + * * @SuppressWarnings(PHPMD.NPathComplexity) * @test * @depends testUndeclared + * @throws \Exception */ public function collectRedundant() { + $schemaDependencyProvider = new DeclarativeSchemaDependencyProvider(); foreach (array_keys(self::$mapDependencies) as $module) { $declared = $this->_getDependencies($module, self::TYPE_HARD, self::MAP_TYPE_DECLARED); $found = array_merge( $this->_getDependencies($module, self::TYPE_HARD, self::MAP_TYPE_FOUND), - $this->_getDependencies($module, self::TYPE_SOFT, self::MAP_TYPE_FOUND) + $this->_getDependencies($module, self::TYPE_SOFT, self::MAP_TYPE_FOUND), + $schemaDependencyProvider->getDeclaredExistingModuleDependencies($module) ); $found['Magento\Framework'] = 'Magento\Framework'; $this->_setDependencies($module, self::TYPE_HARD, self::MAP_TYPE_REDUNDANT, array_diff($declared, $found)); @@ -487,6 +489,7 @@ protected function _prepareFiles($fileType, $files, $skip = null) * Return all files * * @return array + * @throws \Exception */ public function getAllFiles() { @@ -508,12 +511,6 @@ public function getAllFiles() $this->_prepareFiles('config', Files::init()->getConfigFiles()) ); - // Get all configuration files - $files = array_merge( - $files, - $this->_prepareFiles('db_schema', Files::init()->getDbSchemaFiles()) - ); - //Get all layout updates files $files = array_merge( $files, @@ -543,20 +540,6 @@ protected static function _prepareListConfigXml() } } - /** - * Prepare list of db_schema.xml files (by modules) - */ - protected static function _prepareListDbSchemaXml() - { - $files = Files::init()->getDbSchemaFiles('db_schema.xml', [], false); - foreach ($files as $file) { - if (preg_match('/(?[A-Z][a-z]+)[_\/\\\\](?[A-Z][a-zA-Z]+)/', $file, $matches)) { - $module = $matches['namespace'] . '\\' . $matches['module']; - self::$_listDbSchemaXml[$module] = $file; - } - } - } - /** * Prepare list of routes.xml files (by modules) */ From 22bd9b2ab045067886855553d0393ee6c847b57d Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Mon, 12 Nov 2018 09:16:41 -0600 Subject: [PATCH 03/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../Dependency/DeclarativeSchemaRuleTest.php | 98 ------------------- 1 file changed, 98 deletions(-) delete mode 100644 dev/tests/static/framework/tests/unit/testsuite/Magento/TestFramework/Dependency/DeclarativeSchemaRuleTest.php diff --git a/dev/tests/static/framework/tests/unit/testsuite/Magento/TestFramework/Dependency/DeclarativeSchemaRuleTest.php b/dev/tests/static/framework/tests/unit/testsuite/Magento/TestFramework/Dependency/DeclarativeSchemaRuleTest.php deleted file mode 100644 index 269eff8087a91..0000000000000 --- a/dev/tests/static/framework/tests/unit/testsuite/Magento/TestFramework/Dependency/DeclarativeSchemaRuleTest.php +++ /dev/null @@ -1,98 +0,0 @@ -model = new DeclarativeSchemaRule(['some_table' => 'SomeModule']); - } - - /** - * @param string $module - * @param string $file - * @param string $contents - * @param array $expected - * @dataProvider getDependencyInfoDataProvider - */ - public function testGetDependencyInfo($module, $file, $contents, array $expected) - { - $actualDependencies = $this->model->getDependencyInfo($module, 'db_schema', $file, $contents); - $this->assertEquals( - $expected, - $actualDependencies - ); - } - - public function getDependencyInfoDataProvider() - { - return [ - ['any', 'non-db-schema-file.php', 'any', []], - [ - 'any', - '/app/Magento/Module/etc/db_schema.xml', - '
', - [['module' => 'Unknown', 'source' => 'unknown_table']] - ], - [ - 'SomeModule', - '/app/some/path/etc/db_schema.xml', - '
', - [] - ], - [ - 'any', - '/app/some/path/etc/db_schema.xml', - '
', - [ - [ - 'module' => 'SomeModule', - 'type' => \Magento\TestFramework\Dependency\RuleInterface::TYPE_HARD, - 'source' => 'some_table', - ] - ] - ], - [ - 'any', - '/app/some/path/etc/db_schema.xml', - ' - - -
- ', - [ - [ - 'module' => 'SomeModule', - 'type' => \Magento\TestFramework\Dependency\RuleInterface::TYPE_HARD, - 'source' => 'some_table', - ], - [ - 'module' => 'Unknown', - 'source' => 'ref_table', - ] - ] - ] - ]; - } -} From 2c92422b0e5ff6336a4eadbfe3127acd73a7a8a5 Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Fri, 30 Nov 2018 11:21:04 -0600 Subject: [PATCH 04/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../TestFramework/Dependency/DiRule.php | 30 ++++++++++++++----- .../DeclarativeSchemaDependencyProvider.php | 4 +-- .../_files/dependency_test/tables_ce.php | 2 +- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/dev/tests/static/framework/Magento/TestFramework/Dependency/DiRule.php b/dev/tests/static/framework/Magento/TestFramework/Dependency/DiRule.php index cdaa49e8d37fb..8eda14dd28165 100644 --- a/dev/tests/static/framework/Magento/TestFramework/Dependency/DiRule.php +++ b/dev/tests/static/framework/Magento/TestFramework/Dependency/DiRule.php @@ -7,11 +7,13 @@ */ namespace Magento\TestFramework\Dependency; -use DOMDocument; -use DOMXPath; +use Magento\Framework\App\Utility\Classes; use Magento\Framework\App\Utility\Files; use Magento\TestFramework\Dependency\VirtualType\VirtualTypeMapper; +/** + * Class provide dependency rule for di.xml config files. + */ class DiRule implements RuleInterface { /** @@ -33,6 +35,8 @@ public function __construct(VirtualTypeMapper $mapper) } /** + * Get class name pattern. + * * @return string * @throws \Exception */ @@ -99,12 +103,14 @@ public function getDependencyInfo($currentModule, $fileType, $file, &$contents) } /** + * Fetch all possible dependencies. + * * @param string $contents * @return array */ private function fetchPossibleDependencies($contents) { - $doc = new DOMDocument(); + $doc = new \DOMDocument(); $doc->loadXML($contents); return [ RuleInterface::TYPE_SOFT => $this->getSoftDependencies($doc), @@ -113,16 +119,22 @@ private function fetchPossibleDependencies($contents) } /** - * @param DOMDocument $doc + * Collect soft dependencies. + * + * @param \DOMDocument $doc * @return array */ - private function getSoftDependencies(DOMDocument $doc) + private function getSoftDependencies(\DOMDocument $doc) { $result = []; foreach (self::$tagNameMap as $tagName => $attributeNames) { $nodes = $doc->getElementsByTagName($tagName); /** @var \DOMElement $node */ foreach ($nodes as $node) { + if ($tagName === 'virtualType' && !$node->getAttribute('type')) { + $result[] = Classes::resolveVirtualType($node->getAttribute('name')); + continue; + } foreach ($attributeNames as $attributeName) { $result[] = $node->getAttribute($attributeName); } @@ -133,13 +145,15 @@ private function getSoftDependencies(DOMDocument $doc) } /** - * @param DOMDocument $doc + * Collect hard dependencies. + * + * @param \DOMDocument $doc * @return array */ - private function getHardDependencies(DOMDocument $doc) + private function getHardDependencies(\DOMDocument $doc) { $result = []; - $xpath = new DOMXPath($doc); + $xpath = new \DOMXPath($doc); $textNodes = $xpath->query('//*[@xsi:type="object"]'); /** @var \DOMElement $node */ foreach ($textNodes as $node) { diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php b/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php index b9050721c6910..f406291603757 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php @@ -92,7 +92,7 @@ public function getDeclaredExistingModuleDependencies(string $moduleName): array foreach ($dependencies as $dependency) { $checkResult = array_intersect($declared, $dependency); if ($checkResult) { - $existingDeclared = array_merge(array_values($checkResult)); + $existingDeclared = array_merge($existingDeclared, array_values($checkResult)); } } @@ -756,4 +756,4 @@ private function convertModuleName(string $jsonName): string return $moduleName; } -} \ No newline at end of file +} diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/tables_ce.php b/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/tables_ce.php index 530f55504d009..3fb53be2ec400 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/tables_ce.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/tables_ce.php @@ -96,7 +96,7 @@ 'catalogrule_website' => 'Magento\CatalogRule', 'catalogsearch_fulltext' => 'Magento\CatalogSearch', 'catalogsearch_result' => 'Magento\CatalogSearch', - 'search_query' => 'Magento\CatalogSearch', + 'search_query' => 'Magento\Search', 'checkout_agreement' => 'Magento\Checkout', 'checkout_agreement_store' => 'Magento\Checkout', 'cms_block' => 'Magento\Cms', From 8485985b61ac4633943fe6911f4af68f90cce05f Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Sun, 2 Dec 2018 01:29:42 -0600 Subject: [PATCH 05/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../Magento/Test/Integrity/DependencyTest.php | 15 ++++++++++++++- .../undetected_dependencies_ce.php | 10 ++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/undetected_dependencies_ce.php diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php index 83dd58ce45bdd..3b1e4aecedb54 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php @@ -221,7 +221,7 @@ protected static function _initThemes() */ protected static function _initRules() { - $replaceFilePattern = str_replace('\\', '/', realpath(__DIR__)) . '/_files/dependency_test/*.php'; + $replaceFilePattern = str_replace('\\', '/', realpath(__DIR__)) . '/_files/dependency_test/tables_*.php'; $dbRuleTables = []; foreach (glob($replaceFilePattern) as $fileName) { $dbRuleTables = array_merge($dbRuleTables, @include $fileName); @@ -423,6 +423,14 @@ private function collectDependency($dependency, $currentModule, &$undeclared) public function collectRedundant() { $schemaDependencyProvider = new DeclarativeSchemaDependencyProvider(); + + /** TODO: Remove this temporary solution after MC-5806 is closed */ + $filePattern = __DIR__ . '/_files/dependency_test/undetected_dependencies_*.php'; + $undetectedDependencies = []; + foreach (glob($filePattern) as $fileName) { + $undetectedDependencies = array_merge($undetectedDependencies, require $fileName); + } + foreach (array_keys(self::$mapDependencies) as $module) { $declared = $this->_getDependencies($module, self::TYPE_HARD, self::MAP_TYPE_DECLARED); $found = array_merge( @@ -430,6 +438,11 @@ public function collectRedundant() $this->_getDependencies($module, self::TYPE_SOFT, self::MAP_TYPE_FOUND), $schemaDependencyProvider->getDeclaredExistingModuleDependencies($module) ); + /** TODO: Remove this temporary solution after MC-5806 is closed */ + if (!empty($undetectedDependencies[$module])) { + $found = array_merge($found, $undetectedDependencies[$module]); + } + $found['Magento\Framework'] = 'Magento\Framework'; $this->_setDependencies($module, self::TYPE_HARD, self::MAP_TYPE_REDUNDANT, array_diff($declared, $found)); } diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/undetected_dependencies_ce.php b/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/undetected_dependencies_ce.php new file mode 100644 index 0000000000000..407f57ee51257 --- /dev/null +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/undetected_dependencies_ce.php @@ -0,0 +1,10 @@ + ["Magento\CatalogSearch"] +]; From 422e459f6cd012bd381c029915c52dbdfe2d5815 Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Fri, 8 Feb 2019 17:08:33 -0600 Subject: [PATCH 06/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../PageCache/Test/Mftf/Test/NewProductsListWidgetTest.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/code/Magento/PageCache/Test/Mftf/Test/NewProductsListWidgetTest.xml b/app/code/Magento/PageCache/Test/Mftf/Test/NewProductsListWidgetTest.xml index c5871ddc3a373..a3c9e7b39217d 100644 --- a/app/code/Magento/PageCache/Test/Mftf/Test/NewProductsListWidgetTest.xml +++ b/app/code/Magento/PageCache/Test/Mftf/Test/NewProductsListWidgetTest.xml @@ -9,7 +9,7 @@ - + @@ -18,7 +18,7 @@ - + From c8cd9e56dc9b8b32c69489d7ab5e96e159b691b4 Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Tue, 26 Mar 2019 13:47:12 -0500 Subject: [PATCH 07/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../Integrity/DeclarativeDependencyTest.php | 35 ++++++++++++++++++- .../blacklisted_dependencies_ce.php | 9 +++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/blacklisted_dependencies_ce.php diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php index 257da9669d9b5..a7fb86e7dff29 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php @@ -22,6 +22,11 @@ class DeclarativeDependencyTest extends \PHPUnit\Framework\TestCase */ private $dependencyProvider; + /** + * @var array + */ + private $blacklistedDependencies = []; + /** * Sets up data * @@ -45,6 +50,14 @@ protected function setUp() */ public function testUndeclaredDependencies() { + /** TODO: Remove this temporary solution after MC-15534 is closed */ + $filePattern = __DIR__ . '/_files/dependency_test/blacklisted_dependencies_*.php'; + $blacklistedDependencies = []; + foreach (glob($filePattern) as $fileName) { + $blacklistedDependencies = array_merge($blacklistedDependencies, require $fileName); + } + $this->blacklistedDependencies = $blacklistedDependencies; + $invoker = new \Magento\Framework\App\Utility\AggregateInvoker($this); $invoker( /** @@ -71,7 +84,9 @@ function ($file) { $result = []; foreach ($undeclaredDependency as $name => $modules) { $modules = array_unique($modules); - $result[] = $this->getErrorMessage($name) . "\n" . implode("\t\n", $modules); + if ($this->filterBlacklistedDependencies($foundModuleName, $modules)) { + $result[] = $this->getErrorMessage($name) . "\n" . implode("\t\n", $modules); + } } if (count($result)) { $this->fail( @@ -83,6 +98,24 @@ function ($file) { ); } + /** + * Filter blacklisted dependencies. + * + * @todo Remove this temporary solution after MC-15534 is closed + * + * @param string $moduleName + * @param array $dependencies + * @return array + */ + private function filterBlacklistedDependencies(string $moduleName, array $dependencies): array + { + if (!empty($this->blacklistedDependencies[$moduleName])) { + $dependencies = array_diff($dependencies, $this->blacklistedDependencies[$moduleName]); + } + + return $dependencies; + } + /** * Convert file list to data provider structure. * diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/blacklisted_dependencies_ce.php b/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/blacklisted_dependencies_ce.php new file mode 100644 index 0000000000000..270cb99c29caa --- /dev/null +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/_files/dependency_test/blacklisted_dependencies_ce.php @@ -0,0 +1,9 @@ + ["Magento\Inventory"], +]; From d49445a7ff25ec1e7d8413ebeceaa06432786d05 Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Thu, 28 Mar 2019 11:35:16 -0500 Subject: [PATCH 08/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../Magento/Test/Integrity/DeclarativeDependencyTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php index a7fb86e7dff29..b19caa650e5e7 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php @@ -85,7 +85,7 @@ function ($file) { foreach ($undeclaredDependency as $name => $modules) { $modules = array_unique($modules); if ($this->filterBlacklistedDependencies($foundModuleName, $modules)) { - $result[] = $this->getErrorMessage($name) . "\n" . implode("\t\n", $modules); + $result[] = $this->getErrorMessage($name) . "\n" . implode("\t\n", $modules) . "\n"; } } if (count($result)) { @@ -144,12 +144,12 @@ private function getErrorMessage(string $id): string $entityType = $decodedId['entityType']; if ($entityType === DeclarativeSchemaDependencyProvider::SCHEMA_ENTITY_TABLE) { $message = sprintf( - 'Table %s has undeclared dependency on one of the next modules.', + 'Table %s has undeclared dependency on one of the following modules:', $decodedId['tableName'] ); } else { $message = sprintf( - '%s %s from %s table has undeclared dependency on one of the next modules.', + '%s %s from %s table has undeclared dependency on one of the following modules:', ucfirst($entityType), $decodedId['entityName'], $decodedId['tableName'] From aa5b19e7c2371e3af787c1afe1ae96348eb21377 Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Fri, 29 Mar 2019 17:12:56 -0500 Subject: [PATCH 09/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../TestFramework/Dependency/DiRule.php | 1 + .../Integrity/DeclarativeDependencyTest.php | 3 ++- .../DeclarativeSchemaDependencyProvider.php | 5 ++--- .../Magento/Test/Integrity/DependencyTest.php | 22 +++++++++++-------- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/dev/tests/static/framework/Magento/TestFramework/Dependency/DiRule.php b/dev/tests/static/framework/Magento/TestFramework/Dependency/DiRule.php index 8eda14dd28165..b3ee0a2880308 100644 --- a/dev/tests/static/framework/Magento/TestFramework/Dependency/DiRule.php +++ b/dev/tests/static/framework/Magento/TestFramework/Dependency/DiRule.php @@ -77,6 +77,7 @@ private function getPattern() */ public function getDependencyInfo($currentModule, $fileType, $file, &$contents) { + //phpcs:ignore Magento2.Functions.DiscouragedFunction if (pathinfo($file, PATHINFO_BASENAME) !== 'di.xml') { return []; } diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php index b19caa650e5e7..87cc5afd5ecb3 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DeclarativeDependencyTest.php @@ -88,7 +88,7 @@ function ($file) { $result[] = $this->getErrorMessage($name) . "\n" . implode("\t\n", $modules) . "\n"; } } - if (count($result)) { + if (!empty($result)) { $this->fail( 'Module ' . $moduleName . ' has undeclared dependencies: ' . "\n" . implode("\t\n", $result) ); @@ -170,6 +170,7 @@ private function readJsonFile(string $file, bool $asArray = false) { $decodedJson = json_decode(file_get_contents($file), $asArray); if (null == $decodedJson) { + //phpcs:ignore Magento2.Exceptions.DirectThrow throw new \Exception("Invalid Json: $file"); } diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php b/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php index f406291603757..965bc6184144b 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/Dependency/DeclarativeSchemaDependencyProvider.php @@ -174,6 +174,7 @@ private function readJsonFile(string $file, bool $asArray = false) { $decodedJson = json_decode(file_get_contents($file), $asArray); if (null == $decodedJson) { + //phpcs:ignore Magento2.Exceptions.DirectThrow throw new \Exception("Invalid Json: $file"); } @@ -304,10 +305,8 @@ private function resolveEntityDependencies(string $tableName, string $entityType case self::SCHEMA_ENTITY_INDEX: return $this->getDeclarativeSchema() [self::SCHEMA_ENTITY_TABLE][$tableName][$entityType][$entityName]['modules']; - break; case self::SCHEMA_ENTITY_TABLE: return $this->getDeclarativeSchema()[self::SCHEMA_ENTITY_TABLE][$tableName]['modules']; - break; default: return []; } @@ -593,7 +592,7 @@ public static function decodeDependencyId(string $id): array */ private function collectDependencies($currentModuleName, $dependencies = []) { - if (!count($dependencies)) { + if (empty($dependencies)) { return []; } foreach ($dependencies as $dependencyName => $dependency) { diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php index 3e50103445987..35819ff212954 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php @@ -221,6 +221,7 @@ protected static function _initThemes() $defaultThemes = []; foreach (self::$_listConfigXml as $file) { $config = simplexml_load_file($file); + //phpcs:ignore Generic.PHP.NoSilencedErrors $nodes = @($config->xpath("/config/*/design/theme/full_name") ?: []); foreach ($nodes as $node) { $defaultThemes[] = (string)$node; @@ -237,6 +238,7 @@ protected static function _initRules() $replaceFilePattern = str_replace('\\', '/', realpath(__DIR__)) . '/_files/dependency_test/tables_*.php'; $dbRuleTables = []; foreach (glob($replaceFilePattern) as $fileName) { + //phpcs:ignore Generic.PHP.NoSilencedErrors $dbRuleTables = array_merge($dbRuleTables, @include $fileName); } self::$_rulesInstances = [ @@ -339,12 +341,12 @@ function ($fileType, $file) { $result = []; foreach ($undeclaredDependency as $type => $modules) { $modules = array_unique($modules); - if (!count($modules)) { + if (empty($modules)) { continue; } $result[] = sprintf("%s [%s]", $type, implode(', ', $modules)); } - if (count($result)) { + if (!empty($result)) { $this->fail('Module ' . $module . ' has undeclared dependencies: ' . implode(', ', $result)); } }, @@ -473,7 +475,7 @@ public function testRedundant() foreach (array_keys(self::$mapDependencies) as $module) { $result = []; $redundant = $this->_getDependencies($module, self::TYPE_HARD, self::MAP_TYPE_REDUNDANT); - if (count($redundant)) { + if (!empty($redundant)) { $result[] = sprintf( "\r\nModule %s: %s [%s]", $module, @@ -482,11 +484,11 @@ public function testRedundant() ); } - if (count($result)) { + if (!empty($result)) { $output[] = implode(', ', $result); } } - if (count($output)) { + if (!empty($output)) { $this->fail("Redundant dependencies found!\r\n" . implode(' ', $output)); } } @@ -654,7 +656,7 @@ protected static function _prepareMapLayoutBlocks() $area = 'default'; if (preg_match('/[\/](?adminhtml|frontend)[\/]/', $file, $matches)) { $area = $matches['area']; - self::$_mapLayoutBlocks[$area] = @(self::$_mapLayoutBlocks[$area] ?: []); + self::$_mapLayoutBlocks[$area] = self::$_mapLayoutBlocks[$area] ?? []; } if (preg_match('/(?[A-Z][a-z]+)[_\/\\\\](?[A-Z][a-zA-Z]+)/', $file, $matches)) { $module = $matches['namespace'] . '\\' . $matches['module']; @@ -664,7 +666,7 @@ protected static function _prepareMapLayoutBlocks() $attributes = $element->attributes(); $block = (string)$attributes->name; if (!empty($block)) { - self::$_mapLayoutBlocks[$area][$block] = @(self::$_mapLayoutBlocks[$area][$block] ?: []); + self::$_mapLayoutBlocks[$area][$block] = self::$_mapLayoutBlocks[$area][$block] ?? []; self::$_mapLayoutBlocks[$area][$block][$module] = $module; } } @@ -682,7 +684,7 @@ protected static function _prepareMapLayoutHandles() $area = 'default'; if (preg_match('/\/(?adminhtml|frontend)\//', $file, $matches)) { $area = $matches['area']; - self::$_mapLayoutHandles[$area] = @(self::$_mapLayoutHandles[$area] ?: []); + self::$_mapLayoutHandles[$area] = self::$_mapLayoutHandles[$area] ?? []; } if (preg_match('/app\/code\/(?[A-Z][a-z]+)[_\/\\\\](?[A-Z][a-zA-Z]+)/', $file, $matches) ) { @@ -691,7 +693,7 @@ protected static function _prepareMapLayoutHandles() foreach ((array)$xml->xpath('/layout/child::*') as $element) { /** @var \SimpleXMLElement $element */ $handle = $element->getName(); - self::$_mapLayoutHandles[$area][$handle] = @(self::$_mapLayoutHandles[$area][$handle] ?: []); + self::$_mapLayoutHandles[$area][$handle] = self::$_mapLayoutHandles[$area][$handle] ?? []; self::$_mapLayoutHandles[$area][$handle][$module] = $module; } } @@ -751,6 +753,7 @@ protected static function _initDependencies() $contents = file_get_contents($file); $decodedJson = json_decode($contents); if (null == $decodedJson) { + //phpcs:ignore Magento2.Exceptions.DirectThrow throw new \Exception("Invalid Json: $file"); } $json = new \Magento\Framework\Config\Composer\Package(json_decode($contents)); @@ -839,6 +842,7 @@ private static function getPackageModuleMapping(): array $contents = file_get_contents($file); $composerJson = json_decode($contents); if (null == $composerJson) { + //phpcs:ignore Magento2.Exceptions.DirectThrow throw new \Exception("Invalid Json: $file"); } $moduleXml = simplexml_load_file(dirname($file) . '/etc/module.xml'); From 9a0dda1a545287f81e2fd75eb52cfec3aee628e9 Mon Sep 17 00:00:00 2001 From: Max Lesechko Date: Mon, 1 Apr 2019 09:24:56 -0500 Subject: [PATCH 10/10] MC-5421: Create test to check dependencies between modules in Declarative Schema --- .../static/testsuite/Magento/Test/Integrity/DependencyTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php b/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php index 35819ff212954..9c03802602938 100644 --- a/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php +++ b/dev/tests/static/testsuite/Magento/Test/Integrity/DependencyTest.php @@ -392,7 +392,7 @@ protected function getDependenciesFromFiles($module, $fileType, $file, $contents */ protected function _collectDependencies($currentModuleName, $dependencies = []) { - if (!count($dependencies)) { + if (empty($dependencies)) { return []; } $undeclared = [];